Add support to override Azure Blob Storage Domain (#756)

* Add support to override the domain for ABS in `pkg/snapstore/abs_snapstore.go`.

* Adapt example in `example/storage-provider-secrets/00-azure-blob-storage-secret.yaml` to explain the override feature.

* Enhance documentation to explain the override feature in `docs/deployment/getting_started.md`.

docs/deployment/getting_started.md

@@ -24,6 +24,7 @@ The procedure to provide credentials to access the cloud provider object store v
 
 * For `Azure Blob storage`:
    1. The secret file should be provided, and the file path should be made available as environment variables: `AZURE_APPLICATION_CREDENTIALS` or `AZURE_APPLICATION_CREDENTIALS_JSON`.
+   2. The Azure Blob Storage domain can be overridden by providing the optional field `domain` in the secret file, as can be seen in the [example](../../example/storage-provider-secrets/00-azure-blob-storage-secret.yaml) file.
 
 * For `Openstack Swift`:
   1. The secret file should be provided, and the file path should be made available as environment variables: `OPENSTACK_APPLICATION_CREDENTIALS` or `OPENSTACK_APPLICATION_CREDENTIALS_JSON`.

example/storage-provider-secrets/00-azure-blob-storage-secret.yaml

@@ -7,6 +7,7 @@ type: Opaque
 data: 
   storageAccount: YWRtaW4= # admin
   storageKey: YWRtaW4= # admin
+  # domain: blob.core.windows.net # override the ABS domain
 
 #### OR ####
 

pkg/snapstore/abs_snapstore.go

@@ -31,6 +31,7 @@ import (
 
 	"github.com/Azure/azure-storage-blob-go/azblob"
 	"github.com/sirupsen/logrus"
+	"k8s.io/utils/pointer"
 
 	brtypes "github.com/gardener/etcd-backup-restore/pkg/types"
 )
@@ -51,56 +52,63 @@ type ABSSnapStore struct {
 }
 
 type absCredentials struct {
-	BucketName     string `json:"bucketName"`
-	SecretKey      string `json:"storageKey"`
-	StorageAccount string `json:"storageAccount"`
+	BucketName     string  `json:"bucketName"`
+	StorageKey     string  `json:"storageKey"`
+	StorageAccount string  `json:"storageAccount"`
+	Domain         *string `json:"domain,omitempty"`
 }
 
 // NewABSSnapStore create new ABSSnapStore from shared configuration with specified bucket
 func NewABSSnapStore(config *brtypes.SnapstoreConfig) (*ABSSnapStore, error) {
-	storageAccount, storageKey, err := getCredentials(getEnvPrefixString(config.IsSource))
+	absCreds, err := getCredentials(getEnvPrefixString(config.IsSource))
 	if err != nil {
 		return nil, err
 	}
 
-	credentials, err := azblob.NewSharedKeyCredential(storageAccount, storageKey)
+	sharedKeyCredential, err := azblob.NewSharedKeyCredential(absCreds.StorageAccount, absCreds.StorageKey)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create shared key credentials: %v", err)
 	}
 
-	p := azblob.NewPipeline(credentials, azblob.PipelineOptions{
+	p := azblob.NewPipeline(sharedKeyCredential, azblob.PipelineOptions{
 		Retry: azblob.RetryOptions{
 			TryTimeout: downloadTimeout,
 		}})
-	u, err := url.Parse(fmt.Sprintf("https://%s.%s", storageAccount, brtypes.AzureBlobStorageHostName))
+
+	domain := brtypes.AzureBlobStorageGlobalDomain
+	if absCreds.Domain != nil {
+		domain = *absCreds.Domain
+	}
+
+	u, err := url.Parse(fmt.Sprintf("https://%s.%s", absCreds.StorageAccount, domain))
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse service url: %v", err)
 	}
+
 	serviceURL := azblob.NewServiceURL(*u, p)
 	containerURL := serviceURL.NewContainerURL(config.Container)
 
 	return GetABSSnapstoreFromClient(config.Container, config.Prefix, config.TempDir, config.MaxParallelChunkUploads, config.MinChunkSize, &containerURL)
 }
 
-func getCredentials(prefixString string) (string, string, error) {
-
+func getCredentials(prefixString string) (*absCredentials, error) {
 	if filename, isSet := os.LookupEnv(prefixString + absCredentialJSONFile); isSet {
 		credentials, err := readABSCredentialsJSON(filename)
 		if err != nil {
-			return "", "", fmt.Errorf("error getting credentials using %v file", filename)
+			return nil, fmt.Errorf("error getting credentials using %v file with error: %w", filename, err)
 		}
-		return credentials.StorageAccount, credentials.SecretKey, nil
+		return credentials, nil
 	}
 
 	if dir, isSet := os.LookupEnv(prefixString + absCredentialDirectory); isSet {
 		credentials, err := readABSCredentialFiles(dir)
 		if err != nil {
-			return "", "", fmt.Errorf("error getting credentials from %v dir", dir)
+			return credentials, fmt.Errorf("error getting credentials from %v dir with error: %w", dir, err)
 		}
-		return credentials.StorageAccount, credentials.SecretKey, nil
+		return credentials, nil
 	}
 
-	return "", "", fmt.Errorf("unable to get credentials")
+	return nil, fmt.Errorf("unable to get credentials")
 }
 
 func readABSCredentialsJSON(filename string) (*absCredentials, error) {
@@ -132,17 +140,23 @@ func readABSCredentialFiles(dirname string) (*absCredentials, error) {
 
 	for _, file := range files {
 		if file.Name() == "storageAccount" {
-			data, err := os.ReadFile(dirname + "/storageAccount")
+			data, err := os.ReadFile(filepath.Join(dirname, "storageAccount"))
 			if err != nil {
 				return nil, err
 			}
 			absConfig.StorageAccount = string(data)
 		} else if file.Name() == "storageKey" {
-			data, err := os.ReadFile(dirname + "/storageKey")
+			data, err := os.ReadFile(filepath.Join(dirname, "storageKey"))
+			if err != nil {
+				return nil, err
+			}
+			absConfig.StorageKey = string(data)
+		} else if file.Name() == "domain" {
+			data, err := os.ReadFile(filepath.Join(dirname, "domain"))
 			if err != nil {
 				return nil, err
 			}
-			absConfig.SecretKey = string(data)
+			absConfig.Domain = pointer.String(string(data))
 		}
 	}
 
@@ -377,7 +391,7 @@ func GetABSCredentialsLastModifiedTime() (time.Time, error) {
 }
 
 func isABSConfigEmpty(config *absCredentials) error {
-	if len(config.SecretKey) != 0 && len(config.StorageAccount) != 0 {
+	if len(config.StorageKey) != 0 && len(config.StorageAccount) != 0 {
 		return nil
 	}
 	return fmt.Errorf("azure object storage credentials: storageKey or storageAccount is missing")

pkg/snapstore/abs_snapstore_test.go

@@ -41,7 +41,7 @@ func newFakeABSSnapstore() brtypes.SnapStore {
 		newFakePolicyFactory(bucket, prefixV2, objectMap),
 	}
 	p := pipeline.NewPipeline(f, pipeline.Options{HTTPSender: newFakePolicyFactory(bucket, prefixV2, objectMap)})
-	u, err := url.Parse(fmt.Sprintf("https://%s.%s", "dummyaccount", brtypes.AzureBlobStorageHostName))
+	u, err := url.Parse(fmt.Sprintf("https://%s.%s", "dummyaccount", brtypes.AzureBlobStorageGlobalDomain))
 	Expect(err).ShouldNot(HaveOccurred())
 	serviceURL := azblob.NewServiceURL(*u, p)
 	containerURL := serviceURL.NewContainerURL(bucket)

pkg/types/snapstore.go

@@ -52,8 +52,8 @@ const (
 	// SnapshotKindChunk is constant for chunk snapshot kind.
 	SnapshotKindChunk = "Chunk"
 
-	// AzureBlobStorageHostName is the host name for azure blob storage service.
-	AzureBlobStorageHostName = "blob.core.windows.net"
+	// AzureBlobStorageGlobalDomain is the default domain for azure blob storage service.
+	AzureBlobStorageGlobalDomain = "blob.core.windows.net"
 
 	// FinalSuffix is the suffix appended to the names of final snapshots.
 	FinalSuffix = ".final"