Merge pull request #1242 from Azure/dev

v10.7.0 Release

ChangeLog.md

@@ -1,6 +1,21 @@
 
 # Change Log
 
+## Version 10.7.0
+
+### New features
+1. Added support for auto-login when performing data commands(copy/sync/list/make/remove). Please refer to our documentation for more info.
+1. Added ``blob-tags`` flag for setting [blob index tags](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blob-index-how-to?tabs=azure-portal) when performing copy command. Please note that we support setting blob tags only when tags are explicitly specified. Refer to the [public documentations](https://docs.microsoft.com/en-us/rest/api/storageservices/put-blob#remarks) to know more.
+
+### Bug fixes
+
+1. Fixed issue [#1139](https://github.com/Azure/azure-storage-azcopy/issues/1139) to preserve content-type in service-to-service transfer.
+1. Fixed issue to allow snapshot restoring.
+1. Fixed issue with setting content-type of an empty file when performing copy command.
+
+### Improvements
+1. Added support for setting tier directly at the time of [upload](https://docs.microsoft.com/en-us/rest/api/storageservices/put-blob#remarks) API call instead of performing a separate [set tier](https://docs.microsoft.com/en-us/rest/api/storageservices/set-blob-tier) API call.
+
 ## Version 10.6.1
 
 ### Bug fixes
@@ -17,8 +32,8 @@
 
 ### New features
 
-1. ``azcopy sync`` now supports the persistence of ACLs between supported resources (Windows and Azure Files) using the --persist-smb-permissions flag.
-1. ``azcopy sync`` now supports the persistence of SMB property info between supported resources (Windows and Azure Files) using the --persist-smb-info flag. The information that can be preserved is Created Time, Last Write Time and Attributes (e.g. Read Only).
+1. ``azcopy sync`` now supports the persistence of ACLs between supported resources (Azure Files) using the ``--preserve-smb-permissions`` flag.
+1. ``azcopy sync`` now supports the persistence of SMB property info between supported resources (Azure Files) using the ``--preserve-smb-info`` flag. The information that can be preserved is Created Time, Last Write Time and Attributes (e.g. Read Only).
 1. Added support for [higher block & blob size](https://docs.microsoft.com/en-us/rest/api/storageservices/put-block#remarks) 
     - For service version ``2019-12-12`` or higher, the block size can now be less than or equal to ``4000 MiB``. The maximum size of a block blob therefore can be ``190.7 TiB (4000 MiB X 50,000 blocks)``
 1. Added support for [Blob Versioning](https://docs.microsoft.com/en-us/azure/storage/blobs/versioning-overview)

cmd/copy.go

@@ -103,6 +103,8 @@ type rawCopyCmdArgs struct {
 	md5ValidationOption      string
 	CheckLength              bool
 	deleteSnapshotsOption    string
+
+	blobTags string
 	// defines the type of the blob at the destination in case of upload / account to account copy
 	blobType      string
 	blockBlobTier string
@@ -486,6 +488,16 @@ func (raw rawCopyCmdArgs) cookWithId(jobId common.JobID) (cookedCopyCmdArgs, err
 	cooked.preserveLastModifiedTime = raw.preserveLastModifiedTime
 	cooked.includeDirectoryStubs = raw.includeDirectoryStubs
 
+	if cooked.fromTo.To() != common.ELocation.Blob() && raw.blobTags != "" {
+		return cooked, errors.New("blob tags can only be set when transferring to blob storage")
+	}
+	blobTags := common.ToCommonBlobTagsMap(raw.blobTags)
+	err = validateBlobTagsKeyValue(blobTags)
+	if err != nil {
+		return cooked, err
+	}
+	cooked.blobTags = blobTags
+
 	// Make sure the given input is the one of the enums given by the blob SDK
 	err = cooked.deleteSnapshotsOption.Parse(raw.deleteSnapshotsOption)
 	if err != nil {
@@ -648,11 +660,11 @@ func (raw rawCopyCmdArgs) cookWithId(jobId common.JobID) (cookedCopyCmdArgs, err
 		if cooked.blobType != common.EBlobType.Detect() && cooked.fromTo.To() != common.ELocation.Blob() {
 			return cooked, fmt.Errorf("blob-type is not supported for the scenario (%s)", cooked.fromTo.String())
 		}
-		// Disabling blob tier override, when copying block -> block blob or page -> page blob, blob tier will be kept,
-		// For s3 and file, only hot block blob tier is supported.
-		if cooked.blockBlobTier != common.EBlockBlobTier.None() ||
-			cooked.pageBlobTier != common.EPageBlobTier.None() {
-			return cooked, fmt.Errorf("blob-tier is not supported while copying from service to service")
+
+		// Setting blob tier is supported only when destination is a blob storage. Disabling it for all the other transfer scenarios.
+		if (cooked.blockBlobTier != common.EBlockBlobTier.None() || cooked.pageBlobTier != common.EPageBlobTier.None()) &&
+			cooked.fromTo.To() != common.ELocation.Blob() {
+			return cooked, fmt.Errorf("blob-tier is not supported for the scenario (%s)", cooked.fromTo.String())
 		}
 		if cooked.noGuessMimeType {
 			return cooked, fmt.Errorf("no-guess-mime-type is not supported while copying from service to service")
@@ -814,6 +826,53 @@ func validateMd5Option(option common.HashValidationOption, fromTo common.FromTo)
 	return nil
 }
 
+// Valid tag key and value characters include:
+// 1. Lowercase and uppercase letters (a-z, A-Z)
+// 2. Digits (0-9)
+// 3. A space ( )
+// 4. Plus (+), minus (-), period (.), solidus (/), colon (:), equals (=), and underscore (_)
+func isValidBlobTagsKeyValue(keyVal string) bool {
+	for _, c := range keyVal {
+		if !((c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == ' ' || c == '+' ||
+			c == '-' || c == '.' || c == '/' || c == ':' || c == '=' || c == '_') {
+			return false
+		}
+	}
+	return true
+}
+
+// ValidateBlobTagsKeyValue
+// The tag set may contain at most 10 tags. Tag keys and values are case sensitive.
+// Tag keys must be between 1 and 128 characters, and tag values must be between 0 and 256 characters.
+func validateBlobTagsKeyValue(bt common.BlobTags) error {
+	if len(bt) > 10 {
+		return errors.New("at-most 10 tags can be associated with a blob")
+	}
+	for k, v := range bt {
+		key, err := url.QueryUnescape(k)
+		if err != nil {
+			return err
+		}
+		value, err := url.QueryUnescape(v)
+		if err != nil {
+			return err
+		}
+
+		if key == "" || len(key) > 128 || len(value) > 256 {
+			return errors.New("tag keys must be between 1 and 128 characters, and tag values must be between 0 and 256 characters")
+		}
+
+		if !isValidBlobTagsKeyValue(key) {
+			return errors.New("incorrect character set used in key: " + k)
+		}
+
+		if !isValidBlobTagsKeyValue(value) {
+			return errors.New("incorrect character set used in value: " + v)
+		}
+	}
+	return nil
+}
+
 // represents the processed copy command input from the user
 type cookedCopyCmdArgs struct {
 	// from arguments
@@ -845,8 +904,11 @@ type cookedCopyCmdArgs struct {
 	// options from flags
 	blockSize int64
 	// list of blobTypes to exclude while enumerating the transfer
-	excludeBlobType          []azblob.BlobType
-	blobType                 common.BlobType
+	excludeBlobType []azblob.BlobType
+	blobType        common.BlobType
+	// Blob index tags categorize data in your storage account utilizing key-value tag attributes.
+	// These tags are automatically indexed and exposed as a queryable multi-dimensional index to easily find data.
+	blobTags                 common.BlobTags
 	blockBlobTier            common.BlockBlobTier
 	pageBlobTier             common.PageBlobTier
 	metadata                 string
@@ -1111,6 +1173,7 @@ func (cca *cookedCopyCmdArgs) processCopyJobPartOrders() (err error) {
 			PutMd5:                   cca.putMd5,
 			MD5ValidationOption:      cca.md5ValidationOption,
 			DeleteSnapshotsOption:    cca.deleteSnapshotsOption,
+			BlobTagsString:           cca.blobTags.ToString(),
 		},
 		CommandString:  cca.commandString,
 		CredentialInfo: cca.credentialInfo,
@@ -1579,7 +1642,7 @@ func init() {
 	cpCmd.PersistentFlags().Float64Var(&raw.blockSizeMB, "block-size-mb", 0, "Use this block size (specified in MiB) when uploading to Azure Storage, and downloading from Azure Storage. The default value is automatically calculated based on file size. Decimal fractions are allowed (For example: 0.25).")
 	cpCmd.PersistentFlags().StringVar(&raw.logVerbosity, "log-level", "INFO", "Define the log verbosity for the log file, available levels: INFO(all requests/responses), WARNING(slow responses), ERROR(only failed requests), and NONE(no output logs). (default 'INFO').")
 	cpCmd.PersistentFlags().StringVar(&raw.blobType, "blob-type", "Detect", "Defines the type of blob at the destination. This is used for uploading blobs and when copying between accounts (default 'Detect'). Valid values include 'Detect', 'BlockBlob', 'PageBlob', and 'AppendBlob'. "+
-		"When copying between accounts, a value of 'Detect' causes AzCopy to use the type of source blob to determine the type of the destination blob. When uploading a file, 'Detect' determines if the file is a VHD or a VHDX file based on the file extension. If the file is ether a VHD or VHDX file, AzCopy treats the file as a page blob.")
+		"When copying between accounts, a value of 'Detect' causes AzCopy to use the type of source blob to determine the type of the destination blob. When uploading a file, 'Detect' determines if the file is a VHD or a VHDX file based on the file extension. If the file is either a VHD or VHDX file, AzCopy treats the file as a page blob.")
 	cpCmd.PersistentFlags().StringVar(&raw.blockBlobTier, "block-blob-tier", "None", "upload block blob to Azure Storage using this blob tier.")
 	cpCmd.PersistentFlags().StringVar(&raw.pageBlobTier, "page-blob-tier", "None", "Upload page blob to Azure Storage using this blob tier. (default 'None').")
 	cpCmd.PersistentFlags().StringVar(&raw.metadata, "metadata", "", "Upload to Azure Storage with these key-value pairs as metadata.")
@@ -1608,6 +1671,7 @@ func init() {
 	cpCmd.PersistentFlags().BoolVar(&raw.s2sSourceChangeValidation, "s2s-detect-source-changed", false, "Detect if the source file/blob changes while it is being read. (This parameter only applies to service to service copies, because the corresponding check is permanently enabled for uploads and downloads.)")
 	cpCmd.PersistentFlags().StringVar(&raw.s2sInvalidMetadataHandleOption, "s2s-handle-invalid-metadata", common.DefaultInvalidMetadataHandleOption.String(), "Specifies how invalid metadata keys are handled. Available options: ExcludeIfInvalid, FailIfInvalid, RenameIfInvalid. (default 'ExcludeIfInvalid').")
 	cpCmd.PersistentFlags().StringVar(&raw.listOfVersionIDs, "list-of-versions", "", "Specifies a file where each version id is listed on a separate line. Ensure that the source must point to a single blob and all the version ids specified in the file using this flag must belong to the source blob only. AzCopy will download the specified versions in the destination folder provided.")
+	cpCmd.PersistentFlags().StringVar(&raw.blobTags, "blob-tags", "", "Set tags on blobs to categorize data in your storage account")
 	// s2sGetPropertiesInBackend is an optional flag for controlling whether S3 object's or Azure file's full properties are get during enumerating in frontend or
 	// right before transferring in ste(backend).
 	// The traditional behavior of all existing enumerator is to get full properties during enumerating(more specifically listing),

cmd/copyEnumeratorInit.go

@@ -239,6 +239,7 @@ func (cca *cookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde
 			cca.s2sPreserveAccessTier,
 			jobPartOrder.Fpo,
 		)
+		transfer.BlobTags = cca.blobTags
 
 		if shouldSendToSte {
 			return addTransfer(&jobPartOrder, transfer, cca)

cmd/credentialUtil.go

@@ -26,12 +26,13 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/minio/minio-go/pkg/s3utils"
 	"net/http"
 	"net/url"
 	"strings"
 	"sync"
 
+	"github.com/minio/minio-go/pkg/s3utils"
+
 	"github.com/Azure/azure-pipeline-go/pipeline"
 	"github.com/Azure/azure-storage-blob-go/azblob"
 	"github.com/Azure/azure-storage-file-go/azfile"
@@ -42,6 +43,7 @@ import (
 )
 
 var once sync.Once
+var autoOAuth sync.Once
 
 // only one UserOAuthTokenManager should exists in azcopy-v2 process in cmd(FE) module for current user.
 // (given appAppPathFolder is mapped to current user)
@@ -69,6 +71,57 @@ func GetUserOAuthTokenManagerInstance() *common.UserOAuthTokenManager {
 	return currentUserOAuthTokenManager
 }
 
+/*
+ * GetInstanceOAuthTokenInfo returns OAuth token, obtained by auto-login,
+ * for current instance of AzCopy.
+ */
+func GetOAuthTokenManagerInstance() (*common.UserOAuthTokenManager, error) {
+	var err error
+	autoOAuth.Do(func() {
+		var lca loginCmdArgs
+		if glcm.GetEnvironmentVariable(common.EEnvironmentVariable.AutoLoginType()) == "" {
+			err = errors.New("no login type specified")
+			return
+		}
+
+		if tenantID := glcm.GetEnvironmentVariable(common.EEnvironmentVariable.TenantID()); tenantID != "" {
+			lca.tenantID = tenantID
+		}
+
+		if endpoint := glcm.GetEnvironmentVariable(common.EEnvironmentVariable.AADEndpoint()); endpoint != "" {
+			lca.aadEndpoint = endpoint
+		}
+
+		// Fill up lca
+		switch glcm.GetEnvironmentVariable(common.EEnvironmentVariable.AutoLoginType()) {
+		case "SPN":
+			lca.applicationID = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.ApplicationID())
+			lca.certPath = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.CertificatePath())
+			lca.certPass = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.CertificatePassword())
+			lca.clientSecret = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.ClientSecret())
+			lca.servicePrincipal = true
+
+		case "MSI":
+			lca.identityClientID = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.ManagedIdentityClientID())
+			lca.identityObjectID = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.ManagedIdentityObjectID())
+			lca.identityResourceID = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.ManagedIdentityResourceString())
+			lca.identity = true
+
+		case "DEVICE":
+			lca.identity = false
+		}
+
+		lca.persistToken = false
+		err = lca.process()
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return GetUserOAuthTokenManagerInstance(), nil
+}
+
 // ==============================================================================================
 // Get credential type methods
 // ==============================================================================================
@@ -145,7 +198,8 @@ func getBlobCredentialType(ctx context.Context, blobResourceURL string, canBePub
 
 		// No forms of auth are present.no SAS token or OAuth token is present and the resource is not public
 		if !isPublicResource {
-			return common.ECredentialType.Unknown(), isPublicResource, errors.New("no SAS token or OAuth token is present and the resource is not public")
+			return common.ECredentialType.Unknown(), isPublicResource,
+				common.NewAzError(common.EAzError.LoginCredMissing(), "No SAS token or OAuth token is present and the resource is not public")
 		}
 
 		return common.ECredentialType.Anonymous(), isPublicResource, nil
@@ -191,7 +245,8 @@ func getBlobFSCredentialType(ctx context.Context, blobResourceURL string, standa
 	if name != "" && key != "" { // TODO: To remove, use for internal testing, SharedKey should not be supported from commandline
 		return common.ECredentialType.SharedKey(), nil
 	} else {
-		return common.ECredentialType.Unknown(), errors.New("OAuth token, SAS token, or shared key should be provided for Blob FS")
+		return common.ECredentialType.Unknown(),
+			common.NewAzError(common.EAzError.LoginCredMissing(), "OAuth token, SAS token, or shared key should be provided for Blob FS")
 	}
 }
 
@@ -403,15 +458,31 @@ func doGetCredentialTypeForLocation(ctx context.Context, location common.Locatio
 		case common.ELocation.Local(), common.ELocation.Benchmark():
 			credType = common.ECredentialType.Anonymous()
 		case common.ELocation.Blob():
-			if credType, isPublic, err = getBlobCredentialType(ctx, resource, isSource, resourceSAS != ""); err != nil {
+			credType, isPublic, err = getBlobCredentialType(ctx, resource, isSource, resourceSAS != "")
+			if azErr, ok := err.(common.AzError); ok && azErr.Equals(common.EAzError.LoginCredMissing()) {
+				_, autoLoginErr := GetOAuthTokenManagerInstance()
+				if autoLoginErr == nil {
+					err = nil // Autologin succeeded, reset original error
+					credType, isPublic = common.ECredentialType.OAuthToken(), false
+				}
+			}
+			if err != nil {
 				return common.ECredentialType.Unknown(), false, err
 			}
 		case common.ELocation.File():
 			if credType, err = getAzureFileCredentialType(); err != nil {
 				return common.ECredentialType.Unknown(), false, err
 			}
 		case common.ELocation.BlobFS():
-			if credType, err = getBlobFSCredentialType(ctx, resource, resourceSAS != ""); err != nil {
+			credType, err = getBlobFSCredentialType(ctx, resource, resourceSAS != "")
+			if azErr, ok := err.(common.AzError); ok && azErr.Equals(common.EAzError.LoginCredMissing()) {
+				_, autoLoginErr := GetOAuthTokenManagerInstance()
+				if autoLoginErr == nil {
+					err = nil // Autologin succeeded, reset original error
+					credType, isPublic = common.ECredentialType.OAuthToken(), false
+				}
+			}
+			if err != nil {
 				return common.ECredentialType.Unknown(), false, err
 			}
 		case common.ELocation.S3():

cmd/helpMessages.go

@@ -79,6 +79,14 @@ Upload files and directories by using a SAS token and wildcard (*) characters:
 
   - azcopy cp "/path/*foo/*bar*" "https://[account].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" --recursive=true
 
+Upload files and directories to Azure Storage account and set the query-string encoded tags on the blob. 
+
+	- To set tags {key = "bla bla", val = "foo"} and {key = "bla bla 2", val = "bar"}, use the following syntax :
+		- azcopy cp "/path/*foo/*bar*" "https://[account].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" --blob-tags="bla%20bla=foo&bla%20bla%202=bar"
+	- Keys and values are URL encoded and the key-value pairs are separated by an ampersand('&')
+	- https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blob-index-how-to?tabs=azure-portal
+	- While setting tags on the blobs, there are additional permissions('t' for tags) in SAS without which the service will give authorization error back.
+
 Download a single file by using OAuth authentication. If you have not yet logged into AzCopy, please run the azcopy login command before you run the following command.
 
   - azcopy cp "https://[account].blob.core.windows.net/[container]/[path/to/blob]" "/path/to/file.txt"
@@ -158,6 +166,14 @@ Copy all buckets to Blob Storage from an Amazon Web Services (AWS) region by usi
 Copy a subset of buckets by using a wildcard symbol (*) in the bucket name. Like the previous examples, you'll need an access key and a SAS token. Make sure to set the environment variable AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY for AWS S3 source.
 
   - azcopy cp "https://s3.amazonaws.com/[bucket*name]/" "https://[destaccount].blob.core.windows.net?[SAS]" --recursive=true
+
+Transfer files and directories to Azure Storage account and set the given query-string encoded tags on the blob. 
+
+	- To set tags {key = "bla bla", val = "foo"} and {key = "bla bla 2", val = "bar"}, use the following syntax :
+		- azcopy cp "https://[account].blob.core.windows.net/[source_container]/[path/to/directory]?[SAS]" "https://[account].blob.core.windows.net/[destination_container]/[path/to/directory]?[SAS]" --blob-tags="bla%20bla=foo&bla%20bla%202=bar"
+	- Keys and values are URL encoded and the key-value pairs are separated by an ampersand('&')
+	- https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blob-index-how-to?tabs=azure-portal
+	- While setting tags on the blobs, there are additional permissions('t' for tags) in SAS without which the service will give authorization error back.
 `
 
 // ===================================== ENV COMMAND ===================================== //