Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Upgraded to latest SDKs and service version #2402

Merged
merged 4 commits into from
Oct 27, 2023
Merged

Upgraded to latest SDKs and service version #2402

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e263a9e
Upgraded to latest SDKs and service version
gapra-msft Oct 5, 2023
c4def00
upgraded to latest ga of blob and file
gapra-msft Oct 17, 2023
e374ff5
Merge branch 'dev' into gapra/updateSDKs
gapra-msft Oct 17, 2023
d3d9438
Merge branch 'dev' into gapra/updateSDKs
gapra-msft Oct 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cmd/copy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1282,7 +1282,7 @@ func (cca *CookedCopyCmdArgs) processRedirectionDownload(blobResource common.Res
}

// step 1: create client options
options := createClientOptions(common.LogNone, nil, nil)
options := createClientOptions(common.LogNone)

// step 2: parse source url
u, err := blobResource.FullURL()
Expand Down Expand Up @@ -1329,7 +1329,7 @@ func (cca *CookedCopyCmdArgs) processRedirectionUpload(blobResource common.Resou
}

// step 0: initialize pipeline
options := createClientOptions(common.LogNone, nil, nil)
options := createClientOptions(common.LogNone)

// step 1: parse destination url
u, err := blobResource.FullURL()
Expand Down
12 changes: 3 additions & 9 deletions cmd/copyEnumeratorInit.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -444,13 +444,7 @@ func (cca *CookedCopyCmdArgs) createDstContainer(containerName string, dstWithSA
return err
}

var trailingDot *common.TrailingDotOption
var from *common.Location
if cca.FromTo.To() == common.ELocation.File() {
trailingDot = &cca.trailingDot
from = to.Ptr(cca.FromTo.From())
}
options := createClientOptions(logLevel, trailingDot, from)
options := createClientOptions(logLevel)

// Because the only use-cases for createDstContainer will be on service-level S2S and service-level download
// We only need to create "containers" on local and blob.
Expand Down Expand Up @@ -487,7 +481,7 @@ func (cca *CookedCopyCmdArgs) createDstContainer(containerName string, dstWithSA
return err
}

fsc := common.CreateFileServiceClient(accountRoot, dstCredInfo, nil, options)
fsc := common.CreateFileServiceClient(accountRoot, dstCredInfo, nil, options, &cca.trailingDot, to.Ptr(cca.FromTo.From()))
sc := fsc.NewShareClient(containerName)

_, err = sc.GetProperties(ctx, nil)
Expand Down Expand Up @@ -721,4 +715,4 @@ func NewFolderPropertyOption(fromTo common.FromTo, recursive, stripTopDir bool,
"Any empty folders will not be processed, because source and/or destination doesn't have full folder support" +
getSuffix(false)

}
}
8 changes: 4 additions & 4 deletions cmd/credentialUtil.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ func getBlobCredentialType(ctx context.Context, blobResourceURL string, canBePub
RequestLogOptions: ste.RequestLogOptions{
SyslogDisabled: common.IsForceLoggingDisabled(),
},
}, nil, nil)
})
credInfo := common.CredentialInfo{CredentialType: common.ECredentialType.Anonymous()}
if isSASExisted := sas.Signature() != ""; isSASExisted {
if isMDAccount {
Expand Down Expand Up @@ -630,9 +630,9 @@ func getCredentialType(ctx context.Context, raw rawFromToInfo, cpkOptions common
// ==============================================================================================
// pipeline factory methods
// ==============================================================================================
func createClientOptions(logLevel common.LogLevel, trailingDot *common.TrailingDotOption, from *common.Location) azcore.ClientOptions {
func createClientOptions(logLevel common.LogLevel) azcore.ClientOptions {
logOptions := ste.LogOptions{}
logOptions.ShouldLog = func(level common.LogLevel) bool {return level <= logLevel}
logOptions.ShouldLog = func(level common.LogLevel) bool { return level <= logLevel }

if azcopyScanningLogger != nil {
logOptions.Log = azcopyScanningLogger.Log
Expand All @@ -644,7 +644,7 @@ func createClientOptions(logLevel common.LogLevel, trailingDot *common.TrailingD
MaxRetryDelay: ste.UploadMaxRetryDelay,
}, policy.TelemetryOptions{
ApplicationID: glcm.AddUserAgentPrefix(common.UserAgent),
}, ste.NewAzcopyHTTPClient(frontEndMaxIdleConnectionsPerHost), nil, logOptions, trailingDot, from)
}, ste.NewAzcopyHTTPClient(frontEndMaxIdleConnectionsPerHost), nil, logOptions)
}

const frontEndMaxIdleConnectionsPerHost = http.DefaultMaxIdleConnsPerHost
4 changes: 2 additions & 2 deletions cmd/make.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ func (cookedArgs cookedMakeCmdArgs) process() (err error) {
}

// Note : trailing dot is only applicable to file operations anyway, so setting this to false
options := createClientOptions(common.LogNone, to.Ptr(common.ETrailingDotOption.Disable()), &cookedArgs.resourceLocation)
options := createClientOptions(common.LogNone)

switch cookedArgs.resourceLocation {
case common.ELocation.BlobFS():
Expand Down Expand Up @@ -116,7 +116,7 @@ func (cookedArgs cookedMakeCmdArgs) process() (err error) {
return err
}
case common.ELocation.File():
shareClient := common.CreateShareClient(cookedArgs.resourceURL.String(), credentialInfo, nil, options)
shareClient := common.CreateShareClient(cookedArgs.resourceURL.String(), credentialInfo, nil, options, to.Ptr(common.ETrailingDotOption.Disable()), &cookedArgs.resourceLocation)
quota := &cookedArgs.quota
if quota != nil && *quota == 0 {
quota = nil
Expand Down
4 changes: 2 additions & 2 deletions cmd/removeEnumerator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ import (
"strings"
"time"

"github.com/Azure/azure-storage-azcopy/v10/jobsAdmin"
"github.com/Azure/azure-storage-azcopy/v10/common"
"github.com/Azure/azure-storage-azcopy/v10/jobsAdmin"
"github.com/Azure/azure-storage-azcopy/v10/ste"
)

Expand Down Expand Up @@ -135,7 +135,7 @@ func removeBfsResources(cca *CookedCopyCmdArgs) (err error) {
return errors.New("pattern matches are not supported in this command")
}

options := createClientOptions(azcopyLogVerbosity, nil, nil)
options := createClientOptions(azcopyLogVerbosity)
// attempt to parse the source url
sourceURL, err := cca.Source.String()
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion cmd/root.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ func beginDetectNewVersion() chan struct{} {
}

// step 1: initialize pipeline
options := createClientOptions(common.LogNone, nil, nil)
options := createClientOptions(common.LogNone)

// step 2: start download
blobClient, err := blob.NewClientWithNoCredential(versionMetadataUrl, &blob.ClientOptions{ClientOptions: options})
Expand Down
56 changes: 30 additions & 26 deletions cmd/syncProcessor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ func (d *interactiveDeleteProcessor) removeImmediately(object StoredObject) (err
msg := fmt.Sprintf("error %s deleting the object %s", err.Error(), object.relativePath)
glcm.Info(msg + "; check the scanning log file for more details")
if azcopyScanningLogger != nil {
azcopyScanningLogger.Log(common.LogError, msg + ": " + err.Error())
azcopyScanningLogger.Log(common.LogError, msg+": "+err.Error())
}
}

Expand Down Expand Up @@ -233,8 +233,8 @@ func newSyncLocalDeleteProcessor(cca *cookedSyncCmdArgs, fpo common.FolderProper
}

type localFileDeleter struct {
rootPath string
fpo common.FolderPropertyOption
rootPath string
fpo common.FolderPropertyOption
folderManager common.FolderDeletionManager
}

Expand Down Expand Up @@ -288,33 +288,37 @@ func newSyncDeleteProcessor(cca *cookedSyncCmdArgs, fpo common.FolderPropertyOpt
from = to.Ptr(cca.fromTo.From())
}

clientOptions := createClientOptions(azcopyLogVerbosity, trailingDot, from)
clientOptions := createClientOptions(azcopyLogVerbosity)

return newInteractiveDeleteProcessor(newRemoteResourceDeleter(rawURL, cca.credentialInfo, clientOptions, ctx, cca.fromTo.To(), fpo, cca.forceIfReadOnly).delete,
return newInteractiveDeleteProcessor(newRemoteResourceDeleter(rawURL, cca.credentialInfo, clientOptions, ctx, cca.fromTo.To(), fpo, cca.forceIfReadOnly, trailingDot, from).delete,
cca.deleteDestination, cca.fromTo.To().String(), cca.destination, cca.incrementDeletionCount, cca.dryrunMode), nil
}

type remoteResourceDeleter struct {
rootURL *url.URL
credInfo common.CredentialInfo
clientOptions azcore.ClientOptions
ctx context.Context
targetLocation common.Location
folderManager common.FolderDeletionManager
folderOption common.FolderPropertyOption
rootURL *url.URL
credInfo common.CredentialInfo
clientOptions azcore.ClientOptions
ctx context.Context
targetLocation common.Location
folderManager common.FolderDeletionManager
folderOption common.FolderPropertyOption
forceIfReadOnly bool
trailingDot *common.TrailingDotOption
from *common.Location
}

func newRemoteResourceDeleter(rawRootURL *url.URL, credInfo common.CredentialInfo, clientOptions azcore.ClientOptions, ctx context.Context, targetLocation common.Location, fpo common.FolderPropertyOption, forceIfReadOnly bool) *remoteResourceDeleter {
func newRemoteResourceDeleter(rawRootURL *url.URL, credInfo common.CredentialInfo, clientOptions azcore.ClientOptions, ctx context.Context, targetLocation common.Location, fpo common.FolderPropertyOption, forceIfReadOnly bool, trailingDot *common.TrailingDotOption, from *common.Location) *remoteResourceDeleter {
return &remoteResourceDeleter{
rootURL: rawRootURL,
credInfo: credInfo,
clientOptions: clientOptions,
ctx: ctx,
targetLocation: targetLocation,
folderManager: common.NewFolderDeletionManager(ctx, fpo, azcopyScanningLogger),
folderOption: fpo,
rootURL: rawRootURL,
credInfo: credInfo,
clientOptions: clientOptions,
ctx: ctx,
targetLocation: targetLocation,
folderManager: common.NewFolderDeletionManager(ctx, fpo, azcopyScanningLogger),
folderOption: fpo,
forceIfReadOnly: forceIfReadOnly,
trailingDot: trailingDot,
from: from,
}
}

Expand Down Expand Up @@ -392,7 +396,7 @@ func (b *remoteResourceDeleter) delete(object StoredObject) error {
}
fileURLParts.DirectoryOrFilePath = path.Join(fileURLParts.DirectoryOrFilePath, object.relativePath)

fileClient := common.CreateShareFileClient(fileURLParts.String(), b.credInfo, nil, b.clientOptions)
fileClient := common.CreateShareFileClient(fileURLParts.String(), b.credInfo, nil, b.clientOptions, b.trailingDot, b.from)

_, err = fileClient.Delete(b.ctx, nil)
if err != nil && b.forceIfReadOnly && fileerror.HasCode(err, fileerror.ReadOnlyAttribute) {
Expand All @@ -410,7 +414,7 @@ func (b *remoteResourceDeleter) delete(object StoredObject) error {
msg := fmt.Sprintf("error %s removing the read-only attribute from the file %s", err.Error(), object.relativePath)
glcm.Info(msg + "; check the scanning log file for more details")
if azcopyScanningLogger != nil {
azcopyScanningLogger.Log(common.LogError, msg + ": " + err.Error())
azcopyScanningLogger.Log(common.LogError, msg+": "+err.Error())
}
}
}
Expand All @@ -431,7 +435,7 @@ func (b *remoteResourceDeleter) delete(object StoredObject) error {
msg := fmt.Sprintf("error %s deleting the object %s", err.Error(), object.relativePath)
glcm.Info(msg + "; check the scanning log file for more details")
if azcopyScanningLogger != nil {
azcopyScanningLogger.Log(common.LogError, msg + ": " + err.Error())
azcopyScanningLogger.Log(common.LogError, msg+": "+err.Error())
}
}

Expand All @@ -452,7 +456,7 @@ func (b *remoteResourceDeleter) delete(object StoredObject) error {
// HNS endpoint doesn't like delete snapshots on a directory
_, err = blobClient.Delete(b.ctx, nil)
case common.ELocation.File():
directoryClient := common.CreateShareDirectoryClient(objectURL.String(), b.credInfo, nil, b.clientOptions)
directoryClient := common.CreateShareDirectoryClient(objectURL.String(), b.credInfo, nil, b.clientOptions, b.trailingDot, b.from)
_, err = directoryClient.Delete(ctx, nil)

if err != nil && b.forceIfReadOnly && fileerror.HasCode(err, fileerror.ReadOnlyAttribute) {
Expand All @@ -470,7 +474,7 @@ func (b *remoteResourceDeleter) delete(object StoredObject) error {
msg := fmt.Sprintf("error %s removing the read-only attribute from the file %s", err.Error(), object.relativePath)
glcm.Info(msg + "; check the scanning log file for more details")
if azcopyScanningLogger != nil {
azcopyScanningLogger.Log(common.LogError, msg + ": " + err.Error())
azcopyScanningLogger.Log(common.LogError, msg+": "+err.Error())
}
}
}
Expand All @@ -489,4 +493,4 @@ func (b *remoteResourceDeleter) delete(object StoredObject) error {

return nil
}
}
}
10 changes: 5 additions & 5 deletions cmd/zc_enumerator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -438,7 +438,7 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat
blobURLParts.BlobName = ""
blobURLParts.Snapshot = ""
blobURLParts.VersionID = ""
bsc := common.CreateBlobServiceClient(blobURLParts.String(), *credential, &common.CredentialOpOptions{LogError: glcm.Info}, createClientOptions(logLevel, nil, nil))
bsc := common.CreateBlobServiceClient(blobURLParts.String(), *credential, &common.CredentialOpOptions{LogError: glcm.Info}, createClientOptions(logLevel))

if containerName == "" || strings.Contains(containerName, "*") {
if !recursive {
Expand Down Expand Up @@ -473,7 +473,7 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat
fileURLParts.ShareName = ""
fileURLParts.ShareSnapshot = ""
fileURLParts.DirectoryOrFilePath = ""
fsc := common.CreateFileServiceClient(fileURLParts.String(), *credential, &common.CredentialOpOptions{LogError: glcm.Info}, createClientOptions(logLevel, to.Ptr(trailingDot), destination))
fsc := common.CreateFileServiceClient(fileURLParts.String(), *credential, &common.CredentialOpOptions{LogError: glcm.Info}, createClientOptions(logLevel), to.Ptr(trailingDot), destination)

if shareName == "" || strings.Contains(shareName, "*") {
if !recursive {
Expand Down Expand Up @@ -506,7 +506,7 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat
blobURLParts.BlobName = ""
blobURLParts.Snapshot = ""
blobURLParts.VersionID = ""
bsc := common.CreateBlobServiceClient(blobURLParts.String(), *credential, &common.CredentialOpOptions{LogError: glcm.Info}, createClientOptions(logLevel, nil, nil))
bsc := common.CreateBlobServiceClient(blobURLParts.String(), *credential, &common.CredentialOpOptions{LogError: glcm.Info}, createClientOptions(logLevel))

includeDirectoryStubs = true // DFS is supposed to feed folders in
if containerName == "" || strings.Contains(containerName, "*") {
Expand Down Expand Up @@ -606,7 +606,7 @@ type objectProcessor func(storedObject StoredObject) error
//
// Might be easier to debug
//
// modifies a StoredObject, but does NOT process it. Used for modifications, such as pre-pending a parent path
// modifies a StoredObject, but does NOT process it. Used for modifications, such as prepending a parent path
type objectMorpher func(storedObject *StoredObject)

// FollowedBy returns a new objectMorpher, which performs the action of existing followed by the action of additional.
Expand Down Expand Up @@ -809,4 +809,4 @@ func getObjectNameOnly(fullPath string) (nameOnly string) {
}

return
}
}
Loading