Skip to content

Commit

Permalink
Fixed a bug where AzCopy was not reporting performance numbers (#2636)
Browse files Browse the repository at this point in the history
  • Loading branch information
@gapra-msft
gapra-msft authored May 21, 2024
1 parent ddfe6bc commit 7fe523c
Show file tree
Hide file tree
Showing 12 changed files with 263 additions and 218 deletions.
2 changes: 1 addition & 1 deletion azure-pipelines.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ jobs:
- task: PowerShell@2
inputs:
targetType: 'inline'
script: 'Install-Module -Name Az -Scope CurrentUser -Repository PSGallery -AllowClobber -Force'
script: 'Install-Module -Name Az.Accounts -Scope CurrentUser -Repository PSGallery -AllowClobber -Force'
pwsh: 'true'
displayName: 'Install Powershell Az Module'
- task: AzureCLI@2
Expand Down
6 changes: 3 additions & 3 deletions cmd/credentialUtil.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -365,7 +365,7 @@ func isPublic(ctx context.Context, blobResourceURL string, cpkOptions common.Cpk
MaxRetryDelay: ste.UploadMaxRetryDelay,
}, policy.TelemetryOptions{
ApplicationID: glcm.AddUserAgentPrefix(common.UserAgent),
}, nil, nil, ste.LogOptions{}, nil)
}, nil, ste.LogOptions{}, nil)

blobClient, _ := blob.NewClientWithNoCredential(bURLParts.String(), &blob.ClientOptions{ClientOptions: clientOptions})
bURLParts.BlobName = ""
Expand Down Expand Up @@ -398,7 +398,7 @@ func mdAccountNeedsOAuth(ctx context.Context, blobResourceURL string, cpkOptions
MaxRetryDelay: ste.UploadMaxRetryDelay,
}, policy.TelemetryOptions{
ApplicationID: glcm.AddUserAgentPrefix(common.UserAgent),
}, nil, nil, ste.LogOptions{}, nil)
}, nil, ste.LogOptions{}, nil)

blobClient, _ := blob.NewClientWithNoCredential(blobResourceURL, &blob.ClientOptions{ClientOptions: clientOptions})
_, err := blobClient.GetProperties(ctx, &blob.GetPropertiesOptions{CPKInfo: cpkOptions.GetCPKInfo()})
Expand Down Expand Up @@ -592,7 +592,7 @@ func createClientOptions(logger common.ILoggerResetable, srcCred *common.ScopedC
MaxRetryDelay: ste.UploadMaxRetryDelay,
}, policy.TelemetryOptions{
ApplicationID: glcm.AddUserAgentPrefix(common.UserAgent),
}, ste.NewAzcopyHTTPClient(frontEndMaxIdleConnectionsPerHost), nil, logOptions, srcCred)
}, ste.NewAzcopyHTTPClient(frontEndMaxIdleConnectionsPerHost), logOptions, srcCred)
}

const frontEndMaxIdleConnectionsPerHost = http.DefaultMaxIdleConnsPerHost
177 changes: 0 additions & 177 deletions e2etest/newe2e_task_resourcemanagement.go
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,7 @@
package e2etest

import (
"crypto/md5"
"encoding/hex"
"fmt"
"github.com/Azure/azure-storage-azcopy/v10/cmd"
"github.com/Azure/azure-storage-azcopy/v10/common"
"io"
"strings"
)

// ResourceTracker tracks resources
Expand Down Expand Up @@ -72,174 +66,3 @@ func CreateResource[T ResourceManager](a Asserter, base ResourceManager, def Mat

return matchingRes.(T)
}

func ValidatePropertyPtr[T any](a Asserter, name string, expected, real *T) {
if expected == nil {
return
}

a.Assert(name+" must match", Equal{Deep: true}, expected, real)
}

func ValidateMetadata(a Asserter, expected, real common.Metadata) {
if expected == nil {
return
}

a.Assert("Metadata must match", Equal{Deep: true}, expected, real)
}

func ValidateTags(a Asserter, expected, real map[string]string) {
if expected == nil {
return
}

a.Assert("Tags must match", Equal{Deep: true}, expected, real)
}

func ValidateResource[T ResourceManager](a Asserter, target T, definition MatchedResourceDefinition[T], validateObjectContent bool) {
a.AssertNow("Target resource and definition must not be null", Not{IsNil{}}, a, target, definition)
a.AssertNow("Target resource must be at a equal level to the resource definition", Equal{}, target.Level(), definition.DefinitionTarget())

if dryrunner, ok := a.(DryrunAsserter); ok && dryrunner.Dryrun() {
return
}

definition.ApplyDefinition(a, target, map[cmd.LocationLevel]func(Asserter, ResourceManager, ResourceDefinition){
cmd.ELocationLevel.Container(): func(a Asserter, manager ResourceManager, definition ResourceDefinition) {
cRes := manager.(ContainerResourceManager)

if !definition.ShouldExist() {
a.AssertNow("container must not exist", Equal{}, cRes.Exists(), false)
return
}

cProps := cRes.GetProperties(a)
vProps := definition.(ResourceDefinitionContainer).Properties

ValidateMetadata(a, vProps.Metadata, cProps.Metadata)

if manager.Location() == common.ELocation.Blob() || manager.Location() == common.ELocation.BlobFS() {
ValidatePropertyPtr(a, "Public access", vProps.BlobContainerProperties.Access, cProps.BlobContainerProperties.Access)
}

if manager.Location() == common.ELocation.File() {
ValidatePropertyPtr(a, "Enabled protocols", vProps.FileContainerProperties.EnabledProtocols, cProps.FileContainerProperties.EnabledProtocols)
ValidatePropertyPtr(a, "RootSquash", vProps.FileContainerProperties.RootSquash, cProps.FileContainerProperties.RootSquash)
ValidatePropertyPtr(a, "AccessTier", vProps.FileContainerProperties.AccessTier, cProps.FileContainerProperties.AccessTier)
ValidatePropertyPtr(a, "Quota", vProps.FileContainerProperties.Quota, cProps.FileContainerProperties.Quota)
}
},
cmd.ELocationLevel.Object(): func(a Asserter, manager ResourceManager, definition ResourceDefinition) {
objMan := manager.(ObjectResourceManager)
objDef := definition.(ResourceDefinitionObject)

if !objDef.ShouldExist() {
a.Assert(fmt.Sprintf("object %s must not exist", objMan.ObjectName()), Equal{}, objMan.Exists(), false)
return
}

oProps := objMan.GetProperties(a)
vProps := objDef.ObjectProperties

if validateObjectContent && objMan.EntityType() == common.EEntityType.File() && objDef.Body != nil {
objBody := objMan.Download(a)
validationBody := objDef.Body.Reader()

objHash := md5.New()
valHash := md5.New()

_, err := io.Copy(objHash, objBody)
a.NoError("hash object body", err)
_, err = io.Copy(valHash, validationBody)
a.NoError("hash validation body", err)

a.Assert("bodies differ in hash", Equal{Deep: true}, hex.EncodeToString(objHash.Sum(nil)), hex.EncodeToString(valHash.Sum(nil)))
}

// Properties
ValidateMetadata(a, vProps.Metadata, oProps.Metadata)

// HTTP headers
ValidatePropertyPtr(a, "Cache control", vProps.HTTPHeaders.cacheControl, oProps.HTTPHeaders.cacheControl)
ValidatePropertyPtr(a, "Content disposition", vProps.HTTPHeaders.contentDisposition, oProps.HTTPHeaders.contentDisposition)
ValidatePropertyPtr(a, "Content encoding", vProps.HTTPHeaders.contentEncoding, oProps.HTTPHeaders.contentEncoding)
ValidatePropertyPtr(a, "Content language", vProps.HTTPHeaders.contentLanguage, oProps.HTTPHeaders.contentLanguage)
ValidatePropertyPtr(a, "Content type", vProps.HTTPHeaders.contentType, oProps.HTTPHeaders.contentType)

switch manager.Location() {
case common.ELocation.Blob():
ValidatePropertyPtr(a, "Blob type", vProps.BlobProperties.Type, oProps.BlobProperties.Type)
ValidateTags(a, vProps.BlobProperties.Tags, oProps.BlobProperties.Tags)
ValidatePropertyPtr(a, "Block blob access tier", vProps.BlobProperties.BlockBlobAccessTier, oProps.BlobProperties.BlockBlobAccessTier)
ValidatePropertyPtr(a, "Page blob access tier", vProps.BlobProperties.PageBlobAccessTier, oProps.BlobProperties.PageBlobAccessTier)
case common.ELocation.File():
ValidatePropertyPtr(a, "Attributes", vProps.FileProperties.FileAttributes, oProps.FileProperties.FileAttributes)
ValidatePropertyPtr(a, "Creation time", vProps.FileProperties.FileCreationTime, oProps.FileProperties.FileCreationTime)
ValidatePropertyPtr(a, "Last write time", vProps.FileProperties.FileLastWriteTime, oProps.FileProperties.FileLastWriteTime)
ValidatePropertyPtr(a, "Permissions", vProps.FileProperties.FilePermissions, oProps.FileProperties.FilePermissions)
case common.ELocation.BlobFS():
ValidatePropertyPtr(a, "Permissions", vProps.BlobFSProperties.Permissions, oProps.BlobFSProperties.Permissions)
ValidatePropertyPtr(a, "Owner", vProps.BlobFSProperties.Owner, oProps.BlobFSProperties.Owner)
ValidatePropertyPtr(a, "Group", vProps.BlobFSProperties.Group, oProps.BlobFSProperties.Group)
ValidatePropertyPtr(a, "ACL", vProps.BlobFSProperties.ACL, oProps.BlobFSProperties.ACL)
}
},
})
}

type AzCopyOutputKey struct {
Path string
VersionId string
SnapshotId string
}

func ValidateListOutput(a Asserter, stdout AzCopyStdout, expectedObjects map[AzCopyOutputKey]cmd.AzCopyListObject, expectedSummary *cmd.AzCopyListSummary) {
if dryrunner, ok := a.(DryrunAsserter); ok && dryrunner.Dryrun() {
return
}

listStdout, ok := stdout.(*AzCopyParsedListStdout)
a.AssertNow("stdout must be AzCopyParsedListStdout", Equal{}, ok, true)

a.AssertNow("stdout and expected objects must not be null", Not{IsNil{}}, a, stdout, expectedObjects)
a.Assert("map of objects must be equivalent in size", Equal{}, len(expectedObjects), len(listStdout.Items))
a.Assert("map of objects must match", MapContains[AzCopyOutputKey, cmd.AzCopyListObject]{TargetMap: expectedObjects}, listStdout.Items)
a.Assert("summary must match", Equal{}, listStdout.Summary, DerefOrZero(expectedSummary))
}

func ValidateErrorOutput(a Asserter, stdout AzCopyStdout, errorMsg string) {
if dryrunner, ok := a.(DryrunAsserter); ok && dryrunner.Dryrun() {
return
}
for _, line := range stdout.RawStdout() {
if strings.Contains(line, errorMsg) {
return
}
}
fmt.Println(stdout.String())
a.Error("expected error message not found in azcopy output")
}

func ValidateContainsError(a Asserter, stdout AzCopyStdout, errorMsg []string) {
if dryrunner, ok := a.(DryrunAsserter); ok && dryrunner.Dryrun() {
return
}
for _, line := range stdout.RawStdout() {
if checkMultipleErrors(errorMsg, line) {
return
}
}
fmt.Println(stdout.String())
a.Error("expected error message not found in azcopy output")
}

func checkMultipleErrors(errorMsg []string, line string) bool {
for _, e := range errorMsg {
if strings.Contains(line, e) {
return true
}
}

return false
}
Loading

0 comments on commit 7fe523c

Please sign in to comment.