From f80e276ec058b9787c56495de72d0713debb0504 Mon Sep 17 00:00:00 2001 From: tanyasethi-msft <124860586+tanyasethi-msft@users.noreply.github.com> Date: Wed, 13 Dec 2023 18:10:21 +0530 Subject: [PATCH] Encryption scope SAS (#22099) * corrected release date in changelog * STG84 swagger and azcore, azidentity, azblob dependency updates * dependancy updates * Initial changes for encryption scope sas * indirect dependancy * update code generator version * using STG 82 swagger, and revert code gen version * adding transforms for missing header files * re-generate zz_options.go file * reverting to azblob 1.1.0 * reverting to azblob 1.0.0 * Updated input file from main to latest commit id * added tests for encryption scope sas * add test for user delegation sas * lint error fix * Fix lint error in test * created new datalake encryption scope var * removed blob account encryption scope * push recording * add encryption scope in user delegation * recordings * linter error * minor comments resolve * arm template changes * arm template changes * test fix --------- Co-authored-by: Sourav Gupta --- sdk/storage/azdatalake/assets.json | 2 +- sdk/storage/azdatalake/file/client_test.go | 174 ++++++++++++++++++ .../internal/testcommon/clients_auth.go | 15 +- sdk/storage/azdatalake/sas/account.go | 29 +-- sdk/storage/azdatalake/sas/query_params.go | 15 +- sdk/storage/azdatalake/sas/service.go | 34 ++-- sdk/storage/azdatalake/test-resources.json | 33 ++-- 7 files changed, 248 insertions(+), 54 deletions(-) diff --git a/sdk/storage/azdatalake/assets.json b/sdk/storage/azdatalake/assets.json index 2a8f02268497..996f24904f8c 100644 --- a/sdk/storage/azdatalake/assets.json +++ b/sdk/storage/azdatalake/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "go", "TagPrefix": "go/storage/azdatalake", - "Tag": "go/storage/azdatalake_7c0fb050cb" + "Tag": "go/storage/azdatalake_b5323f920e" } \ No newline at end of file diff --git a/sdk/storage/azdatalake/file/client_test.go b/sdk/storage/azdatalake/file/client_test.go index 7294f001d8c4..2716968c3ea9 100644 --- a/sdk/storage/azdatalake/file/client_test.go +++ b/sdk/storage/azdatalake/file/client_test.go @@ -11,6 +11,7 @@ import ( "context" "crypto/md5" "encoding/binary" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/service" "hash/crc64" "io" "math/rand" @@ -1337,6 +1338,179 @@ func (s *UnrecordedTestSuite) TestFileDeleteWithSAS() { _require.NoError(err) } +func (s *UnrecordedTestSuite) TestFileEncryptionScopeSAS() { + _require := require.New(s.T()) + testName := s.T().Name() + svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient := testcommon.CreateNewFileSystem(context.Background(), _require, filesystemName, svcClient) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + encryptionScope, err := testcommon.GetRequiredEnv(testcommon.DataLakeEncryptionScopeEnvVar) + _require.Nil(err) + + cred, err := testcommon.GetGenericSharedKeyCredential(testcommon.TestAccountDatalake) + _require.NoError(err) + + perms := sas.FilePermissions{Read: true, Create: true, Write: true, Move: true, Delete: true, List: true} + sasQueryParams, err := sas.DatalakeSignatureValues{ + Protocol: sas.ProtocolHTTPS, // Users MUST use HTTPS (not HTTP) + ExpiryTime: time.Now().UTC().Add(48 * time.Hour), // 48-hours before expiration + FileSystemName: filesystemName, + Permissions: perms.String(), + EncryptionScope: encryptionScope, + }.SignWithSharedKey(cred) + _require.NoError(err) + + sasToken := sasQueryParams.Encode() + + srcFileClient, err := file.NewClientWithNoCredential(fsClient.DFSURL()+"/file?"+sasToken, nil) + _require.NoError(err) + _require.NotNil(srcFileClient) + + _, err = srcFileClient.Create(context.Background(), nil) + _require.NoError(err) + + response, err := srcFileClient.SetMetadata(context.Background(), testcommon.BasicMetadata, nil) + _require.NoError(err) + _require.Equal(encryptionScope, *response.EncryptionScope) + +} + +func (s *UnrecordedTestSuite) TestAccountEncryptionScopeSAS() { + _require := require.New(s.T()) + testName := s.T().Name() + svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient := testcommon.CreateNewFileSystem(context.Background(), _require, filesystemName, svcClient) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + encryptionScope, err := testcommon.GetRequiredEnv(testcommon.DataLakeEncryptionScopeEnvVar) + _require.Nil(err) + + credential, err := testcommon.GetGenericSharedKeyCredential(testcommon.TestAccountDatalake) + _require.Nil(err) + + sasQueryParams, err := sas.AccountSignatureValues{ + Protocol: sas.ProtocolHTTPS, // Users MUST use HTTPS (not HTTP) + ExpiryTime: time.Now().UTC().Add(48 * time.Hour), // 48-hours before expiration + Permissions: to.Ptr(sas.AccountPermissions{Read: true, Create: true, Write: true, Delete: true}).String(), + ResourceTypes: to.Ptr(sas.AccountResourceTypes{Service: true, Container: true, Object: true}).String(), + EncryptionScope: encryptionScope, + }.SignWithSharedKey(credential) + _require.NoError(err) + + sasToken := sasQueryParams.Encode() + + srcFileClient, err := file.NewClientWithNoCredential(fsClient.DFSURL()+"/file?"+sasToken, nil) + _require.NoError(err) + _require.NotNil(srcFileClient) + + resp, err := srcFileClient.Create(context.Background(), nil) + _require.NoError(err) + _require.NotNil(resp) + + // create local file + _, content := generateData(10 * 1024) + err = os.WriteFile("testFile", content, 0644) + _require.NoError(err) + + defer func() { + err = os.Remove("testFile") + _require.NoError(err) + }() + + fh, err := os.Open("testFile") + _require.NoError(err) + + defer func(fh *os.File) { + err := fh.Close() + _require.NoError(err) + }(fh) + + // upload the file + err = srcFileClient.UploadFile(context.Background(), fh, &file.UploadFileOptions{ + Concurrency: 5, + ChunkSize: 2 * 1024, + }) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + response, err := srcFileClient.DownloadStream(context.Background(), nil) + _require.NoError(err) + testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + _require.Equal(encryptionScope, *response.EncryptionScope) + + // validate the data downloaded + downloadedData, err := io.ReadAll(response.Body) + _require.NoError(err) + _require.Equal(len(content), len(downloadedData)) + _require.EqualValues(content, downloadedData) +} + +func (s *UnrecordedTestSuite) TestGetUserDelegationEncryptionScopeSAS() { + _require := require.New(s.T()) + testName := s.T().Name() + + accountName, _ := testcommon.GetGenericAccountInfo(testcommon.TestAccountDatalake) + _require.Greater(len(accountName), 0) + + cred, err := testcommon.GetGenericTokenCredential() + _require.NoError(err) + + svcClient, err := service.NewClient("https://"+accountName+".dfs.core.windows.net/", cred, nil) + _require.NoError(err) + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient := testcommon.CreateNewFileSystem(context.Background(), _require, filesystemName, svcClient) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + // Set current and past time and create key + currentTime := time.Now().UTC().Add(-10 * time.Second) + pastTime := currentTime.Add(48 * time.Hour) + info := service.KeyInfo{ + Start: to.Ptr(currentTime.UTC().Format(sas.TimeFormat)), + Expiry: to.Ptr(pastTime.UTC().Format(sas.TimeFormat)), + } + + udc, err := svcClient.GetUserDelegationCredential(context.Background(), info, nil) + _require.NoError(err) + + // get permissions and details for sas + encryptionScope, err := testcommon.GetRequiredEnv(testcommon.DataLakeEncryptionScopeEnvVar) + _require.Nil(err) + + // Create Blob Signature Values with desired permissions and sign with user delegation credential + perms := sas.FilePermissions{Read: true, Create: true, Write: true, Move: true, Delete: true, List: true} + sasQueryParams, err := sas.DatalakeSignatureValues{ + Protocol: sas.ProtocolHTTPS, // Users MUST use HTTPS (not HTTP) + StartTime: time.Now().UTC().Add(time.Second * -10), + ExpiryTime: time.Now().UTC().Add(15 * time.Minute), // 15 minutes before expiration + FileSystemName: filesystemName, + Permissions: perms.String(), + EncryptionScope: encryptionScope, + }.SignWithUserDelegation(udc) + _require.Nil(err) + + sasURL := fsClient.DFSURL() + "/file?" + sasQueryParams.Encode() + // This URL can be used to authenticate requests now + srcFileClient, err := file.NewClientWithNoCredential(sasURL, nil) + _require.NoError(err) + + _, err = srcFileClient.Create(context.Background(), nil) + _require.NoError(err) + + response, err := srcFileClient.SetMetadata(context.Background(), testcommon.BasicMetadata, nil) + _require.NoError(err) + _require.Equal(encryptionScope, *response.EncryptionScope) +} + func (s *RecordedTestSuite) TestFileGetAccessControlWithNilAccessConditions() { _require := require.New(s.T()) testName := s.T().Name() diff --git a/sdk/storage/azdatalake/internal/testcommon/clients_auth.go b/sdk/storage/azdatalake/internal/testcommon/clients_auth.go index d79e3c16e2fd..87e1c434809e 100644 --- a/sdk/storage/azdatalake/internal/testcommon/clients_auth.go +++ b/sdk/storage/azdatalake/internal/testcommon/clients_auth.go @@ -27,13 +27,14 @@ import ( ) const ( - DefaultEndpointSuffix = "core.windows.net/" - DefaultBlobEndpointSuffix = "blob.core.windows.net/" - AccountNameEnvVar = "AZURE_STORAGE_ACCOUNT_NAME" - AccountKeyEnvVar = "AZURE_STORAGE_ACCOUNT_KEY" - DefaultEndpointSuffixEnvVar = "AZURE_STORAGE_ENDPOINT_SUFFIX" - SubscriptionID = "SUBSCRIPTION_ID" - ResourceGroupName = "RESOURCE_GROUP_NAME" + DefaultEndpointSuffix = "core.windows.net/" + DefaultBlobEndpointSuffix = "blob.core.windows.net/" + AccountNameEnvVar = "AZURE_STORAGE_ACCOUNT_NAME" + AccountKeyEnvVar = "AZURE_STORAGE_ACCOUNT_KEY" + DefaultEndpointSuffixEnvVar = "AZURE_STORAGE_ENDPOINT_SUFFIX" + DataLakeEncryptionScopeEnvVar = "DATALAKE_AZURE_STORAGE_ENCRYPTION_SCOPE" + SubscriptionID = "SUBSCRIPTION_ID" + ResourceGroupName = "RESOURCE_GROUP_NAME" ) const ( diff --git a/sdk/storage/azdatalake/sas/account.go b/sdk/storage/azdatalake/sas/account.go index e5681c4ebdba..9273dcbddd63 100644 --- a/sdk/storage/azdatalake/sas/account.go +++ b/sdk/storage/azdatalake/sas/account.go @@ -25,13 +25,14 @@ type UserDelegationCredential = exported.UserDelegationCredential // AccountSignatureValues is used to generate a Shared Access Signature (SAS) for an Azure Storage account. // For more information, see https://docs.microsoft.com/rest/api/storageservices/constructing-an-account-sas type AccountSignatureValues struct { - Version string `param:"sv"` // If not specified, this format to SASVersion - Protocol Protocol `param:"spr"` // See the SASProtocol* constants - StartTime time.Time `param:"st"` // Not specified if IsZero - ExpiryTime time.Time `param:"se"` // Not specified if IsZero - Permissions string `param:"sp"` // Create by initializing AccountPermissions and then call String() - IPRange IPRange `param:"sip"` - ResourceTypes string `param:"srt"` // Create by initializing AccountResourceTypes and then call String() + Version string `param:"sv"` // If not specified, this format to SASVersion + Protocol Protocol `param:"spr"` // See the SASProtocol* constants + StartTime time.Time `param:"st"` // Not specified if IsZero + ExpiryTime time.Time `param:"se"` // Not specified if IsZero + Permissions string `param:"sp"` // Create by initializing AccountPermissions and then call String() + IPRange IPRange `param:"sip"` + ResourceTypes string `param:"srt"` // Create by initializing AccountResourceTypes and then call String() + EncryptionScope string `param:"ses"` } // SignWithSharedKey uses an account's shared key credential to sign this signature values to produce @@ -68,6 +69,7 @@ func (v AccountSignatureValues) SignWithSharedKey(sharedKeyCredential *SharedKey v.IPRange.String(), string(v.Protocol), v.Version, + v.EncryptionScope, ""}, // That is right, the account SAS requires a terminating extra newline "\n") @@ -77,12 +79,13 @@ func (v AccountSignatureValues) SignWithSharedKey(sharedKeyCredential *SharedKey } p := QueryParameters{ // Common SAS parameters - version: v.Version, - protocol: v.Protocol, - startTime: v.StartTime, - expiryTime: v.ExpiryTime, - permissions: v.Permissions, - ipRange: v.IPRange, + version: v.Version, + protocol: v.Protocol, + startTime: v.StartTime, + expiryTime: v.ExpiryTime, + permissions: v.Permissions, + ipRange: v.IPRange, + encryptionScope: v.EncryptionScope, // Account-specific SAS parameters services: "b", // will always be "b" diff --git a/sdk/storage/azdatalake/sas/query_params.go b/sdk/storage/azdatalake/sas/query_params.go index 7806687833c0..5e50aa4e301d 100644 --- a/sdk/storage/azdatalake/sas/query_params.go +++ b/sdk/storage/azdatalake/sas/query_params.go @@ -21,7 +21,7 @@ const ( var ( // Version is the default version encoded in the SAS token. - Version = "2020-02-10" + Version = "2020-12-06" ) // TimeFormats ISO 8601 format. @@ -137,6 +137,7 @@ type QueryParameters struct { authorizedObjectID string `param:"saoid"` unauthorizedObjectID string `param:"suoid"` correlationID string `param:"scid"` + encryptionScope string `param:"ses"` // private member used for startTime and expiryTime formatting. stTimeFormat string seTimeFormat string @@ -277,6 +278,11 @@ func (p *QueryParameters) SignedDirectoryDepth() string { return p.signedDirectoryDepth } +// SignedEncryptionScope returns encryptionScope. +func (p *QueryParameters) SignedEncryptionScope() string { + return p.encryptionScope +} + // Encode encodes the SAS query parameters into URL encoded form sorted by key. func (p *QueryParameters) Encode() string { v := url.Values{} @@ -349,6 +355,9 @@ func (p *QueryParameters) Encode() string { if p.correlationID != "" { v.Add("scid", p.correlationID) } + if p.encryptionScope != "" { + v.Add("ses", p.encryptionScope) + } return v.Encode() } @@ -418,6 +427,8 @@ func NewQueryParameters(values url.Values) QueryParameters { p.unauthorizedObjectID = val case "scid": p.correlationID = val + case "ses": + p.encryptionScope = val default: continue // query param didn't get recognized } @@ -495,6 +506,8 @@ func newQueryParameters(values url.Values, deleteSASParametersFromValues bool) Q p.unauthorizedObjectID = val case "scid": p.correlationID = val + case "ses": + p.encryptionScope = val default: isSASKey = false // We didn't recognize the query parameter } diff --git a/sdk/storage/azdatalake/sas/service.go b/sdk/storage/azdatalake/sas/service.go index 22784206a667..306aafc9d7cd 100644 --- a/sdk/storage/azdatalake/sas/service.go +++ b/sdk/storage/azdatalake/sas/service.go @@ -42,6 +42,7 @@ type DatalakeSignatureValues struct { AuthorizedObjectID string // saoid UnauthorizedObjectID string // suoid CorrelationID string // scid + EncryptionScope string `param:"ses"` } //TODO: add snapshot and versioning support in the future @@ -94,7 +95,8 @@ func (v DatalakeSignatureValues) SignWithSharedKey(sharedKeyCredential *SharedKe string(v.Protocol), v.Version, resource, - "", //snapshot not supported + "", //snapshot not supported + v.EncryptionScope, v.CacheControl, // rscc v.ContentDisposition, // rscd v.ContentEncoding, // rsce @@ -109,13 +111,13 @@ func (v DatalakeSignatureValues) SignWithSharedKey(sharedKeyCredential *SharedKe p := QueryParameters{ // Common SAS parameters - version: v.Version, - protocol: v.Protocol, - startTime: v.StartTime, - expiryTime: v.ExpiryTime, - permissions: v.Permissions, - ipRange: v.IPRange, - + version: v.Version, + protocol: v.Protocol, + startTime: v.StartTime, + expiryTime: v.ExpiryTime, + permissions: v.Permissions, + ipRange: v.IPRange, + encryptionScope: v.EncryptionScope, // Container/Blob-specific SAS parameters resource: resource, cacheControl: v.CacheControl, @@ -197,7 +199,8 @@ func (v DatalakeSignatureValues) SignWithUserDelegation(userDelegationCredential string(v.Protocol), v.Version, resource, - "", //snapshot not supported + "", //snapshot not supported + v.EncryptionScope, v.CacheControl, // rscc v.ContentDisposition, // rscd v.ContentEncoding, // rsce @@ -212,12 +215,13 @@ func (v DatalakeSignatureValues) SignWithUserDelegation(userDelegationCredential p := QueryParameters{ // Common SAS parameters - version: v.Version, - protocol: v.Protocol, - startTime: v.StartTime, - expiryTime: v.ExpiryTime, - permissions: v.Permissions, - ipRange: v.IPRange, + version: v.Version, + protocol: v.Protocol, + startTime: v.StartTime, + expiryTime: v.ExpiryTime, + permissions: v.Permissions, + ipRange: v.IPRange, + encryptionScope: v.EncryptionScope, // Container/Blob-specific SAS parameters resource: resource, diff --git a/sdk/storage/azdatalake/test-resources.json b/sdk/storage/azdatalake/test-resources.json index 88c380e64a8f..a85273bfea5d 100644 --- a/sdk/storage/azdatalake/test-resources.json +++ b/sdk/storage/azdatalake/test-resources.json @@ -27,11 +27,11 @@ "blobDataOwnerRoleId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/b7e6dc6d-f1e8-4753-8033-0f276bb0955b')]", "primaryAccountName": "[concat(parameters('baseName'), 'prim')]", "immutableAccountName": "[concat(parameters('baseName'), 'imm')]", - "primaryEncryptionScopeName": "encryptionScope", - "primaryEncryptionScope": "[concat(parameters('baseName'), 'prim', concat('/', variables('primaryEncryptionScopeName')))]", "secondaryAccountName": "[concat(parameters('baseName'), 'sec')]", "premiumAccountName": "[concat(parameters('baseName'), 'prem')]", "dataLakeAccountName": "[concat(parameters('baseName'), 'dtlk')]", + "dataLakeEncryptionScopeName": "encryptionScope", + "dataLakeEncryptionScope": "[concat(parameters('baseName'), 'dtlk', concat('/', variables('dataLakeEncryptionScopeName')))]", "softDeleteAccountName": "[concat(parameters('baseName'), 'sftdl')]", "premiumFileAccountName": "[concat(parameters('baseName'), 'pfile')]", "webjobsPrimaryAccountName": "[concat(parameters('baseName'), 'wjprim')]", @@ -160,18 +160,6 @@ "[variables('immutableAccountName')]" ] }, - { - "type": "Microsoft.Storage/storageAccounts/encryptionScopes", - "apiVersion": "[variables('mgmtApiVersion')]", - "name": "[variables('primaryEncryptionScope')]", - "properties": { - "source": "Microsoft.Storage", - "state": "Enabled" - }, - "dependsOn": [ - "[variables('primaryAccountName')]" - ] - }, { "type": "Microsoft.Storage/storageAccounts", "apiVersion": "[variables('mgmtApiVersion')]", @@ -206,6 +194,18 @@ "accessTier": "Hot" } }, + { + "type": "Microsoft.Storage/storageAccounts/encryptionScopes", + "apiVersion": "[variables('mgmtApiVersion')]", + "name": "[variables('dataLakeEncryptionScope')]", + "properties": { + "source": "Microsoft.Storage", + "state": "Enabled" + }, + "dependsOn": [ + "[variables('dataLakeAccountName')]" + ] + }, { "type": "Microsoft.Storage/storageAccounts", "apiVersion": "[variables('mgmtApiVersion')]", @@ -575,10 +575,9 @@ "type": "string", "value": "[variables('location')]" }, - "AZURE_STORAGE_ENCRYPTION_SCOPE": { + "DATALAKE_AZURE_STORAGE_ENCRYPTION_SCOPE": { "type": "string", - "value": "[variables('primaryEncryptionScopeName')]" + "value": "[variables('dataLakeEncryptionScopeName')]" } } } - \ No newline at end of file