[storage] port HNS permission support from storage/stable (#26563)

Port PR #25675

- update blob service version to 2023-01-03
- add "appendBlock - append large block" test
- port file-datalake source changes
- port file-datalake tests and add recordings

sdk/storage/storage-blob/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "js",
   "TagPrefix": "js/storage/storage-blob",
-  "Tag": "js/storage/storage-blob_8c8488b395"
+  "Tag": "js/storage/storage-blob_0dfb9bf85a"
 }

sdk/storage/storage-blob/src/utils/constants.ts

@@ -2,7 +2,7 @@
 // Licensed under the MIT license.
 
 export const SDK_VERSION: string = "12.20.0";
-export const SERVICE_VERSION: string = "2021-10-04";
+export const SERVICE_VERSION: string = "2023-01-03";
 
 export const BLOCK_BLOB_MAX_UPLOAD_BLOB_BYTES: number = 256 * 1024 * 1024; // 256MB
 export const BLOCK_BLOB_MAX_STAGE_BLOCK_BYTES: number = 4000 * 1024 * 1024; // 4000MB

sdk/storage/storage-blob/swagger/README.md

@@ -1464,4 +1464,13 @@ directive:
         ];
 ```
 
+### Update service version from "2021-12-02" to "2023-01-03"
+
+```yaml
+directive:
+  - from: swagger-document
+    where: $.parameters.ApiVersionParameter
+    transform: $.enum = [ "2023-01-03" ];
+```
+
 ![Impressions](https://azure-sdk-impressions.azurewebsites.net/api/impressions/azure-sdk-for-js%2Fsdk%2Fstorage%2Fstorage-blob%2Fswagger%2FREADME.png)

sdk/storage/storage-blob/test/node/appendblobclient.spec.ts

@@ -24,7 +24,7 @@ import {
 } from "../utils";
 import { TokenCredential } from "@azure/core-auth";
 import { assertClientUsesTokenCredential } from "../utils/assert";
-import { Recorder } from "@azure-tools/test-recorder";
+import { Recorder, isLiveMode } from "@azure-tools/test-recorder";
 import { Test_CPK_INFO } from "../utils/fakeTestSecrets";
 import { Context } from "mocha";
 
@@ -37,6 +37,8 @@ describe("AppendBlobClient Node.js only", () => {
   let recorder: Recorder;
 
   let blobServiceClient: BlobServiceClient;
+  const timeoutForLargeFileUploadingTest = 20 * 60 * 1000;
+
   beforeEach(async function (this: Context) {
     recorder = new Recorder(this.currentTest);
     await recorder.start(recorderEnvSetup);
@@ -339,4 +341,23 @@ describe("AppendBlobClient Node.js only", () => {
     assert.equal(await bodyToString(downloadResponse, content.length * 2), content + content);
     assert.equal(downloadResponse.contentLength!, content.length * 2);
   });
+
+  it("appendBlock - append large block", async function (this: Context) {
+    if (!isLiveMode()) {
+      // Recorder file larger than github limitation
+      this.skip();
+    }
+    await appendBlobClient.create();
+
+    const largeBlockSize = 100 * 1024 * 1024;
+    const content = new Uint8Array(largeBlockSize);
+    for (let i = 0; i < largeBlockSize; i = i + 1000) {
+      content[i] = i;
+    }
+    await appendBlobClient.appendBlock(content, content.length);
+
+    const downloadResponse = await appendBlobClient.downloadToBuffer(0);
+    assert.deepStrictEqual(downloadResponse, content);
+    assert.equal(downloadResponse.length, content.length);
+  }).timeout(timeoutForLargeFileUploadingTest);
 });

sdk/storage/storage-file-datalake/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "js",
   "TagPrefix": "js/storage/storage-file-datalake",
-  "Tag": "js/storage/storage-file-datalake_1bed51a03a"
+  "Tag": "js/storage/storage-file-datalake_0d90641797"
 }

sdk/storage/storage-file-datalake/review/storage-file-datalake.api.md

@@ -678,6 +678,8 @@ export interface FileReadHeaders {
     // (undocumented)
     fileContentMD5?: Uint8Array;
     // (undocumented)
+    group?: string;
+    // (undocumented)
     isServerEncrypted?: boolean;
     // (undocumented)
     lastModified?: Date;
@@ -690,6 +692,10 @@ export interface FileReadHeaders {
     // (undocumented)
     metadata?: Metadata;
     // (undocumented)
+    owner?: string;
+    // (undocumented)
+    permissions?: PathPermissions;
+    // (undocumented)
     requestId?: string;
     // (undocumented)
     version?: string;
@@ -1474,6 +1480,8 @@ export interface PathGetPropertiesHeaders {
     etag?: string;
     expiresOn?: Date;
     // (undocumented)
+    group?: string;
+    // (undocumented)
     isIncrementalCopy?: boolean;
     // (undocumented)
     isServerEncrypted?: boolean;
@@ -1488,6 +1496,10 @@ export interface PathGetPropertiesHeaders {
     // (undocumented)
     metadata?: Metadata;
     // (undocumented)
+    owner?: string;
+    // (undocumented)
+    permissions?: PathPermissions;
+    // (undocumented)
     requestId?: string;
     // (undocumented)
     version?: string;

sdk/storage/storage-file-datalake/src/clients.ts

@@ -103,7 +103,7 @@ import {
   assertResponse,
   ensureCpkIfSpecified,
   getURLPathAndQuery,
-  ParseEncryptionContextHeaderValue,
+  ParsePathGetPropertiesExtraHeaderValues,
   setURLPath,
   setURLQueries,
 } from "./utils/utils.common";
@@ -677,7 +677,7 @@ export class DataLakePathClient extends StorageClient {
           customerProvidedKey: toBlobCpkInfo(options.customerProvidedKey),
           tracingOptions: updatedOptions.tracingOptions,
         });
-        return ParseEncryptionContextHeaderValue(response as PathGetPropertiesResponse);
+        return ParsePathGetPropertiesExtraHeaderValues(response as PathGetPropertiesResponse);
       }
     );
   }
@@ -1230,7 +1230,7 @@ export class DataLakeFileClient extends DataLakePathClient {
         customerProvidedKey: toBlobCpkInfo(updatedOptions.customerProvidedKey),
       });
 
-      const response = ParseEncryptionContextHeaderValue(
+      const response = ParsePathGetPropertiesExtraHeaderValues(
         rawResponse as FileReadResponse
       ) as FileReadResponse;
       if (!isNode && !response.contentAsBlob) {

sdk/storage/storage-file-datalake/src/models.ts

@@ -975,6 +975,9 @@ export interface PathGetPropertiesHeaders {
    * Optional. Specifies the encryption context to set on the file.
    */
   encryptionContext?: string;
+  owner?: string;
+  group?: string;
+  permissions?: PathPermissions;
 }
 
 export type PathGetPropertiesResponse = WithResponse<
@@ -1195,6 +1198,9 @@ export interface FileReadHeaders {
    * Specifies the encryption context to set on the file.
    */
   encryptionContext?: string;
+  owner?: string;
+  group?: string;
+  permissions?: PathPermissions;
 }
 
 export type FileReadResponse = WithResponse<

sdk/storage/storage-file-datalake/src/utils/constants.ts

@@ -2,7 +2,7 @@
 // Licensed under the MIT license.
 
 export const SDK_VERSION: string = "12.20.0";
-export const SERVICE_VERSION: string = "2021-12-02";
+export const SERVICE_VERSION: string = "2023-01-03";
 
 export const KB: number = 1024;
 export const MB: number = KB * 1024;

sdk/storage/storage-file-datalake/src/utils/utils.common.ts

@@ -3,7 +3,7 @@
 import { createHttpHeaders, HttpHeaders } from "@azure/core-rest-pipeline";
 import { isNode } from "@azure/core-util";
 import { ContainerEncryptionScope, WithResponse } from "@azure/storage-blob";
-import { CpkInfo, FileSystemEncryptionScope } from "../models";
+import { CpkInfo, FileSystemEncryptionScope, PathPermissions } from "../models";
 
 import {
   DevelopmentConnectionString,
@@ -14,6 +14,7 @@ import {
 } from "./constants";
 import { HttpResponse } from "@azure/storage-blob";
 import { HttpHeadersLike } from "@azure/core-http-compat";
+import { toPermissions } from "../transforms";
 
 /**
  * Reserved URL characters must be properly escaped for Storage services like Blob or File.
@@ -631,30 +632,51 @@ export function assertResponse<T extends object, Headers = undefined, Body = und
   throw new TypeError(`Unexpected response object ${response}`);
 }
 
-export interface RawResponseWithEncryptionContextLike {
+export interface PathGetPropertiesRawResponseWithExtraPropertiesLike {
   encryptionContext?: string;
+  owner?: string;
+  group?: string;
+  permissions?: PathPermissions;
   _response: HttpResponse & {
     parsedHeaders: {
       encryptionContext?: string;
+      owner?: string;
+      group?: string;
+      permissions?: PathPermissions;
     };
   };
 }
 
-/**
- * Parse value of encryption context from headers in raw response.
- */
-export function ParseEncryptionContextHeaderValue(
-  rawResponse: RawResponseWithEncryptionContextLike
-): RawResponseWithEncryptionContextLike {
-  const response = rawResponse;
+function ParseHeaderValue(
+  rawResponse: PathGetPropertiesRawResponseWithExtraPropertiesLike,
+  headerName: string
+): string | undefined {
   if (rawResponse._response) {
     const headers = rawResponse._response.headers as HttpHeadersLike;
     if (headers) {
-      response.encryptionContext = headers.get("x-ms-encryption-context");
-      if (response._response.parsedHeaders) {
-        response._response.parsedHeaders.encryptionContext = response.encryptionContext;
-      }
+      return headers.get(headerName);
     }
   }
+
+  return undefined;
+}
+
+/**
+ * Parse extra properties values from headers in raw response.
+ */
+export function ParsePathGetPropertiesExtraHeaderValues(
+  rawResponse: PathGetPropertiesRawResponseWithExtraPropertiesLike
+): PathGetPropertiesRawResponseWithExtraPropertiesLike {
+  const response = rawResponse;
+  response.encryptionContext = ParseHeaderValue(rawResponse, "x-ms-encryption-context");
+  response.owner = ParseHeaderValue(rawResponse, "x-ms-owner");
+  response.group = ParseHeaderValue(rawResponse, "x-ms-group");
+  response.permissions = toPermissions(ParseHeaderValue(rawResponse, "x-ms-permissions"));
+  if (response._response?.parsedHeaders) {
+    response._response.parsedHeaders.encryptionContext = response.encryptionContext;
+    response._response.parsedHeaders.owner = response.owner;
+    response._response.parsedHeaders.group = response.group;
+    response._response.parsedHeaders.permissions = response.permissions;
+  }
   return response;
 }

sdk/storage/storage-file-datalake/swagger/README.md

@@ -354,5 +354,5 @@ directive:
 directive:
   - from: swagger-document
     where: $.parameters.ApiVersionParameter
-    transform: $.enum = [ "2022-11-02" ];
+    transform: $.enum = [ "2023-01-03" ];
 ```