Http stream end to end tests

.eslintrc.json

@@ -29,6 +29,7 @@
         ],
         "deprecation/deprecation": "error",
         "@typescript-eslint/no-empty-interface": "off",
+        "@typescript-eslint/no-empty-function": "off",
         "@typescript-eslint/no-explicit-any": "off",
         "@typescript-eslint/no-namespace": "off",
         "@typescript-eslint/no-unused-vars": ["error", { "argsIgnorePattern": "^_" }],

app/v3/httpTriggerRandomDelay/function.json

@@ -0,0 +1,20 @@
+{
+  "bindings": [
+    {
+      "authLevel": "anonymous",
+      "type": "httpTrigger",
+      "direction": "in",
+      "name": "req",
+      "methods": [
+        "get",
+        "post"
+      ]
+    },
+    {
+      "type": "http",
+      "direction": "out",
+      "name": "res"
+    }
+  ],
+  "scriptFile": "../dist/httpTriggerRandomDelay/index.js"
+}

app/v3/httpTriggerRandomDelay/index.ts

@@ -0,0 +1,19 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License.
+
+import { AzureFunction, Context, HttpRequest } from '@azure/functions';
+import { addRandomDelay } from '../utils/getRandomTestData';
+
+const httpTrigger: AzureFunction = async function (context: Context, req: HttpRequest): Promise<void> {
+    context.log(`Http function processed request for url "${req.url}"`);
+
+    await addRandomDelay();
+
+    const name = req.query.name || req.body || 'world';
+
+    context.res = {
+        body: `Hello, ${name}!`,
+    };
+};
+
+export default httpTrigger;

app/v3/utils/delay.ts

@@ -0,0 +1,13 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License.
+
+export async function delay(ms: number): Promise<void> {
+    await new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export function delaySync(ms: number): void {
+    const endTime = Date.now() + ms;
+    while (Date.now() < endTime) {
+        // wait
+    }
+}

app/v3/utils/getRandomTestData.ts

@@ -0,0 +1,32 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License.
+
+import * as crypto from 'crypto';
+import { delay, delaySync } from './delay';
+
+export function getRandomTestData(): string {
+    // This should start with non-numeric data to prevent this bug from causing a test failure
+    // https://github.com/Azure/azure-functions-nodejs-library/issues/90
+    return `testData${getRandomHexString()}`;
+}
+
+export function getRandomHexString(length = 10): string {
+    const buffer: Buffer = crypto.randomBytes(Math.ceil(length / 2));
+    return buffer.toString('hex').slice(0, length);
+}
+
+export function getRandomInt(min: number, max: number): number {
+    return Math.floor(Math.random() * (max - min) + min);
+}
+
+export function getRandomBoolean(percentTrue: number): boolean {
+    return Math.random() * 100 > percentTrue;
+}
+
+export async function addRandomDelay(): Promise<void> {
+    if (getRandomBoolean(95)) {
+        await delay(getRandomInt(0, 250));
+    } else {
+        delaySync(getRandomInt(0, 10));
+    }
+}

app/v4-oldConfig/src/setup.ts

@@ -0,0 +1,6 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License.
+
+import { app } from '@azure/functions';
+
+app.setup({ enableHttpStream: false });

app/v4/package.json

@@ -12,13 +12,13 @@
     "test": "echo \"No tests yet...\""
   },
   "dependencies": {
-    "@azure/functions": "^4.0.0"
+    "@azure/functions": "file:azure-functions-4.1.0.tgz"
   },
   "devDependencies": {
     "@types/long": "^4.0.0",
     "@types/node": "^18.x",
-    "typescript": "^4.0.0",
-    "rimraf": "^5.0.0"
+    "rimraf": "^5.0.0",
+    "typescript": "^4.0.0"
   },
-  "main": "dist/src/{index.js,functions/*.js}"
-}
+  "main": "dist/src/{index.js,setup.js,functions/*.js}"
+}

app/v4/src/functions/httpTriggerRandomDelay.ts

@@ -0,0 +1,24 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License.
+
+import { app, HttpRequest, HttpResponseInit, InvocationContext } from '@azure/functions';
+import { addRandomDelay } from '../utils/getRandomTestData';
+
+export async function httpTriggerRandomDelay(
+    request: HttpRequest,
+    context: InvocationContext
+): Promise<HttpResponseInit> {
+    context.log(`Http function processed request for url "${request.url}"`);
+
+    await addRandomDelay();
+
+    const name = request.query.get('name') || (await request.text()) || 'world';
+
+    return { body: `Hello, ${name}!` };
+}
+
+app.http('httpTriggerRandomDelay', {
+    methods: ['GET', 'POST'],
+    authLevel: 'anonymous',
+    handler: httpTriggerRandomDelay,
+});

app/v4/src/functions/httpTriggerReceiveStream.ts

@@ -0,0 +1,22 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License.
+
+import { app, HttpRequest, HttpResponseInit, InvocationContext } from '@azure/functions';
+import { receiveStreamWithProgress } from '../utils/streamHttp';
+
+export async function httpTriggerReceiveStream(
+    request: HttpRequest,
+    context: InvocationContext
+): Promise<HttpResponseInit> {
+    context.log(`Http function processed request for url "${request.url}"`);
+
+    const bytesReceived = await receiveStreamWithProgress(request.body);
+
+    return { body: `Bytes received: ${bytesReceived}` };
+}
+
+app.http('httpTriggerReceiveStream', {
+    methods: ['GET', 'POST'],
+    authLevel: 'anonymous',
+    handler: httpTriggerReceiveStream,
+});

app/v4/src/functions/httpTriggerSendStream.ts

@@ -0,0 +1,22 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License.
+
+import { app, HttpRequest, HttpResponseInit, InvocationContext } from '@azure/functions';
+import { createRandomStream } from '../utils/streamHttp';
+
+export async function httpTriggerSendStream(
+    request: HttpRequest,
+    context: InvocationContext
+): Promise<HttpResponseInit> {
+    context.log(`Http function processed request for url "${request.url}"`);
+
+    const lengthInMb = request.query.get('lengthInMb');
+    const stream = createRandomStream(Number(lengthInMb));
+    return { body: stream };
+}
+
+app.http('httpTriggerSendStream', {
+    methods: ['GET', 'POST'],
+    authLevel: 'anonymous',
+    handler: httpTriggerSendStream,
+});

app/v4/src/setup.ts

@@ -0,0 +1,6 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License.
+
+import { app } from '@azure/functions';
+
+app.setup({ enableHttpStream: true });

app/v4/src/utils/delay.ts

@@ -0,0 +1,13 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License.
+
+export async function delay(ms: number): Promise<void> {
+    await new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export function delaySync(ms: number): void {
+    const endTime = Date.now() + ms;
+    while (Date.now() < endTime) {
+        // wait
+    }
+}

app/v4/src/utils/getRandomTestData.ts

@@ -0,0 +1,32 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License.
+
+import * as crypto from 'crypto';
+import { delay, delaySync } from './delay';
+
+export function getRandomTestData(): string {
+    // This should start with non-numeric data to prevent this bug from causing a test failure
+    // https://github.com/Azure/azure-functions-nodejs-library/issues/90
+    return `testData${getRandomHexString()}`;
+}
+
+export function getRandomHexString(length = 10): string {
+    const buffer: Buffer = crypto.randomBytes(Math.ceil(length / 2));
+    return buffer.toString('hex').slice(0, length);
+}
+
+export function getRandomInt(min: number, max: number): number {
+    return Math.floor(Math.random() * (max - min) + min);
+}
+
+export function getRandomBoolean(percentTrue: number): boolean {
+    return Math.random() * 100 > percentTrue;
+}
+
+export async function addRandomDelay(): Promise<void> {
+    if (getRandomBoolean(95)) {
+        await delay(getRandomInt(0, 250));
+    } else {
+        delaySync(getRandomInt(0, 10));
+    }
+}

app/v4/src/utils/streamHttp.ts

@@ -0,0 +1,57 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License.
+
+import * as crypto from 'crypto';
+import { Readable } from 'stream';
+import { delay } from './delay';
+
+export function createRandomStream(lengthInMb: number): Readable {
+    const stream = new Readable();
+    stream._read = () => {};
+    setTimeout(() => {
+        void sendRandomData(stream, lengthInMb);
+    }, 5);
+    return stream;
+}
+
+async function sendRandomData(stream: Readable, lengthInMb: number): Promise<void> {
+    const maxChunkSize = oneMb;
+    let remainingBytes = convertMbToB(lengthInMb);
+    do {
+        if (stream.readableLength > maxChunkSize) {
+            await delay(5);
+        } else {
+            const chunkSize = Math.min(maxChunkSize, remainingBytes);
+            stream.push(crypto.randomBytes(chunkSize));
+            remainingBytes -= chunkSize;
+        }
+    } while (remainingBytes > 0);
+    stream.push(null);
+}
+
+export async function receiveStreamWithProgress(stream: {
+    [Symbol.asyncIterator](): AsyncIterableIterator<string | Buffer>;
+}): Promise<number> {
+    let bytesReceived = 0;
+    const logInterval = 500;
+    let nextLogTime = Date.now();
+    for await (const chunk of stream) {
+        if (Date.now() > nextLogTime) {
+            nextLogTime = Date.now() + logInterval;
+            console.log(`Progress: ${convertBToMb(bytesReceived)}mb`);
+        }
+
+        bytesReceived += chunk.length;
+    }
+    return bytesReceived;
+}
+
+const oneMb = 1024 * 1024;
+
+export function convertMbToB(mb: number): number {
+    return mb * oneMb;
+}
+
+function convertBToMb(bytes: number) {
+    return Math.round(bytes / oneMb);
+}

azure-pipelines/test.yml

@@ -89,9 +89,10 @@ jobs:
   - script: mv *.tgz package.tgz
     displayName: 'Rename v4 tgz file' # because the publish command below requires an exact path
     workingDirectory: '$(Pipeline.Workspace)/nodeLibraryV4/drop'
-  - script: npm install "$(Pipeline.Workspace)/nodeLibraryV4/drop/package.tgz"
-    displayName: 'install latest v4 library'
-    workingDirectory: 'app/v4'
+  # TODO: Temporary change until actual v4 drop exists with http stream changes
+  #- script: npm install "$(Pipeline.Workspace)/nodeLibraryV4/drop/package.tgz"
+  #  displayName: 'install latest v4 library'
+  #  workingDirectory: 'app/v4'
   - script: npm run build
     displayName: 'build v4 app'
     workingDirectory: 'app/v4'
@@ -117,9 +118,10 @@ jobs:
   - script: npm ci
     displayName: 'install deps v4 app (oldConfig)'
     workingDirectory: 'app/combined/v4-oldConfig'
-  - script: npm install "$(Pipeline.Workspace)/nodeLibraryV4/drop/package.tgz"
-    displayName: 'install latest v4 library (oldConfig)'
-    workingDirectory: 'app/combined/v4-oldConfig'
+  # TODO: Temporary change until actual v4 drop exists with http stream changes
+  #- script: npm install "$(Pipeline.Workspace)/nodeLibraryV4/drop/package.tgz"
+  #  displayName: 'install latest v4 library (oldConfig)'
+  #  workingDirectory: 'app/combined/v4-oldConfig'
   - script: npm run build
     displayName: 'build v4 app (oldConfig)'
     workingDirectory: 'app/combined/v4-oldConfig'