Http stream end to end tests

.eslintrc.json

@@ -29,6 +29,7 @@
         ],
         "deprecation/deprecation": "error",
         "@typescript-eslint/no-empty-interface": "off",
+        "@typescript-eslint/no-empty-function": "off",
         "@typescript-eslint/no-explicit-any": "off",
         "@typescript-eslint/no-namespace": "off",
         "@typescript-eslint/no-unused-vars": ["error", { "argsIgnorePattern": "^_" }],

app/v3/httpTriggerRandomDelay/function.json

@@ -0,0 +1,20 @@
+{
+  "bindings": [
+    {
+      "authLevel": "anonymous",
+      "type": "httpTrigger",
+      "direction": "in",
+      "name": "req",
+      "methods": [
+        "get",
+        "post"
+      ]
+    },
+    {
+      "type": "http",
+      "direction": "out",
+      "name": "res"
+    }
+  ],
+  "scriptFile": "../dist/httpTriggerRandomDelay/index.js"
+}

app/v3/httpTriggerRandomDelay/index.ts

@@ -0,0 +1,19 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License.
+
+import { AzureFunction, Context, HttpRequest } from '@azure/functions';
+import { addRandomDelay } from '../utils/getRandomTestData';
+
+const httpTrigger: AzureFunction = async function (context: Context, req: HttpRequest): Promise<void> {
+    context.log(`Http function processed request for url "${req.url}"`);
+
+    await addRandomDelay();
+
+    const name = req.query.name || req.body || 'world';
+
+    context.res = {
+        body: `Hello, ${name}!`,
+    };
+};
+
+export default httpTrigger;

app/v3/utils/delay.ts

@@ -0,0 +1,13 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License.
+
+export async function delay(ms: number): Promise<void> {
+    await new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export function delaySync(ms: number): void {
+    const endTime = Date.now() + ms;
+    while (Date.now() < endTime) {
+        // wait
+    }
+}

app/v3/utils/getRandomTestData.ts

@@ -0,0 +1,32 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License.
+
+import * as crypto from 'crypto';
+import { delay, delaySync } from './delay';
+
+export function getRandomTestData(): string {
+    // This should start with non-numeric data to prevent this bug from causing a test failure
+    // https://github.com/Azure/azure-functions-nodejs-library/issues/90
+    return `testData${getRandomHexString()}`;
+}
+
+export function getRandomHexString(length = 10): string {
+    const buffer: Buffer = crypto.randomBytes(Math.ceil(length / 2));
+    return buffer.toString('hex').slice(0, length);
+}
+
+export function getRandomInt(min: number, max: number): number {
+    return Math.floor(Math.random() * (max - min) + min);
+}
+
+export function getRandomBoolean(percentTrue: number): boolean {
+    return Math.random() * 100 > percentTrue;
+}
+
+export async function addRandomDelay(): Promise<void> {
+    if (getRandomBoolean(95)) {
+        await delay(getRandomInt(0, 250));
+    } else {
+        delaySync(getRandomInt(0, 10));
+    }
+}

app/v4-oldConfig/src/setup.ts

@@ -0,0 +1,6 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License.
+
+import { app } from '@azure/functions';
+
+app.setup({ enableHttpStream: false });

app/v4/package.json

@@ -12,13 +12,13 @@
     "test": "echo \"No tests yet...\""
   },
   "dependencies": {
-    "@azure/functions": "^4.0.0"
+    "@azure/functions": "^4.2.0"
   },
   "devDependencies": {
     "@types/long": "^4.0.0",
     "@types/node": "^18.x",
-    "typescript": "^4.0.0",
-    "rimraf": "^5.0.0"
+    "rimraf": "^5.0.0",
+    "typescript": "^4.0.0"
   },
-  "main": "dist/src/{index.js,functions/*.js}"
-}
+  "main": "dist/src/{index.js,setup.js,functions/*.js}"
+}

app/v4/src/functions/httpTriggerRandomDelay.ts

@@ -0,0 +1,24 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License.
+
+import { app, HttpRequest, HttpResponseInit, InvocationContext } from '@azure/functions';
+import { addRandomDelay } from '../utils/getRandomTestData';
+
+export async function httpTriggerRandomDelay(
+    request: HttpRequest,
+    context: InvocationContext
+): Promise<HttpResponseInit> {
+    context.log(`Http function processed request for url "${request.url}"`);
+
+    await addRandomDelay();
+
+    const name = request.query.get('name') || (await request.text()) || 'world';
+
+    return { body: `Hello, ${name}!` };
+}
+
+app.http('httpTriggerRandomDelay', {
+    methods: ['GET', 'POST'],
+    authLevel: 'anonymous',
+    handler: httpTriggerRandomDelay,
+});

app/v4/src/functions/httpTriggerReceiveStream.ts

@@ -0,0 +1,22 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License.
+
+import { app, HttpRequest, HttpResponseInit, InvocationContext } from '@azure/functions';
+import { receiveStreamWithProgress } from '../utils/streamHttp';
+
+export async function httpTriggerReceiveStream(
+    request: HttpRequest,
+    context: InvocationContext
+): Promise<HttpResponseInit> {
+    context.log(`Http function processed request for url "${request.url}"`);
+
+    const bytesReceived = await receiveStreamWithProgress(request.body);
+
+    return { body: `Bytes received: ${bytesReceived}` };
+}
+
+app.http('httpTriggerReceiveStream', {
+    methods: ['GET', 'POST'],
+    authLevel: 'anonymous',
+    handler: httpTriggerReceiveStream,
+});

app/v4/src/functions/httpTriggerSendStream.ts

@@ -0,0 +1,22 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License.
+
+import { app, HttpRequest, HttpResponseInit, InvocationContext } from '@azure/functions';
+import { createRandomStream } from '../utils/streamHttp';
+
+export async function httpTriggerSendStream(
+    request: HttpRequest,
+    context: InvocationContext
+): Promise<HttpResponseInit> {
+    context.log(`Http function processed request for url "${request.url}"`);
+
+    const lengthInMb = request.query.get('lengthInMb');
+    const stream = createRandomStream(Number(lengthInMb));
+    return { body: stream };
+}
+
+app.http('httpTriggerSendStream', {
+    methods: ['GET', 'POST'],
+    authLevel: 'anonymous',
+    handler: httpTriggerSendStream,
+});

app/v4/src/setup.ts

@@ -0,0 +1,6 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License.
+
+import { app } from '@azure/functions';
+
+app.setup({ enableHttpStream: true });

app/v4/src/utils/delay.ts

@@ -0,0 +1,13 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License.
+
+export async function delay(ms: number): Promise<void> {
+    await new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export function delaySync(ms: number): void {
+    const endTime = Date.now() + ms;
+    while (Date.now() < endTime) {
+        // wait
+    }
+}

app/v4/src/utils/getRandomTestData.ts

@@ -0,0 +1,32 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License.
+
+import * as crypto from 'crypto';
+import { delay, delaySync } from './delay';
+
+export function getRandomTestData(): string {
+    // This should start with non-numeric data to prevent this bug from causing a test failure
+    // https://github.com/Azure/azure-functions-nodejs-library/issues/90
+    return `testData${getRandomHexString()}`;
+}
+
+export function getRandomHexString(length = 10): string {
+    const buffer: Buffer = crypto.randomBytes(Math.ceil(length / 2));
+    return buffer.toString('hex').slice(0, length);
+}
+
+export function getRandomInt(min: number, max: number): number {
+    return Math.floor(Math.random() * (max - min) + min);
+}
+
+export function getRandomBoolean(percentTrue: number): boolean {
+    return Math.random() * 100 > percentTrue;
+}
+
+export async function addRandomDelay(): Promise<void> {
+    if (getRandomBoolean(95)) {
+        await delay(getRandomInt(0, 250));
+    } else {
+        delaySync(getRandomInt(0, 10));
+    }
+}

app/v4/src/utils/streamHttp.ts

@@ -0,0 +1,57 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License.
+
+import * as crypto from 'crypto';
+import { Readable } from 'stream';
+import { delay } from './delay';
+
+export function createRandomStream(lengthInMb: number): Readable {
+    const stream = new Readable();
+    stream._read = () => {};
+    setTimeout(() => {
+        void sendRandomData(stream, lengthInMb);
+    }, 5);
+    return stream;
+}
+
+async function sendRandomData(stream: Readable, lengthInMb: number): Promise<void> {
+    const maxChunkSize = oneMb;
+    let remainingBytes = convertMbToB(lengthInMb);
+    do {
+        if (stream.readableLength > maxChunkSize) {
+            await delay(5);
+        } else {
+            const chunkSize = Math.min(maxChunkSize, remainingBytes);
+            stream.push(crypto.randomBytes(chunkSize));
+            remainingBytes -= chunkSize;
+        }
+    } while (remainingBytes > 0);
+    stream.push(null);
+}
+
+export async function receiveStreamWithProgress(stream: {
+    [Symbol.asyncIterator](): AsyncIterableIterator<string | Buffer>;
+}): Promise<number> {
+    let bytesReceived = 0;
+    const logInterval = 500;
+    let nextLogTime = Date.now();
+    for await (const chunk of stream) {
+        if (Date.now() > nextLogTime) {
+            nextLogTime = Date.now() + logInterval;
+            console.log(`Progress: ${convertBToMb(bytesReceived)}mb`);
+        }
+
+        bytesReceived += chunk.length;
+    }
+    return bytesReceived;
+}
+
+const oneMb = 1024 * 1024;
+
+export function convertMbToB(mb: number): number {
+    return mb * oneMb;
+}
+
+function convertBToMb(bytes: number) {
+    return Math.round(bytes / oneMb);
+}

src/global.test.ts

@@ -25,6 +25,12 @@ export let isOldConfig: boolean;
 let childProc: cp.ChildProcess | undefined;
 let testsDone = false;
 
+interface FuncCliSettings {
+    hideOutput?: boolean;
+}
+
+export const funcCliSettings: FuncCliSettings = {};
+
 before(async function (this: Mocha.Context): Promise<void> {
     model = getModelArg();
     if (model === 'v4' && semver.lt(process.versions.node, '18.0.0')) {
@@ -108,6 +114,7 @@ async function startFuncProcess(appPath: string): Promise<void> {
                     [EnvVarNames.cosmosDB]: cosmosDBConnectionString,
                     [EnvVarNames.serviceBus]: serviceBusConnectionString,
                     [EnvVarNames.sql]: sqlConnectionString,
+                    FUNCTIONS_REQUEST_BODY_SIZE_LIMIT: '4294967296',
                 },
             },
             null,
@@ -123,7 +130,9 @@ async function startFuncProcess(appPath: string): Promise<void> {
 
     childProc.stdout?.on('data', (data: string | Buffer) => {
         data = data.toString();
-        process.stdout.write(data);
+        if (!funcCliSettings.hideOutput) {
+            process.stdout.write(data);
+        }
         perTestFuncOutput += data;
         fullFuncOutput += data;
     });