From 2469b758a9c92267594da1b7ecd2b9e16de91e6c Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Sun, 5 Jan 2025 19:23:16 +0100 Subject: [PATCH 01/14] split into callLambdaSync(), callLambdaAsync(), callLambdaStreaming() --- package.json | 1 + .../src/api/get-compositions-on-lambda.ts | 4 +- .../lambda/src/api/get-render-progress.ts | 4 +- .../lambda/src/api/render-media-on-lambda.ts | 4 +- .../lambda/src/api/render-still-on-lambda.ts | 2 +- .../src/functions/helpers/stream-renderer.ts | 2 +- packages/lambda/src/functions/start.ts | 33 +-- .../lambda/src/shared/call-lambda-async.ts | 37 +++ .../src/shared/call-lambda-streaming.ts | 218 ++++++++++++++ .../lambda/src/shared/call-lambda-sync.ts | 54 ++++ packages/lambda/src/shared/call-lambda.ts | 272 +----------------- .../lambda/src/shared/get-function-version.ts | 4 +- .../src/test/integration/handlers.test.ts | 4 +- .../integration/renders/old-version.test.ts | 4 +- .../integration/simulate-lambda-render.ts | 6 +- .../src/test/integration/webhooks.test.ts | 8 +- .../lambda/src/test/unit/handlers.test.ts | 4 +- 17 files changed, 346 insertions(+), 315 deletions(-) create mode 100644 packages/lambda/src/shared/call-lambda-async.ts create mode 100644 packages/lambda/src/shared/call-lambda-streaming.ts create mode 100644 packages/lambda/src/shared/call-lambda-sync.ts diff --git a/package.json b/package.json index cd866395cac..67d441ecb52 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "ci": "turbo run make test --concurrency=1 --no-update-notifier", "watch": "turbo watch make --concurrency=2", "watchwebcodecs": "turbo watch make --filter='@remotion/media-parser' --filter='@remotion/webcodecs'", + "watchserverless": "turbo watch make --filter='@remotion/lambda' --filter='@remotion/serverless' --filter='@remotion/streaming'", "release-alpha": "pnpm recursive publish --tag=alpha --no-git-checks && pnpm recursive run --sequential publishprivate", "release": "pnpm recursive publish && pnpm recursive run --sequential publishprivate && git push --tags && git push", "clean": "turbo run clean && rm -rf packages/**/dist && rm -rf packages/**/node_modules && rm -rf node_modules && rm -rf .cache && rm -rf packages/**/tsconfig.tsbuildinfo && rm -f packages/tsconfig.tsbuildinfo && rm -rf packages/**/.turbo && rm -rf .turbo" diff --git a/packages/lambda/src/api/get-compositions-on-lambda.ts b/packages/lambda/src/api/get-compositions-on-lambda.ts index 549926909f6..537331816cf 100644 --- a/packages/lambda/src/api/get-compositions-on-lambda.ts +++ b/packages/lambda/src/api/get-compositions-on-lambda.ts @@ -10,7 +10,7 @@ import type {VideoConfig} from 'remotion/no-react'; import {VERSION} from 'remotion/version'; import type {AwsRegion} from '../client'; import {awsImplementation} from '../functions/aws-implementation'; -import {callLambda} from '../shared/call-lambda'; +import {callLambdaSync} from '../shared/call-lambda-sync'; export type GetCompositionsOnLambdaInput = { chromiumOptions?: ChromiumOptions; @@ -66,7 +66,7 @@ export const getCompositionsOnLambda = async ({ }); try { - const res = await callLambda({ + const res = await callLambdaSync({ functionName, type: ServerlessRoutines.compositions, payload: { diff --git a/packages/lambda/src/api/get-render-progress.ts b/packages/lambda/src/api/get-render-progress.ts index 740acff819e..c8d4d3de96a 100644 --- a/packages/lambda/src/api/get-render-progress.ts +++ b/packages/lambda/src/api/get-render-progress.ts @@ -8,7 +8,7 @@ import { import {getProgress} from '../functions/helpers/get-progress'; import {parseFunctionName} from '../functions/helpers/parse-function-name'; import type {AwsRegion} from '../regions'; -import {callLambda} from '../shared/call-lambda'; +import {callLambdaSync} from '../shared/call-lambda-sync'; import type {RenderProgress} from '../shared/constants'; import {getRenderProgressPayload} from './make-lambda-payload'; @@ -56,7 +56,7 @@ export const getRenderProgress = async ( }); } - const result = await callLambda({ + const result = await callLambdaSync({ functionName: input.functionName, type: ServerlessRoutines.status, payload: getRenderProgressPayload(input), diff --git a/packages/lambda/src/api/render-media-on-lambda.ts b/packages/lambda/src/api/render-media-on-lambda.ts index fdb8f8510dd..daec6f09f3e 100644 --- a/packages/lambda/src/api/render-media-on-lambda.ts +++ b/packages/lambda/src/api/render-media-on-lambda.ts @@ -19,7 +19,7 @@ import type { import {ServerlessRoutines} from '@remotion/serverless/client'; import type {AwsProvider} from '../functions/aws-implementation'; import type {AwsRegion} from '../regions'; -import {callLambda} from '../shared/call-lambda'; +import {callLambdaSync} from '../shared/call-lambda-sync'; import { getCloudwatchMethodUrl, getCloudwatchRendererUrl, @@ -88,7 +88,7 @@ export const internalRenderMediaOnLambdaRaw = async ( const {functionName, region, rendererFunctionName} = input; try { - const res = await callLambda({ + const res = await callLambdaSync({ functionName, type: ServerlessRoutines.start, payload: await makeLambdaRenderMediaPayload(input), diff --git a/packages/lambda/src/api/render-still-on-lambda.ts b/packages/lambda/src/api/render-still-on-lambda.ts index 118aae677e3..7aa7caabd9c 100644 --- a/packages/lambda/src/api/render-still-on-lambda.ts +++ b/packages/lambda/src/api/render-still-on-lambda.ts @@ -6,7 +6,7 @@ import type { import type {BrowserSafeApis} from '@remotion/renderer/client'; import type {DownloadBehavior} from '@remotion/serverless/client'; import {ServerlessRoutines} from '@remotion/serverless/client'; -import {callLambdaWithStreaming} from '../shared/call-lambda'; +import {callLambdaWithStreaming} from '../shared/call-lambda-streaming'; import {wrapWithErrorHandling} from '@remotion/renderer/error-handling'; import type { diff --git a/packages/lambda/src/functions/helpers/stream-renderer.ts b/packages/lambda/src/functions/helpers/stream-renderer.ts index 5ae66309c9c..3bc42ee3196 100644 --- a/packages/lambda/src/functions/helpers/stream-renderer.ts +++ b/packages/lambda/src/functions/helpers/stream-renderer.ts @@ -9,7 +9,7 @@ import type {ServerlessPayload} from '@remotion/serverless/client'; import {ServerlessRoutines} from '@remotion/serverless/client'; import {writeFileSync} from 'fs'; import {join} from 'path'; -import {callLambdaWithStreaming} from '../../shared/call-lambda'; +import {callLambdaWithStreaming} from '../../shared/call-lambda-streaming'; import type {OverallProgressHelper} from './overall-render-progress'; type StreamRendererResponse = diff --git a/packages/lambda/src/functions/start.ts b/packages/lambda/src/functions/start.ts index aad816d1263..9a8f3ad919d 100644 --- a/packages/lambda/src/functions/start.ts +++ b/packages/lambda/src/functions/start.ts @@ -1,4 +1,3 @@ -import {InvokeCommand} from '@aws-sdk/client-lambda'; import type {CloudProvider, ProviderSpecifics} from '@remotion/serverless'; import type {ServerlessPayload} from '@remotion/serverless/client'; import { @@ -8,7 +7,7 @@ import { } from '@remotion/serverless/client'; import {VERSION} from 'remotion/version'; import type {AwsRegion} from '../regions'; -import {getLambdaClient} from '../shared/aws-clients'; +import {callLambdaAsync} from '../shared/call-lambda-async'; import {validateDeleteAfter} from './helpers/lifecycle'; import {makeInitialOverallRenderProgress} from './helpers/overall-render-progress'; @@ -123,29 +122,13 @@ export const startHandler = async ( metadata: params.metadata, }; - const stringifiedPayload = JSON.stringify(payload); - - if (stringifiedPayload.length > 256 * 1024) { - throw new Error( - `Payload is too big: ${stringifiedPayload.length} bytes. Maximum size is 256 KB. This should not happen, please report this to the Remotion team. Payload: ${stringifiedPayload}`, - ); - } - - // Don't replace with callLambda(), we want to return before the render is snone - const result = await getLambdaClient( - providerSpecifics.getCurrentRegionInFunction() as AwsRegion, - ).send( - new InvokeCommand({ - FunctionName: process.env.AWS_LAMBDA_FUNCTION_NAME, - Payload: stringifiedPayload, - InvocationType: 'Event', - }), - ); - if (result.FunctionError) { - throw new Error( - `Lambda function returned error: ${result.FunctionError} ${result.LogResult}`, - ); - } + await callLambdaAsync({ + functionName: process.env.AWS_LAMBDA_FUNCTION_NAME as string, + type: ServerlessRoutines.launch, + payload, + region: region as AwsRegion, + timeoutInTest: options.timeoutInMilliseconds, + }); await initialFile; diff --git a/packages/lambda/src/shared/call-lambda-async.ts b/packages/lambda/src/shared/call-lambda-async.ts new file mode 100644 index 00000000000..f42094b172b --- /dev/null +++ b/packages/lambda/src/shared/call-lambda-async.ts @@ -0,0 +1,37 @@ +import {InvokeCommand} from '@aws-sdk/client-lambda'; +import type {CloudProvider} from '@remotion/serverless'; +import type {ServerlessRoutines} from '@remotion/serverless/client'; +import type {AwsRegion} from '../regions'; +import {getLambdaClient} from './aws-clients'; +import type {CallLambdaOptions} from './call-lambda'; + +export const callLambdaAsync = async < + T extends ServerlessRoutines, + Provider extends CloudProvider, +>({ + functionName, + payload, + region, + timeoutInTest, +}: CallLambdaOptions): Promise => { + const stringifiedPayload = JSON.stringify(payload); + if (stringifiedPayload.length > 256 * 1024) { + throw new Error( + `Payload is too big: ${stringifiedPayload.length} bytes. Maximum size is 256 KB. This should not happen, please report this to the Remotion team. Payload: ${stringifiedPayload}`, + ); + } + + const result = await getLambdaClient(region as AwsRegion, timeoutInTest).send( + new InvokeCommand({ + FunctionName: functionName, + Payload: stringifiedPayload, + InvocationType: 'Event', + }), + ); + + if (result.FunctionError) { + throw new Error( + `Lambda function returned error: ${result.FunctionError} ${result.LogResult}`, + ); + } +}; diff --git a/packages/lambda/src/shared/call-lambda-streaming.ts b/packages/lambda/src/shared/call-lambda-streaming.ts new file mode 100644 index 00000000000..8182cb35e90 --- /dev/null +++ b/packages/lambda/src/shared/call-lambda-streaming.ts @@ -0,0 +1,218 @@ +import type {InvokeWithResponseStreamCommandOutput} from '@aws-sdk/client-lambda'; +import { + InvokeWithResponseStreamCommand, + type InvokeWithResponseStreamResponseEvent, +} from '@aws-sdk/client-lambda'; +import type { + CloudProvider, + OnMessage, + StreamingMessage, +} from '@remotion/serverless'; +import type { + MessageTypeId, + ServerlessRoutines, +} from '@remotion/serverless/client'; +import { + formatMap, + messageTypeIdToMessageType, +} from '@remotion/serverless/client'; +import {makeStreamer} from '@remotion/streaming'; +import type {AwsRegion} from '../regions'; +import {getLambdaClient} from './aws-clients'; +import type {CallLambdaOptions} from './call-lambda'; + +const STREAM_STALL_TIMEOUT = 30000; +const LAMBDA_STREAM_STALL = `AWS did not invoke Lambda in ${STREAM_STALL_TIMEOUT}ms`; + +const parseJsonOrThrowSource = (data: Uint8Array, type: string) => { + const asString = new TextDecoder('utf-8').decode(data); + try { + return JSON.parse(asString); + } catch { + throw new Error(`Invalid JSON (${type}): ${asString}`); + } +}; + +const invokeStreamOrTimeout = async ({ + region, + timeoutInTest, + functionName, + type, + payload, +}: { + region: Provider['region']; + timeoutInTest: number; + functionName: string; + type: string; + payload: Record; +}) => { + const resProm = getLambdaClient(region as AwsRegion, timeoutInTest).send( + new InvokeWithResponseStreamCommand({ + FunctionName: functionName, + Payload: JSON.stringify({type, ...payload}), + }), + ); + + let cleanup = () => undefined; + + const timeout = new Promise( + (_resolve, reject) => { + const int = setTimeout(() => { + reject(new Error(LAMBDA_STREAM_STALL)); + }, STREAM_STALL_TIMEOUT); + cleanup = () => { + clearTimeout(int); + }; + }, + ); + + const res = await Promise.race([resProm, timeout]); + + cleanup(); + + return res; +}; + +const INVALID_JSON_MESSAGE = 'Cannot parse Lambda response as JSON'; + +const callLambdaWithStreamingWithoutRetry = async < + T extends ServerlessRoutines, + Provider extends CloudProvider, +>({ + functionName, + type, + payload, + region, + timeoutInTest, + receivedStreamingPayload, +}: CallLambdaOptions & { + receivedStreamingPayload: OnMessage; +}): Promise => { + const res = await invokeStreamOrTimeout({ + functionName, + payload, + region, + timeoutInTest, + type, + }); + + const {onData, clear} = makeStreamer((status, messageTypeId, data) => { + const messageType = messageTypeIdToMessageType( + messageTypeId as MessageTypeId, + ); + const innerPayload = + formatMap[messageType] === 'json' + ? parseJsonOrThrowSource(data, messageType) + : data; + + const message: StreamingMessage = { + successType: status, + message: { + type: messageType, + payload: innerPayload, + }, + }; + + receivedStreamingPayload(message); + }); + + const dumpBuffers = () => { + clear(); + }; + + // @ts-expect-error - We are adding a listener to a global variable + if (globalThis._dumpUnreleasedBuffers) { + // @ts-expect-error - We are adding a listener to a global variable + (globalThis._dumpUnreleasedBuffers as EventEmitter).addListener( + 'dump-unreleased-buffers', + dumpBuffers, + ); + } + + const events = + res.EventStream as AsyncIterable; + + for await (const event of events) { + // There are two types of events you can get on a stream. + + // `PayloadChunk`: These contain the actual raw bytes of the chunk + // It has a single property: `Payload` + if (event.PayloadChunk && event.PayloadChunk.Payload) { + onData(event.PayloadChunk.Payload); + } + + if (event.InvokeComplete) { + if (event.InvokeComplete.ErrorCode) { + const logs = `https://${region}.console.aws.amazon.com/cloudwatch/home?region=${region}#logsV2:logs-insights$3FqueryDetail$3D~(end~0~start~-3600~timeType~'RELATIVE~unit~'seconds~editorString~'fields*20*40timestamp*2c*20*40requestId*2c*20*40message*0a*7c*20filter*20*40requestId*20like*20*${res.$metadata.requestId}*22*0a*7c*20sort*20*40timestamp*20asc~source~(~'*2faws*2flambda*2f${functionName}))`; + if (event.InvokeComplete.ErrorCode === 'Unhandled') { + throw new Error( + `Lambda function ${functionName} failed with an unhandled error: ${ + event.InvokeComplete.ErrorDetails as string + } See ${logs} to see the logs of this invocation.`, + ); + } + + throw new Error( + `Lambda function ${functionName} failed with error code ${event.InvokeComplete.ErrorCode}: ${event.InvokeComplete.ErrorDetails}. See ${logs} to see the logs of this invocation.`, + ); + } + } + + // Don't put a `break` statement here, as it will cause the socket to not properly exit. + } + + // @ts-expect-error - We are adding a listener to a global variable + if (globalThis._dumpUnreleasedBuffers) { + // @ts-expect-error - We are adding a listener to a global variable + (globalThis._dumpUnreleasedBuffers as EventEmitter).removeListener( + 'dump-unreleased-buffers', + dumpBuffers, + ); + } + + clear(); +}; + +export const callLambdaWithStreaming = async < + Provider extends CloudProvider, + T extends ServerlessRoutines, +>( + options: CallLambdaOptions & { + receivedStreamingPayload: OnMessage; + retriesRemaining: number; + }, +): Promise => { + // As of August 2023, Lambda streaming sometimes misses parts of the JSON response. + // Handling this for now by applying a retry mechanism. + + try { + // Do not remove this await + await callLambdaWithStreamingWithoutRetry(options); + } catch (err) { + if ((err as Error).stack?.includes('TooManyRequestsException')) { + throw new Error( + `AWS Concurrency limit reached (Original Error: ${(err as Error).message}). See https://www.remotion.dev/docs/lambda/troubleshooting/rate-limit for tips to fix this.`, + ); + } + + if ( + !(err as Error).message.includes(INVALID_JSON_MESSAGE) && + !(err as Error).message.includes(LAMBDA_STREAM_STALL) && + !(err as Error).message.includes('aborted') + ) { + throw err; + } + + console.error('Retries remaining:', options.retriesRemaining); + if (options.retriesRemaining === 0) { + console.error('Throwing error:'); + throw err; + } + + console.error(err); + return callLambdaWithStreaming({ + ...options, + retriesRemaining: options.retriesRemaining - 1, + }); + } +}; diff --git a/packages/lambda/src/shared/call-lambda-sync.ts b/packages/lambda/src/shared/call-lambda-sync.ts new file mode 100644 index 00000000000..a6082375f2f --- /dev/null +++ b/packages/lambda/src/shared/call-lambda-sync.ts @@ -0,0 +1,54 @@ +import {InvokeCommand} from '@aws-sdk/client-lambda'; +import type {CloudProvider} from '@remotion/serverless'; +import type {ServerlessRoutines} from '@remotion/serverless/client'; +import type {OrError} from '../functions'; +import type {AwsRegion} from '../regions'; +import {getLambdaClient} from './aws-clients'; +import type {CallLambdaOptions} from './call-lambda'; +import type {LambdaReturnValues} from './return-values'; + +const callLambdaSyncWithoutRetry = async < + T extends ServerlessRoutines, + Provider extends CloudProvider, +>({ + functionName, + type, + payload, + region, + timeoutInTest, +}: CallLambdaOptions): Promise< + OrError[T]> +> => { + const Payload = JSON.stringify(payload); + const res = await getLambdaClient(region as AwsRegion, timeoutInTest).send( + new InvokeCommand({ + FunctionName: functionName, + Payload, + InvocationType: 'RequestResponse', + }), + ); + + const decoded = new TextDecoder('utf-8').decode(res.Payload); + + try { + return JSON.parse(decoded) as OrError[T]>; + } catch { + throw new Error(`Invalid JSON (${type}): ${JSON.stringify(decoded)}`); + } +}; + +export const callLambdaSync = async < + Provider extends CloudProvider, + T extends ServerlessRoutines, +>( + options: CallLambdaOptions, +): Promise[T]> => { + const res = await callLambdaSyncWithoutRetry(options); + if (res.type === 'error') { + const err = new Error(res.message); + err.stack = res.stack; + throw err; + } + + return res; +}; diff --git a/packages/lambda/src/shared/call-lambda.ts b/packages/lambda/src/shared/call-lambda.ts index 88d0db6268a..1bb8855c938 100644 --- a/packages/lambda/src/shared/call-lambda.ts +++ b/packages/lambda/src/shared/call-lambda.ts @@ -1,278 +1,16 @@ -/* eslint-disable @typescript-eslint/no-use-before-define */ +import type {CloudProvider} from '@remotion/serverless'; import type { - InvokeWithResponseStreamCommandOutput, - InvokeWithResponseStreamResponseEvent, -} from '@aws-sdk/client-lambda'; -import { - InvokeCommand, - InvokeWithResponseStreamCommand, -} from '@aws-sdk/client-lambda'; -import type { - CloudProvider, - OnMessage, - StreamingMessage, -} from '@remotion/serverless'; -import type { - MessageTypeId, ServerlessPayloads, ServerlessRoutines, } from '@remotion/serverless/client'; -import { - formatMap, - messageTypeIdToMessageType, -} from '@remotion/serverless/client'; -import {makeStreamer} from '@remotion/streaming'; -import type {EventEmitter} from 'node:events'; -import type {OrError} from '../functions'; -import type {AwsRegion} from '../regions'; -import {getLambdaClient} from './aws-clients'; -import type {LambdaReturnValues} from './return-values'; - -const INVALID_JSON_MESSAGE = 'Cannot parse Lambda response as JSON'; -type Options = { +export type CallLambdaOptions< + T extends ServerlessRoutines, + Provider extends CloudProvider, +> = { functionName: string; type: T; payload: Omit[T], 'type'>; region: Provider['region']; timeoutInTest: number; }; - -const parseJsonOrThrowSource = (data: Uint8Array, type: string) => { - const asString = new TextDecoder('utf-8').decode(data); - try { - return JSON.parse(asString); - } catch { - throw new Error(`Invalid JSON (${type}): ${asString}`); - } -}; - -export const callLambda = async < - Provider extends CloudProvider, - T extends ServerlessRoutines, ->( - options: Options, -): Promise[T]> => { - // Do not remove this await - const res = await callLambdaWithoutRetry(options); - if (res.type === 'error') { - const err = new Error(res.message); - err.stack = res.stack; - throw err; - } - - return res; -}; - -export const callLambdaWithStreaming = async < - Provider extends CloudProvider, - T extends ServerlessRoutines, ->( - options: Options & { - receivedStreamingPayload: OnMessage; - retriesRemaining: number; - }, -): Promise => { - // As of August 2023, Lambda streaming sometimes misses parts of the JSON response. - // Handling this for now by applying a retry mechanism. - - try { - // Do not remove this await - await callLambdaWithStreamingWithoutRetry(options); - } catch (err) { - if ((err as Error).stack?.includes('TooManyRequestsException')) { - throw new Error( - `AWS Concurrency limit reached (Original Error: ${(err as Error).message}). See https://www.remotion.dev/docs/lambda/troubleshooting/rate-limit for tips to fix this.`, - ); - } - - if ( - !(err as Error).message.includes(INVALID_JSON_MESSAGE) && - !(err as Error).message.includes(LAMBDA_STREAM_STALL) && - !(err as Error).message.includes('aborted') - ) { - throw err; - } - - console.error('Retries remaining:', options.retriesRemaining); - if (options.retriesRemaining === 0) { - console.error('Throwing error:'); - throw err; - } - - console.error(err); - return callLambdaWithStreaming({ - ...options, - retriesRemaining: options.retriesRemaining - 1, - }); - } -}; - -const callLambdaWithoutRetry = async < - T extends ServerlessRoutines, - Provider extends CloudProvider, ->({ - functionName, - type, - payload, - region, - timeoutInTest, -}: Options): Promise[T]>> => { - const Payload = JSON.stringify({type, ...payload}); - const res = await getLambdaClient(region as AwsRegion, timeoutInTest).send( - new InvokeCommand({ - FunctionName: functionName, - Payload, - InvocationType: 'RequestResponse', - }), - ); - - const decoded = new TextDecoder('utf-8').decode(res.Payload); - - try { - return JSON.parse(decoded) as OrError[T]>; - } catch { - throw new Error(`Invalid JSON (${type}): ${JSON.stringify(decoded)}`); - } -}; - -const STREAM_STALL_TIMEOUT = 30000; -const LAMBDA_STREAM_STALL = `AWS did not invoke Lambda in ${STREAM_STALL_TIMEOUT}ms`; - -const invokeStreamOrTimeout = async ({ - region, - timeoutInTest, - functionName, - type, - payload, -}: { - region: Provider['region']; - timeoutInTest: number; - functionName: string; - type: string; - payload: Record; -}) => { - const resProm = getLambdaClient(region as AwsRegion, timeoutInTest).send( - new InvokeWithResponseStreamCommand({ - FunctionName: functionName, - Payload: JSON.stringify({type, ...payload}), - }), - ); - - let cleanup = () => undefined; - - const timeout = new Promise( - (_resolve, reject) => { - const int = setTimeout(() => { - reject(new Error(LAMBDA_STREAM_STALL)); - }, STREAM_STALL_TIMEOUT); - cleanup = () => { - clearTimeout(int); - }; - }, - ); - - const res = await Promise.race([resProm, timeout]); - - cleanup(); - - return res; -}; - -const callLambdaWithStreamingWithoutRetry = async < - T extends ServerlessRoutines, - Provider extends CloudProvider, ->({ - functionName, - type, - payload, - region, - timeoutInTest, - receivedStreamingPayload, -}: Options & { - receivedStreamingPayload: OnMessage; -}): Promise => { - const res = await invokeStreamOrTimeout({ - functionName, - payload, - region, - timeoutInTest, - type, - }); - - const {onData, clear} = makeStreamer((status, messageTypeId, data) => { - const messageType = messageTypeIdToMessageType( - messageTypeId as MessageTypeId, - ); - const innerPayload = - formatMap[messageType] === 'json' - ? parseJsonOrThrowSource(data, messageType) - : data; - - const message: StreamingMessage = { - successType: status, - message: { - type: messageType, - payload: innerPayload, - }, - }; - - receivedStreamingPayload(message); - }); - - const dumpBuffers = () => { - clear(); - }; - - // @ts-expect-error - We are adding a listener to a global variable - if (globalThis._dumpUnreleasedBuffers) { - // @ts-expect-error - We are adding a listener to a global variable - (globalThis._dumpUnreleasedBuffers as EventEmitter).addListener( - 'dump-unreleased-buffers', - dumpBuffers, - ); - } - - const events = - res.EventStream as AsyncIterable; - - for await (const event of events) { - // There are two types of events you can get on a stream. - - // `PayloadChunk`: These contain the actual raw bytes of the chunk - // It has a single property: `Payload` - if (event.PayloadChunk && event.PayloadChunk.Payload) { - onData(event.PayloadChunk.Payload); - } - - if (event.InvokeComplete) { - if (event.InvokeComplete.ErrorCode) { - const logs = `https://${region}.console.aws.amazon.com/cloudwatch/home?region=${region}#logsV2:logs-insights$3FqueryDetail$3D~(end~0~start~-3600~timeType~'RELATIVE~unit~'seconds~editorString~'fields*20*40timestamp*2c*20*40requestId*2c*20*40message*0a*7c*20filter*20*40requestId*20like*20*${res.$metadata.requestId}*22*0a*7c*20sort*20*40timestamp*20asc~source~(~'*2faws*2flambda*2f${functionName}))`; - if (event.InvokeComplete.ErrorCode === 'Unhandled') { - throw new Error( - `Lambda function ${functionName} failed with an unhandled error: ${ - event.InvokeComplete.ErrorDetails as string - } See ${logs} to see the logs of this invocation.`, - ); - } - - throw new Error( - `Lambda function ${functionName} failed with error code ${event.InvokeComplete.ErrorCode}: ${event.InvokeComplete.ErrorDetails}. See ${logs} to see the logs of this invocation.`, - ); - } - } - - // Don't put a `break` statement here, as it will cause the socket to not properly exit. - } - - // @ts-expect-error - We are adding a listener to a global variable - if (globalThis._dumpUnreleasedBuffers) { - // @ts-expect-error - We are adding a listener to a global variable - (globalThis._dumpUnreleasedBuffers as EventEmitter).removeListener( - 'dump-unreleased-buffers', - dumpBuffers, - ); - } - - clear(); -}; diff --git a/packages/lambda/src/shared/get-function-version.ts b/packages/lambda/src/shared/get-function-version.ts index 058aebfc645..1acfcc5bcf8 100644 --- a/packages/lambda/src/shared/get-function-version.ts +++ b/packages/lambda/src/shared/get-function-version.ts @@ -1,7 +1,7 @@ import type {LogLevel} from '@remotion/renderer'; import {ServerlessRoutines} from '@remotion/serverless/client'; import type {AwsRegion} from '../regions'; -import {callLambda} from './call-lambda'; +import {callLambdaSync} from './call-lambda-sync'; import {COMMAND_NOT_FOUND} from './constants'; export const getFunctionVersion = async ({ @@ -14,7 +14,7 @@ export const getFunctionVersion = async ({ logLevel: LogLevel; }): Promise => { try { - const result = await callLambda({ + const result = await callLambdaSync({ functionName, payload: { logLevel, diff --git a/packages/lambda/src/test/integration/handlers.test.ts b/packages/lambda/src/test/integration/handlers.test.ts index f1b7c3336e2..5b2d1449396 100644 --- a/packages/lambda/src/test/integration/handlers.test.ts +++ b/packages/lambda/src/test/integration/handlers.test.ts @@ -1,11 +1,11 @@ import {ServerlessRoutines} from '@remotion/serverless/client'; import {VERSION} from 'remotion/version'; import {expect, test} from 'vitest'; -import {callLambda} from '../../shared/call-lambda'; +import {callLambdaSync} from '../../shared/call-lambda-sync'; test('Call function locally', async () => { expect( - await callLambda({ + await callLambdaSync({ payload: { logLevel: 'info', }, diff --git a/packages/lambda/src/test/integration/renders/old-version.test.ts b/packages/lambda/src/test/integration/renders/old-version.test.ts index e227ff41d3f..5295444dfef 100644 --- a/packages/lambda/src/test/integration/renders/old-version.test.ts +++ b/packages/lambda/src/test/integration/renders/old-version.test.ts @@ -1,7 +1,7 @@ import {RenderInternals} from '@remotion/renderer'; import {ServerlessRoutines} from '@remotion/serverless/client'; import {afterAll, expect, test} from 'vitest'; -import {callLambda} from '../../../shared/call-lambda'; +import {callLambdaSync} from '../../../shared/call-lambda-sync'; afterAll(async () => { await RenderInternals.killAllBrowsers(); @@ -11,7 +11,7 @@ test('Should fail when using an incompatible version', async () => { process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE = '2048'; try { - const aha = await callLambda({ + const aha = await callLambdaSync({ type: ServerlessRoutines.launch, payload: { serveUrl: 'https://competent-mccarthy-56f7c9.netlify.app/', diff --git a/packages/lambda/src/test/integration/simulate-lambda-render.ts b/packages/lambda/src/test/integration/simulate-lambda-render.ts index e150cc1e530..ac0b084212f 100644 --- a/packages/lambda/src/test/integration/simulate-lambda-render.ts +++ b/packages/lambda/src/test/integration/simulate-lambda-render.ts @@ -6,14 +6,14 @@ import {makeLambdaRenderMediaPayload} from '../../api/make-lambda-payload'; import type {RenderMediaOnLambdaInput} from '../../api/render-media-on-lambda'; import {renderMediaOnLambdaOptionalToRequired} from '../../api/render-media-on-lambda'; import type {AwsProvider} from '../../functions/aws-implementation'; -import {callLambda} from '../../shared/call-lambda'; +import {callLambdaSync} from '../../shared/call-lambda-sync'; import {mockImplementation} from '../mock-implementation'; const functionName = 'remotion-dev-render'; const waitUntilDone = async (bucketName: string, renderId: string) => { while (true) { - const progress = await callLambda({ + const progress = await callLambdaSync({ type: ServerlessRoutines.status, payload: { bucketName, @@ -68,7 +68,7 @@ export const simulateLambdaRender = async ( }), ); - const res = await callLambda({ + const res = await callLambdaSync({ type: ServerlessRoutines.start, payload, functionName: 'remotion-dev-lambda', diff --git a/packages/lambda/src/test/integration/webhooks.test.ts b/packages/lambda/src/test/integration/webhooks.test.ts index e4a64e5bb74..80b42f8b053 100644 --- a/packages/lambda/src/test/integration/webhooks.test.ts +++ b/packages/lambda/src/test/integration/webhooks.test.ts @@ -3,7 +3,7 @@ import {ServerlessRoutines} from '@remotion/serverless/client'; import path from 'path'; import {VERSION} from 'remotion/version'; import {beforeAll, beforeEach, describe, expect, test, vi} from 'vitest'; -import {callLambda} from '../../shared/call-lambda'; +import {callLambdaSync} from '../../shared/call-lambda-sync'; import {mockableHttpClients} from '../../shared/invoke-webhook'; const originalFetch = mockableHttpClients.http; @@ -57,7 +57,7 @@ describe('Webhooks', () => { forceIPv4: false, }); - const res = await callLambda({ + const res = await callLambdaSync({ type: ServerlessRoutines.start, payload: { serveUrl: `http://localhost:${port}`, @@ -121,7 +121,7 @@ describe('Webhooks', () => { }); const parsed = res; - await callLambda({ + await callLambdaSync({ type: ServerlessRoutines.status, payload: { bucketName: parsed.bucketName, @@ -173,7 +173,7 @@ describe('Webhooks', () => { forceIPv4: false, }); - await callLambda({ + await callLambdaSync({ functionName: 'remotion-dev-lambda', region: 'us-east-1', type: ServerlessRoutines.launch, diff --git a/packages/lambda/src/test/unit/handlers.test.ts b/packages/lambda/src/test/unit/handlers.test.ts index 03c6768379e..a3a4bb0d191 100644 --- a/packages/lambda/src/test/unit/handlers.test.ts +++ b/packages/lambda/src/test/unit/handlers.test.ts @@ -1,9 +1,9 @@ import {ServerlessRoutines} from '@remotion/serverless/client'; import {expect, test} from 'vitest'; -import {callLambda} from '../../shared/call-lambda'; +import {callLambdaSync} from '../../shared/call-lambda-sync'; test('Info handler should return version', async () => { - const response = await callLambda({ + const response = await callLambdaSync({ type: ServerlessRoutines.info, payload: { logLevel: 'info', From 502489659d89e0a14b7112c1fbf3e967aacb3c10 Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Sun, 5 Jan 2025 19:27:02 +0100 Subject: [PATCH 02/14] implement validateafter --- .../src/functions/aws-implementation.ts | 28 +++++++++++++++++++ .../lambda/src/functions/helpers/lifecycle.ts | 26 ----------------- packages/lambda/src/functions/index.ts | 7 ++--- packages/lambda/src/functions/start.ts | 3 +- .../lambda/src/test/mock-implementation.ts | 1 + .../serverless/src/provider-implementation.ts | 1 + 6 files changed, 33 insertions(+), 33 deletions(-) diff --git a/packages/lambda/src/functions/aws-implementation.ts b/packages/lambda/src/functions/aws-implementation.ts index 1eaf165514c..9df16aff02f 100644 --- a/packages/lambda/src/functions/aws-implementation.ts +++ b/packages/lambda/src/functions/aws-implementation.ts @@ -1,4 +1,5 @@ import {type ProviderSpecifics} from '@remotion/serverless'; +import {expiryDays} from '@remotion/serverless/client'; import {EventEmitter} from 'node:events'; import {bucketExistsInRegionImplementation} from '../api/bucket-exists'; import {createBucket} from '../api/create-bucket'; @@ -43,6 +44,32 @@ export type AwsProvider = { }; }; +const validateDeleteAfter = (lifeCycleValue: unknown) => { + if (lifeCycleValue === null) { + return; + } + + if (lifeCycleValue === undefined) { + return; + } + + if (typeof lifeCycleValue !== 'string') { + throw new TypeError( + `Expected life cycle value to be a string, got ${JSON.stringify( + lifeCycleValue, + )}`, + ); + } + + if (!(lifeCycleValue in expiryDays)) { + throw new TypeError( + `Expected deleteAfter value to be one of ${Object.keys(expiryDays).join( + ', ', + )}, got ${lifeCycleValue}`, + ); + } +}; + export const awsImplementation: ProviderSpecifics = { getChromiumPath() { return '/opt/bin/chromium'; @@ -62,4 +89,5 @@ export const awsImplementation: ProviderSpecifics = { printLoggingHelper: true, getFolderFiles, makeArtifactWithDetails: makeAwsArtifact, + validateDeleteAfter, }; diff --git a/packages/lambda/src/functions/helpers/lifecycle.ts b/packages/lambda/src/functions/helpers/lifecycle.ts index cd22b9f0827..fa3d94fb7f3 100644 --- a/packages/lambda/src/functions/helpers/lifecycle.ts +++ b/packages/lambda/src/functions/helpers/lifecycle.ts @@ -36,29 +36,3 @@ export const generateRandomHashWithLifeCycleRule = < ) => { return [deleteAfter, providerSpecifics.randomHash()].filter(truthy).join('-'); }; - -export const validateDeleteAfter = (lifeCycleValue: unknown) => { - if (lifeCycleValue === null) { - return; - } - - if (lifeCycleValue === undefined) { - return; - } - - if (typeof lifeCycleValue !== 'string') { - throw new TypeError( - `Expected life cycle value to be a string, got ${JSON.stringify( - lifeCycleValue, - )}`, - ); - } - - if (!(lifeCycleValue in expiryDays)) { - throw new TypeError( - `Expected deleteAfter value to be one of ${Object.keys(expiryDays).join( - ', ', - )}, got ${lifeCycleValue}`, - ); - } -}; diff --git a/packages/lambda/src/functions/index.ts b/packages/lambda/src/functions/index.ts index 6afd2404937..c3f4f6eb93b 100644 --- a/packages/lambda/src/functions/index.ts +++ b/packages/lambda/src/functions/index.ts @@ -22,10 +22,7 @@ import {awsImplementation} from './aws-implementation'; import {deleteTmpDir} from './helpers/clean-tmpdir'; import {getWarm, setWarm} from './helpers/is-warm'; import {setCurrentRequestId, stopLeakDetection} from './helpers/leak-detection'; -import { - generateRandomHashWithLifeCycleRule, - validateDeleteAfter, -} from './helpers/lifecycle'; +import {generateRandomHashWithLifeCycleRule} from './helpers/lifecycle'; import {printLoggingGrepHelper} from './helpers/print-logging-helper'; import type {RequestContext} from './helpers/request-context'; import {streamifyResponse} from './helpers/streamify-response'; @@ -68,7 +65,7 @@ const innerHandler = async ({ const currentUserId = context.invokedFunctionArn.split(':')[4]; if (params.type === ServerlessRoutines.still) { - validateDeleteAfter(params.deleteAfter); + providerSpecifics.validateDeleteAfter(params.deleteAfter); const renderId = generateRandomHashWithLifeCycleRule( params.deleteAfter, providerSpecifics, diff --git a/packages/lambda/src/functions/start.ts b/packages/lambda/src/functions/start.ts index 9a8f3ad919d..df59997e9c4 100644 --- a/packages/lambda/src/functions/start.ts +++ b/packages/lambda/src/functions/start.ts @@ -8,7 +8,6 @@ import { import {VERSION} from 'remotion/version'; import type {AwsRegion} from '../regions'; import {callLambdaAsync} from '../shared/call-lambda-async'; -import {validateDeleteAfter} from './helpers/lifecycle'; import {makeInitialOverallRenderProgress} from './helpers/overall-render-progress'; type Options = { @@ -57,7 +56,7 @@ export const startHandler = async ( bucketName, }); - validateDeleteAfter(params.deleteAfter); + providerSpecifics.validateDeleteAfter(params.deleteAfter); const initialFile = providerSpecifics.writeFile({ bucketName, diff --git a/packages/lambda/src/test/mock-implementation.ts b/packages/lambda/src/test/mock-implementation.ts index f483f2ce949..059f9fbaa53 100644 --- a/packages/lambda/src/test/mock-implementation.ts +++ b/packages/lambda/src/test/mock-implementation.ts @@ -119,4 +119,5 @@ export const mockImplementation: ProviderSpecifics = { s3Url: 'https://s3.af-south-1.amazonaws.com/bucket/key', s3Key: 'key', }), + validateDeleteAfter: () => {}, }; diff --git a/packages/serverless/src/provider-implementation.ts b/packages/serverless/src/provider-implementation.ts index 690511fe869..43eeb5da267 100644 --- a/packages/serverless/src/provider-implementation.ts +++ b/packages/serverless/src/provider-implementation.ts @@ -135,4 +135,5 @@ export type ProviderSpecifics = { printLoggingHelper: boolean; getFolderFiles: GetFolderFiles; makeArtifactWithDetails: MakeArtifactWithDetails; + validateDeleteAfter: (lifeCycleValue: unknown) => void; }; From 3206afea9ee185075d51eeec0c465a8172b4274a Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Sun, 5 Jan 2025 20:26:29 +0100 Subject: [PATCH 03/14] hmm --- packages/lambda/src/client.ts | 3 +- .../helpers/calculate-price-from-bucket.ts | 8 +- .../helpers/create-post-render-data.ts | 19 ++-- .../helpers/find-output-file-in-bucket.ts | 3 +- .../helpers/get-overall-progress-s3.ts | 7 +- .../src/functions/helpers/get-progress.ts | 5 +- .../src/functions/helpers/get-retry-stats.ts | 5 - .../src/functions/helpers/merge-chunks.ts | 9 +- .../src/functions/helpers/stream-renderer.ts | 7 +- packages/lambda/src/functions/http-client.ts | 19 ++++ packages/lambda/src/functions/launch.ts | 17 +++- packages/lambda/src/functions/progress.ts | 7 +- packages/lambda/src/functions/start.ts | 7 +- packages/lambda/src/functions/still.ts | 2 +- packages/lambda/src/index.ts | 3 +- packages/lambda/src/internals.ts | 2 +- packages/lambda/src/shared/constants.ts | 92 +------------------ .../src/shared/parse-lambda-timings-key.ts | 5 - .../src/test/integration/webhooks.test.ts | 2 +- .../src}/calculate-chunk-times.ts | 2 +- packages/serverless/src/compositions.ts | 2 +- packages/serverless/src/compress-props.ts | 2 +- packages/serverless/src/constants.ts | 33 ++++++- packages/serverless/src/expected-out-name.ts | 2 +- .../serverless/src/get-browser-instance.ts | 2 +- .../serverless/src/get-custom-out-name.ts | 2 +- .../serverless/src/get-or-create-bucket.ts | 2 +- packages/serverless/src/index.ts | 21 +++-- packages/serverless/src/info.ts | 2 +- .../src}/invoke-webhook.ts | 39 +++----- packages/serverless/src/make-bucket-name.ts | 2 +- .../helpers => serverless/src}/min-max.ts | 0 .../src/most-expensive-chunks.ts} | 2 +- .../src}/overall-render-progress.ts | 18 ++-- .../serverless/src/provider-implementation.ts | 2 +- packages/serverless/src/render-metadata.ts | 2 +- packages/serverless/src/render-progress.ts | 58 ++++++++++++ .../serverless/src/streaming/streaming.ts | 2 +- .../src/test}/invoke-webhook.test.ts | 4 +- .../src/test}/min-max.test.ts | 5 +- .../src/test/most-expensive.test.ts} | 5 +- .../serverless/src/{still.ts => types.ts} | 12 +++ .../serverless/src/validate-composition.ts | 2 +- packages/serverless/src/write-lambda-error.ts | 2 +- 44 files changed, 241 insertions(+), 206 deletions(-) delete mode 100644 packages/lambda/src/functions/helpers/get-retry-stats.ts create mode 100644 packages/lambda/src/functions/http-client.ts delete mode 100644 packages/lambda/src/shared/parse-lambda-timings-key.ts rename packages/{lambda/src/functions/helpers => serverless/src}/calculate-chunk-times.ts (93%) rename packages/{lambda/src/shared => serverless/src}/invoke-webhook.ts (81%) rename packages/{lambda/src/functions/helpers => serverless/src}/min-max.ts (100%) rename packages/{lambda/src/shared/get-most-expensive-chunks.ts => serverless/src/most-expensive-chunks.ts} (93%) rename packages/{lambda/src/functions/helpers => serverless/src}/overall-render-progress.ts (95%) create mode 100644 packages/serverless/src/render-progress.ts rename packages/{lambda/src/test/unit => serverless/src/test}/invoke-webhook.test.ts (78%) rename packages/{lambda/src/test/unit => serverless/src/test}/min-max.test.ts (85%) rename packages/{lambda/src/test/unit/most-expensive-chunks.test.ts => serverless/src/test/most-expensive.test.ts} (96%) rename packages/serverless/src/{still.ts => types.ts} (83%) diff --git a/packages/lambda/src/client.ts b/packages/lambda/src/client.ts index efad02e9281..b2f6f5ce051 100644 --- a/packages/lambda/src/client.ts +++ b/packages/lambda/src/client.ts @@ -24,7 +24,7 @@ import type {SpeculateFunctionNameInput} from './api/speculate-function-name'; import {speculateFunctionName} from './api/speculate-function-name'; import {validateWebhookSignature} from './api/validate-webhook-signature'; import type {RenderProgress} from './shared/constants'; -import type {WebhookPayload} from './shared/invoke-webhook'; +export type {WebhookPayload} from '@remotion/serverless'; export {CustomCredentials, DeleteAfter} from '@remotion/serverless/client'; export { @@ -56,5 +56,4 @@ export type { RenderStillOnLambdaInput, RenderStillOnLambdaOutput, SpeculateFunctionNameInput, - WebhookPayload, }; diff --git a/packages/lambda/src/functions/helpers/calculate-price-from-bucket.ts b/packages/lambda/src/functions/helpers/calculate-price-from-bucket.ts index cb63802d1f7..a973f8f8142 100644 --- a/packages/lambda/src/functions/helpers/calculate-price-from-bucket.ts +++ b/packages/lambda/src/functions/helpers/calculate-price-from-bucket.ts @@ -1,9 +1,11 @@ -import type {CloudProvider} from '@remotion/serverless'; +import { + calculateChunkTimes, + type CloudProvider, + type ParsedTiming, +} from '@remotion/serverless'; import type {RenderMetadata} from '@remotion/serverless/client'; import {estimatePrice} from '../../api/estimate-price'; import type {AwsRegion} from '../../regions'; -import type {ParsedTiming} from '../../shared/parse-lambda-timings-key'; -import {calculateChunkTimes} from './calculate-chunk-times'; export const estimatePriceFromBucket = ({ renderMetadata, diff --git a/packages/lambda/src/functions/helpers/create-post-render-data.ts b/packages/lambda/src/functions/helpers/create-post-render-data.ts index 9b2b4b4e613..36d93fb1dcf 100644 --- a/packages/lambda/src/functions/helpers/create-post-render-data.ts +++ b/packages/lambda/src/functions/helpers/create-post-render-data.ts @@ -1,16 +1,19 @@ -import type {CloudProvider, EnhancedErrorInfo} from '@remotion/serverless'; +import type { + CloudProvider, + EnhancedErrorInfo, + OverallRenderProgress, + PostRenderData, +} from '@remotion/serverless'; +import { + OVERHEAD_TIME_PER_LAMBDA, + calculateChunkTimes, + getMostExpensiveChunks, +} from '@remotion/serverless'; import type {RenderMetadata} from '@remotion/serverless/client'; import {estimatePrice} from '../../api/estimate-price'; import type {AwsRegion} from '../../regions'; -import type {PostRenderData} from '../../shared/constants'; import {MAX_EPHEMERAL_STORAGE_IN_MB} from '../../shared/constants'; -import { - OVERHEAD_TIME_PER_LAMBDA, - getMostExpensiveChunks, -} from '../../shared/get-most-expensive-chunks'; -import {calculateChunkTimes} from './calculate-chunk-times'; import type {OutputFileMetadata} from './find-output-file-in-bucket'; -import type {OverallRenderProgress} from './overall-render-progress'; export const createPostRenderData = ({ region, diff --git a/packages/lambda/src/functions/helpers/find-output-file-in-bucket.ts b/packages/lambda/src/functions/helpers/find-output-file-in-bucket.ts index 68aa167c68b..424cb094143 100644 --- a/packages/lambda/src/functions/helpers/find-output-file-in-bucket.ts +++ b/packages/lambda/src/functions/helpers/find-output-file-in-bucket.ts @@ -4,7 +4,6 @@ import { type CustomCredentials, type RenderMetadata, } from '@remotion/serverless/client'; -import {ROLE_NAME} from '../../api/iam-validation/suggested-policy'; import {getOutputUrlFromMetadata} from './get-output-url-from-metadata'; export type OutputFileMetadata = { @@ -69,7 +68,7 @@ export const findOutputFileInBucket = async ({ customCredentials?.endpoint ? `(S3 Endpoint = ${customCredentials?.endpoint})` : '' - }. The "${ROLE_NAME}" role must have permission for both "s3:GetObject" and "s3:ListBucket" actions.`, + }. The Lambda role must have permission for both "s3:GetObject" and "s3:ListBucket" actions.`, ); } diff --git a/packages/lambda/src/functions/helpers/get-overall-progress-s3.ts b/packages/lambda/src/functions/helpers/get-overall-progress-s3.ts index 65bb3b38890..eaf7c36b407 100644 --- a/packages/lambda/src/functions/helpers/get-overall-progress-s3.ts +++ b/packages/lambda/src/functions/helpers/get-overall-progress-s3.ts @@ -1,6 +1,9 @@ -import type {CloudProvider, ProviderSpecifics} from '@remotion/serverless'; +import type { + CloudProvider, + OverallRenderProgress, + ProviderSpecifics, +} from '@remotion/serverless'; import {overallProgressKey, streamToString} from '@remotion/serverless/client'; -import type {OverallRenderProgress} from './overall-render-progress'; export const getOverallProgressS3 = async ({ renderId, diff --git a/packages/lambda/src/functions/helpers/get-progress.ts b/packages/lambda/src/functions/helpers/get-progress.ts index 11af07b7921..9f1ce6540da 100644 --- a/packages/lambda/src/functions/helpers/get-progress.ts +++ b/packages/lambda/src/functions/helpers/get-progress.ts @@ -1,9 +1,12 @@ import {NoReactAPIs} from '@remotion/renderer/pure'; import type { + CleanupInfo, CloudProvider, EnhancedErrorInfo, + GenericRenderProgress, ProviderSpecifics, } from '@remotion/serverless'; +import {calculateChunkTimes} from '@remotion/serverless'; import { getExpectedOutName, truthy, @@ -11,9 +14,7 @@ import { } from '@remotion/serverless/client'; import {NoReactInternals} from 'remotion/no-react'; import type {AwsRegion} from '../../regions'; -import type {CleanupInfo, GenericRenderProgress} from '../../shared/constants'; import {MAX_EPHEMERAL_STORAGE_IN_MB} from '../../shared/constants'; -import {calculateChunkTimes} from './calculate-chunk-times'; import {estimatePriceFromBucket} from './calculate-price-from-bucket'; import {formatCostsInfo} from './format-costs-info'; import {getOverallProgress} from './get-overall-progress'; diff --git a/packages/lambda/src/functions/helpers/get-retry-stats.ts b/packages/lambda/src/functions/helpers/get-retry-stats.ts deleted file mode 100644 index 020866bc454..00000000000 --- a/packages/lambda/src/functions/helpers/get-retry-stats.ts +++ /dev/null @@ -1,5 +0,0 @@ -export type ChunkRetry = { - chunk: number; - attempt: number; - time: number; -}; diff --git a/packages/lambda/src/functions/helpers/merge-chunks.ts b/packages/lambda/src/functions/helpers/merge-chunks.ts index 468203babcb..31076d4bcad 100644 --- a/packages/lambda/src/functions/helpers/merge-chunks.ts +++ b/packages/lambda/src/functions/helpers/merge-chunks.ts @@ -1,5 +1,10 @@ import type {AudioCodec, LogLevel} from '@remotion/renderer'; -import type {CloudProvider, ProviderSpecifics} from '@remotion/serverless'; +import type { + CloudProvider, + OverallProgressHelper, + PostRenderData, + ProviderSpecifics, +} from '@remotion/serverless'; import type { CustomCredentials, DownloadBehavior, @@ -9,13 +14,11 @@ import type { ServerlessCodec, } from '@remotion/serverless/client'; import fs from 'fs'; -import type {PostRenderData} from '../../shared/constants'; import {cleanupProps} from './cleanup-props'; import {concatVideos} from './concat-videos'; import {createPostRenderData} from './create-post-render-data'; import {getOutputUrlFromMetadata} from './get-output-url-from-metadata'; import {inspectErrors} from './inspect-errors'; -import type {OverallProgressHelper} from './overall-render-progress'; import {timer} from './timer'; export const mergeChunksAndFinishRender = async < diff --git a/packages/lambda/src/functions/helpers/stream-renderer.ts b/packages/lambda/src/functions/helpers/stream-renderer.ts index 3bc42ee3196..883890838d1 100644 --- a/packages/lambda/src/functions/helpers/stream-renderer.ts +++ b/packages/lambda/src/functions/helpers/stream-renderer.ts @@ -1,6 +1,10 @@ import type {EmittedArtifact, LogLevel} from '@remotion/renderer'; import {RenderInternals} from '@remotion/renderer'; -import type {CloudProvider, OnMessage} from '@remotion/serverless'; +import type { + CloudProvider, + OnMessage, + OverallProgressHelper, +} from '@remotion/serverless'; import { deserializeArtifact, type ProviderSpecifics, @@ -10,7 +14,6 @@ import {ServerlessRoutines} from '@remotion/serverless/client'; import {writeFileSync} from 'fs'; import {join} from 'path'; import {callLambdaWithStreaming} from '../../shared/call-lambda-streaming'; -import type {OverallProgressHelper} from './overall-render-progress'; type StreamRendererResponse = | { diff --git a/packages/lambda/src/functions/http-client.ts b/packages/lambda/src/functions/http-client.ts new file mode 100644 index 00000000000..0f0cbb26a92 --- /dev/null +++ b/packages/lambda/src/functions/http-client.ts @@ -0,0 +1,19 @@ +import http from 'node:http'; +import https from 'node:https'; + +export const mockableHttpClients = { + http: http.request, + https: https.request, +}; + +export const getWebhookClient = (url: string) => { + if (url.startsWith('https://')) { + return mockableHttpClients.https; + } + + if (url.startsWith('http://')) { + return mockableHttpClients.http; + } + + throw new Error('Can only request URLs starting with http:// or https://'); +}; diff --git a/packages/lambda/src/functions/launch.ts b/packages/lambda/src/functions/launch.ts index 95b3ad3c6c3..e8442b9fe8d 100644 --- a/packages/lambda/src/functions/launch.ts +++ b/packages/lambda/src/functions/launch.ts @@ -1,11 +1,18 @@ /* eslint-disable @typescript-eslint/no-use-before-define */ import type {EmittedArtifact, LogOptions} from '@remotion/renderer'; import {RenderInternals} from '@remotion/renderer'; -import type {CloudProvider, ProviderSpecifics} from '@remotion/serverless'; +import type { + CloudProvider, + OverallProgressHelper, + PostRenderData, + ProviderSpecifics, +} from '@remotion/serverless'; import { forgetBrowserEventLoop, getBrowserInstance, getTmpDirStateIfENoSp, + invokeWebhook, + makeOverallRenderProgress, validateComposition, validateOutname, } from '@remotion/serverless'; @@ -26,13 +33,11 @@ import {existsSync, mkdirSync, rmSync} from 'fs'; import {type EventEmitter} from 'node:events'; import {join} from 'path'; import {VERSION} from 'remotion/version'; -import type {PostRenderData} from '../shared/constants'; import { CONCAT_FOLDER_TOKEN, MAX_FUNCTIONS_PER_RENDER, } from '../shared/constants'; import {DOCS_URL} from '../shared/docs-url'; -import {invokeWebhook} from '../shared/invoke-webhook'; import { validateDimension, validateDurationInFrames, @@ -45,10 +50,9 @@ import {bestFramesPerLambdaParam} from './helpers/best-frames-per-lambda-param'; import {cleanupProps} from './helpers/cleanup-props'; import {findOutputFileInBucket} from './helpers/find-output-file-in-bucket'; import {mergeChunksAndFinishRender} from './helpers/merge-chunks'; -import type {OverallProgressHelper} from './helpers/overall-render-progress'; -import {makeOverallRenderProgress} from './helpers/overall-render-progress'; import {streamRendererFunctionWithRetry} from './helpers/stream-renderer'; import {timer} from './helpers/timer'; +import {getWebhookClient} from './http-client'; type Options = { expectedBucketOwner: string; @@ -596,6 +600,7 @@ export const launchHandler = async ( customData: params.webhook.customData ?? null, }, redirectsSoFar: 0, + client: getWebhookClient(params.webhook.url), }, params.logLevel, ); @@ -699,6 +704,7 @@ export const launchHandler = async ( costs: postRenderData.cost, }, redirectsSoFar: 0, + client: getWebhookClient(params.webhook.url), }, params.logLevel, ); @@ -793,6 +799,7 @@ export const launchHandler = async ( })), }, redirectsSoFar: 0, + client: getWebhookClient(params.webhook.url), }, params.logLevel, ); diff --git a/packages/lambda/src/functions/progress.ts b/packages/lambda/src/functions/progress.ts index c8474546037..014e8ad657e 100644 --- a/packages/lambda/src/functions/progress.ts +++ b/packages/lambda/src/functions/progress.ts @@ -1,8 +1,11 @@ -import type {CloudProvider, ProviderSpecifics} from '@remotion/serverless'; +import type { + CloudProvider, + GenericRenderProgress, + ProviderSpecifics, +} from '@remotion/serverless'; import type {ServerlessPayload} from '@remotion/serverless/client'; import {ServerlessRoutines} from '@remotion/serverless/client'; import {VERSION} from 'remotion/version'; -import type {GenericRenderProgress} from '../shared/constants'; import {getProgress} from './helpers/get-progress'; type Options = { diff --git a/packages/lambda/src/functions/start.ts b/packages/lambda/src/functions/start.ts index df59997e9c4..bba6296199b 100644 --- a/packages/lambda/src/functions/start.ts +++ b/packages/lambda/src/functions/start.ts @@ -1,4 +1,8 @@ -import type {CloudProvider, ProviderSpecifics} from '@remotion/serverless'; +import { + makeInitialOverallRenderProgress, + type CloudProvider, + type ProviderSpecifics, +} from '@remotion/serverless'; import type {ServerlessPayload} from '@remotion/serverless/client'; import { ServerlessRoutines, @@ -8,7 +12,6 @@ import { import {VERSION} from 'remotion/version'; import type {AwsRegion} from '../regions'; import {callLambdaAsync} from '../shared/call-lambda-async'; -import {makeInitialOverallRenderProgress} from './helpers/overall-render-progress'; type Options = { expectedBucketOwner: string; diff --git a/packages/lambda/src/functions/still.ts b/packages/lambda/src/functions/still.ts index 5fbd0d4356a..4abfb2882f8 100644 --- a/packages/lambda/src/functions/still.ts +++ b/packages/lambda/src/functions/still.ts @@ -11,6 +11,7 @@ import { getBrowserInstance, getCredentialsFromOutName, getTmpDirStateIfENoSp, + makeInitialOverallRenderProgress, validateComposition, validateOutname, type ProviderSpecifics, @@ -41,7 +42,6 @@ import {validatePrivacy} from '../shared/validate-privacy'; import {formatCostsInfo} from './helpers/format-costs-info'; import {getOutputUrlFromMetadata} from './helpers/get-output-url-from-metadata'; import {onDownloadsHelper} from './helpers/on-downloads-logger'; -import {makeInitialOverallRenderProgress} from './helpers/overall-render-progress'; type Options = { params: ServerlessPayload; diff --git a/packages/lambda/src/index.ts b/packages/lambda/src/index.ts index 443ff4be311..574d0aa912b 100644 --- a/packages/lambda/src/index.ts +++ b/packages/lambda/src/index.ts @@ -73,7 +73,7 @@ import { } from './internals'; import type {AwsRegion} from './regions'; import type {RenderProgress} from './shared/constants'; -import type {WebhookPayload} from './shared/invoke-webhook'; +export type {WebhookPayload} from '@remotion/serverless'; /** * @deprecated Import this from `@remotion/lambda/client` instead @@ -191,7 +191,6 @@ export type { RenderStillOnLambdaOutput, SimulatePermissionsInput, SimulatePermissionsOutput, - WebhookPayload, }; export {_InternalAwsProvider, _InternalOverallRenderProgress}; diff --git a/packages/lambda/src/internals.ts b/packages/lambda/src/internals.ts index f3470014dd8..9c712ea62c4 100644 --- a/packages/lambda/src/internals.ts +++ b/packages/lambda/src/internals.ts @@ -14,5 +14,5 @@ export const LambdaInternals = { internalDeploySite, }; +export {OverallRenderProgress as _InternalOverallRenderProgress} from '@remotion/serverless'; export {AwsProvider as _InternalAwsProvider} from './functions/aws-implementation'; -export {OverallRenderProgress as _InternalOverallRenderProgress} from './functions/helpers/overall-render-progress'; diff --git a/packages/lambda/src/shared/constants.ts b/packages/lambda/src/shared/constants.ts index ec63171dbc1..ea0383b8b10 100644 --- a/packages/lambda/src/shared/constants.ts +++ b/packages/lambda/src/shared/constants.ts @@ -1,18 +1,8 @@ -import type { - CloudProvider, - EnhancedErrorInfo, - ReceivedArtifact, -} from '@remotion/serverless'; -import { - type DeleteAfter, - type Privacy, - type RenderMetadata, -} from '@remotion/serverless/client'; +import type {GenericRenderProgress} from '@remotion/serverless'; +import {type Privacy} from '@remotion/serverless/client'; import {NoReactInternals} from 'remotion/no-react'; import type {AwsProvider} from '../functions/aws-implementation'; -import type {ChunkRetry} from '../functions/helpers/get-retry-stats'; import type {AwsRegion} from '../regions'; -import type {ExpensiveChunk} from './get-most-expensive-chunks'; export const MIN_MEMORY = 512; export const MAX_MEMORY = 10240; @@ -54,35 +44,6 @@ export const CONCAT_FOLDER_TOKEN = 'remotion-concat'; export const REMOTION_CONCATED_TOKEN = 'remotion-concated-token'; export const REMOTION_FILELIST_TOKEN = 'remotion-filelist'; -export type AfterRenderCost = { - estimatedCost: number; - estimatedDisplayCost: string; - currency: string; - disclaimer: string; -}; - -export type PostRenderData = { - cost: AfterRenderCost; - outputFile: string; - outputSize: number; - renderSize: number; - timeToFinish: number; - timeToRenderFrames: number; - errors: EnhancedErrorInfo[]; - startTime: number; - endTime: number; - filesCleanedUp: number; - timeToEncode: number; - timeToCleanUp: number; - timeToRenderChunks: number; - retriesInfo: ChunkRetry[]; - mostExpensiveFrameRanges: ExpensiveChunk[] | undefined; - estimatedBillingDurationInMilliseconds: number; - deleteAfter: DeleteAfter | null; - timeToCombine: number | null; - artifactProgress: ReceivedArtifact[]; -}; - export type CostsInfo = { accruedSoFar: number; displayCost: string; @@ -90,55 +51,6 @@ export type CostsInfo = { disclaimer: string; }; -export type CleanupInfo = { - doneIn: number | null; - minFilesToDelete: number; - filesDeleted: number; -}; - -type EncodingProgress = { - framesEncoded: number; - combinedFrames: number; - timeToCombine: number | null; -}; - -export type GenericRenderProgress = { - chunks: number; - done: boolean; - encodingStatus: EncodingProgress | null; - costs: CostsInfo; - renderId: string; - renderMetadata: RenderMetadata | null; - bucket: string; - outputFile: string | null; - outKey: string | null; - outBucket: string | null; - timeToFinish: number | null; - errors: EnhancedErrorInfo[]; - fatalErrorEncountered: boolean; - currentTime: number; - renderSize: number; - lambdasInvoked: number; - cleanup: CleanupInfo | null; - timeToFinishChunks: number | null; - timeToRenderFrames: number | null; - timeToEncode: number | null; - overallProgress: number; - retriesInfo: ChunkRetry[]; - mostExpensiveFrameRanges: ExpensiveChunk[] | null; - framesRendered: number; - outputSizeInBytes: number | null; - type: 'success'; - estimatedBillingDurationInMilliseconds: number | null; - combinedFrames: number; - timeToCombine: number | null; - timeoutTimestamp: number; - functionLaunched: number; - serveUrlOpened: number | null; - compositionValidated: number | null; - artifacts: ReceivedArtifact[]; -}; - export type RenderProgress = GenericRenderProgress; export const LAMBDA_CONCURRENCY_LIMIT_QUOTA = 'L-B99A9384'; diff --git a/packages/lambda/src/shared/parse-lambda-timings-key.ts b/packages/lambda/src/shared/parse-lambda-timings-key.ts deleted file mode 100644 index 952fca82f83..00000000000 --- a/packages/lambda/src/shared/parse-lambda-timings-key.ts +++ /dev/null @@ -1,5 +0,0 @@ -export type ParsedTiming = { - chunk: number; - start: number; - rendered: number; -}; diff --git a/packages/lambda/src/test/integration/webhooks.test.ts b/packages/lambda/src/test/integration/webhooks.test.ts index 80b42f8b053..71641d7a015 100644 --- a/packages/lambda/src/test/integration/webhooks.test.ts +++ b/packages/lambda/src/test/integration/webhooks.test.ts @@ -3,8 +3,8 @@ import {ServerlessRoutines} from '@remotion/serverless/client'; import path from 'path'; import {VERSION} from 'remotion/version'; import {beforeAll, beforeEach, describe, expect, test, vi} from 'vitest'; +import {mockableHttpClients} from '../../functions/http-client'; import {callLambdaSync} from '../../shared/call-lambda-sync'; -import {mockableHttpClients} from '../../shared/invoke-webhook'; const originalFetch = mockableHttpClients.http; beforeEach(() => { diff --git a/packages/lambda/src/functions/helpers/calculate-chunk-times.ts b/packages/serverless/src/calculate-chunk-times.ts similarity index 93% rename from packages/lambda/src/functions/helpers/calculate-chunk-times.ts rename to packages/serverless/src/calculate-chunk-times.ts index 70274624d2b..9f8edf8c515 100644 --- a/packages/lambda/src/functions/helpers/calculate-chunk-times.ts +++ b/packages/serverless/src/calculate-chunk-times.ts @@ -1,5 +1,5 @@ -import type {ParsedTiming} from '../../shared/parse-lambda-timings-key'; import {max, min} from './min-max'; +import type {ParsedTiming} from './types'; const getAbsoluteTime = (parsedTimings: ParsedTiming[]) => { if (parsedTimings.length === 0) { diff --git a/packages/serverless/src/compositions.ts b/packages/serverless/src/compositions.ts index 4595536df65..f5ea4a12867 100644 --- a/packages/serverless/src/compositions.ts +++ b/packages/serverless/src/compositions.ts @@ -9,7 +9,7 @@ import { } from './get-browser-instance'; import {internalGetOrCreateBucket} from './get-or-create-bucket'; import type {ProviderSpecifics} from './provider-implementation'; -import type {CloudProvider} from './still'; +import type {CloudProvider} from './types'; type Options = { expectedBucketOwner: string; diff --git a/packages/serverless/src/compress-props.ts b/packages/serverless/src/compress-props.ts index 57da93c3e11..0e70def1687 100644 --- a/packages/serverless/src/compress-props.ts +++ b/packages/serverless/src/compress-props.ts @@ -3,8 +3,8 @@ import type {SerializedInputProps} from './constants'; import {internalGetOrCreateBucket} from './get-or-create-bucket'; import {inputPropsKey, resolvedPropsKey} from './input-props-keys'; import type {ProviderSpecifics} from './provider-implementation'; -import type {CloudProvider} from './still'; import {streamToString} from './stream-to-string'; +import type {CloudProvider} from './types'; import {MAX_WEBHOOK_CUSTOM_DATA_SIZE} from './validate-webhook'; type PropsType = 'input-props' | 'resolved-props'; diff --git a/packages/serverless/src/constants.ts b/packages/serverless/src/constants.ts index 846cbca3897..22134a30d19 100644 --- a/packages/serverless/src/constants.ts +++ b/packages/serverless/src/constants.ts @@ -12,7 +12,9 @@ import type { X264Preset, } from '@remotion/renderer'; import type {BrowserSafeApis} from '@remotion/renderer/client'; -import type {CloudProvider} from './still'; +import type {ExpensiveChunk} from './most-expensive-chunks'; +import type {ChunkRetry, CloudProvider, ReceivedArtifact} from './types'; +import type {EnhancedErrorInfo} from './write-lambda-error'; // Needs to be in sync with renderer/src/options/delete-after.ts#L7 export const expiryDays = { @@ -357,3 +359,32 @@ export const overallProgressKey = (renderId: string) => export const artifactName = (renderId: string, name: string) => `${rendersPrefix(renderId)}/artifacts/${name}`; + +export type PostRenderData = { + cost: AfterRenderCost; + outputFile: string; + outputSize: number; + renderSize: number; + timeToFinish: number; + timeToRenderFrames: number; + errors: EnhancedErrorInfo[]; + startTime: number; + endTime: number; + filesCleanedUp: number; + timeToEncode: number; + timeToCleanUp: number; + timeToRenderChunks: number; + retriesInfo: ChunkRetry[]; + mostExpensiveFrameRanges: ExpensiveChunk[] | undefined; + estimatedBillingDurationInMilliseconds: number; + deleteAfter: DeleteAfter | null; + timeToCombine: number | null; + artifactProgress: ReceivedArtifact[]; +}; + +export type AfterRenderCost = { + estimatedCost: number; + estimatedDisplayCost: string; + currency: string; + disclaimer: string; +}; diff --git a/packages/serverless/src/expected-out-name.ts b/packages/serverless/src/expected-out-name.ts index b0271af939c..045e4912657 100644 --- a/packages/serverless/src/expected-out-name.ts +++ b/packages/serverless/src/expected-out-name.ts @@ -10,7 +10,7 @@ import { } from './constants'; import {getCustomOutName} from './get-custom-out-name'; import type {RenderMetadata} from './render-metadata'; -import type {CloudProvider} from './still'; +import type {CloudProvider} from './types'; import {validateOutname} from './validate-outname'; export const getCredentialsFromOutName = ( diff --git a/packages/serverless/src/get-browser-instance.ts b/packages/serverless/src/get-browser-instance.ts index 2ba0545de96..af84b15100c 100644 --- a/packages/serverless/src/get-browser-instance.ts +++ b/packages/serverless/src/get-browser-instance.ts @@ -3,7 +3,7 @@ import {RenderInternals} from '@remotion/renderer'; import {VERSION} from 'remotion/version'; import type {Await} from './await'; import type {ProviderSpecifics} from './provider-implementation'; -import type {CloudProvider} from './still'; +import type {CloudProvider} from './types'; type LaunchedBrowser = { instance: Await>; diff --git a/packages/serverless/src/get-custom-out-name.ts b/packages/serverless/src/get-custom-out-name.ts index e66dfbfd36c..2ace4d6afc8 100644 --- a/packages/serverless/src/get-custom-out-name.ts +++ b/packages/serverless/src/get-custom-out-name.ts @@ -1,6 +1,6 @@ import type {CustomCredentials, OutNameInput} from './constants'; import type {RenderMetadata} from './render-metadata'; -import type {CloudProvider} from './still'; +import type {CloudProvider} from './types'; export const getCustomOutName = ({ renderMetadata, diff --git a/packages/serverless/src/get-or-create-bucket.ts b/packages/serverless/src/get-or-create-bucket.ts index 6eea5db5aa5..05231a01517 100644 --- a/packages/serverless/src/get-or-create-bucket.ts +++ b/packages/serverless/src/get-or-create-bucket.ts @@ -2,7 +2,7 @@ import type {CustomCredentials} from './constants'; import {REMOTION_BUCKET_PREFIX} from './constants'; import {makeBucketName} from './make-bucket-name'; import type {ProviderSpecifics} from './provider-implementation'; -import type {CloudProvider} from './still'; +import type {CloudProvider} from './types'; type GetOrCreateBucketInputInner = { region: Provider['region']; diff --git a/packages/serverless/src/index.ts b/packages/serverless/src/index.ts index 0c51edb5266..03bb79454a4 100644 --- a/packages/serverless/src/index.ts +++ b/packages/serverless/src/index.ts @@ -1,23 +1,31 @@ export {compositionsHandler} from './compositions'; +export {calculateChunkTimes} from './calculate-chunk-times'; +export {PostRenderData} from './constants'; export {getCredentialsFromOutName} from './expected-out-name'; export { forgetBrowserEventLoop, getBrowserInstance, } from './get-browser-instance'; export {infoHandler} from './info'; +export {WebhookPayload, invokeWebhook} from './invoke-webhook'; +export { + OVERHEAD_TIME_PER_LAMBDA, + getMostExpensiveChunks, +} from './most-expensive-chunks'; +export { + OverallProgressHelper, + OverallRenderProgress, + makeInitialOverallRenderProgress, + makeOverallRenderProgress, +} from './overall-render-progress'; export { MakeArtifactWithDetails, ProviderSpecifics, WriteFileInput, } from './provider-implementation'; +export type {CleanupInfo, GenericRenderProgress} from './render-progress'; export {deserializeArtifact, serializeArtifact} from './serialize-artifact'; -export { - CloudProvider, - CostsInfo, - ReceivedArtifact, - RenderStillLambdaResponsePayload, -} from './still'; export {ResponseStream} from './streaming/response-stream'; export {ResponseStreamWriter, streamWriter} from './streaming/stream-writer'; export { @@ -26,6 +34,7 @@ export { type StreamingMessage, type StreamingPayload, } from './streaming/streaming'; +export * from './types'; export {validateComposition} from './validate-composition'; export {validateOutname} from './validate-outname'; export { diff --git a/packages/serverless/src/info.ts b/packages/serverless/src/info.ts index 5ee7f788e43..926a926760b 100644 --- a/packages/serverless/src/info.ts +++ b/packages/serverless/src/info.ts @@ -1,7 +1,7 @@ import {VERSION} from 'remotion/version'; import type {ServerlessPayload} from './constants'; import {ServerlessRoutines} from './constants'; -import type {CloudProvider} from './still'; +import type {CloudProvider} from './types'; export const infoHandler = ( serverlessParams: ServerlessPayload, diff --git a/packages/lambda/src/shared/invoke-webhook.ts b/packages/serverless/src/invoke-webhook.ts similarity index 81% rename from packages/lambda/src/shared/invoke-webhook.ts rename to packages/serverless/src/invoke-webhook.ts index f1e0643309a..3417be969b3 100644 --- a/packages/lambda/src/shared/invoke-webhook.ts +++ b/packages/serverless/src/invoke-webhook.ts @@ -1,18 +1,11 @@ import type {LogLevel} from '@remotion/renderer'; import {RenderInternals} from '@remotion/renderer'; -import type {EnhancedErrorInfo} from '@remotion/serverless'; -import https from 'https'; +import type https from 'https'; import * as Crypto from 'node:crypto'; -import http from 'node:http'; +import type http from 'node:http'; import type {AfterRenderCost} from './constants'; +import type {EnhancedErrorInfo} from './write-lambda-error'; -/** - * @description Calculates cryptographically secure signature for webhooks using Hmac. - * @link https://remotion.dev/docs/lambda/webhooks#validate-webhooks - * @param payload Stringified request body to encode in the signature. - * @param secret User-provided webhook secret used to sign the request. - * @returns {string} Calculated signature - */ export function calculateSignature(payload: string, secret: string | null) { if (!secret) { return 'NO_SECRET_PROVIDED'; @@ -51,32 +44,20 @@ export type WebhookPayload = { customData: Record | null; } & DynamicWebhookPayload; -export const mockableHttpClients = { - http: http.request, - https: https.request, -}; - // Don't handle 304 status code (Not Modified) as a redirect, // since the browser will display the right page. const redirectStatusCodes = [301, 302, 303, 307, 308]; -const getWebhookClient = (url: string) => { - if (url.startsWith('https://')) { - return mockableHttpClients.https; - } - - if (url.startsWith('http://')) { - return mockableHttpClients.http; - } - - throw new Error('Can only request URLs starting with http:// or https://'); -}; - type InvokeWebhookOptions = { payload: WebhookPayload; url: string; secret: string | null; redirectsSoFar: number; + client: ( + url: string | URL, + options: https.RequestOptions, + callback?: (res: http.IncomingMessage) => void, + ) => http.ClientRequest; }; function invokeWebhookRaw({ @@ -84,11 +65,12 @@ function invokeWebhookRaw({ secret, url, redirectsSoFar, + client, }: InvokeWebhookOptions): Promise { const jsonPayload = JSON.stringify(payload); return new Promise((resolve, reject) => { - const req = getWebhookClient(url)( + const req = client( url, { method: 'POST', @@ -123,6 +105,7 @@ function invokeWebhookRaw({ secret, url: res.headers.location, redirectsSoFar: redirectsSoFar + 1, + client, }) .then(resolve) .catch(reject); diff --git a/packages/serverless/src/make-bucket-name.ts b/packages/serverless/src/make-bucket-name.ts index 54ac2bd2ada..dbfc06a2c09 100644 --- a/packages/serverless/src/make-bucket-name.ts +++ b/packages/serverless/src/make-bucket-name.ts @@ -1,6 +1,6 @@ import {REMOTION_BUCKET_PREFIX} from './constants'; import type {ProviderSpecifics} from './provider-implementation'; -import type {CloudProvider} from './still'; +import type {CloudProvider} from './types'; export const makeBucketName = ( region: Provider['region'], diff --git a/packages/lambda/src/functions/helpers/min-max.ts b/packages/serverless/src/min-max.ts similarity index 100% rename from packages/lambda/src/functions/helpers/min-max.ts rename to packages/serverless/src/min-max.ts diff --git a/packages/lambda/src/shared/get-most-expensive-chunks.ts b/packages/serverless/src/most-expensive-chunks.ts similarity index 93% rename from packages/lambda/src/shared/get-most-expensive-chunks.ts rename to packages/serverless/src/most-expensive-chunks.ts index 460afe4703d..8ecd66288b7 100644 --- a/packages/lambda/src/shared/get-most-expensive-chunks.ts +++ b/packages/serverless/src/most-expensive-chunks.ts @@ -1,4 +1,4 @@ -import type {ParsedTiming} from './parse-lambda-timings-key'; +import type {ParsedTiming} from './types'; export const OVERHEAD_TIME_PER_LAMBDA = 100; diff --git a/packages/lambda/src/functions/helpers/overall-render-progress.ts b/packages/serverless/src/overall-render-progress.ts similarity index 95% rename from packages/lambda/src/functions/helpers/overall-render-progress.ts rename to packages/serverless/src/overall-render-progress.ts index 521b377affa..3ca94d2e17f 100644 --- a/packages/lambda/src/functions/helpers/overall-render-progress.ts +++ b/packages/serverless/src/overall-render-progress.ts @@ -1,18 +1,16 @@ import type {LogLevel} from '@remotion/renderer'; import {RenderInternals} from '@remotion/renderer'; +import type {PostRenderData} from './constants'; +import {overallProgressKey} from './constants'; +import type {ProviderSpecifics} from './provider-implementation'; +import type {RenderMetadata} from './render-metadata'; import type { + ChunkRetry, CloudProvider, - LambdaErrorInfo, - ProviderSpecifics, + ParsedTiming, ReceivedArtifact, -} from '@remotion/serverless'; -import { - overallProgressKey, - type RenderMetadata, -} from '@remotion/serverless/client'; -import type {PostRenderData} from '../../shared/constants'; -import type {ParsedTiming} from '../../shared/parse-lambda-timings-key'; -import type {ChunkRetry} from './get-retry-stats'; +} from './types'; +import type {LambdaErrorInfo} from './write-lambda-error'; export type OverallRenderProgress = { chunks: number[]; diff --git a/packages/serverless/src/provider-implementation.ts b/packages/serverless/src/provider-implementation.ts index 43eeb5da267..b98094bea36 100644 --- a/packages/serverless/src/provider-implementation.ts +++ b/packages/serverless/src/provider-implementation.ts @@ -2,7 +2,7 @@ import type {EmittedArtifact} from '@remotion/renderer'; import type {Readable} from 'stream'; import type {CustomCredentials, DownloadBehavior, Privacy} from './constants'; import type {GetFolderFiles} from './get-files-in-folder'; -import type {CloudProvider, ReceivedArtifact} from './still'; +import type {CloudProvider, ReceivedArtifact} from './types'; export type BucketWithLocation = { name: string; diff --git a/packages/serverless/src/render-metadata.ts b/packages/serverless/src/render-metadata.ts index 8e83b0be415..a0f26b2666c 100644 --- a/packages/serverless/src/render-metadata.ts +++ b/packages/serverless/src/render-metadata.ts @@ -11,7 +11,7 @@ import type { SerializedInputProps, ServerlessCodec, } from './constants'; -import type {CloudProvider} from './still'; +import type {CloudProvider} from './types'; type Discriminated = | { diff --git a/packages/serverless/src/render-progress.ts b/packages/serverless/src/render-progress.ts new file mode 100644 index 00000000000..08853097bde --- /dev/null +++ b/packages/serverless/src/render-progress.ts @@ -0,0 +1,58 @@ +import type {ExpensiveChunk} from './most-expensive-chunks'; +import type {RenderMetadata} from './render-metadata'; +import type { + ChunkRetry, + CloudProvider, + CostsInfo, + ReceivedArtifact, +} from './types'; +import type {EnhancedErrorInfo} from './write-lambda-error'; + +export type CleanupInfo = { + doneIn: number | null; + minFilesToDelete: number; + filesDeleted: number; +}; + +type EncodingProgress = { + framesEncoded: number; + combinedFrames: number; + timeToCombine: number | null; +}; + +export type GenericRenderProgress = { + chunks: number; + done: boolean; + encodingStatus: EncodingProgress | null; + costs: CostsInfo; + renderId: string; + renderMetadata: RenderMetadata | null; + bucket: string; + outputFile: string | null; + outKey: string | null; + outBucket: string | null; + timeToFinish: number | null; + errors: EnhancedErrorInfo[]; + fatalErrorEncountered: boolean; + currentTime: number; + renderSize: number; + lambdasInvoked: number; + cleanup: CleanupInfo | null; + timeToFinishChunks: number | null; + timeToRenderFrames: number | null; + timeToEncode: number | null; + overallProgress: number; + retriesInfo: ChunkRetry[]; + mostExpensiveFrameRanges: ExpensiveChunk[] | null; + framesRendered: number; + outputSizeInBytes: number | null; + type: 'success'; + estimatedBillingDurationInMilliseconds: number | null; + combinedFrames: number; + timeToCombine: number | null; + timeoutTimestamp: number; + functionLaunched: number; + serveUrlOpened: number | null; + compositionValidated: number | null; + artifacts: ReceivedArtifact[]; +}; diff --git a/packages/serverless/src/streaming/streaming.ts b/packages/serverless/src/streaming/streaming.ts index d684731360b..bc0ea5ee04d 100644 --- a/packages/serverless/src/streaming/streaming.ts +++ b/packages/serverless/src/streaming/streaming.ts @@ -1,6 +1,6 @@ import {makeStreamPayloadMessage} from '@remotion/streaming'; import type {SerializedArtifact} from '../serialize-artifact'; -import type {CloudProvider, RenderStillLambdaResponsePayload} from '../still'; +import type {CloudProvider, RenderStillLambdaResponsePayload} from '../types'; import type {LambdaErrorInfo} from '../write-lambda-error'; const framesRendered = 'frames-rendered' as const; diff --git a/packages/lambda/src/test/unit/invoke-webhook.test.ts b/packages/serverless/src/test/invoke-webhook.test.ts similarity index 78% rename from packages/lambda/src/test/unit/invoke-webhook.test.ts rename to packages/serverless/src/test/invoke-webhook.test.ts index 51b1e71e1d3..1f90be01d63 100644 --- a/packages/lambda/src/test/unit/invoke-webhook.test.ts +++ b/packages/serverless/src/test/invoke-webhook.test.ts @@ -1,5 +1,5 @@ -import {expect, test} from 'vitest'; -import {calculateSignature} from '../../shared/invoke-webhook'; +import {expect, test} from 'bun:test'; +import {calculateSignature} from '../invoke-webhook'; test('calculateSignature should return correct sha512 signature', () => { const signature = calculateSignature( diff --git a/packages/lambda/src/test/unit/min-max.test.ts b/packages/serverless/src/test/min-max.test.ts similarity index 85% rename from packages/lambda/src/test/unit/min-max.test.ts rename to packages/serverless/src/test/min-max.test.ts index 3fbf53cbad4..b47ad7bdb6e 100644 --- a/packages/lambda/src/test/unit/min-max.test.ts +++ b/packages/serverless/src/test/min-max.test.ts @@ -1,6 +1,5 @@ -import {describe, expect, test} from 'vitest'; -import {max, min} from '../../functions/helpers/min-max'; - +import {describe, expect, test} from 'bun:test'; +import {max, min} from '../min-max'; describe('min() and max()', () => { test('Implemented min() functions correctly', () => { expect(min([0, -30, 90, -120, 0])).toBe(-120); diff --git a/packages/lambda/src/test/unit/most-expensive-chunks.test.ts b/packages/serverless/src/test/most-expensive.test.ts similarity index 96% rename from packages/lambda/src/test/unit/most-expensive-chunks.test.ts rename to packages/serverless/src/test/most-expensive.test.ts index 617014a5e99..aa03d114bf0 100644 --- a/packages/lambda/src/test/unit/most-expensive-chunks.test.ts +++ b/packages/serverless/src/test/most-expensive.test.ts @@ -1,6 +1,5 @@ -import {expect, test} from 'vitest'; - -import {getMostExpensiveChunks} from '../../shared/get-most-expensive-chunks'; +import {expect, test} from 'bun:test'; +import {getMostExpensiveChunks} from '../most-expensive-chunks'; test('Should calculate most expensive chunks', () => { const most = getMostExpensiveChunks( diff --git a/packages/serverless/src/still.ts b/packages/serverless/src/types.ts similarity index 83% rename from packages/serverless/src/still.ts rename to packages/serverless/src/types.ts index 71f0967993c..393ce2fdfc8 100644 --- a/packages/serverless/src/still.ts +++ b/packages/serverless/src/types.ts @@ -35,3 +35,15 @@ export type RenderStillLambdaResponsePayload = { renderId: string; receivedArtifacts: ReceivedArtifact[]; }; + +export type ChunkRetry = { + chunk: number; + attempt: number; + time: number; +}; + +export type ParsedTiming = { + chunk: number; + start: number; + rendered: number; +}; diff --git a/packages/serverless/src/validate-composition.ts b/packages/serverless/src/validate-composition.ts index 05814caf410..df9eb13ba65 100644 --- a/packages/serverless/src/validate-composition.ts +++ b/packages/serverless/src/validate-composition.ts @@ -9,7 +9,7 @@ import {RenderInternals} from '@remotion/renderer'; import type {VideoConfig} from 'remotion/no-react'; import type {Await} from './await'; import type {ProviderSpecifics} from './provider-implementation'; -import type {CloudProvider} from './still'; +import type {CloudProvider} from './types'; type ValidateCompositionOptions = { serveUrl: string; diff --git a/packages/serverless/src/write-lambda-error.ts b/packages/serverless/src/write-lambda-error.ts index 1ab708811a0..0e39fb67415 100644 --- a/packages/serverless/src/write-lambda-error.ts +++ b/packages/serverless/src/write-lambda-error.ts @@ -1,7 +1,7 @@ import {errorIsOutOfSpaceError} from './error-category'; import type {FileNameAndSize} from './get-files-in-folder'; import type {ProviderSpecifics} from './provider-implementation'; -import type {CloudProvider} from './still'; +import type {CloudProvider} from './types'; export type LambdaErrorInfo = { type: 'renderer' | 'browser' | 'stitcher' | 'webhook' | 'artifact'; From aa70a449c4a17e9606323dc962779261c55c660c Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Sun, 5 Jan 2025 20:43:36 +0100 Subject: [PATCH 04/14] compositions --- packages/lambda/src/api/get-compositions-on-lambda.ts | 1 + .../src/functions/helpers/calculate-price-from-bucket.ts | 7 ++----- .../src/functions/helpers/create-post-render-data.ts | 2 +- packages/lambda/src/functions/helpers/get-progress.ts | 2 +- packages/lambda/src/shared/call-lambda-sync.ts | 3 +-- packages/lambda/src/shared/call-lambda.ts | 2 +- packages/lambda/src/shared/get-function-version.ts | 1 + packages/lambda/src/test/integration/handlers.test.ts | 1 + .../src/test/integration/renders/old-version.test.ts | 1 + .../lambda/src/test/integration/simulate-lambda-render.ts | 1 + packages/lambda/src/test/integration/webhooks.test.ts | 3 +++ packages/lambda/src/test/unit/handlers.test.ts | 1 + packages/serverless/src/client.ts | 1 + packages/serverless/src/index.ts | 1 - 14 files changed, 16 insertions(+), 11 deletions(-) diff --git a/packages/lambda/src/api/get-compositions-on-lambda.ts b/packages/lambda/src/api/get-compositions-on-lambda.ts index 537331816cf..183e1ae000f 100644 --- a/packages/lambda/src/api/get-compositions-on-lambda.ts +++ b/packages/lambda/src/api/get-compositions-on-lambda.ts @@ -70,6 +70,7 @@ export const getCompositionsOnLambda = async ({ functionName, type: ServerlessRoutines.compositions, payload: { + type: ServerlessRoutines.compositions, chromiumOptions: chromiumOptions ?? {}, serveUrl, envVariables, diff --git a/packages/lambda/src/functions/helpers/calculate-price-from-bucket.ts b/packages/lambda/src/functions/helpers/calculate-price-from-bucket.ts index a973f8f8142..0483bcfce59 100644 --- a/packages/lambda/src/functions/helpers/calculate-price-from-bucket.ts +++ b/packages/lambda/src/functions/helpers/calculate-price-from-bucket.ts @@ -1,9 +1,6 @@ -import { - calculateChunkTimes, - type CloudProvider, - type ParsedTiming, -} from '@remotion/serverless'; +import {type CloudProvider, type ParsedTiming} from '@remotion/serverless'; import type {RenderMetadata} from '@remotion/serverless/client'; +import {calculateChunkTimes} from '@remotion/serverless/client'; import {estimatePrice} from '../../api/estimate-price'; import type {AwsRegion} from '../../regions'; diff --git a/packages/lambda/src/functions/helpers/create-post-render-data.ts b/packages/lambda/src/functions/helpers/create-post-render-data.ts index 36d93fb1dcf..61c138793bb 100644 --- a/packages/lambda/src/functions/helpers/create-post-render-data.ts +++ b/packages/lambda/src/functions/helpers/create-post-render-data.ts @@ -6,10 +6,10 @@ import type { } from '@remotion/serverless'; import { OVERHEAD_TIME_PER_LAMBDA, - calculateChunkTimes, getMostExpensiveChunks, } from '@remotion/serverless'; import type {RenderMetadata} from '@remotion/serverless/client'; +import {calculateChunkTimes} from '@remotion/serverless/client'; import {estimatePrice} from '../../api/estimate-price'; import type {AwsRegion} from '../../regions'; import {MAX_EPHEMERAL_STORAGE_IN_MB} from '../../shared/constants'; diff --git a/packages/lambda/src/functions/helpers/get-progress.ts b/packages/lambda/src/functions/helpers/get-progress.ts index 9f1ce6540da..a1ccbeb2e4f 100644 --- a/packages/lambda/src/functions/helpers/get-progress.ts +++ b/packages/lambda/src/functions/helpers/get-progress.ts @@ -6,8 +6,8 @@ import type { GenericRenderProgress, ProviderSpecifics, } from '@remotion/serverless'; -import {calculateChunkTimes} from '@remotion/serverless'; import { + calculateChunkTimes, getExpectedOutName, truthy, type CustomCredentials, diff --git a/packages/lambda/src/shared/call-lambda-sync.ts b/packages/lambda/src/shared/call-lambda-sync.ts index a6082375f2f..913908e9138 100644 --- a/packages/lambda/src/shared/call-lambda-sync.ts +++ b/packages/lambda/src/shared/call-lambda-sync.ts @@ -12,7 +12,6 @@ const callLambdaSyncWithoutRetry = async < Provider extends CloudProvider, >({ functionName, - type, payload, region, timeoutInTest, @@ -33,7 +32,7 @@ const callLambdaSyncWithoutRetry = async < try { return JSON.parse(decoded) as OrError[T]>; } catch { - throw new Error(`Invalid JSON (${type}): ${JSON.stringify(decoded)}`); + throw new Error(`Invalid JSON: ${JSON.stringify(decoded)}`); } }; diff --git a/packages/lambda/src/shared/call-lambda.ts b/packages/lambda/src/shared/call-lambda.ts index 1bb8855c938..c353e8d8b61 100644 --- a/packages/lambda/src/shared/call-lambda.ts +++ b/packages/lambda/src/shared/call-lambda.ts @@ -10,7 +10,7 @@ export type CallLambdaOptions< > = { functionName: string; type: T; - payload: Omit[T], 'type'>; + payload: ServerlessPayloads[T]; region: Provider['region']; timeoutInTest: number; }; diff --git a/packages/lambda/src/shared/get-function-version.ts b/packages/lambda/src/shared/get-function-version.ts index 1acfcc5bcf8..d6869366f4c 100644 --- a/packages/lambda/src/shared/get-function-version.ts +++ b/packages/lambda/src/shared/get-function-version.ts @@ -18,6 +18,7 @@ export const getFunctionVersion = async ({ functionName, payload: { logLevel, + type: ServerlessRoutines.info, }, region, type: ServerlessRoutines.info, diff --git a/packages/lambda/src/test/integration/handlers.test.ts b/packages/lambda/src/test/integration/handlers.test.ts index 5b2d1449396..929965cf07f 100644 --- a/packages/lambda/src/test/integration/handlers.test.ts +++ b/packages/lambda/src/test/integration/handlers.test.ts @@ -7,6 +7,7 @@ test('Call function locally', async () => { expect( await callLambdaSync({ payload: { + type: ServerlessRoutines.info, logLevel: 'info', }, type: ServerlessRoutines.info, diff --git a/packages/lambda/src/test/integration/renders/old-version.test.ts b/packages/lambda/src/test/integration/renders/old-version.test.ts index 5295444dfef..00de4ec526c 100644 --- a/packages/lambda/src/test/integration/renders/old-version.test.ts +++ b/packages/lambda/src/test/integration/renders/old-version.test.ts @@ -14,6 +14,7 @@ test('Should fail when using an incompatible version', async () => { const aha = await callLambdaSync({ type: ServerlessRoutines.launch, payload: { + type: ServerlessRoutines.launch, serveUrl: 'https://competent-mccarthy-56f7c9.netlify.app/', chromiumOptions: {}, codec: 'h264', diff --git a/packages/lambda/src/test/integration/simulate-lambda-render.ts b/packages/lambda/src/test/integration/simulate-lambda-render.ts index ac0b084212f..969d4abe629 100644 --- a/packages/lambda/src/test/integration/simulate-lambda-render.ts +++ b/packages/lambda/src/test/integration/simulate-lambda-render.ts @@ -16,6 +16,7 @@ const waitUntilDone = async (bucketName: string, renderId: string) => { const progress = await callLambdaSync({ type: ServerlessRoutines.status, payload: { + type: ServerlessRoutines.status, bucketName, renderId, version: VERSION, diff --git a/packages/lambda/src/test/integration/webhooks.test.ts b/packages/lambda/src/test/integration/webhooks.test.ts index 71641d7a015..8c772a1b143 100644 --- a/packages/lambda/src/test/integration/webhooks.test.ts +++ b/packages/lambda/src/test/integration/webhooks.test.ts @@ -60,6 +60,7 @@ describe('Webhooks', () => { const res = await callLambdaSync({ type: ServerlessRoutines.start, payload: { + type: ServerlessRoutines.start, serveUrl: `http://localhost:${port}`, chromiumOptions: {}, codec: 'h264', @@ -124,6 +125,7 @@ describe('Webhooks', () => { await callLambdaSync({ type: ServerlessRoutines.status, payload: { + type: ServerlessRoutines.status, bucketName: parsed.bucketName, renderId: parsed.renderId, version: VERSION, @@ -178,6 +180,7 @@ describe('Webhooks', () => { region: 'us-east-1', type: ServerlessRoutines.launch, payload: { + type: ServerlessRoutines.launch, offthreadVideoCacheSizeInBytes: null, serveUrl: `http://localhost:${port}`, chromiumOptions: {}, diff --git a/packages/lambda/src/test/unit/handlers.test.ts b/packages/lambda/src/test/unit/handlers.test.ts index a3a4bb0d191..3fe52060450 100644 --- a/packages/lambda/src/test/unit/handlers.test.ts +++ b/packages/lambda/src/test/unit/handlers.test.ts @@ -7,6 +7,7 @@ test('Info handler should return version', async () => { type: ServerlessRoutines.info, payload: { logLevel: 'info', + type: ServerlessRoutines.info, }, functionName: 'remotion-dev-lambda', region: 'us-east-1', diff --git a/packages/serverless/src/client.ts b/packages/serverless/src/client.ts index 12b3f0fc22e..b57eca8025c 100644 --- a/packages/serverless/src/client.ts +++ b/packages/serverless/src/client.ts @@ -44,6 +44,7 @@ export { isErrInsufficientResourcesErr, } from './error-category'; +export {calculateChunkTimes} from './calculate-chunk-times'; export {getExpectedOutName} from './expected-out-name'; export {FileNameAndSize, GetFolderFiles} from './get-files-in-folder'; export {inputPropsKey, resolvedPropsKey} from './input-props-keys'; diff --git a/packages/serverless/src/index.ts b/packages/serverless/src/index.ts index 03bb79454a4..e5e03580987 100644 --- a/packages/serverless/src/index.ts +++ b/packages/serverless/src/index.ts @@ -1,6 +1,5 @@ export {compositionsHandler} from './compositions'; -export {calculateChunkTimes} from './calculate-chunk-times'; export {PostRenderData} from './constants'; export {getCredentialsFromOutName} from './expected-out-name'; export { From 3ee7f79b88195b4b79ba032063777e6a9069a746 Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Sun, 5 Jan 2025 21:38:33 +0100 Subject: [PATCH 05/14] getting there --- .../src/api/get-compositions-on-lambda.ts | 3 +- .../lambda/src/api/get-render-progress.ts | 16 ++-- .../lambda/src/api/render-media-on-lambda.ts | 8 +- .../lambda/src/api/render-still-on-lambda.ts | 69 ++++++++------- .../src/functions/aws-implementation.ts | 6 ++ .../src/functions/helpers/stream-renderer.ts | 20 ++--- packages/lambda/src/functions/index.ts | 9 +- packages/lambda/src/functions/start.ts | 3 +- .../src/shared/__mocks__/aws-clients.ts | 55 ------------ .../lambda/src/shared/call-lambda-async.ts | 7 +- .../src/shared/call-lambda-streaming.ts | 10 +-- .../lambda/src/shared/call-lambda-sync.ts | 22 ++--- packages/lambda/src/shared/call-lambda.ts | 16 ---- .../lambda/src/shared/get-function-version.ts | 4 +- packages/lambda/src/shared/return-values.ts | 21 ----- .../src/test/integration/handlers.test.ts | 4 +- .../integration/renders/old-version.test.ts | 4 +- .../integration/simulate-lambda-render.ts | 19 ++-- .../src/test/integration/webhooks.test.ts | 8 +- .../lambda/src/test/mock-implementation.ts | 8 ++ packages/lambda/src/test/mocks/aws-clients.ts | 88 +++++++++++++++++++ packages/lambda/src/test/setup.ts | 3 - .../lambda/src/test/unit/handlers.test.ts | 4 +- packages/serverless/src/index.ts | 7 +- .../serverless/src/provider-implementation.ts | 47 +++++++++- packages/serverless/src/return-values.ts | 45 ++++++++++ packages/serverless/src/types.ts | 13 +++ 27 files changed, 314 insertions(+), 205 deletions(-) delete mode 100644 packages/lambda/src/shared/__mocks__/aws-clients.ts delete mode 100644 packages/lambda/src/shared/call-lambda.ts delete mode 100644 packages/lambda/src/shared/return-values.ts create mode 100644 packages/lambda/src/test/mocks/aws-clients.ts create mode 100644 packages/serverless/src/return-values.ts diff --git a/packages/lambda/src/api/get-compositions-on-lambda.ts b/packages/lambda/src/api/get-compositions-on-lambda.ts index 183e1ae000f..8a79104cd6f 100644 --- a/packages/lambda/src/api/get-compositions-on-lambda.ts +++ b/packages/lambda/src/api/get-compositions-on-lambda.ts @@ -10,7 +10,6 @@ import type {VideoConfig} from 'remotion/no-react'; import {VERSION} from 'remotion/version'; import type {AwsRegion} from '../client'; import {awsImplementation} from '../functions/aws-implementation'; -import {callLambdaSync} from '../shared/call-lambda-sync'; export type GetCompositionsOnLambdaInput = { chromiumOptions?: ChromiumOptions; @@ -66,7 +65,7 @@ export const getCompositionsOnLambda = async ({ }); try { - const res = await callLambdaSync({ + const res = await awsImplementation.callFunctionSync({ functionName, type: ServerlessRoutines.compositions, payload: { diff --git a/packages/lambda/src/api/get-render-progress.ts b/packages/lambda/src/api/get-render-progress.ts index c8d4d3de96a..23d8e5b8ad0 100644 --- a/packages/lambda/src/api/get-render-progress.ts +++ b/packages/lambda/src/api/get-render-progress.ts @@ -8,7 +8,6 @@ import { import {getProgress} from '../functions/helpers/get-progress'; import {parseFunctionName} from '../functions/helpers/parse-function-name'; import type {AwsRegion} from '../regions'; -import {callLambdaSync} from '../shared/call-lambda-sync'; import type {RenderProgress} from '../shared/constants'; import {getRenderProgressPayload} from './make-lambda-payload'; @@ -56,12 +55,13 @@ export const getRenderProgress = async ( }); } - const result = await callLambdaSync({ - functionName: input.functionName, - type: ServerlessRoutines.status, - payload: getRenderProgressPayload(input), - region: input.region, - timeoutInTest: 120000, - }); + const result = + await awsImplementation.callFunctionSync({ + functionName: input.functionName, + type: ServerlessRoutines.status, + payload: getRenderProgressPayload(input), + region: input.region, + timeoutInTest: 120000, + }); return result; }; diff --git a/packages/lambda/src/api/render-media-on-lambda.ts b/packages/lambda/src/api/render-media-on-lambda.ts index daec6f09f3e..9dd7fcbbd84 100644 --- a/packages/lambda/src/api/render-media-on-lambda.ts +++ b/packages/lambda/src/api/render-media-on-lambda.ts @@ -17,9 +17,11 @@ import type { WebhookOption, } from '@remotion/serverless/client'; import {ServerlessRoutines} from '@remotion/serverless/client'; -import type {AwsProvider} from '../functions/aws-implementation'; +import { + awsImplementation, + type AwsProvider, +} from '../functions/aws-implementation'; import type {AwsRegion} from '../regions'; -import {callLambdaSync} from '../shared/call-lambda-sync'; import { getCloudwatchMethodUrl, getCloudwatchRendererUrl, @@ -88,7 +90,7 @@ export const internalRenderMediaOnLambdaRaw = async ( const {functionName, region, rendererFunctionName} = input; try { - const res = await callLambdaSync({ + const res = await awsImplementation.callFunctionSync({ functionName, type: ServerlessRoutines.start, payload: await makeLambdaRenderMediaPayload(input), diff --git a/packages/lambda/src/api/render-still-on-lambda.ts b/packages/lambda/src/api/render-still-on-lambda.ts index 7aa7caabd9c..efdea652b04 100644 --- a/packages/lambda/src/api/render-still-on-lambda.ts +++ b/packages/lambda/src/api/render-still-on-lambda.ts @@ -6,7 +6,6 @@ import type { import type {BrowserSafeApis} from '@remotion/renderer/client'; import type {DownloadBehavior} from '@remotion/serverless/client'; import {ServerlessRoutines} from '@remotion/serverless/client'; -import {callLambdaWithStreaming} from '../shared/call-lambda-streaming'; import {wrapWithErrorHandling} from '@remotion/renderer/error-handling'; import type { @@ -14,7 +13,10 @@ import type { RenderStillLambdaResponsePayload, } from '@remotion/serverless'; import type {OutNameInput, Privacy} from '@remotion/serverless/client'; -import type {AwsProvider} from '../functions/aws-implementation'; +import { + awsImplementation, + type AwsProvider, +} from '../functions/aws-implementation'; import type {AwsRegion} from '../regions'; import type {CostsInfo} from '../shared/constants'; import {DEFAULT_MAX_RETRIES} from '../shared/constants'; @@ -87,40 +89,41 @@ const internalRenderStillOnLambda = async ( const res = await new Promise< RenderStillLambdaResponsePayload >((resolve, reject) => { - callLambdaWithStreaming({ - functionName, - type: ServerlessRoutines.still, - payload, - region, - receivedStreamingPayload: ({message}) => { - if (message.type === 'render-id-determined') { - onInit?.({ - renderId: message.payload.renderId, - cloudWatchLogs: getCloudwatchMethodUrl({ - functionName, - method: ServerlessRoutines.still, - region, - rendererFunctionName: null, + awsImplementation + .callFunctionStreaming({ + functionName, + type: ServerlessRoutines.still, + payload, + region, + receivedStreamingPayload: ({message}) => { + if (message.type === 'render-id-determined') { + onInit?.({ renderId: message.payload.renderId, - }), - lambdaInsightsUrl: getLambdaInsightsUrl({ - functionName, - region, - }), - }); - } + cloudWatchLogs: getCloudwatchMethodUrl({ + functionName, + method: ServerlessRoutines.still, + region, + rendererFunctionName: null, + renderId: message.payload.renderId, + }), + lambdaInsightsUrl: getLambdaInsightsUrl({ + functionName, + region, + }), + }); + } - if (message.type === 'error-occurred') { - reject(new Error(message.payload.error)); - } + if (message.type === 'error-occurred') { + reject(new Error(message.payload.error)); + } - if (message.type === 'still-rendered') { - resolve(message.payload); - } - }, - timeoutInTest: 120000, - retriesRemaining: input.maxRetries, - }) + if (message.type === 'still-rendered') { + resolve(message.payload); + } + }, + timeoutInTest: 120000, + retriesRemaining: input.maxRetries, + }) .then(() => { reject(new Error('Expected response to be streamed')); }) diff --git a/packages/lambda/src/functions/aws-implementation.ts b/packages/lambda/src/functions/aws-implementation.ts index 9df16aff02f..6f29dc5055e 100644 --- a/packages/lambda/src/functions/aws-implementation.ts +++ b/packages/lambda/src/functions/aws-implementation.ts @@ -10,6 +10,9 @@ import {lambdaLsImplementation} from '../io/list-objects'; import {lambdaReadFileImplementation} from '../io/read-file'; import {lambdaWriteFileImplementation} from '../io/write-file'; import type {AwsRegion} from '../regions'; +import {callFunctionAsyncImplementation} from '../shared/call-lambda-async'; +import {callFunctionWithStreamingImplementation} from '../shared/call-lambda-streaming'; +import {callFunctionSyncImplementation} from '../shared/call-lambda-sync'; import {convertToServeUrlImplementation} from '../shared/convert-to-serve-url'; import {applyLifeCyleOperation} from '../shared/lifecycle-rules'; import {randomHashImplementation} from '../shared/random-hash'; @@ -90,4 +93,7 @@ export const awsImplementation: ProviderSpecifics = { getFolderFiles, makeArtifactWithDetails: makeAwsArtifact, validateDeleteAfter, + callFunctionAsync: callFunctionAsyncImplementation, + callFunctionStreaming: callFunctionWithStreamingImplementation, + callFunctionSync: callFunctionSyncImplementation, }; diff --git a/packages/lambda/src/functions/helpers/stream-renderer.ts b/packages/lambda/src/functions/helpers/stream-renderer.ts index 883890838d1..794155f8a98 100644 --- a/packages/lambda/src/functions/helpers/stream-renderer.ts +++ b/packages/lambda/src/functions/helpers/stream-renderer.ts @@ -13,7 +13,6 @@ import type {ServerlessPayload} from '@remotion/serverless/client'; import {ServerlessRoutines} from '@remotion/serverless/client'; import {writeFileSync} from 'fs'; import {join} from 'path'; -import {callLambdaWithStreaming} from '../../shared/call-lambda-streaming'; type StreamRendererResponse = | { @@ -161,15 +160,16 @@ const streamRenderer = ({ throw new Error(`Unknown message type ${message.type}`); }; - callLambdaWithStreaming({ - functionName, - payload, - retriesRemaining: 1, - region: providerSpecifics.getCurrentRegionInFunction(), - timeoutInTest: 12000, - type: ServerlessRoutines.renderer, - receivedStreamingPayload, - }) + providerSpecifics + .callFunctionStreaming({ + functionName, + payload, + retriesRemaining: 1, + region: providerSpecifics.getCurrentRegionInFunction(), + timeoutInTest: 12000, + type: ServerlessRoutines.renderer, + receivedStreamingPayload, + }) .then(() => { resolve({ type: 'success', diff --git a/packages/lambda/src/functions/index.ts b/packages/lambda/src/functions/index.ts index c3f4f6eb93b..86fd8dd5bb1 100644 --- a/packages/lambda/src/functions/index.ts +++ b/packages/lambda/src/functions/index.ts @@ -1,6 +1,7 @@ import {RenderInternals} from '@remotion/renderer'; import type { CloudProvider, + OrError, ProviderSpecifics, ResponseStream, ResponseStreamWriter, @@ -329,14 +330,6 @@ const innerHandler = async ({ throw new Error(COMMAND_NOT_FOUND); }; -export type OrError = - | T - | { - type: 'error'; - message: string; - stack: string; - }; - export const innerRoutine = async ( params: ServerlessPayload, responseStream: ResponseStream, diff --git a/packages/lambda/src/functions/start.ts b/packages/lambda/src/functions/start.ts index bba6296199b..053e72ac6c4 100644 --- a/packages/lambda/src/functions/start.ts +++ b/packages/lambda/src/functions/start.ts @@ -11,7 +11,6 @@ import { } from '@remotion/serverless/client'; import {VERSION} from 'remotion/version'; import type {AwsRegion} from '../regions'; -import {callLambdaAsync} from '../shared/call-lambda-async'; type Options = { expectedBucketOwner: string; @@ -124,7 +123,7 @@ export const startHandler = async ( metadata: params.metadata, }; - await callLambdaAsync({ + await providerSpecifics.callFunctionAsync({ functionName: process.env.AWS_LAMBDA_FUNCTION_NAME as string, type: ServerlessRoutines.launch, payload, diff --git a/packages/lambda/src/shared/__mocks__/aws-clients.ts b/packages/lambda/src/shared/__mocks__/aws-clients.ts deleted file mode 100644 index b12e3a9248d..00000000000 --- a/packages/lambda/src/shared/__mocks__/aws-clients.ts +++ /dev/null @@ -1,55 +0,0 @@ -import type {LambdaClient} from '@aws-sdk/client-lambda'; - -import {ResponseStream} from '@remotion/serverless'; -import type {getLambdaClient as original} from '../../shared/aws-clients'; -import {mockImplementation} from '../../test/mock-implementation'; - -export const getLambdaClient: typeof original = (_region, timeoutInTest) => { - return { - config: { - requestHandler: {}, - apiVersion: 'fake', - }, - destroy: () => undefined, - middlewareStack: undefined, - send: async (params: { - input: { - FunctionName: undefined; - Payload: string; - InvocationType: 'Event' | 'RequestResponse' | undefined; - }; - }) => { - const payload = JSON.parse(params.input.Payload); - - const {innerRoutine} = await import('../../functions/index'); - - const responseStream = new ResponseStream(); - const prom = innerRoutine( - payload, - responseStream, - { - invokedFunctionArn: 'arn:fake', - getRemainingTimeInMillis: () => timeoutInTest ?? 120000, - awsRequestId: 'fake', - }, - mockImplementation, - ); - if ( - params.input.InvocationType === 'RequestResponse' || - params.input.InvocationType === 'Event' - ) { - await prom; - return {Payload: responseStream.getBufferedData()}; - } - - prom.then(() => { - responseStream._finish(); - responseStream.end(); - }); - // When streaming, we should not consume the response - return { - EventStream: responseStream, - }; - }, - } as unknown as LambdaClient; -}; diff --git a/packages/lambda/src/shared/call-lambda-async.ts b/packages/lambda/src/shared/call-lambda-async.ts index f42094b172b..ea5feef81cc 100644 --- a/packages/lambda/src/shared/call-lambda-async.ts +++ b/packages/lambda/src/shared/call-lambda-async.ts @@ -1,11 +1,10 @@ import {InvokeCommand} from '@aws-sdk/client-lambda'; -import type {CloudProvider} from '@remotion/serverless'; +import type {CallFunctionOptions, CloudProvider} from '@remotion/serverless'; import type {ServerlessRoutines} from '@remotion/serverless/client'; import type {AwsRegion} from '../regions'; import {getLambdaClient} from './aws-clients'; -import type {CallLambdaOptions} from './call-lambda'; -export const callLambdaAsync = async < +export const callFunctionAsyncImplementation = async < T extends ServerlessRoutines, Provider extends CloudProvider, >({ @@ -13,7 +12,7 @@ export const callLambdaAsync = async < payload, region, timeoutInTest, -}: CallLambdaOptions): Promise => { +}: CallFunctionOptions): Promise => { const stringifiedPayload = JSON.stringify(payload); if (stringifiedPayload.length > 256 * 1024) { throw new Error( diff --git a/packages/lambda/src/shared/call-lambda-streaming.ts b/packages/lambda/src/shared/call-lambda-streaming.ts index 8182cb35e90..6599201e434 100644 --- a/packages/lambda/src/shared/call-lambda-streaming.ts +++ b/packages/lambda/src/shared/call-lambda-streaming.ts @@ -4,6 +4,7 @@ import { type InvokeWithResponseStreamResponseEvent, } from '@aws-sdk/client-lambda'; import type { + CallFunctionOptions, CloudProvider, OnMessage, StreamingMessage, @@ -19,7 +20,6 @@ import { import {makeStreamer} from '@remotion/streaming'; import type {AwsRegion} from '../regions'; import {getLambdaClient} from './aws-clients'; -import type {CallLambdaOptions} from './call-lambda'; const STREAM_STALL_TIMEOUT = 30000; const LAMBDA_STREAM_STALL = `AWS did not invoke Lambda in ${STREAM_STALL_TIMEOUT}ms`; @@ -85,7 +85,7 @@ const callLambdaWithStreamingWithoutRetry = async < region, timeoutInTest, receivedStreamingPayload, -}: CallLambdaOptions & { +}: CallFunctionOptions & { receivedStreamingPayload: OnMessage; }): Promise => { const res = await invokeStreamOrTimeout({ @@ -173,11 +173,11 @@ const callLambdaWithStreamingWithoutRetry = async < clear(); }; -export const callLambdaWithStreaming = async < +export const callFunctionWithStreamingImplementation = async < Provider extends CloudProvider, T extends ServerlessRoutines, >( - options: CallLambdaOptions & { + options: CallFunctionOptions & { receivedStreamingPayload: OnMessage; retriesRemaining: number; }, @@ -210,7 +210,7 @@ export const callLambdaWithStreaming = async < } console.error(err); - return callLambdaWithStreaming({ + return callFunctionWithStreamingImplementation({ ...options, retriesRemaining: options.retriesRemaining - 1, }); diff --git a/packages/lambda/src/shared/call-lambda-sync.ts b/packages/lambda/src/shared/call-lambda-sync.ts index 913908e9138..37bd33c832a 100644 --- a/packages/lambda/src/shared/call-lambda-sync.ts +++ b/packages/lambda/src/shared/call-lambda-sync.ts @@ -1,11 +1,13 @@ import {InvokeCommand} from '@aws-sdk/client-lambda'; -import type {CloudProvider} from '@remotion/serverless'; +import type { + CallFunctionOptions, + CloudProvider, + OrError, + ServerlessReturnValues, +} from '@remotion/serverless'; import type {ServerlessRoutines} from '@remotion/serverless/client'; -import type {OrError} from '../functions'; import type {AwsRegion} from '../regions'; import {getLambdaClient} from './aws-clients'; -import type {CallLambdaOptions} from './call-lambda'; -import type {LambdaReturnValues} from './return-values'; const callLambdaSyncWithoutRetry = async < T extends ServerlessRoutines, @@ -15,8 +17,8 @@ const callLambdaSyncWithoutRetry = async < payload, region, timeoutInTest, -}: CallLambdaOptions): Promise< - OrError[T]> +}: CallFunctionOptions): Promise< + OrError[T]> > => { const Payload = JSON.stringify(payload); const res = await getLambdaClient(region as AwsRegion, timeoutInTest).send( @@ -30,18 +32,18 @@ const callLambdaSyncWithoutRetry = async < const decoded = new TextDecoder('utf-8').decode(res.Payload); try { - return JSON.parse(decoded) as OrError[T]>; + return JSON.parse(decoded) as OrError[T]>; } catch { throw new Error(`Invalid JSON: ${JSON.stringify(decoded)}`); } }; -export const callLambdaSync = async < +export const callFunctionSyncImplementation = async < Provider extends CloudProvider, T extends ServerlessRoutines, >( - options: CallLambdaOptions, -): Promise[T]> => { + options: CallFunctionOptions, +): Promise[T]> => { const res = await callLambdaSyncWithoutRetry(options); if (res.type === 'error') { const err = new Error(res.message); diff --git a/packages/lambda/src/shared/call-lambda.ts b/packages/lambda/src/shared/call-lambda.ts deleted file mode 100644 index c353e8d8b61..00000000000 --- a/packages/lambda/src/shared/call-lambda.ts +++ /dev/null @@ -1,16 +0,0 @@ -import type {CloudProvider} from '@remotion/serverless'; -import type { - ServerlessPayloads, - ServerlessRoutines, -} from '@remotion/serverless/client'; - -export type CallLambdaOptions< - T extends ServerlessRoutines, - Provider extends CloudProvider, -> = { - functionName: string; - type: T; - payload: ServerlessPayloads[T]; - region: Provider['region']; - timeoutInTest: number; -}; diff --git a/packages/lambda/src/shared/get-function-version.ts b/packages/lambda/src/shared/get-function-version.ts index d6869366f4c..d19b13c7bb2 100644 --- a/packages/lambda/src/shared/get-function-version.ts +++ b/packages/lambda/src/shared/get-function-version.ts @@ -1,7 +1,7 @@ import type {LogLevel} from '@remotion/renderer'; import {ServerlessRoutines} from '@remotion/serverless/client'; +import {awsImplementation} from '../functions/aws-implementation'; import type {AwsRegion} from '../regions'; -import {callLambdaSync} from './call-lambda-sync'; import {COMMAND_NOT_FOUND} from './constants'; export const getFunctionVersion = async ({ @@ -14,7 +14,7 @@ export const getFunctionVersion = async ({ logLevel: LogLevel; }): Promise => { try { - const result = await callLambdaSync({ + const result = await awsImplementation.callFunctionSync({ functionName, payload: { logLevel, diff --git a/packages/lambda/src/shared/return-values.ts b/packages/lambda/src/shared/return-values.ts deleted file mode 100644 index 64fa0386ad5..00000000000 --- a/packages/lambda/src/shared/return-values.ts +++ /dev/null @@ -1,21 +0,0 @@ -import type { - CloudProvider, - compositionsHandler, - infoHandler, -} from '@remotion/serverless'; -import type {ServerlessRoutines} from '@remotion/serverless/client'; -import type {launchHandler} from '../functions/launch'; -import type {progressHandler} from '../functions/progress'; -import type {rendererHandler} from '../functions/renderer'; -import type {startHandler} from '../functions/start'; -import type {stillHandler} from '../functions/still'; - -export interface LambdaReturnValues { - [ServerlessRoutines.start]: ReturnType; - [ServerlessRoutines.launch]: ReturnType; - [ServerlessRoutines.renderer]: ReturnType; - [ServerlessRoutines.status]: ReturnType>; - [ServerlessRoutines.info]: ReturnType>; - [ServerlessRoutines.still]: ReturnType; - [ServerlessRoutines.compositions]: ReturnType; -} diff --git a/packages/lambda/src/test/integration/handlers.test.ts b/packages/lambda/src/test/integration/handlers.test.ts index 929965cf07f..42316151e5d 100644 --- a/packages/lambda/src/test/integration/handlers.test.ts +++ b/packages/lambda/src/test/integration/handlers.test.ts @@ -1,11 +1,11 @@ import {ServerlessRoutines} from '@remotion/serverless/client'; import {VERSION} from 'remotion/version'; import {expect, test} from 'vitest'; -import {callLambdaSync} from '../../shared/call-lambda-sync'; +import {mockImplementation} from '../mock-implementation'; test('Call function locally', async () => { expect( - await callLambdaSync({ + await mockImplementation.callFunctionSync({ payload: { type: ServerlessRoutines.info, logLevel: 'info', diff --git a/packages/lambda/src/test/integration/renders/old-version.test.ts b/packages/lambda/src/test/integration/renders/old-version.test.ts index 00de4ec526c..13c4ced49b5 100644 --- a/packages/lambda/src/test/integration/renders/old-version.test.ts +++ b/packages/lambda/src/test/integration/renders/old-version.test.ts @@ -1,7 +1,7 @@ import {RenderInternals} from '@remotion/renderer'; import {ServerlessRoutines} from '@remotion/serverless/client'; import {afterAll, expect, test} from 'vitest'; -import {callLambdaSync} from '../../../shared/call-lambda-sync'; +import {mockImplementation} from '../../mock-implementation'; afterAll(async () => { await RenderInternals.killAllBrowsers(); @@ -11,7 +11,7 @@ test('Should fail when using an incompatible version', async () => { process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE = '2048'; try { - const aha = await callLambdaSync({ + const aha = await mockImplementation.callFunctionSync({ type: ServerlessRoutines.launch, payload: { type: ServerlessRoutines.launch, diff --git a/packages/lambda/src/test/integration/simulate-lambda-render.ts b/packages/lambda/src/test/integration/simulate-lambda-render.ts index 969d4abe629..e9f3121ff52 100644 --- a/packages/lambda/src/test/integration/simulate-lambda-render.ts +++ b/packages/lambda/src/test/integration/simulate-lambda-render.ts @@ -5,15 +5,13 @@ import {VERSION} from 'remotion/version'; import {makeLambdaRenderMediaPayload} from '../../api/make-lambda-payload'; import type {RenderMediaOnLambdaInput} from '../../api/render-media-on-lambda'; import {renderMediaOnLambdaOptionalToRequired} from '../../api/render-media-on-lambda'; -import type {AwsProvider} from '../../functions/aws-implementation'; -import {callLambdaSync} from '../../shared/call-lambda-sync'; import {mockImplementation} from '../mock-implementation'; const functionName = 'remotion-dev-render'; const waitUntilDone = async (bucketName: string, renderId: string) => { while (true) { - const progress = await callLambdaSync({ + const progress = await mockImplementation.callFunctionSync({ type: ServerlessRoutines.status, payload: { type: ServerlessRoutines.status, @@ -69,13 +67,14 @@ export const simulateLambdaRender = async ( }), ); - const res = await callLambdaSync({ - type: ServerlessRoutines.start, - payload, - functionName: 'remotion-dev-lambda', - region: 'eu-central-1', - timeoutInTest: 120000, - }); + const res = + await mockImplementation.callFunctionSync({ + type: ServerlessRoutines.start, + payload, + functionName: 'remotion-dev-lambda', + region: 'eu-central-1', + timeoutInTest: 120000, + }); const progress = await waitUntilDone(res.bucketName, res.renderId); diff --git a/packages/lambda/src/test/integration/webhooks.test.ts b/packages/lambda/src/test/integration/webhooks.test.ts index 8c772a1b143..b4dee223184 100644 --- a/packages/lambda/src/test/integration/webhooks.test.ts +++ b/packages/lambda/src/test/integration/webhooks.test.ts @@ -4,7 +4,7 @@ import path from 'path'; import {VERSION} from 'remotion/version'; import {beforeAll, beforeEach, describe, expect, test, vi} from 'vitest'; import {mockableHttpClients} from '../../functions/http-client'; -import {callLambdaSync} from '../../shared/call-lambda-sync'; +import {mockImplementation} from '../mock-implementation'; const originalFetch = mockableHttpClients.http; beforeEach(() => { @@ -57,7 +57,7 @@ describe('Webhooks', () => { forceIPv4: false, }); - const res = await callLambdaSync({ + const res = await mockImplementation.callFunctionSync({ type: ServerlessRoutines.start, payload: { type: ServerlessRoutines.start, @@ -122,7 +122,7 @@ describe('Webhooks', () => { }); const parsed = res; - await callLambdaSync({ + await mockImplementation.callFunctionSync({ type: ServerlessRoutines.status, payload: { type: ServerlessRoutines.status, @@ -175,7 +175,7 @@ describe('Webhooks', () => { forceIPv4: false, }); - await callLambdaSync({ + await mockImplementation.callFunctionSync({ functionName: 'remotion-dev-lambda', region: 'us-east-1', type: ServerlessRoutines.launch, diff --git a/packages/lambda/src/test/mock-implementation.ts b/packages/lambda/src/test/mock-implementation.ts index 059f9fbaa53..c5785182333 100644 --- a/packages/lambda/src/test/mock-implementation.ts +++ b/packages/lambda/src/test/mock-implementation.ts @@ -2,6 +2,11 @@ import type {ProviderSpecifics} from '@remotion/serverless'; import {Readable} from 'stream'; import type {AwsProvider} from '../functions/aws-implementation'; import {convertToServeUrlImplementation} from '../shared/convert-to-serve-url'; +import { + getMockCallFunctionAsync, + getMockCallFunctionStreaming, + getMockCallFunctionSync, +} from './mocks/aws-clients'; import { addMockBucket, getMockBuckets, @@ -120,4 +125,7 @@ export const mockImplementation: ProviderSpecifics = { s3Key: 'key', }), validateDeleteAfter: () => {}, + callFunctionAsync: getMockCallFunctionAsync, + callFunctionStreaming: getMockCallFunctionStreaming, + callFunctionSync: getMockCallFunctionSync, }; diff --git a/packages/lambda/src/test/mocks/aws-clients.ts b/packages/lambda/src/test/mocks/aws-clients.ts new file mode 100644 index 00000000000..4c8596b0045 --- /dev/null +++ b/packages/lambda/src/test/mocks/aws-clients.ts @@ -0,0 +1,88 @@ +import type { + CallFunctionAsync, + CallFunctionOptions, + CallFunctionStreaming, + CallFunctionSync, + OnMessage, + ServerlessReturnValues, + ServerlessRoutines, +} from '@remotion/serverless'; +import {ResponseStream} from '@remotion/serverless'; +import type {AwsProvider} from '../../functions/aws-implementation'; +import {mockImplementation} from '../mock-implementation'; + +export const getMockCallFunctionStreaming: CallFunctionStreaming< + AwsProvider +> = async ( + params: CallFunctionOptions & { + receivedStreamingPayload: OnMessage; + retriesRemaining: number; + }, +) => { + const {innerRoutine} = await import('../../functions/index'); + + const responseStream = new ResponseStream(); + await innerRoutine( + params.payload, + responseStream, + { + invokedFunctionArn: 'arn:fake', + getRemainingTimeInMillis: () => params.timeoutInTest ?? 120000, + awsRequestId: 'fake', + }, + mockImplementation, + ); + + responseStream._finish(); + responseStream.end(); +}; + +export const getMockCallFunctionAsync: CallFunctionAsync = async < + T extends ServerlessRoutines, +>( + params: CallFunctionOptions, +) => { + const {innerRoutine} = await import('../../functions/index'); + + const responseStream = new ResponseStream(); + await innerRoutine( + params.payload, + responseStream, + { + invokedFunctionArn: 'arn:fake', + getRemainingTimeInMillis: () => params.timeoutInTest ?? 120000, + awsRequestId: 'fake', + }, + mockImplementation, + ); + + responseStream._finish(); + responseStream.end(); +}; + +export const getMockCallFunctionSync: CallFunctionSync = async < + T extends ServerlessRoutines, +>( + params: CallFunctionOptions, +): Promise[T]> => { + const {innerRoutine} = await import('../../functions/index'); + + const responseStream = new ResponseStream(); + await innerRoutine( + params.payload, + responseStream, + { + invokedFunctionArn: 'arn:fake', + getRemainingTimeInMillis: () => params.timeoutInTest ?? 120000, + awsRequestId: 'fake', + }, + mockImplementation, + ); + + responseStream._finish(); + responseStream.end(); + + return JSON.parse( + new TextDecoder().decode(responseStream.getBufferedData()), + ) as ServerlessReturnValues[T]; +}; diff --git a/packages/lambda/src/test/setup.ts b/packages/lambda/src/test/setup.ts index 3fb988c311c..33806afbf16 100644 --- a/packages/lambda/src/test/setup.ts +++ b/packages/lambda/src/test/setup.ts @@ -18,9 +18,6 @@ vi.mock('../shared/bundle-site', () => vi.mock('../shared/get-account-id', () => vi.importActual('../shared/__mocks__/get-account-id'), ); -vi.mock('../shared/aws-clients', () => - vi.importActual('../shared/__mocks__/aws-clients'), -); vi.mock('../shared/read-dir', () => vi.importActual('../shared/__mocks__/read-dir'), ); diff --git a/packages/lambda/src/test/unit/handlers.test.ts b/packages/lambda/src/test/unit/handlers.test.ts index 3fe52060450..2aa6b520110 100644 --- a/packages/lambda/src/test/unit/handlers.test.ts +++ b/packages/lambda/src/test/unit/handlers.test.ts @@ -1,9 +1,9 @@ import {ServerlessRoutines} from '@remotion/serverless/client'; import {expect, test} from 'vitest'; -import {callLambdaSync} from '../../shared/call-lambda-sync'; +import {mockImplementation} from '../mock-implementation'; test('Info handler should return version', async () => { - const response = await callLambdaSync({ + const response = await mockImplementation.callFunctionSync({ type: ServerlessRoutines.info, payload: { logLevel: 'info', diff --git a/packages/serverless/src/index.ts b/packages/serverless/src/index.ts index e5e03580987..e15bbbc1221 100644 --- a/packages/serverless/src/index.ts +++ b/packages/serverless/src/index.ts @@ -1,6 +1,6 @@ export {compositionsHandler} from './compositions'; -export {PostRenderData} from './constants'; +export {PostRenderData, ServerlessRoutines} from './constants'; export {getCredentialsFromOutName} from './expected-out-name'; export { forgetBrowserEventLoop, @@ -19,11 +19,16 @@ export { makeOverallRenderProgress, } from './overall-render-progress'; export { + BucketWithLocation, + CallFunctionAsync, + CallFunctionStreaming, + CallFunctionSync, MakeArtifactWithDetails, ProviderSpecifics, WriteFileInput, } from './provider-implementation'; export type {CleanupInfo, GenericRenderProgress} from './render-progress'; +export {OrError, ServerlessReturnValues} from './return-values'; export {deserializeArtifact, serializeArtifact} from './serialize-artifact'; export {ResponseStream} from './streaming/response-stream'; export {ResponseStreamWriter, streamWriter} from './streaming/stream-writer'; diff --git a/packages/serverless/src/provider-implementation.ts b/packages/serverless/src/provider-implementation.ts index b98094bea36..735451fafdb 100644 --- a/packages/serverless/src/provider-implementation.ts +++ b/packages/serverless/src/provider-implementation.ts @@ -1,8 +1,19 @@ import type {EmittedArtifact} from '@remotion/renderer'; import type {Readable} from 'stream'; -import type {CustomCredentials, DownloadBehavior, Privacy} from './constants'; +import type { + CustomCredentials, + DownloadBehavior, + Privacy, + ServerlessRoutines, +} from './constants'; import type {GetFolderFiles} from './get-files-in-folder'; -import type {CloudProvider, ReceivedArtifact} from './types'; +import type {ServerlessReturnValues} from './return-values'; +import type {OnMessage} from './streaming/streaming'; +import type { + CallFunctionOptions, + CloudProvider, + ReceivedArtifact, +} from './types'; export type BucketWithLocation = { name: string; @@ -118,6 +129,35 @@ export type MakeArtifactWithDetails = (params: { artifact: EmittedArtifact; }) => ReceivedArtifact; +export type CallFunctionAsync = < + T extends ServerlessRoutines, +>({ + functionName, + payload, + region, + timeoutInTest, +}: CallFunctionOptions) => Promise; + +export type CallFunctionStreaming = < + T extends ServerlessRoutines, +>( + options: CallFunctionOptions & { + receivedStreamingPayload: OnMessage; + retriesRemaining: number; + }, +) => Promise; + +export type CallFunctionSync = < + T extends ServerlessRoutines, +>({ + functionName, + payload, + region, + timeoutInTest, +}: CallFunctionOptions) => Promise< + ServerlessReturnValues[T] +>; + export type ProviderSpecifics = { getChromiumPath: () => string | null; getCurrentRegionInFunction: () => Provider['region']; @@ -136,4 +176,7 @@ export type ProviderSpecifics = { getFolderFiles: GetFolderFiles; makeArtifactWithDetails: MakeArtifactWithDetails; validateDeleteAfter: (lifeCycleValue: unknown) => void; + callFunctionAsync: CallFunctionAsync; + callFunctionStreaming: CallFunctionStreaming; + callFunctionSync: CallFunctionSync; }; diff --git a/packages/serverless/src/return-values.ts b/packages/serverless/src/return-values.ts new file mode 100644 index 00000000000..d5a3fba3e4a --- /dev/null +++ b/packages/serverless/src/return-values.ts @@ -0,0 +1,45 @@ +import type {VideoConfig} from 'remotion/no-react'; +import type {ServerlessRoutines} from './constants'; +import type {GenericRenderProgress} from './render-progress'; +import type {CloudProvider} from './types'; + +export type OrError = + | T + | { + type: 'error'; + message: string; + stack: string; + }; + +export interface ServerlessReturnValues { + [ServerlessRoutines.start]: Promise<{ + type: 'success'; + bucketName: string; + renderId: string; + }>; + [ServerlessRoutines.launch]: Promise<{ + type: 'success'; + }>; + [ServerlessRoutines.renderer]: Promise<{ + type: 'success'; + }>; + [ServerlessRoutines.status]: Promise>; + [ServerlessRoutines.info]: { + version: string; + type: 'success'; + }; + [ServerlessRoutines.still]: Promise< + | { + type: 'success'; + } + | { + type: 'error'; + message: string; + stack: string; + } + >; + [ServerlessRoutines.compositions]: Promise<{ + compositions: VideoConfig[]; + type: 'success'; + }>; +} diff --git a/packages/serverless/src/types.ts b/packages/serverless/src/types.ts index 393ce2fdfc8..5e25269f793 100644 --- a/packages/serverless/src/types.ts +++ b/packages/serverless/src/types.ts @@ -1,3 +1,5 @@ +import type {ServerlessPayloads, ServerlessRoutines} from './constants'; + export interface CloudProvider< Region extends string = string, ReceivedArtifactType extends Record = Record< @@ -47,3 +49,14 @@ export type ParsedTiming = { start: number; rendered: number; }; + +export type CallFunctionOptions< + T extends ServerlessRoutines, + Provider extends CloudProvider, +> = { + functionName: string; + type: T; + payload: ServerlessPayloads[T]; + region: Provider['region']; + timeoutInTest: number; +}; From 58b12d0ee0b3f68996d327d53c2c6e2a6654d430 Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Sun, 5 Jan 2025 22:01:57 +0100 Subject: [PATCH 06/14] nice --- .../src/functions/helpers/merge-chunks.ts | 4 ++ packages/lambda/src/functions/index.ts | 8 ++-- .../src/shared/call-lambda-streaming.ts | 2 +- packages/lambda/src/test/mocks/aws-clients.ts | 46 +++++++++++++++++-- 4 files changed, 51 insertions(+), 9 deletions(-) diff --git a/packages/lambda/src/functions/helpers/merge-chunks.ts b/packages/lambda/src/functions/helpers/merge-chunks.ts index 31076d4bcad..04a213e5c8f 100644 --- a/packages/lambda/src/functions/helpers/merge-chunks.ts +++ b/packages/lambda/src/functions/helpers/merge-chunks.ts @@ -63,6 +63,10 @@ export const mergeChunksAndFinishRender = async < throw new Error('Cannot merge stills'); } + if (options.files.length === 0) { + throw new Error('No files to merge'); + } + const {outfile, cleanupChunksProm} = await concatVideos({ onProgress, numberOfFrames: options.numberOfFrames, diff --git a/packages/lambda/src/functions/index.ts b/packages/lambda/src/functions/index.ts index 86fd8dd5bb1..ef7f78d73e6 100644 --- a/packages/lambda/src/functions/index.ts +++ b/packages/lambda/src/functions/index.ts @@ -332,12 +332,10 @@ const innerHandler = async ({ export const innerRoutine = async ( params: ServerlessPayload, - responseStream: ResponseStream, + responseWriter: ResponseStreamWriter, context: RequestContext, providerSpecifics: ProviderSpecifics, ): Promise => { - const responseWriter = streamWriter(responseStream); - try { await innerHandler({ params, @@ -362,7 +360,9 @@ export const routine = ( responseStream: ResponseStream, context: RequestContext, ): Promise => { - return innerRoutine(params, responseStream, context, awsImplementation); + const responseWriter = streamWriter(responseStream); + + return innerRoutine(params, responseWriter, context, awsImplementation); }; export const handler = streamifyResponse(routine); diff --git a/packages/lambda/src/shared/call-lambda-streaming.ts b/packages/lambda/src/shared/call-lambda-streaming.ts index 6599201e434..4f475a54558 100644 --- a/packages/lambda/src/shared/call-lambda-streaming.ts +++ b/packages/lambda/src/shared/call-lambda-streaming.ts @@ -24,7 +24,7 @@ import {getLambdaClient} from './aws-clients'; const STREAM_STALL_TIMEOUT = 30000; const LAMBDA_STREAM_STALL = `AWS did not invoke Lambda in ${STREAM_STALL_TIMEOUT}ms`; -const parseJsonOrThrowSource = (data: Uint8Array, type: string) => { +export const parseJsonOrThrowSource = (data: Uint8Array, type: string) => { const asString = new TextDecoder('utf-8').decode(data); try { return JSON.parse(asString); diff --git a/packages/lambda/src/test/mocks/aws-clients.ts b/packages/lambda/src/test/mocks/aws-clients.ts index 4c8596b0045..36acd52afdd 100644 --- a/packages/lambda/src/test/mocks/aws-clients.ts +++ b/packages/lambda/src/test/mocks/aws-clients.ts @@ -6,9 +6,17 @@ import type { OnMessage, ServerlessReturnValues, ServerlessRoutines, + StreamingMessage, } from '@remotion/serverless'; -import {ResponseStream} from '@remotion/serverless'; +import {ResponseStream, streamWriter} from '@remotion/serverless'; +import type {MessageTypeId} from '@remotion/serverless/client'; +import { + formatMap, + messageTypeIdToMessageType, +} from '@remotion/serverless/client'; +import {makeStreamer} from '@remotion/streaming'; import type {AwsProvider} from '../../functions/aws-implementation'; +import {parseJsonOrThrowSource} from '../../shared/call-lambda-streaming'; import {mockImplementation} from '../mock-implementation'; export const getMockCallFunctionStreaming: CallFunctionStreaming< @@ -22,9 +30,39 @@ export const getMockCallFunctionStreaming: CallFunctionStreaming< const {innerRoutine} = await import('../../functions/index'); const responseStream = new ResponseStream(); + + const {onData, clear} = makeStreamer((status, messageTypeId, data) => { + const messageType = messageTypeIdToMessageType( + messageTypeId as MessageTypeId, + ); + const innerPayload = + formatMap[messageType] === 'json' + ? parseJsonOrThrowSource(data, messageType) + : data; + + const message: StreamingMessage = { + successType: status, + message: { + type: messageType, + payload: innerPayload, + }, + }; + + params.receivedStreamingPayload(message); + }); + await innerRoutine( params.payload, - responseStream, + { + write(message) { + onData(message); + return Promise.resolve(); + }, + end() { + clear(); + return Promise.resolve(); + }, + }, { invokedFunctionArn: 'arn:fake', getRemainingTimeInMillis: () => params.timeoutInTest ?? 120000, @@ -47,7 +85,7 @@ export const getMockCallFunctionAsync: CallFunctionAsync = async < const responseStream = new ResponseStream(); await innerRoutine( params.payload, - responseStream, + streamWriter(responseStream), { invokedFunctionArn: 'arn:fake', getRemainingTimeInMillis: () => params.timeoutInTest ?? 120000, @@ -70,7 +108,7 @@ export const getMockCallFunctionSync: CallFunctionSync = async < const responseStream = new ResponseStream(); await innerRoutine( params.payload, - responseStream, + streamWriter(responseStream), { invokedFunctionArn: 'arn:fake', getRemainingTimeInMillis: () => params.timeoutInTest ?? 120000, From ccc9a380222b6fdaf0e943cb17614f0acfede54b Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Sun, 5 Jan 2025 22:04:19 +0100 Subject: [PATCH 07/14] Update aws-clients.ts --- packages/lambda/src/test/mocks/aws-clients.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/lambda/src/test/mocks/aws-clients.ts b/packages/lambda/src/test/mocks/aws-clients.ts index 36acd52afdd..2d53c75fac5 100644 --- a/packages/lambda/src/test/mocks/aws-clients.ts +++ b/packages/lambda/src/test/mocks/aws-clients.ts @@ -4,6 +4,7 @@ import type { CallFunctionStreaming, CallFunctionSync, OnMessage, + OrError, ServerlessReturnValues, ServerlessRoutines, StreamingMessage, @@ -120,7 +121,12 @@ export const getMockCallFunctionSync: CallFunctionSync = async < responseStream._finish(); responseStream.end(); - return JSON.parse( + const parsed = JSON.parse( new TextDecoder().decode(responseStream.getBufferedData()), - ) as ServerlessReturnValues[T]; + ) as OrError[T]>>; + if (parsed.type === 'error') { + throw new Error(parsed.message); + } + + return parsed; }; From 66210d66d20ce95dc1871864e4af4e3756e7a0df Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Sun, 5 Jan 2025 22:09:53 +0100 Subject: [PATCH 08/14] move start handler to `@remotion/serverless` --- .../src/functions/aws-implementation.ts | 8 +++++++ packages/lambda/src/functions/index.ts | 2 +- .../lambda/src/test/mock-implementation.ts | 7 ++++++ .../src/{ => handlers}/compositions.ts | 14 ++++++------ .../src/handlers}/start.ts | 22 +++++++------------ packages/serverless/src/index.ts | 3 ++- .../serverless/src/provider-implementation.ts | 1 + 7 files changed, 34 insertions(+), 23 deletions(-) rename packages/serverless/src/{ => handlers}/compositions.ts (90%) rename packages/{lambda/src/functions => serverless/src/handlers}/start.ts (90%) diff --git a/packages/lambda/src/functions/aws-implementation.ts b/packages/lambda/src/functions/aws-implementation.ts index 6f29dc5055e..a4a1b043169 100644 --- a/packages/lambda/src/functions/aws-implementation.ts +++ b/packages/lambda/src/functions/aws-implementation.ts @@ -96,4 +96,12 @@ export const awsImplementation: ProviderSpecifics = { callFunctionAsync: callFunctionAsyncImplementation, callFunctionStreaming: callFunctionWithStreamingImplementation, callFunctionSync: callFunctionSyncImplementation, + getCurrentFunctionName() { + const name = process.env.AWS_LAMBDA_FUNCTION_NAME; + if (!name) { + throw new Error('Expected AWS_LAMBDA_FUNCTION_NAME to be set'); + } + + return name; + }, }; diff --git a/packages/lambda/src/functions/index.ts b/packages/lambda/src/functions/index.ts index ef7f78d73e6..da945318654 100644 --- a/packages/lambda/src/functions/index.ts +++ b/packages/lambda/src/functions/index.ts @@ -10,6 +10,7 @@ import type { import { compositionsHandler, infoHandler, + startHandler, streamWriter, } from '@remotion/serverless'; import type {ServerlessPayload} from '@remotion/serverless/client'; @@ -30,7 +31,6 @@ import {streamifyResponse} from './helpers/streamify-response'; import {launchHandler} from './launch'; import {progressHandler} from './progress'; import {rendererHandler} from './renderer'; -import {startHandler} from './start'; import {stillHandler} from './still'; const innerHandler = async ({ diff --git a/packages/lambda/src/test/mock-implementation.ts b/packages/lambda/src/test/mock-implementation.ts index c5785182333..425db25e699 100644 --- a/packages/lambda/src/test/mock-implementation.ts +++ b/packages/lambda/src/test/mock-implementation.ts @@ -1,5 +1,6 @@ import type {ProviderSpecifics} from '@remotion/serverless'; import {Readable} from 'stream'; +import {speculateFunctionName} from '../client'; import type {AwsProvider} from '../functions/aws-implementation'; import {convertToServeUrlImplementation} from '../shared/convert-to-serve-url'; import { @@ -128,4 +129,10 @@ export const mockImplementation: ProviderSpecifics = { callFunctionAsync: getMockCallFunctionAsync, callFunctionStreaming: getMockCallFunctionStreaming, callFunctionSync: getMockCallFunctionSync, + getCurrentFunctionName: () => + speculateFunctionName({ + diskSizeInMb: 10240, + memorySizeInMb: 3009, + timeoutInSeconds: 120, + }), }; diff --git a/packages/serverless/src/compositions.ts b/packages/serverless/src/handlers/compositions.ts similarity index 90% rename from packages/serverless/src/compositions.ts rename to packages/serverless/src/handlers/compositions.ts index f5ea4a12867..1e75dee82ba 100644 --- a/packages/serverless/src/compositions.ts +++ b/packages/serverless/src/handlers/compositions.ts @@ -1,15 +1,15 @@ import {RenderInternals} from '@remotion/renderer'; import {VERSION} from 'remotion/version'; -import {decompressInputProps} from './compress-props'; -import type {ServerlessPayload} from './constants'; -import {ServerlessRoutines} from './constants'; +import {decompressInputProps} from '../compress-props'; +import type {ServerlessPayload} from '../constants'; +import {ServerlessRoutines} from '../constants'; import { forgetBrowserEventLoop, getBrowserInstance, -} from './get-browser-instance'; -import {internalGetOrCreateBucket} from './get-or-create-bucket'; -import type {ProviderSpecifics} from './provider-implementation'; -import type {CloudProvider} from './types'; +} from '../get-browser-instance'; +import {internalGetOrCreateBucket} from '../get-or-create-bucket'; +import type {ProviderSpecifics} from '../provider-implementation'; +import type {CloudProvider} from '../types'; type Options = { expectedBucketOwner: string; diff --git a/packages/lambda/src/functions/start.ts b/packages/serverless/src/handlers/start.ts similarity index 90% rename from packages/lambda/src/functions/start.ts rename to packages/serverless/src/handlers/start.ts index 053e72ac6c4..7fc3726f331 100644 --- a/packages/lambda/src/functions/start.ts +++ b/packages/serverless/src/handlers/start.ts @@ -1,16 +1,10 @@ -import { - makeInitialOverallRenderProgress, - type CloudProvider, - type ProviderSpecifics, -} from '@remotion/serverless'; -import type {ServerlessPayload} from '@remotion/serverless/client'; -import { - ServerlessRoutines, - internalGetOrCreateBucket, - overallProgressKey, -} from '@remotion/serverless/client'; import {VERSION} from 'remotion/version'; -import type {AwsRegion} from '../regions'; +import type {ServerlessPayload} from '../constants'; +import {ServerlessRoutines, overallProgressKey} from '../constants'; +import {internalGetOrCreateBucket} from '../get-or-create-bucket'; +import {makeInitialOverallRenderProgress} from '../overall-render-progress'; +import type {ProviderSpecifics} from '../provider-implementation'; +import type {CloudProvider} from '../types'; type Options = { expectedBucketOwner: string; @@ -124,10 +118,10 @@ export const startHandler = async ( }; await providerSpecifics.callFunctionAsync({ - functionName: process.env.AWS_LAMBDA_FUNCTION_NAME as string, + functionName: providerSpecifics.getCurrentFunctionName(), type: ServerlessRoutines.launch, payload, - region: region as AwsRegion, + region, timeoutInTest: options.timeoutInMilliseconds, }); diff --git a/packages/serverless/src/index.ts b/packages/serverless/src/index.ts index e15bbbc1221..29023c4dd38 100644 --- a/packages/serverless/src/index.ts +++ b/packages/serverless/src/index.ts @@ -1,4 +1,5 @@ -export {compositionsHandler} from './compositions'; +export {compositionsHandler} from './handlers/compositions'; +export {startHandler} from './handlers/start'; export {PostRenderData, ServerlessRoutines} from './constants'; export {getCredentialsFromOutName} from './expected-out-name'; diff --git a/packages/serverless/src/provider-implementation.ts b/packages/serverless/src/provider-implementation.ts index 735451fafdb..9b5f0ea3152 100644 --- a/packages/serverless/src/provider-implementation.ts +++ b/packages/serverless/src/provider-implementation.ts @@ -179,4 +179,5 @@ export type ProviderSpecifics = { callFunctionAsync: CallFunctionAsync; callFunctionStreaming: CallFunctionStreaming; callFunctionSync: CallFunctionSync; + getCurrentFunctionName: () => string; }; From 6a0305c81b7b58cc117d6918e662e79b3e000644 Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Sun, 5 Jan 2025 22:39:44 +0100 Subject: [PATCH 09/14] extract more to `@remotion/serverless` --- packages/lambda/src/api/delete-render.ts | 4 +- packages/lambda/src/api/download-media.ts | 4 +- .../lambda/src/api/get-render-progress.ts | 2 +- .../lambda/src/api/render-still-on-lambda.ts | 2 +- packages/lambda/src/cli/index.ts | 3 +- .../src/functions/aws-implementation.ts | 14 +++++ .../src/functions/helpers/merge-chunks.ts | 2 +- packages/lambda/src/functions/index.ts | 2 +- packages/lambda/src/functions/launch.ts | 2 +- packages/lambda/src/functions/still.ts | 2 +- .../lambda/src/shared/check-credentials.ts | 2 +- packages/lambda/src/shared/constants.ts | 7 --- .../lambda/src/shared/convert-to-serve-url.ts | 2 +- packages/lambda/src/shared/get-aws-urls.ts | 18 ++----- .../lambda/src/test/mock-implementation.ts | 10 ++++ .../src/test/unit/price-calculation.test.ts | 4 +- packages/serverless/src/client.ts | 1 + .../src/shared => serverless/src}/docs-url.ts | 0 .../src/estimate-price-from-bucket.ts} | 29 +++++----- .../src}/format-costs-info.ts | 2 +- .../src/get-overall-progress-from-storage.ts} | 15 +++--- .../src}/get-overall-progress.ts | 0 .../src/handlers}/progress.ts | 14 +++-- packages/serverless/src/index.ts | 16 +++--- .../src/inspect-error.ts} | 7 ++- .../src}/make-timeout-error.ts | 9 +++- .../src}/make-timeout-message.ts | 54 ++++++++++--------- .../src/progress.ts} | 40 ++++++-------- .../serverless/src/provider-implementation.ts | 37 +++++++++++++ .../src}/render-has-audio-video.ts | 4 +- 30 files changed, 179 insertions(+), 129 deletions(-) rename packages/{lambda/src/shared => serverless/src}/docs-url.ts (100%) rename packages/{lambda/src/functions/helpers/calculate-price-from-bucket.ts => serverless/src/estimate-price-from-bucket.ts} (63%) rename packages/{lambda/src/functions/helpers => serverless/src}/format-costs-info.ts (90%) rename packages/{lambda/src/functions/helpers/get-overall-progress-s3.ts => serverless/src/get-overall-progress-from-storage.ts} (67%) rename packages/{lambda/src/functions/helpers => serverless/src}/get-overall-progress.ts (100%) rename packages/{lambda/src/functions => serverless/src/handlers}/progress.ts (88%) rename packages/{lambda/src/functions/helpers/inspect-errors.ts => serverless/src/inspect-error.ts} (90%) rename packages/{lambda/src/functions/helpers => serverless/src}/make-timeout-error.ts (73%) rename packages/{lambda/src/functions/helpers => serverless/src}/make-timeout-message.ts (69%) rename packages/{lambda/src/functions/helpers/get-progress.ts => serverless/src/progress.ts} (92%) rename packages/{lambda/src/functions/helpers => serverless/src}/render-has-audio-video.ts (83%) diff --git a/packages/lambda/src/api/delete-render.ts b/packages/lambda/src/api/delete-render.ts index db997b25b42..0b3fa533f4f 100644 --- a/packages/lambda/src/api/delete-render.ts +++ b/packages/lambda/src/api/delete-render.ts @@ -1,12 +1,12 @@ import type {ProviderSpecifics} from '@remotion/serverless'; import { getExpectedOutName, + getOverallProgressFromStorage, rendersPrefix, type CustomCredentials, } from '@remotion/serverless/client'; import type {AwsProvider} from '../functions/aws-implementation'; import {awsImplementation} from '../functions/aws-implementation'; -import {getOverallProgressS3} from '../functions/helpers/get-overall-progress-s3'; import type {AwsRegion} from '../regions'; import {getAccountId} from '../shared/get-account-id'; import {cleanItems} from './clean-items'; @@ -28,7 +28,7 @@ export const internalDeleteRender = async ( const expectedBucketOwner = await getAccountId({ region: input.region, }); - const progress = await getOverallProgressS3({ + const progress = await getOverallProgressFromStorage({ bucketName: input.bucketName, expectedBucketOwner, region: input.region, diff --git a/packages/lambda/src/api/download-media.ts b/packages/lambda/src/api/download-media.ts index acec2781b6a..181e701a00f 100644 --- a/packages/lambda/src/api/download-media.ts +++ b/packages/lambda/src/api/download-media.ts @@ -3,12 +3,12 @@ import {RenderInternals} from '@remotion/renderer'; import type {ProviderSpecifics} from '@remotion/serverless'; import { getExpectedOutName, + getOverallProgressFromStorage, type CustomCredentials, } from '@remotion/serverless/client'; import path from 'node:path'; import type {AwsProvider} from '../functions/aws-implementation'; import {awsImplementation} from '../functions/aws-implementation'; -import {getOverallProgressS3} from '../functions/helpers/get-overall-progress-s3'; import type {LambdaReadFileProgress} from '../functions/helpers/read-with-progress'; import {lambdaDownloadFileWithProgress} from '../functions/helpers/read-with-progress'; import type {AwsRegion} from '../regions'; @@ -39,7 +39,7 @@ export const internalDownloadMedia = async ( const expectedBucketOwner = await getAccountId({ region: input.region, }); - const overallProgress = await getOverallProgressS3({ + const overallProgress = await getOverallProgressFromStorage({ bucketName: input.bucketName, expectedBucketOwner, region: input.region, diff --git a/packages/lambda/src/api/get-render-progress.ts b/packages/lambda/src/api/get-render-progress.ts index 23d8e5b8ad0..d54045dfe1b 100644 --- a/packages/lambda/src/api/get-render-progress.ts +++ b/packages/lambda/src/api/get-render-progress.ts @@ -1,11 +1,11 @@ import type {LogLevel} from '@remotion/renderer'; +import {getProgress} from '@remotion/serverless'; import type {CustomCredentials} from '@remotion/serverless/client'; import {ServerlessRoutines} from '@remotion/serverless/client'; import { awsImplementation, type AwsProvider, } from '../functions/aws-implementation'; -import {getProgress} from '../functions/helpers/get-progress'; import {parseFunctionName} from '../functions/helpers/parse-function-name'; import type {AwsRegion} from '../regions'; import type {RenderProgress} from '../shared/constants'; diff --git a/packages/lambda/src/api/render-still-on-lambda.ts b/packages/lambda/src/api/render-still-on-lambda.ts index efdea652b04..7f6f77d06d2 100644 --- a/packages/lambda/src/api/render-still-on-lambda.ts +++ b/packages/lambda/src/api/render-still-on-lambda.ts @@ -9,6 +9,7 @@ import {ServerlessRoutines} from '@remotion/serverless/client'; import {wrapWithErrorHandling} from '@remotion/renderer/error-handling'; import type { + CostsInfo, ReceivedArtifact, RenderStillLambdaResponsePayload, } from '@remotion/serverless'; @@ -18,7 +19,6 @@ import { type AwsProvider, } from '../functions/aws-implementation'; import type {AwsRegion} from '../regions'; -import type {CostsInfo} from '../shared/constants'; import {DEFAULT_MAX_RETRIES} from '../shared/constants'; import { getCloudwatchMethodUrl, diff --git a/packages/lambda/src/cli/index.ts b/packages/lambda/src/cli/index.ts index 91b79c14aa4..a03ba198d1f 100644 --- a/packages/lambda/src/cli/index.ts +++ b/packages/lambda/src/cli/index.ts @@ -1,13 +1,12 @@ import {CliInternals} from '@remotion/cli'; import type {LogLevel} from '@remotion/renderer'; import {RenderInternals} from '@remotion/renderer'; -import type {ProviderSpecifics} from '@remotion/serverless'; +import {DOCS_URL, type ProviderSpecifics} from '@remotion/serverless'; import {ROLE_NAME} from '../api/iam-validation/suggested-policy'; import {BINARY_NAME} from '../defaults'; import type {AwsProvider} from '../functions/aws-implementation'; import {awsImplementation} from '../functions/aws-implementation'; import {checkCredentials} from '../shared/check-credentials'; -import {DOCS_URL} from '../shared/docs-url'; import {parsedLambdaCli} from './args'; import { COMPOSITIONS_COMMAND, diff --git a/packages/lambda/src/functions/aws-implementation.ts b/packages/lambda/src/functions/aws-implementation.ts index a4a1b043169..6d4db65c44d 100644 --- a/packages/lambda/src/functions/aws-implementation.ts +++ b/packages/lambda/src/functions/aws-implementation.ts @@ -3,7 +3,9 @@ import {expiryDays} from '@remotion/serverless/client'; import {EventEmitter} from 'node:events'; import {bucketExistsInRegionImplementation} from '../api/bucket-exists'; import {createBucket} from '../api/create-bucket'; +import {estimatePrice} from '../api/estimate-price'; import {getRemotionBuckets} from '../api/get-buckets'; +import {MAX_EPHEMERAL_STORAGE_IN_MB} from '../defaults'; import {lambdaDeleteFileImplementation} from '../io/delete-file'; import {lambdaHeadFileImplementation} from '../io/head-file'; import {lambdaLsImplementation} from '../io/list-objects'; @@ -14,6 +16,10 @@ import {callFunctionAsyncImplementation} from '../shared/call-lambda-async'; import {callFunctionWithStreamingImplementation} from '../shared/call-lambda-streaming'; import {callFunctionSyncImplementation} from '../shared/call-lambda-sync'; import {convertToServeUrlImplementation} from '../shared/convert-to-serve-url'; +import { + getCloudwatchMethodUrl, + getCloudwatchRendererUrl, +} from '../shared/get-aws-urls'; import {applyLifeCyleOperation} from '../shared/lifecycle-rules'; import {randomHashImplementation} from '../shared/random-hash'; import {getCurrentRegionInFunctionImplementation} from './helpers/get-current-region'; @@ -104,4 +110,12 @@ export const awsImplementation: ProviderSpecifics = { return name; }, + getEphemeralStorageForPriceCalculation() { + // We cannot determine the ephemeral storage size, so we + // overestimate the price, but will only have a miniscule effect (~0.2%) + return MAX_EPHEMERAL_STORAGE_IN_MB; + }, + estimatePrice, + getLoggingUrlForMethod: getCloudwatchMethodUrl, + getLoggingUrlForRendererFunction: getCloudwatchRendererUrl, }; diff --git a/packages/lambda/src/functions/helpers/merge-chunks.ts b/packages/lambda/src/functions/helpers/merge-chunks.ts index 04a213e5c8f..5bd50a3a06f 100644 --- a/packages/lambda/src/functions/helpers/merge-chunks.ts +++ b/packages/lambda/src/functions/helpers/merge-chunks.ts @@ -5,6 +5,7 @@ import type { PostRenderData, ProviderSpecifics, } from '@remotion/serverless'; +import {inspectErrors} from '@remotion/serverless'; import type { CustomCredentials, DownloadBehavior, @@ -18,7 +19,6 @@ import {cleanupProps} from './cleanup-props'; import {concatVideos} from './concat-videos'; import {createPostRenderData} from './create-post-render-data'; import {getOutputUrlFromMetadata} from './get-output-url-from-metadata'; -import {inspectErrors} from './inspect-errors'; import {timer} from './timer'; export const mergeChunksAndFinishRender = async < diff --git a/packages/lambda/src/functions/index.ts b/packages/lambda/src/functions/index.ts index da945318654..94063794fc0 100644 --- a/packages/lambda/src/functions/index.ts +++ b/packages/lambda/src/functions/index.ts @@ -10,6 +10,7 @@ import type { import { compositionsHandler, infoHandler, + progressHandler, startHandler, streamWriter, } from '@remotion/serverless'; @@ -29,7 +30,6 @@ import {printLoggingGrepHelper} from './helpers/print-logging-helper'; import type {RequestContext} from './helpers/request-context'; import {streamifyResponse} from './helpers/streamify-response'; import {launchHandler} from './launch'; -import {progressHandler} from './progress'; import {rendererHandler} from './renderer'; import {stillHandler} from './still'; diff --git a/packages/lambda/src/functions/launch.ts b/packages/lambda/src/functions/launch.ts index e8442b9fe8d..001871aff45 100644 --- a/packages/lambda/src/functions/launch.ts +++ b/packages/lambda/src/functions/launch.ts @@ -8,6 +8,7 @@ import type { ProviderSpecifics, } from '@remotion/serverless'; import { + DOCS_URL, forgetBrowserEventLoop, getBrowserInstance, getTmpDirStateIfENoSp, @@ -37,7 +38,6 @@ import { CONCAT_FOLDER_TOKEN, MAX_FUNCTIONS_PER_RENDER, } from '../shared/constants'; -import {DOCS_URL} from '../shared/docs-url'; import { validateDimension, validateDurationInFrames, diff --git a/packages/lambda/src/functions/still.ts b/packages/lambda/src/functions/still.ts index 4abfb2882f8..6edab9fbe68 100644 --- a/packages/lambda/src/functions/still.ts +++ b/packages/lambda/src/functions/still.ts @@ -8,6 +8,7 @@ import type { } from '@remotion/serverless'; import { forgetBrowserEventLoop, + formatCostsInfo, getBrowserInstance, getCredentialsFromOutName, getTmpDirStateIfENoSp, @@ -39,7 +40,6 @@ import {cleanupSerializedInputProps} from '../shared/cleanup-serialized-input-pr import {isFlakyError} from '../shared/is-flaky-error'; import {validateDownloadBehavior} from '../shared/validate-download-behavior'; import {validatePrivacy} from '../shared/validate-privacy'; -import {formatCostsInfo} from './helpers/format-costs-info'; import {getOutputUrlFromMetadata} from './helpers/get-output-url-from-metadata'; import {onDownloadsHelper} from './helpers/on-downloads-logger'; diff --git a/packages/lambda/src/shared/check-credentials.ts b/packages/lambda/src/shared/check-credentials.ts index 3ab7465feed..c95698f9edc 100644 --- a/packages/lambda/src/shared/check-credentials.ts +++ b/packages/lambda/src/shared/check-credentials.ts @@ -1,6 +1,6 @@ +import {DOCS_URL} from '@remotion/serverless'; import {truthy} from '@remotion/serverless/client'; import {getIsCli} from '../cli/is-cli'; -import {DOCS_URL} from './docs-url'; import {isLikelyToHaveAwsProfile} from './is-likely-to-have-aws-profile'; const messageForVariable = (variable: string) => { diff --git a/packages/lambda/src/shared/constants.ts b/packages/lambda/src/shared/constants.ts index ea0383b8b10..257243d8a23 100644 --- a/packages/lambda/src/shared/constants.ts +++ b/packages/lambda/src/shared/constants.ts @@ -44,13 +44,6 @@ export const CONCAT_FOLDER_TOKEN = 'remotion-concat'; export const REMOTION_CONCATED_TOKEN = 'remotion-concated-token'; export const REMOTION_FILELIST_TOKEN = 'remotion-filelist'; -export type CostsInfo = { - accruedSoFar: number; - displayCost: string; - currency: string; - disclaimer: string; -}; - export type RenderProgress = GenericRenderProgress; export const LAMBDA_CONCURRENCY_LIMIT_QUOTA = 'L-B99A9384'; diff --git a/packages/lambda/src/shared/convert-to-serve-url.ts b/packages/lambda/src/shared/convert-to-serve-url.ts index fc82c2725ee..d1180a74f67 100644 --- a/packages/lambda/src/shared/convert-to-serve-url.ts +++ b/packages/lambda/src/shared/convert-to-serve-url.ts @@ -1,5 +1,5 @@ +import {DOCS_URL} from '@remotion/serverless'; import type {AwsRegion} from '../regions'; -import {DOCS_URL} from './docs-url'; export const convertToServeUrlImplementation = ({ urlOrId, diff --git a/packages/lambda/src/shared/get-aws-urls.ts b/packages/lambda/src/shared/get-aws-urls.ts index 5597cdc3c70..f7bc2795ecb 100644 --- a/packages/lambda/src/shared/get-aws-urls.ts +++ b/packages/lambda/src/shared/get-aws-urls.ts @@ -1,5 +1,7 @@ +import type {GetLoggingUrlForRendererFunction} from '@remotion/serverless'; import type {ServerlessRoutines} from '@remotion/serverless/client'; import type {AwsRegion} from '../client'; +import type {AwsProvider} from '../functions/aws-implementation'; import {encodeAwsUrlParams} from './encode-aws-url-params'; const cloudWatchUrlWithQuery = ({ @@ -45,19 +47,9 @@ export const getLambdaInsightsUrl = ({ return `https://${region}.console.aws.amazon.com/cloudwatch/home?region=${region}#lambda-insights:functions/${functionName}`; }; -export const getCloudwatchRendererUrl = ({ - region, - functionName, - renderId, - rendererFunctionName, - chunk, -}: { - region: AwsRegion; - functionName: string; - rendererFunctionName: string | null; - renderId: string; - chunk: null | number; -}) => { +export const getCloudwatchRendererUrl: GetLoggingUrlForRendererFunction< + AwsProvider +> = ({region, functionName, renderId, rendererFunctionName, chunk}) => { const functionNameToUse = rendererFunctionName ?? functionName; const query = `"method=renderer,renderId=${renderId}${ chunk === null ? '' : `,chunk=${chunk},` diff --git a/packages/lambda/src/test/mock-implementation.ts b/packages/lambda/src/test/mock-implementation.ts index 425db25e699..2ab1c17f0d1 100644 --- a/packages/lambda/src/test/mock-implementation.ts +++ b/packages/lambda/src/test/mock-implementation.ts @@ -1,8 +1,14 @@ import type {ProviderSpecifics} from '@remotion/serverless'; import {Readable} from 'stream'; +import {estimatePrice} from '../api/estimate-price'; import {speculateFunctionName} from '../client'; +import {MAX_EPHEMERAL_STORAGE_IN_MB} from '../defaults'; import type {AwsProvider} from '../functions/aws-implementation'; import {convertToServeUrlImplementation} from '../shared/convert-to-serve-url'; +import { + getCloudwatchMethodUrl, + getCloudwatchRendererUrl, +} from '../shared/get-aws-urls'; import { getMockCallFunctionAsync, getMockCallFunctionStreaming, @@ -23,6 +29,9 @@ export const mockImplementation: ProviderSpecifics = { getChromiumPath() { return null; }, + getEphemeralStorageForPriceCalculation: () => MAX_EPHEMERAL_STORAGE_IN_MB, + getLoggingUrlForMethod: getCloudwatchMethodUrl, + getLoggingUrlForRendererFunction: getCloudwatchRendererUrl, getCurrentRegionInFunction: () => 'eu-central-1', createBucket: (input) => { addMockBucket({ @@ -135,4 +144,5 @@ export const mockImplementation: ProviderSpecifics = { memorySizeInMb: 3009, timeoutInSeconds: 120, }), + estimatePrice, }; diff --git a/packages/lambda/src/test/unit/price-calculation.test.ts b/packages/lambda/src/test/unit/price-calculation.test.ts index 1fd46bf2b46..635969bb2c4 100644 --- a/packages/lambda/src/test/unit/price-calculation.test.ts +++ b/packages/lambda/src/test/unit/price-calculation.test.ts @@ -1,5 +1,6 @@ +import {estimatePriceFromBucket} from '@remotion/serverless'; import {expect, test} from 'vitest'; -import {estimatePriceFromBucket} from '../../functions/helpers/calculate-price-from-bucket'; +import {awsImplementation} from '../../functions/aws-implementation'; test('Should not throw while calculating prices when time shifts occur', () => { const aDate = Date.now(); @@ -54,6 +55,7 @@ test('Should not throw while calculating prices when time shifts occur', () => { }, ], region: 'eu-central-1', + providerSpecifics: awsImplementation, }); expect(price?.accruedSoFar).toBeGreaterThanOrEqual(0); }); diff --git a/packages/serverless/src/client.ts b/packages/serverless/src/client.ts index b57eca8025c..e47023de84f 100644 --- a/packages/serverless/src/client.ts +++ b/packages/serverless/src/client.ts @@ -47,6 +47,7 @@ export { export {calculateChunkTimes} from './calculate-chunk-times'; export {getExpectedOutName} from './expected-out-name'; export {FileNameAndSize, GetFolderFiles} from './get-files-in-folder'; +export {getOverallProgressFromStorage} from './get-overall-progress-from-storage'; export {inputPropsKey, resolvedPropsKey} from './input-props-keys'; export {makeBucketName} from './make-bucket-name'; export {RenderMetadata} from './render-metadata'; diff --git a/packages/lambda/src/shared/docs-url.ts b/packages/serverless/src/docs-url.ts similarity index 100% rename from packages/lambda/src/shared/docs-url.ts rename to packages/serverless/src/docs-url.ts diff --git a/packages/lambda/src/functions/helpers/calculate-price-from-bucket.ts b/packages/serverless/src/estimate-price-from-bucket.ts similarity index 63% rename from packages/lambda/src/functions/helpers/calculate-price-from-bucket.ts rename to packages/serverless/src/estimate-price-from-bucket.ts index 0483bcfce59..ad9d6066feb 100644 --- a/packages/lambda/src/functions/helpers/calculate-price-from-bucket.ts +++ b/packages/serverless/src/estimate-price-from-bucket.ts @@ -1,8 +1,7 @@ -import {type CloudProvider, type ParsedTiming} from '@remotion/serverless'; -import type {RenderMetadata} from '@remotion/serverless/client'; -import {calculateChunkTimes} from '@remotion/serverless/client'; -import {estimatePrice} from '../../api/estimate-price'; -import type {AwsRegion} from '../../regions'; +import {calculateChunkTimes} from './calculate-chunk-times'; +import type {ProviderSpecifics} from './provider-implementation'; +import type {RenderMetadata} from './render-metadata'; +import type {CloudProvider, ParsedTiming} from './types'; export const estimatePriceFromBucket = ({ renderMetadata, @@ -11,13 +10,15 @@ export const estimatePriceFromBucket = ({ lambdasInvoked, timings, region, + providerSpecifics, }: { renderMetadata: RenderMetadata | null; memorySizeInMb: number; diskSizeInMb: number; lambdasInvoked: number; timings: ParsedTiming[]; - region: AwsRegion; + region: Provider['region']; + providerSpecifics: ProviderSpecifics; }) => { if (!renderMetadata) { return null; @@ -43,13 +44,15 @@ export const estimatePriceFromBucket = ({ }) + timeElapsedOfUnfinished; const accruedSoFar = Number( - estimatePrice({ - region, - durationInMilliseconds: estimatedBillingDurationInMilliseconds, - memorySizeInMb, - diskSizeInMb, - lambdasInvoked, - }).toPrecision(5), + providerSpecifics + .estimatePrice({ + region, + durationInMilliseconds: estimatedBillingDurationInMilliseconds, + memorySizeInMb, + diskSizeInMb, + lambdasInvoked, + }) + .toPrecision(5), ); return {accruedSoFar, estimatedBillingDurationInMilliseconds}; diff --git a/packages/lambda/src/functions/helpers/format-costs-info.ts b/packages/serverless/src/format-costs-info.ts similarity index 90% rename from packages/lambda/src/functions/helpers/format-costs-info.ts rename to packages/serverless/src/format-costs-info.ts index b95bc5273f9..9a53dbbe79b 100644 --- a/packages/lambda/src/functions/helpers/format-costs-info.ts +++ b/packages/serverless/src/format-costs-info.ts @@ -1,4 +1,4 @@ -import type {CostsInfo} from '../../shared/constants'; +import type {CostsInfo} from './types'; const display = (accrued: number) => { if (accrued < 0.001) { diff --git a/packages/lambda/src/functions/helpers/get-overall-progress-s3.ts b/packages/serverless/src/get-overall-progress-from-storage.ts similarity index 67% rename from packages/lambda/src/functions/helpers/get-overall-progress-s3.ts rename to packages/serverless/src/get-overall-progress-from-storage.ts index eaf7c36b407..8c2fcc7e3f7 100644 --- a/packages/lambda/src/functions/helpers/get-overall-progress-s3.ts +++ b/packages/serverless/src/get-overall-progress-from-storage.ts @@ -1,11 +1,12 @@ -import type { - CloudProvider, - OverallRenderProgress, - ProviderSpecifics, -} from '@remotion/serverless'; -import {overallProgressKey, streamToString} from '@remotion/serverless/client'; +import {overallProgressKey} from './constants'; +import type {OverallRenderProgress} from './overall-render-progress'; +import type {ProviderSpecifics} from './provider-implementation'; +import {streamToString} from './stream-to-string'; +import type {CloudProvider} from './types'; -export const getOverallProgressS3 = async ({ +export const getOverallProgressFromStorage = async < + Provider extends CloudProvider, +>({ renderId, bucketName, expectedBucketOwner, diff --git a/packages/lambda/src/functions/helpers/get-overall-progress.ts b/packages/serverless/src/get-overall-progress.ts similarity index 100% rename from packages/lambda/src/functions/helpers/get-overall-progress.ts rename to packages/serverless/src/get-overall-progress.ts diff --git a/packages/lambda/src/functions/progress.ts b/packages/serverless/src/handlers/progress.ts similarity index 88% rename from packages/lambda/src/functions/progress.ts rename to packages/serverless/src/handlers/progress.ts index 014e8ad657e..563ddcb3ba2 100644 --- a/packages/lambda/src/functions/progress.ts +++ b/packages/serverless/src/handlers/progress.ts @@ -1,12 +1,10 @@ -import type { - CloudProvider, - GenericRenderProgress, - ProviderSpecifics, -} from '@remotion/serverless'; -import type {ServerlessPayload} from '@remotion/serverless/client'; -import {ServerlessRoutines} from '@remotion/serverless/client'; import {VERSION} from 'remotion/version'; -import {getProgress} from './helpers/get-progress'; +import type {ServerlessPayload} from '../constants'; +import {ServerlessRoutines} from '../constants'; +import {getProgress} from '../progress'; +import type {ProviderSpecifics} from '../provider-implementation'; +import type {GenericRenderProgress} from '../render-progress'; +import type {CloudProvider} from '../types'; type Options = { expectedBucketOwner: string; diff --git a/packages/serverless/src/index.ts b/packages/serverless/src/index.ts index 29023c4dd38..e55bda96607 100644 --- a/packages/serverless/src/index.ts +++ b/packages/serverless/src/index.ts @@ -1,13 +1,18 @@ export {compositionsHandler} from './handlers/compositions'; +export {progressHandler} from './handlers/progress'; export {startHandler} from './handlers/start'; export {PostRenderData, ServerlessRoutines} from './constants'; +export {DOCS_URL} from './docs-url'; +export {estimatePriceFromBucket} from './estimate-price-from-bucket'; export {getCredentialsFromOutName} from './expected-out-name'; +export {formatCostsInfo} from './format-costs-info'; export { forgetBrowserEventLoop, getBrowserInstance, } from './get-browser-instance'; export {infoHandler} from './info'; +export {inspectErrors} from './inspect-error'; export {WebhookPayload, invokeWebhook} from './invoke-webhook'; export { OVERHEAD_TIME_PER_LAMBDA, @@ -19,15 +24,8 @@ export { makeInitialOverallRenderProgress, makeOverallRenderProgress, } from './overall-render-progress'; -export { - BucketWithLocation, - CallFunctionAsync, - CallFunctionStreaming, - CallFunctionSync, - MakeArtifactWithDetails, - ProviderSpecifics, - WriteFileInput, -} from './provider-implementation'; +export {getProgress} from './progress'; +export * from './provider-implementation'; export type {CleanupInfo, GenericRenderProgress} from './render-progress'; export {OrError, ServerlessReturnValues} from './return-values'; export {deserializeArtifact, serializeArtifact} from './serialize-artifact'; diff --git a/packages/lambda/src/functions/helpers/inspect-errors.ts b/packages/serverless/src/inspect-error.ts similarity index 90% rename from packages/lambda/src/functions/helpers/inspect-errors.ts rename to packages/serverless/src/inspect-error.ts index db88c3acd70..1bbd4b9e930 100644 --- a/packages/lambda/src/functions/helpers/inspect-errors.ts +++ b/packages/serverless/src/inspect-error.ts @@ -1,11 +1,10 @@ -import type {EnhancedErrorInfo, LambdaErrorInfo} from '@remotion/serverless'; +import {DOCS_URL} from './docs-url'; import { errorIsOutOfSpaceError, isBrowserCrashedError, isErrInsufficientResourcesErr, -} from '@remotion/serverless/client'; - -import {DOCS_URL} from '../../shared/docs-url'; +} from './error-category'; +import type {EnhancedErrorInfo, LambdaErrorInfo} from './write-lambda-error'; const FAILED_TO_LAUNCH_TOKEN = 'Failed to launch browser.'; diff --git a/packages/lambda/src/functions/helpers/make-timeout-error.ts b/packages/serverless/src/make-timeout-error.ts similarity index 73% rename from packages/lambda/src/functions/helpers/make-timeout-error.ts rename to packages/serverless/src/make-timeout-error.ts index 2026ddc46ce..24ef43655ce 100644 --- a/packages/lambda/src/functions/helpers/make-timeout-error.ts +++ b/packages/serverless/src/make-timeout-error.ts @@ -1,6 +1,8 @@ -import type {CloudProvider, EnhancedErrorInfo} from '@remotion/serverless'; -import type {RenderMetadata} from '@remotion/serverless/client'; import {makeTimeoutMessage} from './make-timeout-message'; +import type {ProviderSpecifics} from './provider-implementation'; +import type {RenderMetadata} from './render-metadata'; +import type {CloudProvider} from './types'; +import type {EnhancedErrorInfo} from './write-lambda-error'; export const makeTimeoutError = ({ timeoutInMilliseconds, @@ -9,6 +11,7 @@ export const makeTimeoutError = ({ renderId, functionName, region, + providerSpecifics, }: { timeoutInMilliseconds: number; renderMetadata: RenderMetadata; @@ -16,6 +19,7 @@ export const makeTimeoutError = ({ missingChunks: number[]; functionName: string; region: Provider['region']; + providerSpecifics: ProviderSpecifics; }): EnhancedErrorInfo => { const message = makeTimeoutMessage({ missingChunks, @@ -24,6 +28,7 @@ export const makeTimeoutError = ({ renderId, functionName, region, + providerSpecifics, }); const error = new Error(message); diff --git a/packages/lambda/src/functions/helpers/make-timeout-message.ts b/packages/serverless/src/make-timeout-message.ts similarity index 69% rename from packages/lambda/src/functions/helpers/make-timeout-message.ts rename to packages/serverless/src/make-timeout-message.ts index 7c17cf5ff21..71846b67448 100644 --- a/packages/lambda/src/functions/helpers/make-timeout-message.ts +++ b/packages/serverless/src/make-timeout-message.ts @@ -1,12 +1,8 @@ -import type {CloudProvider} from '@remotion/serverless'; -import type {RenderMetadata} from '@remotion/serverless/client'; -import {ServerlessRoutines} from '@remotion/serverless/client'; -import type {AwsRegion} from '../../regions'; -import {DOCS_URL} from '../../shared/docs-url'; -import { - getCloudwatchMethodUrl, - getCloudwatchRendererUrl, -} from '../../shared/get-aws-urls'; +import {ServerlessRoutines} from './constants'; +import {DOCS_URL} from './docs-url'; +import type {ProviderSpecifics} from './provider-implementation'; +import type {RenderMetadata} from './render-metadata'; +import type {CloudProvider} from './types'; const MAX_MISSING_CHUNKS = 5; @@ -14,10 +10,12 @@ const makeChunkMissingMessage = ({ missingChunks, renderMetadata, region, + providerSpecifics, }: { missingChunks: number[]; renderMetadata: RenderMetadata; region: Provider['region']; + providerSpecifics: ProviderSpecifics; }) => { if (missingChunks.length === 0) { return 'All chunks have been successfully rendered, but the main function has timed out.'; @@ -43,13 +41,15 @@ const makeChunkMissingMessage = ({ return [ msg, - `â–¸ Logs for chunk ${ch}: ${getCloudwatchRendererUrl({ - functionName: process.env.AWS_LAMBDA_FUNCTION_NAME as string, - region: region as AwsRegion, - rendererFunctionName: null, - renderId: renderMetadata.renderId, - chunk: ch, - })}`, + `â–¸ Logs for chunk ${ch}: ${providerSpecifics.getLoggingUrlForRendererFunction( + { + functionName: providerSpecifics.getCurrentFunctionName(), + region, + rendererFunctionName: null, + renderId: renderMetadata.renderId, + chunk: ch, + }, + )}`, ].join('\n'); }) .slice(0, 5), @@ -63,6 +63,7 @@ export const makeTimeoutMessage = ({ renderId, functionName, region, + providerSpecifics, }: { timeoutInMilliseconds: number; missingChunks: number[]; @@ -70,20 +71,22 @@ export const makeTimeoutMessage = ({ renderId: string; region: Provider['region']; functionName: string; + providerSpecifics: ProviderSpecifics; }) => { - const cloudWatchRendererUrl = getCloudwatchRendererUrl({ - renderId, - functionName, - region: region as AwsRegion, - rendererFunctionName: functionName, - chunk: null, - }); + const cloudWatchRendererUrl = + providerSpecifics.getLoggingUrlForRendererFunction({ + renderId, + functionName, + region, + rendererFunctionName: functionName, + chunk: null, + }); - const cloudWatchLaunchUrl = getCloudwatchMethodUrl({ + const cloudWatchLaunchUrl = providerSpecifics.getLoggingUrlForMethod({ renderId, functionName, method: ServerlessRoutines.launch, - region: region as AwsRegion, + region, rendererFunctionName: functionName, }); const message = [ @@ -92,6 +95,7 @@ export const makeTimeoutMessage = ({ missingChunks, renderMetadata, region, + providerSpecifics, }), '', `Consider increasing the timeout of your function.`, diff --git a/packages/lambda/src/functions/helpers/get-progress.ts b/packages/serverless/src/progress.ts similarity index 92% rename from packages/lambda/src/functions/helpers/get-progress.ts rename to packages/serverless/src/progress.ts index a1ccbeb2e4f..1e15de906f7 100644 --- a/packages/lambda/src/functions/helpers/get-progress.ts +++ b/packages/serverless/src/progress.ts @@ -1,27 +1,21 @@ import {NoReactAPIs} from '@remotion/renderer/pure'; -import type { - CleanupInfo, - CloudProvider, - EnhancedErrorInfo, - GenericRenderProgress, - ProviderSpecifics, -} from '@remotion/serverless'; -import { - calculateChunkTimes, - getExpectedOutName, - truthy, - type CustomCredentials, -} from '@remotion/serverless/client'; + import {NoReactInternals} from 'remotion/no-react'; -import type {AwsRegion} from '../../regions'; -import {MAX_EPHEMERAL_STORAGE_IN_MB} from '../../shared/constants'; -import {estimatePriceFromBucket} from './calculate-price-from-bucket'; +import {calculateChunkTimes} from './calculate-chunk-times'; +import type {CustomCredentials} from './constants'; +import {estimatePriceFromBucket} from './estimate-price-from-bucket'; +import {getExpectedOutName} from './expected-out-name'; import {formatCostsInfo} from './format-costs-info'; import {getOverallProgress} from './get-overall-progress'; -import {getOverallProgressS3} from './get-overall-progress-s3'; -import {inspectErrors} from './inspect-errors'; +import {getOverallProgressFromStorage} from './get-overall-progress-from-storage'; +import {inspectErrors} from './inspect-error'; import {makeTimeoutError} from './make-timeout-error'; +import type {ProviderSpecifics} from './provider-implementation'; import {lambdaRenderHasAudioVideo} from './render-has-audio-video'; +import type {CleanupInfo, GenericRenderProgress} from './render-progress'; +import {truthy} from './truthy'; +import type {CloudProvider} from './types'; +import type {EnhancedErrorInfo} from './write-lambda-error'; export const getProgress = async ({ bucketName, @@ -46,7 +40,7 @@ export const getProgress = async ({ forcePathStyle: boolean; functionName: string; }): Promise> => { - const overallProgress = await getOverallProgressS3({ + const overallProgress = await getOverallProgressFromStorage({ renderId, bucketName, expectedBucketOwner, @@ -207,11 +201,10 @@ export const getProgress = async ({ renderMetadata, memorySizeInMb, lambdasInvoked: renderMetadata.estimatedRenderLambdaInvokations ?? 0, - // We cannot determine the ephemeral storage size, so we - // overestimate the price, but will only have a miniscule effect (~0.2%) - diskSizeInMb: MAX_EPHEMERAL_STORAGE_IN_MB, + diskSizeInMb: providerSpecifics.getEphemeralStorageForPriceCalculation(), timings: overallProgress.timings ?? [], - region: region as AwsRegion, + region, + providerSpecifics, }); const chunkMultiplier = [hasAudio, hasVideo].filter(truthy).length; @@ -261,6 +254,7 @@ export const getProgress = async ({ missingChunks: missingChunks ?? [], region, functionName, + providerSpecifics, }) : null, ...errorExplanations, diff --git a/packages/serverless/src/provider-implementation.ts b/packages/serverless/src/provider-implementation.ts index 9b5f0ea3152..1dadee0d705 100644 --- a/packages/serverless/src/provider-implementation.ts +++ b/packages/serverless/src/provider-implementation.ts @@ -158,6 +158,39 @@ export type CallFunctionSync = < ServerlessReturnValues[T] >; +export type EstimatePriceInput = { + region: Provider['region']; + memorySizeInMb: number; + diskSizeInMb: number; + lambdasInvoked: number; + durationInMilliseconds: number; +}; + +export type EstimatePrice = ({ + region, + memorySizeInMb, + diskSizeInMb, + lambdasInvoked, + ...other +}: EstimatePriceInput) => number; + +export type GetLoggingUrlForRendererFunction = + (options: { + region: Provider['region']; + functionName: string; + rendererFunctionName: string | null; + renderId: string; + chunk: null | number; + }) => string; + +export type GetLoggingUrlForMethod = (options: { + region: Provider['region']; + functionName: string; + method: ServerlessRoutines; + rendererFunctionName: string | null; + renderId: string; +}) => string; + export type ProviderSpecifics = { getChromiumPath: () => string | null; getCurrentRegionInFunction: () => Provider['region']; @@ -180,4 +213,8 @@ export type ProviderSpecifics = { callFunctionStreaming: CallFunctionStreaming; callFunctionSync: CallFunctionSync; getCurrentFunctionName: () => string; + estimatePrice: EstimatePrice; + getLoggingUrlForRendererFunction: GetLoggingUrlForRendererFunction; + getLoggingUrlForMethod: GetLoggingUrlForMethod; + getEphemeralStorageForPriceCalculation: () => number; }; diff --git a/packages/lambda/src/functions/helpers/render-has-audio-video.ts b/packages/serverless/src/render-has-audio-video.ts similarity index 83% rename from packages/lambda/src/functions/helpers/render-has-audio-video.ts rename to packages/serverless/src/render-has-audio-video.ts index 02b3eed1418..d8b7600bc4b 100644 --- a/packages/lambda/src/functions/helpers/render-has-audio-video.ts +++ b/packages/serverless/src/render-has-audio-video.ts @@ -1,6 +1,6 @@ import {NoReactAPIs} from '@remotion/renderer/pure'; -import type {CloudProvider} from '@remotion/serverless'; -import type {RenderMetadata} from '@remotion/serverless/client'; +import type {RenderMetadata} from './render-metadata'; +import type {CloudProvider} from './types'; export const lambdaRenderHasAudioVideo = ( renderMetadata: RenderMetadata, From 098251f1585f9976ddb7e7809e6ed31ccd1fa01d Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Sun, 5 Jan 2025 23:37:57 +0100 Subject: [PATCH 10/14] crazy --- packages/lambda/src/api/deploy-site.ts | 3 +- .../lambda/src/api/make-lambda-payload.ts | 6 +- .../lambda/src/cli/commands/render/render.ts | 10 +- packages/lambda/src/cli/commands/still.ts | 2 +- .../src/functions/aws-implementation.ts | 14 ++- .../src/functions/chunk-optimization/types.ts | 8 -- .../helpers/__mocks__/get-browser-instance.ts | 16 --- .../helpers/__mocks__/leak-detection.ts | 11 -- .../src/functions/helpers/__mocks__/timer.ts | 8 -- .../helpers/get-output-url-from-metadata.ts | 22 ++-- .../helpers/print-concurrency-curve.ts | 9 -- .../src/functions/helpers/request-context.ts | 5 - .../lambda/src/functions/helpers/timer.ts | 3 +- packages/lambda/src/functions/index.ts | 19 ++-- packages/lambda/src/shared/constants.ts | 8 -- .../lambda/src/shared/get-service-client.ts | 2 +- .../lambda/src/test/mock-implementation.ts | 24 +++- packages/lambda/src/test/setup.ts | 9 -- .../unit/best-frames-per-lambda-param.test.ts | 15 --- .../src/best-frames-per-function-param.ts} | 2 +- .../src}/can-concat-seamlessly.ts | 0 .../src}/cleanup-props.ts | 7 +- .../src}/cleanup-serialized-input-props.ts | 10 +- packages/serverless/src/client.ts | 4 + .../src}/concat-videos.ts | 20 ++-- packages/serverless/src/constants.ts | 9 ++ .../src}/create-post-render-data.ts | 44 ++++---- .../src}/find-output-file-in-bucket.ts | 16 ++- .../serverless/src/get-browser-instance.ts | 17 ++- .../serverless/src/handlers/compositions.ts | 9 +- .../src/handlers}/launch.ts | 104 +++++++++--------- .../src/handlers}/renderer.ts | 65 +++++------ .../src/handlers}/still.ts | 88 +++++++-------- packages/serverless/src/index.ts | 17 +-- packages/serverless/src/invoke-webhook.ts | 12 +- .../src}/leak-detection.ts | 5 +- .../src}/merge-chunks.ts | 41 ++++--- .../serverless/src/most-expensive-chunks.ts | 17 ++- .../src/on-downloads-helpers.ts} | 0 .../src}/plan-frame-ranges.ts | 0 .../serverless/src/provider-implementation.ts | 41 ++++++- .../shared => serverless/src}/stackback.ts | 0 .../src}/stream-renderer.ts | 19 ++-- .../src/test/best-frames-per-function.test.ts | 15 +++ .../src/test/most-expensive.test.ts | 24 ++-- .../src/test}/plan-frame-ranges.test.ts | 4 +- packages/serverless/src/types.ts | 9 ++ .../src}/validate-download-behavior.ts | 2 +- .../src/validate-frames-per-function.ts} | 2 +- .../src}/validate-privacy.ts | 2 +- .../src/shared => serverless/src}/validate.ts | 0 .../src}/why-is-node-running.ts | 1 + 52 files changed, 410 insertions(+), 390 deletions(-) delete mode 100644 packages/lambda/src/functions/chunk-optimization/types.ts delete mode 100644 packages/lambda/src/functions/helpers/__mocks__/get-browser-instance.ts delete mode 100644 packages/lambda/src/functions/helpers/__mocks__/leak-detection.ts delete mode 100644 packages/lambda/src/functions/helpers/__mocks__/timer.ts delete mode 100644 packages/lambda/src/functions/helpers/print-concurrency-curve.ts delete mode 100644 packages/lambda/src/functions/helpers/request-context.ts delete mode 100644 packages/lambda/src/test/unit/best-frames-per-lambda-param.test.ts rename packages/{lambda/src/functions/helpers/best-frames-per-lambda-param.ts => serverless/src/best-frames-per-function-param.ts} (90%) rename packages/{lambda/src/functions/helpers => serverless/src}/can-concat-seamlessly.ts (100%) rename packages/{lambda/src/functions/helpers => serverless/src}/cleanup-props.ts (81%) rename packages/{lambda/src/shared => serverless/src}/cleanup-serialized-input-props.ts (83%) rename packages/{lambda/src/functions/helpers => serverless/src}/concat-videos.ts (85%) rename packages/{lambda/src/functions/helpers => serverless/src}/create-post-render-data.ts (74%) rename packages/{lambda/src/functions/helpers => serverless/src}/find-output-file-in-bucket.ts (82%) rename packages/{lambda/src/functions => serverless/src/handlers}/launch.ts (91%) rename packages/{lambda/src/functions => serverless/src/handlers}/renderer.ts (90%) rename packages/{lambda/src/functions => serverless/src/handlers}/still.ts (88%) rename packages/{lambda/src/functions/helpers => serverless/src}/leak-detection.ts (88%) rename packages/{lambda/src/functions/helpers => serverless/src}/merge-chunks.ts (84%) rename packages/{lambda/src/functions/helpers/on-downloads-logger.ts => serverless/src/on-downloads-helpers.ts} (100%) rename packages/{lambda/src/functions/chunk-optimization => serverless/src}/plan-frame-ranges.ts (100%) rename packages/{lambda/src/shared => serverless/src}/stackback.ts (100%) rename packages/{lambda/src/functions/helpers => serverless/src}/stream-renderer.ts (93%) create mode 100644 packages/serverless/src/test/best-frames-per-function.test.ts rename packages/{lambda/src/test/unit => serverless/src/test}/plan-frame-ranges.test.ts (86%) rename packages/{lambda/src/shared => serverless/src}/validate-download-behavior.ts (91%) rename packages/{lambda/src/shared/validate-frames-per-lambda.ts => serverless/src/validate-frames-per-function.ts} (96%) rename packages/{lambda/src/shared => serverless/src}/validate-privacy.ts (89%) rename packages/{lambda/src/shared => serverless/src}/validate.ts (100%) rename packages/{lambda/src/shared => serverless/src}/why-is-node-running.ts (98%) diff --git a/packages/lambda/src/api/deploy-site.ts b/packages/lambda/src/api/deploy-site.ts index 4b315d415af..db812f860bf 100644 --- a/packages/lambda/src/api/deploy-site.ts +++ b/packages/lambda/src/api/deploy-site.ts @@ -3,7 +3,7 @@ import type {ToOptions} from '@remotion/renderer'; import type {BrowserSafeApis} from '@remotion/renderer/client'; import {wrapWithErrorHandling} from '@remotion/renderer/error-handling'; import type {ProviderSpecifics} from '@remotion/serverless'; -import {validateBucketName} from '@remotion/serverless/client'; +import {validateBucketName, validatePrivacy} from '@remotion/serverless/client'; import fs from 'node:fs'; import type {AwsProvider} from '../functions/aws-implementation'; import {awsImplementation} from '../functions/aws-implementation'; @@ -14,7 +14,6 @@ import {getAccountId} from '../shared/get-account-id'; import {getS3DiffOperations} from '../shared/get-s3-operations'; import {makeS3ServeUrl} from '../shared/make-s3-url'; import {validateAwsRegion} from '../shared/validate-aws-region'; -import {validatePrivacy} from '../shared/validate-privacy'; import {validateSiteName} from '../shared/validate-site-name'; import type {UploadDirProgress} from './upload-dir'; import {uploadDir} from './upload-dir'; diff --git a/packages/lambda/src/api/make-lambda-payload.ts b/packages/lambda/src/api/make-lambda-payload.ts index 7e9716496a0..81e6cd59434 100644 --- a/packages/lambda/src/api/make-lambda-payload.ts +++ b/packages/lambda/src/api/make-lambda-payload.ts @@ -26,6 +26,8 @@ import { compressInputProps, getNeedsToUpload, serializeOrThrow, + validateDownloadBehavior, + validateFramesPerFunction, } from '@remotion/serverless/client'; import {VERSION} from 'remotion/version'; import type {AwsRegion, DeleteAfter} from '../client'; @@ -34,8 +36,6 @@ import {awsImplementation} from '../functions/aws-implementation'; import {validateWebhook} from '@remotion/serverless/client'; import {NoReactInternals} from 'remotion/no-react'; -import {validateDownloadBehavior} from '../shared/validate-download-behavior'; -import {validateFramesPerLambda} from '../shared/validate-frames-per-lambda'; import {validateLambdaCodec} from '../shared/validate-lambda-codec'; import {validateServeUrl} from '../shared/validate-serveurl'; import type {GetRenderProgressInput} from './get-render-progress'; @@ -136,7 +136,7 @@ export const makeLambdaRenderMediaPayload = async ({ > => { const actualCodec = validateLambdaCodec(codec); validateServeUrl(serveUrl); - validateFramesPerLambda({ + validateFramesPerFunction({ framesPerLambda: framesPerLambda ?? null, durationInFrames: 1, }); diff --git a/packages/lambda/src/cli/commands/render/render.ts b/packages/lambda/src/cli/commands/render/render.ts index d13d8ed996c..a8554d3a6f1 100644 --- a/packages/lambda/src/cli/commands/render/render.ts +++ b/packages/lambda/src/cli/commands/render/render.ts @@ -10,7 +10,11 @@ import {getRenderProgress} from '../../../api/get-render-progress'; import {internalRenderMediaOnLambdaRaw} from '../../../api/render-media-on-lambda'; import type {EnhancedErrorInfo, ProviderSpecifics} from '@remotion/serverless'; -import type {ServerlessCodec} from '@remotion/serverless/client'; +import { + validateFramesPerFunction, + validatePrivacy, + type ServerlessCodec, +} from '@remotion/serverless/client'; import type {AwsProvider} from '../../../functions/aws-implementation'; import {parseFunctionName} from '../../../functions/helpers/parse-function-name'; import { @@ -19,8 +23,6 @@ import { DEFAULT_OUTPUT_PRIVACY, } from '../../../shared/constants'; import {sleep} from '../../../shared/sleep'; -import {validateFramesPerLambda} from '../../../shared/validate-frames-per-lambda'; -import {validatePrivacy} from '../../../shared/validate-privacy'; import {validateMaxRetries} from '../../../shared/validate-retries'; import {validateServeUrl} from '../../../shared/validate-serveurl'; import {parsedLambdaCli} from '../../args'; @@ -270,7 +272,7 @@ export const renderCommand = async ( const privacy = parsedLambdaCli.privacy ?? DEFAULT_OUTPUT_PRIVACY; validatePrivacy(privacy, true); const framesPerLambda = parsedLambdaCli['frames-per-lambda'] ?? undefined; - validateFramesPerLambda({framesPerLambda, durationInFrames: 1}); + validateFramesPerFunction({framesPerLambda, durationInFrames: 1}); const webhookCustomData = getWebhookCustomData(logLevel); diff --git a/packages/lambda/src/cli/commands/still.ts b/packages/lambda/src/cli/commands/still.ts index 26ebedd470f..6a82e37356d 100644 --- a/packages/lambda/src/cli/commands/still.ts +++ b/packages/lambda/src/cli/commands/still.ts @@ -4,6 +4,7 @@ import type {ChromiumOptions, LogLevel} from '@remotion/renderer'; import {RenderInternals} from '@remotion/renderer'; import {BrowserSafeApis} from '@remotion/renderer/client'; import type {ProviderSpecifics} from '@remotion/serverless'; +import {validatePrivacy} from '@remotion/serverless/client'; import path from 'path'; import {NoReactInternals} from 'remotion/no-react'; import {internalDownloadMedia} from '../../api/download-media'; @@ -15,7 +16,6 @@ import { DEFAULT_OUTPUT_PRIVACY, } from '../../shared/constants'; import {getS3RenderUrl} from '../../shared/get-aws-urls'; -import {validatePrivacy} from '../../shared/validate-privacy'; import {validateMaxRetries} from '../../shared/validate-retries'; import {validateServeUrl} from '../../shared/validate-serveurl'; import {parsedLambdaCli} from '../args'; diff --git a/packages/lambda/src/functions/aws-implementation.ts b/packages/lambda/src/functions/aws-implementation.ts index 6d4db65c44d..392e5cfbe65 100644 --- a/packages/lambda/src/functions/aws-implementation.ts +++ b/packages/lambda/src/functions/aws-implementation.ts @@ -1,4 +1,8 @@ -import {type ProviderSpecifics} from '@remotion/serverless'; +import { + forgetBrowserEventLoopImplementation, + getBrowserInstanceImplementation, + type ProviderSpecifics, +} from '@remotion/serverless'; import {expiryDays} from '@remotion/serverless/client'; import {EventEmitter} from 'node:events'; import {bucketExistsInRegionImplementation} from '../api/bucket-exists'; @@ -20,11 +24,14 @@ import { getCloudwatchMethodUrl, getCloudwatchRendererUrl, } from '../shared/get-aws-urls'; +import {isFlakyError} from '../shared/is-flaky-error'; import {applyLifeCyleOperation} from '../shared/lifecycle-rules'; import {randomHashImplementation} from '../shared/random-hash'; import {getCurrentRegionInFunctionImplementation} from './helpers/get-current-region'; import {getFolderFiles} from './helpers/get-folder-files'; +import {getOutputUrlFromMetadata} from './helpers/get-output-url-from-metadata'; import {makeAwsArtifact} from './helpers/make-aws-artifact'; +import {timer} from './helpers/timer'; if ( /^AWS_Lambda_nodejs(?:18|20)[.]x$/.test( @@ -118,4 +125,9 @@ export const awsImplementation: ProviderSpecifics = { estimatePrice, getLoggingUrlForMethod: getCloudwatchMethodUrl, getLoggingUrlForRendererFunction: getCloudwatchRendererUrl, + isFlakyError, + getOutputUrl: getOutputUrlFromMetadata, + timer, + forgetBrowserEventLoop: forgetBrowserEventLoopImplementation, + getBrowserInstance: getBrowserInstanceImplementation, }; diff --git a/packages/lambda/src/functions/chunk-optimization/types.ts b/packages/lambda/src/functions/chunk-optimization/types.ts deleted file mode 100644 index a15e797d39c..00000000000 --- a/packages/lambda/src/functions/chunk-optimization/types.ts +++ /dev/null @@ -1,8 +0,0 @@ -export type ObjectChunkTimingData = { - chunk: number; - frameRange: [number, number]; - startDate: number; - timings: { - [key: number]: number; - }; -}; diff --git a/packages/lambda/src/functions/helpers/__mocks__/get-browser-instance.ts b/packages/lambda/src/functions/helpers/__mocks__/get-browser-instance.ts deleted file mode 100644 index 25c5274854c..00000000000 --- a/packages/lambda/src/functions/helpers/__mocks__/get-browser-instance.ts +++ /dev/null @@ -1,16 +0,0 @@ -import {openBrowser} from '@remotion/renderer'; -import type { - forgetBrowserEventLoop as originalForget, - getBrowserInstance as originalGetBrowserInstance, -} from '@remotion/serverless'; -import type {Await} from '@remotion/serverless/client'; - -let _browserInstance: Await> | null; - -export const getBrowserInstance: typeof originalGetBrowserInstance = - async () => { - _browserInstance = await openBrowser('chrome'); - return {instance: _browserInstance, configurationString: 'chrome'}; - }; - -export const forgetBrowserEventLoop: typeof originalForget = () => {}; diff --git a/packages/lambda/src/functions/helpers/__mocks__/leak-detection.ts b/packages/lambda/src/functions/helpers/__mocks__/leak-detection.ts deleted file mode 100644 index cdc18e71507..00000000000 --- a/packages/lambda/src/functions/helpers/__mocks__/leak-detection.ts +++ /dev/null @@ -1,11 +0,0 @@ -import type { - setCurrentRequestId as originalSetCurrentRequestId, - startLeakDetection as originalStartLeakDetection, - stopLeakDetection as originalStopLeakDetection, -} from '../leak-detection'; - -export const stopLeakDetection: typeof originalStopLeakDetection = () => {}; - -export const setCurrentRequestId: typeof originalSetCurrentRequestId = () => {}; - -export const startLeakDetection: typeof originalStartLeakDetection = () => {}; diff --git a/packages/lambda/src/functions/helpers/__mocks__/timer.ts b/packages/lambda/src/functions/helpers/__mocks__/timer.ts deleted file mode 100644 index f58fcc89e61..00000000000 --- a/packages/lambda/src/functions/helpers/__mocks__/timer.ts +++ /dev/null @@ -1,8 +0,0 @@ -import type {timer as original} from '../../../functions/helpers/timer'; - -// Turn off timers while in testing. very noisy. -export const timer: typeof original = () => { - return { - end: () => undefined, - }; -}; diff --git a/packages/lambda/src/functions/helpers/get-output-url-from-metadata.ts b/packages/lambda/src/functions/helpers/get-output-url-from-metadata.ts index e4e36099ad0..49c2d4b9ca0 100644 --- a/packages/lambda/src/functions/helpers/get-output-url-from-metadata.ts +++ b/packages/lambda/src/functions/helpers/get-output-url-from-metadata.ts @@ -1,21 +1,19 @@ -import type {CloudProvider} from '@remotion/serverless'; -import type {RenderMetadata} from '@remotion/serverless/client'; -import { - getExpectedOutName, - type CustomCredentials, -} from '@remotion/serverless/client'; +import type {GetOutputUrl} from '@remotion/serverless'; +import {getExpectedOutName} from '@remotion/serverless/client'; +import type {AwsProvider} from '../aws-implementation'; -export const getOutputUrlFromMetadata = ( - renderMetadata: RenderMetadata, - bucketName: string, - customCredentials: CustomCredentials | null, - currentRegion: Provider['region'], -) => { +export const getOutputUrlFromMetadata: GetOutputUrl = ({ + renderMetadata, + bucketName, + customCredentials, + currentRegion, +}) => { const {key, renderBucketName} = getExpectedOutName( renderMetadata, bucketName, customCredentials, ); + return { url: `https://s3.${currentRegion}.amazonaws.com/${renderBucketName}/${key}`, key, diff --git a/packages/lambda/src/functions/helpers/print-concurrency-curve.ts b/packages/lambda/src/functions/helpers/print-concurrency-curve.ts deleted file mode 100644 index 9ba928236d9..00000000000 --- a/packages/lambda/src/functions/helpers/print-concurrency-curve.ts +++ /dev/null @@ -1,9 +0,0 @@ -import {bestFramesPerLambdaParam} from './best-frames-per-lambda-param'; - -const entries: [number, number][] = []; - -for (let i = 0; i < 18000; i += 100) { - entries.push([i, bestFramesPerLambdaParam(i)]); -} - -console.log(entries.map((e) => e.join(',')).join('\n')); diff --git a/packages/lambda/src/functions/helpers/request-context.ts b/packages/lambda/src/functions/helpers/request-context.ts deleted file mode 100644 index fba5d21614a..00000000000 --- a/packages/lambda/src/functions/helpers/request-context.ts +++ /dev/null @@ -1,5 +0,0 @@ -export type RequestContext = { - invokedFunctionArn: string; - getRemainingTimeInMillis: () => number; - awsRequestId: string; -}; diff --git a/packages/lambda/src/functions/helpers/timer.ts b/packages/lambda/src/functions/helpers/timer.ts index d80a55b5214..577c4b2c962 100644 --- a/packages/lambda/src/functions/helpers/timer.ts +++ b/packages/lambda/src/functions/helpers/timer.ts @@ -1,11 +1,12 @@ import type {LogLevel} from '@remotion/renderer'; import {RenderInternals} from '@remotion/renderer'; +import type {DebuggingTimer} from '@remotion/serverless'; const formatTime = (time: number) => { return time + 'ms'; }; -export const timer = (label: string, logLevel: LogLevel) => { +export const timer: DebuggingTimer = (label: string, logLevel: LogLevel) => { const start = Date.now(); RenderInternals.Log.verbose({indent: false, logLevel}, `${label} - start\n`); diff --git a/packages/lambda/src/functions/index.ts b/packages/lambda/src/functions/index.ts index 94063794fc0..055205f6002 100644 --- a/packages/lambda/src/functions/index.ts +++ b/packages/lambda/src/functions/index.ts @@ -3,6 +3,7 @@ import type { CloudProvider, OrError, ProviderSpecifics, + RequestContext, ResponseStream, ResponseStreamWriter, StreamingPayload, @@ -10,8 +11,13 @@ import type { import { compositionsHandler, infoHandler, + launchHandler, progressHandler, + rendererHandler, + setCurrentRequestId, startHandler, + stillHandler, + stopLeakDetection, streamWriter, } from '@remotion/serverless'; import type {ServerlessPayload} from '@remotion/serverless/client'; @@ -24,14 +30,10 @@ import type {AwsProvider} from './aws-implementation'; import {awsImplementation} from './aws-implementation'; import {deleteTmpDir} from './helpers/clean-tmpdir'; import {getWarm, setWarm} from './helpers/is-warm'; -import {setCurrentRequestId, stopLeakDetection} from './helpers/leak-detection'; import {generateRandomHashWithLifeCycleRule} from './helpers/lifecycle'; import {printLoggingGrepHelper} from './helpers/print-logging-helper'; -import type {RequestContext} from './helpers/request-context'; import {streamifyResponse} from './helpers/streamify-response'; -import {launchHandler} from './launch'; -import {rendererHandler} from './renderer'; -import {stillHandler} from './still'; +import {getWebhookClient} from './http-client'; const innerHandler = async ({ params, @@ -189,14 +191,15 @@ const innerHandler = async ({ ); } - const response = await launchHandler( + const response = await launchHandler({ params, - { + options: { expectedBucketOwner: currentUserId, getRemainingTimeInMillis: context.getRemainingTimeInMillis, }, providerSpecifics, - ); + client: getWebhookClient(params.webhook?.url ?? 'http://localhost:3000'), + }); await responseWriter.write(Buffer.from(JSON.stringify(response))); await responseWriter.end(); diff --git a/packages/lambda/src/shared/constants.ts b/packages/lambda/src/shared/constants.ts index 257243d8a23..6c30ace9e4c 100644 --- a/packages/lambda/src/shared/constants.ts +++ b/packages/lambda/src/shared/constants.ts @@ -12,7 +12,6 @@ export const DEFAULT_TIMEOUT = 120; export const MIN_TIMEOUT = 15; export const MAX_TIMEOUT = 900; -export const MINIMUM_FRAMES_PER_LAMBDA = 4; export const DEFAULT_FRAMES_PER_LAMBDA = 20; export const BINARY_NAME = 'remotion lambda'; @@ -20,8 +19,6 @@ export const COMMAND_NOT_FOUND = 'Command not found'; export const DEFAULT_REGION: AwsRegion = 'us-east-1'; export const DEFAULT_MAX_RETRIES = 1; -export const MAX_FUNCTIONS_PER_RENDER = 200; - export const MAX_EPHEMERAL_STORAGE_IN_MB = 10240; export const DEFAULT_EPHEMERAL_STORAGE_IN_MB = NoReactInternals.ENABLE_V5_BREAKING_CHANGES @@ -39,11 +36,6 @@ export const LAMBDA_INSIGHTS_PREFIX = '/aws/lambda-insights'; export const getSitesKey = (siteId: string) => `sites/${siteId}`; -export const RENDERER_PATH_TOKEN = 'remotion-bucket'; -export const CONCAT_FOLDER_TOKEN = 'remotion-concat'; -export const REMOTION_CONCATED_TOKEN = 'remotion-concated-token'; -export const REMOTION_FILELIST_TOKEN = 'remotion-filelist'; - export type RenderProgress = GenericRenderProgress; export const LAMBDA_CONCURRENCY_LIMIT_QUOTA = 'L-B99A9384'; diff --git a/packages/lambda/src/shared/get-service-client.ts b/packages/lambda/src/shared/get-service-client.ts index ff8d2aa8fcc..75bc1e7e14f 100644 --- a/packages/lambda/src/shared/get-service-client.ts +++ b/packages/lambda/src/shared/get-service-client.ts @@ -4,12 +4,12 @@ import {LambdaClient} from '@aws-sdk/client-lambda'; import {S3Client} from '@aws-sdk/client-s3'; import {ServiceQuotasClient} from '@aws-sdk/client-service-quotas'; import {STSClient} from '@aws-sdk/client-sts'; +import {MAX_FUNCTIONS_PER_RENDER} from '@remotion/serverless/client'; import {random} from 'remotion/no-react'; import type {CustomCredentials} from '../client'; import type {AwsProvider} from '../functions/aws-implementation'; import type {AwsRegion} from '../regions'; import {checkCredentials} from './check-credentials'; -import {MAX_FUNCTIONS_PER_RENDER} from './constants'; import {getCredentials} from './get-credentials'; export type ServiceMapping = { diff --git a/packages/lambda/src/test/mock-implementation.ts b/packages/lambda/src/test/mock-implementation.ts index 2ab1c17f0d1..7a7b0b61e0b 100644 --- a/packages/lambda/src/test/mock-implementation.ts +++ b/packages/lambda/src/test/mock-implementation.ts @@ -1,4 +1,5 @@ -import type {ProviderSpecifics} from '@remotion/serverless'; +import {openBrowser} from '@remotion/renderer'; +import type {GetBrowserInstance, ProviderSpecifics} from '@remotion/serverless'; import {Readable} from 'stream'; import {estimatePrice} from '../api/estimate-price'; import {speculateFunctionName} from '../client'; @@ -9,6 +10,7 @@ import { getCloudwatchMethodUrl, getCloudwatchRendererUrl, } from '../shared/get-aws-urls'; +import {isFlakyError} from '../shared/is-flaky-error'; import { getMockCallFunctionAsync, getMockCallFunctionStreaming, @@ -24,6 +26,14 @@ import { writeMockS3File, } from './mocks/mock-store'; +type Await = T extends PromiseLike ? U : T; +let _browserInstance: Await> | null; + +export const getBrowserInstance: GetBrowserInstance = async () => { + _browserInstance = await openBrowser('chrome'); + return {instance: _browserInstance, configurationString: 'chrome'}; +}; + export const mockImplementation: ProviderSpecifics = { applyLifeCycle: () => Promise.resolve(), getChromiumPath() { @@ -145,4 +155,16 @@ export const mockImplementation: ProviderSpecifics = { timeoutInSeconds: 120, }), estimatePrice, + getOutputUrl: () => { + return { + key: 'mock/mock.mp4', + url: 'https://s3.mock-region-1.amazonaws.com/bucket/mock.mp4', + }; + }, + isFlakyError, + timer: () => ({ + end: () => {}, + }), + forgetBrowserEventLoop: () => {}, + getBrowserInstance, }; diff --git a/packages/lambda/src/test/setup.ts b/packages/lambda/src/test/setup.ts index 33806afbf16..646812234ee 100644 --- a/packages/lambda/src/test/setup.ts +++ b/packages/lambda/src/test/setup.ts @@ -3,15 +3,6 @@ import {vi} from 'vitest'; vi.mock('../cli/helpers/quit', () => vi.importActual('../cli/helpers/__mocks__/quit'), ); -vi.mock('../functions/helpers/timer', () => - vi.importActual('../functions/helpers/__mocks__/timer'), -); -vi.mock('../functions/helpers/get-browser-instance', () => - vi.importActual('../functions/helpers/__mocks__/get-browser-instance'), -); -vi.mock('../functions/helpers/leak-detection', () => - vi.importActual('../functions/helpers/__mocks__/leak-detection'), -); vi.mock('../shared/bundle-site', () => vi.importActual('../shared/__mocks__/bundle-site'), ); diff --git a/packages/lambda/src/test/unit/best-frames-per-lambda-param.test.ts b/packages/lambda/src/test/unit/best-frames-per-lambda-param.test.ts deleted file mode 100644 index 391e6f68b22..00000000000 --- a/packages/lambda/src/test/unit/best-frames-per-lambda-param.test.ts +++ /dev/null @@ -1,15 +0,0 @@ -import {expect, test} from 'vitest'; -import {bestFramesPerLambdaParam} from '../../functions/helpers/best-frames-per-lambda-param'; - -test('Get reasonable framesPerLambda defaults', () => { - expect(bestFramesPerLambdaParam(20)).toEqual(20); - expect(bestFramesPerLambdaParam(21)).toEqual(11); - expect(bestFramesPerLambdaParam(100)).toEqual(20); - expect(bestFramesPerLambdaParam(2000)).toEqual(24); - expect(bestFramesPerLambdaParam(4000)).toEqual(44); - expect(bestFramesPerLambdaParam(8000)).toEqual(74); - expect(bestFramesPerLambdaParam(10000)).toEqual(86); - expect(bestFramesPerLambdaParam(14000)).toEqual(105); - expect(bestFramesPerLambdaParam(18000)).toEqual(120); - expect(bestFramesPerLambdaParam(216000)).toEqual(1440); -}); diff --git a/packages/lambda/src/functions/helpers/best-frames-per-lambda-param.ts b/packages/serverless/src/best-frames-per-function-param.ts similarity index 90% rename from packages/lambda/src/functions/helpers/best-frames-per-lambda-param.ts rename to packages/serverless/src/best-frames-per-function-param.ts index a7fef5117f3..412c9850308 100644 --- a/packages/lambda/src/functions/helpers/best-frames-per-lambda-param.ts +++ b/packages/serverless/src/best-frames-per-function-param.ts @@ -2,7 +2,7 @@ import {interpolate} from 'remotion/no-react'; // Always update the code in docs/lambda/concurrency.md too -export const bestFramesPerLambdaParam = (frameCount: number) => { +export const bestFramesPerFunctionParam = (frameCount: number) => { // Between 0 and 10 minutes (at 30fps), interpolate the concurrency from 75 to 150 const concurrency = interpolate(frameCount, [0, 18000], [75, 150], { extrapolateRight: 'clamp', diff --git a/packages/lambda/src/functions/helpers/can-concat-seamlessly.ts b/packages/serverless/src/can-concat-seamlessly.ts similarity index 100% rename from packages/lambda/src/functions/helpers/can-concat-seamlessly.ts rename to packages/serverless/src/can-concat-seamlessly.ts diff --git a/packages/lambda/src/functions/helpers/cleanup-props.ts b/packages/serverless/src/cleanup-props.ts similarity index 81% rename from packages/lambda/src/functions/helpers/cleanup-props.ts rename to packages/serverless/src/cleanup-props.ts index adfd971a28e..0a0b20cac39 100644 --- a/packages/lambda/src/functions/helpers/cleanup-props.ts +++ b/packages/serverless/src/cleanup-props.ts @@ -1,9 +1,10 @@ -import type {CloudProvider, ProviderSpecifics} from '@remotion/serverless'; -import type {SerializedInputProps} from '@remotion/serverless/client'; import { cleanupSerializedInputProps, cleanupSerializedResolvedProps, -} from '../../shared/cleanup-serialized-input-props'; +} from './cleanup-serialized-input-props'; +import type {SerializedInputProps} from './constants'; +import type {ProviderSpecifics} from './provider-implementation'; +import type {CloudProvider} from './types'; export const cleanupProps = ({ serializedResolvedProps, diff --git a/packages/lambda/src/shared/cleanup-serialized-input-props.ts b/packages/serverless/src/cleanup-serialized-input-props.ts similarity index 83% rename from packages/lambda/src/shared/cleanup-serialized-input-props.ts rename to packages/serverless/src/cleanup-serialized-input-props.ts index aa0f8e6a16c..f5e30b03f54 100644 --- a/packages/lambda/src/shared/cleanup-serialized-input-props.ts +++ b/packages/serverless/src/cleanup-serialized-input-props.ts @@ -1,9 +1,7 @@ -import type {CloudProvider, ProviderSpecifics} from '@remotion/serverless'; -import { - inputPropsKey, - resolvedPropsKey, - type SerializedInputProps, -} from '@remotion/serverless/client'; +import type {SerializedInputProps} from './constants'; +import {inputPropsKey, resolvedPropsKey} from './input-props-keys'; +import type {ProviderSpecifics} from './provider-implementation'; +import type {CloudProvider} from './types'; export const cleanupSerializedInputProps = async < Provider extends CloudProvider, diff --git a/packages/serverless/src/client.ts b/packages/serverless/src/client.ts index e47023de84f..d0cd97e53b1 100644 --- a/packages/serverless/src/client.ts +++ b/packages/serverless/src/client.ts @@ -37,6 +37,8 @@ export { GetOrCreateBucketOutput, internalGetOrCreateBucket, } from './get-or-create-bucket'; +export {validateDownloadBehavior} from './validate-download-behavior'; +export {validateFramesPerFunction} from './validate-frames-per-function'; export { errorIsOutOfSpaceError, @@ -45,6 +47,7 @@ export { } from './error-category'; export {calculateChunkTimes} from './calculate-chunk-times'; +export {MAX_FUNCTIONS_PER_RENDER} from './constants'; export {getExpectedOutName} from './expected-out-name'; export {FileNameAndSize, GetFolderFiles} from './get-files-in-folder'; export {getOverallProgressFromStorage} from './get-overall-progress-from-storage'; @@ -60,4 +63,5 @@ export { } from './streaming/streaming'; export {truthy} from './truthy'; export {validateBucketName} from './validate-bucket-name'; +export {validatePrivacy} from './validate-privacy'; export {validateWebhook} from './validate-webhook'; diff --git a/packages/lambda/src/functions/helpers/concat-videos.ts b/packages/serverless/src/concat-videos.ts similarity index 85% rename from packages/lambda/src/functions/helpers/concat-videos.ts rename to packages/serverless/src/concat-videos.ts index c06eef6a94b..ad1714a1472 100644 --- a/packages/lambda/src/functions/helpers/concat-videos.ts +++ b/packages/serverless/src/concat-videos.ts @@ -1,19 +1,21 @@ import type {AudioCodec, CancelSignal, LogLevel} from '@remotion/renderer'; import {RenderInternals} from '@remotion/renderer'; -import type {ServerlessCodec} from '@remotion/serverless/client'; import fs from 'node:fs'; import {join} from 'node:path'; -import { - REMOTION_CONCATED_TOKEN, - REMOTION_FILELIST_TOKEN, -} from '../../shared/constants'; + import { canConcatAudioSeamlessly, canConcatVideoSeamlessly, } from './can-concat-seamlessly'; -import {timer} from './timer'; +import { + REMOTION_CONCATED_TOKEN, + REMOTION_FILELIST_TOKEN, + type ServerlessCodec, +} from './constants'; +import type {ProviderSpecifics} from './provider-implementation'; +import type {CloudProvider} from './types'; -export const concatVideos = async ({ +export const concatVideos = async ({ onProgress, numberOfFrames, codec, @@ -30,6 +32,7 @@ export const concatVideos = async ({ preferLossless, muted, metadata, + providerSpecifics, }: { onProgress: (frames: number) => void; numberOfFrames: number; @@ -47,12 +50,13 @@ export const concatVideos = async ({ preferLossless: boolean; muted: boolean; metadata: Record | null; + providerSpecifics: ProviderSpecifics; }) => { const outfile = join( RenderInternals.tmpDir(REMOTION_CONCATED_TOKEN), `concat.${RenderInternals.getFileExtensionFromCodec(codec, audioCodec)}`, ); - const combine = timer('Combine chunks', logLevel); + const combine = providerSpecifics.timer('Combine chunks', logLevel); const filelistDir = RenderInternals.tmpDir(REMOTION_FILELIST_TOKEN); const chunkDurationInSeconds = framesPerLambda / fps; diff --git a/packages/serverless/src/constants.ts b/packages/serverless/src/constants.ts index 22134a30d19..b4146908ca8 100644 --- a/packages/serverless/src/constants.ts +++ b/packages/serverless/src/constants.ts @@ -388,3 +388,12 @@ export type AfterRenderCost = { currency: string; disclaimer: string; }; + +export const CONCAT_FOLDER_TOKEN = 'remotion-concat'; +export const MAX_FUNCTIONS_PER_RENDER = 200; +export const MINIMUM_FRAMES_PER_LAMBDA = 4; + +export const REMOTION_CONCATED_TOKEN = 'remotion-concated-token'; +export const REMOTION_FILELIST_TOKEN = 'remotion-filelist'; + +export const RENDERER_PATH_TOKEN = 'remotion-bucket'; diff --git a/packages/lambda/src/functions/helpers/create-post-render-data.ts b/packages/serverless/src/create-post-render-data.ts similarity index 74% rename from packages/lambda/src/functions/helpers/create-post-render-data.ts rename to packages/serverless/src/create-post-render-data.ts index 61c138793bb..43f8b270e64 100644 --- a/packages/lambda/src/functions/helpers/create-post-render-data.ts +++ b/packages/serverless/src/create-post-render-data.ts @@ -1,19 +1,15 @@ -import type { - CloudProvider, - EnhancedErrorInfo, - OverallRenderProgress, - PostRenderData, -} from '@remotion/serverless'; +import {calculateChunkTimes} from './calculate-chunk-times'; +import type {PostRenderData} from './constants'; +import type {OutputFileMetadata} from './find-output-file-in-bucket'; import { - OVERHEAD_TIME_PER_LAMBDA, getMostExpensiveChunks, -} from '@remotion/serverless'; -import type {RenderMetadata} from '@remotion/serverless/client'; -import {calculateChunkTimes} from '@remotion/serverless/client'; -import {estimatePrice} from '../../api/estimate-price'; -import type {AwsRegion} from '../../regions'; -import {MAX_EPHEMERAL_STORAGE_IN_MB} from '../../shared/constants'; -import type {OutputFileMetadata} from './find-output-file-in-bucket'; + OVERHEAD_TIME_PER_LAMBDA, +} from './most-expensive-chunks'; +import type {OverallRenderProgress} from './overall-render-progress'; +import type {ProviderSpecifics} from './provider-implementation'; +import type {RenderMetadata} from './render-metadata'; +import type {CloudProvider} from './types'; +import type {EnhancedErrorInfo} from './write-lambda-error'; export const createPostRenderData = ({ region, @@ -26,6 +22,7 @@ export const createPostRenderData = ({ overallProgress, timeToFinish, outputSize, + providerSpecifics, }: { region: Provider['region']; memorySizeInMb: number; @@ -37,6 +34,7 @@ export const createPostRenderData = ({ overallProgress: OverallRenderProgress; timeToFinish: number; outputSize: number; + providerSpecifics: ProviderSpecifics; }): PostRenderData => { const parsedTimings = overallProgress.timings; @@ -44,14 +42,12 @@ export const createPostRenderData = ({ .map((p) => p.rendered - p.start + OVERHEAD_TIME_PER_LAMBDA) .reduce((a, b) => a + b); - const cost = estimatePrice({ + const cost = providerSpecifics.estimatePrice({ durationInMilliseconds: estimatedBillingDurationInMilliseconds, memorySizeInMb, - region: region as AwsRegion, + region, lambdasInvoked: renderMetadata.estimatedTotalLambdaInvokations, - // We cannot determine the ephemeral storage size, so we - // overestimate the price, but will only have a miniscule effect (~0.2%) - diskSizeInMb: MAX_EPHEMERAL_STORAGE_IN_MB, + diskSizeInMb: providerSpecifics.getEphemeralStorageForPriceCalculation(), }); if (!outputFile) { @@ -98,12 +94,12 @@ export const createPostRenderData = ({ mostExpensiveFrameRanges: renderMetadata.type === 'still' ? [] - : getMostExpensiveChunks( + : getMostExpensiveChunks({ parsedTimings, - renderMetadata.framesPerLambda, - renderMetadata.frameRange[0], - renderMetadata.frameRange[1], - ), + framesPerLambda: renderMetadata.framesPerLambda, + firstFrame: renderMetadata.frameRange[0], + lastFrame: renderMetadata.frameRange[1], + }), deleteAfter: renderMetadata.deleteAfter, estimatedBillingDurationInMilliseconds, timeToCombine: timeToCombine ?? null, diff --git a/packages/lambda/src/functions/helpers/find-output-file-in-bucket.ts b/packages/serverless/src/find-output-file-in-bucket.ts similarity index 82% rename from packages/lambda/src/functions/helpers/find-output-file-in-bucket.ts rename to packages/serverless/src/find-output-file-in-bucket.ts index 424cb094143..5103012917d 100644 --- a/packages/lambda/src/functions/helpers/find-output-file-in-bucket.ts +++ b/packages/serverless/src/find-output-file-in-bucket.ts @@ -1,10 +1,8 @@ -import type {CloudProvider, ProviderSpecifics} from '@remotion/serverless'; -import { - getExpectedOutName, - type CustomCredentials, - type RenderMetadata, -} from '@remotion/serverless/client'; -import {getOutputUrlFromMetadata} from './get-output-url-from-metadata'; +import type {CustomCredentials} from './constants'; +import {getExpectedOutName} from './expected-out-name'; +import type {ProviderSpecifics} from './provider-implementation'; +import type {RenderMetadata} from './render-metadata'; +import type {CloudProvider} from './types'; export type OutputFileMetadata = { url: string; @@ -46,12 +44,12 @@ export const findOutputFileInBucket = async ({ forcePathStyle, }); return { - url: getOutputUrlFromMetadata( + url: providerSpecifics.getOutputUrl({ renderMetadata, bucketName, customCredentials, currentRegion, - ).url, + }).url, }; } catch (err) { if ((err as Error).name === 'NotFound') { diff --git a/packages/serverless/src/get-browser-instance.ts b/packages/serverless/src/get-browser-instance.ts index af84b15100c..c57e385c4f3 100644 --- a/packages/serverless/src/get-browser-instance.ts +++ b/packages/serverless/src/get-browser-instance.ts @@ -2,10 +2,13 @@ import type {ChromiumOptions, LogLevel, openBrowser} from '@remotion/renderer'; import {RenderInternals} from '@remotion/renderer'; import {VERSION} from 'remotion/version'; import type {Await} from './await'; -import type {ProviderSpecifics} from './provider-implementation'; +import type { + GetBrowserInstance, + ProviderSpecifics, +} from './provider-implementation'; import type {CloudProvider} from './types'; -type LaunchedBrowser = { +export type LaunchedBrowser = { instance: Await>; configurationString: string; }; @@ -45,7 +48,7 @@ const waitForLaunched = () => { }); }; -export const forgetBrowserEventLoop = (logLevel: LogLevel) => { +export const forgetBrowserEventLoopImplementation = (logLevel: LogLevel) => { RenderInternals.Log.info( {indent: false, logLevel}, 'Keeping browser open for next invocation', @@ -54,7 +57,9 @@ export const forgetBrowserEventLoop = (logLevel: LogLevel) => { _browserInstance?.instance.runner.deleteBrowserCaches(); }; -export const getBrowserInstance = async ({ +export const getBrowserInstanceImplementation: GetBrowserInstance = async < + Provider extends CloudProvider, +>({ logLevel, indent, chromiumOptions, @@ -117,7 +122,7 @@ export const getBrowserInstance = async ({ {indent: false, logLevel}, 'Browser disconnected or crashed.', ); - forgetBrowserEventLoop(logLevel); + providerSpecifics.forgetBrowserEventLoop(logLevel); _browserInstance?.instance?.close(true, logLevel, indent).catch((err) => { RenderInternals.Log.info( {indent: false, logLevel}, @@ -144,7 +149,7 @@ export const getBrowserInstance = async ({ _browserInstance.instance.runner.rememberEventLoop(); await _browserInstance.instance.close(true, logLevel, indent); _browserInstance = null; - return getBrowserInstance({ + return providerSpecifics.getBrowserInstance({ logLevel, indent, chromiumOptions, diff --git a/packages/serverless/src/handlers/compositions.ts b/packages/serverless/src/handlers/compositions.ts index 1e75dee82ba..8e24e508cce 100644 --- a/packages/serverless/src/handlers/compositions.ts +++ b/packages/serverless/src/handlers/compositions.ts @@ -3,10 +3,7 @@ import {VERSION} from 'remotion/version'; import {decompressInputProps} from '../compress-props'; import type {ServerlessPayload} from '../constants'; import {ServerlessRoutines} from '../constants'; -import { - forgetBrowserEventLoop, - getBrowserInstance, -} from '../get-browser-instance'; +import {} from '../get-browser-instance'; import {internalGetOrCreateBucket} from '../get-or-create-bucket'; import type {ProviderSpecifics} from '../provider-implementation'; import type {CloudProvider} from '../types'; @@ -39,7 +36,7 @@ export const compositionsHandler = async ( try { const region = providerSpecifics.getCurrentRegionInFunction(); - const browserInstancePromise = getBrowserInstance({ + const browserInstancePromise = providerSpecifics.getBrowserInstance({ logLevel: lambdaParams.logLevel, indent: false, chromiumOptions: lambdaParams.chromiumOptions, @@ -99,6 +96,6 @@ export const compositionsHandler = async ( type: 'success' as const, }); } finally { - forgetBrowserEventLoop(lambdaParams.logLevel); + providerSpecifics.forgetBrowserEventLoop(lambdaParams.logLevel); } }; diff --git a/packages/lambda/src/functions/launch.ts b/packages/serverless/src/handlers/launch.ts similarity index 91% rename from packages/lambda/src/functions/launch.ts rename to packages/serverless/src/handlers/launch.ts index 001871aff45..a9cc3f4feb2 100644 --- a/packages/lambda/src/functions/launch.ts +++ b/packages/serverless/src/handlers/launch.ts @@ -1,58 +1,50 @@ /* eslint-disable @typescript-eslint/no-use-before-define */ import type {EmittedArtifact, LogOptions} from '@remotion/renderer'; import {RenderInternals} from '@remotion/renderer'; -import type { - CloudProvider, - OverallProgressHelper, - PostRenderData, - ProviderSpecifics, -} from '@remotion/serverless'; -import { - DOCS_URL, - forgetBrowserEventLoop, - getBrowserInstance, - getTmpDirStateIfENoSp, - invokeWebhook, - makeOverallRenderProgress, - validateComposition, - validateOutname, -} from '@remotion/serverless'; -import type { - RenderMetadata, - ServerlessPayload, -} from '@remotion/serverless/client'; + +import {existsSync, mkdirSync, rmSync} from 'fs'; +import {type EventEmitter} from 'node:events'; +import {join} from 'path'; +import {VERSION} from 'remotion/version'; import { - ServerlessRoutines, - artifactName, compressInputProps, decompressInputProps, - getExpectedOutName, getNeedsToUpload, serializeOrThrow, -} from '@remotion/serverless/client'; -import {existsSync, mkdirSync, rmSync} from 'fs'; -import {type EventEmitter} from 'node:events'; -import {join} from 'path'; -import {VERSION} from 'remotion/version'; +} from '../compress-props'; +import type {PostRenderData, ServerlessPayload} from '../constants'; import { CONCAT_FOLDER_TOKEN, MAX_FUNCTIONS_PER_RENDER, -} from '../shared/constants'; + ServerlessRoutines, + artifactName, +} from '../constants'; +import {DOCS_URL} from '../docs-url'; +import {getExpectedOutName} from '../expected-out-name'; +import type {WebhookClient} from '../invoke-webhook'; +import {invokeWebhook} from '../invoke-webhook'; +import type {OverallProgressHelper} from '../overall-render-progress'; +import {makeOverallRenderProgress} from '../overall-render-progress'; +import type {ProviderSpecifics} from '../provider-implementation'; +import type {RenderMetadata} from '../render-metadata'; + +import {bestFramesPerFunctionParam} from '../best-frames-per-function-param'; +import {cleanupProps} from '../cleanup-props'; +import {findOutputFileInBucket} from '../find-output-file-in-bucket'; +import {mergeChunksAndFinishRender} from '../merge-chunks'; +import {planFrameRanges} from '../plan-frame-ranges'; +import {streamRendererFunctionWithRetry} from '../stream-renderer'; +import type {CloudProvider} from '../types'; import { validateDimension, validateDurationInFrames, validateFps, -} from '../shared/validate'; -import {validateFramesPerLambda} from '../shared/validate-frames-per-lambda'; -import {validatePrivacy} from '../shared/validate-privacy'; -import {planFrameRanges} from './chunk-optimization/plan-frame-ranges'; -import {bestFramesPerLambdaParam} from './helpers/best-frames-per-lambda-param'; -import {cleanupProps} from './helpers/cleanup-props'; -import {findOutputFileInBucket} from './helpers/find-output-file-in-bucket'; -import {mergeChunksAndFinishRender} from './helpers/merge-chunks'; -import {streamRendererFunctionWithRetry} from './helpers/stream-renderer'; -import {timer} from './helpers/timer'; -import {getWebhookClient} from './http-client'; +} from '../validate'; +import {validateComposition} from '../validate-composition'; +import {validateFramesPerFunction} from '../validate-frames-per-function'; +import {validateOutname} from '../validate-outname'; +import {validatePrivacy} from '../validate-privacy'; +import {getTmpDirStateIfENoSp} from '../write-lambda-error'; type Options = { expectedBucketOwner: string; @@ -80,7 +72,7 @@ const innerLaunchHandler = async ({ const startedDate = Date.now(); - const browserInstance = getBrowserInstance({ + const browserInstance = providerSpecifics.getBrowserInstance({ logLevel: params.logLevel, indent: false, chromiumOptions: params.chromiumOptions, @@ -173,9 +165,9 @@ const innerLaunchHandler = async ({ ); const framesPerLambda = - params.framesPerLambda ?? bestFramesPerLambdaParam(frameCount.length); + params.framesPerLambda ?? bestFramesPerFunctionParam(frameCount.length); - validateFramesPerLambda({ + validateFramesPerFunction({ framesPerLambda, durationInFrames: frameCount.length, }); @@ -348,7 +340,7 @@ const innerLaunchHandler = async ({ ); if (!params.overwrite) { - const findOutputFile = timer( + const findOutputFile = providerSpecifics.timer( 'Checking if output file already exists', params.logLevel, ); @@ -503,11 +495,17 @@ const innerLaunchHandler = async ({ type CleanupTask = () => Promise; -export const launchHandler = async ( - params: ServerlessPayload, - options: Options, - providerSpecifics: ProviderSpecifics, -): Promise<{ +export const launchHandler = async ({ + params, + options, + providerSpecifics, + client, +}: { + params: ServerlessPayload; + options: Options; + providerSpecifics: ProviderSpecifics; + client: WebhookClient; +}): Promise<{ type: 'success'; }> => { if (params.type !== ServerlessRoutines.launch) { @@ -600,7 +598,7 @@ export const launchHandler = async ( customData: params.webhook.customData ?? null, }, redirectsSoFar: 0, - client: getWebhookClient(params.webhook.url), + client, }, params.logLevel, ); @@ -704,7 +702,7 @@ export const launchHandler = async ( costs: postRenderData.cost, }, redirectsSoFar: 0, - client: getWebhookClient(params.webhook.url), + client, }, params.logLevel, ); @@ -799,7 +797,7 @@ export const launchHandler = async ( })), }, redirectsSoFar: 0, - client: getWebhookClient(params.webhook.url), + client, }, params.logLevel, ); @@ -837,6 +835,6 @@ export const launchHandler = async ( throw err; } finally { - forgetBrowserEventLoop(params.logLevel); + providerSpecifics.forgetBrowserEventLoop(params.logLevel); } }; diff --git a/packages/lambda/src/functions/renderer.ts b/packages/serverless/src/handlers/renderer.ts similarity index 90% rename from packages/lambda/src/functions/renderer.ts rename to packages/serverless/src/handlers/renderer.ts index 9ce1dc431da..a68018beb2f 100644 --- a/packages/lambda/src/functions/renderer.ts +++ b/packages/serverless/src/handlers/renderer.ts @@ -6,44 +6,38 @@ import type { } from '@remotion/renderer'; import {RenderInternals} from '@remotion/renderer'; import {NoReactAPIs} from '@remotion/renderer/pure'; -import type { - CloudProvider, - OnStream, - ProviderSpecifics, -} from '@remotion/serverless'; -import { - forgetBrowserEventLoop, - getBrowserInstance, - getTmpDirStateIfENoSp, - serializeArtifact, -} from '@remotion/serverless'; -import type {ServerlessPayload} from '@remotion/serverless/client'; -import { - ServerlessRoutines, - decompressInputProps, - truthy, -} from '@remotion/serverless/client'; + import fs from 'node:fs'; import path from 'node:path'; import {VERSION} from 'remotion/version'; -import {RENDERER_PATH_TOKEN} from '../shared/constants'; -import {isFlakyError} from '../shared/is-flaky-error'; -import {enableNodeIntrospection} from '../shared/why-is-node-running'; -import type {ObjectChunkTimingData} from './chunk-optimization/types'; import { canConcatAudioSeamlessly, canConcatVideoSeamlessly, -} from './helpers/can-concat-seamlessly'; -import {startLeakDetection} from './helpers/leak-detection'; -import {onDownloadsHelper} from './helpers/on-downloads-logger'; -import type {RequestContext} from './helpers/request-context'; -import {timer} from './helpers/timer'; +} from '../can-concat-seamlessly'; +import {decompressInputProps} from '../compress-props'; +import type {ServerlessPayload} from '../constants'; +import {RENDERER_PATH_TOKEN, ServerlessRoutines} from '../constants'; +import {startLeakDetection} from '../leak-detection'; +import {onDownloadsHelper} from '../on-downloads-helpers'; +import type {ProviderSpecifics} from '../provider-implementation'; +import {serializeArtifact} from '../serialize-artifact'; +import type {OnStream} from '../streaming/streaming'; +import {truthy} from '../truthy'; +import type {CloudProvider, ObjectChunkTimingData} from '../types'; +import {enableNodeIntrospection} from '../why-is-node-running'; +import {getTmpDirStateIfENoSp} from '../write-lambda-error'; type Options = { expectedBucketOwner: string; isWarm: boolean; }; +export type RequestContext = { + invokedFunctionArn: string; + getRemainingTimeInMillis: () => number; + awsRequestId: string; +}; + const renderHandler = async ({ params, options, @@ -87,7 +81,7 @@ const renderHandler = async ({ forcePathStyle: params.forcePathStyle, }); - const browserInstance = await getBrowserInstance({ + const browserInstance = await providerSpecifics.getBrowserInstance({ logLevel: params.logLevel, indent: false, chromiumOptions: params.chromiumOptions, @@ -343,13 +337,16 @@ const renderHandler = async ({ .catch((err) => reject(err)); }); - const streamTimer = timer( + const streamTimer = providerSpecifics.timer( 'Streaming chunk to the main function', params.logLevel, ); if (audioOutputLocation) { - const audioChunkTimer = timer('Sending audio chunk', params.logLevel); + const audioChunkTimer = providerSpecifics.timer( + 'Sending audio chunk', + params.logLevel, + ); await onStream({ type: 'audio-chunk-rendered', payload: fs.readFileSync(audioOutputLocation), @@ -358,7 +355,10 @@ const renderHandler = async ({ } if (videoOutputLocation) { - const videoChunkTimer = timer('Sending main chunk', params.logLevel); + const videoChunkTimer = providerSpecifics.timer( + 'Sending main chunk', + params.logLevel, + ); await onStream({ type: NoReactAPIs.isAudioCodec(params.codec) ? 'audio-chunk-rendered' @@ -441,13 +441,14 @@ export const rendererHandler = async ({ }; } catch (err) { if (process.env.NODE_ENV === 'test') { + // eslint-disable-next-line no-console console.log({err}); throw err; } // If this error is encountered, we can just retry as it // is a very rare error to occur - const isRetryableError = isFlakyError(err as Error); + const isRetryableError = providerSpecifics.isFlakyError(err as Error); if (isRetryableError) { shouldKeepBrowserOpen = false; } @@ -493,7 +494,7 @@ export const rendererHandler = async ({ throw err; } finally { if (shouldKeepBrowserOpen) { - forgetBrowserEventLoop(params.logLevel); + providerSpecifics.forgetBrowserEventLoop(params.logLevel); } else { RenderInternals.Log.info( {indent: false, logLevel: params.logLevel}, diff --git a/packages/lambda/src/functions/still.ts b/packages/serverless/src/handlers/still.ts similarity index 88% rename from packages/lambda/src/functions/still.ts rename to packages/serverless/src/handlers/still.ts index 6edab9fbe68..2c1427bba1d 100644 --- a/packages/lambda/src/functions/still.ts +++ b/packages/serverless/src/handlers/still.ts @@ -1,47 +1,39 @@ import type {EmittedArtifact, StillImageFormat} from '@remotion/renderer'; import {RenderInternals} from '@remotion/renderer'; -import type { - CloudProvider, - OnStream, - ReceivedArtifact, - RenderStillLambdaResponsePayload, -} from '@remotion/serverless'; -import { - forgetBrowserEventLoop, - formatCostsInfo, - getBrowserInstance, - getCredentialsFromOutName, - getTmpDirStateIfENoSp, - makeInitialOverallRenderProgress, - validateComposition, - validateOutname, - type ProviderSpecifics, -} from '@remotion/serverless'; -import type { - RenderMetadata, - ServerlessPayload, -} from '@remotion/serverless/client'; -import { - ServerlessRoutines, - artifactName, - decompressInputProps, - getExpectedOutName, - internalGetOrCreateBucket, - overallProgressKey, -} from '@remotion/serverless/client'; + import fs from 'node:fs'; import path from 'node:path'; import {NoReactInternals} from 'remotion/no-react'; import {VERSION} from 'remotion/version'; -import {estimatePrice} from '../api/estimate-price'; -import {MAX_EPHEMERAL_STORAGE_IN_MB} from '../defaults'; -import type {AwsRegion} from '../regions'; -import {cleanupSerializedInputProps} from '../shared/cleanup-serialized-input-props'; -import {isFlakyError} from '../shared/is-flaky-error'; -import {validateDownloadBehavior} from '../shared/validate-download-behavior'; -import {validatePrivacy} from '../shared/validate-privacy'; -import {getOutputUrlFromMetadata} from './helpers/get-output-url-from-metadata'; -import {onDownloadsHelper} from './helpers/on-downloads-logger'; +import {cleanupSerializedInputProps} from '../cleanup-serialized-input-props'; +import {decompressInputProps} from '../compress-props'; +import type {ServerlessPayload} from '../constants'; +import { + ServerlessRoutines, + artifactName, + overallProgressKey, +} from '../constants'; +import { + getCredentialsFromOutName, + getExpectedOutName, +} from '../expected-out-name'; +import {formatCostsInfo} from '../format-costs-info'; +import {internalGetOrCreateBucket} from '../get-or-create-bucket'; +import {onDownloadsHelper} from '../on-downloads-helpers'; +import {makeInitialOverallRenderProgress} from '../overall-render-progress'; +import type {ProviderSpecifics} from '../provider-implementation'; +import type {RenderMetadata} from '../render-metadata'; +import type {OnStream} from '../streaming/streaming'; +import type { + CloudProvider, + ReceivedArtifact, + RenderStillLambdaResponsePayload, +} from '../types'; +import {validateComposition} from '../validate-composition'; +import {validateDownloadBehavior} from '../validate-download-behavior'; +import {validateOutname} from '../validate-outname'; +import {validatePrivacy} from '../validate-privacy'; +import {getTmpDirStateIfENoSp} from '../write-lambda-error'; type Options = { params: ServerlessPayload; @@ -90,7 +82,7 @@ const innerStillHandler = async ( const start = Date.now(); - const browserInstancePromise = getBrowserInstance({ + const browserInstancePromise = providerSpecifics.getBrowserInstance({ logLevel: lambdaParams.logLevel, indent: false, chromiumOptions: lambdaParams.chromiumOptions, @@ -342,22 +334,20 @@ const innerStillHandler = async ( server.closeServer(true), ]); - const estimatedPrice = estimatePrice({ + const estimatedPrice = providerSpecifics.estimatePrice({ durationInMilliseconds: Date.now() - start + 100, memorySizeInMb: Number(process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE), - region: providerSpecifics.getCurrentRegionInFunction() as AwsRegion, + region: providerSpecifics.getCurrentRegionInFunction(), lambdasInvoked: 1, - // We cannot determine the ephemeral storage size, so we - // overestimate the price, but will only have a miniscule effect (~0.2%) - diskSizeInMb: MAX_EPHEMERAL_STORAGE_IN_MB, + diskSizeInMb: providerSpecifics.getEphemeralStorageForPriceCalculation(), }); - const {key: outKey, url} = getOutputUrlFromMetadata( + const {key: outKey, url} = providerSpecifics.getOutputUrl({ renderMetadata, bucketName, customCredentials, - providerSpecifics.getCurrentRegionInFunction(), - ); + currentRegion: providerSpecifics.getCurrentRegionInFunction(), + }); const payload: RenderStillLambdaResponsePayload = { type: 'success' as const, @@ -407,7 +397,7 @@ export const stillHandler = async ( } catch (err) { // If this error is encountered, we can just retry as it // is a very rare error to occur - const isBrowserError = isFlakyError(err as Error); + const isBrowserError = options.providerSpecifics.isFlakyError(err as Error); const willRetry = isBrowserError || params.maxRetries > 0; RenderInternals.Log.error( @@ -452,7 +442,7 @@ export const stillHandler = async ( stack: (err as Error).stack as string, }; } finally { - forgetBrowserEventLoop( + options.providerSpecifics.forgetBrowserEventLoop( options.params.type === ServerlessRoutines.still ? options.params.logLevel : 'error', diff --git a/packages/serverless/src/index.ts b/packages/serverless/src/index.ts index e55bda96607..3ba0b658a60 100644 --- a/packages/serverless/src/index.ts +++ b/packages/serverless/src/index.ts @@ -1,19 +1,22 @@ -export {compositionsHandler} from './handlers/compositions'; -export {progressHandler} from './handlers/progress'; -export {startHandler} from './handlers/start'; - export {PostRenderData, ServerlessRoutines} from './constants'; export {DOCS_URL} from './docs-url'; export {estimatePriceFromBucket} from './estimate-price-from-bucket'; export {getCredentialsFromOutName} from './expected-out-name'; export {formatCostsInfo} from './format-costs-info'; export { - forgetBrowserEventLoop, - getBrowserInstance, + forgetBrowserEventLoopImplementation, + getBrowserInstanceImplementation, } from './get-browser-instance'; +export {compositionsHandler} from './handlers/compositions'; +export {launchHandler} from './handlers/launch'; +export {progressHandler} from './handlers/progress'; +export {RequestContext, rendererHandler} from './handlers/renderer'; +export {startHandler} from './handlers/start'; +export {stillHandler} from './handlers/still'; export {infoHandler} from './info'; export {inspectErrors} from './inspect-error'; -export {WebhookPayload, invokeWebhook} from './invoke-webhook'; +export {WebhookClient, WebhookPayload, invokeWebhook} from './invoke-webhook'; +export {setCurrentRequestId, stopLeakDetection} from './leak-detection'; export { OVERHEAD_TIME_PER_LAMBDA, getMostExpensiveChunks, diff --git a/packages/serverless/src/invoke-webhook.ts b/packages/serverless/src/invoke-webhook.ts index 3417be969b3..2bfda4ae8c6 100644 --- a/packages/serverless/src/invoke-webhook.ts +++ b/packages/serverless/src/invoke-webhook.ts @@ -48,16 +48,18 @@ export type WebhookPayload = { // since the browser will display the right page. const redirectStatusCodes = [301, 302, 303, 307, 308]; +export type WebhookClient = ( + url: string | URL, + options: https.RequestOptions, + callback?: (res: http.IncomingMessage) => void, +) => http.ClientRequest; + type InvokeWebhookOptions = { payload: WebhookPayload; url: string; secret: string | null; redirectsSoFar: number; - client: ( - url: string | URL, - options: https.RequestOptions, - callback?: (res: http.IncomingMessage) => void, - ) => http.ClientRequest; + client: WebhookClient; }; function invokeWebhookRaw({ diff --git a/packages/lambda/src/functions/helpers/leak-detection.ts b/packages/serverless/src/leak-detection.ts similarity index 88% rename from packages/lambda/src/functions/helpers/leak-detection.ts rename to packages/serverless/src/leak-detection.ts index d42aa92cf5d..ddf4e3646c3 100644 --- a/packages/lambda/src/functions/helpers/leak-detection.ts +++ b/packages/serverless/src/leak-detection.ts @@ -1,5 +1,6 @@ -import type {NodeIntrospection} from '../../shared/why-is-node-running'; -import {whyIsNodeRunning} from '../../shared/why-is-node-running'; +/* eslint-disable no-console */ +import type {NodeIntrospection} from './why-is-node-running'; +import {whyIsNodeRunning} from './why-is-node-running'; type LeakTimeout = { timeout: Timer; diff --git a/packages/lambda/src/functions/helpers/merge-chunks.ts b/packages/serverless/src/merge-chunks.ts similarity index 84% rename from packages/lambda/src/functions/helpers/merge-chunks.ts rename to packages/serverless/src/merge-chunks.ts index 5bd50a3a06f..8641cea05c1 100644 --- a/packages/lambda/src/functions/helpers/merge-chunks.ts +++ b/packages/serverless/src/merge-chunks.ts @@ -1,25 +1,22 @@ import type {AudioCodec, LogLevel} from '@remotion/renderer'; -import type { - CloudProvider, - OverallProgressHelper, - PostRenderData, - ProviderSpecifics, -} from '@remotion/serverless'; -import {inspectErrors} from '@remotion/serverless'; + +import fs from 'fs'; +import {cleanupProps} from './cleanup-props'; +import {concatVideos} from './concat-videos'; import type { CustomCredentials, DownloadBehavior, + PostRenderData, Privacy, - RenderMetadata, SerializedInputProps, ServerlessCodec, -} from '@remotion/serverless/client'; -import fs from 'fs'; -import {cleanupProps} from './cleanup-props'; -import {concatVideos} from './concat-videos'; +} from './constants'; import {createPostRenderData} from './create-post-render-data'; -import {getOutputUrlFromMetadata} from './get-output-url-from-metadata'; -import {timer} from './timer'; +import {inspectErrors} from './inspect-error'; +import type {OverallProgressHelper} from './overall-render-progress'; +import type {ProviderSpecifics} from './provider-implementation'; +import type {RenderMetadata} from './render-metadata'; +import type {CloudProvider} from './types'; export const mergeChunksAndFinishRender = async < Provider extends CloudProvider, @@ -84,13 +81,14 @@ export const mergeChunksAndFinishRender = async < preferLossless: options.preferLossless, muted: options.renderMetadata.muted, metadata: options.renderMetadata.metadata, + providerSpecifics: options.providerSpecifics, }); const encodingStop = Date.now(); options.overallProgress.setTimeToCombine(encodingStop - encodingStart); const outputSize = fs.statSync(outfile).size; - const writeToBucket = timer( + const writeToBucket = options.providerSpecifics.timer( `Writing to bucket (${outputSize} bytes)`, options.logLevel, ); @@ -120,12 +118,12 @@ export const mergeChunksAndFinishRender = async < forcePathStyle: options.forcePathStyle, }); - const {url: outputUrl} = getOutputUrlFromMetadata( - options.renderMetadata, - options.bucketName, - options.customCredentials, - options.providerSpecifics.getCurrentRegionInFunction(), - ); + const {url: outputUrl} = options.providerSpecifics.getOutputUrl({ + bucketName: options.renderBucketName, + currentRegion: options.providerSpecifics.getCurrentRegionInFunction(), + customCredentials: options.customCredentials, + renderMetadata: options.renderMetadata, + }); const postRenderData = createPostRenderData({ region: options.providerSpecifics.getCurrentRegionInFunction(), @@ -140,6 +138,7 @@ export const mergeChunksAndFinishRender = async < timeToCombine: encodingStop - encodingStart, overallProgress: options.overallProgress.get(), timeToFinish: Date.now() - options.startTime, + providerSpecifics: options.providerSpecifics, }); options.overallProgress.setPostRenderData(postRenderData); diff --git a/packages/serverless/src/most-expensive-chunks.ts b/packages/serverless/src/most-expensive-chunks.ts index 8ecd66288b7..412425fd8ab 100644 --- a/packages/serverless/src/most-expensive-chunks.ts +++ b/packages/serverless/src/most-expensive-chunks.ts @@ -8,12 +8,17 @@ export type ExpensiveChunk = { timeInMilliseconds: number; }; -export const getMostExpensiveChunks = ( - parsedTimings: ParsedTiming[], - framesPerLambda: number, - firstFrame: number, - lastFrame: number, -): ExpensiveChunk[] => { +export const getMostExpensiveChunks = ({ + parsedTimings, + framesPerLambda, + firstFrame, + lastFrame, +}: { + parsedTimings: ParsedTiming[]; + framesPerLambda: number; + firstFrame: number; + lastFrame: number; +}): ExpensiveChunk[] => { const mostExpensiveChunks = parsedTimings .slice(0) .sort((a, b) => { diff --git a/packages/lambda/src/functions/helpers/on-downloads-logger.ts b/packages/serverless/src/on-downloads-helpers.ts similarity index 100% rename from packages/lambda/src/functions/helpers/on-downloads-logger.ts rename to packages/serverless/src/on-downloads-helpers.ts diff --git a/packages/lambda/src/functions/chunk-optimization/plan-frame-ranges.ts b/packages/serverless/src/plan-frame-ranges.ts similarity index 100% rename from packages/lambda/src/functions/chunk-optimization/plan-frame-ranges.ts rename to packages/serverless/src/plan-frame-ranges.ts diff --git a/packages/serverless/src/provider-implementation.ts b/packages/serverless/src/provider-implementation.ts index 1dadee0d705..7dee2e315ad 100644 --- a/packages/serverless/src/provider-implementation.ts +++ b/packages/serverless/src/provider-implementation.ts @@ -1,4 +1,8 @@ -import type {EmittedArtifact} from '@remotion/renderer'; +import type { + ChromiumOptions, + EmittedArtifact, + LogLevel, +} from '@remotion/renderer'; import type {Readable} from 'stream'; import type { CustomCredentials, @@ -6,7 +10,9 @@ import type { Privacy, ServerlessRoutines, } from './constants'; +import type {LaunchedBrowser} from './get-browser-instance'; import type {GetFolderFiles} from './get-files-in-folder'; +import type {RenderMetadata} from './render-metadata'; import type {ServerlessReturnValues} from './return-values'; import type {OnMessage} from './streaming/streaming'; import type { @@ -129,6 +135,13 @@ export type MakeArtifactWithDetails = (params: { artifact: EmittedArtifact; }) => ReceivedArtifact; +export type DebuggingTimer = ( + label: string, + logLevel: LogLevel, +) => { + end: () => void; +}; + export type CallFunctionAsync = < T extends ServerlessRoutines, >({ @@ -191,6 +204,27 @@ export type GetLoggingUrlForMethod = (options: { renderId: string; }) => string; +export type GetOutputUrl = (options: { + renderMetadata: RenderMetadata; + bucketName: string; + customCredentials: CustomCredentials | null; + currentRegion: Provider['region']; +}) => {url: string; key: string}; + +export type GetBrowserInstance = ({ + logLevel, + indent, + chromiumOptions, + providerSpecifics, +}: { + logLevel: LogLevel; + indent: boolean; + chromiumOptions: ChromiumOptions; + providerSpecifics: ProviderSpecifics; +}) => Promise; + +export type ForgetBrowserEventLoop = (logLevel: LogLevel) => void; + export type ProviderSpecifics = { getChromiumPath: () => string | null; getCurrentRegionInFunction: () => Provider['region']; @@ -217,4 +251,9 @@ export type ProviderSpecifics = { getLoggingUrlForRendererFunction: GetLoggingUrlForRendererFunction; getLoggingUrlForMethod: GetLoggingUrlForMethod; getEphemeralStorageForPriceCalculation: () => number; + timer: DebuggingTimer; + getOutputUrl: GetOutputUrl; + isFlakyError: (err: Error) => boolean; + getBrowserInstance: GetBrowserInstance; + forgetBrowserEventLoop: ForgetBrowserEventLoop; }; diff --git a/packages/lambda/src/shared/stackback.ts b/packages/serverless/src/stackback.ts similarity index 100% rename from packages/lambda/src/shared/stackback.ts rename to packages/serverless/src/stackback.ts diff --git a/packages/lambda/src/functions/helpers/stream-renderer.ts b/packages/serverless/src/stream-renderer.ts similarity index 93% rename from packages/lambda/src/functions/helpers/stream-renderer.ts rename to packages/serverless/src/stream-renderer.ts index 794155f8a98..3e2fbaecaba 100644 --- a/packages/lambda/src/functions/helpers/stream-renderer.ts +++ b/packages/serverless/src/stream-renderer.ts @@ -1,18 +1,15 @@ import type {EmittedArtifact, LogLevel} from '@remotion/renderer'; import {RenderInternals} from '@remotion/renderer'; -import type { - CloudProvider, - OnMessage, - OverallProgressHelper, -} from '@remotion/serverless'; -import { - deserializeArtifact, - type ProviderSpecifics, -} from '@remotion/serverless'; -import type {ServerlessPayload} from '@remotion/serverless/client'; -import {ServerlessRoutines} from '@remotion/serverless/client'; + import {writeFileSync} from 'fs'; import {join} from 'path'; +import type {ServerlessPayload} from './constants'; +import {ServerlessRoutines} from './constants'; +import type {OverallProgressHelper} from './overall-render-progress'; +import type {ProviderSpecifics} from './provider-implementation'; +import {deserializeArtifact} from './serialize-artifact'; +import type {OnMessage} from './streaming/streaming'; +import type {CloudProvider} from './types'; type StreamRendererResponse = | { diff --git a/packages/serverless/src/test/best-frames-per-function.test.ts b/packages/serverless/src/test/best-frames-per-function.test.ts new file mode 100644 index 00000000000..60d96b23d10 --- /dev/null +++ b/packages/serverless/src/test/best-frames-per-function.test.ts @@ -0,0 +1,15 @@ +import {expect, test} from 'bun:test'; +import {bestFramesPerFunctionParam} from '../best-frames-per-function-param'; + +test('Get reasonable framesPerLambda defaults', () => { + expect(bestFramesPerFunctionParam(20)).toEqual(20); + expect(bestFramesPerFunctionParam(21)).toEqual(11); + expect(bestFramesPerFunctionParam(100)).toEqual(20); + expect(bestFramesPerFunctionParam(2000)).toEqual(24); + expect(bestFramesPerFunctionParam(4000)).toEqual(44); + expect(bestFramesPerFunctionParam(8000)).toEqual(74); + expect(bestFramesPerFunctionParam(10000)).toEqual(86); + expect(bestFramesPerFunctionParam(14000)).toEqual(105); + expect(bestFramesPerFunctionParam(18000)).toEqual(120); + expect(bestFramesPerFunctionParam(216000)).toEqual(1440); +}); diff --git a/packages/serverless/src/test/most-expensive.test.ts b/packages/serverless/src/test/most-expensive.test.ts index aa03d114bf0..bf3ef7b3397 100644 --- a/packages/serverless/src/test/most-expensive.test.ts +++ b/packages/serverless/src/test/most-expensive.test.ts @@ -2,8 +2,8 @@ import {expect, test} from 'bun:test'; import {getMostExpensiveChunks} from '../most-expensive-chunks'; test('Should calculate most expensive chunks', () => { - const most = getMostExpensiveChunks( - [ + const most = getMostExpensiveChunks({ + parsedTimings: [ { chunk: 0, rendered: 1000, @@ -40,10 +40,10 @@ test('Should calculate most expensive chunks', () => { start: 2000, }, ], - 10, - 0, - 99, - ); + framesPerLambda: 10, + firstFrame: 0, + lastFrame: 99, + }); expect(most).toEqual([ {timeInMilliseconds: 98000, chunk: 4, frameRange: [40, 49]}, @@ -58,8 +58,8 @@ test('Render starting from frame 10 should have correct offset', () => { const framesPerLambda = 10; const firstFrame = 10; const lastFrame = 99; - const most = getMostExpensiveChunks( - [ + const most = getMostExpensiveChunks({ + parsedTimings: [ { chunk: 0, rendered: 1000, @@ -99,7 +99,7 @@ test('Render starting from frame 10 should have correct offset', () => { framesPerLambda, firstFrame, lastFrame, - ); + }); expect(most).toEqual([ {timeInMilliseconds: 98000, chunk: 4, frameRange: [50, 59]}, @@ -114,8 +114,8 @@ test('Render starting from frame 10 and last chunk in most expensive should be c const framesPerLambda = 10; const firstFrame = 10; const lastFrame = 79; - const most = getMostExpensiveChunks( - [ + const most = getMostExpensiveChunks({ + parsedTimings: [ { chunk: 0, rendered: 1000, @@ -155,7 +155,7 @@ test('Render starting from frame 10 and last chunk in most expensive should be c framesPerLambda, firstFrame, lastFrame, - ); + }); expect(most).toEqual([ {timeInMilliseconds: 123000, chunk: 6, frameRange: [70, 79]}, diff --git a/packages/lambda/src/test/unit/plan-frame-ranges.test.ts b/packages/serverless/src/test/plan-frame-ranges.test.ts similarity index 86% rename from packages/lambda/src/test/unit/plan-frame-ranges.test.ts rename to packages/serverless/src/test/plan-frame-ranges.test.ts index a2327605389..831c96cbc07 100644 --- a/packages/lambda/src/test/unit/plan-frame-ranges.test.ts +++ b/packages/serverless/src/test/plan-frame-ranges.test.ts @@ -1,5 +1,5 @@ -import {expect, test} from 'vitest'; -import {planFrameRanges} from '../../functions/chunk-optimization/plan-frame-ranges'; +import {expect, test} from 'bun:test'; +import {planFrameRanges} from '../plan-frame-ranges'; test('Plan frame ranges should respect everyNthFrame', () => { const planned = planFrameRanges({ diff --git a/packages/serverless/src/types.ts b/packages/serverless/src/types.ts index 5e25269f793..eb32d098e14 100644 --- a/packages/serverless/src/types.ts +++ b/packages/serverless/src/types.ts @@ -60,3 +60,12 @@ export type CallFunctionOptions< region: Provider['region']; timeoutInTest: number; }; + +export type ObjectChunkTimingData = { + chunk: number; + frameRange: [number, number]; + startDate: number; + timings: { + [key: number]: number; + }; +}; diff --git a/packages/lambda/src/shared/validate-download-behavior.ts b/packages/serverless/src/validate-download-behavior.ts similarity index 91% rename from packages/lambda/src/shared/validate-download-behavior.ts rename to packages/serverless/src/validate-download-behavior.ts index e05d5ccd660..66c53b7f779 100644 --- a/packages/lambda/src/shared/validate-download-behavior.ts +++ b/packages/serverless/src/validate-download-behavior.ts @@ -1,4 +1,4 @@ -import type {DownloadBehavior} from '@remotion/serverless/client'; +import type {DownloadBehavior} from './constants'; export const validateDownloadBehavior = (downloadBehavior: unknown) => { if (downloadBehavior === null || downloadBehavior === undefined) { diff --git a/packages/lambda/src/shared/validate-frames-per-lambda.ts b/packages/serverless/src/validate-frames-per-function.ts similarity index 96% rename from packages/lambda/src/shared/validate-frames-per-lambda.ts rename to packages/serverless/src/validate-frames-per-function.ts index cd04b76dd68..6f34e6afe85 100644 --- a/packages/lambda/src/shared/validate-frames-per-lambda.ts +++ b/packages/serverless/src/validate-frames-per-function.ts @@ -1,6 +1,6 @@ import {MINIMUM_FRAMES_PER_LAMBDA} from './constants'; -export const validateFramesPerLambda = ({ +export const validateFramesPerFunction = ({ framesPerLambda, durationInFrames, }: { diff --git a/packages/lambda/src/shared/validate-privacy.ts b/packages/serverless/src/validate-privacy.ts similarity index 89% rename from packages/lambda/src/shared/validate-privacy.ts rename to packages/serverless/src/validate-privacy.ts index a53f976c83d..222240edb5e 100644 --- a/packages/lambda/src/shared/validate-privacy.ts +++ b/packages/serverless/src/validate-privacy.ts @@ -1,4 +1,4 @@ -import type {Privacy} from '@remotion/serverless/client'; +import type {Privacy} from './constants'; export function validatePrivacy( privacy: unknown, diff --git a/packages/lambda/src/shared/validate.ts b/packages/serverless/src/validate.ts similarity index 100% rename from packages/lambda/src/shared/validate.ts rename to packages/serverless/src/validate.ts diff --git a/packages/lambda/src/shared/why-is-node-running.ts b/packages/serverless/src/why-is-node-running.ts similarity index 98% rename from packages/lambda/src/shared/why-is-node-running.ts rename to packages/serverless/src/why-is-node-running.ts index 8a370ab316d..95a54b61e7c 100644 --- a/packages/lambda/src/shared/why-is-node-running.ts +++ b/packages/serverless/src/why-is-node-running.ts @@ -1,3 +1,4 @@ +/* eslint-disable no-console */ import asyncHooks from 'async_hooks'; import fs from 'fs'; import {sep} from 'path'; From 2f2aae82aae3a730dcb8613868558742e90da7a8 Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Mon, 6 Jan 2025 11:28:14 +0100 Subject: [PATCH 11/14] nice --- .../src/lambda/lambda-browser-bundle.test.ts | 3 ++- packages/lambda/build.ts | 19 +++++++++++-------- packages/lambda/src/api/get-aws-client.ts | 2 +- .../src/api/get-compositions-on-lambda.ts | 2 +- .../lambda/src/api/make-lambda-payload.ts | 3 ++- packages/lambda/src/cli/args.ts | 3 ++- .../src/cli/commands/quotas/increase.ts | 2 +- .../src/cli/helpers/find-function-name.ts | 2 +- packages/lambda/src/cli/index.ts | 3 ++- .../lambda/src/shared/check-credentials.ts | 3 +-- .../lambda/src/shared/convert-to-serve-url.ts | 2 +- packages/lambda/src/shared/get-aws-urls.ts | 2 +- .../lambda/src/shared/validate-bucketname.ts | 2 +- .../lambda/src/test/mock-implementation.ts | 2 +- packages/serverless/src/client.ts | 1 + packages/serverless/src/index.ts | 1 - .../src/test/dont-contain-forbidden.test.ts | 14 ++++++++++++++ 17 files changed, 43 insertions(+), 23 deletions(-) create mode 100644 packages/serverless/src/test/dont-contain-forbidden.test.ts diff --git a/packages/it-tests/src/lambda/lambda-browser-bundle.test.ts b/packages/it-tests/src/lambda/lambda-browser-bundle.test.ts index 3da8ab3a5b3..5ae2a63d91c 100644 --- a/packages/it-tests/src/lambda/lambda-browser-bundle.test.ts +++ b/packages/it-tests/src/lambda/lambda-browser-bundle.test.ts @@ -43,7 +43,8 @@ describe('Should be able to bundle @remotion/lambda/client with ESBuild', () => test('Bundle should not include Renderer', async () => { const file = await fs.promises.readFile(outfile, 'utf-8'); - expect(file).not.toContain('@remotion/renderer'); + expect(file.includes('"@remotion/renderer"')).toBe(false); + expect(file.includes("'@remotion/renderer'")).toBe(false); }); test('Should be able to delete it', () => { diff --git a/packages/lambda/build.ts b/packages/lambda/build.ts index 8c36c1cfdbe..d55aeb5ea0f 100644 --- a/packages/lambda/build.ts +++ b/packages/lambda/build.ts @@ -1,4 +1,3 @@ -import {BundlerInternals} from '@remotion/bundler'; import {dir} from '@remotion/compositor-linux-arm64-gnu'; import fs, {cpSync, readdirSync} from 'node:fs'; import path from 'node:path'; @@ -18,15 +17,19 @@ const template = require.resolve( path.join(__dirname, 'src', 'functions', 'index'), ); -await BundlerInternals.esbuild.build({ - platform: 'node', - target: 'node16', - bundle: true, - outfile, - entryPoints: [template], - treeShaking: true, +const {outputs, success, logs} = await Bun.build({ + target: 'node', + minify: true, + entrypoints: [template], external: [], }); +if (!success) { + console.error(logs); + process.exit(1); +} + +const text = await outputs[0].text(); +await Bun.write(outfile, text); const filesInCwd = readdirSync(dir); const filesToCopy = filesInCwd.filter( diff --git a/packages/lambda/src/api/get-aws-client.ts b/packages/lambda/src/api/get-aws-client.ts index d05887e4aba..5112f8e9eb1 100644 --- a/packages/lambda/src/api/get-aws-client.ts +++ b/packages/lambda/src/api/get-aws-client.ts @@ -5,8 +5,8 @@ import * as S3SDK from '@aws-sdk/client-s3'; import * as ServiceQuotasSDK from '@aws-sdk/client-service-quotas'; import * as StsSdk from '@aws-sdk/client-sts'; import type {CustomCredentials} from '@remotion/serverless/client'; -import type {AwsRegion} from '../client'; import type {AwsProvider} from '../functions/aws-implementation'; +import type {AwsRegion} from '../regions'; import { getServiceClient, type ServiceMapping, diff --git a/packages/lambda/src/api/get-compositions-on-lambda.ts b/packages/lambda/src/api/get-compositions-on-lambda.ts index 8a79104cd6f..56de2b353bd 100644 --- a/packages/lambda/src/api/get-compositions-on-lambda.ts +++ b/packages/lambda/src/api/get-compositions-on-lambda.ts @@ -8,8 +8,8 @@ import { } from '@remotion/serverless/client'; import type {VideoConfig} from 'remotion/no-react'; import {VERSION} from 'remotion/version'; -import type {AwsRegion} from '../client'; import {awsImplementation} from '../functions/aws-implementation'; +import type {AwsRegion} from '../regions'; export type GetCompositionsOnLambdaInput = { chromiumOptions?: ChromiumOptions; diff --git a/packages/lambda/src/api/make-lambda-payload.ts b/packages/lambda/src/api/make-lambda-payload.ts index 81e6cd59434..8afd2aae11e 100644 --- a/packages/lambda/src/api/make-lambda-payload.ts +++ b/packages/lambda/src/api/make-lambda-payload.ts @@ -30,12 +30,13 @@ import { validateFramesPerFunction, } from '@remotion/serverless/client'; import {VERSION} from 'remotion/version'; -import type {AwsRegion, DeleteAfter} from '../client'; +import type {DeleteAfter} from '../client'; import type {AwsProvider} from '../functions/aws-implementation'; import {awsImplementation} from '../functions/aws-implementation'; import {validateWebhook} from '@remotion/serverless/client'; import {NoReactInternals} from 'remotion/no-react'; +import type {AwsRegion} from '../regions'; import {validateLambdaCodec} from '../shared/validate-lambda-codec'; import {validateServeUrl} from '../shared/validate-serveurl'; import type {GetRenderProgressInput} from './get-render-progress'; diff --git a/packages/lambda/src/cli/args.ts b/packages/lambda/src/cli/args.ts index 545b8536957..26e1037149a 100644 --- a/packages/lambda/src/cli/args.ts +++ b/packages/lambda/src/cli/args.ts @@ -1,8 +1,9 @@ import {CliInternals} from '@remotion/cli'; import type {BrowserSafeApis} from '@remotion/renderer/client'; -import type {AwsRegion, DeleteAfter} from '../client'; +import type {DeleteAfter} from '../client'; import type {Privacy} from '@remotion/serverless/client'; +import {AwsRegion} from '../regions'; import type {RuntimePreference} from '../shared/get-layers'; type LambdaCommandLineOptions = { diff --git a/packages/lambda/src/cli/commands/quotas/increase.ts b/packages/lambda/src/cli/commands/quotas/increase.ts index 65be03f7036..37d3e2c9725 100644 --- a/packages/lambda/src/cli/commands/quotas/increase.ts +++ b/packages/lambda/src/cli/commands/quotas/increase.ts @@ -7,8 +7,8 @@ import { import type {LogLevel, LogOptions} from '@remotion/renderer'; import {exit} from 'node:process'; import {QUOTAS_COMMAND} from '.'; -import type {AwsRegion} from '../../../client'; import {BINARY_NAME, LAMBDA_CONCURRENCY_LIMIT_QUOTA} from '../../../defaults'; +import {AwsRegion} from '../../../regions'; import {getServiceQuotasClient} from '../../../shared/aws-clients'; import {forceFlagProvided} from '../../args'; import {getAwsRegion} from '../../get-aws-region'; diff --git a/packages/lambda/src/cli/helpers/find-function-name.ts b/packages/lambda/src/cli/helpers/find-function-name.ts index 70d32c43c3d..59109e84408 100644 --- a/packages/lambda/src/cli/helpers/find-function-name.ts +++ b/packages/lambda/src/cli/helpers/find-function-name.ts @@ -1,7 +1,7 @@ import type {LogLevel, LogOptions} from '@remotion/renderer'; import {VERSION} from 'remotion/version'; import {getFunctions} from '../../api/get-functions'; -import {speculateFunctionName} from '../../client'; +import {speculateFunctionName} from '../../api/speculate-function-name'; import {BINARY_NAME} from '../../shared/constants'; import {parsedLambdaCli} from '../args'; import {FUNCTIONS_COMMAND} from '../commands/functions'; diff --git a/packages/lambda/src/cli/index.ts b/packages/lambda/src/cli/index.ts index a03ba198d1f..f36a44e22a2 100644 --- a/packages/lambda/src/cli/index.ts +++ b/packages/lambda/src/cli/index.ts @@ -1,7 +1,8 @@ import {CliInternals} from '@remotion/cli'; import type {LogLevel} from '@remotion/renderer'; import {RenderInternals} from '@remotion/renderer'; -import {DOCS_URL, type ProviderSpecifics} from '@remotion/serverless'; +import {type ProviderSpecifics} from '@remotion/serverless'; +import {DOCS_URL} from '@remotion/serverless/client'; import {ROLE_NAME} from '../api/iam-validation/suggested-policy'; import {BINARY_NAME} from '../defaults'; import type {AwsProvider} from '../functions/aws-implementation'; diff --git a/packages/lambda/src/shared/check-credentials.ts b/packages/lambda/src/shared/check-credentials.ts index c95698f9edc..1df5aedb810 100644 --- a/packages/lambda/src/shared/check-credentials.ts +++ b/packages/lambda/src/shared/check-credentials.ts @@ -1,5 +1,4 @@ -import {DOCS_URL} from '@remotion/serverless'; -import {truthy} from '@remotion/serverless/client'; +import {DOCS_URL, truthy} from '@remotion/serverless/client'; import {getIsCli} from '../cli/is-cli'; import {isLikelyToHaveAwsProfile} from './is-likely-to-have-aws-profile'; diff --git a/packages/lambda/src/shared/convert-to-serve-url.ts b/packages/lambda/src/shared/convert-to-serve-url.ts index d1180a74f67..b3f0b242be9 100644 --- a/packages/lambda/src/shared/convert-to-serve-url.ts +++ b/packages/lambda/src/shared/convert-to-serve-url.ts @@ -1,4 +1,4 @@ -import {DOCS_URL} from '@remotion/serverless'; +import {DOCS_URL} from '@remotion/serverless/client'; import type {AwsRegion} from '../regions'; export const convertToServeUrlImplementation = ({ diff --git a/packages/lambda/src/shared/get-aws-urls.ts b/packages/lambda/src/shared/get-aws-urls.ts index f7bc2795ecb..91bd482aa3a 100644 --- a/packages/lambda/src/shared/get-aws-urls.ts +++ b/packages/lambda/src/shared/get-aws-urls.ts @@ -1,7 +1,7 @@ import type {GetLoggingUrlForRendererFunction} from '@remotion/serverless'; import type {ServerlessRoutines} from '@remotion/serverless/client'; -import type {AwsRegion} from '../client'; import type {AwsProvider} from '../functions/aws-implementation'; +import type {AwsRegion} from '../regions'; import {encodeAwsUrlParams} from './encode-aws-url-params'; const cloudWatchUrlWithQuery = ({ diff --git a/packages/lambda/src/shared/validate-bucketname.ts b/packages/lambda/src/shared/validate-bucketname.ts index 7b6500bfe4d..fbbfb15fb05 100644 --- a/packages/lambda/src/shared/validate-bucketname.ts +++ b/packages/lambda/src/shared/validate-bucketname.ts @@ -1,5 +1,5 @@ import {REMOTION_BUCKET_PREFIX} from '@remotion/serverless/client'; -import type {AwsRegion} from '../client'; +import type {AwsRegion} from '../regions'; import {AWS_REGIONS} from '../regions'; export const parseBucketName = ( diff --git a/packages/lambda/src/test/mock-implementation.ts b/packages/lambda/src/test/mock-implementation.ts index 7a7b0b61e0b..f097dcd26dc 100644 --- a/packages/lambda/src/test/mock-implementation.ts +++ b/packages/lambda/src/test/mock-implementation.ts @@ -2,7 +2,7 @@ import {openBrowser} from '@remotion/renderer'; import type {GetBrowserInstance, ProviderSpecifics} from '@remotion/serverless'; import {Readable} from 'stream'; import {estimatePrice} from '../api/estimate-price'; -import {speculateFunctionName} from '../client'; +import {speculateFunctionName} from '../api/speculate-function-name'; import {MAX_EPHEMERAL_STORAGE_IN_MB} from '../defaults'; import type {AwsProvider} from '../functions/aws-implementation'; import {convertToServeUrlImplementation} from '../shared/convert-to-serve-url'; diff --git a/packages/serverless/src/client.ts b/packages/serverless/src/client.ts index d0cd97e53b1..76d6f93d371 100644 --- a/packages/serverless/src/client.ts +++ b/packages/serverless/src/client.ts @@ -48,6 +48,7 @@ export { export {calculateChunkTimes} from './calculate-chunk-times'; export {MAX_FUNCTIONS_PER_RENDER} from './constants'; +export {DOCS_URL} from './docs-url'; export {getExpectedOutName} from './expected-out-name'; export {FileNameAndSize, GetFolderFiles} from './get-files-in-folder'; export {getOverallProgressFromStorage} from './get-overall-progress-from-storage'; diff --git a/packages/serverless/src/index.ts b/packages/serverless/src/index.ts index 3ba0b658a60..88b8d5ef723 100644 --- a/packages/serverless/src/index.ts +++ b/packages/serverless/src/index.ts @@ -1,5 +1,4 @@ export {PostRenderData, ServerlessRoutines} from './constants'; -export {DOCS_URL} from './docs-url'; export {estimatePriceFromBucket} from './estimate-price-from-bucket'; export {getCredentialsFromOutName} from './expected-out-name'; export {formatCostsInfo} from './format-costs-info'; diff --git a/packages/serverless/src/test/dont-contain-forbidden.test.ts b/packages/serverless/src/test/dont-contain-forbidden.test.ts new file mode 100644 index 00000000000..d51f0639a30 --- /dev/null +++ b/packages/serverless/src/test/dont-contain-forbidden.test.ts @@ -0,0 +1,14 @@ +import {expect, test} from 'bun:test'; +import path from 'path'; + +test('Should not contain forbidden imports', async () => { + const output = await Bun.build({ + external: ['@remotion/renderer', 'react/jsx-runtime'], + entrypoints: [path.join(__filename, '..', '..', 'client.ts')], + }); + expect(output.outputs.length).toBe(1); + const text = await output.outputs[0].text(); + expect(text).not.toContain('jsx-runtime'); + expect(text).not.toContain('"@remotion/renderer"'); + expect(text).not.toContain("'@remotion/renderer'"); +}); From 72f593b84be8d7486a9ccfadef8215bd3aa8e418 Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Mon, 6 Jan 2025 11:49:01 +0100 Subject: [PATCH 12/14] looks good now --- packages/lambda/build.ts | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/packages/lambda/build.ts b/packages/lambda/build.ts index d55aeb5ea0f..8c36c1cfdbe 100644 --- a/packages/lambda/build.ts +++ b/packages/lambda/build.ts @@ -1,3 +1,4 @@ +import {BundlerInternals} from '@remotion/bundler'; import {dir} from '@remotion/compositor-linux-arm64-gnu'; import fs, {cpSync, readdirSync} from 'node:fs'; import path from 'node:path'; @@ -17,19 +18,15 @@ const template = require.resolve( path.join(__dirname, 'src', 'functions', 'index'), ); -const {outputs, success, logs} = await Bun.build({ - target: 'node', - minify: true, - entrypoints: [template], +await BundlerInternals.esbuild.build({ + platform: 'node', + target: 'node16', + bundle: true, + outfile, + entryPoints: [template], + treeShaking: true, external: [], }); -if (!success) { - console.error(logs); - process.exit(1); -} - -const text = await outputs[0].text(); -await Bun.write(outfile, text); const filesInCwd = readdirSync(dir); const filesToCopy = filesInCwd.filter( From 8ff3b5b0292c6df28d246d26b9178864b33c8c8a Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Mon, 6 Jan 2025 15:55:33 +0100 Subject: [PATCH 13/14] split serverless and server provider --- .../lambda/src/api/get-render-progress.ts | 3 +- .../src/functions/aws-implementation.ts | 10 +---- .../functions/aws-server-implementation.ts | 12 +++++ packages/lambda/src/functions/index.ts | 36 ++++++++++++--- .../lambda/src/test/mock-implementation.ts | 13 ++++-- packages/lambda/src/test/mocks/aws-clients.ts | 45 +++++++++++-------- packages/serverless/src/client.ts | 1 + .../serverless/src/get-browser-instance.ts | 8 +++- .../serverless/src/handlers/compositions.ts | 11 +++-- packages/serverless/src/handlers/launch.ts | 15 +++++-- packages/serverless/src/handlers/renderer.ts | 15 +++++-- packages/serverless/src/handlers/still.ts | 12 +++-- packages/serverless/src/index.ts | 1 - .../serverless/src/provider-implementation.ts | 11 +++-- 14 files changed, 135 insertions(+), 58 deletions(-) create mode 100644 packages/lambda/src/functions/aws-server-implementation.ts diff --git a/packages/lambda/src/api/get-render-progress.ts b/packages/lambda/src/api/get-render-progress.ts index d54045dfe1b..a027e0d60f5 100644 --- a/packages/lambda/src/api/get-render-progress.ts +++ b/packages/lambda/src/api/get-render-progress.ts @@ -1,7 +1,6 @@ import type {LogLevel} from '@remotion/renderer'; -import {getProgress} from '@remotion/serverless'; import type {CustomCredentials} from '@remotion/serverless/client'; -import {ServerlessRoutines} from '@remotion/serverless/client'; +import {getProgress, ServerlessRoutines} from '@remotion/serverless/client'; import { awsImplementation, type AwsProvider, diff --git a/packages/lambda/src/functions/aws-implementation.ts b/packages/lambda/src/functions/aws-implementation.ts index 392e5cfbe65..e919f97e537 100644 --- a/packages/lambda/src/functions/aws-implementation.ts +++ b/packages/lambda/src/functions/aws-implementation.ts @@ -1,8 +1,4 @@ -import { - forgetBrowserEventLoopImplementation, - getBrowserInstanceImplementation, - type ProviderSpecifics, -} from '@remotion/serverless'; +import type {ProviderSpecifics} from '@remotion/serverless'; import {expiryDays} from '@remotion/serverless/client'; import {EventEmitter} from 'node:events'; import {bucketExistsInRegionImplementation} from '../api/bucket-exists'; @@ -31,7 +27,6 @@ import {getCurrentRegionInFunctionImplementation} from './helpers/get-current-re import {getFolderFiles} from './helpers/get-folder-files'; import {getOutputUrlFromMetadata} from './helpers/get-output-url-from-metadata'; import {makeAwsArtifact} from './helpers/make-aws-artifact'; -import {timer} from './helpers/timer'; if ( /^AWS_Lambda_nodejs(?:18|20)[.]x$/.test( @@ -127,7 +122,4 @@ export const awsImplementation: ProviderSpecifics = { getLoggingUrlForRendererFunction: getCloudwatchRendererUrl, isFlakyError, getOutputUrl: getOutputUrlFromMetadata, - timer, - forgetBrowserEventLoop: forgetBrowserEventLoopImplementation, - getBrowserInstance: getBrowserInstanceImplementation, }; diff --git a/packages/lambda/src/functions/aws-server-implementation.ts b/packages/lambda/src/functions/aws-server-implementation.ts new file mode 100644 index 00000000000..da7531c63e8 --- /dev/null +++ b/packages/lambda/src/functions/aws-server-implementation.ts @@ -0,0 +1,12 @@ +import type {ServerProviderSpecifics} from '@remotion/serverless'; +import { + forgetBrowserEventLoopImplementation, + getBrowserInstanceImplementation, +} from '@remotion/serverless'; +import {timer} from './helpers/timer'; + +export const serverAwsImplementation: ServerProviderSpecifics = { + forgetBrowserEventLoop: forgetBrowserEventLoopImplementation, + getBrowserInstance: getBrowserInstanceImplementation, + timer, +}; diff --git a/packages/lambda/src/functions/index.ts b/packages/lambda/src/functions/index.ts index 055205f6002..e5dff89388c 100644 --- a/packages/lambda/src/functions/index.ts +++ b/packages/lambda/src/functions/index.ts @@ -6,6 +6,7 @@ import type { RequestContext, ResponseStream, ResponseStreamWriter, + ServerProviderSpecifics, StreamingPayload, } from '@remotion/serverless'; import { @@ -28,6 +29,7 @@ import { import {COMMAND_NOT_FOUND} from '../shared/constants'; import type {AwsProvider} from './aws-implementation'; import {awsImplementation} from './aws-implementation'; +import {serverAwsImplementation} from './aws-server-implementation'; import {deleteTmpDir} from './helpers/clean-tmpdir'; import {getWarm, setWarm} from './helpers/is-warm'; import {generateRandomHashWithLifeCycleRule} from './helpers/lifecycle'; @@ -40,11 +42,13 @@ const innerHandler = async ({ responseWriter, context, providerSpecifics, + serverProviderSpecifics, }: { params: ServerlessPayload; responseWriter: ResponseStreamWriter; context: RequestContext; providerSpecifics: ProviderSpecifics; + serverProviderSpecifics: ServerProviderSpecifics; }): Promise => { setCurrentRequestId(context.awsRequestId); process.env.__RESERVED_IS_INSIDE_REMOTION_LAMBDA = 'true'; @@ -129,6 +133,7 @@ const innerHandler = async ({ onStream, timeoutInMilliseconds, providerSpecifics, + serverProviderSpecifics, }) .then((r) => { resolve(r); @@ -199,6 +204,7 @@ const innerHandler = async ({ }, providerSpecifics, client: getWebhookClient(params.webhook?.url ?? 'http://localhost:3000'), + serverProviderSpecifics, }); await responseWriter.write(Buffer.from(JSON.stringify(response))); @@ -274,6 +280,7 @@ const innerHandler = async ({ }, requestContext: context, providerSpecifics, + serverProviderSpecifics, }) .then((res) => { resolve(res); @@ -322,6 +329,7 @@ const innerHandler = async ({ expectedBucketOwner: currentUserId, }, providerSpecifics, + serverProviderSpecifics, ); await responseWriter.write(Buffer.from(JSON.stringify(response))); @@ -333,18 +341,26 @@ const innerHandler = async ({ throw new Error(COMMAND_NOT_FOUND); }; -export const innerRoutine = async ( - params: ServerlessPayload, - responseWriter: ResponseStreamWriter, - context: RequestContext, - providerSpecifics: ProviderSpecifics, -): Promise => { +export const innerRoutine = async ({ + params, + responseWriter, + context, + providerSpecifics, + serverProviderSpecifics, +}: { + params: ServerlessPayload; + responseWriter: ResponseStreamWriter; + context: RequestContext; + providerSpecifics: ProviderSpecifics; + serverProviderSpecifics: ServerProviderSpecifics; +}): Promise => { try { await innerHandler({ params, responseWriter, context, providerSpecifics, + serverProviderSpecifics, }); } catch (err) { const res: OrError<0> = { @@ -365,7 +381,13 @@ export const routine = ( ): Promise => { const responseWriter = streamWriter(responseStream); - return innerRoutine(params, responseWriter, context, awsImplementation); + return innerRoutine({ + params, + responseWriter, + context, + providerSpecifics: awsImplementation, + serverProviderSpecifics: serverAwsImplementation, + }); }; export const handler = streamifyResponse(routine); diff --git a/packages/lambda/src/test/mock-implementation.ts b/packages/lambda/src/test/mock-implementation.ts index f097dcd26dc..f938173cba6 100644 --- a/packages/lambda/src/test/mock-implementation.ts +++ b/packages/lambda/src/test/mock-implementation.ts @@ -1,5 +1,9 @@ import {openBrowser} from '@remotion/renderer'; -import type {GetBrowserInstance, ProviderSpecifics} from '@remotion/serverless'; +import type { + GetBrowserInstance, + ProviderSpecifics, + ServerProviderSpecifics, +} from '@remotion/serverless'; import {Readable} from 'stream'; import {estimatePrice} from '../api/estimate-price'; import {speculateFunctionName} from '../api/speculate-function-name'; @@ -162,9 +166,12 @@ export const mockImplementation: ProviderSpecifics = { }; }, isFlakyError, +}; + +export const mockServerImplementation: ServerProviderSpecifics = { + forgetBrowserEventLoop: () => {}, + getBrowserInstance, timer: () => ({ end: () => {}, }), - forgetBrowserEventLoop: () => {}, - getBrowserInstance, }; diff --git a/packages/lambda/src/test/mocks/aws-clients.ts b/packages/lambda/src/test/mocks/aws-clients.ts index 2d53c75fac5..990bdb87dcb 100644 --- a/packages/lambda/src/test/mocks/aws-clients.ts +++ b/packages/lambda/src/test/mocks/aws-clients.ts @@ -18,7 +18,10 @@ import { import {makeStreamer} from '@remotion/streaming'; import type {AwsProvider} from '../../functions/aws-implementation'; import {parseJsonOrThrowSource} from '../../shared/call-lambda-streaming'; -import {mockImplementation} from '../mock-implementation'; +import { + mockImplementation, + mockServerImplementation, +} from '../mock-implementation'; export const getMockCallFunctionStreaming: CallFunctionStreaming< AwsProvider @@ -52,9 +55,9 @@ export const getMockCallFunctionStreaming: CallFunctionStreaming< params.receivedStreamingPayload(message); }); - await innerRoutine( - params.payload, - { + await innerRoutine({ + params: params.payload, + responseWriter: { write(message) { onData(message); return Promise.resolve(); @@ -64,13 +67,15 @@ export const getMockCallFunctionStreaming: CallFunctionStreaming< return Promise.resolve(); }, }, - { + context: { invokedFunctionArn: 'arn:fake', getRemainingTimeInMillis: () => params.timeoutInTest ?? 120000, awsRequestId: 'fake', }, - mockImplementation, - ); + + providerSpecifics: mockImplementation, + serverProviderSpecifics: mockServerImplementation, + }); responseStream._finish(); responseStream.end(); @@ -84,16 +89,17 @@ export const getMockCallFunctionAsync: CallFunctionAsync = async < const {innerRoutine} = await import('../../functions/index'); const responseStream = new ResponseStream(); - await innerRoutine( - params.payload, - streamWriter(responseStream), - { + await innerRoutine({ + context: { invokedFunctionArn: 'arn:fake', getRemainingTimeInMillis: () => params.timeoutInTest ?? 120000, awsRequestId: 'fake', }, - mockImplementation, - ); + providerSpecifics: mockImplementation, + serverProviderSpecifics: mockServerImplementation, + params: params.payload, + responseWriter: streamWriter(responseStream), + }); responseStream._finish(); responseStream.end(); @@ -107,16 +113,17 @@ export const getMockCallFunctionSync: CallFunctionSync = async < const {innerRoutine} = await import('../../functions/index'); const responseStream = new ResponseStream(); - await innerRoutine( - params.payload, - streamWriter(responseStream), - { + await innerRoutine({ + context: { invokedFunctionArn: 'arn:fake', getRemainingTimeInMillis: () => params.timeoutInTest ?? 120000, awsRequestId: 'fake', }, - mockImplementation, - ); + params: params.payload, + responseWriter: streamWriter(responseStream), + providerSpecifics: mockImplementation, + serverProviderSpecifics: mockServerImplementation, + }); responseStream._finish(); responseStream.end(); diff --git a/packages/serverless/src/client.ts b/packages/serverless/src/client.ts index 76d6f93d371..620609b5cf2 100644 --- a/packages/serverless/src/client.ts +++ b/packages/serverless/src/client.ts @@ -54,6 +54,7 @@ export {FileNameAndSize, GetFolderFiles} from './get-files-in-folder'; export {getOverallProgressFromStorage} from './get-overall-progress-from-storage'; export {inputPropsKey, resolvedPropsKey} from './input-props-keys'; export {makeBucketName} from './make-bucket-name'; +export {getProgress} from './progress'; export {RenderMetadata} from './render-metadata'; export {streamToString} from './stream-to-string'; export { diff --git a/packages/serverless/src/get-browser-instance.ts b/packages/serverless/src/get-browser-instance.ts index c57e385c4f3..5ed288dd18a 100644 --- a/packages/serverless/src/get-browser-instance.ts +++ b/packages/serverless/src/get-browser-instance.ts @@ -5,6 +5,7 @@ import type {Await} from './await'; import type { GetBrowserInstance, ProviderSpecifics, + ServerProviderSpecifics, } from './provider-implementation'; import type {CloudProvider} from './types'; @@ -64,11 +65,13 @@ export const getBrowserInstanceImplementation: GetBrowserInstance = async < indent, chromiumOptions, providerSpecifics, + serverProviderSpecifics, }: { logLevel: LogLevel; indent: boolean; chromiumOptions: ChromiumOptions; providerSpecifics: ProviderSpecifics; + serverProviderSpecifics: ServerProviderSpecifics; }): Promise => { const actualChromiumOptions: ChromiumOptions = { ...chromiumOptions, @@ -122,7 +125,7 @@ export const getBrowserInstanceImplementation: GetBrowserInstance = async < {indent: false, logLevel}, 'Browser disconnected or crashed.', ); - providerSpecifics.forgetBrowserEventLoop(logLevel); + serverProviderSpecifics.forgetBrowserEventLoop(logLevel); _browserInstance?.instance?.close(true, logLevel, indent).catch((err) => { RenderInternals.Log.info( {indent: false, logLevel}, @@ -149,11 +152,12 @@ export const getBrowserInstanceImplementation: GetBrowserInstance = async < _browserInstance.instance.runner.rememberEventLoop(); await _browserInstance.instance.close(true, logLevel, indent); _browserInstance = null; - return providerSpecifics.getBrowserInstance({ + return serverProviderSpecifics.getBrowserInstance({ logLevel, indent, chromiumOptions, providerSpecifics, + serverProviderSpecifics, }); } diff --git a/packages/serverless/src/handlers/compositions.ts b/packages/serverless/src/handlers/compositions.ts index 8e24e508cce..6fd5eead52d 100644 --- a/packages/serverless/src/handlers/compositions.ts +++ b/packages/serverless/src/handlers/compositions.ts @@ -5,7 +5,10 @@ import type {ServerlessPayload} from '../constants'; import {ServerlessRoutines} from '../constants'; import {} from '../get-browser-instance'; import {internalGetOrCreateBucket} from '../get-or-create-bucket'; -import type {ProviderSpecifics} from '../provider-implementation'; +import type { + ProviderSpecifics, + ServerProviderSpecifics, +} from '../provider-implementation'; import type {CloudProvider} from '../types'; type Options = { @@ -16,6 +19,7 @@ export const compositionsHandler = async ( lambdaParams: ServerlessPayload, options: Options, providerSpecifics: ProviderSpecifics, + serverProviderSpecifics: ServerProviderSpecifics, ) => { if (lambdaParams.type !== ServerlessRoutines.compositions) { throw new TypeError('Expected info compositions'); @@ -36,11 +40,12 @@ export const compositionsHandler = async ( try { const region = providerSpecifics.getCurrentRegionInFunction(); - const browserInstancePromise = providerSpecifics.getBrowserInstance({ + const browserInstancePromise = serverProviderSpecifics.getBrowserInstance({ logLevel: lambdaParams.logLevel, indent: false, chromiumOptions: lambdaParams.chromiumOptions, providerSpecifics, + serverProviderSpecifics, }); const bucketNamePromise = lambdaParams.bucketName ? Promise.resolve(lambdaParams.bucketName) @@ -96,6 +101,6 @@ export const compositionsHandler = async ( type: 'success' as const, }); } finally { - providerSpecifics.forgetBrowserEventLoop(lambdaParams.logLevel); + serverProviderSpecifics.forgetBrowserEventLoop(lambdaParams.logLevel); } }; diff --git a/packages/serverless/src/handlers/launch.ts b/packages/serverless/src/handlers/launch.ts index a9cc3f4feb2..a99b880691a 100644 --- a/packages/serverless/src/handlers/launch.ts +++ b/packages/serverless/src/handlers/launch.ts @@ -25,7 +25,10 @@ import type {WebhookClient} from '../invoke-webhook'; import {invokeWebhook} from '../invoke-webhook'; import type {OverallProgressHelper} from '../overall-render-progress'; import {makeOverallRenderProgress} from '../overall-render-progress'; -import type {ProviderSpecifics} from '../provider-implementation'; +import type { + ProviderSpecifics, + ServerProviderSpecifics, +} from '../provider-implementation'; import type {RenderMetadata} from '../render-metadata'; import {bestFramesPerFunctionParam} from '../best-frames-per-function-param'; @@ -58,6 +61,7 @@ const innerLaunchHandler = async ({ overallProgress, registerCleanupTask, providerSpecifics, + serverProviderSpecifics, }: { functionName: string; params: ServerlessPayload; @@ -65,6 +69,7 @@ const innerLaunchHandler = async ({ overallProgress: OverallProgressHelper; registerCleanupTask: (cleanupTask: CleanupTask) => void; providerSpecifics: ProviderSpecifics; + serverProviderSpecifics: ServerProviderSpecifics; }): Promise> => { if (params.type !== ServerlessRoutines.launch) { throw new Error('Expected launch type'); @@ -72,11 +77,12 @@ const innerLaunchHandler = async ({ const startedDate = Date.now(); - const browserInstance = providerSpecifics.getBrowserInstance({ + const browserInstance = serverProviderSpecifics.getBrowserInstance({ logLevel: params.logLevel, indent: false, chromiumOptions: params.chromiumOptions, providerSpecifics, + serverProviderSpecifics, }); const inputPropsPromise = decompressInputProps({ @@ -500,10 +506,12 @@ export const launchHandler = async ({ options, providerSpecifics, client, + serverProviderSpecifics, }: { params: ServerlessPayload; options: Options; providerSpecifics: ProviderSpecifics; + serverProviderSpecifics: ServerProviderSpecifics; client: WebhookClient; }): Promise<{ type: 'success'; @@ -675,6 +683,7 @@ export const launchHandler = async ({ overallProgress, registerCleanupTask, providerSpecifics, + serverProviderSpecifics, }); clearTimeout(webhookDueToTimeout); @@ -835,6 +844,6 @@ export const launchHandler = async ({ throw err; } finally { - providerSpecifics.forgetBrowserEventLoop(params.logLevel); + serverProviderSpecifics.forgetBrowserEventLoop(params.logLevel); } }; diff --git a/packages/serverless/src/handlers/renderer.ts b/packages/serverless/src/handlers/renderer.ts index a68018beb2f..53648a23a60 100644 --- a/packages/serverless/src/handlers/renderer.ts +++ b/packages/serverless/src/handlers/renderer.ts @@ -19,7 +19,10 @@ import type {ServerlessPayload} from '../constants'; import {RENDERER_PATH_TOKEN, ServerlessRoutines} from '../constants'; import {startLeakDetection} from '../leak-detection'; import {onDownloadsHelper} from '../on-downloads-helpers'; -import type {ProviderSpecifics} from '../provider-implementation'; +import type { + ProviderSpecifics, + ServerProviderSpecifics, +} from '../provider-implementation'; import {serializeArtifact} from '../serialize-artifact'; import type {OnStream} from '../streaming/streaming'; import {truthy} from '../truthy'; @@ -44,12 +47,14 @@ const renderHandler = async ({ logs, onStream, providerSpecifics, + serverProviderSpecifics, }: { params: ServerlessPayload; options: Options; logs: BrowserLog[]; onStream: OnStream; providerSpecifics: ProviderSpecifics; + serverProviderSpecifics: ServerProviderSpecifics; }): Promise<{}> => { if (params.type !== ServerlessRoutines.renderer) { throw new Error('Params must be renderer'); @@ -81,11 +86,12 @@ const renderHandler = async ({ forcePathStyle: params.forcePathStyle, }); - const browserInstance = await providerSpecifics.getBrowserInstance({ + const browserInstance = await serverProviderSpecifics.getBrowserInstance({ logLevel: params.logLevel, indent: false, chromiumOptions: params.chromiumOptions, providerSpecifics, + serverProviderSpecifics, }); const outputPath = RenderInternals.tmpDir('remotion-render-'); @@ -410,12 +416,14 @@ export const rendererHandler = async ({ params, providerSpecifics, requestContext, + serverProviderSpecifics, }: { params: ServerlessPayload; options: Options; onStream: OnStream; requestContext: RequestContext; providerSpecifics: ProviderSpecifics; + serverProviderSpecifics: ServerProviderSpecifics; }): Promise<{ type: 'success'; }> => { @@ -435,6 +443,7 @@ export const rendererHandler = async ({ logs, onStream, providerSpecifics, + serverProviderSpecifics, }); return { type: 'success', @@ -494,7 +503,7 @@ export const rendererHandler = async ({ throw err; } finally { if (shouldKeepBrowserOpen) { - providerSpecifics.forgetBrowserEventLoop(params.logLevel); + serverProviderSpecifics.forgetBrowserEventLoop(params.logLevel); } else { RenderInternals.Log.info( {indent: false, logLevel: params.logLevel}, diff --git a/packages/serverless/src/handlers/still.ts b/packages/serverless/src/handlers/still.ts index 2c1427bba1d..3733f98ff43 100644 --- a/packages/serverless/src/handlers/still.ts +++ b/packages/serverless/src/handlers/still.ts @@ -21,7 +21,10 @@ import {formatCostsInfo} from '../format-costs-info'; import {internalGetOrCreateBucket} from '../get-or-create-bucket'; import {onDownloadsHelper} from '../on-downloads-helpers'; import {makeInitialOverallRenderProgress} from '../overall-render-progress'; -import type {ProviderSpecifics} from '../provider-implementation'; +import type { + ProviderSpecifics, + ServerProviderSpecifics, +} from '../provider-implementation'; import type {RenderMetadata} from '../render-metadata'; import type {OnStream} from '../streaming/streaming'; import type { @@ -42,6 +45,7 @@ type Options = { onStream: OnStream; timeoutInMilliseconds: number; providerSpecifics: ProviderSpecifics; + serverProviderSpecifics: ServerProviderSpecifics; }; const innerStillHandler = async ( @@ -52,6 +56,7 @@ const innerStillHandler = async ( onStream, timeoutInMilliseconds, providerSpecifics, + serverProviderSpecifics, }: Options, cleanup: CleanupFn[], ) => { @@ -82,11 +87,12 @@ const innerStillHandler = async ( const start = Date.now(); - const browserInstancePromise = providerSpecifics.getBrowserInstance({ + const browserInstancePromise = serverProviderSpecifics.getBrowserInstance({ logLevel: lambdaParams.logLevel, indent: false, chromiumOptions: lambdaParams.chromiumOptions, providerSpecifics, + serverProviderSpecifics, }); const bucketNamePromise = lambdaParams.bucketName ?? @@ -442,7 +448,7 @@ export const stillHandler = async ( stack: (err as Error).stack as string, }; } finally { - options.providerSpecifics.forgetBrowserEventLoop( + options.serverProviderSpecifics.forgetBrowserEventLoop( options.params.type === ServerlessRoutines.still ? options.params.logLevel : 'error', diff --git a/packages/serverless/src/index.ts b/packages/serverless/src/index.ts index 88b8d5ef723..a07fd78648f 100644 --- a/packages/serverless/src/index.ts +++ b/packages/serverless/src/index.ts @@ -26,7 +26,6 @@ export { makeInitialOverallRenderProgress, makeOverallRenderProgress, } from './overall-render-progress'; -export {getProgress} from './progress'; export * from './provider-implementation'; export type {CleanupInfo, GenericRenderProgress} from './render-progress'; export {OrError, ServerlessReturnValues} from './return-values'; diff --git a/packages/serverless/src/provider-implementation.ts b/packages/serverless/src/provider-implementation.ts index 7dee2e315ad..1c3976d3743 100644 --- a/packages/serverless/src/provider-implementation.ts +++ b/packages/serverless/src/provider-implementation.ts @@ -216,15 +216,23 @@ export type GetBrowserInstance = ({ indent, chromiumOptions, providerSpecifics, + serverProviderSpecifics, }: { logLevel: LogLevel; indent: boolean; chromiumOptions: ChromiumOptions; providerSpecifics: ProviderSpecifics; + serverProviderSpecifics: ServerProviderSpecifics; }) => Promise; export type ForgetBrowserEventLoop = (logLevel: LogLevel) => void; +export type ServerProviderSpecifics = { + getBrowserInstance: GetBrowserInstance; + forgetBrowserEventLoop: ForgetBrowserEventLoop; + timer: DebuggingTimer; +}; + export type ProviderSpecifics = { getChromiumPath: () => string | null; getCurrentRegionInFunction: () => Provider['region']; @@ -251,9 +259,6 @@ export type ProviderSpecifics = { getLoggingUrlForRendererFunction: GetLoggingUrlForRendererFunction; getLoggingUrlForMethod: GetLoggingUrlForMethod; getEphemeralStorageForPriceCalculation: () => number; - timer: DebuggingTimer; getOutputUrl: GetOutputUrl; isFlakyError: (err: Error) => boolean; - getBrowserInstance: GetBrowserInstance; - forgetBrowserEventLoop: ForgetBrowserEventLoop; }; From df19a38d31db86a65b4cf1759ee0fcfd1fc93352 Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Mon, 6 Jan 2025 16:00:26 +0100 Subject: [PATCH 14/14] fix --- packages/serverless/src/concat-videos.ts | 11 +++++------ packages/serverless/src/handlers/launch.ts | 3 ++- packages/serverless/src/handlers/renderer.ts | 6 +++--- packages/serverless/src/merge-chunks.ts | 10 +++++++--- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/packages/serverless/src/concat-videos.ts b/packages/serverless/src/concat-videos.ts index ad1714a1472..7c1e9648ec3 100644 --- a/packages/serverless/src/concat-videos.ts +++ b/packages/serverless/src/concat-videos.ts @@ -12,10 +12,9 @@ import { REMOTION_FILELIST_TOKEN, type ServerlessCodec, } from './constants'; -import type {ProviderSpecifics} from './provider-implementation'; -import type {CloudProvider} from './types'; +import type {ServerProviderSpecifics} from './provider-implementation'; -export const concatVideos = async ({ +export const concatVideos = async ({ onProgress, numberOfFrames, codec, @@ -32,7 +31,7 @@ export const concatVideos = async ({ preferLossless, muted, metadata, - providerSpecifics, + serverProviderSpecifics, }: { onProgress: (frames: number) => void; numberOfFrames: number; @@ -50,13 +49,13 @@ export const concatVideos = async ({ preferLossless: boolean; muted: boolean; metadata: Record | null; - providerSpecifics: ProviderSpecifics; + serverProviderSpecifics: ServerProviderSpecifics; }) => { const outfile = join( RenderInternals.tmpDir(REMOTION_CONCATED_TOKEN), `concat.${RenderInternals.getFileExtensionFromCodec(codec, audioCodec)}`, ); - const combine = providerSpecifics.timer('Combine chunks', logLevel); + const combine = serverProviderSpecifics.timer('Combine chunks', logLevel); const filelistDir = RenderInternals.tmpDir(REMOTION_FILELIST_TOKEN); const chunkDurationInSeconds = framesPerLambda / fps; diff --git a/packages/serverless/src/handlers/launch.ts b/packages/serverless/src/handlers/launch.ts index a99b880691a..7ae4c86e836 100644 --- a/packages/serverless/src/handlers/launch.ts +++ b/packages/serverless/src/handlers/launch.ts @@ -346,7 +346,7 @@ const innerLaunchHandler = async ({ ); if (!params.overwrite) { - const findOutputFile = providerSpecifics.timer( + const findOutputFile = serverProviderSpecifics.timer( 'Checking if output file already exists', params.logLevel, ); @@ -494,6 +494,7 @@ const innerLaunchHandler = async ({ startTime, providerSpecifics, forcePathStyle: params.forcePathStyle, + serverProviderSpecifics, }); return postRenderData; diff --git a/packages/serverless/src/handlers/renderer.ts b/packages/serverless/src/handlers/renderer.ts index 53648a23a60..792a7a398e7 100644 --- a/packages/serverless/src/handlers/renderer.ts +++ b/packages/serverless/src/handlers/renderer.ts @@ -343,13 +343,13 @@ const renderHandler = async ({ .catch((err) => reject(err)); }); - const streamTimer = providerSpecifics.timer( + const streamTimer = serverProviderSpecifics.timer( 'Streaming chunk to the main function', params.logLevel, ); if (audioOutputLocation) { - const audioChunkTimer = providerSpecifics.timer( + const audioChunkTimer = serverProviderSpecifics.timer( 'Sending audio chunk', params.logLevel, ); @@ -361,7 +361,7 @@ const renderHandler = async ({ } if (videoOutputLocation) { - const videoChunkTimer = providerSpecifics.timer( + const videoChunkTimer = serverProviderSpecifics.timer( 'Sending main chunk', params.logLevel, ); diff --git a/packages/serverless/src/merge-chunks.ts b/packages/serverless/src/merge-chunks.ts index 8641cea05c1..5472f4a0d52 100644 --- a/packages/serverless/src/merge-chunks.ts +++ b/packages/serverless/src/merge-chunks.ts @@ -14,7 +14,10 @@ import type { import {createPostRenderData} from './create-post-render-data'; import {inspectErrors} from './inspect-error'; import type {OverallProgressHelper} from './overall-render-progress'; -import type {ProviderSpecifics} from './provider-implementation'; +import type { + ProviderSpecifics, + ServerProviderSpecifics, +} from './provider-implementation'; import type {RenderMetadata} from './render-metadata'; import type {CloudProvider} from './types'; @@ -49,6 +52,7 @@ export const mergeChunksAndFinishRender = async < overallProgress: OverallProgressHelper; startTime: number; providerSpecifics: ProviderSpecifics; + serverProviderSpecifics: ServerProviderSpecifics; forcePathStyle: boolean; }): Promise> => { const onProgress = (framesEncoded: number) => { @@ -81,14 +85,14 @@ export const mergeChunksAndFinishRender = async < preferLossless: options.preferLossless, muted: options.renderMetadata.muted, metadata: options.renderMetadata.metadata, - providerSpecifics: options.providerSpecifics, + serverProviderSpecifics: options.serverProviderSpecifics, }); const encodingStop = Date.now(); options.overallProgress.setTimeToCombine(encodingStop - encodingStart); const outputSize = fs.statSync(outfile).size; - const writeToBucket = options.providerSpecifics.timer( + const writeToBucket = options.serverProviderSpecifics.timer( `Writing to bucket (${outputSize} bytes)`, options.logLevel, );