Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(itk-wasm): #1087 - Add query params options to pipeline URL #1093

Merged
merged 2 commits into from
Mar 11, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,18 @@ describe('runPipeline', () => {
})
})

it('fetches Wasm files from a custom URL and query params', () => {
cy.window().then(async (win) => {
const itk = win.itk

const args = []
const outputs = null
const inputs = null
const stdoutStderrPath = 'stdout-stderr-test'
const { webWorker, returnValue, stdout, stderr } = await itk.runPipeline(stdoutStderrPath, args, outputs, inputs, { pipelineBaseUrl, pipelineWorkerUrl, pipelineQueryParams: {key: 'value'} })
})
})

it('fetches Wasm files from a custom pipelineBaseUrl string', () => {
cy.window().then(async (win) => {
const itk = win.itk
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import axios from 'axios'
import RunPipelineOptions from './run-pipeline-options'

async function createWebWorker (pipelineWorkerUrl?: string | null): Promise<Worker> {
async function createWebWorker (pipelineWorkerUrl?: string | null, queryParams?: RunPipelineOptions['pipelineQueryParams']): Promise<Worker> {
const workerUrl = pipelineWorkerUrl
let worker = null
if (workerUrl === null) {
Expand All @@ -10,7 +11,7 @@ async function createWebWorker (pipelineWorkerUrl?: string | null): Promise<Work
worker = new Worker(new URL('./web-workers/itk-wasm-pipeline.worker.js', import.meta.url), { type: 'module' })
} else {
if ((workerUrl as string).startsWith('http')) {
const response = await axios.get((workerUrl as string), { responseType: 'blob' })
const response = await axios.get((workerUrl as string), { responseType: 'blob', params: queryParams })
const workerObjectUrl = URL.createObjectURL(response.data as Blob)
worker = new Worker(workerObjectUrl, { type: 'module' })
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import * as Comlink from 'comlink'
import WorkerProxy from './web-workers/worker-proxy.js'
import createWebWorker from './create-web-worker.js'
import ItkWorker from './itk-worker.js'
import RunPipelineOptions from './run-pipeline-options.js'

interface createWorkerProxyResult {
workerProxy: WorkerProxy
Expand All @@ -24,7 +25,7 @@ function workerToWorkerProxy (worker: Worker): createWorkerProxyResult {
}

// Internal function to create a web worker proxy
async function createWorkerProxy (existingWorker: Worker | null, pipelineWorkerUrl?: string | null): Promise<createWorkerProxyResult> {
async function createWorkerProxy (existingWorker: Worker | null, pipelineWorkerUrl?: string | null, queryParams?: RunPipelineOptions['pipelineQueryParams']): Promise<createWorkerProxyResult> {
let workerProxy: WorkerProxy
if (existingWorker != null) {
// See if we have a worker promise attached the worker, if so reuse it. This ensures
Expand All @@ -38,7 +39,7 @@ async function createWorkerProxy (existingWorker: Worker | null, pipelineWorkerU
}
}

const worker = await createWebWorker(pipelineWorkerUrl)
const worker = await createWebWorker(pipelineWorkerUrl, queryParams)

return workerToWorkerProxy(worker)
}
Expand Down
1 change: 1 addition & 0 deletions packages/core/typescript/itk-wasm/src/pipeline/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,5 @@ export type { default as ItkWorker } from './itk-worker.js'

export * from './pipeline-worker-url.js'
export * from './pipelines-base-url.js'
export * from './pipelines-query-params.js'
export * from './default-web-worker.js'
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
import axios from 'axios'

import EmscriptenModule from '../itk-wasm-emscripten-module.js'
import RunPipelineOptions from '../run-pipeline-options.js'

async function loadEmscriptenModuleMainThread (moduleRelativePathOrURL: string | URL, baseUrl?: string): Promise<EmscriptenModule> {
async function loadEmscriptenModuleMainThread (moduleRelativePathOrURL: string | URL, baseUrl?: string, queryParams?: RunPipelineOptions['pipelineQueryParams']): Promise<EmscriptenModule> {
let modulePrefix: string = 'unknown'
if (typeof moduleRelativePathOrURL !== 'string') {
modulePrefix = moduleRelativePathOrURL.href
Expand All @@ -18,7 +19,7 @@ async function loadEmscriptenModuleMainThread (moduleRelativePathOrURL: string |
modulePrefix = modulePrefix.substring(0, modulePrefix.length - 5)
}
const wasmBinaryPath = `${modulePrefix}.wasm`
const response = await axios.get(wasmBinaryPath, { responseType: 'arraybuffer' })
const response = await axios.get(wasmBinaryPath, { responseType: 'arraybuffer', params: queryParams })
const wasmBinary = response.data
const fullModulePath = `${modulePrefix}.js`
const result = await import(/* webpackIgnore: true */ /* @vite-ignore */ fullModulePath)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,16 @@ import axios from 'axios'
import { ZSTDDecoder } from '@thewtex/zstddec'

import ITKWasmEmscriptenModule from '../itk-wasm-emscripten-module.js'
import RunPipelineOptions from '../run-pipeline-options.js'

const decoder = new ZSTDDecoder()
let decoderInitialized = false

// Load the Emscripten module in the browser in a WebWorker.
//
// baseUrl is usually taken from 'getPipelinesBaseUrl()', but a different value
// could be passed.
async function loadEmscriptenModuleWebWorker (moduleRelativePathOrURL: string | URL, baseUrl: string): Promise<ITKWasmEmscriptenModule> {
async function loadEmscriptenModuleWebWorker (moduleRelativePathOrURL: string | URL, baseUrl: string, queryParams?: RunPipelineOptions['pipelineQueryParams']): Promise<ITKWasmEmscriptenModule> {
let modulePrefix = null
if (typeof moduleRelativePathOrURL !== 'string') {
modulePrefix = moduleRelativePathOrURL.href
Expand All @@ -26,7 +28,7 @@ async function loadEmscriptenModuleWebWorker (moduleRelativePathOrURL: string |
modulePrefix = modulePrefix.substring(0, modulePrefix.length - 5)
}
const wasmBinaryPath = `${modulePrefix}.wasm`
const response = await axios.get(`${wasmBinaryPath}.zst`, { responseType: 'arraybuffer' })
const response = await axios.get(`${wasmBinaryPath}.zst`, { responseType: 'arraybuffer', params: queryParams })
if (!decoderInitialized) {
await decoder.init()
decoderInitialized = true
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
import RunPipelineOptions from "./run-pipeline-options"

let pipelinesQueryParams: RunPipelineOptions['pipelineQueryParams'] | undefined

export function setPipelinesQueryParams (queryParams: RunPipelineOptions['pipelineQueryParams']): void {
pipelinesQueryParams = queryParams
}

export function getPipelinesQueryParams (): RunPipelineOptions['pipelineQueryParams'] | undefined {
return pipelinesQueryParams
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ interface RunPipelineOptions {
/** Url where WebAssembly pipelines are hosted. */
pipelineBaseUrl?: string | URL

/** Query params to use when requesting for WebAssembly pipelines */
pipelineQueryParams?: Record<string, string>

/** Url where the pipeline web worker is hosted. */
pipelineWorkerUrl?: string | URL | null

Expand Down
28 changes: 23 additions & 5 deletions packages/core/typescript/itk-wasm/src/pipeline/run-pipeline.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import polyDataTransferables from './internal/poly-data-transferables.js'
import TypedArray from '../typed-array.js'
import RunPipelineWorkerResult from './web-workers/run-pipeline-worker-result.js'
import { getPipelinesBaseUrl } from './pipelines-base-url.js'
import { getPipelinesQueryParams } from './pipelines-query-params.js'
import { getPipelineWorkerUrl } from './pipeline-worker-url.js'

// To cache loaded pipeline modules
Expand All @@ -46,9 +47,18 @@ function defaultPipelinesBaseUrl (): string {
return result as string
}

function defaultPipelinesQueryParams (): RunPipelineOptions['pipelineQueryParams'] {
let result = getPipelinesQueryParams()
if (typeof result === 'undefined') {
result = {}
}
return result
}

async function loadPipelineModule (
pipelinePath: string | URL,
pipelineBaseUrl?: string | URL
pipelineBaseUrl?: string | URL,
pipelineQueryParams?: RunPipelineOptions['pipelineQueryParams']
): Promise<PipelineEmscriptenModule> {
let moduleRelativePathOrURL: string | URL = pipelinePath as string
let pipeline = pipelinePath as string
Expand All @@ -61,7 +71,8 @@ async function loadPipelineModule (
} else {
const pipelineModule = (await loadEmscriptenModuleMainThread(
pipelinePath,
pipelineBaseUrl?.toString() ?? defaultPipelinesBaseUrl()
pipelineBaseUrl?.toString() ?? defaultPipelinesBaseUrl(),
pipelineQueryParams ?? defaultPipelinesQueryParams()
)) as PipelineEmscriptenModule
pipelineToModule.set(pipeline, pipelineModule)
return pipelineModule
Expand All @@ -83,15 +94,21 @@ async function runPipeline (
const webWorker = options?.webWorker ?? null

if (webWorker === false) {
const pipelineModule = await loadPipelineModule(pipelinePath.toString(), options?.pipelineBaseUrl)
const pipelineModule = await loadPipelineModule(
pipelinePath.toString(),
options?.pipelineBaseUrl,
options?.pipelineQueryParams ?? defaultPipelinesQueryParams(),
)
const result = runPipelineEmscripten(pipelineModule, args, outputs, inputs)
return result
}
let worker = webWorker
const pipelineWorkerUrl = options?.pipelineWorkerUrl ?? defaultPipelineWorkerUrl()
const pipelineWorkerUrlString = typeof pipelineWorkerUrl !== 'string' && typeof pipelineWorkerUrl?.href !== 'undefined' ? pipelineWorkerUrl.href : pipelineWorkerUrl
const { workerProxy, worker: usedWorker } = await createWorkerProxy(
worker as Worker | null, pipelineWorkerUrlString as string | undefined | null
worker as Worker | null,
pipelineWorkerUrlString as string | undefined | null,
options?.pipelineQueryParams ?? defaultPipelinesQueryParams(),
)
worker = usedWorker
const transferables: Array<ArrayBuffer | TypedArray | null> = []
Expand Down Expand Up @@ -130,7 +147,8 @@ async function runPipeline (
pipelineBaseUrlString as string,
args,
outputs,
transferedInputs
transferedInputs,
options?.pipelineQueryParams ?? defaultPipelinesQueryParams()
)
return {
returnValue: result.returnValue,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,11 @@ import PipelineInput from '../pipeline-input.js'
import RunPipelineWorkerResult from './run-pipeline-worker-result.js'
import loadPipelineModule from './load-pipeline-module.js'
import runPipeline from './run-pipeline.js'
import RunPipelineOptions from '../run-pipeline-options.js'

const workerOperations = {
runPipeline: async function (pipelinePath: string, pipelineBaseUrl: string, args: string[], outputs: PipelineOutput[] | null, inputs: PipelineInput[] | null): Promise<RunPipelineWorkerResult> {
const pipelineModule = await loadPipelineModule(pipelinePath, pipelineBaseUrl)
runPipeline: async function (pipelinePath: string, pipelineBaseUrl: string, args: string[], outputs: PipelineOutput[] | null, inputs: PipelineInput[] | null, pipelineQueryParams?: RunPipelineOptions['pipelineQueryParams']): Promise<RunPipelineWorkerResult> {
const pipelineModule = await loadPipelineModule(pipelinePath, pipelineBaseUrl, pipelineQueryParams)
return await runPipeline(pipelineModule, args, outputs, inputs)
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
import loadEmscriptenModule from '../internal/load-emscripten-module-web-worker.js'
import PipelineEmscriptenModule from '../pipeline-emscripten-module.js'
import RunPipelineOptions from '../run-pipeline-options.js'

// To cache loaded pipeline modules wrapped in a Promise
const pipelineToModule: Map<string, Promise<PipelineEmscriptenModule>> = new Map()

async function loadPipelineModule (pipelinePath: string | object, baseUrl: string): Promise<PipelineEmscriptenModule> {
async function loadPipelineModule (pipelinePath: string | object, baseUrl: string, queryParams?: RunPipelineOptions['pipelineQueryParams']): Promise<PipelineEmscriptenModule> {
let moduleRelativePathOrURL: string | URL = pipelinePath as string
let pipeline = pipelinePath as string
let pipelineModule = null
Expand All @@ -15,7 +16,7 @@ async function loadPipelineModule (pipelinePath: string | object, baseUrl: strin
if (pipelineToModule.has(pipeline)) {
pipelineModule = await pipelineToModule.get(pipeline) as PipelineEmscriptenModule
} else {
pipelineToModule.set(pipeline, loadEmscriptenModule(moduleRelativePathOrURL, baseUrl) as Promise<PipelineEmscriptenModule>)
pipelineToModule.set(pipeline, loadEmscriptenModule(moduleRelativePathOrURL, baseUrl, queryParams) as Promise<PipelineEmscriptenModule>)
pipelineModule = await pipelineToModule.get(pipeline) as PipelineEmscriptenModule
}
return pipelineModule
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
import PipelineOutput from '../pipeline-output.js'
import PipelineInput from '../pipeline-input.js'
import RunPipelineWorkerResult from './run-pipeline-worker-result.js'
import RunPipelineOptions from '../run-pipeline-options.js'

interface WorkerOperations {
runPipeline: (pipelinePath: string, pipelineBaseUrl: string, args: string[], outputs: PipelineOutput[] | null, inputs: PipelineInput[] | null) => RunPipelineWorkerResult
runPipeline: (pipelinePath: string, pipelineBaseUrl: string, args: string[], outputs: PipelineOutput[] | null, inputs: PipelineInput[] | null, pipelineQueryParams?: RunPipelineOptions['pipelineQueryParams']) => RunPipelineWorkerResult
}

export default WorkerOperations
Loading