From 6f124bf88f0daf6991274398f1299511ee93316e Mon Sep 17 00:00:00 2001 From: Hari Seshadri Date: Fri, 30 Nov 2018 14:26:32 -0800 Subject: [PATCH 1/6] Fix operator support table to include ImageScaler operator for cpu backend --- docs/operators.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operators.md b/docs/operators.md index 90da244e..5eef7a13 100644 --- a/docs/operators.md +++ b/docs/operators.md @@ -26,7 +26,7 @@ The supported platforms are Windows 10 + Edge/Chrome/Firefox/Electron/Node.js. | [GlobalMaxPool](https://github.com/onnx/onnx/blob/rel-1.2.3/docs/Operators.md#GlobalMaxPool) | x | x | x | | [Greater](https://github.com/onnx/onnx/blob/rel-1.2.3/docs/Operators.md#Greater) | | | x | | [Identity](https://github.com/onnx/onnx/blob/rel-1.2.3/docs/Operators.md#Identity) | | | x | -| [ImageScaler](https://github.com/onnx/onnx/blob/rel-1.2.3/docs/Operators.md#ImageScaler) | | | x | +| [ImageScaler](https://github.com/onnx/onnx/blob/rel-1.2.3/docs/Operators.md#ImageScaler) | x | | x | | [LeakyRelu](https://github.com/onnx/onnx/blob/rel-1.2.3/docs/Operators.md#LeakyRelu) | x | | x | | [Less](https://github.com/onnx/onnx/blob/rel-1.2.3/docs/Operators.md#Less) | | | x | | [Log](https://github.com/onnx/onnx/blob/rel-1.2.3/docs/Operators.md#Log) | x | | x | From c46b2ef503e3a0fc3761bc17afbed6d39f70b0a0 Mon Sep 17 00:00:00 2001 From: hasesh Date: Thu, 13 Dec 2018 04:03:17 -0800 Subject: [PATCH 2/6] Update to latest remote commit for data submodule --- deps/data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/data b/deps/data index f4556abc..2c20f6cd 160000 --- a/deps/data +++ b/deps/data @@ -1 +1 @@ -Subproject commit f4556abcc05d13262d502c0e255bb68289459ddc +Subproject commit 2c20f6cd16c6f4dfc7019a186a447b523d8c8a66 From 9b35505f93058a623ef953ce2b58845360c3cfec Mon Sep 17 00:00:00 2001 From: hariharans29 Date: Thu, 24 Jan 2019 12:39:24 -0800 Subject: [PATCH 3/6] InstanceNormalization op for cpu and wasm backends --- lib/backends/cpu/ops-resolve.ts | 3 + .../cpu/ops/instance-normalization.ts | 69 +++++++++++++++++++ .../wasm/ops/instance-normalization.ts | 51 ++++++++++++++ lib/backends/wasm/session-handler.ts | 3 + lib/ops/instance-normalization.ts | 45 ++++++++++++ src/wasm-build-config.json | 1 + src/wasm-ops/batch-normalization.cpp | 4 +- src/wasm-ops/instance-normalization.cpp | 57 +++++++++++++++ src/wasm-ops/instance-normalization.h | 12 ++++ test/unittest-whitelist.jsonc | 6 +- 10 files changed, 248 insertions(+), 3 deletions(-) create mode 100644 lib/backends/cpu/ops/instance-normalization.ts create mode 100644 lib/backends/wasm/ops/instance-normalization.ts create mode 100644 lib/ops/instance-normalization.ts create mode 100644 src/wasm-ops/instance-normalization.cpp create mode 100644 src/wasm-ops/instance-normalization.h diff --git a/lib/backends/cpu/ops-resolve.ts b/lib/backends/cpu/ops-resolve.ts index 05eda208..d097cf57 100644 --- a/lib/backends/cpu/ops-resolve.ts +++ b/lib/backends/cpu/ops-resolve.ts @@ -13,6 +13,7 @@ import {CpuDropout} from './ops/dropout'; import {CpuGather} from './ops/gather'; import {CpuGemm} from './ops/gemm'; import {CpuImageScaler} from './ops/image-scaler'; +import {CpuInstanceNormalization} from './ops/instance-normalization'; import {CpuLrn} from './ops/lrn'; import {CpuMatMul} from './ops/matmul'; import {CpuAveragePool, CpuGlobalAveragePool, CpuGlobalMaxPool, CpuMaxPool} from './ops/pool'; @@ -118,6 +119,8 @@ function createOperator(node: Graph.Node, domain: string, version: number): Oper return new CpuGlobalMaxPool(); case 'GlobalAveragePool': return new CpuGlobalAveragePool(); + case 'InstanceNormalization': + return new CpuInstanceNormalization(); case 'PRelu': return new CpuBinaryOp(NUMBER_TYPES, (e1, e2) => (e1 >= 0 ? e1 : e1 * e2)); case 'Reshape': diff --git a/lib/backends/cpu/ops/instance-normalization.ts b/lib/backends/cpu/ops/instance-normalization.ts new file mode 100644 index 00000000..11e46c02 --- /dev/null +++ b/lib/backends/cpu/ops/instance-normalization.ts @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +import {InstanceNormalization} from '../../../ops/instance-normalization'; +import {Tensor} from '../../../tensor'; +import {CpuInferenceHandler} from '../inference-handler'; + +export class CpuInstanceNormalization extends InstanceNormalization { + run(inferenceHandler: CpuInferenceHandler, inputs: Tensor[]): Tensor[] { + const output = instanceNormalization(inputs[0], inputs[1], inputs[2], this.epsilon); + return [output]; + } +} + +export function instanceNormalization(x: Tensor, scale: Tensor, b: Tensor, epsilon: number) { + const inputDimensions = x.dims; + const N = inputDimensions[0]; + const C = inputDimensions[1]; + + // calculate channel size (i.e.) data points per channel + let channelSize = 1; + for (let i = 2; i < inputDimensions.length; i++) { + channelSize *= inputDimensions[i]; + } + const sampleSize = C * channelSize; + + const output = new Tensor(x.dims, x.type); + + const X = x.floatData; + const Y = output.floatData; + const scaleData = scale.numberData; + const bData = b.numberData; + + let temp: number; + let mean: number; + let variance: number; + let sampleOffset: number; + let physicalOffset: number; + let iterEnd: number; + + for (let n = 0; n < N; ++n) { + sampleOffset = n * sampleSize; + for (let c = 0; c < C; ++c) { + physicalOffset = sampleOffset + c * channelSize; + iterEnd = physicalOffset + channelSize; + + // compute mean for this channel + temp = 0; + for (let i = physicalOffset; i < iterEnd; ++i) { + temp += X[i]; + } + mean = temp / channelSize; + + // compute variance for this channel + temp = 0; + for (let i = physicalOffset; i < iterEnd; ++i) { + temp += Math.pow(X[i] - mean, 2); + } + variance = temp / channelSize; + + // compute normalized value for data in this channel + for (let i = physicalOffset; i < iterEnd; ++i) { + Y[i] = scaleData[c] * ((X[i] - mean) / Math.sqrt(variance + epsilon)) + bData[c]; + } + } + } + + return output; +} diff --git a/lib/backends/wasm/ops/instance-normalization.ts b/lib/backends/wasm/ops/instance-normalization.ts new file mode 100644 index 00000000..c624a8ed --- /dev/null +++ b/lib/backends/wasm/ops/instance-normalization.ts @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +import {InstanceNormalization} from '../../../ops/instance-normalization'; +import {Tensor} from '../../../tensor'; +import {WasmBinding} from '../../../wasm-binding'; +import {WasmInferenceHandler} from '../inference-handler'; + +export class WasmInstanceNormalization extends InstanceNormalization { + run(inferenceHandler: WasmInferenceHandler, inputs: Tensor[]): Tensor[] { + const x = inputs[0]; + const scale = inputs[1]; + const b = inputs[2]; + + // calculate channel size (i.e.) data points per channel + let channelSize = 1; + for (let i = 2; i < x.dims.length; i++) { + channelSize *= x.dims[i]; + } + + // create output Tensor after determining output size + const y = new Tensor(x.dims, x.type); + WasmBinding.getInstance().ccall( + '_instance_normalization_f32', [x.floatData, 'float32ptr'], [y.floatData, 'float32ptr', 'out'], + [x.dims[0], 'int32'], [x.dims[1], 'int32'], [channelSize, 'int32'], [scale.floatData, 'float32ptr'], + [b.floatData, 'float32ptr'], [this.epsilon, 'float32']); + + return [y]; + } + + // overriding the checkInputTypes() in the base class because Wasm backend has special type limitations + checkInputTypes(inputs: Tensor[]): boolean { + const X = inputs[0]; + const scale = inputs[1]; + const B = inputs[2]; + + // input should atleast have three dimensions - N,C,dim1,...,dimn + // other inputs need to be one dimensional + if (X.dims.length < 3 || scale.dims.length !== 1 || B.dims.length !== 1) { + return false; + } + if (scale.dims[0] !== X.dims[1] || B.dims[0] !== X.dims[1]) { + return false; + } + // currently Wasm backend only supports 'float32' input type + if (X.type !== 'float32' || scale.type !== 'float32' || B.type !== 'float32') { + return false; + } + return true; + } +} diff --git a/lib/backends/wasm/session-handler.ts b/lib/backends/wasm/session-handler.ts index 5a907469..eed12277 100644 --- a/lib/backends/wasm/session-handler.ts +++ b/lib/backends/wasm/session-handler.ts @@ -10,6 +10,7 @@ import {WasmInferenceHandler} from './inference-handler'; import {WasmBatchNormalization} from './ops/batch-normalization'; import {WasmConv} from './ops/conv'; import {WasmGemm} from './ops/gemm'; +import {WasmInstanceNormalization} from './ops/instance-normalization'; import {WasmAveragePool, WasmGlobalAveragePool, WasmGlobalMaxPool, WasmMaxPool} from './ops/pool'; import {WasmSoftmax} from './ops/softmax'; import {WasmSum} from './ops/sum'; @@ -50,6 +51,8 @@ export class WasmSessionHandler implements SessionHandler { return new WasmGlobalMaxPool(); case 'GlobalAveragePool': return new WasmGlobalAveragePool(); + case 'InstanceNormalization': + return new WasmInstanceNormalization(); default: if (this.fallbackToCpuOps) { return resolve(node, domain, version); diff --git a/lib/ops/instance-normalization.ts b/lib/ops/instance-normalization.ts new file mode 100644 index 00000000..bba059df --- /dev/null +++ b/lib/ops/instance-normalization.ts @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +import {Attribute} from '../attribute'; +import {InferenceHandler} from '../backend'; +import {Operator} from '../operators'; +import {Tensor} from '../tensor'; + +export abstract class InstanceNormalization implements Operator { + abstract run(inferenceHandler: InferenceHandler, inputs: Tensor[]): Tensor[]|Promise; + + initialize(attributes: Attribute): void { + this.epsilon = attributes.getFloat('epsilon', 1e-5); + } + + checkInputs(inputs: Tensor[]): boolean { + if (!inputs || inputs.length !== 3) { + return false; + } + + return this.checkInputTypes(inputs); + } + + protected checkInputTypes(inputs: Tensor[]): boolean { + const X = inputs[0]; + const scale = inputs[1]; + const B = inputs[2]; + + // input should atleast have three dimensions - N,C,dim1,...,dimn + // other inputs can have only one dimensions + if (X.dims.length < 3 || scale.dims.length !== 1 || B.dims.length !== 1) { + return false; + } + if (scale.dims[0] !== X.dims[1] || B.dims[0] !== X.dims[1]) { + return false; + } + if ((X.type !== 'float32' && X.type !== 'float64') || (scale.type !== 'float32' && scale.type !== 'float64') || + (B.type !== 'float32' && B.type !== 'float64')) { + return false; + } + return true; + } + + protected epsilon: number; +} diff --git a/src/wasm-build-config.json b/src/wasm-build-config.json index e17c0c3b..a12f6691 100644 --- a/src/wasm-build-config.json +++ b/src/wasm-build-config.json @@ -9,6 +9,7 @@ "_pool_f32", "_gemm_f32", "_batch_normalization_f32", + "_instance_normalization_f32", "_sum_f32", "_softmax_f32" ] diff --git a/src/wasm-ops/batch-normalization.cpp b/src/wasm-ops/batch-normalization.cpp index 2e1f7a76..a75ce947 100644 --- a/src/wasm-ops/batch-normalization.cpp +++ b/src/wasm-ops/batch-normalization.cpp @@ -18,8 +18,8 @@ void batch_normalization_f32(void *data) { } // Core operator implementation -void batch_normalization_f32_imp(float *X, float *Y, int batch_size, - int num_channels, int channel_size, +void batch_normalization_f32_imp(float *X, float *Y, int32_t batch_size, + int32_t num_channels, int32_t channel_size, float *scale, float *bias, float *mean, float *variance, float epsilon) { for (size_t nc = 0; nc < batch_size * num_channels; ++nc) { diff --git a/src/wasm-ops/instance-normalization.cpp b/src/wasm-ops/instance-normalization.cpp new file mode 100644 index 00000000..75b46f47 --- /dev/null +++ b/src/wasm-ops/instance-normalization.cpp @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +#include "instance-normalization.h" +#include "common.h" +#include + +// Wasm interop method +void instance_normalization_f32(void *data) { + uint32_t *dataIndex = static_cast(data); + uint32_t const argc = dataIndex[0]; + instance_normalization_f32_imp( + PARAM_FLOAT_PTR(data, dataIndex[1]), PARAM_FLOAT_PTR(data, dataIndex[2]), + PARAM_INT32(data, dataIndex[3]), PARAM_INT32(data, dataIndex[4]), + PARAM_INT32(data, dataIndex[5]), PARAM_FLOAT_PTR(data, dataIndex[6]), + PARAM_FLOAT_PTR(data, dataIndex[7]), PARAM_FLOAT(data, dataIndex[8])); +} + +// Core operator implementation +void instance_normalization_f32_imp(float *X, float *Y, int32_t batch_size, + int32_t num_channels, int32_t channel_size, + float *scale, float *bias, float epsilon) { + float temp; + float mean; + float variance; + size_t sampleOffset; + size_t physicalOffset; + size_t iterEnd; + size_t sample_size = channel_size * num_channels; + + for (size_t n = 0; n < batch_size; ++n) { + sampleOffset = n * sample_size; + for (size_t c = 0; c < num_channels; ++c) { + physicalOffset = sampleOffset + c * channel_size; + iterEnd = physicalOffset + channel_size; + + // compute mean for this channel + temp = 0; + for (int32_t i = physicalOffset; i < iterEnd; ++i) { + temp += X[i]; + } + mean = temp / channel_size; + + // compute variance for this channel + temp = 0; + for (size_t i = physicalOffset; i < iterEnd; ++i) { + temp += pow(X[i] - mean, 2); + } + variance = temp / channel_size; + + // compute normalized value for data in this channel + for (size_t i = physicalOffset; i < iterEnd; ++i) { + Y[i] = scale[c] * ((X[i] - mean) / sqrt(variance + epsilon)) + bias[c]; + } + } + } +} diff --git a/src/wasm-ops/instance-normalization.h b/src/wasm-ops/instance-normalization.h new file mode 100644 index 00000000..8b2d5fb3 --- /dev/null +++ b/src/wasm-ops/instance-normalization.h @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +#pragma once + +#include + +extern "C" { +void instance_normalization_f32(void *); +void instance_normalization_f32_imp(float *, float *, int32_t, int32_t, int32_t, + float *, float *, float); +} diff --git a/test/unittest-whitelist.jsonc b/test/unittest-whitelist.jsonc index 8f47e8d7..7beef6d5 100644 --- a/test/unittest-whitelist.jsonc +++ b/test/unittest-whitelist.jsonc @@ -70,6 +70,8 @@ "test_globalmaxpool_precomputed", "test_globalmaxpool", "test_identity", + "test_instancenorm_epsilon", + "test_instancenorm_example", "test_leakyrelu_default", "test_leakyrelu_example", "test_leakyrelu", @@ -515,7 +517,9 @@ "test_globalaveragepool_precomputed", "test_globalaveragepool", "test_globalmaxpool_precomputed", - "test_globalmaxpool" + "test_globalmaxpool", + "test_instancenorm_epsilon", + "test_instancenorm_example" ], "ops": [ // Check in op tests that have native Wasm implementations From 411069c10d6f8bb546947cd529874891fa079d7c Mon Sep 17 00:00:00 2001 From: hariharans29 Date: Thu, 24 Jan 2019 12:40:07 -0800 Subject: [PATCH 4/6] Formatiing changes --- src/wasm-ops/instance-normalization.cpp | 4 ++-- src/wasm-ops/instance-normalization.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/wasm-ops/instance-normalization.cpp b/src/wasm-ops/instance-normalization.cpp index 75b46f47..28144369 100644 --- a/src/wasm-ops/instance-normalization.cpp +++ b/src/wasm-ops/instance-normalization.cpp @@ -18,8 +18,8 @@ void instance_normalization_f32(void *data) { // Core operator implementation void instance_normalization_f32_imp(float *X, float *Y, int32_t batch_size, - int32_t num_channels, int32_t channel_size, - float *scale, float *bias, float epsilon) { + int32_t num_channels, int32_t channel_size, + float *scale, float *bias, float epsilon) { float temp; float mean; float variance; diff --git a/src/wasm-ops/instance-normalization.h b/src/wasm-ops/instance-normalization.h index 8b2d5fb3..03a8a737 100644 --- a/src/wasm-ops/instance-normalization.h +++ b/src/wasm-ops/instance-normalization.h @@ -8,5 +8,5 @@ extern "C" { void instance_normalization_f32(void *); void instance_normalization_f32_imp(float *, float *, int32_t, int32_t, int32_t, - float *, float *, float); + float *, float *, float); } From f1f63578fa8ee3aa707b5eca08a4e2e9e0c3effe Mon Sep 17 00:00:00 2001 From: hariharans29 Date: Thu, 24 Jan 2019 12:50:41 -0800 Subject: [PATCH 5/6] Cleanup to make code more concise --- .../cpu/ops/instance-normalization.ts | 44 +++++++++---------- src/wasm-ops/instance-normalization.cpp | 43 +++++++++--------- 2 files changed, 40 insertions(+), 47 deletions(-) diff --git a/lib/backends/cpu/ops/instance-normalization.ts b/lib/backends/cpu/ops/instance-normalization.ts index 11e46c02..d0f182cb 100644 --- a/lib/backends/cpu/ops/instance-normalization.ts +++ b/lib/backends/cpu/ops/instance-normalization.ts @@ -22,8 +22,6 @@ export function instanceNormalization(x: Tensor, scale: Tensor, b: Tensor, epsil for (let i = 2; i < inputDimensions.length; i++) { channelSize *= inputDimensions[i]; } - const sampleSize = C * channelSize; - const output = new Tensor(x.dims, x.type); const X = x.floatData; @@ -34,34 +32,32 @@ export function instanceNormalization(x: Tensor, scale: Tensor, b: Tensor, epsil let temp: number; let mean: number; let variance: number; - let sampleOffset: number; let physicalOffset: number; let iterEnd: number; + let currentChannel: number; - for (let n = 0; n < N; ++n) { - sampleOffset = n * sampleSize; - for (let c = 0; c < C; ++c) { - physicalOffset = sampleOffset + c * channelSize; - iterEnd = physicalOffset + channelSize; + for (let nc = 0; nc < N * C; nc++) { + physicalOffset = nc * channelSize; + iterEnd = physicalOffset + channelSize; + currentChannel = nc % C; - // compute mean for this channel - temp = 0; - for (let i = physicalOffset; i < iterEnd; ++i) { - temp += X[i]; - } - mean = temp / channelSize; + // compute mean for this channel + temp = 0; + for (let i = physicalOffset; i < iterEnd; ++i) { + temp += X[i]; + } + mean = temp / channelSize; - // compute variance for this channel - temp = 0; - for (let i = physicalOffset; i < iterEnd; ++i) { - temp += Math.pow(X[i] - mean, 2); - } - variance = temp / channelSize; + // compute variance for this channel + temp = 0; + for (let i = physicalOffset; i < iterEnd; ++i) { + temp += Math.pow(X[i] - mean, 2); + } + variance = temp / channelSize; - // compute normalized value for data in this channel - for (let i = physicalOffset; i < iterEnd; ++i) { - Y[i] = scaleData[c] * ((X[i] - mean) / Math.sqrt(variance + epsilon)) + bData[c]; - } + // compute normalized value for data in this channel + for (let i = physicalOffset; i < iterEnd; ++i) { + Y[i] = scaleData[currentChannel] * ((X[i] - mean) / Math.sqrt(variance + epsilon)) + bData[currentChannel]; } } diff --git a/src/wasm-ops/instance-normalization.cpp b/src/wasm-ops/instance-normalization.cpp index 28144369..29a15e3b 100644 --- a/src/wasm-ops/instance-normalization.cpp +++ b/src/wasm-ops/instance-normalization.cpp @@ -23,35 +23,32 @@ void instance_normalization_f32_imp(float *X, float *Y, int32_t batch_size, float temp; float mean; float variance; - size_t sampleOffset; size_t physicalOffset; size_t iterEnd; - size_t sample_size = channel_size * num_channels; + size_t currentChannel; - for (size_t n = 0; n < batch_size; ++n) { - sampleOffset = n * sample_size; - for (size_t c = 0; c < num_channels; ++c) { - physicalOffset = sampleOffset + c * channel_size; - iterEnd = physicalOffset + channel_size; + for (size_t nc = 0; nc < batch_size * channel_size; nc++) { + physicalOffset = nc * channel_size; + iterEnd = physicalOffset + channel_size; + currentChannel = nc % num_channels; - // compute mean for this channel - temp = 0; - for (int32_t i = physicalOffset; i < iterEnd; ++i) { - temp += X[i]; - } - mean = temp / channel_size; + // compute mean for this channel + temp = 0; + for (size_t i = physicalOffset; i < iterEnd; ++i) { + temp += X[i]; + } + mean = temp / channel_size; - // compute variance for this channel - temp = 0; - for (size_t i = physicalOffset; i < iterEnd; ++i) { - temp += pow(X[i] - mean, 2); - } - variance = temp / channel_size; + // compute variance for this channel + temp = 0; + for (size_t i = physicalOffset; i < iterEnd; ++i) { + temp += pow(X[i] - mean, 2); + } + variance = temp / channel_size; - // compute normalized value for data in this channel - for (size_t i = physicalOffset; i < iterEnd; ++i) { - Y[i] = scale[c] * ((X[i] - mean) / sqrt(variance + epsilon)) + bias[c]; - } + // compute normalized value for data in this channel + for (size_t i = physicalOffset; i < iterEnd; ++i) { + Y[i] = scale[currentChannel] * ((X[i] - mean) / sqrt(variance + epsilon)) + bias[currentChannel]; } } } From 00b8d4bf9c27201f63b8fb74fcf31c316ab45a6b Mon Sep 17 00:00:00 2001 From: hariharans29 Date: Thu, 24 Jan 2019 12:51:16 -0800 Subject: [PATCH 6/6] Formatting --- src/wasm-ops/instance-normalization.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/wasm-ops/instance-normalization.cpp b/src/wasm-ops/instance-normalization.cpp index 29a15e3b..6bac9358 100644 --- a/src/wasm-ops/instance-normalization.cpp +++ b/src/wasm-ops/instance-normalization.cpp @@ -48,7 +48,9 @@ void instance_normalization_f32_imp(float *X, float *Y, int32_t batch_size, // compute normalized value for data in this channel for (size_t i = physicalOffset; i < iterEnd; ++i) { - Y[i] = scale[currentChannel] * ((X[i] - mean) / sqrt(variance + epsilon)) + bias[currentChannel]; + Y[i] = + scale[currentChannel] * ((X[i] - mean) / sqrt(variance + epsilon)) + + bias[currentChannel]; } } }