cpu: update unary operators

lib/backends/cpu/op-resolve-rules.ts

@@ -27,63 +27,73 @@ import {CpuSum} from './ops/sum';
 import {CpuTile} from './ops/tile';
 import {CpuTranspose} from './ops/transpose';
 import * as unaryOps from './ops/unary-op';
+import {CpuUnaryOp} from './ops/unary-op';
 import {CpuUnsqueeze} from './ops/unsqueeze';
 
 export const CPU_OP_RESOLVE_RULES: ReadonlyArray<OpSet.ResolveRule> = [
-  ['Abs', '', '6+', () => new unaryOps.CpuUnaryOp(NUMBER_TYPES, unaryOps.abs)],
-  ['Acos', '', '7+', () => new unaryOps.CpuUnaryOp(NUMBER_TYPES, unaryOps.acos)],
+  ['Abs', '', '6+', () => new CpuUnaryOp(NUMBER_TYPES, unaryOps.abs)],
+  ['Acos', '', '7+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.acos)],
+  ['Acosh', '', '9+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.acosh)],
   ['Add', '', '7+', () => new CpuBinaryOp(NUMBER_TYPES, (e1, e2) => (e1 + e2))],
   ['And', '', '7+', () => new CpuBinaryOp(['bool'], (e1, e2) => (e1 && e2))],
   ['ArgMax', '', '1+', () => new CpuArgMax()],
-  ['Asin', '', '7+', () => new unaryOps.CpuUnaryOp(NUMBER_TYPES, unaryOps.asin)],
-  ['Atan', '', '7+', () => new unaryOps.CpuUnaryOp(NUMBER_TYPES, unaryOps.atan)],
+  ['Asin', '', '7+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.asin)],
+  ['Asinh', '', '9+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.asinh)],
+  ['Atan', '', '7+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.atan)],
+  ['Atanh', '', '9+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.atanh)],
   ['AveragePool', '', '7+', () => new CpuAveragePool()],  // TODO: support new attributes for AveragePool-10
   ['BatchNormalization', '', '7+', () => new CpuBatchNormalization()],
-  ['Ceil', '', '6+', () => new unaryOps.CpuUnaryOp(NUMBER_TYPES, unaryOps.ceil)],
-  ['Clip', '', '6+', () => new unaryOps.CpuUnaryOp(FLOAT_TYPES, unaryOps.clip)],
+  ['Ceil', '', '6+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.ceil)],
+  ['Clip', '', '6+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.clip, unaryOps.clipInitializer)],
   ['Concat', '', '4+', () => new CpuConcat()],
   ['Conv', '', '1+', () => new CpuConv()],
-  ['Cos', '', '7+', () => new unaryOps.CpuUnaryOp(NUMBER_TYPES, unaryOps.cos)],
+  ['Cos', '', '7+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.cos)],
+  ['Cosh', '', '9+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.cosh)],
   ['Div', '', '7+', () => new CpuBinaryOp(NUMBER_TYPES, (e1, e2) => (e1 / e2))],
   ['Dropout', '', '7+', () => new CpuDropout()],
-  ['Elu', '', '6+', () => new unaryOps.CpuUnaryOp(FLOAT_TYPES, unaryOps.elu)],
-  ['Exp', '', '6+', () => new unaryOps.CpuUnaryOp(NUMBER_TYPES, unaryOps.exp)],
+  ['Elu', '', '6+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.elu, unaryOps.eluInitializer)],
+  ['Exp', '', '6+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.exp)],
   ['Flatten', '', '1+', () => new CpuFlatten()],
-  ['Floor', '', '6+', () => new unaryOps.CpuUnaryOp(NUMBER_TYPES, unaryOps.floor)],
+  ['Floor', '', '6+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.floor)],
   ['Gather', '', '1+', () => new CpuGather()],
   ['Gemm', '', '7+', () => new CpuGemm()],
   ['GlobalAveragePool', '', '1+', () => new CpuGlobalAveragePool()],
   ['GlobalMaxPool', '', '1+', () => new CpuGlobalMaxPool()],
   ['ImageScaler', '', '1+', () => new CpuImageScaler()],
   ['InstanceNormalization', '', '6+', () => new CpuInstanceNormalization()],
-  ['LeakyRelu', '', '6+', () => new unaryOps.CpuUnaryOp(FLOAT_TYPES, unaryOps.leakyRelu)],
-  ['Log', '', '6+', () => new unaryOps.CpuUnaryOp(NUMBER_TYPES, unaryOps.log)],
+  ['IsNaN', '', '9+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.isNan, undefined, 'bool')],
+  ['LeakyRelu', '', '6+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.leakyRelu, unaryOps.leakyReluInitializer)],
+  ['Log', '', '6+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.log)],
   ['LRN', '', '1+', () => new CpuLrn()],
   ['MatMul', '', '1+', () => new CpuMatMul()],
   ['MaxPool', '', '1+', () => new CpuMaxPool()],  // TODO: support new attributes for MaxPool-8 and MaxPool-10
   ['Mul', '', '7+', () => new CpuBinaryOp(NUMBER_TYPES, (e1, e2) => (e1 * e2))],
-  ['Neg', '', '6+', () => new unaryOps.CpuUnaryOp(NUMBER_TYPES, unaryOps.neg)],
+  ['Neg', '', '6+', () => new CpuUnaryOp(NUMBER_TYPES, unaryOps.neg)],
+  ['Not', '', '1+', () => new CpuUnaryOp(['bool'], unaryOps.not, undefined, 'bool')],
   ['Or', '', '7+', () => new CpuBinaryOp(['bool'], (e1, e2) => (e1 || e2))],
   ['PRelu', '', '7+', () => new CpuBinaryOp(NUMBER_TYPES, (e1, e2) => (e1 >= 0 ? e1 : e1 * e2))],
+  ['Reciprocal', '', '6+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.reciprocal)],
   ['ReduceLogSum', '', '1+', () => new cpuReduce.CpuReduceLogSum()],
   ['ReduceMax', '', '1+', () => new cpuReduce.CpuReduceMax()],
   ['ReduceMean', '', '1+', () => new cpuReduce.CpuReduceMean()],
   ['ReduceMin', '', '1+', () => new cpuReduce.CpuReduceMin()],
   ['ReduceProd', '', '1+', () => new cpuReduce.CpuReduceProd()],
   ['ReduceSum', '', '1+', () => new cpuReduce.CpuReduceSum()],
   ['ReduceSumSquare', '', '1+', () => new cpuReduce.CpuReduceSumSquare()],
-  ['Relu', '', '6+', () => new unaryOps.CpuUnaryOp(NUMBER_TYPES, unaryOps.relu)],
+  ['Relu', '', '6+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.relu)],
   ['Reshape', '', '5+', () => new CpuReshape()],
-  ['Sigmoid', '', '6+', () => new unaryOps.CpuUnaryOp(NUMBER_TYPES, unaryOps.sigmoid)],
-  ['Sin', '', '7+', () => new unaryOps.CpuUnaryOp(NUMBER_TYPES, unaryOps.sin)],
+  ['Sigmoid', '', '6+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.sigmoid)],
+  ['Sign', '', '9+', () => new CpuUnaryOp(NUMBER_TYPES, unaryOps.sign)],
+  ['Sin', '', '7+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.sin)],
+  ['Sinh', '', '9+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.sinh)],
   ['Slice', '', '1+', () => new CpuSlice()],
   ['Softmax', '', '1+', () => new CpuSoftmax()],
-  ['Sqrt', '', '6+', () => new unaryOps.CpuUnaryOp(NUMBER_TYPES, unaryOps.sqrt)],
+  ['Sqrt', '', '6+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.sqrt)],
   ['Squeeze', '', '1+', () => new CpuSqueeze()],
   ['Sub', '', '7+', () => new CpuBinaryOp(NUMBER_TYPES, (e1, e2) => (e1 - e2))],
   ['Sum', '', '6+', () => new CpuSum()],  // TODO: support multidirectional broadcast for Sum-8
-  ['Tan', '', '7+', () => new unaryOps.CpuUnaryOp(NUMBER_TYPES, unaryOps.tan)],
-  ['Tanh', '', '6+', () => new unaryOps.CpuUnaryOp(NUMBER_TYPES, unaryOps.tanh)],
+  ['Tan', '', '7+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.tan)],
+  ['Tanh', '', '6+', () => new CpuUnaryOp(FLOAT_TYPES, unaryOps.tanh)],
   ['Tile', '', '6+', () => new CpuTile()],
   ['Transpose', '', '1+', () => new CpuTranspose()],
   ['Unsqueeze', '', '1+', () => new CpuUnsqueeze()],

lib/backends/cpu/ops/unary-op.ts

@@ -6,14 +6,23 @@ import {UnaryOp} from '../../../ops/unary-op';
 import {Tensor} from '../../../tensor';
 import {CpuInferenceHandler} from '../inference-handler';
 
-type UnaryOpCoreFunction = (input: Tensor.NumberType, output: Tensor.NumberType, attributes: Attribute) => void;
+type UnaryOpCoreFunction<T> = (input: Tensor.NumberType, output: Tensor.NumberType, attributes?: T) => void;
+
+export class CpuUnaryOp<T = unknown> extends UnaryOp {
+  private attributes?: T;
 
-export class CpuUnaryOp extends UnaryOp {
   constructor(
-      typeConstraint: ReadonlyArray<Tensor.DataType>, private func: UnaryOpCoreFunction, resultType?: Tensor.DataType) {
+      typeConstraint: ReadonlyArray<Tensor.DataType>, private func: UnaryOpCoreFunction<T>,
+      private attributesInitializer?: (attributes: Attribute) => T, resultType?: Tensor.DataType) {
     super(typeConstraint, resultType);
   }
 
+  initialize(attributes: Attribute) {
+    if (this.attributesInitializer) {
+      this.attributes = this.attributesInitializer(attributes);
+    }
+  }
+
   run(inferenceHandler: CpuInferenceHandler, inputs: Tensor[]): Tensor[] {
     // TODO:  use webpack + ts-loader + CustomTransformer
     // tslint:disable-next-line:max-line-length
@@ -23,8 +32,8 @@ export class CpuUnaryOp extends UnaryOp {
   }
 }
 
-export function unaryOp(
-    x: Tensor, func: UnaryOpCoreFunction, attributes: Attribute, resultType?: Tensor.DataType): Tensor {
+export function unaryOp<T>(
+    x: Tensor, func: UnaryOpCoreFunction<T>, attributes: T, resultType?: Tensor.DataType): Tensor {
   const output = new Tensor(x.dims, resultType ? resultType : x.type);
   const inputNumberData = x.data as Tensor.NumberType;
   const outputNumberData = output.data as Tensor.NumberType;
@@ -37,123 +46,192 @@ export function unaryOp(
 // that approach was found to be detrimental to performance
 // so we use this approach which involves slight code duplication
 
-export function abs(input: Tensor.NumberType, output: Tensor.NumberType, attributes: Attribute) {
+export function abs(input: Tensor.NumberType, output: Tensor.NumberType) {
   for (let i = 0; i < input.length; i++) {
     output[i] = Math.abs(input[i]);
   }
 }
 
-export function neg(input: Tensor.NumberType, output: Tensor.NumberType, attributes: Attribute) {
+export function acos(input: Tensor.NumberType, output: Tensor.NumberType) {
   for (let i = 0; i < input.length; i++) {
-    output[i] = -input[i];
+    output[i] = Math.acos(input[i]);
   }
 }
 
-export function acos(input: Tensor.NumberType, output: Tensor.NumberType, attributes: Attribute) {
+export function acosh(input: Tensor.NumberType, output: Tensor.NumberType) {
   for (let i = 0; i < input.length; i++) {
-    output[i] = Math.acos(input[i]);
+    output[i] = Math.acosh(input[i]);
   }
 }
 
-export function ceil(input: Tensor.NumberType, output: Tensor.NumberType, attributes: Attribute) {
+export function asin(input: Tensor.NumberType, output: Tensor.NumberType) {
   for (let i = 0; i < input.length; i++) {
-    output[i] = Math.ceil(input[i]);
+    output[i] = Math.asin(input[i]);
   }
 }
 
-export function cos(input: Tensor.NumberType, output: Tensor.NumberType, attributes: Attribute) {
+export function asinh(input: Tensor.NumberType, output: Tensor.NumberType) {
   for (let i = 0; i < input.length; i++) {
-    output[i] = Math.cos(input[i]);
+    output[i] = Math.asinh(input[i]);
+  }
+}
+
+export function atan(input: Tensor.NumberType, output: Tensor.NumberType) {
+  for (let i = 0; i < input.length; i++) {
+    output[i] = Math.atan(input[i]);
+  }
+}
+
+export function atanh(input: Tensor.NumberType, output: Tensor.NumberType) {
+  for (let i = 0; i < input.length; i++) {
+    output[i] = Math.atanh(input[i]);
+  }
+}
+
+export function ceil(input: Tensor.NumberType, output: Tensor.NumberType) {
+  for (let i = 0; i < input.length; i++) {
+    output[i] = Math.ceil(input[i]);
   }
 }
 
-export function clip(input: Tensor.NumberType, output: Tensor.NumberType, attributes: Attribute) {
-  const min = attributes.getFloat('min', -3.4028234663852886e+38);
-  const max = attributes.getFloat('max', 3.4028234663852886e+38);
+export function clipInitializer(attributes: Attribute) {
+  return {
+    min: attributes.getFloat('min', -3.4028234663852886e+38),
+    max: attributes.getFloat('max', 3.4028234663852886e+38)
+  };
+}
+
+export function clip(input: Tensor.NumberType, output: Tensor.NumberType, attributes: {min: number, max: number}) {
+  const min = attributes.min;
+  const max = attributes.max;
   for (let i = 0; i < input.length; i++) {
     const value = input[i];
     output[i] = (value < min) ? min : (value > max) ? max : value;
   }
 }
 
-export function sin(input: Tensor.NumberType, output: Tensor.NumberType, attributes: Attribute) {
+export function cos(input: Tensor.NumberType, output: Tensor.NumberType) {
   for (let i = 0; i < input.length; i++) {
-    output[i] = Math.sin(input[i]);
+    output[i] = Math.cos(input[i]);
   }
 }
 
-export function tan(input: Tensor.NumberType, output: Tensor.NumberType, attributes: Attribute) {
+export function cosh(input: Tensor.NumberType, output: Tensor.NumberType) {
   for (let i = 0; i < input.length; i++) {
-    output[i] = Math.tan(input[i]);
+    output[i] = Math.cosh(input[i]);
   }
 }
 
-export function tanh(input: Tensor.NumberType, output: Tensor.NumberType, attributes: Attribute) {
+export function eluInitializer(attributes: Attribute) {
+  return attributes.getFloat('alpha', 1.0);
+}
+
+export function elu(input: Tensor.NumberType, output: Tensor.NumberType, attributes: number) {
+  const alpha = attributes;
   for (let i = 0; i < input.length; i++) {
-    output[i] = Math.tanh(input[i]);
+    const value = input[i];
+    output[i] = value >= 0 ? value : alpha * (Math.exp(value) - 1.0);
   }
 }
 
-export function exp(input: Tensor.NumberType, output: Tensor.NumberType, attributes: Attribute) {
+export function exp(input: Tensor.NumberType, output: Tensor.NumberType) {
   for (let i = 0; i < input.length; i++) {
     output[i] = Math.exp(input[i]);
   }
 }
 
-export function floor(input: Tensor.NumberType, output: Tensor.NumberType, attributes: Attribute) {
+export function floor(input: Tensor.NumberType, output: Tensor.NumberType) {
   for (let i = 0; i < input.length; i++) {
     output[i] = Math.floor(input[i]);
   }
 }
 
-export function atan(input: Tensor.NumberType, output: Tensor.NumberType, attributes: Attribute) {
+export function isNan(input: Tensor.NumberType, output: Tensor.BooleanType) {
   for (let i = 0; i < input.length; i++) {
-    output[i] = Math.atan(input[i]);
+    output[i] = Number.isNaN(input[i]) ? 1 : 0;
   }
 }
 
-export function relu(input: Tensor.NumberType, output: Tensor.NumberType, attributes: Attribute) {
+export function leakyReluInitializer(attributes: Attribute) {
+  return attributes.getFloat('alpha', 0.01);
+}
+
+export function leakyRelu(input: Tensor.NumberType, output: Tensor.NumberType, attributes: number) {
+  const alpha = attributes;
+  for (let i = 0; i < input.length; i++) {
+    const value = input[i];
+    output[i] = value >= 0 ? value : alpha * value;
+  }
+}
+
+export function log(input: Tensor.NumberType, output: Tensor.NumberType) {
+  for (let i = 0; i < input.length; i++) {
+    output[i] = Math.log(input[i]);
+  }
+}
+
+export function neg(input: Tensor.NumberType, output: Tensor.NumberType) {
+  for (let i = 0; i < input.length; i++) {
+    output[i] = -input[i];
+  }
+}
+
+export function not(input: Tensor.BooleanType, output: Tensor.BooleanType) {
+  for (let i = 0; i < input.length; i++) {
+    output[i] = input[i] ? 0 : 1;
+  }
+}
+
+export function reciprocal(input: Tensor.NumberType, output: Tensor.NumberType) {
+  for (let i = 0; i < input.length; i++) {
+    output[i] = 1.0 / input[i];
+  }
+}
+
+export function relu(input: Tensor.NumberType, output: Tensor.NumberType) {
   for (let i = 0; i < input.length; i++) {
     output[i] = Math.max(0, input[i]);
   }
 }
 
-export function leakyRelu(input: Tensor.NumberType, output: Tensor.NumberType, attributes: Attribute) {
-  const alpha = attributes.getFloat('alpha', 0.01);
+export function sigmoid(input: Tensor.NumberType, output: Tensor.NumberType) {
   for (let i = 0; i < input.length; i++) {
-    const value = input[i];
-    output[i] = value >= 0 ? value : alpha * value;
+    output[i] = (1 / (1 + Math.exp(-input[i])));
   }
 }
 
-export function elu(input: Tensor.NumberType, output: Tensor.NumberType, attributes: Attribute) {
-  const alpha = attributes.getFloat('alpha', 1.0);
+export function sign(input: Tensor.NumberType, output: Tensor.NumberType) {
   for (let i = 0; i < input.length; i++) {
-    const value = input[i];
-    output[i] = value >= 0 ? value : alpha * (Math.exp(value) - 1.0);
+    output[i] = input[i] > 0 ? 1 : input[i] < 0 ? -1 : 0;
   }
 }
 
-export function log(input: Tensor.NumberType, output: Tensor.NumberType, attributes: Attribute) {
+export function sin(input: Tensor.NumberType, output: Tensor.NumberType) {
   for (let i = 0; i < input.length; i++) {
-    output[i] = Math.log(input[i]);
+    output[i] = Math.sin(input[i]);
+  }
+}
+
+export function sinh(input: Tensor.NumberType, output: Tensor.NumberType) {
+  for (let i = 0; i < input.length; i++) {
+    output[i] = Math.sinh(input[i]);
   }
 }
 
-export function sqrt(input: Tensor.NumberType, output: Tensor.NumberType, attributes: Attribute) {
+export function sqrt(input: Tensor.NumberType, output: Tensor.NumberType) {
   for (let i = 0; i < input.length; i++) {
     output[i] = Math.sqrt(input[i]);
   }
 }
 
-export function asin(input: Tensor.NumberType, output: Tensor.NumberType, attributes: Attribute) {
+export function tan(input: Tensor.NumberType, output: Tensor.NumberType) {
   for (let i = 0; i < input.length; i++) {
-    output[i] = Math.asin(input[i]);
+    output[i] = Math.tan(input[i]);
   }
 }
 
-export function sigmoid(input: Tensor.NumberType, output: Tensor.NumberType, attributes: Attribute) {
+export function tanh(input: Tensor.NumberType, output: Tensor.NumberType) {
   for (let i = 0; i < input.length; i++) {
-    output[i] = (1 / (1 + Math.exp(-input[i])));
+    output[i] = Math.tanh(input[i]);
   }
 }

lib/backends/webgl/ops/unary-op.ts

@@ -1,7 +1,6 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license.
 
-import {Attribute} from '../../../attribute';
 import {UnaryOp} from '../../../ops/unary-op';
 import {Tensor} from '../../../tensor';
 import {FunctionType, GlslValueFunction} from '../glsl-definitions';
@@ -15,7 +14,6 @@ export class WebGLUnaryOp extends UnaryOp implements WebGLOperator {
   constructor(protected typeConstraint: ReadonlyArray<Tensor.DataType>, protected glslFunc: GlslValueFunction) {
     super(typeConstraint);
   }
-  initialize(attributes: Attribute): void {}
   run(inferenceHandler: WebGLInferenceHandler, inputs: Tensor[]): Tensor[] {
     return WebGLOperatorHelper.run(this, inferenceHandler, inputs);
   }

lib/ops/unary-op.ts

@@ -11,9 +11,7 @@ export abstract class UnaryOp implements Operator {
 
   abstract run(inferenceHandler: InferenceHandler, inputs: Tensor[]): Tensor[]|Promise<Tensor[]>;
 
-  initialize(attributes: Attribute): void {
-    this.attributes = attributes;
-  }
+  initialize(attributes: Attribute): void {}
 
   checkInputs(inputs: Tensor[]): boolean {
     if (!inputs || inputs.length !== 1) {
@@ -30,6 +28,4 @@ export abstract class UnaryOp implements Operator {
 
     return true;
   }
-
-  protected attributes: Attribute;
 }