feat: add swappable hasher, default to noble-hashes (#314)

* Add swappable hasher, default to noble-hashes

* Use as-sha256 hasher in benchmarks

* Add some comments

* Use as-sha256 hasher in tests

* Increase test timeout

.benchrc.yaml

@@ -2,6 +2,7 @@
 colors: true
 require:
   - ts-node/register
+  - setHasher.mjs
 
 # benchmark opts
 threshold: 3

.eslintrc.js

@@ -81,6 +81,9 @@ module.exports = {
 
     // Prevents accidentally pushing a commit with .only in Mocha tests
     "no-only-tests/no-only-tests": "error",
+
+    // TEMP Disabled while eslint-plugin-import support ESM (Typescript does support it) https://github.com/import-js/eslint-plugin-import/issues/2170
+    "import/no-unresolved": "off",
   },
   overrides: [
     {

packages/as-sha256/package.json

@@ -13,6 +13,19 @@
     "url": "git+https://github.com/chainsafe/as-sha256.git"
   },
   "main": "lib/index.js",
+  "exports": {
+    ".": "./lib/index.js",
+    "./hashObject": "./lib/hashObject.js"
+  },
+  "typesVersions": {
+    "*": {
+      "*": [
+        "*",
+        "lib/*",
+        "lib/*/index"
+      ]
+    }
+  },
   "types": "lib/index.d.ts",
   "files": [
     "lib",

packages/persistent-merkle-tree/.mocharc.yaml

@@ -1,2 +1,4 @@
 colors: true
-require: ts-node/register
+require:
+- ts-node/register
+- ../../setHasher.mjs

packages/persistent-merkle-tree/package.json

@@ -3,6 +3,21 @@
   "version": "0.5.0",
   "description": "Merkle tree implemented as a persistent datastructure",
   "main": "lib/index.js",
+  "exports": {
+    ".": "./lib/index.js",
+    "./hasher": "./lib/hasher/index.js",
+    "./hasher/as-sha256": "./lib/hasher/as-sha256.js",
+    "./hasher/noble": "./lib/hasher/noble.js"
+  },
+  "typesVersions": {
+    "*": {
+      "*": [
+        "*",
+        "lib/*",
+        "lib/*/index"
+      ]
+    }
+  },
   "files": [
     "lib"
   ],
@@ -35,6 +50,7 @@
   },
   "homepage": "https://github.com/ChainSafe/persistent-merkle-tree#readme",
   "dependencies": {
-    "@chainsafe/as-sha256": "^0.3.1"
+    "@chainsafe/as-sha256": "^0.3.1",
+    "@noble/hashes": "^1.3.0"
   }
 }

packages/persistent-merkle-tree/src/hasher/as-sha256.ts

@@ -0,0 +1,7 @@
+import {digest2Bytes32, digest64HashObjects} from "@chainsafe/as-sha256";
+import type {Hasher} from "./types";
+
+export const hasher: Hasher = {
+  digest64: digest2Bytes32,
+  digest64HashObjects,
+};

packages/persistent-merkle-tree/src/hasher/index.ts

@@ -0,0 +1,20 @@
+import {Hasher} from "./types";
+import {hasher as nobleHasher} from "./noble";
+
+export {HashObject} from "@chainsafe/as-sha256/hashObject";
+export * from "./types";
+export * from "./util";
+
+/**
+ * Hasher used across the SSZ codebase
+ */
+export let hasher: Hasher = nobleHasher;
+
+/**
+ * Set the hasher to be used across the SSZ codebase
+ *
+ * WARNING: This function is intended for power users and must be executed before any other SSZ code is imported
+ */
+export function setHasher(newHasher: Hasher): void {
+  hasher = newHasher;
+}

packages/persistent-merkle-tree/src/hasher/noble.ts

@@ -0,0 +1,10 @@
+import {sha256} from "@noble/hashes/sha256";
+import type {Hasher} from "./types";
+import {hashObjectToUint8Array, uint8ArrayToHashObject} from "./util";
+
+const digest64 = (a: Uint8Array, b: Uint8Array): Uint8Array => sha256.create().update(a).update(b).digest();
+
+export const hasher: Hasher = {
+  digest64,
+  digest64HashObjects: (a, b) => uint8ArrayToHashObject(digest64(hashObjectToUint8Array(a), hashObjectToUint8Array(b))),
+};

packages/persistent-merkle-tree/src/hasher/types.ts

@@ -0,0 +1,12 @@
+import type {HashObject} from "@chainsafe/as-sha256/hashObject";
+
+export type Hasher = {
+  /**
+   * Hash two 32-byte Uint8Arrays
+   */
+  digest64(a32Bytes: Uint8Array, b32Bytes: Uint8Array): Uint8Array;
+  /**
+   * Hash two 32-byte HashObjects
+   */
+  digest64HashObjects(a: HashObject, b: HashObject): HashObject;
+};

packages/persistent-merkle-tree/src/hasher/util.ts

@@ -0,0 +1,11 @@
+import {byteArrayToHashObject, HashObject, hashObjectToByteArray} from "@chainsafe/as-sha256/hashObject";
+
+export function hashObjectToUint8Array(obj: HashObject): Uint8Array {
+  const byteArr = new Uint8Array(32);
+  hashObjectToByteArray(obj, byteArr, 0);
+  return byteArr;
+}
+
+export function uint8ArrayToHashObject(byteArr: Uint8Array): HashObject {
+  return byteArrayToHashObject(byteArr);
+}

packages/persistent-merkle-tree/src/index.ts

@@ -1,5 +1,5 @@
 export * from "./gindex";
-export * from "./hash";
+export * from "./hasher";
 export * from "./node";
 export * from "./packedNode";
 export * from "./proof";

packages/persistent-merkle-tree/src/node.ts

@@ -1,5 +1,5 @@
-import {HashObject} from "@chainsafe/as-sha256";
-import {hashObjectToUint8Array, hashTwoObjects, uint8ArrayToHashObject} from "./hash";
+import {HashObject} from "@chainsafe/as-sha256/hashObject";
+import {hashObjectToUint8Array, hasher, uint8ArrayToHashObject} from "./hasher";
 
 const TWO_POWER_32 = 2 ** 32;
 
@@ -72,7 +72,7 @@ export class BranchNode extends Node {
 
   get rootHashObject(): HashObject {
     if (this.h0 === null) {
-      super.applyHash(hashTwoObjects(this.left.rootHashObject, this.right.rootHashObject));
+      super.applyHash(hasher.digest64HashObjects(this.left.rootHashObject, this.right.rootHashObject));
     }
     return this;
   }

packages/persistent-merkle-tree/test/perf/hasher.test.ts

@@ -0,0 +1,33 @@
+import {itBench} from "@dapplion/benchmark";
+import {uint8ArrayToHashObject} from "../../src/hasher";
+import {hasher as asShaHasher} from "../../src/hasher/as-sha256";
+import {hasher as nobleHasher} from "../../src/hasher/noble";
+
+describe("hasher", () => {
+  const root1 = new Uint8Array(32);
+  const root2 = new Uint8Array(32);
+  for (let i = 0; i < root1.length; i++) {
+    root1[i] = 1;
+  }
+  for (let i = 0; i < root2.length; i++) {
+    root2[i] = 2;
+  }
+
+  // total number of time running hash for 250_000 validators
+  const iterations = 2250026;
+
+  for (const {hasher, name} of [
+    {hasher: asShaHasher, name: "as-sha256"},
+    {hasher: nobleHasher, name: "noble"},
+  ]) {
+    itBench(`hash 2 Uint8Array ${iterations} times - ${name}`, () => {
+      for (let j = 0; j < iterations; j++) hasher.digest64(root1, root2);
+    });
+
+    const obj1 = uint8ArrayToHashObject(root1);
+    const obj2 = uint8ArrayToHashObject(root2);
+    itBench(`hashTwoObjects ${iterations} times - ${name}`, () => {
+      for (let j = 0; j < iterations; j++) hasher.digest64HashObjects(obj1, obj2);
+    });
+  }
+});

packages/persistent-merkle-tree/test/hash.test.ts → packages/persistent-merkle-tree/test/unit/hasher.test.ts

@@ -1,15 +1,15 @@
 import { expect } from "chai";
-import {uint8ArrayToHashObject, hash, hashTwoObjects, hashObjectToUint8Array} from "../src/hash";
+import {uint8ArrayToHashObject, hasher, hashObjectToUint8Array} from "../../src/hasher";
 
-describe("hash", function () {
-  it("hash and hashTwoObjects should be the same", () => {
+describe("hasher", function () {
+  it("hasher methods should be the same", () => {
     const root1 = Buffer.alloc(32, 1);
     const root2 = Buffer.alloc(32, 2);
-    const root = hash(root1, root2);
+    const root = hasher.digest64(root1, root2);
 
     const obj1 = uint8ArrayToHashObject(root1);
     const obj2 = uint8ArrayToHashObject(root2);
-    const obj = hashTwoObjects(obj1, obj2);
+    const obj = hasher.digest64HashObjects(obj1, obj2);
     const newRoot = hashObjectToUint8Array(obj);
     expect(newRoot).to.be.deep.equal(root, "hash and hash2 is not equal");
   });

packages/persistent-merkle-tree/test/unit/proof/index.test.ts

@@ -29,7 +29,7 @@ describe("proof equivalence", () => {
     }
   });
   it("should compute the same root from different proof types - multiple leaves", function () {
-    this.timeout(2000);
+    this.timeout(10_000);
     const depth = 6;
     const maxIndex = 2 ** depth;
     const node = createTree(depth);

packages/ssz/.mocharc.yaml

@@ -1,5 +1,7 @@
 colors: true
-require: ts-node/register
+require:
+- ts-node/register
+- ../../setHasher.mjs
 # benchmark
 maxMs: 30_000
 threshold: 3

packages/ssz/src/branchNodeStruct.ts

@@ -1,4 +1,4 @@
-import {HashObject} from "@chainsafe/as-sha256";
+import {HashObject} from "@chainsafe/as-sha256/hashObject";
 import {hashObjectToUint8Array, Node} from "@chainsafe/persistent-merkle-tree";
 
 /**

packages/ssz/src/util/merkleize.ts

@@ -1,8 +1,8 @@
-import {digest2Bytes32} from "@chainsafe/as-sha256";
+import {hasher} from "@chainsafe/persistent-merkle-tree/hasher";
 import {zeroHash} from "./zeros";
 
 export function hash64(bytes32A: Uint8Array, bytes32B: Uint8Array): Uint8Array {
-  return digest2Bytes32(bytes32A, bytes32B);
+  return hasher.digest64(bytes32A, bytes32B);
 }
 
 export function merkleize(chunks: Uint8Array[], padFor: number): Uint8Array {

packages/ssz/src/util/zeros.ts

@@ -1,12 +1,12 @@
-import {digest2Bytes32} from "@chainsafe/as-sha256";
+import {hasher} from "@chainsafe/persistent-merkle-tree/hasher";
 
 // create array of "zero hashes", successively hashed zero chunks
 const zeroHashes = [new Uint8Array(32)];
 
 export function zeroHash(depth: number): Uint8Array {
   if (depth >= zeroHashes.length) {
     for (let i = zeroHashes.length; i <= depth; i++) {
-      zeroHashes[i] = digest2Bytes32(zeroHashes[i - 1], zeroHashes[i - 1]);
+      zeroHashes[i] = hasher.digest64(zeroHashes[i - 1], zeroHashes[i - 1]);
     }
   }
   return zeroHashes[depth];

packages/ssz/test/perf/eth2/hashTreeRoot.test.ts

@@ -1,5 +1,5 @@
 import {itBench} from "@dapplion/benchmark";
-import {hashTwoObjects, uint8ArrayToHashObject} from "@chainsafe/persistent-merkle-tree";
+import {hasher, uint8ArrayToHashObject} from "@chainsafe/persistent-merkle-tree";
 import * as sszPhase0 from "../../lodestarTypes/phase0/sszTypes";
 import * as sszAltair from "../../lodestarTypes/altair/sszTypes";
 import {
@@ -130,7 +130,7 @@ describe("HashTreeRoot individual components", () => {
     });
 
     itBench(`hashTwoObjects x${count}`, () => {
-      for (let i = 0; i < count; i++) hashTwoObjects(ho, ho);
+      for (let i = 0; i < count; i++) hasher.digest64HashObjects(ho, ho);
     });
   }
 

setHasher.mjs

@@ -0,0 +1,5 @@
+// Set the hasher to as-sha256
+// Used to run benchmarks with with visibility into as-sha256 performance, useful for Lodestar
+import {setHasher} from "@chainsafe/persistent-merkle-tree/hasher";
+import {hasher} from "@chainsafe/persistent-merkle-tree/hasher/as-sha256";
+setHasher(hasher);

yarn.lock

@@ -1669,6 +1669,7 @@ __metadata:
   resolution: "@chainsafe/persistent-merkle-tree@workspace:packages/persistent-merkle-tree"
   dependencies:
     "@chainsafe/as-sha256": ^0.3.1
+    "@noble/hashes": ^1.3.0
   languageName: unknown
   linkType: soft
 
@@ -1920,6 +1921,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@noble/hashes@npm:^1.3.0":
+  version: 1.3.0
+  resolution: "@noble/hashes@npm:1.3.0"
+  checksum: 06d27f9e7dfbe379dbfb02073358aa2c25a6363acc3a2b09e074f0f47f630da1d5a81c731696faa9407594371b61a27c496076fdecc2f60fb1c6a24247a5dbef
+  languageName: node
+  linkType: hard
+
 "@nodelib/fs.scandir@npm:2.1.5":
   version: 2.1.5
   resolution: "@nodelib/fs.scandir@npm:2.1.5"