diff --git a/.benchrc.yaml b/.benchrc.yaml index 04c74137..4772b534 100644 --- a/.benchrc.yaml +++ b/.benchrc.yaml @@ -2,6 +2,7 @@ colors: true require: - ts-node/register + - setHasher.mjs # benchmark opts threshold: 3 diff --git a/.eslintrc.js b/.eslintrc.js index 6a14b236..bce205f9 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -81,6 +81,9 @@ module.exports = { // Prevents accidentally pushing a commit with .only in Mocha tests "no-only-tests/no-only-tests": "error", + + // TEMP Disabled while eslint-plugin-import support ESM (Typescript does support it) https://github.com/import-js/eslint-plugin-import/issues/2170 + "import/no-unresolved": "off", }, overrides: [ { diff --git a/packages/as-sha256/package.json b/packages/as-sha256/package.json index 7c48e393..90d992ad 100644 --- a/packages/as-sha256/package.json +++ b/packages/as-sha256/package.json @@ -13,6 +13,19 @@ "url": "git+https://github.com/chainsafe/as-sha256.git" }, "main": "lib/index.js", + "exports": { + ".": "./lib/index.js", + "./hashObject": "./lib/hashObject.js" + }, + "typesVersions": { + "*": { + "*": [ + "*", + "lib/*", + "lib/*/index" + ] + } + }, "types": "lib/index.d.ts", "files": [ "lib", diff --git a/packages/persistent-merkle-tree/.mocharc.yaml b/packages/persistent-merkle-tree/.mocharc.yaml index b2f7befe..52a2d88f 100644 --- a/packages/persistent-merkle-tree/.mocharc.yaml +++ b/packages/persistent-merkle-tree/.mocharc.yaml @@ -1,2 +1,4 @@ colors: true -require: ts-node/register +require: +- ts-node/register +- ../../setHasher.mjs diff --git a/packages/persistent-merkle-tree/package.json b/packages/persistent-merkle-tree/package.json index a9c7c621..5bfe8b6d 100644 --- a/packages/persistent-merkle-tree/package.json +++ b/packages/persistent-merkle-tree/package.json @@ -3,6 +3,21 @@ "version": "0.5.0", "description": "Merkle tree implemented as a persistent datastructure", "main": "lib/index.js", + "exports": { + ".": "./lib/index.js", + "./hasher": "./lib/hasher/index.js", + "./hasher/as-sha256": "./lib/hasher/as-sha256.js", + "./hasher/noble": "./lib/hasher/noble.js" + }, + "typesVersions": { + "*": { + "*": [ + "*", + "lib/*", + "lib/*/index" + ] + } + }, "files": [ "lib" ], @@ -35,6 +50,7 @@ }, "homepage": "https://github.com/ChainSafe/persistent-merkle-tree#readme", "dependencies": { - "@chainsafe/as-sha256": "^0.3.1" + "@chainsafe/as-sha256": "^0.3.1", + "@noble/hashes": "^1.3.0" } } diff --git a/packages/persistent-merkle-tree/src/hash.ts b/packages/persistent-merkle-tree/src/hash.ts deleted file mode 100644 index 4952a9ea..00000000 --- a/packages/persistent-merkle-tree/src/hash.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { - byteArrayToHashObject, - digest64, - digest64HashObjects, - HashObject, - hashObjectToByteArray, -} from "@chainsafe/as-sha256"; - -const input = new Uint8Array(64); - -/** - * Hash two 32 byte arrays - */ -export function hash(a: Uint8Array, b: Uint8Array): Uint8Array { - input.set(a, 0); - input.set(b, 32); - return digest64(input); -} - -/** - * Hash 2 objects, each store 8 numbers (equivalent to Uint8Array(32)) - */ -export function hashTwoObjects(a: HashObject, b: HashObject): HashObject { - return digest64HashObjects(a, b); -} - -export function hashObjectToUint8Array(obj: HashObject): Uint8Array { - const byteArr = new Uint8Array(32); - hashObjectToByteArray(obj, byteArr, 0); - return byteArr; -} - -export function uint8ArrayToHashObject(byteArr: Uint8Array): HashObject { - return byteArrayToHashObject(byteArr); -} - -export function isHashObject(hash: HashObject | Uint8Array): hash is HashObject { - // @ts-ignore - return hash.length === undefined; -} diff --git a/packages/persistent-merkle-tree/src/hasher/as-sha256.ts b/packages/persistent-merkle-tree/src/hasher/as-sha256.ts new file mode 100644 index 00000000..07095345 --- /dev/null +++ b/packages/persistent-merkle-tree/src/hasher/as-sha256.ts @@ -0,0 +1,7 @@ +import {digest2Bytes32, digest64HashObjects} from "@chainsafe/as-sha256"; +import type {Hasher} from "./types"; + +export const hasher: Hasher = { + digest64: digest2Bytes32, + digest64HashObjects, +}; diff --git a/packages/persistent-merkle-tree/src/hasher/index.ts b/packages/persistent-merkle-tree/src/hasher/index.ts new file mode 100644 index 00000000..b2b549d3 --- /dev/null +++ b/packages/persistent-merkle-tree/src/hasher/index.ts @@ -0,0 +1,20 @@ +import {Hasher} from "./types"; +import {hasher as nobleHasher} from "./noble"; + +export {HashObject} from "@chainsafe/as-sha256/hashObject"; +export * from "./types"; +export * from "./util"; + +/** + * Hasher used across the SSZ codebase + */ +export let hasher: Hasher = nobleHasher; + +/** + * Set the hasher to be used across the SSZ codebase + * + * WARNING: This function is intended for power users and must be executed before any other SSZ code is imported + */ +export function setHasher(newHasher: Hasher): void { + hasher = newHasher; +} diff --git a/packages/persistent-merkle-tree/src/hasher/noble.ts b/packages/persistent-merkle-tree/src/hasher/noble.ts new file mode 100644 index 00000000..7877f97e --- /dev/null +++ b/packages/persistent-merkle-tree/src/hasher/noble.ts @@ -0,0 +1,10 @@ +import {sha256} from "@noble/hashes/sha256"; +import type {Hasher} from "./types"; +import {hashObjectToUint8Array, uint8ArrayToHashObject} from "./util"; + +const digest64 = (a: Uint8Array, b: Uint8Array): Uint8Array => sha256.create().update(a).update(b).digest(); + +export const hasher: Hasher = { + digest64, + digest64HashObjects: (a, b) => uint8ArrayToHashObject(digest64(hashObjectToUint8Array(a), hashObjectToUint8Array(b))), +}; diff --git a/packages/persistent-merkle-tree/src/hasher/types.ts b/packages/persistent-merkle-tree/src/hasher/types.ts new file mode 100644 index 00000000..96b7c942 --- /dev/null +++ b/packages/persistent-merkle-tree/src/hasher/types.ts @@ -0,0 +1,12 @@ +import type {HashObject} from "@chainsafe/as-sha256/hashObject"; + +export type Hasher = { + /** + * Hash two 32-byte Uint8Arrays + */ + digest64(a32Bytes: Uint8Array, b32Bytes: Uint8Array): Uint8Array; + /** + * Hash two 32-byte HashObjects + */ + digest64HashObjects(a: HashObject, b: HashObject): HashObject; +}; diff --git a/packages/persistent-merkle-tree/src/hasher/util.ts b/packages/persistent-merkle-tree/src/hasher/util.ts new file mode 100644 index 00000000..6756372f --- /dev/null +++ b/packages/persistent-merkle-tree/src/hasher/util.ts @@ -0,0 +1,11 @@ +import {byteArrayToHashObject, HashObject, hashObjectToByteArray} from "@chainsafe/as-sha256/hashObject"; + +export function hashObjectToUint8Array(obj: HashObject): Uint8Array { + const byteArr = new Uint8Array(32); + hashObjectToByteArray(obj, byteArr, 0); + return byteArr; +} + +export function uint8ArrayToHashObject(byteArr: Uint8Array): HashObject { + return byteArrayToHashObject(byteArr); +} diff --git a/packages/persistent-merkle-tree/src/index.ts b/packages/persistent-merkle-tree/src/index.ts index 6ae7c6f1..d3ff35a5 100644 --- a/packages/persistent-merkle-tree/src/index.ts +++ b/packages/persistent-merkle-tree/src/index.ts @@ -1,5 +1,5 @@ export * from "./gindex"; -export * from "./hash"; +export * from "./hasher"; export * from "./node"; export * from "./packedNode"; export * from "./proof"; diff --git a/packages/persistent-merkle-tree/src/node.ts b/packages/persistent-merkle-tree/src/node.ts index d31cc389..3b0d9f00 100644 --- a/packages/persistent-merkle-tree/src/node.ts +++ b/packages/persistent-merkle-tree/src/node.ts @@ -1,5 +1,5 @@ -import {HashObject} from "@chainsafe/as-sha256"; -import {hashObjectToUint8Array, hashTwoObjects, uint8ArrayToHashObject} from "./hash"; +import {HashObject} from "@chainsafe/as-sha256/hashObject"; +import {hashObjectToUint8Array, hasher, uint8ArrayToHashObject} from "./hasher"; const TWO_POWER_32 = 2 ** 32; @@ -72,7 +72,7 @@ export class BranchNode extends Node { get rootHashObject(): HashObject { if (this.h0 === null) { - super.applyHash(hashTwoObjects(this.left.rootHashObject, this.right.rootHashObject)); + super.applyHash(hasher.digest64HashObjects(this.left.rootHashObject, this.right.rootHashObject)); } return this; } diff --git a/packages/persistent-merkle-tree/test/perf/hash.test.ts b/packages/persistent-merkle-tree/test/perf/hash.test.ts deleted file mode 100644 index e5e7d69a..00000000 --- a/packages/persistent-merkle-tree/test/perf/hash.test.ts +++ /dev/null @@ -1,26 +0,0 @@ -import {itBench} from "@dapplion/benchmark"; -import {uint8ArrayToHashObject, hash, hashTwoObjects} from "../../src/hash"; - -describe("hash", () => { - const root1 = new Uint8Array(32); - const root2 = new Uint8Array(32); - for (let i = 0; i < root1.length; i++) { - root1[i] = 1; - } - for (let i = 0; i < root2.length; i++) { - root2[i] = 2; - } - - // total number of time running hash for 250_000 validators - const iterations = 2250026; - - itBench(`hash 2 Uint8Array ${iterations} times`, () => { - for (let j = 0; j < iterations; j++) hash(root1, root2); - }); - - const obj1 = uint8ArrayToHashObject(root1); - const obj2 = uint8ArrayToHashObject(root2); - itBench(`hashTwoObjects ${iterations} times`, () => { - for (let j = 0; j < iterations; j++) hashTwoObjects(obj1, obj2); - }); -}); diff --git a/packages/persistent-merkle-tree/test/perf/hasher.test.ts b/packages/persistent-merkle-tree/test/perf/hasher.test.ts new file mode 100644 index 00000000..e375057f --- /dev/null +++ b/packages/persistent-merkle-tree/test/perf/hasher.test.ts @@ -0,0 +1,33 @@ +import {itBench} from "@dapplion/benchmark"; +import {uint8ArrayToHashObject} from "../../src/hasher"; +import {hasher as asShaHasher} from "../../src/hasher/as-sha256"; +import {hasher as nobleHasher} from "../../src/hasher/noble"; + +describe("hasher", () => { + const root1 = new Uint8Array(32); + const root2 = new Uint8Array(32); + for (let i = 0; i < root1.length; i++) { + root1[i] = 1; + } + for (let i = 0; i < root2.length; i++) { + root2[i] = 2; + } + + // total number of time running hash for 250_000 validators + const iterations = 2250026; + + for (const {hasher, name} of [ + {hasher: asShaHasher, name: "as-sha256"}, + {hasher: nobleHasher, name: "noble"}, + ]) { + itBench(`hash 2 Uint8Array ${iterations} times - ${name}`, () => { + for (let j = 0; j < iterations; j++) hasher.digest64(root1, root2); + }); + + const obj1 = uint8ArrayToHashObject(root1); + const obj2 = uint8ArrayToHashObject(root2); + itBench(`hashTwoObjects ${iterations} times - ${name}`, () => { + for (let j = 0; j < iterations; j++) hasher.digest64HashObjects(obj1, obj2); + }); + } +}); diff --git a/packages/persistent-merkle-tree/test/hash.test.ts b/packages/persistent-merkle-tree/test/unit/hasher.test.ts similarity index 55% rename from packages/persistent-merkle-tree/test/hash.test.ts rename to packages/persistent-merkle-tree/test/unit/hasher.test.ts index ec10735b..f51ca461 100644 --- a/packages/persistent-merkle-tree/test/hash.test.ts +++ b/packages/persistent-merkle-tree/test/unit/hasher.test.ts @@ -1,15 +1,15 @@ import { expect } from "chai"; -import {uint8ArrayToHashObject, hash, hashTwoObjects, hashObjectToUint8Array} from "../src/hash"; +import {uint8ArrayToHashObject, hasher, hashObjectToUint8Array} from "../../src/hasher"; -describe("hash", function () { - it("hash and hashTwoObjects should be the same", () => { +describe("hasher", function () { + it("hasher methods should be the same", () => { const root1 = Buffer.alloc(32, 1); const root2 = Buffer.alloc(32, 2); - const root = hash(root1, root2); + const root = hasher.digest64(root1, root2); const obj1 = uint8ArrayToHashObject(root1); const obj2 = uint8ArrayToHashObject(root2); - const obj = hashTwoObjects(obj1, obj2); + const obj = hasher.digest64HashObjects(obj1, obj2); const newRoot = hashObjectToUint8Array(obj); expect(newRoot).to.be.deep.equal(root, "hash and hash2 is not equal"); }); diff --git a/packages/persistent-merkle-tree/test/unit/proof/index.test.ts b/packages/persistent-merkle-tree/test/unit/proof/index.test.ts index 2a7da945..d5664245 100644 --- a/packages/persistent-merkle-tree/test/unit/proof/index.test.ts +++ b/packages/persistent-merkle-tree/test/unit/proof/index.test.ts @@ -29,7 +29,7 @@ describe("proof equivalence", () => { } }); it("should compute the same root from different proof types - multiple leaves", function () { - this.timeout(2000); + this.timeout(10_000); const depth = 6; const maxIndex = 2 ** depth; const node = createTree(depth); diff --git a/packages/ssz/.mocharc.yaml b/packages/ssz/.mocharc.yaml index a2596399..e129395e 100644 --- a/packages/ssz/.mocharc.yaml +++ b/packages/ssz/.mocharc.yaml @@ -1,5 +1,7 @@ colors: true -require: ts-node/register +require: +- ts-node/register +- ../../setHasher.mjs # benchmark maxMs: 30_000 threshold: 3 diff --git a/packages/ssz/src/branchNodeStruct.ts b/packages/ssz/src/branchNodeStruct.ts index 19f65851..4eb32cab 100644 --- a/packages/ssz/src/branchNodeStruct.ts +++ b/packages/ssz/src/branchNodeStruct.ts @@ -1,4 +1,4 @@ -import {HashObject} from "@chainsafe/as-sha256"; +import {HashObject} from "@chainsafe/as-sha256/hashObject"; import {hashObjectToUint8Array, Node} from "@chainsafe/persistent-merkle-tree"; /** diff --git a/packages/ssz/src/util/merkleize.ts b/packages/ssz/src/util/merkleize.ts index b35b73e9..eff32ff0 100644 --- a/packages/ssz/src/util/merkleize.ts +++ b/packages/ssz/src/util/merkleize.ts @@ -1,8 +1,8 @@ -import {digest2Bytes32} from "@chainsafe/as-sha256"; +import {hasher} from "@chainsafe/persistent-merkle-tree/hasher"; import {zeroHash} from "./zeros"; export function hash64(bytes32A: Uint8Array, bytes32B: Uint8Array): Uint8Array { - return digest2Bytes32(bytes32A, bytes32B); + return hasher.digest64(bytes32A, bytes32B); } export function merkleize(chunks: Uint8Array[], padFor: number): Uint8Array { diff --git a/packages/ssz/src/util/zeros.ts b/packages/ssz/src/util/zeros.ts index 9dc14d1e..c6d22ff4 100644 --- a/packages/ssz/src/util/zeros.ts +++ b/packages/ssz/src/util/zeros.ts @@ -1,4 +1,4 @@ -import {digest2Bytes32} from "@chainsafe/as-sha256"; +import {hasher} from "@chainsafe/persistent-merkle-tree/hasher"; // create array of "zero hashes", successively hashed zero chunks const zeroHashes = [new Uint8Array(32)]; @@ -6,7 +6,7 @@ const zeroHashes = [new Uint8Array(32)]; export function zeroHash(depth: number): Uint8Array { if (depth >= zeroHashes.length) { for (let i = zeroHashes.length; i <= depth; i++) { - zeroHashes[i] = digest2Bytes32(zeroHashes[i - 1], zeroHashes[i - 1]); + zeroHashes[i] = hasher.digest64(zeroHashes[i - 1], zeroHashes[i - 1]); } } return zeroHashes[depth]; diff --git a/packages/ssz/test/perf/eth2/hashTreeRoot.test.ts b/packages/ssz/test/perf/eth2/hashTreeRoot.test.ts index 0d8bb00d..00923197 100644 --- a/packages/ssz/test/perf/eth2/hashTreeRoot.test.ts +++ b/packages/ssz/test/perf/eth2/hashTreeRoot.test.ts @@ -1,5 +1,5 @@ import {itBench} from "@dapplion/benchmark"; -import {hashTwoObjects, uint8ArrayToHashObject} from "@chainsafe/persistent-merkle-tree"; +import {hasher, uint8ArrayToHashObject} from "@chainsafe/persistent-merkle-tree"; import * as sszPhase0 from "../../lodestarTypes/phase0/sszTypes"; import * as sszAltair from "../../lodestarTypes/altair/sszTypes"; import { @@ -130,7 +130,7 @@ describe("HashTreeRoot individual components", () => { }); itBench(`hashTwoObjects x${count}`, () => { - for (let i = 0; i < count; i++) hashTwoObjects(ho, ho); + for (let i = 0; i < count; i++) hasher.digest64HashObjects(ho, ho); }); } diff --git a/setHasher.mjs b/setHasher.mjs new file mode 100644 index 00000000..9d6c3800 --- /dev/null +++ b/setHasher.mjs @@ -0,0 +1,5 @@ +// Set the hasher to as-sha256 +// Used to run benchmarks with with visibility into as-sha256 performance, useful for Lodestar +import {setHasher} from "@chainsafe/persistent-merkle-tree/hasher"; +import {hasher} from "@chainsafe/persistent-merkle-tree/hasher/as-sha256"; +setHasher(hasher); diff --git a/yarn.lock b/yarn.lock index 11ea2368..f7d830fe 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1669,6 +1669,7 @@ __metadata: resolution: "@chainsafe/persistent-merkle-tree@workspace:packages/persistent-merkle-tree" dependencies: "@chainsafe/as-sha256": ^0.3.1 + "@noble/hashes": ^1.3.0 languageName: unknown linkType: soft @@ -1920,6 +1921,13 @@ __metadata: languageName: node linkType: hard +"@noble/hashes@npm:^1.3.0": + version: 1.3.0 + resolution: "@noble/hashes@npm:1.3.0" + checksum: 06d27f9e7dfbe379dbfb02073358aa2c25a6363acc3a2b09e074f0f47f630da1d5a81c731696faa9407594371b61a27c496076fdecc2f60fb1c6a24247a5dbef + languageName: node + linkType: hard + "@nodelib/fs.scandir@npm:2.1.5": version: 2.1.5 resolution: "@nodelib/fs.scandir@npm:2.1.5"