diff --git a/packages/ssz/src/type/arrayComposite.ts b/packages/ssz/src/type/arrayComposite.ts index 986b0e0a..d77e89dc 100644 --- a/packages/ssz/src/type/arrayComposite.ts +++ b/packages/ssz/src/type/arrayComposite.ts @@ -211,29 +211,29 @@ export function tree_deserializeFromBytesArrayComposite>( +export function value_getBlocksBytesArrayComposite>( elementType: ElementType, length: number, value: ValueOf[], - chunkBytesBuffer: Uint8Array + blocksBuffer: Uint8Array ): Uint8Array { - const isOddChunk = length % 2 === 1; - const chunkBytesLen = isOddChunk ? length * 32 + 32 : length * 32; - if (chunkBytesLen > chunkBytesBuffer.length) { - throw new Error(`chunkBytesBuffer is too small: ${chunkBytesBuffer.length} < ${chunkBytesLen}`); + const blockBytesLen = Math.ceil(length / 2) * 64; + if (blockBytesLen > blocksBuffer.length) { + throw new Error(`blocksBuffer is too small: ${blocksBuffer.length} < ${blockBytesLen}`); } - const chunkBytes = chunkBytesBuffer.subarray(0, chunkBytesLen); + const blocksBytes = blocksBuffer.subarray(0, blockBytesLen); for (let i = 0; i < length; i++) { - elementType.hashTreeRootInto(value[i], chunkBytes, i * 32); + elementType.hashTreeRootInto(value[i], blocksBytes, i * 32); } + const isOddChunk = length % 2 === 1; if (isOddChunk) { // similar to append zeroHash(0) - chunkBytes.subarray(length * 32, chunkBytesLen).fill(0); + blocksBytes.subarray(length * 32, blockBytesLen).fill(0); } - return chunkBytes; + return blocksBytes; } function readOffsetsArrayComposite( diff --git a/packages/ssz/src/type/bitArray.ts b/packages/ssz/src/type/bitArray.ts index d485de27..469cd131 100644 --- a/packages/ssz/src/type/bitArray.ts +++ b/packages/ssz/src/type/bitArray.ts @@ -4,7 +4,7 @@ import {CompositeType, LENGTH_GINDEX} from "./composite"; import {BitArray} from "../value/bitArray"; import {BitArrayTreeView} from "../view/bitArray"; import {BitArrayTreeViewDU} from "../viewDU/bitArray"; -import {getChunkBytes} from "./byteArray"; +import {getBlocksBytes} from "./byteArray"; /* eslint-disable @typescript-eslint/member-ordering */ @@ -40,15 +40,13 @@ export abstract class BitArrayType extends CompositeType this.chunkBytesBuffer.length) { + protected getBlocksBytes(value: BitArray): Uint8Array { + // reallocate this.blocksBuffer if needed + if (value.uint8Array.length > this.blocksBuffer.length) { const chunkCount = Math.ceil(value.bitLen / 8 / 32); - const chunkBytes = chunkCount * 32; - // pad 1 chunk if maxChunkCount is not even - this.chunkBytesBuffer = chunkCount % 2 === 1 ? new Uint8Array(chunkBytes + 32) : new Uint8Array(chunkBytes); + this.blocksBuffer = new Uint8Array(Math.ceil(chunkCount / 2) * 64); } - return getChunkBytes(value.uint8Array, this.chunkBytesBuffer); + return getBlocksBytes(value.uint8Array, this.blocksBuffer); } // Proofs diff --git a/packages/ssz/src/type/bitList.ts b/packages/ssz/src/type/bitList.ts index ba1c419a..c343ac48 100644 --- a/packages/ssz/src/type/bitList.ts +++ b/packages/ssz/src/type/bitList.ts @@ -1,7 +1,7 @@ import {allocUnsafe} from "@chainsafe/as-sha256"; import { getNodesAtDepth, - merkleizeInto, + merkleizeBlocksBytes, Node, packedNodeRootsToBytes, packedRootsBytesToNode, @@ -36,11 +36,11 @@ export class BitListType extends BitArrayType { readonly maxSize: number; readonly maxChunkCount: number; readonly isList = true; - readonly mixInLengthChunkBytes = new Uint8Array(64); + readonly mixInLengthBlockBytes = new Uint8Array(64); readonly mixInLengthBuffer = Buffer.from( - this.mixInLengthChunkBytes.buffer, - this.mixInLengthChunkBytes.byteOffset, - this.mixInLengthChunkBytes.byteLength + this.mixInLengthBlockBytes.buffer, + this.mixInLengthBlockBytes.byteOffset, + this.mixInLengthBlockBytes.byteLength ); constructor(readonly limitBits: number, opts?: BitListOptions) { @@ -120,12 +120,12 @@ export class BitListType extends BitArrayType { } hashTreeRootInto(value: BitArray, output: Uint8Array, offset: number): void { - super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0); + super.hashTreeRootInto(value, this.mixInLengthBlockBytes, 0); // mixInLength this.mixInLengthBuffer.writeUIntLE(value.bitLen, 32, 6); // one for hashTreeRoot(value), one for length const chunkCount = 2; - merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset); + merkleizeBlocksBytes(this.mixInLengthBlockBytes, chunkCount, output, offset); } // Proofs: inherited from BitArrayType diff --git a/packages/ssz/src/type/byteArray.ts b/packages/ssz/src/type/byteArray.ts index 78e6ae30..fc67037b 100644 --- a/packages/ssz/src/type/byteArray.ts +++ b/packages/ssz/src/type/byteArray.ts @@ -89,15 +89,13 @@ export abstract class ByteArrayType extends CompositeType this.chunkBytesBuffer.length) { + protected getBlocksBytes(value: ByteArray): Uint8Array { + // reallocate this.blocksBuffer if needed + if (value.length > this.blocksBuffer.length) { const chunkCount = Math.ceil(value.length / 32); - const chunkBytes = chunkCount * 32; - // pad 1 chunk if maxChunkCount is not even - this.chunkBytesBuffer = chunkCount % 2 === 1 ? new Uint8Array(chunkBytes + 32) : new Uint8Array(chunkBytes); + this.blocksBuffer = new Uint8Array(Math.ceil(chunkCount / 2) * 64); } - return getChunkBytes(value, this.chunkBytesBuffer); + return getBlocksBytes(value, this.blocksBuffer); } // Proofs @@ -162,15 +160,15 @@ export abstract class ByteArrayType extends CompositeType merkleBytesBuffer.length) { - throw new Error(`data length ${data.length} exceeds merkleBytesBuffer length ${merkleBytesBuffer.length}`); +export function getBlocksBytes(value: Uint8Array, blocksBuffer: Uint8Array): Uint8Array { + if (value.length > blocksBuffer.length) { + throw new Error(`data length ${value.length} exceeds blocksBuffer length ${blocksBuffer.length}`); } - merkleBytesBuffer.set(data); - const valueLen = data.length; - const chunkByteLen = Math.ceil(valueLen / 64) * 64; + blocksBuffer.set(value); + const valueLen = value.length; + const blockByteLen = Math.ceil(valueLen / 64) * 64; // all padding bytes must be zero, this is similar to set zeroHash(0) - merkleBytesBuffer.subarray(valueLen, chunkByteLen).fill(0); - return merkleBytesBuffer.subarray(0, chunkByteLen); + blocksBuffer.subarray(valueLen, blockByteLen).fill(0); + return blocksBuffer.subarray(0, blockByteLen); } diff --git a/packages/ssz/src/type/byteList.ts b/packages/ssz/src/type/byteList.ts index 53c46d39..0ceaab7a 100644 --- a/packages/ssz/src/type/byteList.ts +++ b/packages/ssz/src/type/byteList.ts @@ -4,7 +4,8 @@ import { Node, packedNodeRootsToBytes, packedRootsBytesToNode, - merkleizeInto, + merkleizeBlocksBytes, + merkleizeBlockArray, } from "@chainsafe/persistent-merkle-tree"; import {maxChunksToDepth} from "../util/merkleize"; import {Require} from "../util/types"; @@ -40,11 +41,13 @@ export class ByteListType extends ByteArrayType { readonly maxSize: number; readonly maxChunkCount: number; readonly isList = true; - readonly mixInLengthChunkBytes = new Uint8Array(64); + readonly blockArray: Uint8Array[] = []; + private blockBytesLen = 0; + readonly mixInLengthBlockBytes = new Uint8Array(64); readonly mixInLengthBuffer = Buffer.from( - this.mixInLengthChunkBytes.buffer, - this.mixInLengthChunkBytes.byteOffset, - this.mixInLengthChunkBytes.byteLength + this.mixInLengthBlockBytes.buffer, + this.mixInLengthBlockBytes.byteOffset, + this.mixInLengthBlockBytes.byteLength ); constructor(readonly limitBytes: number, opts?: ByteListOptions) { @@ -106,13 +109,44 @@ export class ByteListType extends ByteArrayType { return root; } + /** + * Use merkleizeBlockArray() instead of merkleizeBlocksBytes() to avoid big memory allocation + */ hashTreeRootInto(value: Uint8Array, output: Uint8Array, offset: number): void { - super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0); + // should not call super.hashTreeRoot() here + // use merkleizeBlockArray() instead of merkleizeBlocksBytes() to avoid big memory allocation + // reallocate this.blockArray if needed + if (value.length > this.blockBytesLen) { + const newBlockCount = Math.ceil(value.length / 64); + // this.blockBytesLen should be a multiple of 64 + const oldBlockCount = Math.ceil(this.blockBytesLen / 64); + const blockDiff = newBlockCount - oldBlockCount; + const newBlocksBytes = new Uint8Array(blockDiff * 64); + for (let i = 0; i < blockDiff; i++) { + this.blockArray.push(newBlocksBytes.subarray(i * 64, (i + 1) * 64)); + this.blockBytesLen += 64; + } + } + + // populate this.blockArray + for (let i = 0; i < value.length; i += 64) { + const block = this.blockArray[i / 64]; + // zero out the last block if it's over value.length + if (i + 64 > value.length) { + block.fill(0); + } + block.set(value.subarray(i, Math.min(i + 64, value.length))); + } + + // compute hashTreeRoot + const blockLimit = Math.ceil(value.length / 64); + merkleizeBlockArray(this.blockArray, blockLimit, this.maxChunkCount, this.mixInLengthBlockBytes, 0); + // mixInLength this.mixInLengthBuffer.writeUIntLE(value.length, 32, 6); // one for hashTreeRoot(value), one for length const chunkCount = 2; - merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset); + merkleizeBlocksBytes(this.mixInLengthBlockBytes, chunkCount, output, offset); } // Proofs: inherited from BitArrayType diff --git a/packages/ssz/src/type/composite.ts b/packages/ssz/src/type/composite.ts index 9f33ca5e..ce70be4b 100644 --- a/packages/ssz/src/type/composite.ts +++ b/packages/ssz/src/type/composite.ts @@ -8,7 +8,7 @@ import { Proof, ProofType, Tree, - merkleizeInto, + merkleizeBlocksBytes, HashComputationLevel, } from "@chainsafe/persistent-merkle-tree"; import {byteArrayEquals} from "../util/byteArray"; @@ -61,7 +61,7 @@ export abstract class CompositeType extends Type { * Required for ContainerNodeStruct to ensure no dangerous types are constructed. */ abstract readonly isViewMutable: boolean; - protected chunkBytesBuffer = new Uint8Array(0); + protected blocksBuffer = new Uint8Array(0); constructor( /** @@ -238,8 +238,8 @@ export abstract class CompositeType extends Type { } } - const merkleBytes = this.getChunkBytes(value); - merkleizeInto(merkleBytes, this.maxChunkCount, output, offset); + const blocksBuffer = this.getBlocksBytes(value); + merkleizeBlocksBytes(blocksBuffer, this.maxChunkCount, output, offset); if (this.cachePermanentRootStruct) { cacheRoot(value as ValueWithCachedPermanentRoot, output, offset, safeCache); } @@ -258,10 +258,10 @@ export abstract class CompositeType extends Type { // to hashObject and back. /** - * Get merkle bytes of each value, the returned Uint8Array should be multiple of 64 bytes. + * Get multiple SHA256 blocks, each is 64 bytes long. * If chunk count is not even, need to append zeroHash(0) */ - protected abstract getChunkBytes(value: V): Uint8Array; + protected abstract getBlocksBytes(value: V): Uint8Array; // Proofs API diff --git a/packages/ssz/src/type/container.ts b/packages/ssz/src/type/container.ts index 1ed46a89..2d6505ea 100644 --- a/packages/ssz/src/type/container.ts +++ b/packages/ssz/src/type/container.ts @@ -131,8 +131,7 @@ export class ContainerType>> extends this.TreeView = opts?.getContainerTreeViewClass?.(this) ?? getContainerTreeViewClass(this); this.TreeViewDU = opts?.getContainerTreeViewDUClass?.(this) ?? getContainerTreeViewDUClass(this); const fieldBytes = this.fieldsEntries.length * 32; - const chunkBytes = Math.ceil(fieldBytes / 64) * 64; - this.chunkBytesBuffer = new Uint8Array(chunkBytes); + this.blocksBuffer = new Uint8Array(Math.ceil(fieldBytes / 64) * 64); } static named>>( @@ -275,13 +274,13 @@ export class ContainerType>> extends // Merkleization - protected getChunkBytes(struct: ValueOfFields): Uint8Array { + protected getBlocksBytes(struct: ValueOfFields): Uint8Array { for (let i = 0; i < this.fieldsEntries.length; i++) { const {fieldName, fieldType} = this.fieldsEntries[i]; - fieldType.hashTreeRootInto(struct[fieldName], this.chunkBytesBuffer, i * 32); + fieldType.hashTreeRootInto(struct[fieldName], this.blocksBuffer, i * 32); } // remaining bytes are zeroed as we never write them - return this.chunkBytesBuffer; + return this.blocksBuffer; } // Proofs diff --git a/packages/ssz/src/type/listBasic.ts b/packages/ssz/src/type/listBasic.ts index 4dd63f08..9af933d7 100644 --- a/packages/ssz/src/type/listBasic.ts +++ b/packages/ssz/src/type/listBasic.ts @@ -1,4 +1,4 @@ -import {HashComputationLevel, LeafNode, Node, Tree, merkleizeInto} from "@chainsafe/persistent-merkle-tree"; +import {HashComputationLevel, LeafNode, Node, Tree, merkleizeBlocksBytes} from "@chainsafe/persistent-merkle-tree"; import {ValueOf} from "./abstract"; import {BasicType} from "./basic"; import {ByteViews} from "./composite"; @@ -47,11 +47,11 @@ export class ListBasicType> readonly maxSize: number; readonly isList = true; readonly isViewMutable = true; - readonly mixInLengthChunkBytes = new Uint8Array(64); + readonly mixInLengthBlockBytes = new Uint8Array(64); readonly mixInLengthBuffer = Buffer.from( - this.mixInLengthChunkBytes.buffer, - this.mixInLengthChunkBytes.byteOffset, - this.mixInLengthChunkBytes.byteLength + this.mixInLengthBlockBytes.buffer, + this.mixInLengthBlockBytes.byteOffset, + this.mixInLengthBlockBytes.byteLength ); protected readonly defaultLen = 0; @@ -193,34 +193,34 @@ export class ListBasicType> } } - super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0); + super.hashTreeRootInto(value, this.mixInLengthBlockBytes, 0); // mixInLength this.mixInLengthBuffer.writeUIntLE(value.length, 32, 6); // one for hashTreeRoot(value), one for length const chunkCount = 2; - merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset); + merkleizeBlocksBytes(this.mixInLengthBlockBytes, chunkCount, output, offset); if (this.cachePermanentRootStruct) { cacheRoot(value as ValueWithCachedPermanentRoot, output, offset, safeCache); } } - protected getChunkBytes(value: ValueOf[]): Uint8Array { + protected getBlocksBytes(value: ValueOf[]): Uint8Array { const byteLen = this.value_serializedSize(value); - const chunkByteLen = Math.ceil(byteLen / 64) * 64; - // reallocate this.verkleBytes if needed - if (byteLen > this.chunkBytesBuffer.length) { + const blockByteLen = Math.ceil(byteLen / 64) * 64; + // reallocate this.blocksBuffer if needed + if (byteLen > this.blocksBuffer.length) { // pad 1 chunk if maxChunkCount is not even - this.chunkBytesBuffer = new Uint8Array(chunkByteLen); + this.blocksBuffer = new Uint8Array(blockByteLen); } - const chunkBytes = this.chunkBytesBuffer.subarray(0, chunkByteLen); - const uint8Array = chunkBytes.subarray(0, byteLen); + const blockBytes = this.blocksBuffer.subarray(0, blockByteLen); + const uint8Array = blockBytes.subarray(0, byteLen); const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength); value_serializeToBytesArrayBasic(this.elementType, value.length, {uint8Array, dataView}, 0, value); // all padding bytes must be zero, this is similar to set zeroHash(0) - this.chunkBytesBuffer.subarray(byteLen, chunkByteLen).fill(0); - return chunkBytes; + this.blocksBuffer.subarray(byteLen, blockByteLen).fill(0); + return blockBytes; } // JSON: inherited from ArrayType diff --git a/packages/ssz/src/type/listComposite.ts b/packages/ssz/src/type/listComposite.ts index 821b9504..5487f700 100644 --- a/packages/ssz/src/type/listComposite.ts +++ b/packages/ssz/src/type/listComposite.ts @@ -1,4 +1,10 @@ -import {HashComputationLevel, Node, Tree, merkleizeInto} from "@chainsafe/persistent-merkle-tree"; +import { + HashComputationLevel, + Node, + Tree, + merkleizeBlocksBytes, + merkleizeBlockArray, +} from "@chainsafe/persistent-merkle-tree"; import {cacheRoot, maxChunksToDepth, symbolCachedPermanentRoot, ValueWithCachedPermanentRoot} from "../util/merkleize"; import {Require} from "../util/types"; import {namedClass} from "../util/named"; @@ -13,7 +19,6 @@ import { tree_deserializeFromBytesArrayComposite, tree_serializeToBytesArrayComposite, maxSizeArrayComposite, - value_getChunkBytesArrayComposite, } from "./arrayComposite"; import {ArrayCompositeType} from "../view/arrayComposite"; import {ListCompositeTreeView} from "../view/listComposite"; @@ -52,11 +57,12 @@ export class ListCompositeType< readonly maxSize: number; readonly isList = true; readonly isViewMutable = true; - readonly mixInLengthChunkBytes = new Uint8Array(64); + readonly blockArray: Uint8Array[] = []; + readonly mixInLengthBlockBytes = new Uint8Array(64); readonly mixInLengthBuffer = Buffer.from( - this.mixInLengthChunkBytes.buffer, - this.mixInLengthChunkBytes.byteOffset, - this.mixInLengthChunkBytes.byteLength + this.mixInLengthBlockBytes.buffer, + this.mixInLengthBlockBytes.byteOffset, + this.mixInLengthBlockBytes.byteLength ); protected readonly defaultLen = 0; @@ -200,25 +206,48 @@ export class ListCompositeType< } } - super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0); + // should not call super.hashTreeRootInto() here + // use merkleizeBlockArray() instead of merkleizeBlocksBytes() to avoid big memory allocation + // reallocate this.blockArray if needed + if (value.length > this.blockArray.length) { + const blockDiff = value.length - this.blockArray.length; + const newBlocksBytes = new Uint8Array(blockDiff * 64); + for (let i = 0; i < blockDiff; i++) { + this.blockArray.push(newBlocksBytes.subarray(i * 64, (i + 1) * 64)); + } + } + + // populate this.blockArray + for (let i = 0; i < value.length; i++) { + // 2 values share a block + const block = this.blockArray[Math.floor(i / 2)]; + const offset = i % 2 === 0 ? 0 : 32; + this.elementType.hashTreeRootInto(value[i], block, offset); + } + + const blockLimit = Math.ceil(value.length / 2); + // zero out the last block if needed + if (value.length % 2 === 1) { + this.blockArray[blockLimit - 1].fill(0, 32); + } + + // compute hashTreeRoot + merkleizeBlockArray(this.blockArray, blockLimit, this.maxChunkCount, this.mixInLengthBlockBytes, 0); + // mixInLength this.mixInLengthBuffer.writeUIntLE(value.length, 32, 6); // one for hashTreeRoot(value), one for length const chunkCount = 2; - merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset); + merkleizeBlocksBytes(this.mixInLengthBlockBytes, chunkCount, output, offset); if (this.cachePermanentRootStruct) { cacheRoot(value as ValueWithCachedPermanentRoot, output, offset, safeCache); } } - protected getChunkBytes(value: ValueOf[]): Uint8Array { - const byteLen = value.length * 32; - const chunkByteLen = this.chunkBytesBuffer.byteLength; - if (byteLen > chunkByteLen) { - this.chunkBytesBuffer = new Uint8Array(Math.ceil(byteLen / 64) * 64); - } - return value_getChunkBytesArrayComposite(this.elementType, value.length, value, this.chunkBytesBuffer); + protected getBlocksBytes(): Uint8Array { + // we use merkleizeBlockArray for hashTreeRoot() computation + throw Error("getBlockBytes should not be called for ListCompositeType"); } // JSON: inherited from ArrayType diff --git a/packages/ssz/src/type/optional.ts b/packages/ssz/src/type/optional.ts index 3d2925fb..1473e6d7 100644 --- a/packages/ssz/src/type/optional.ts +++ b/packages/ssz/src/type/optional.ts @@ -1,7 +1,7 @@ import { concatGindices, Gindex, - merkleizeInto, + merkleizeBlocksBytes, Node, Tree, zeroNode, @@ -48,11 +48,11 @@ export class OptionalType> extends CompositeTy readonly maxSize: number; readonly isList = true; readonly isViewMutable = true; - readonly mixInLengthChunkBytes = new Uint8Array(64); + readonly mixInLengthBlockBytes = new Uint8Array(64); readonly mixInLengthBuffer = Buffer.from( - this.mixInLengthChunkBytes.buffer, - this.mixInLengthChunkBytes.byteOffset, - this.mixInLengthChunkBytes.byteLength + this.mixInLengthBlockBytes.buffer, + this.mixInLengthBlockBytes.byteOffset, + this.mixInLengthBlockBytes.byteLength ); constructor(readonly elementType: ElementType, opts?: OptionalOpts) { @@ -66,7 +66,8 @@ export class OptionalType> extends CompositeTy this.minSize = 0; // Max size includes prepended 0x01 byte this.maxSize = elementType.maxSize + 1; - this.chunkBytesBuffer = new Uint8Array(32); + // maxChunkCount = 1 so this.blocksBuffer.length = 32 in this case + this.blocksBuffer = new Uint8Array(32); } static named>( @@ -185,21 +186,21 @@ export class OptionalType> extends CompositeTy } hashTreeRootInto(value: ValueOfType, output: Uint8Array, offset: number): void { - super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0); + super.hashTreeRootInto(value, this.mixInLengthBlockBytes, 0); const selector = value === null ? 0 : 1; this.mixInLengthBuffer.writeUIntLE(selector, 32, 6); // one for hashTreeRoot(value), one for selector const chunkCount = 2; - merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset); + merkleizeBlocksBytes(this.mixInLengthBlockBytes, chunkCount, output, offset); } - protected getChunkBytes(value: ValueOfType): Uint8Array { + protected getBlocksBytes(value: ValueOfType): Uint8Array { if (value === null) { - this.chunkBytesBuffer.fill(0); + this.blocksBuffer.fill(0); } else { - this.elementType.hashTreeRootInto(value, this.chunkBytesBuffer, 0); + this.elementType.hashTreeRootInto(value, this.blocksBuffer, 0); } - return this.chunkBytesBuffer; + return this.blocksBuffer; } // Proofs diff --git a/packages/ssz/src/type/profile.ts b/packages/ssz/src/type/profile.ts index 1ac440d7..06b2cf76 100644 --- a/packages/ssz/src/type/profile.ts +++ b/packages/ssz/src/type/profile.ts @@ -6,7 +6,7 @@ import { Gindex, toGindex, concatGindices, - merkleizeInto, + merkleizeBlocksBytes, getNode, BranchNode, zeroHash, @@ -158,8 +158,7 @@ export class ProfileType>> extends C this.TreeView = opts?.getProfileTreeViewClass?.(this) ?? getProfileTreeViewClass(this); this.TreeViewDU = opts?.getProfileTreeViewDUClass?.(this) ?? getProfileTreeViewDUClass(this); const fieldBytes = this.activeFields.bitLen * 32; - const chunkBytes = Math.ceil(fieldBytes / 64) * 64; - this.chunkBytesBuffer = new Uint8Array(chunkBytes); + this.blocksBuffer = new Uint8Array(Math.ceil(fieldBytes / 64) * 64); } static named>>( @@ -378,8 +377,8 @@ export class ProfileType>> extends C } } - const merkleBytes = this.getChunkBytes(value); - merkleizeInto(merkleBytes, this.maxChunkCount, this.tempRoot, 0); + const blocksBytes = this.getBlocksBytes(value); + merkleizeBlocksBytes(blocksBytes, this.maxChunkCount, this.tempRoot, 0); mixInActiveFields(this.tempRoot, this.activeFields, output, offset); if (this.cachePermanentRootStruct) { @@ -387,18 +386,18 @@ export class ProfileType>> extends C } } - protected getChunkBytes(struct: ValueOfFields): Uint8Array { - this.chunkBytesBuffer.fill(0); + protected getBlocksBytes(struct: ValueOfFields): Uint8Array { + this.blocksBuffer.fill(0); for (let i = 0; i < this.fieldsEntries.length; i++) { const {fieldName, fieldType, chunkIndex, optional} = this.fieldsEntries[i]; if (optional && struct[fieldName] == null) { - this.chunkBytesBuffer.set(zeroHash(0), chunkIndex * 32); + this.blocksBuffer.set(zeroHash(0), chunkIndex * 32); } else { - fieldType.hashTreeRootInto(struct[fieldName], this.chunkBytesBuffer, chunkIndex * 32); + fieldType.hashTreeRootInto(struct[fieldName], this.blocksBuffer, chunkIndex * 32); } } // remaining bytes are zeroed as we never write them - return this.chunkBytesBuffer; + return this.blocksBuffer; } // Proofs diff --git a/packages/ssz/src/type/stableContainer.ts b/packages/ssz/src/type/stableContainer.ts index 35415ad8..b64ed331 100644 --- a/packages/ssz/src/type/stableContainer.ts +++ b/packages/ssz/src/type/stableContainer.ts @@ -11,7 +11,7 @@ import { getNode, zeroNode, zeroHash, - merkleizeInto, + merkleizeBlocksBytes, countToDepth, getNodeH, setNode, @@ -150,8 +150,7 @@ export class StableContainerType>> e this.TreeView = opts?.getContainerTreeViewClass?.(this) ?? getContainerTreeViewClass(this); this.TreeViewDU = opts?.getContainerTreeViewDUClass?.(this) ?? getContainerTreeViewDUClass(this); const fieldBytes = this.fieldsEntries.length * 32; - const chunkBytes = Math.ceil(fieldBytes / 64) * 64; - this.chunkBytesBuffer = new Uint8Array(chunkBytes); + this.blocksBuffer = new Uint8Array(Math.ceil(fieldBytes / 64) * 64); } static named>>( @@ -351,8 +350,8 @@ export class StableContainerType>> e } } - const merkleBytes = this.getChunkBytes(value); - merkleizeInto(merkleBytes, this.maxChunkCount, this.tempRoot, 0); + const blockBytes = this.getBlocksBytes(value); + merkleizeBlocksBytes(blockBytes, this.maxChunkCount, this.tempRoot, 0); // compute active field bitvector const activeFields = BitArray.fromBoolArray([ ...this.fieldsEntries.map(({fieldName}) => value[fieldName] != null), @@ -365,18 +364,18 @@ export class StableContainerType>> e } } - protected getChunkBytes(struct: ValueOfFields): Uint8Array { - this.chunkBytesBuffer.fill(0); + protected getBlocksBytes(struct: ValueOfFields): Uint8Array { + this.blocksBuffer.fill(0); for (let i = 0; i < this.fieldsEntries.length; i++) { const {fieldName, fieldType, optional} = this.fieldsEntries[i]; if (optional && struct[fieldName] == null) { - this.chunkBytesBuffer.set(zeroHash(0), i * 32); + this.blocksBuffer.set(zeroHash(0), i * 32); } else { - fieldType.hashTreeRootInto(struct[fieldName], this.chunkBytesBuffer, i * 32); + fieldType.hashTreeRootInto(struct[fieldName], this.blocksBuffer, i * 32); } } - return this.chunkBytesBuffer; + return this.blocksBuffer; } // Proofs @@ -817,24 +816,24 @@ export function setActiveField(rootNode: Node, bitLen: number, fieldIndex: numbe return new BranchNode(rootNode.left, newActiveFieldsNode); } -// This is a global buffer to avoid creating a new one for each call to getChunkBytes -const mixInActiveFieldsChunkBytes = new Uint8Array(64); -const activeFieldsSingleChunk = mixInActiveFieldsChunkBytes.subarray(32); +// This is a global buffer to avoid creating a new one for each call to getBlocksBytes +const mixInActiveFieldsBlockBytes = new Uint8Array(64); +const activeFieldsSingleChunk = mixInActiveFieldsBlockBytes.subarray(32); export function mixInActiveFields(root: Uint8Array, activeFields: BitArray, output: Uint8Array, offset: number): void { // fast path for depth 1, the bitvector fits in one chunk - mixInActiveFieldsChunkBytes.set(root, 0); + mixInActiveFieldsBlockBytes.set(root, 0); if (activeFields.bitLen <= 256) { activeFieldsSingleChunk.fill(0); activeFieldsSingleChunk.set(activeFields.uint8Array); // 1 chunk for root, 1 chunk for activeFields const chunkCount = 2; - merkleizeInto(mixInActiveFieldsChunkBytes, chunkCount, output, offset); + merkleizeBlocksBytes(mixInActiveFieldsBlockBytes, chunkCount, output, offset); return; } const chunkCount = Math.ceil(activeFields.uint8Array.length / 32); - merkleizeInto(activeFields.uint8Array, chunkCount, activeFieldsSingleChunk, 0); + merkleizeBlocksBytes(activeFields.uint8Array, chunkCount, activeFieldsSingleChunk, 0); // 1 chunk for root, 1 chunk for activeFields - merkleizeInto(mixInActiveFieldsChunkBytes, 2, output, offset); + merkleizeBlocksBytes(mixInActiveFieldsBlockBytes, 2, output, offset); } diff --git a/packages/ssz/src/type/union.ts b/packages/ssz/src/type/union.ts index 6a6117dd..908d9604 100644 --- a/packages/ssz/src/type/union.ts +++ b/packages/ssz/src/type/union.ts @@ -4,7 +4,7 @@ import { Gindex, Node, Tree, - merkleizeInto, + merkleizeBlocksBytes, getHashComputations, HashComputationLevel, } from "@chainsafe/persistent-merkle-tree"; @@ -49,11 +49,11 @@ export class UnionType[]> extends CompositeType< readonly maxSize: number; readonly isList = true; readonly isViewMutable = true; - readonly mixInLengthChunkBytes = new Uint8Array(64); + readonly mixInLengthBlockBytes = new Uint8Array(64); readonly mixInLengthBuffer = Buffer.from( - this.mixInLengthChunkBytes.buffer, - this.mixInLengthChunkBytes.byteOffset, - this.mixInLengthChunkBytes.byteLength + this.mixInLengthBlockBytes.buffer, + this.mixInLengthBlockBytes.byteOffset, + this.mixInLengthBlockBytes.byteLength ); protected readonly maxSelector: number; @@ -92,7 +92,8 @@ export class UnionType[]> extends CompositeType< this.minSize = 1 + Math.min(...minLens); this.maxSize = 1 + Math.max(...maxLens); this.maxSelector = this.types.length - 1; - this.chunkBytesBuffer = new Uint8Array(32); + // maxChunkCount = 1 so this.blocksBuffer.length = 32 in this case + this.blocksBuffer = new Uint8Array(32); } static named[]>(types: Types, opts: Require): UnionType { @@ -184,15 +185,15 @@ export class UnionType[]> extends CompositeType< } hashTreeRootInto(value: ValueOfTypes, output: Uint8Array, offset: number): void { - super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0); + super.hashTreeRootInto(value, this.mixInLengthBlockBytes, 0); this.mixInLengthBuffer.writeUIntLE(value.selector, 32, 6); const chunkCount = 2; - merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset); + merkleizeBlocksBytes(this.mixInLengthBlockBytes, chunkCount, output, offset); } - protected getChunkBytes(value: ValueOfTypes): Uint8Array { - this.types[value.selector].hashTreeRootInto(value.value, this.chunkBytesBuffer, 0); - return this.chunkBytesBuffer; + protected getBlocksBytes(value: ValueOfTypes): Uint8Array { + this.types[value.selector].hashTreeRootInto(value.value, this.blocksBuffer, 0); + return this.blocksBuffer; } // Proofs diff --git a/packages/ssz/src/type/vectorBasic.ts b/packages/ssz/src/type/vectorBasic.ts index bb189044..0c528c96 100644 --- a/packages/ssz/src/type/vectorBasic.ts +++ b/packages/ssz/src/type/vectorBasic.ts @@ -59,10 +59,7 @@ export class VectorBasicType> this.minSize = this.fixedSize; this.maxSize = this.fixedSize; this.defaultLen = length; - // pad 1 chunk if maxChunkCount is not even - this.chunkBytesBuffer = new Uint8Array( - this.maxChunkCount % 2 === 1 ? this.maxChunkCount * 32 + 32 : this.maxChunkCount * 32 - ); + this.blocksBuffer = new Uint8Array(Math.ceil(this.maxChunkCount / 2) * 64); } static named>( @@ -150,13 +147,13 @@ export class VectorBasicType> // Merkleization - protected getChunkBytes(value: ValueOf[]): Uint8Array { - const uint8Array = this.chunkBytesBuffer.subarray(0, this.fixedSize); + protected getBlocksBytes(value: ValueOf[]): Uint8Array { + const uint8Array = this.blocksBuffer.subarray(0, this.fixedSize); const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength); value_serializeToBytesArrayBasic(this.elementType, this.length, {uint8Array, dataView}, 0, value); - // remaining bytes from this.fixedSize to this.chunkBytesBuffer.length must be zeroed - return this.chunkBytesBuffer; + // remaining bytes from this.fixedSize to this.blocksBuffer.length must be zeroed + return this.blocksBuffer; } // JSON: inherited from ArrayType diff --git a/packages/ssz/src/type/vectorComposite.ts b/packages/ssz/src/type/vectorComposite.ts index 28990c43..908f4f9b 100644 --- a/packages/ssz/src/type/vectorComposite.ts +++ b/packages/ssz/src/type/vectorComposite.ts @@ -13,7 +13,7 @@ import { tree_serializeToBytesArrayComposite, maxSizeArrayComposite, minSizeArrayComposite, - value_getChunkBytesArrayComposite, + value_getBlocksBytesArrayComposite, } from "./arrayComposite"; import {ArrayCompositeType, ArrayCompositeTreeView} from "../view/arrayComposite"; import {ArrayCompositeTreeViewDU} from "../viewDU/arrayComposite"; @@ -65,10 +65,7 @@ export class VectorCompositeType< this.minSize = minSizeArrayComposite(elementType, length); this.maxSize = maxSizeArrayComposite(elementType, length); this.defaultLen = length; - this.chunkBytesBuffer = - this.maxChunkCount % 2 === 1 - ? new Uint8Array(this.maxChunkCount * 32 + 32) - : new Uint8Array(this.maxChunkCount * 32); + this.blocksBuffer = new Uint8Array(Math.ceil(this.maxChunkCount / 2) * 64); } // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -157,8 +154,8 @@ export class VectorCompositeType< // Merkleization - protected getChunkBytes(value: ValueOf[]): Uint8Array { - return value_getChunkBytesArrayComposite(this.elementType, this.length, value, this.chunkBytesBuffer); + protected getBlocksBytes(value: ValueOf[]): Uint8Array { + return value_getBlocksBytesArrayComposite(this.elementType, this.length, value, this.blocksBuffer); } // JSON: inherited from ArrayType diff --git a/packages/ssz/test/perf/merkleize.test.ts b/packages/ssz/test/perf/merkleize.test.ts index a900015a..af70868e 100644 --- a/packages/ssz/test/perf/merkleize.test.ts +++ b/packages/ssz/test/perf/merkleize.test.ts @@ -1,6 +1,6 @@ import {itBench} from "@dapplion/benchmark"; import {bitLength, merkleize} from "../../src/util/merkleize"; -import {merkleizeInto} from "@chainsafe/persistent-merkle-tree"; +import {merkleizeBlockArray, merkleizeBlocksBytes} from "@chainsafe/persistent-merkle-tree"; describe("merkleize / bitLength", () => { for (const n of [50, 8000, 250000]) { @@ -14,20 +14,33 @@ describe("merkleize / bitLength", () => { } }); -describe("merkleize vs persistent-merkle-tree merkleizeInto", () => { - const chunkCounts = [4, 8, 16, 32]; +describe("merkleize vs persistent-merkle-tree merkleizeBlocksBytes", () => { + const chunkCounts = [32, 128, 512, 1024]; for (const chunkCount of chunkCounts) { const rootArr = Array.from({length: chunkCount}, (_, i) => Buffer.alloc(32, i)); - const roots = Buffer.concat(rootArr); + const blocksBytes = Buffer.concat(rootArr); + if (blocksBytes.length % 64 !== 0) { + throw new Error("blockBytes length must be a multiple of 64"); + } + const blockArray: Uint8Array[] = []; + for (let i = 0; i < blocksBytes.length; i += 64) { + blockArray.push(blocksBytes.slice(i, i + 64)); + } + const result = Buffer.alloc(32); - itBench(`merkleizeInto ${chunkCount} chunks`, () => { - merkleizeInto(roots, chunkCount, result, 0); - }); itBench(`merkleize ${chunkCount} chunks`, () => { merkleize(rootArr, chunkCount); }); + + itBench(`merkleizeBlocksBytes ${chunkCount} chunks`, () => { + merkleizeBlocksBytes(blocksBytes, chunkCount, result, 0); + }); + + itBench(`merkleizeBlockArray ${chunkCount} chunks`, () => { + merkleizeBlockArray(blockArray, blockArray.length, chunkCount, result, 0); + }); } }); diff --git a/packages/ssz/test/unit/byType/byteList/value.test.ts b/packages/ssz/test/unit/byType/byteList/value.test.ts new file mode 100644 index 00000000..0033443f --- /dev/null +++ b/packages/ssz/test/unit/byType/byteList/value.test.ts @@ -0,0 +1,28 @@ +import {expect} from "chai"; +import {ByteListType} from "../../../../src"; + +describe("ByteListValue", () => { + const type = new ByteListType(1024); + + it("should zero out the last sha256 block if it's over value.length", () => { + const value = Buffer.alloc(65, 1); + const expectedRoot = type.hashTreeRoot(value); + // now hash another value which make the cached blocks non zero + type.hashTreeRoot(Buffer.alloc(1024, 2)); + const actualRoot = type.hashTreeRoot(value); + expect(actualRoot).to.deep.equal(expectedRoot); + }); + + it("should increase blockArray size if needed", () => { + const value0 = Buffer.alloc(65, 1); + const expectedRoot0 = type.hashTreeRoot(value0); + const value1 = Buffer.alloc(1024, 3); + const expectedRoot1 = type.hashTreeRoot(value1); + // now increase block array size + type.hashTreeRoot(Buffer.alloc(1024, 2)); + + // hash again + expect(type.hashTreeRoot(value0)).to.deep.equal(expectedRoot0); + expect(type.hashTreeRoot(value1)).to.deep.equal(expectedRoot1); + }); +}); diff --git a/packages/ssz/test/unit/byType/container/tree.test.ts b/packages/ssz/test/unit/byType/container/tree.test.ts index 83fad27a..5c3bbd54 100644 --- a/packages/ssz/test/unit/byType/container/tree.test.ts +++ b/packages/ssz/test/unit/byType/container/tree.test.ts @@ -1,5 +1,4 @@ import {expect} from "chai"; -import {Tree} from "@chainsafe/persistent-merkle-tree"; import { BitArray, BitListType, @@ -13,7 +12,6 @@ import { ListCompositeType, NoneType, toHexString, - Type, UintNumberType, UnionType, ValueOf, @@ -22,7 +20,6 @@ import { } from "../../../../src"; import {uint64NumInfType, uint64NumType} from "../../../utils/primitiveTypes"; import {runViewTestMutation} from "../runViewTestMutation"; -import {upgradeToNewType} from "../../../../src/util/upgrade"; // Test both ContainerType, ContainerNodeStructType only if // - All fields are immutable @@ -244,7 +241,7 @@ describe("ContainerViewDU batchHashTreeRoot", function () { a: uint64NumType, b: new BooleanType(), c: unionType, - d: new ByteListType(64), + d: new ByteListType(1024), e: new ByteVectorType(64), // a child container type f: childContainerType, @@ -262,7 +259,8 @@ describe("ContainerViewDU batchHashTreeRoot", function () { a: 10, b: true, c: {selector: 1, value: 100}, - d: Buffer.alloc(64, 2), + // make this not divisible by 64 to test edge case + d: Buffer.alloc(65, 2), e: Buffer.alloc(64, 1), f: {f0: 100, f1: 101}, g: {g0: 100, g1: 101}, @@ -274,6 +272,7 @@ describe("ContainerViewDU batchHashTreeRoot", function () { m: BitArray.fromSingleBit(4, 1), }; const expectedRoot = parentContainerType.toView(value).hashTreeRoot(); + expect(parentContainerType.hashTreeRoot(value)).to.be.deep.equal(expectedRoot); it("fresh ViewDU", () => { expect(parentContainerType.toViewDU(value).batchHashTreeRoot()).to.be.deep.equal(expectedRoot); @@ -330,9 +329,10 @@ describe("ContainerViewDU batchHashTreeRoot", function () { it("full hash then modify ByteListType", () => { const viewDU = parentContainerType.toViewDU(value); + viewDU.d = Buffer.alloc(1024, 3); viewDU.batchHashTreeRoot(); - // this takes more than 1 chunk so the resulting node is a branch node - viewDU.d = viewDU.d.slice(); + // set back to the original value, this takes more than 1 chunk so the resulting node is a branch node + viewDU.d = Buffer.alloc(65, 2); expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); // assign again but commit before batchHashTreeRoot() @@ -638,53 +638,3 @@ describe("ContainerNodeStruct batchHashTreeRoot", function () { expect(viewDU.batchHashTreeRoot()).to.be.deep.equal(expectedRoot); }); }); - -describe("upgradeToNewType utility", () => { - const numFields = [2, 7, 15, 17, 31, 33, 63, 65, 127, 129]; - for (const [i, numField] of numFields.entries()) { - it(`upgradeToNewType with ${numField} fields`, () => { - const fields: Record> = {}; - for (let j = 0; j < numField; j++) { - fields[`f${j}`] = uint64NumInfType; - } - const oldType = new ContainerType(fields); - const view = oldType.defaultView(); - const viewDU = oldType.defaultViewDU(); - for (let j = 0; j < numField; j++) { - (view as Record)[`f${j}`] = j; - (viewDU as Record)[`f${j}`] = j; - } - - for (let j = i + 1; j < numFields.length; j++) { - const newFields: Record> = {}; - for (let k = 0; k < numFields[j]; k++) { - (newFields as Record>)[`f${k}`] = uint64NumInfType; - } - - const newType = new ContainerType(newFields); - const newView = newType.getView(new Tree(upgradeToNewType(view.node, oldType, newType))); - // commit view DU to make sure the view is updated before accessing viewDU.node - viewDU.commit(); - const newViewDU = newType.getViewDU(upgradeToNewType(viewDU.node, oldType, newType)); - for (let k = i + 1; k < numFields[j]; k++) { - (newView as Record)[`f${k}`] = k; - (newViewDU as Record)[`f${k}`] = k; - } - newViewDU.commit(); - - const expectedValue = newType.defaultValue(); - for (let k = 0; k < numFields[j]; k++) { - (expectedValue as Record)[`f${k}`] = k; - } - const expectedViewDU = newType.toViewDU(expectedValue); - - expect(newView.toValue()).to.be.deep.equal(expectedValue); - expect(newView.hashTreeRoot()).to.be.deep.equal(expectedViewDU.hashTreeRoot()); - expect(newView.serialize()).to.be.deep.equal(expectedViewDU.serialize()); - expect(newViewDU.toValue()).to.be.deep.equal(expectedValue); - expect(newViewDU.hashTreeRoot()).to.be.deep.equal(expectedViewDU.hashTreeRoot()); - expect(newViewDU.serialize()).to.be.deep.equal(expectedViewDU.serialize()); - } - }); - } -}); diff --git a/packages/ssz/test/unit/byType/listComposite/tree.test.ts b/packages/ssz/test/unit/byType/listComposite/tree.test.ts index 95b39746..f428365b 100644 --- a/packages/ssz/test/unit/byType/listComposite/tree.test.ts +++ b/packages/ssz/test/unit/byType/listComposite/tree.test.ts @@ -226,7 +226,21 @@ describe("ListCompositeType.sliceFrom", () => { } }); -describe("ListCompositeType batchHashTreeRoot", () => { +describe("ListCompositeType hashTreeRoot", () => { + it("shouldzero out the last sha256 block", () => { + const listType = new ListCompositeType(ssz.Root, 1024); + const value0 = Array.from({length: 65}, (_, i) => Buffer.alloc(32, i)); + const value1 = Array.from({length: 120}, (_, i) => Buffer.alloc(32, i)); + const expectedRoot0 = listType.hashTreeRoot(value0); + const expectedRoot1 = listType.hashTreeRoot(value1); + // now increase block array size + listType.hashTreeRoot(Array.from({length: 1024}, () => Buffer.alloc(32, 3))); + expect(listType.hashTreeRoot(value0)).to.deep.equal(expectedRoot0); + expect(listType.hashTreeRoot(value1)).to.deep.equal(expectedRoot1); + }); +}); + +describe("ListCompositeType ViewDU batchHashTreeRoot", () => { const value = [ {a: 1, b: 2}, {a: 3, b: 4}, @@ -242,6 +256,7 @@ describe("ListCompositeType batchHashTreeRoot", () => { for (const list of [listOfContainersType, listOfContainersType2]) { const typeName = list.typeName; const expectedRoot = list.toView(value).hashTreeRoot(); + expect(listOfContainersType2.hashTreeRoot(value)).to.be.deep.equal(expectedRoot); it(`${typeName} - fresh ViewDU`, () => { expect(listOfContainersType.toViewDU(value).batchHashTreeRoot()).to.be.deep.equal(expectedRoot);