From 43237408b076ec03e3f19828f6463f830a8bca9e Mon Sep 17 00:00:00 2001 From: Cayman Date: Wed, 1 May 2024 16:55:58 -0400 Subject: [PATCH 1/9] fix: avoid Buffer.from copies --- packages/api/src/beacon/server/beacon.ts | 4 ++-- packages/api/src/beacon/server/debug.ts | 2 +- packages/api/src/beacon/server/proof.ts | 4 ++-- .../beacon-node/src/network/gossip/encoding.ts | 2 +- packages/beacon-node/src/util/sszBytes.ts | 10 ++++++---- packages/reqresp/src/encoders/responseEncode.ts | 4 +++- .../src/encodingStrategies/sszSnappy/encode.ts | 3 ++- packages/reqresp/src/utils/errorMessage.ts | 6 +++--- packages/state-transition/package.json | 7 ++----- .../state-transition/src/block/processRandao.ts | 11 +++++++++-- packages/state-transition/src/cache/pubkeyCache.ts | 2 +- packages/state-transition/src/util/shuffle.ts | 2 +- .../src/repositories/metaDataRepository.ts | 4 ++-- .../attestation/attestationByTargetRepository.ts | 4 ++-- .../attestation/attestationLowerBoundRepository.ts | 4 ++-- .../block/blockBySlotRepository.ts | 4 ++-- .../minMaxSurround/distanceStoreRepository.ts | 4 ++-- yarn.lock | 14 -------------- 18 files changed, 43 insertions(+), 48 deletions(-) diff --git a/packages/api/src/beacon/server/beacon.ts b/packages/api/src/beacon/server/beacon.ts index c71decd1ac8..6fd4f89e38c 100644 --- a/packages/api/src/beacon/server/beacon.ts +++ b/packages/api/src/beacon/server/beacon.ts @@ -23,7 +23,7 @@ export function getRoutes(config: ChainForkConfig, api: ServerApi): ServerR const response = await api.getBlock(...reqSerializers.getBlock.parseReq(req)); if (response instanceof Uint8Array) { // Fastify 3.x.x will automatically add header `Content-Type: application/octet-stream` if Buffer - return Buffer.from(response); + return Buffer.from(response.buffer, response.byteOffset, response.byteLength); } else { return returnTypes.getBlock.toJson(response); } @@ -38,7 +38,7 @@ export function getRoutes(config: ChainForkConfig, api: ServerApi): ServerR const version = config.getForkName(slot); void res.header("Eth-Consensus-Version", version); // Fastify 3.x.x will automatically add header `Content-Type: application/octet-stream` if Buffer - return Buffer.from(response); + return Buffer.from(response.buffer, response.byteOffset, response.byteLength); } else { void res.header("Eth-Consensus-Version", response.version); return returnTypes.getBlockV2.toJson(response); diff --git a/packages/api/src/beacon/server/debug.ts b/packages/api/src/beacon/server/debug.ts index 34edf91a580..c8f4a928492 100644 --- a/packages/api/src/beacon/server/debug.ts +++ b/packages/api/src/beacon/server/debug.ts @@ -24,7 +24,7 @@ export function getRoutes(config: ChainForkConfig, api: ServerApi): ServerR const response = await api.getState(...reqSerializers.getState.parseReq(req)); if (response instanceof Uint8Array) { // Fastify 3.x.x will automatically add header `Content-Type: application/octet-stream` if Buffer - return Buffer.from(response); + return Buffer.from(response.buffer, response.byteOffset, response.byteLength); } else { return returnTypes.getState.toJson(response); } diff --git a/packages/api/src/beacon/server/proof.ts b/packages/api/src/beacon/server/proof.ts index a03ae472bf7..d33541749af 100644 --- a/packages/api/src/beacon/server/proof.ts +++ b/packages/api/src/beacon/server/proof.ts @@ -25,7 +25,7 @@ export function getRoutes(config: ChainForkConfig, api: ServerApi): ServerR response.set(leaves[i], i * 32); } // Fastify 3.x.x will automatically add header `Content-Type: application/octet-stream` if Buffer - return Buffer.from(response); + return Buffer.from(response.buffer, response.byteOffset, response.byteLength); }, }, getBlockProof: { @@ -39,7 +39,7 @@ export function getRoutes(config: ChainForkConfig, api: ServerApi): ServerR response.set(leaves[i], i * 32); } // Fastify 3.x.x will automatically add header `Content-Type: application/octet-stream` if Buffer - return Buffer.from(response); + return Buffer.from(response.buffer, response.byteOffset, response.byteLength); }, }, }; diff --git a/packages/beacon-node/src/network/gossip/encoding.ts b/packages/beacon-node/src/network/gossip/encoding.ts index bcb6d9181bc..3b87f42747a 100644 --- a/packages/beacon-node/src/network/gossip/encoding.ts +++ b/packages/beacon-node/src/network/gossip/encoding.ts @@ -58,7 +58,7 @@ export function msgIdFn(gossipTopicCache: GossipTopicCache, msg: Message): Uint8 vec = [MESSAGE_DOMAIN_VALID_SNAPPY, intToBytes(msg.topic.length, 8), Buffer.from(msg.topic), msg.data]; } - return Buffer.from(digest(Buffer.concat(vec))).subarray(0, 20); + return digest(Buffer.concat(vec)).subarray(0, 20); } export class DataTransformSnappy { diff --git a/packages/beacon-node/src/util/sszBytes.ts b/packages/beacon-node/src/util/sszBytes.ts index cd12c4bd9c1..52ff7de6cac 100644 --- a/packages/beacon-node/src/util/sszBytes.ts +++ b/packages/beacon-node/src/util/sszBytes.ts @@ -25,6 +25,10 @@ const SLOT_SIZE = 8; const ATTESTATION_DATA_SIZE = 128; const SIGNATURE_SIZE = 96; +function toBuffer(x: Uint8Array): Buffer { + return Buffer.from(x.buffer, x.byteOffset, x.byteLength); +} + /** * Extract slot from attestation serialized bytes. * Return null if data is not long enough to extract slot. @@ -59,9 +63,7 @@ export function getAttDataBase64FromAttestationSerialized(data: Uint8Array): Att } // base64 is a bit efficient than hex - return Buffer.from(data.slice(VARIABLE_FIELD_OFFSET, VARIABLE_FIELD_OFFSET + ATTESTATION_DATA_SIZE)).toString( - "base64" - ); + return toBuffer(data.slice(VARIABLE_FIELD_OFFSET, VARIABLE_FIELD_OFFSET + ATTESTATION_DATA_SIZE)).toString("base64"); } /** @@ -150,7 +152,7 @@ export function getAttDataBase64FromSignedAggregateAndProofSerialized(data: Uint } // base64 is a bit efficient than hex - return Buffer.from( + return toBuffer( data.slice(SIGNED_AGGREGATE_AND_PROOF_SLOT_OFFSET, SIGNED_AGGREGATE_AND_PROOF_SLOT_OFFSET + ATTESTATION_DATA_SIZE) ).toString("base64"); } diff --git a/packages/reqresp/src/encoders/responseEncode.ts b/packages/reqresp/src/encoders/responseEncode.ts index 3e2d875685e..c5320ffc1ce 100644 --- a/packages/reqresp/src/encoders/responseEncode.ts +++ b/packages/reqresp/src/encoders/responseEncode.ts @@ -3,6 +3,8 @@ import {encodeErrorMessage} from "../utils/index.js"; import {ContextBytesType, ContextBytesFactory, MixedProtocol, Protocol, ResponseOutgoing} from "../types.js"; import {RespStatus, RpcResponseStatusError} from "../interface.js"; +const SUCCESS_BUFFER = Buffer.from([RespStatus.SUCCESS]); + /** * Yields byte chunks for a `` with a zero response code `` * ```bnf @@ -24,7 +26,7 @@ export function responseEncodeSuccess( cbs.onChunk(chunkIndex++); // - yield Buffer.from([RespStatus.SUCCESS]); + yield SUCCESS_BUFFER; // - from altair const contextBytes = getContextBytes(protocol.contextBytes, chunk); diff --git a/packages/reqresp/src/encodingStrategies/sszSnappy/encode.ts b/packages/reqresp/src/encodingStrategies/sszSnappy/encode.ts index 930aa242766..f4559b91f3f 100644 --- a/packages/reqresp/src/encodingStrategies/sszSnappy/encode.ts +++ b/packages/reqresp/src/encodingStrategies/sszSnappy/encode.ts @@ -15,7 +15,8 @@ export const writeSszSnappyPayload = encodeSszSnappy as (bytes: Uint8Array) => A */ export async function* encodeSszSnappy(bytes: Buffer): AsyncGenerator { // MUST encode the length of the raw SSZ bytes, encoded as an unsigned protobuf varint - yield Buffer.from(varintEncode(bytes.length)); + const varint = varintEncode(bytes.length); + yield Buffer.from(varint.buffer, varint.byteOffset, varint.byteLength); // By first computing and writing the SSZ byte length, the SSZ encoder can then directly // write the chunk contents to the stream. Snappy writer compresses frame by frame diff --git a/packages/reqresp/src/utils/errorMessage.ts b/packages/reqresp/src/utils/errorMessage.ts index c9ca6505d28..3af32a8f5c5 100644 --- a/packages/reqresp/src/utils/errorMessage.ts +++ b/packages/reqresp/src/utils/errorMessage.ts @@ -1,4 +1,4 @@ -import {encodeSszSnappy} from "../encodingStrategies/sszSnappy/encode.js"; +import {writeSszSnappyPayload} from "../encodingStrategies/sszSnappy/encode.js"; import {Encoding} from "../types.js"; // ErrorMessage schema: @@ -17,11 +17,11 @@ import {Encoding} from "../types.js"; */ export async function* encodeErrorMessage(errorMessage: string, encoding: Encoding): AsyncGenerator { const encoder = new TextEncoder(); - const bytes = Buffer.from(encoder.encode(errorMessage).slice(0, 256)); + const bytes = encoder.encode(errorMessage).slice(0, 256); switch (encoding) { case Encoding.SSZ_SNAPPY: - yield* encodeSszSnappy(bytes); + yield* writeSszSnappyPayload(bytes); } } diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index 2fd71665ae1..9a65755a647 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -68,12 +68,9 @@ "@lodestar/params": "^1.18.0", "@lodestar/types": "^1.18.0", "@lodestar/utils": "^1.18.0", - "bigint-buffer": "^1.1.5", - "buffer-xor": "^2.0.2" - }, - "devDependencies": { - "@types/buffer-xor": "^2.0.0" + "bigint-buffer": "^1.1.5" }, + "devDependencies": {}, "keywords": [ "ethereum", "eth-consensus", diff --git a/packages/state-transition/src/block/processRandao.ts b/packages/state-transition/src/block/processRandao.ts index cf9ea193b94..effeb34390c 100644 --- a/packages/state-transition/src/block/processRandao.ts +++ b/packages/state-transition/src/block/processRandao.ts @@ -1,4 +1,3 @@ -import xor from "buffer-xor"; import {digest} from "@chainsafe/as-sha256"; import {allForks} from "@lodestar/types"; import {EPOCHS_PER_HISTORICAL_VECTOR} from "@lodestar/params"; @@ -28,6 +27,14 @@ export function processRandao( } // mix in RANDAO reveal - const randaoMix = xor(Buffer.from(getRandaoMix(state, epoch)), Buffer.from(digest(randaoReveal))); + const randaoMix = xor(getRandaoMix(state, epoch), digest(randaoReveal)); state.randaoMixes.set(epoch % EPOCHS_PER_HISTORICAL_VECTOR, randaoMix); } + +function xor(a: Uint8Array, b: Uint8Array): Uint8Array { + const length = Math.min(a.length, b.length); + for (let i = 0; i < length; i++) { + a[i] = a[i] ^ b[i]; + } + return a; +} diff --git a/packages/state-transition/src/cache/pubkeyCache.ts b/packages/state-transition/src/cache/pubkeyCache.ts index 64a3f4f23c8..fdc343ff4ec 100644 --- a/packages/state-transition/src/cache/pubkeyCache.ts +++ b/packages/state-transition/src/cache/pubkeyCache.ts @@ -23,7 +23,7 @@ function toMemoryEfficientHexStr(hex: Uint8Array | string): string { return hex; } - return Buffer.from(hex).toString("hex"); + return Buffer.from(hex.buffer, hex.byteOffset, hex.byteLength).toString("hex"); } export class PubkeyIndexMap { diff --git a/packages/state-transition/src/util/shuffle.ts b/packages/state-transition/src/util/shuffle.ts index 07457c5975d..539ffd4c1b3 100644 --- a/packages/state-transition/src/util/shuffle.ts +++ b/packages/state-transition/src/util/shuffle.ts @@ -101,7 +101,7 @@ function innerShuffleList(input: Shuffleable, seed: Bytes32, dir: boolean): void // Seed is always the first 32 bytes of the hash input, we never have to change this part of the buffer. const _seed = seed; - Buffer.from(_seed).copy(buf, 0, 0, _SHUFFLE_H_SEED_SIZE); + buf.set(_seed.subarray(0, _SHUFFLE_H_SEED_SIZE), 0); // initial values here are not used: overwritten first within the inner for loop. let source = seed; // just setting it to a Bytes32 diff --git a/packages/validator/src/repositories/metaDataRepository.ts b/packages/validator/src/repositories/metaDataRepository.ts index c7824094741..0ba1565a665 100644 --- a/packages/validator/src/repositories/metaDataRepository.ts +++ b/packages/validator/src/repositories/metaDataRepository.ts @@ -25,7 +25,7 @@ export class MetaDataRepository { } async setGenesisValidatorsRoot(genesisValidatorsRoot: Root): Promise { - await this.db.put(this.encodeKey(GENESIS_VALIDATORS_ROOT), Buffer.from(genesisValidatorsRoot), this.dbReqOpts); + await this.db.put(this.encodeKey(GENESIS_VALIDATORS_ROOT), genesisValidatorsRoot, this.dbReqOpts); } async getGenesisTime(): Promise { @@ -34,7 +34,7 @@ export class MetaDataRepository { } async setGenesisTime(genesisTime: UintNum64): Promise { - await this.db.put(this.encodeKey(GENESIS_TIME), Buffer.from(ssz.UintNum64.serialize(genesisTime)), this.dbReqOpts); + await this.db.put(this.encodeKey(GENESIS_TIME), ssz.UintNum64.serialize(genesisTime), this.dbReqOpts); } private encodeKey(key: Uint8Array): Uint8Array { diff --git a/packages/validator/src/slashingProtection/attestation/attestationByTargetRepository.ts b/packages/validator/src/slashingProtection/attestation/attestationByTargetRepository.ts index 7606d31338c..61c13bb17ce 100644 --- a/packages/validator/src/slashingProtection/attestation/attestationByTargetRepository.ts +++ b/packages/validator/src/slashingProtection/attestation/attestationByTargetRepository.ts @@ -50,7 +50,7 @@ export class AttestationByTargetRepository { await this.db.batchPut( atts.map((att) => ({ key: this.encodeKey(pubkey, att.targetEpoch), - value: Buffer.from(this.type.serialize(att)), + value: this.type.serialize(att), })), this.dbReqOpts ); @@ -62,7 +62,7 @@ export class AttestationByTargetRepository { } private encodeKey(pubkey: BLSPubkey, targetEpoch: Epoch): Uint8Array { - return encodeKey(this.bucket, Buffer.concat([Buffer.from(pubkey), intToBytes(BigInt(targetEpoch), uintLen, "be")])); + return encodeKey(this.bucket, Buffer.concat([pubkey, intToBytes(BigInt(targetEpoch), uintLen, "be")])); } private decodeKey(key: Uint8Array): {pubkey: BLSPubkey; targetEpoch: Epoch} { diff --git a/packages/validator/src/slashingProtection/attestation/attestationLowerBoundRepository.ts b/packages/validator/src/slashingProtection/attestation/attestationLowerBoundRepository.ts index d45814e6648..84a4e7032a7 100644 --- a/packages/validator/src/slashingProtection/attestation/attestationLowerBoundRepository.ts +++ b/packages/validator/src/slashingProtection/attestation/attestationLowerBoundRepository.ts @@ -35,10 +35,10 @@ export class AttestationLowerBoundRepository { } async set(pubkey: BLSPubkey, value: SlashingProtectionLowerBound): Promise { - await this.db.put(this.encodeKey(pubkey), Buffer.from(this.type.serialize(value)), this.dbReqOpts); + await this.db.put(this.encodeKey(pubkey), this.type.serialize(value), this.dbReqOpts); } private encodeKey(pubkey: BLSPubkey): Uint8Array { - return encodeKey(this.bucket, Buffer.from(pubkey)); + return encodeKey(this.bucket, pubkey); } } diff --git a/packages/validator/src/slashingProtection/block/blockBySlotRepository.ts b/packages/validator/src/slashingProtection/block/blockBySlotRepository.ts index 25b2fee7e8e..eb1ae092cf3 100644 --- a/packages/validator/src/slashingProtection/block/blockBySlotRepository.ts +++ b/packages/validator/src/slashingProtection/block/blockBySlotRepository.ts @@ -54,7 +54,7 @@ export class BlockBySlotRepository { await this.db.batchPut( blocks.map((block) => ({ key: this.encodeKey(pubkey, block.slot), - value: Buffer.from(this.type.serialize(block)), + value: this.type.serialize(block), })), this.dbReqOpts ); @@ -66,7 +66,7 @@ export class BlockBySlotRepository { } private encodeKey(pubkey: BLSPubkey, slot: Slot): Uint8Array { - return encodeKey(this.bucket, Buffer.concat([Buffer.from(pubkey), intToBytes(BigInt(slot), uintLen, "be")])); + return encodeKey(this.bucket, Buffer.concat([pubkey, intToBytes(BigInt(slot), uintLen, "be")])); } private decodeKey(key: Uint8Array): {pubkey: BLSPubkey; slot: Slot} { diff --git a/packages/validator/src/slashingProtection/minMaxSurround/distanceStoreRepository.ts b/packages/validator/src/slashingProtection/minMaxSurround/distanceStoreRepository.ts index db8345a90cc..45315bf95b0 100644 --- a/packages/validator/src/slashingProtection/minMaxSurround/distanceStoreRepository.ts +++ b/packages/validator/src/slashingProtection/minMaxSurround/distanceStoreRepository.ts @@ -45,13 +45,13 @@ class SpanDistanceRepository { await this.db.batchPut( values.map((value) => ({ key: this.encodeKey(pubkey, value.source), - value: Buffer.from(this.type.serialize(value.distance)), + value: this.type.serialize(value.distance), })), this.dbReqOpts ); } private encodeKey(pubkey: BLSPubkey, epoch: Epoch): Uint8Array { - return encodeKey(this.bucket, Buffer.concat([Buffer.from(pubkey), intToBytes(BigInt(epoch), 8, "be")])); + return encodeKey(this.bucket, Buffer.concat([pubkey, intToBytes(BigInt(epoch), 8, "be")])); } } diff --git a/yarn.lock b/yarn.lock index b27b4382b39..8bf77769e6a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2836,13 +2836,6 @@ resolved "https://registry.yarnpkg.com/@types/argparse/-/argparse-1.0.38.tgz#a81fd8606d481f873a3800c6ebae4f1d768a56a9" integrity sha512-ebDJ9b0e702Yr7pWgB0jzm+CX4Srzz8RcXtLJDJB+BSccqMa36uyH/zUsSYao5+BD1ytv3k3rPYCq4mAE1hsXA== -"@types/buffer-xor@^2.0.0": - version "2.0.2" - resolved "https://registry.yarnpkg.com/@types/buffer-xor/-/buffer-xor-2.0.2.tgz#d8c463583b8fbb322ea824562dc78a0c3cea2ca6" - integrity sha512-OqdCua7QCTupPnJgmyGJUpxWgbuOi0IMIVslXTSePS2o+qDrDB6f2Pg44zRyqhUA5GbFAf39U8z0+mH4WG0fLQ== - dependencies: - "@types/node" "*" - "@types/cacheable-request@^6.0.1": version "6.0.3" resolved "https://registry.yarnpkg.com/@types/cacheable-request/-/cacheable-request-6.0.3.tgz#a430b3260466ca7b5ca5bfd735693b36e7a9d183" @@ -4461,13 +4454,6 @@ buffer-xor@^1.0.3: resolved "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz" integrity sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk= -buffer-xor@^2.0.2: - version "2.0.2" - resolved "https://registry.npmjs.org/buffer-xor/-/buffer-xor-2.0.2.tgz" - integrity sha512-eHslX0bin3GB+Lx2p7lEYRShRewuNZL3fUl4qlVJGGiwoPGftmt8JQgk2Y9Ji5/01TnVDo33E5b5O3vUB1HdqQ== - dependencies: - safe-buffer "^5.1.1" - buffer@4.9.2, buffer@^4.3.0: version "4.9.2" resolved "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz" From ae68206311a384b1f1d869ff36f175d727593920 Mon Sep 17 00:00:00 2001 From: Cayman Date: Thu, 2 May 2024 14:56:31 -0400 Subject: [PATCH 2/9] chore: simplify shuffling --- .../beacon-node/test/perf/util/bytes.test.ts | 36 +++++++++++++++++++ packages/state-transition/src/util/shuffle.ts | 5 +-- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/packages/beacon-node/test/perf/util/bytes.test.ts b/packages/beacon-node/test/perf/util/bytes.test.ts index bc3f6cb72e0..d1d2e968b18 100644 --- a/packages/beacon-node/test/perf/util/bytes.test.ts +++ b/packages/beacon-node/test/perf/util/bytes.test.ts @@ -34,4 +34,40 @@ describe("bytes utils", function () { } }, }); + + itBench({ + id: "Buffer.copy", + fn: () => { + const arr = Buffer.alloc(32 * count); + let offset = 0; + for (const b of buffers) { + b.copy(arr, offset, 0, b.length); + offset += b.length; + } + }, + }); + + itBench({ + id: "Uint8Array.set - with subarray", + fn: () => { + const arr = new Uint8Array(32 * count); + let offset = 0; + for (const b of roots) { + arr.set(b.subarray(0, b.length), offset); + offset += b.length; + } + }, + }); + + itBench({ + id: "Uint8Array.set - without subarray", + fn: () => { + const arr = new Uint8Array(32 * count); + let offset = 0; + for (const b of roots) { + arr.set(b, offset); + offset += b.length; + } + }, + }); }); diff --git a/packages/state-transition/src/util/shuffle.ts b/packages/state-transition/src/util/shuffle.ts index 539ffd4c1b3..176d4f33cd8 100644 --- a/packages/state-transition/src/util/shuffle.ts +++ b/packages/state-transition/src/util/shuffle.ts @@ -90,6 +90,8 @@ function innerShuffleList(input: Shuffleable, seed: Bytes32, dir: boolean): void const listSize = input.length >>> 0; // check if list size fits in uint32 assert.equal(listSize, input.length, "input length does not fit uint32"); + // check that the seed is 32 bytes + assert.equal(seed.length, 32, "seed length is not 32 bytes"); const buf = Buffer.alloc(_SHUFFLE_H_TOTAL_SIZE); let r = 0; @@ -100,8 +102,7 @@ function innerShuffleList(input: Shuffleable, seed: Bytes32, dir: boolean): void } // Seed is always the first 32 bytes of the hash input, we never have to change this part of the buffer. - const _seed = seed; - buf.set(_seed.subarray(0, _SHUFFLE_H_SEED_SIZE), 0); + buf.set(seed, 0); // initial values here are not used: overwritten first within the inner for loop. let source = seed; // just setting it to a Bytes32 From 5d26127a9447deb4fc1f3503973e8ecb176469ea Mon Sep 17 00:00:00 2001 From: Cayman Date: Thu, 2 May 2024 15:30:54 -0400 Subject: [PATCH 3/9] fix: use subarray instead of slice in shuffling --- packages/state-transition/src/util/shuffle.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/state-transition/src/util/shuffle.ts b/packages/state-transition/src/util/shuffle.ts index 176d4f33cd8..ea18ae1286e 100644 --- a/packages/state-transition/src/util/shuffle.ts +++ b/packages/state-transition/src/util/shuffle.ts @@ -115,8 +115,8 @@ function innerShuffleList(input: Shuffleable, seed: Bytes32, dir: boolean): void buf[_SHUFFLE_H_SEED_SIZE] = r; // Seed is already in place, now just hash the correct part of the buffer, and take a uint64 from it, // and modulo it to get a pivot within range. - const h = digest(buf.slice(0, _SHUFFLE_H_PIVOT_VIEW_SIZE)); - const pivot = Number(bytesToBigInt(h.slice(0, 8)) % BigInt(listSize)) >>> 0; + const h = digest(buf.subarray(0, _SHUFFLE_H_PIVOT_VIEW_SIZE)); + const pivot = Number(bytesToBigInt(h.subarray(0, 8)) % BigInt(listSize)) >>> 0; // Split up the for-loop in two: // 1. Handle the part from 0 (incl) to pivot (incl). This is mirrored around (pivot / 2) From 3e5984e493dd9e5fc7d73309b73dcdf3e9379c60 Mon Sep 17 00:00:00 2001 From: Cayman Date: Thu, 2 May 2024 15:32:59 -0400 Subject: [PATCH 4/9] chore: remove unnecessary devDependencies --- packages/state-transition/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index 9a65755a647..2d179ee93af 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -70,7 +70,6 @@ "@lodestar/utils": "^1.18.0", "bigint-buffer": "^1.1.5" }, - "devDependencies": {}, "keywords": [ "ethereum", "eth-consensus", From 5a71120d8f37e2b20518a2d39be504b2addb82b6 Mon Sep 17 00:00:00 2001 From: Cayman Date: Thu, 2 May 2024 15:40:20 -0400 Subject: [PATCH 5/9] chore: rely on fastify 4.x behavior --- packages/api/src/beacon/server/beacon.ts | 8 ++++---- packages/api/src/beacon/server/debug.ts | 8 ++++---- packages/api/src/beacon/server/proof.ts | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/api/src/beacon/server/beacon.ts b/packages/api/src/beacon/server/beacon.ts index 6fd4f89e38c..cd1ed72fb58 100644 --- a/packages/api/src/beacon/server/beacon.ts +++ b/packages/api/src/beacon/server/beacon.ts @@ -22,8 +22,8 @@ export function getRoutes(config: ChainForkConfig, api: ServerApi): ServerR handler: async (req) => { const response = await api.getBlock(...reqSerializers.getBlock.parseReq(req)); if (response instanceof Uint8Array) { - // Fastify 3.x.x will automatically add header `Content-Type: application/octet-stream` if Buffer - return Buffer.from(response.buffer, response.byteOffset, response.byteLength); + // Fastify 4.x.x will automatically add header `Content-Type: application/octet-stream` if TypedArray + return response; } else { return returnTypes.getBlock.toJson(response); } @@ -37,8 +37,8 @@ export function getRoutes(config: ChainForkConfig, api: ServerApi): ServerR const slot = extractSlotFromBlockBytes(response); const version = config.getForkName(slot); void res.header("Eth-Consensus-Version", version); - // Fastify 3.x.x will automatically add header `Content-Type: application/octet-stream` if Buffer - return Buffer.from(response.buffer, response.byteOffset, response.byteLength); + // Fastify 4.x.x will automatically add header `Content-Type: application/octet-stream` if TypedArray + return response; } else { void res.header("Eth-Consensus-Version", response.version); return returnTypes.getBlockV2.toJson(response); diff --git a/packages/api/src/beacon/server/debug.ts b/packages/api/src/beacon/server/debug.ts index c8f4a928492..553e08afddd 100644 --- a/packages/api/src/beacon/server/debug.ts +++ b/packages/api/src/beacon/server/debug.ts @@ -23,8 +23,8 @@ export function getRoutes(config: ChainForkConfig, api: ServerApi): ServerR handler: async (req) => { const response = await api.getState(...reqSerializers.getState.parseReq(req)); if (response instanceof Uint8Array) { - // Fastify 3.x.x will automatically add header `Content-Type: application/octet-stream` if Buffer - return Buffer.from(response.buffer, response.byteOffset, response.byteLength); + // Fastify 4.x.x will automatically add header `Content-Type: application/octet-stream` if TypedArray + return response; } else { return returnTypes.getState.toJson(response); } @@ -38,8 +38,8 @@ export function getRoutes(config: ChainForkConfig, api: ServerApi): ServerR const slot = extractSlotFromStateBytes(response); const version = config.getForkName(slot); void res.header("Eth-Consensus-Version", version); - // Fastify 3.x.x will automatically add header `Content-Type: application/octet-stream` if Buffer - return Buffer.from(response.buffer, response.byteOffset, response.byteLength); + // Fastify 4.x.x will automatically add header `Content-Type: application/octet-stream` if TypedArray + return response; } else { void res.header("Eth-Consensus-Version", response.version); return returnTypes.getStateV2.toJson(response); diff --git a/packages/api/src/beacon/server/proof.ts b/packages/api/src/beacon/server/proof.ts index d33541749af..bc865626ba7 100644 --- a/packages/api/src/beacon/server/proof.ts +++ b/packages/api/src/beacon/server/proof.ts @@ -24,8 +24,8 @@ export function getRoutes(config: ChainForkConfig, api: ServerApi): ServerR for (let i = 0; i < leaves.length; i++) { response.set(leaves[i], i * 32); } - // Fastify 3.x.x will automatically add header `Content-Type: application/octet-stream` if Buffer - return Buffer.from(response.buffer, response.byteOffset, response.byteLength); + // Fastify 4.x.x will automatically add header `Content-Type: application/octet-stream` if TypedArray + return response; }, }, getBlockProof: { @@ -38,8 +38,8 @@ export function getRoutes(config: ChainForkConfig, api: ServerApi): ServerR for (let i = 0; i < leaves.length; i++) { response.set(leaves[i], i * 32); } - // Fastify 3.x.x will automatically add header `Content-Type: application/octet-stream` if Buffer - return Buffer.from(response.buffer, response.byteOffset, response.byteLength); + // Fastify 4.x.x will automatically add header `Content-Type: application/octet-stream` if TypedArray + return response; }, }, }; From dc549ded28726c5bc5a06d3ded062788c5c30104 Mon Sep 17 00:00:00 2001 From: Cayman Date: Thu, 2 May 2024 15:41:56 -0400 Subject: [PATCH 6/9] chore: avoid copy in verifyMerkleBranch --- packages/utils/src/verifyMerkleBranch.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/utils/src/verifyMerkleBranch.ts b/packages/utils/src/verifyMerkleBranch.ts index e109813dd29..7bb090da06e 100644 --- a/packages/utils/src/verifyMerkleBranch.ts +++ b/packages/utils/src/verifyMerkleBranch.ts @@ -23,5 +23,5 @@ export function verifyMerkleBranch( value = digest64(Buffer.concat([value, proof[i]])); } } - return Buffer.from(value).equals(root); + return Buffer.from(value.buffer, value.byteOffset, value.byteLength).equals(root); } From 9f9904ebe07a3c8ddee7dd5d10774837a251416c Mon Sep 17 00:00:00 2001 From: Cayman Date: Thu, 23 May 2024 09:33:13 -0400 Subject: [PATCH 7/9] use toBase64 --- packages/beacon-node/src/util/sszBytes.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/beacon-node/src/util/sszBytes.ts b/packages/beacon-node/src/util/sszBytes.ts index 52ff7de6cac..802b9a266ab 100644 --- a/packages/beacon-node/src/util/sszBytes.ts +++ b/packages/beacon-node/src/util/sszBytes.ts @@ -25,10 +25,6 @@ const SLOT_SIZE = 8; const ATTESTATION_DATA_SIZE = 128; const SIGNATURE_SIZE = 96; -function toBuffer(x: Uint8Array): Buffer { - return Buffer.from(x.buffer, x.byteOffset, x.byteLength); -} - /** * Extract slot from attestation serialized bytes. * Return null if data is not long enough to extract slot. @@ -63,7 +59,7 @@ export function getAttDataBase64FromAttestationSerialized(data: Uint8Array): Att } // base64 is a bit efficient than hex - return toBuffer(data.slice(VARIABLE_FIELD_OFFSET, VARIABLE_FIELD_OFFSET + ATTESTATION_DATA_SIZE)).toString("base64"); + return toBase64(data.slice(VARIABLE_FIELD_OFFSET, VARIABLE_FIELD_OFFSET + ATTESTATION_DATA_SIZE)); } /** @@ -152,9 +148,9 @@ export function getAttDataBase64FromSignedAggregateAndProofSerialized(data: Uint } // base64 is a bit efficient than hex - return toBuffer( + return toBase64( data.slice(SIGNED_AGGREGATE_AND_PROOF_SLOT_OFFSET, SIGNED_AGGREGATE_AND_PROOF_SLOT_OFFSET + ATTESTATION_DATA_SIZE) - ).toString("base64"); + ); } /** @@ -208,3 +204,7 @@ function getSlotFromOffset(data: Uint8Array, offset: number): Slot { // Read only the first 4 bytes of Slot, max value is 4,294,967,295 will be reached 1634 years after genesis return dv.getUint32(offset, true); } + +function toBase64(data: Uint8Array): string { + return Buffer.from(data.buffer, data.byteOffset, data.byteLength).toString("base64"); +} From c2832c2c4f8443ae78a00762a9d86d1a02789d36 Mon Sep 17 00:00:00 2001 From: Cayman Date: Thu, 23 May 2024 12:28:52 -0400 Subject: [PATCH 8/9] relax assertions in shuffle function --- packages/state-transition/src/util/shuffle.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/state-transition/src/util/shuffle.ts b/packages/state-transition/src/util/shuffle.ts index ea18ae1286e..f8ef418ad5b 100644 --- a/packages/state-transition/src/util/shuffle.ts +++ b/packages/state-transition/src/util/shuffle.ts @@ -91,7 +91,7 @@ function innerShuffleList(input: Shuffleable, seed: Bytes32, dir: boolean): void // check if list size fits in uint32 assert.equal(listSize, input.length, "input length does not fit uint32"); // check that the seed is 32 bytes - assert.equal(seed.length, 32, "seed length is not 32 bytes"); + assert.lte(seed.length, 32, "seed length is not lte 32 bytes"); const buf = Buffer.alloc(_SHUFFLE_H_TOTAL_SIZE); let r = 0; From 1c687b76075800aa021600264754d640e46162e6 Mon Sep 17 00:00:00 2001 From: Cayman Date: Fri, 24 May 2024 09:14:10 -0400 Subject: [PATCH 9/9] Update packages/state-transition/src/util/shuffle.ts Co-authored-by: twoeths --- packages/state-transition/src/util/shuffle.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/state-transition/src/util/shuffle.ts b/packages/state-transition/src/util/shuffle.ts index f8ef418ad5b..39137bca69d 100644 --- a/packages/state-transition/src/util/shuffle.ts +++ b/packages/state-transition/src/util/shuffle.ts @@ -91,7 +91,7 @@ function innerShuffleList(input: Shuffleable, seed: Bytes32, dir: boolean): void // check if list size fits in uint32 assert.equal(listSize, input.length, "input length does not fit uint32"); // check that the seed is 32 bytes - assert.lte(seed.length, 32, "seed length is not lte 32 bytes"); + assert.lte(seed.length, _SHUFFLE_H_SEED_SIZE, `seed length is not lte ${_SHUFFLE_H_SEED_SIZE} bytes`); const buf = Buffer.alloc(_SHUFFLE_H_TOTAL_SIZE); let r = 0;