From f5fb78d5156278f14c6dde98660c5865285c1361 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Tue, 7 May 2024 09:37:19 -0500 Subject: [PATCH 001/142] Encode empty hex-string with quotes --- .../pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index 3ab2e9eea8..bc7154ecf6 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -65,7 +65,14 @@ def get_default_yaml(): def _represent_none(self, _): return self.represent_scalar('tag:yaml.org,2002:null', 'null') + def _represent_str(self, data): + if data.startswith("0x"): + # Without this, a zero-byte hex string is represented without quotes. + return self.represent_scalar('tag:yaml.org,2002:str', data, style="'") + return self.represent_str(data) + yaml.representer.add_representer(type(None), _represent_none) + yaml.representer.add_representer(str, _represent_str) return yaml From f97538719ba4bcd08fbbc8601cc274e74a492066 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Tue, 14 May 2024 17:15:44 +0200 Subject: [PATCH 002/142] p2p: Deprecate TTFB, RESP_TIMEOUT, introduce rate limiting recommendations As part of the discussions surrounding EIP-7594 (peerdas), it was highlighted that during sampling and/or data requests, the sampler does not have timing information for when a samplee will have data available. It is desireable to not introduce a deadline, since this artificially introduces latency for the typical scenario where data becomes available earlier than an agreed-upon deadline. Similarly, when a client issues a request for blocks, it does often not know what rate limiting policy of the serving end and must either pessimistically rate limit itself or run the risk of getting disconnected for spamming the server - outcomes which lead to unnecessarily slow syncing as well as testnet mess with peer scoring and disconnection issues. This PR solves both problems by: * removing the time-to-first-byte and response timeouts allowing requesters to optimistically queue requests - the timeouts have historically not been implemented fully in clients to this date * introducing a hard limit in the number of concurrent requests that a client may issue, per protocol * introducing a recommendation for rate limiting that allows optimal bandwidth usage without protocol changes or additional messaging roundtrips On the server side, an "open" request does not consume significant resources while it's resting, meaning that allowing the server to manage resource allocation by slowing down data serving is safe, as long as concurrency is adequately limited. On the client side, clients must be prepared to handle slow servers already and they can simply apply their existing strategy both to uncertainty and rate-limiting scenarios (how long before timeout, what to do in "slow peer" scenarios). Token / leaky buckets are a classic option for rate limiting with desireable properties both for the case when we're sending requests to many clients concurrently (getting good burst performance) and when the requestees are busy (by keeping long-term resource usage in check and fairly serving clients) --- specs/phase0/p2p-interface.md | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index bc29b9ad92..4c8f70f442 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -194,8 +194,8 @@ This section outlines configurations that are used in this spec. | `EPOCHS_PER_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | Number of epochs on a subnet subscription (~27 hours) | | `MIN_EPOCHS_FOR_BLOCK_REQUESTS` | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months) | The minimum epoch range over which a node must serve blocks | | `MAX_CHUNK_SIZE` | `10 * 2**20` (=10485760, 10 MiB) | The maximum allowed size of uncompressed req/resp chunked responses. | -| `TTFB_TIMEOUT` | `5` | The maximum duration in **seconds** to wait for first byte of request response (time-to-first-byte). | -| `RESP_TIMEOUT` | `10` | The maximum duration in **seconds** for complete response transfer. | +| `TTFB_TIMEOUT` | N/A | TTFB should remain disabled. | +| `RESP_TIMEOUT` | N/A | Response timeouts are request-specific | | `ATTESTATION_PROPAGATION_SLOT_RANGE` | `32` | The maximum number of slots during which an attestation can be propagated. | | `MAXIMUM_GOSSIP_CLOCK_DISPARITY` | `500` | The maximum **milliseconds** of clock disparity assumed between honest nodes. | | `MESSAGE_DOMAIN_INVALID_SNAPPY` | `DomainType('0x00000000')` | 4-byte domain for gossip message-id isolation of *invalid* snappy messages | @@ -561,10 +561,9 @@ The request MUST be encoded according to the encoding strategy. The requester MUST close the write side of the stream once it finishes writing the request message. At this point, the stream will be half-closed. -The requester MUST wait a maximum of `TTFB_TIMEOUT` for the first response byte to arrive (time to first byte—or TTFB—timeout). -On that happening, the requester allows a further `RESP_TIMEOUT` for each subsequent `response_chunk` received. +The requester MUST NOT make more than two concurrent requests with the same ID. -If any of these timeouts fire, the requester SHOULD reset the stream and deem the req/resp operation to have failed. +If a timeout happens on the requesting side, they SHOULD reset the stream. A requester SHOULD read from the stream until either: 1. An error result is received in one of the chunks (the error payload MAY be read before stopping). @@ -593,10 +592,10 @@ The responder MUST: If steps (1), (2), or (3) fail due to invalid, malformed, or inconsistent data, the responder MUST respond in error. Clients tracking peer reputation MAY record such failures, as well as unexpected events, e.g. early stream resets. -The entire request should be read in no more than `RESP_TIMEOUT`. -Upon a timeout, the responder SHOULD reset the stream. +The responder MAY rate-limit chunks by withholding each chunk until capacity is available. The responding side MUST NOT respond with an error or close the stream when rate limiting. + +When rate limiting, the responder MUST send each `response_chunk` in full promptly but may introduce delays between each chunk. -The responder SHOULD send a `response_chunk` promptly. Chunks start with a **single-byte** response code which determines the contents of the `response_chunk` (`result` particle in the BNF grammar above). For multiple chunks, only the last chunk is allowed to have a non-zero error code (i.e. The chunk stream is terminated once an error occurs). @@ -626,6 +625,8 @@ The `ErrorMessage` schema is: *Note*: By convention, the `error_message` is a sequence of bytes that MAY be interpreted as a UTF-8 string (for debugging purposes). Clients MUST treat as valid any byte sequences. +The responder MAY penalise peers that concurrently open more than two streams for the same request type, for the protocol ID:s defined in this specification. + #### Encoding strategies The token of the negotiated protocol ID specifies the type of encoding to be used for the req/resp interaction. @@ -1455,6 +1456,20 @@ For this reason, we remove and replace semver with ordinals that require explici Req/Resp is used to avoid confusion with JSON-RPC and similar user-client interaction mechanisms. +#### What is a typical rate limiting strategy? + +The serving side typically will want to rate limit requests to protect against spam and to manage resource consumption, while the requesting side will want to maximise performance based on its own resource allocation strategy. For the network, it is beneficial if available resources are used optimally. + +Broadly, the requesting side does not know the capacity / limit of each server but can derive it from the rate of responses for the purpose of selecting the next peer for a request. + +Because the server withholds the response until capacity is available, a client can optimistically send requests without risking running into negative scoring situations or sub-optimal rate polling. + +A typical approach for the requesting side is to implement a timeout on the request that depends on the nature of the request and on connectivity parameters in general - for example when requesting blocks, a peer might choose to send a request to a second peer if the first peer does not respond within a reasonable time, and to reset the request to the first peer if the second peer responds faster. Clients may use past response perforance to reward fast peers when implementing peer scoring. + +A typical approach for the responding side is to implement a two-level token/leaky bucket with a per-peer limit and a global limit. The granularity of rate limiting may be based either on full requests or individual chunks with the latter being preferable. A token cost may be assigned to the request itself and separately each chunk in the response so as to remain protected both against large and frequent requests. + +For requesters, rate limiting is not distinguishable from other conditions causing slow responses (slow peers, congestion etc) and since the latter conditions must be handled anyway, including rate limiting in this strategy keeps the implementation simple. + #### Why do we allow empty responses in block requests? When requesting blocks by range or root, it may happen that there are no blocks in the selected range or the responding node does not have the requested blocks. From f349bfcddc9dfc3d8719def876ad727dd8f8bc80 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Tue, 21 May 2024 13:21:25 +0200 Subject: [PATCH 003/142] lint --- specs/phase0/p2p-interface.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 4c8f70f442..2240c21765 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -94,6 +94,7 @@ It consists of four main sections: - [Why are messages length-prefixed with a protobuf varint in the SSZ-encoding?](#why-are-messages-length-prefixed-with-a-protobuf-varint-in-the-ssz-encoding) - [Why do we version protocol strings with ordinals instead of semver?](#why-do-we-version-protocol-strings-with-ordinals-instead-of-semver) - [Why is it called Req/Resp and not RPC?](#why-is-it-called-reqresp-and-not-rpc) + - [What is a typical rate limiting strategy?](#what-is-a-typical-rate-limiting-strategy) - [Why do we allow empty responses in block requests?](#why-do-we-allow-empty-responses-in-block-requests) - [Why does `BeaconBlocksByRange` let the server choose which branch to send blocks from?](#why-does-beaconblocksbyrange-let-the-server-choose-which-branch-to-send-blocks-from) - [Why are `BlocksByRange` requests only required to be served for the latest `MIN_EPOCHS_FOR_BLOCK_REQUESTS` epochs?](#why-are-blocksbyrange-requests-only-required-to-be-served-for-the-latest-min_epochs_for_block_requests-epochs) @@ -1464,7 +1465,7 @@ Broadly, the requesting side does not know the capacity / limit of each server b Because the server withholds the response until capacity is available, a client can optimistically send requests without risking running into negative scoring situations or sub-optimal rate polling. -A typical approach for the requesting side is to implement a timeout on the request that depends on the nature of the request and on connectivity parameters in general - for example when requesting blocks, a peer might choose to send a request to a second peer if the first peer does not respond within a reasonable time, and to reset the request to the first peer if the second peer responds faster. Clients may use past response perforance to reward fast peers when implementing peer scoring. +A typical approach for the requesting side is to implement a timeout on the request that depends on the nature of the request and on connectivity parameters in general - for example when requesting blocks, a peer might choose to send a request to a second peer if the first peer does not respond within a reasonable time, and to reset the request to the first peer if the second peer responds faster. Clients may use past response performance to reward fast peers when implementing peer scoring. A typical approach for the responding side is to implement a two-level token/leaky bucket with a per-peer limit and a global limit. The granularity of rate limiting may be based either on full requests or individual chunks with the latter being preferable. A token cost may be assigned to the request itself and separately each chunk in the response so as to remain protected both against large and frequent requests. From e4919d71d80cfc345dcd36cd81116190a51c8b32 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Wed, 22 May 2024 07:57:43 +0200 Subject: [PATCH 004/142] Update specs/phase0/p2p-interface.md Co-authored-by: Pop Chunhapanya --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 2240c21765..4560d4890e 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -626,7 +626,7 @@ The `ErrorMessage` schema is: *Note*: By convention, the `error_message` is a sequence of bytes that MAY be interpreted as a UTF-8 string (for debugging purposes). Clients MUST treat as valid any byte sequences. -The responder MAY penalise peers that concurrently open more than two streams for the same request type, for the protocol ID:s defined in this specification. +The responder MAY penalise peers that concurrently open more than two streams for the same request type, for the protocol IDs defined in this specification. #### Encoding strategies From e872fdb936c6e27125a0773d20f35c1a373ef844 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Fri, 2 Aug 2024 14:19:26 +0200 Subject: [PATCH 005/142] engine_getBlobV1_clarifications --- specs/deneb/p2p-interface.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 41e2fa9d3f..ef5985f7f8 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -184,6 +184,7 @@ The following validations MUST pass before forwarding the `blob_sidecar` on the - _[REJECT]_ The sidecar's inclusion proof is valid as verified by `verify_blob_sidecar_inclusion_proof(blob_sidecar)`. - _[REJECT]_ The sidecar's blob is valid as verified by `verify_blob_kzg_proof(blob_sidecar.blob, blob_sidecar.kzg_commitment, blob_sidecar.kzg_proof)`. - _[IGNORE]_ The sidecar is the first sidecar for the tuple `(block_header.slot, block_header.proposer_index, blob_sidecar.index)` with valid header signature, sidecar inclusion proof, and kzg proof. +*Note:* If client obtains the blob via `engine_getBlobV1` or any other method before receiving the corresponding sidecar via this topic, it MUST NOT consider the tuple as "seen" (i.e. it MUST participate in the dissemination of the blob_sidecar even if it already obtained the blob) - _[REJECT]_ The sidecar is proposed by the expected `proposer_index` for the block's slot in the context of the current shuffling (defined by `block_header.parent_root`/`block_header.slot`). If the `proposer_index` cannot immediately be verified against the expected shuffling, the sidecar MAY be queued for later processing while proposers for the block's branch are calculated -- in such a case _do not_ `REJECT`, instead `IGNORE` this message. @@ -295,7 +296,7 @@ Before consuming the next response chunk, the response reader SHOULD verify the No more than `MAX_REQUEST_BLOB_SIDECARS` may be requested at a time. -`BlobSidecarsByRoot` is primarily used to recover recent blobs (e.g. when receiving a block with a transaction whose corresponding blob is missing). +`BlobSidecarsByRoot` is primarily used to recover recent blobs (e.g. when receiving a block with a transaction whose corresponding blob is missing). In addition, it is possible to retrieve recent blobs by quering the Execution Layer via `engine_getBlobV1` engine API method. The response MUST consist of zero or more `response_chunk`. Each _successful_ `response_chunk` MUST contain a single `BlobSidecar` payload. From 09966cd07071740e67d2851ab37aed6efa4176ed Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Fri, 2 Aug 2024 14:23:19 +0200 Subject: [PATCH 006/142] fix method name --- specs/deneb/p2p-interface.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index ef5985f7f8..cde726e2f4 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -184,7 +184,7 @@ The following validations MUST pass before forwarding the `blob_sidecar` on the - _[REJECT]_ The sidecar's inclusion proof is valid as verified by `verify_blob_sidecar_inclusion_proof(blob_sidecar)`. - _[REJECT]_ The sidecar's blob is valid as verified by `verify_blob_kzg_proof(blob_sidecar.blob, blob_sidecar.kzg_commitment, blob_sidecar.kzg_proof)`. - _[IGNORE]_ The sidecar is the first sidecar for the tuple `(block_header.slot, block_header.proposer_index, blob_sidecar.index)` with valid header signature, sidecar inclusion proof, and kzg proof. -*Note:* If client obtains the blob via `engine_getBlobV1` or any other method before receiving the corresponding sidecar via this topic, it MUST NOT consider the tuple as "seen" (i.e. it MUST participate in the dissemination of the blob_sidecar even if it already obtained the blob) +*Note:* If client obtains the blob via `engine_getBlobsV1` or any other method before receiving the corresponding sidecar via this topic, it MUST NOT consider the tuple as "seen" (i.e. it MUST participate in the dissemination of the blob_sidecar even if it already obtained the blob) - _[REJECT]_ The sidecar is proposed by the expected `proposer_index` for the block's slot in the context of the current shuffling (defined by `block_header.parent_root`/`block_header.slot`). If the `proposer_index` cannot immediately be verified against the expected shuffling, the sidecar MAY be queued for later processing while proposers for the block's branch are calculated -- in such a case _do not_ `REJECT`, instead `IGNORE` this message. @@ -296,7 +296,7 @@ Before consuming the next response chunk, the response reader SHOULD verify the No more than `MAX_REQUEST_BLOB_SIDECARS` may be requested at a time. -`BlobSidecarsByRoot` is primarily used to recover recent blobs (e.g. when receiving a block with a transaction whose corresponding blob is missing). In addition, it is possible to retrieve recent blobs by quering the Execution Layer via `engine_getBlobV1` engine API method. +`BlobSidecarsByRoot` is primarily used to recover recent blobs (e.g. when receiving a block with a transaction whose corresponding blob is missing). In addition, it is possible to retrieve recent blobs by quering the Execution Layer via `engine_getBlobsV1` engine API method. The response MUST consist of zero or more `response_chunk`. Each _successful_ `response_chunk` MUST contain a single `BlobSidecar` payload. From 6f219159e577f834f8c8e6a08d9c3920062f46ae Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Fri, 2 Aug 2024 17:04:04 +0200 Subject: [PATCH 007/142] typo --- specs/deneb/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index cde726e2f4..da95e5786c 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -296,7 +296,7 @@ Before consuming the next response chunk, the response reader SHOULD verify the No more than `MAX_REQUEST_BLOB_SIDECARS` may be requested at a time. -`BlobSidecarsByRoot` is primarily used to recover recent blobs (e.g. when receiving a block with a transaction whose corresponding blob is missing). In addition, it is possible to retrieve recent blobs by quering the Execution Layer via `engine_getBlobsV1` engine API method. +`BlobSidecarsByRoot` is primarily used to recover recent blobs (e.g. when receiving a block with a transaction whose corresponding blob is missing). In addition, it is possible to retrieve recent blobs by querying the Execution Layer via `engine_getBlobsV1` engine API method. The response MUST consist of zero or more `response_chunk`. Each _successful_ `response_chunk` MUST contain a single `BlobSidecar` payload. From 99bf3c112d1408a0920d16e22c0ddfd8b13bec45 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Wed, 14 Aug 2024 20:33:17 +0200 Subject: [PATCH 008/142] Avoid generating empty transactions in tests Transactions cannot be empty, they always have at least 1 byte. Random tests should produce valid CL data by default. There are still individual tests for invalid transactions. --- tests/core/pyspec/eth2spec/test/helpers/execution_payload.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py index 1fbb12d7ba..99399abd41 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py @@ -64,7 +64,7 @@ def compute_trie_root_from_indexed_data(data): t = HexaryTrie(db={}) for i, obj in enumerate(data): k = encode(i, big_endian_int) - t.set(k, obj) + t.set(k, obj) # Implicitly skipped if `obj == b''` (invalid RLP) return t.root_hash @@ -359,4 +359,4 @@ def build_state_with_execution_payload_header(spec, state, execution_payload_hea def get_random_tx(rng): - return get_random_bytes_list(rng, rng.randint(0, 1000)) + return get_random_bytes_list(rng, rng.randint(1, 1000)) From e1eaa7ffa194f70f044970689b11b27c9d68cb66 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Wed, 14 Aug 2024 22:55:05 +0200 Subject: [PATCH 009/142] Synchronously check all `transactions` to have non-zero length As part of `newPayload` block hash verification, the `transactionsRoot` is computed by the EL. Because Merkle-Patricia Tries cannot contain `[]` entries, MPT implementations typically treat setting a key to `[]` as deleting the entry for the key. This means that if a CL receives a block with `transactions` containing one or more zero-length transactions, that such transactions will effectively be skipped when computing the `transactionsRoot`. Note that `transactions` are opaque to the CL and zero-length transactions are not filtered out before `newPayload`. ```python # https://eips.ethereum.org/EIPS/eip-2718 def compute_trie_root_from_indexed_data(data): """ Computes the root hash of `patriciaTrie(rlp(Index) => Data)` for a data array. """ t = HexaryTrie(db={}) for i, obj in enumerate(data): k = encode(i, big_endian_int) t.set(k, obj) # Implicitly skipped if `obj == b''` (invalid RLP) return t.root_hash ``` In any case, the `blockHash` validation may still succeed, resulting in a potential `SYNCING/ACCEPTED` result to `newPayload` by spec. Note, however, that there is an effective hash collision if a payload is modified by appending one or more zero-length transactions to the end of `transactions` list: In the trivial case, a block with zero transactions has the same `transactionsRoot` (and `blockHash`) as one of a block with one `[]` transaction (as that one is skipped). This means that the same `blockHash` can refer to a valid block (without extra `[]` transactions added), but also can refer to an invalid block. Because `forkchoiceUpdated` refers to blocks by `blockHash`, outcome may be nondeterministic and implementation dependent. If `forkchoiceUpdated` deems the `blockHash` to refer to a `VALID` object (obtained from a src that does not have the extra `[]` transactions, e.g., devp2p), then this could result in honest attestations to a CL beacon block with invalid `[]` transactions in its `ExecutionPayload`, risking finalizing it. The problem can be avoided by returning `INVALID` in `newPayload` if there are any zero-length `transactions` entries, preventing optimistic import of such blocks by the CL. --- specs/bellatrix/beacon-chain.md | 11 +++++++++-- specs/deneb/beacon-chain.md | 4 ++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/specs/bellatrix/beacon-chain.md b/specs/bellatrix/beacon-chain.md index 6f67be96a7..51d570fe2d 100644 --- a/specs/bellatrix/beacon-chain.md +++ b/specs/bellatrix/beacon-chain.md @@ -355,10 +355,17 @@ def verify_and_notify_new_payload(self: ExecutionEngine, """ Return ``True`` if and only if ``new_payload_request`` is valid with respect to ``self.execution_state``. """ - if not self.is_valid_block_hash(new_payload_request.execution_payload): + execution_payload = new_payload_request.execution_payload + + if b'' in execution_payload.transactions: return False - if not self.notify_new_payload(new_payload_request.execution_payload): + + if not self.is_valid_block_hash(execution_payload): return False + + if not self.notify_new_payload(execution_payload): + return False + return True ``` diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 0f6a8fc076..1b7a322c2f 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -293,9 +293,13 @@ def verify_and_notify_new_payload(self: ExecutionEngine, """ Return ``True`` if and only if ``new_payload_request`` is valid with respect to ``self.execution_state``. """ + # [Modified in Deneb:EIP4788] execution_payload = new_payload_request.execution_payload parent_beacon_block_root = new_payload_request.parent_beacon_block_root + if b'' in execution_payload.transactions: + return False + # [Modified in Deneb:EIP4788] if not self.is_valid_block_hash(execution_payload, parent_beacon_block_root): return False From 647a45d3b1a885ae77979a46a3fe635f1190292d Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 21 Aug 2024 17:08:14 -0500 Subject: [PATCH 010/142] Rebase eip7594 onto Electra --- pysetup/md_doc_paths.py | 2 +- pysetup/spec_builders/eip7594.py | 2 +- specs/_features/eip7594/fork.md | 17 +++++++++++++++-- .../pyspec/eth2spec/test/helpers/constants.py | 2 +- tests/generators/merkle_proof/main.py | 2 +- 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/pysetup/md_doc_paths.py b/pysetup/md_doc_paths.py index d99fc122ac..0f0d1c8593 100644 --- a/pysetup/md_doc_paths.py +++ b/pysetup/md_doc_paths.py @@ -22,7 +22,7 @@ DENEB: CAPELLA, ELECTRA: DENEB, WHISK: CAPELLA, - EIP7594: DENEB, + EIP7594: ELECTRA, EIP6800: DENEB, EIP7732: ELECTRA, } diff --git a/pysetup/spec_builders/eip7594.py b/pysetup/spec_builders/eip7594.py index 109ad8736f..bbc46294ca 100644 --- a/pysetup/spec_builders/eip7594.py +++ b/pysetup/spec_builders/eip7594.py @@ -10,7 +10,7 @@ class EIP7594SpecBuilder(BaseSpecBuilder): @classmethod def imports(cls, preset_name: str): return f''' -from eth2spec.deneb import {preset_name} as deneb +from eth2spec.electra import {preset_name} as electra ''' diff --git a/specs/_features/eip7594/fork.md b/specs/_features/eip7594/fork.md index 790ab0287d..bb688c0dcd 100644 --- a/specs/_features/eip7594/fork.md +++ b/specs/_features/eip7594/fork.md @@ -44,6 +44,8 @@ def compute_fork_version(epoch: Epoch) -> Version: """ if epoch >= EIP7594_FORK_EPOCH: return EIP7594_FORK_VERSION + if epoch >= ELECTRA_FORK_EPOCH: + return ELECTRA_FORK_VERSION if epoch >= DENEB_FORK_EPOCH: return DENEB_FORK_VERSION if epoch >= CAPELLA_FORK_EPOCH: @@ -71,8 +73,8 @@ If `state.slot % SLOTS_PER_EPOCH == 0` and `compute_epoch_at_slot(state.slot) == an irregular state change is made to upgrade to EIP7594. ```python -def upgrade_to_eip7594(pre: deneb.BeaconState) -> BeaconState: - epoch = deneb.get_current_epoch(pre) +def upgrade_to_eip7594(pre: electra.BeaconState) -> BeaconState: + epoch = electra.get_current_epoch(pre) post = BeaconState( # Versioning genesis_time=pre.genesis_time, @@ -119,6 +121,17 @@ def upgrade_to_eip7594(pre: deneb.BeaconState) -> BeaconState: next_withdrawal_validator_index=pre.next_withdrawal_validator_index, # Deep history valid from Capella onwards historical_summaries=pre.historical_summaries, + # On-chain deposits + deposit_requests_start_index=pre.deposit_requests_start_index, + deposit_balance_to_consume=pre.deposit_balance_to_consume, + # Consolidations + exit_balance_to_consume=pre.exit_balance_to_consume, + earliest_exit_epoch=pre.earliest_exit_epoch, + consolidation_balance_to_consume=pre.consolidation_balance_to_consume, + earliest_consolidation_epoch=pre.earliest_consolidation_epoch, + pending_balance_deposits=pre.pending_balance_deposits, + pending_partial_withdrawals=pre.pending_partial_withdrawals, + pending_consolidations=pre.pending_consolidations, ) return post diff --git a/tests/core/pyspec/eth2spec/test/helpers/constants.py b/tests/core/pyspec/eth2spec/test/helpers/constants.py index 7e5d0a0f4c..97ed1134e2 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/constants.py +++ b/tests/core/pyspec/eth2spec/test/helpers/constants.py @@ -57,7 +57,7 @@ ELECTRA: DENEB, # Experimental patches WHISK: CAPELLA, - EIP7594: DENEB, + EIP7594: ELECTRA, EIP7732: ELECTRA, } diff --git a/tests/generators/merkle_proof/main.py b/tests/generators/merkle_proof/main.py index 69500137ab..bf52111e96 100644 --- a/tests/generators/merkle_proof/main.py +++ b/tests/generators/merkle_proof/main.py @@ -10,7 +10,7 @@ 'single_merkle_proof', ]} electra_mods = deneb_mods - eip_7594_mods = combine_mods(_new_eip7594_mods, deneb_mods) + eip_7594_mods = combine_mods(_new_eip7594_mods, electra_mods) all_mods = { DENEB: deneb_mods, From b24d53f23afa2f7e4910512a31acd0c546849188 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 21 Aug 2024 20:04:32 -0500 Subject: [PATCH 011/142] Fix comment --- specs/_features/eip7594/fork.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/_features/eip7594/fork.md b/specs/_features/eip7594/fork.md index bb688c0dcd..4f2c17561f 100644 --- a/specs/_features/eip7594/fork.md +++ b/specs/_features/eip7594/fork.md @@ -123,8 +123,8 @@ def upgrade_to_eip7594(pre: electra.BeaconState) -> BeaconState: historical_summaries=pre.historical_summaries, # On-chain deposits deposit_requests_start_index=pre.deposit_requests_start_index, - deposit_balance_to_consume=pre.deposit_balance_to_consume, # Consolidations + deposit_balance_to_consume=pre.deposit_balance_to_consume, exit_balance_to_consume=pre.exit_balance_to_consume, earliest_exit_epoch=pre.earliest_exit_epoch, consolidation_balance_to_consume=pre.consolidation_balance_to_consume, From 08e020e98d5fbeef8d49e1f074e0564df3ea85b7 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Fri, 23 Aug 2024 13:37:33 +0200 Subject: [PATCH 012/142] remove timeout constants --- specs/phase0/p2p-interface.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 4560d4890e..269981a807 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -195,8 +195,6 @@ This section outlines configurations that are used in this spec. | `EPOCHS_PER_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | Number of epochs on a subnet subscription (~27 hours) | | `MIN_EPOCHS_FOR_BLOCK_REQUESTS` | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months) | The minimum epoch range over which a node must serve blocks | | `MAX_CHUNK_SIZE` | `10 * 2**20` (=10485760, 10 MiB) | The maximum allowed size of uncompressed req/resp chunked responses. | -| `TTFB_TIMEOUT` | N/A | TTFB should remain disabled. | -| `RESP_TIMEOUT` | N/A | Response timeouts are request-specific | | `ATTESTATION_PROPAGATION_SLOT_RANGE` | `32` | The maximum number of slots during which an attestation can be propagated. | | `MAXIMUM_GOSSIP_CLOCK_DISPARITY` | `500` | The maximum **milliseconds** of clock disparity assumed between honest nodes. | | `MESSAGE_DOMAIN_INVALID_SNAPPY` | `DomainType('0x00000000')` | 4-byte domain for gossip message-id isolation of *invalid* snappy messages | From 0b95012c2ece8f12436727b4a2c0beb0c024cb4f Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Mon, 26 Aug 2024 15:54:59 +0200 Subject: [PATCH 013/142] Separate type for unaggregated network attestations As a complement to https://github.com/ethereum/consensus-specs/pull/3787, this PR introduces a `SingleAttestation` type used for network propagation only. In Electra, the on-chain attestation format introduced in [EIP-7549](https://github.com/ethereum/consensus-specs/pull/3559) presents several difficulties - not only are the new fields to be interpreted differently during network processing and onchain which adds complexity in clients, they also introduce inefficiency both in hash computation and bandwidth. The new type puts the validator and committee indices directly in the attestation type, this simplifying processing and increasing security. * placing the validator index directly in the attestation allows verifying the signature without computing a shuffling - this closes a loophole where clients either must drop attestations or risk being overwhelmed by shuffling computations during attestation verification * the simpler "structure" of the attestation saves several hash calls during processing (a single-item List has significant hashing overhead compared to a field) * we save a few bytes here and there - we can also put stricter bounds on message size on the attestation topic because `SingleAttestation` is now fixed-size * the ambiguity of interpreting the `attestation_bits` list indices which became contextual under EIP-7549 is removed Because this change only affects the network encoding (and not block contents), the implementation impact on clients should be minimal. --- specs/electra/beacon-chain.md | 14 +++++++++++++- specs/electra/p2p-interface.md | 5 +++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/specs/electra/beacon-chain.md b/specs/electra/beacon-chain.md index cb5ee6a7a9..6a79a7012a 100644 --- a/specs/electra/beacon-chain.md +++ b/specs/electra/beacon-chain.md @@ -30,6 +30,7 @@ - [`WithdrawalRequest`](#withdrawalrequest) - [`ConsolidationRequest`](#consolidationrequest) - [`PendingConsolidation`](#pendingconsolidation) + - [`SingleAttestation`](#singleattestation) - [Modified Containers](#modified-containers) - [`AttesterSlashing`](#attesterslashing) - [Extended Containers](#extended-containers) @@ -258,6 +259,17 @@ class PendingConsolidation(Container): target_index: ValidatorIndex ``` + +#### `SingleAttestation` + +```python +class SingleAttestation(Container): + committee_index: CommitteeIndex + attester_index: ValidatorIndex + data: AttestationData + signature: BLSSignature +``` + ### Modified Containers #### `AttesterSlashing` @@ -875,7 +887,7 @@ def process_pending_balance_deposits(state: BeaconState) -> None: if processed_amount + deposit.amount > available_for_processing: break # Deposit fits in the churn, process it. Increase balance and consume churn. - else: + else: increase_balance(state, deposit.index, deposit.amount) processed_amount += deposit.amount # Regardless of how the deposit was handled, we move on in the queue. diff --git a/specs/electra/p2p-interface.md b/specs/electra/p2p-interface.md index 88d14813b3..aa6ed3789d 100644 --- a/specs/electra/p2p-interface.md +++ b/specs/electra/p2p-interface.md @@ -54,9 +54,10 @@ The following validations are added: ##### `beacon_attestation_{subnet_id}` +The topic is updated to propagate `SingleAttestation` objects. + The following convenience variables are re-defined -- `index = get_committee_indices(attestation.committee_bits)[0]` +- `index = attestation.committee_index` The following validations are added: -* [REJECT] `len(committee_indices) == 1`, where `committee_indices = get_committee_indices(attestation)`. * [REJECT] `attestation.data.index == 0` From 5761fb4d974b794d6d6dc0a0f82c3dff95ad618d Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Wed, 28 Aug 2024 07:31:49 +0200 Subject: [PATCH 014/142] update list of checks --- specs/electra/p2p-interface.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/specs/electra/p2p-interface.md b/specs/electra/p2p-interface.md index aa6ed3789d..9e4eff0a99 100644 --- a/specs/electra/p2p-interface.md +++ b/specs/electra/p2p-interface.md @@ -56,8 +56,16 @@ The following validations are added: The topic is updated to propagate `SingleAttestation` objects. -The following convenience variables are re-defined +The following convenience variables are re-defined: - `index = attestation.committee_index` The following validations are added: -* [REJECT] `attestation.data.index == 0` +- _[REJECT]_ `attestation.data.index == 0` +- _[REJECT]_ The attester is a member of the committtee -- i.e. + `atestation.attester_index in get_beacon_committee(state, attestation.data.slot, index)`. + +The following validations are removed: +- _[REJECT]_ The attestation is unaggregated -- + that is, it has exactly one participating validator (`len([bit for bit in aggregation_bits if bit]) == 1`, i.e. exactly 1 bit is set). +- _[REJECT]_ The number of aggregation bits matches the committee size -- i.e. + `len(aggregation_bits) == len(get_beacon_committee(state, attestation.data.slot, index))`. From 0833551328dc94e6f55f514c806291bb99743e6a Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Wed, 28 Aug 2024 07:35:12 +0200 Subject: [PATCH 015/142] spelling --- specs/electra/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/electra/p2p-interface.md b/specs/electra/p2p-interface.md index 9e4eff0a99..b8370c77f8 100644 --- a/specs/electra/p2p-interface.md +++ b/specs/electra/p2p-interface.md @@ -61,7 +61,7 @@ The following convenience variables are re-defined: The following validations are added: - _[REJECT]_ `attestation.data.index == 0` -- _[REJECT]_ The attester is a member of the committtee -- i.e. +- _[REJECT]_ The attester is a member of the committee -- i.e. `atestation.attester_index in get_beacon_committee(state, attestation.data.slot, index)`. The following validations are removed: From d61f95e138bca0069b69c10111d4ade30c4d29b1 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Wed, 28 Aug 2024 12:46:53 +0200 Subject: [PATCH 016/142] Fix blob transaction serialization to use RLP EIP-4844 uses RLP not SSZ for blob transactions. --- .../test_process_execution_payload.py | 28 ++--- .../test/deneb/fork_choice/test_on_block.py | 6 +- .../merkle_proof/test_single_merkle_proof.py | 6 +- .../eth2spec/test/deneb/sanity/test_blocks.py | 6 +- .../test_polynomial_commitments.py | 2 +- .../unittests/validator/test_validator.py | 8 +- .../merkle_proof/test_single_merkle_proof.py | 6 +- .../test/eip7594/unittests/das/test_das.py | 2 +- .../test_polynomial_commitments.py | 2 +- .../test/helpers/{sharding.py => blob.py} | 101 +++++++----------- .../test/utils/randomized_block_tests.py | 6 +- 11 files changed, 77 insertions(+), 96 deletions(-) rename tests/core/pyspec/eth2spec/test/helpers/{sharding.py => blob.py} (57%) diff --git a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py index 85175a3b9c..d0c4a6f22a 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py @@ -10,8 +10,8 @@ expect_assertion_error, with_deneb_and_later ) -from eth2spec.test.helpers.sharding import ( - get_sample_opaque_tx, +from eth2spec.test.helpers.blob import ( + get_sample_blob_tx, ) @@ -74,7 +74,7 @@ def test_incorrect_blob_tx_type(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec) opaque_tx = b'\x04' + opaque_tx[1:] # incorrect tx type execution_payload.transactions = [opaque_tx] @@ -91,7 +91,7 @@ def test_incorrect_transaction_length_1_extra_byte(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec) opaque_tx = opaque_tx + b'\x12' # incorrect tx length, longer execution_payload.transactions = [opaque_tx] @@ -108,7 +108,7 @@ def test_incorrect_transaction_length_1_byte_short(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec) opaque_tx = opaque_tx[:-1] # incorrect tx length, shorter execution_payload.transactions = [opaque_tx] @@ -125,7 +125,7 @@ def test_incorrect_transaction_length_empty(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec) opaque_tx = opaque_tx[0:0] # incorrect tx length, empty execution_payload.transactions = [opaque_tx] @@ -142,7 +142,7 @@ def test_incorrect_transaction_length_32_extra_bytes(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec) opaque_tx = opaque_tx + b'\x12' * 32 # incorrect tx length execution_payload.transactions = [opaque_tx] @@ -159,7 +159,7 @@ def test_no_transactions_with_commitments(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - _, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + _, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec) execution_payload.transactions = [] execution_payload.block_hash = compute_el_block_hash(spec, execution_payload, state) @@ -175,7 +175,7 @@ def test_incorrect_commitment(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec) blob_kzg_commitments[0] = b'\x12' * 48 # incorrect commitment execution_payload.transactions = [opaque_tx] @@ -192,7 +192,7 @@ def test_incorrect_commitments_order(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=2, rng=Random(1111)) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec, blob_count=2, rng=Random(1111)) blob_kzg_commitments = [blob_kzg_commitments[1], blob_kzg_commitments[0]] # incorrect order execution_payload.transactions = [opaque_tx] @@ -206,7 +206,7 @@ def test_incorrect_commitments_order(spec, state): def test_incorrect_block_hash(spec, state): execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec) execution_payload.transactions = [opaque_tx] execution_payload.block_hash = b'\x12' * 32 # incorrect block hash @@ -223,7 +223,7 @@ def test_zeroed_commitment(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=1, is_valid_blob=False) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec, blob_count=1, is_valid_blob=False) assert all(commitment == b'\x00' * 48 for commitment in blob_kzg_commitments) execution_payload.transactions = [opaque_tx] @@ -240,7 +240,7 @@ def test_invalid_correct_input__execution_invalid(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec) execution_payload.transactions = [opaque_tx] execution_payload.block_hash = compute_el_block_hash(spec, execution_payload, state) @@ -254,7 +254,7 @@ def test_invalid_correct_input__execution_invalid(spec, state): def test_invalid_exceed_max_blobs_per_block(spec, state): execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=spec.config.MAX_BLOBS_PER_BLOCK + 1) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec, blob_count=spec.config.MAX_BLOBS_PER_BLOCK + 1) execution_payload.transactions = [opaque_tx] execution_payload.block_hash = compute_el_block_hash(spec, execution_payload, state) diff --git a/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py b/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py index a7e7f784e4..905cb390eb 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py +++ b/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py @@ -25,14 +25,14 @@ from eth2spec.test.helpers.state import ( state_transition_and_sign_block, ) -from eth2spec.test.helpers.sharding import ( - get_sample_opaque_tx, +from eth2spec.test.helpers.blob import ( + get_sample_blob_tx, ) def get_block_with_blob(spec, state, rng=None): block = build_empty_block_for_next_slot(spec, state) - opaque_tx, blobs, blob_kzg_commitments, blob_kzg_proofs = get_sample_opaque_tx(spec, blob_count=1, rng=rng) + opaque_tx, blobs, blob_kzg_commitments, blob_kzg_proofs = get_sample_blob_tx(spec, blob_count=1, rng=rng) block.body.execution_payload.transactions = [opaque_tx] block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload, state) block.body.blob_kzg_commitments = blob_kzg_commitments diff --git a/tests/core/pyspec/eth2spec/test/deneb/merkle_proof/test_single_merkle_proof.py b/tests/core/pyspec/eth2spec/test/deneb/merkle_proof/test_single_merkle_proof.py index 1dda310123..6deccd31e2 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/merkle_proof/test_single_merkle_proof.py +++ b/tests/core/pyspec/eth2spec/test/deneb/merkle_proof/test_single_merkle_proof.py @@ -12,8 +12,8 @@ from eth2spec.test.helpers.execution_payload import ( compute_el_block_hash, ) -from eth2spec.test.helpers.sharding import ( - get_sample_opaque_tx, +from eth2spec.test.helpers.blob import ( + get_sample_blob_tx, ) from eth2spec.debug.random_value import ( RandomizationMode, @@ -22,7 +22,7 @@ def _run_blob_kzg_commitment_merkle_proof_test(spec, state, rng=None): - opaque_tx, blobs, blob_kzg_commitments, proofs = get_sample_opaque_tx(spec, blob_count=1) + opaque_tx, blobs, blob_kzg_commitments, proofs = get_sample_blob_tx(spec, blob_count=1) if rng is None: block = build_empty_block_for_next_slot(spec, state) else: diff --git a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py index 94db2c34fc..b019175369 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py @@ -14,8 +14,8 @@ compute_el_block_hash, get_random_tx, ) -from eth2spec.test.helpers.sharding import ( - get_sample_opaque_tx, +from eth2spec.test.helpers.blob import ( + get_sample_blob_tx, ) @@ -27,7 +27,7 @@ def run_block_with_blobs(spec, state, blob_count, tx_count=1, blob_gas_used=1, e txs = [] blob_kzg_commitments = [] for _ in range(tx_count): - opaque_tx, _, commits, _ = get_sample_opaque_tx(spec, blob_count=blob_count) + opaque_tx, _, commits, _ = get_sample_blob_tx(spec, blob_count=blob_count) txs.append(opaque_tx) blob_kzg_commitments += commits diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/polynomial_commitments/test_polynomial_commitments.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/polynomial_commitments/test_polynomial_commitments.py index 1d43d07caf..964fe16608 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/unittests/polynomial_commitments/test_polynomial_commitments.py +++ b/tests/core/pyspec/eth2spec/test/deneb/unittests/polynomial_commitments/test_polynomial_commitments.py @@ -7,7 +7,7 @@ expect_assertion_error, always_bls ) -from eth2spec.test.helpers.sharding import ( +from eth2spec.test.helpers.blob import ( get_sample_blob, get_poly_in_both_forms, eval_poly_in_coeff_form, diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py index 6724e8304a..0e02ab8c3c 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py +++ b/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py @@ -6,8 +6,8 @@ from eth2spec.test.helpers.execution_payload import ( compute_el_block_hash, ) -from eth2spec.test.helpers.sharding import ( - get_sample_opaque_tx, +from eth2spec.test.helpers.blob import ( + get_sample_blob_tx, ) from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, @@ -20,8 +20,8 @@ def _get_sample_sidecars(spec, state, rng): # 2 txs, each has 2 blobs blob_count = 2 - opaque_tx_1, blobs_1, blob_kzg_commitments_1, proofs_1 = get_sample_opaque_tx(spec, blob_count=blob_count, rng=rng) - opaque_tx_2, blobs_2, blob_kzg_commitments_2, proofs_2 = get_sample_opaque_tx(spec, blob_count=blob_count, rng=rng) + opaque_tx_1, blobs_1, blob_kzg_commitments_1, proofs_1 = get_sample_blob_tx(spec, blob_count=blob_count, rng=rng) + opaque_tx_2, blobs_2, blob_kzg_commitments_2, proofs_2 = get_sample_blob_tx(spec, blob_count=blob_count, rng=rng) assert opaque_tx_1 != opaque_tx_2 block.body.blob_kzg_commitments = blob_kzg_commitments_1 + blob_kzg_commitments_2 diff --git a/tests/core/pyspec/eth2spec/test/eip7594/merkle_proof/test_single_merkle_proof.py b/tests/core/pyspec/eth2spec/test/eip7594/merkle_proof/test_single_merkle_proof.py index 98c751508d..4bd5fb4912 100644 --- a/tests/core/pyspec/eth2spec/test/eip7594/merkle_proof/test_single_merkle_proof.py +++ b/tests/core/pyspec/eth2spec/test/eip7594/merkle_proof/test_single_merkle_proof.py @@ -12,8 +12,8 @@ from eth2spec.test.helpers.execution_payload import ( compute_el_block_hash, ) -from eth2spec.test.helpers.sharding import ( - get_sample_opaque_tx, +from eth2spec.test.helpers.blob import ( + get_sample_blob_tx, ) from eth2spec.debug.random_value import ( RandomizationMode, @@ -22,7 +22,7 @@ def _run_blob_kzg_commitments_merkle_proof_test(spec, state, rng=None): - opaque_tx, blobs, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=1) + opaque_tx, blobs, blob_kzg_commitments, _ = get_sample_blob_tx(spec, blob_count=1) if rng is None: block = build_empty_block_for_next_slot(spec, state) else: diff --git a/tests/core/pyspec/eth2spec/test/eip7594/unittests/das/test_das.py b/tests/core/pyspec/eth2spec/test/eip7594/unittests/das/test_das.py index 625136b73e..dd0a80fda4 100644 --- a/tests/core/pyspec/eth2spec/test/eip7594/unittests/das/test_das.py +++ b/tests/core/pyspec/eth2spec/test/eip7594/unittests/das/test_das.py @@ -6,7 +6,7 @@ with_config_overrides, with_eip7594_and_later, ) -from eth2spec.test.helpers.sharding import ( +from eth2spec.test.helpers.blob import ( get_sample_blob, ) diff --git a/tests/core/pyspec/eth2spec/test/eip7594/unittests/polynomial_commitments/test_polynomial_commitments.py b/tests/core/pyspec/eth2spec/test/eip7594/unittests/polynomial_commitments/test_polynomial_commitments.py index c68c606765..4baf9fc83f 100644 --- a/tests/core/pyspec/eth2spec/test/eip7594/unittests/polynomial_commitments/test_polynomial_commitments.py +++ b/tests/core/pyspec/eth2spec/test/eip7594/unittests/polynomial_commitments/test_polynomial_commitments.py @@ -5,7 +5,7 @@ expect_assertion_error, with_eip7594_and_later, ) -from eth2spec.test.helpers.sharding import ( +from eth2spec.test.helpers.blob import ( get_sample_blob, ) from eth2spec.utils.bls import BLS_MODULUS diff --git a/tests/core/pyspec/eth2spec/test/helpers/sharding.py b/tests/core/pyspec/eth2spec/test/helpers/blob.py similarity index 57% rename from tests/core/pyspec/eth2spec/test/helpers/sharding.py rename to tests/core/pyspec/eth2spec/test/helpers/blob.py index c06b1aa9cb..e8117ece6c 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/sharding.py +++ b/tests/core/pyspec/eth2spec/test/helpers/blob.py @@ -1,57 +1,28 @@ import random -from eth2spec.utils.ssz.ssz_typing import ( - Container, - Bytes20, Bytes32, - ByteList, - List, - Union, - boolean, - uint256, uint64, - uint8, -) -from eth2spec.utils.ssz.ssz_impl import serialize - - -# -# Containers from EIP-4844 -# -MAX_CALLDATA_SIZE = 2**24 -MAX_VERSIONED_HASHES_LIST_SIZE = 2**24 -MAX_ACCESS_LIST_STORAGE_KEYS = 2**24 -MAX_ACCESS_LIST_SIZE = 2**24 - - -BLOB_TX_TYPE = uint8(0x03) - - -class AccessTuple(Container): - address: Bytes20 # Address = Bytes20 - storage_keys: List[Bytes32, MAX_ACCESS_LIST_STORAGE_KEYS] - - -class ECDSASignature(Container): - y_parity: boolean - r: uint256 - s: uint256 - - -class BlobTransaction(Container): - chain_id: uint256 - nonce: uint64 - max_priority_fee_per_gas: uint256 - max_fee_per_gas: uint256 - gas: uint64 - to: Union[None, Bytes20] # Address = Bytes20 - value: uint256 - data: ByteList[MAX_CALLDATA_SIZE] - access_list: List[AccessTuple, MAX_ACCESS_LIST_SIZE] - max_fee_per_blob_gas: uint256 - blob_versioned_hashes: List[Bytes32, MAX_VERSIONED_HASHES_LIST_SIZE] - - -class SignedBlobTransaction(Container): - message: BlobTransaction - signature: ECDSASignature +from rlp import encode, Serializable +from rlp.sedes import Binary, CountableList, List as RLPList, big_endian_int, binary + + +class Eip4844RlpTransaction(Serializable): + fields = ( + ('chain_id', big_endian_int), + ('nonce', big_endian_int), + ('max_priority_fee_per_gas', big_endian_int), + ('max_fee_per_gas', big_endian_int), + ('gas_limit', big_endian_int), + ('to', Binary(20, 20)), + ('value', big_endian_int), + ('data', binary), + ('access_list', CountableList(RLPList([ + Binary(20, 20), + CountableList(Binary(32, 32)), + ]))), + ('max_fee_per_blob_gas', big_endian_int), + ('blob_versioned_hashes', CountableList(Binary(32, 32))), + ('signature_y_parity', big_endian_int), + ('signature_r', big_endian_int), + ('signature_s', big_endian_int), + ) def get_sample_blob(spec, rng=random.Random(5566), is_valid_blob=True): @@ -99,7 +70,7 @@ def get_poly_in_both_forms(spec, rng=None): return coeffs, evals -def get_sample_opaque_tx(spec, blob_count=1, rng=random.Random(5566), is_valid_blob=True): +def get_sample_blob_tx(spec, blob_count=1, rng=random.Random(5566), is_valid_blob=True): blobs = [] blob_kzg_commitments = [] blob_kzg_proofs = [] @@ -118,11 +89,21 @@ def get_sample_opaque_tx(spec, blob_count=1, rng=random.Random(5566), is_valid_b blob_kzg_proofs.append(blob_kzg_proof) blob_versioned_hashes.append(blob_versioned_hash) - signed_blob_tx = SignedBlobTransaction( - message=BlobTransaction( - blob_versioned_hashes=blob_versioned_hashes, - ) + signed_blob_tx = Eip4844RlpTransaction( + chain_id=0, + nonce=0, + max_priority_fee_per_gas=0, + max_fee_per_gas=0, + gas_limit=0, + to=bytes.fromhex("0000000000000000000000000000000000000000"), + value=0, + data=bytes.fromhex(""), + access_list=[], + max_fee_per_blob_gas=0, + blob_versioned_hashes=[bytes(h) for h in blob_versioned_hashes], + signature_y_parity=0, + signature_r=0, + signature_s=0, ) - serialized_tx = serialize(signed_blob_tx) - opaque_tx = spec.uint_to_bytes(BLOB_TX_TYPE) + serialized_tx + opaque_tx = bytes([0x03]) + encode(signed_blob_tx) return opaque_tx, blobs, blob_kzg_commitments, blob_kzg_proofs diff --git a/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py b/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py index 78802a6ddd..6e6cdd7683 100644 --- a/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py +++ b/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py @@ -24,8 +24,8 @@ randomize_state as randomize_state_helper, patch_state_to_non_leaking, ) -from eth2spec.test.helpers.sharding import ( - get_sample_opaque_tx, +from eth2spec.test.helpers.blob import ( + get_sample_blob_tx, ) from eth2spec.test.helpers.state import ( next_slot, @@ -250,7 +250,7 @@ def random_block_capella(spec, state, signed_blocks, scenario_state, rng=Random( def random_block_deneb(spec, state, signed_blocks, scenario_state, rng=Random(3456)): block = random_block_capella(spec, state, signed_blocks, scenario_state, rng=rng) # TODO: more commitments. blob_kzg_commitments: List[KZGCommitment, MAX_BLOBS_PER_BLOCK] - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx( + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx( spec, blob_count=rng.randint(0, spec.config.MAX_BLOBS_PER_BLOCK), rng=rng) block.body.execution_payload.transactions.append(opaque_tx) block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload, state) From 1c529a858c13cccae60b69b3f35c14b02b214c69 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Fri, 20 Sep 2024 09:25:35 +0200 Subject: [PATCH 017/142] use SingleAttestation in honest validator spec --- specs/electra/validator.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/specs/electra/validator.md b/specs/electra/validator.md index f589e963c5..c27f02b9d1 100644 --- a/specs/electra/validator.md +++ b/specs/electra/validator.md @@ -152,11 +152,12 @@ def prepare_execution_payload(state: BeaconState, ### Construct attestation -- Set `attestation_data.index = 0`. -- Let `attestation.aggregation_bits` be a `Bitlist[MAX_VALIDATORS_PER_COMMITTEE * MAX_COMMITTEES_PER_SLOT]` of length `len(committee)`, where the bit of the index of the validator in the `committee` is set to `0b1`. -- Let `attestation.committee_bits` be a `Bitvector[MAX_COMMITTEES_PER_SLOT]`, where the bit at the index associated with the validator's committee is set to `0b1`. +The validator creates `attestation` as a `SingleAttestation` container +with updated field assignments: -*Note*: Calling `get_attesting_indices(state, attestation)` should return a list of length equal to 1, containing `validator_index`. +- Set `attestation_data.index = 0`. +- Set `attestation.committee_index` to the index associated with the validator's committee. +- Set `attestation.attester_index` to the index of the validator. ## Attestation aggregation @@ -164,4 +165,4 @@ def prepare_execution_payload(state: BeaconState, - Set `attestation_data.index = 0`. - Let `aggregation_bits` be a `Bitlist[MAX_VALIDATORS_PER_COMMITTEE * MAX_COMMITTEES_PER_SLOT]` of length `len(committee)`, where each bit set from each individual attestation is set to `0b1`. -- Set `attestation.committee_bits = committee_bits`, where `committee_bits` has the same value as in each individual attestation. +- Set `attestation.committee_bits = committee_bits`, where `committee_bits` has the bit set corresponding to `committee_index` in each individual attestation. From 768fb454b1236c26ddee472501a8746bbc8b6558 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Fri, 20 Sep 2024 09:30:55 +0200 Subject: [PATCH 018/142] merge cleanup --- specs/electra/beacon-chain.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/electra/beacon-chain.md b/specs/electra/beacon-chain.md index 84d6b2fce3..2029c70843 100644 --- a/specs/electra/beacon-chain.md +++ b/specs/electra/beacon-chain.md @@ -30,6 +30,7 @@ - [`ConsolidationRequest`](#consolidationrequest) - [`PendingConsolidation`](#pendingconsolidation) - [`SingleAttestation`](#singleattestation) + - [`ExecutionRequests`](#executionrequests) - [Modified Containers](#modified-containers) - [`AttesterSlashing`](#attesterslashing) - [`BeaconBlockBody`](#beaconblockbody) @@ -257,7 +258,6 @@ class PendingConsolidation(Container): target_index: ValidatorIndex ``` - #### `SingleAttestation` ```python @@ -266,6 +266,7 @@ class SingleAttestation(Container): attester_index: ValidatorIndex data: AttestationData signature: BLSSignature +``` #### `ExecutionRequests` From 223d7346f9f592c12b5f02bcf5b55d919dcb397e Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Mon, 23 Sep 2024 14:23:38 -0500 Subject: [PATCH 019/142] Move deposit contract rules to its own Makefile --- Makefile | 33 +-------------- solidity_deposit_contract/Makefile | 41 +++++++++++++++++++ .../web3_tester/requirements.txt | 6 +-- .../web3_tester/tests/conftest.py | 2 +- .../web3_tester/tests/test_deposit.py | 8 ++-- 5 files changed, 50 insertions(+), 40 deletions(-) create mode 100644 solidity_deposit_contract/Makefile diff --git a/Makefile b/Makefile index c805b456b0..b1e8b2fd62 100644 --- a/Makefile +++ b/Makefile @@ -8,10 +8,6 @@ ETH2SPEC_MODULE_DIR = $(PY_SPEC_DIR)/eth2spec TEST_REPORT_DIR = $(PY_SPEC_DIR)/test-reports TEST_VECTOR_DIR = ../consensus-spec-tests/tests GENERATOR_DIR = ./tests/generators -SOLIDITY_DEPOSIT_CONTRACT_DIR = ./solidity_deposit_contract -SOLIDITY_DEPOSIT_CONTRACT_SOURCE = ${SOLIDITY_DEPOSIT_CONTRACT_DIR}/deposit_contract.sol -SOLIDITY_FILE_NAME = deposit_contract.json -DEPOSIT_CONTRACT_TESTER_DIR = ${SOLIDITY_DEPOSIT_CONTRACT_DIR}/web3_tester CONFIGS_DIR = ./configs TEST_PRESET_TYPE ?= minimal # Collect a list of generator names @@ -50,15 +46,8 @@ GENERATOR_ERROR_LOG_FILE = $(CURRENT_DIR)/$(TEST_VECTOR_DIR)/testgen_error_log.t SCRIPTS_DIR = ${CURRENT_DIR}/scripts -export DAPP_SKIP_BUILD:=1 -export DAPP_SRC:=$(SOLIDITY_DEPOSIT_CONTRACT_DIR) -export DAPP_LIB:=$(SOLIDITY_DEPOSIT_CONTRACT_DIR)/lib -export DAPP_JSON:=build/combined.json - .PHONY: clean partial_clean all test citest lint generate_tests pyspec install_test open_cov \ - install_deposit_contract_tester test_deposit_contract install_deposit_contract_compiler \ - compile_deposit_contract test_compile_deposit_contract check_toc \ - detect_generator_incomplete detect_generator_error_log + check_toc detect_generator_incomplete detect_generator_error_log all: $(PY_SPEC_ALL_TARGETS) @@ -162,26 +151,6 @@ lint_generators: pyspec . venv/bin/activate; cd $(TEST_GENERATORS_DIR); \ flake8 --config $(LINTER_CONFIG_FILE) -compile_deposit_contract: - @cd $(SOLIDITY_DEPOSIT_CONTRACT_DIR) - @git submodule update --recursive --init - @solc --metadata-literal --optimize --optimize-runs 5000000 --bin --abi --combined-json=abi,bin,bin-runtime,srcmap,srcmap-runtime,ast,metadata,storage-layout --overwrite -o build $(SOLIDITY_DEPOSIT_CONTRACT_SOURCE) $(SOLIDITY_DEPOSIT_CONTRACT_DIR)/tests/deposit_contract.t.sol - @/bin/echo -n '{"abi": ' > $(SOLIDITY_FILE_NAME) - @cat build/DepositContract.abi >> $(SOLIDITY_FILE_NAME) - @/bin/echo -n ', "bytecode": "0x' >> $(SOLIDITY_FILE_NAME) - @cat build/DepositContract.bin >> $(SOLIDITY_FILE_NAME) - @/bin/echo -n '"}' >> $(SOLIDITY_FILE_NAME) - -test_deposit_contract: - dapp test -v --fuzz-runs 5 - -install_deposit_contract_web3_tester: - cd $(DEPOSIT_CONTRACT_TESTER_DIR); python3 -m venv venv; . venv/bin/activate; python3 -m pip install -r requirements.txt - -test_deposit_contract_web3_tests: - cd $(DEPOSIT_CONTRACT_TESTER_DIR); . venv/bin/activate; \ - python3 -m pytest . - # Runs a generator, identified by param 1 define run_generator # Started! diff --git a/solidity_deposit_contract/Makefile b/solidity_deposit_contract/Makefile new file mode 100644 index 0000000000..164df20a99 --- /dev/null +++ b/solidity_deposit_contract/Makefile @@ -0,0 +1,41 @@ +SOLIDITY_FILE_NAME = deposit_contract.json +SOLIDITY_DEPOSIT_CONTRACT_SOURCE = deposit_contract.sol +DEPOSIT_CONTRACT_TESTER_DIR = web3_tester + +export DAPP_SKIP_BUILD:=1 +export DAPP_SRC:=$(CURDIR) +export DAPP_LIB:=$(CURDIR)/lib +export DAPP_JSON:=build/combined.json + +all: \ + compile_deposit_contract \ + install_deposit_contract_web3_tester \ + test_deposit_contract_web3_tests + +compile_deposit_contract: + @git submodule update --recursive --init + @solc --metadata-literal --optimize --optimize-runs 5000000 --bin --abi \ + --combined-json=abi,bin,bin-runtime,srcmap,srcmap-runtime,ast,metadata,storage-layout \ + --overwrite -o build $(SOLIDITY_DEPOSIT_CONTRACT_SOURCE) tests/deposit_contract.t.sol + @/bin/echo -n '{"abi": ' > $(SOLIDITY_FILE_NAME) + @cat build/DepositContract.abi >> $(SOLIDITY_FILE_NAME) + @/bin/echo -n ', "bytecode": "0x' >> $(SOLIDITY_FILE_NAME) + @cat build/DepositContract.bin >> $(SOLIDITY_FILE_NAME) + @/bin/echo -n '"}' >> $(SOLIDITY_FILE_NAME) + +test_deposit_contract: + @dapp test -v --fuzz-runs 5 + +install_deposit_contract_web3_tester: + @cd $(DEPOSIT_CONTRACT_TESTER_DIR); \ + python3 -m venv venv; \ + source venv/bin/activate; \ + python3 -m pip install -r requirements.txt + +test_deposit_contract_web3_tests: + @cd $(DEPOSIT_CONTRACT_TESTER_DIR); \ + source venv/bin/activate; \ + python3 -m pytest . + +clean: + @git clean -fdx \ No newline at end of file diff --git a/solidity_deposit_contract/web3_tester/requirements.txt b/solidity_deposit_contract/web3_tester/requirements.txt index 8dadab0bc2..92c8298bfb 100644 --- a/solidity_deposit_contract/web3_tester/requirements.txt +++ b/solidity_deposit_contract/web3_tester/requirements.txt @@ -1,5 +1,5 @@ -eth-tester[py-evm]>=0.3.0b1,<0.4 -web3==5.4.0 -pytest==3.6.1 +eth-tester[py-evm]>=0.12.0b1 +web3>=6.11.0 +pytest>=4.4 # The eth2spec ../../ diff --git a/solidity_deposit_contract/web3_tester/tests/conftest.py b/solidity_deposit_contract/web3_tester/tests/conftest.py index 57b5f6b7a7..31686eabf3 100644 --- a/solidity_deposit_contract/web3_tester/tests/conftest.py +++ b/solidity_deposit_contract/web3_tester/tests/conftest.py @@ -51,7 +51,7 @@ def registration_contract(w3, tester): abi=contract_abi, bytecode=contract_bytecode) tx_hash = registration.constructor().transact() - tx_receipt = w3.eth.waitForTransactionReceipt(tx_hash) + tx_receipt = w3.eth.wait_for_transaction_receipt(tx_hash) registration_deployed = w3.eth.contract( address=tx_receipt.contractAddress, abi=contract_abi diff --git a/solidity_deposit_contract/web3_tester/tests/test_deposit.py b/solidity_deposit_contract/web3_tester/tests/test_deposit.py index 4b16446a16..e4b794ffbf 100644 --- a/solidity_deposit_contract/web3_tester/tests/test_deposit.py +++ b/solidity_deposit_contract/web3_tester/tests/test_deposit.py @@ -2,7 +2,7 @@ import pytest import eth_utils -from eth2spec.phase0.spec import DepositData +from eth2spec.phase0.mainnet import DepositData from eth2spec.utils.ssz.ssz_typing import List from eth2spec.utils.ssz.ssz_impl import hash_tree_root @@ -119,7 +119,7 @@ def test_deposit_inputs(registration_contract, def test_deposit_event_log(registration_contract, a0, w3): - log_filter = registration_contract.events.DepositEvent.createFilter( + log_filter = registration_contract.events.DepositEvent.create_filter( fromBlock='latest', ) deposit_amount_list = [randint(MIN_DEPOSIT_AMOUNT, FULL_DEPOSIT_AMOUNT * 2) for _ in range(3)] @@ -154,7 +154,7 @@ def test_deposit_event_log(registration_contract, a0, w3): def test_deposit_tree(registration_contract, w3, assert_tx_failed): - log_filter = registration_contract.events.DepositEvent.createFilter( + log_filter = registration_contract.events.DepositEvent.create_filter( fromBlock='latest', ) @@ -178,7 +178,7 @@ def test_deposit_tree(registration_contract, w3, assert_tx_failed): tx_hash = registration_contract.functions.deposit( *deposit_input, ).transact({"value": deposit_amount_list[i] * eth_utils.denoms.gwei}) - receipt = w3.eth.getTransactionReceipt(tx_hash) + receipt = w3.eth.get_transaction_receipt(tx_hash) print("deposit transaction consumes %d gas" % receipt['gasUsed']) logs = log_filter.get_new_entries() From 01793e5e4391c01717d0da8d58db3ed62d9987dc Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Mon, 23 Sep 2024 14:28:21 -0500 Subject: [PATCH 020/142] Update README and CI workflow --- .circleci/config.yml | 92 ----------------------------- solidity_deposit_contract/README.md | 6 +- 2 files changed, 3 insertions(+), 95 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7ac281dea9..17db49b3a9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,19 +44,6 @@ commands: venv_name: v25-pyspec reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "requirements_preinstallation.txt" }} venv_path: ./venv - restore_deposit_contract_tester_cached_venv: - description: "Restore the venv from cache for the deposit contract tester" - steps: - - restore_cached_venv: - venv_name: v23-deposit-contract-tester - reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "requirements_preinstallation.txt" }}-{{ checksum "solidity_deposit_contract/web3_tester/requirements.txt" }} - save_deposit_contract_tester_cached_venv: - description: "Save the venv to cache for later use of the deposit contract tester" - steps: - - save_cached_venv: - venv_name: v23-deposit-contract-tester - reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "requirements_preinstallation.txt" }}-{{ checksum "solidity_deposit_contract/web3_tester/requirements.txt" }} - venv_path: ./solidity_deposit_contract/web3_tester/venv jobs: checkout_specs: docker: @@ -226,71 +213,6 @@ jobs: - run: name: Run linter for test generators command: make lint_generators - build_deposit_contract: - docker: - - image: ethereum/solc:0.6.11-alpine - steps: - - checkout - - run: - name: Install build essentials - command: | - apk update - apk add git make - - run: - name: Compile the contract - command: | - make compile_deposit_contract - git diff --color --exit-code - - persist_to_workspace: - root: . - paths: - - ./solidity_deposit_contract/deposit_contract.json - - ./build/combined.json - - ./solidity_deposit_contract/lib - test_deposit_contract: - docker: - - image: nixorg/nix:circleci - steps: - - checkout - - restore_cache: - key: nix-store-test-v2 - - attach_workspace: - at: /tmp/ - - run: - name: Test the contract - command: | - mkdir build - cp -r /tmp/build/* build - cp -r /tmp/solidity_deposit_contract/lib/* ./solidity_deposit_contract/lib - cp -r /tmp/solidity_deposit_contract/deposit_contract.json ./solidity_deposit_contract/deposit_contract.json - nix-shell --command 'make test_deposit_contract' ./solidity_deposit_contract/shell.nix - - save_cache: - key: nix-store-test-v2 - paths: - - /nix - install_deposit_contract_web3_tester: - docker: - - image: cimg/python:3.12.4 - working_directory: ~/specs-repo - steps: - - restore_cache: - key: v3-specs-repo-{{ .Branch }}-{{ .Revision }} - - restore_deposit_contract_tester_cached_venv - - run: - name: Install deposit contract tester requirements - command: make install_deposit_contract_web3_tester - - save_deposit_contract_tester_cached_venv - test_deposit_contract_web3_tests: - docker: - - image: cimg/python:3.12.4 - working_directory: ~/specs-repo - steps: - - restore_cache: - key: v3-specs-repo-{{ .Branch }}-{{ .Revision }} - - restore_deposit_contract_tester_cached_venv - - run: - name: Run deposit contract test with web3.py - command: make test_deposit_contract_web3_tests workflows: version: 2.1 test_spec: @@ -328,17 +250,3 @@ workflows: - lint: requires: - install_pyspec_test - # NOTE: Since phase 0 has been launched, we disabled the deposit contract tests. - # - install_deposit_contract_web3_tester: - # requires: - # - checkout_specs - # - test_deposit_contract_web3_tests: - # requires: - # - install_deposit_contract_web3_tester - build_and_test_deposit_contract: - jobs: - - build_deposit_contract - # NOTE: Since phase 0 has been launched, we disabled the deposit contract tests. - # - test_deposit_contract: - # requires: - # - build_deposit_contract diff --git a/solidity_deposit_contract/README.md b/solidity_deposit_contract/README.md index 0388d7d2f5..298ea92fef 100644 --- a/solidity_deposit_contract/README.md +++ b/solidity_deposit_contract/README.md @@ -13,7 +13,7 @@ In August 2020, version `r2` was released with metadata modifications and relice ## Compiling solidity deposit contract -In the `eth2.0-specs` directory run: +In this directory run: ```sh make compile_deposit_contract ``` @@ -31,8 +31,8 @@ solc --optimize --optimize-runs 5000000 --metadata-literal --bin deposit_contrac ## Running web3 tests -1. In the `eth2.0-specs` directory run `make install_deposit_contract_web3_tester` to install the tools needed (make sure to have Python 3.7 and pip installed). -2. In the `eth2.0-specs` directory run `make test_deposit_contract_web3_tests` to execute the tests. +1. In this directory run `make install_deposit_contract_web3_tester` to install the tools needed (make sure to have Python 3 and pip installed). +2. In this directory run `make test_deposit_contract_web3_tests` to execute the tests. ## Running randomized `dapp` tests: From f1d23ebb8520b0d60d07af2e1ed80e0480c11131 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Wed, 2 Oct 2024 12:14:50 +0200 Subject: [PATCH 021/142] update --- specs/deneb/p2p-interface.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index da95e5786c..394dab31bc 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -296,7 +296,7 @@ Before consuming the next response chunk, the response reader SHOULD verify the No more than `MAX_REQUEST_BLOB_SIDECARS` may be requested at a time. -`BlobSidecarsByRoot` is primarily used to recover recent blobs (e.g. when receiving a block with a transaction whose corresponding blob is missing). In addition, it is possible to retrieve recent blobs by querying the Execution Layer via `engine_getBlobsV1` engine API method. +`BlobSidecarsByRoot` is primarily used to recover recent blobs (e.g. when receiving a block with a transaction whose corresponding blob is missing). The response MUST consist of zero or more `response_chunk`. Each _successful_ `response_chunk` MUST contain a single `BlobSidecar` payload. @@ -310,6 +310,14 @@ Clients SHOULD include a sidecar in the response as soon as it passes the gossip Clients SHOULD NOT respond with sidecars related to blocks that fail gossip validation rules. Clients SHOULD NOT respond with sidecars related to blocks that fail the beacon chain state transition +###### Usage of `engine_getBlobsV1` engine API method +In addition to `BlobSidecarsByRoot` requests, recent blobs recovery MAY also be done by querying the Execution Layer via `engine_getBlobsV1`. +Implementers are encouraged to leverege this method to encrease the likelyhood of incorporating and attesting to the last block when its proposer is not able to publish blobs on time. + +Client MAY use this method when serving `BlobSidecarsByRoot` requests. + +Client MUST publish the corresponding `blob_sidecar` whenever succesfully recovers blobs via `engine_getBlobsV1`. + ##### BlobSidecarsByRange v1 **Protocol ID:** `/eth2/beacon_chain/req/blob_sidecars_by_range/1/` From 37d6d37ce4346474bdfecf7bbd54a637db5ac6cd Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Wed, 2 Oct 2024 12:17:09 +0200 Subject: [PATCH 022/142] fix typos --- specs/deneb/p2p-interface.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 394dab31bc..0189643de2 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -312,11 +312,11 @@ Clients SHOULD NOT respond with sidecars related to blocks that fail the beacon ###### Usage of `engine_getBlobsV1` engine API method In addition to `BlobSidecarsByRoot` requests, recent blobs recovery MAY also be done by querying the Execution Layer via `engine_getBlobsV1`. -Implementers are encouraged to leverege this method to encrease the likelyhood of incorporating and attesting to the last block when its proposer is not able to publish blobs on time. +Implementers are encouraged to leverege this method to encrease the likelihood of incorporating and attesting to the last block when its proposer is not able to publish blobs on time. Client MAY use this method when serving `BlobSidecarsByRoot` requests. -Client MUST publish the corresponding `blob_sidecar` whenever succesfully recovers blobs via `engine_getBlobsV1`. +Client MUST publish the corresponding `blob_sidecar` whenever successfully recovers blobs via `engine_getBlobsV1`. ##### BlobSidecarsByRange v1 From 2acf362f81fad2664d22fa42f9b17033c5d7499f Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Wed, 2 Oct 2024 12:22:48 +0200 Subject: [PATCH 023/142] fix toc --- specs/deneb/p2p-interface.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 0189643de2..44471d9e6b 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -34,6 +34,7 @@ The specification of these changes continues in the same format as the network s - [BeaconBlocksByRange v2](#beaconblocksbyrange-v2) - [BeaconBlocksByRoot v2](#beaconblocksbyroot-v2) - [BlobSidecarsByRoot v1](#blobsidecarsbyroot-v1) + - [Usage of `engine_getBlobsV1` engine API method](#usage-of-engine_getblobsv1-engine-api-method) - [BlobSidecarsByRange v1](#blobsidecarsbyrange-v1) - [Design decision rationale](#design-decision-rationale) - [Why are blobs relayed as a sidecar, separate from beacon blocks?](#why-are-blobs-relayed-as-a-sidecar-separate-from-beacon-blocks) From 918a394ce1679e63bbe82332142b89441e4c3914 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Wed, 2 Oct 2024 12:28:03 +0200 Subject: [PATCH 024/142] remove usage in RPC serving sentence --- specs/deneb/p2p-interface.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 44471d9e6b..3cd85bf09e 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -315,8 +315,6 @@ Clients SHOULD NOT respond with sidecars related to blocks that fail the beacon In addition to `BlobSidecarsByRoot` requests, recent blobs recovery MAY also be done by querying the Execution Layer via `engine_getBlobsV1`. Implementers are encouraged to leverege this method to encrease the likelihood of incorporating and attesting to the last block when its proposer is not able to publish blobs on time. -Client MAY use this method when serving `BlobSidecarsByRoot` requests. - Client MUST publish the corresponding `blob_sidecar` whenever successfully recovers blobs via `engine_getBlobsV1`. ##### BlobSidecarsByRange v1 From 63ad07b450d304b3972042b8255e552b7dd9ca0d Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Wed, 2 Oct 2024 12:29:44 +0200 Subject: [PATCH 025/142] typos --- specs/deneb/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 3cd85bf09e..f9b5681998 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -313,7 +313,7 @@ Clients SHOULD NOT respond with sidecars related to blocks that fail the beacon ###### Usage of `engine_getBlobsV1` engine API method In addition to `BlobSidecarsByRoot` requests, recent blobs recovery MAY also be done by querying the Execution Layer via `engine_getBlobsV1`. -Implementers are encouraged to leverege this method to encrease the likelihood of incorporating and attesting to the last block when its proposer is not able to publish blobs on time. +Implementers are encouraged to leverage this method to increase the likelihood of incorporating and attesting to the last block when its proposer is not able to publish blobs on time. Client MUST publish the corresponding `blob_sidecar` whenever successfully recovers blobs via `engine_getBlobsV1`. From 4531fe629c2bcc5f9e06604a90492c1573dce6e9 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Wed, 2 Oct 2024 12:33:42 +0200 Subject: [PATCH 026/142] remove explicit reference to engine method --- specs/deneb/p2p-interface.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index f9b5681998..0c4658f0f2 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -34,7 +34,7 @@ The specification of these changes continues in the same format as the network s - [BeaconBlocksByRange v2](#beaconblocksbyrange-v2) - [BeaconBlocksByRoot v2](#beaconblocksbyroot-v2) - [BlobSidecarsByRoot v1](#blobsidecarsbyroot-v1) - - [Usage of `engine_getBlobsV1` engine API method](#usage-of-engine_getblobsv1-engine-api-method) + - [Blobs recovery via local execution layer client](#blobs-recovery-via-local-execution-layer-client) - [BlobSidecarsByRange v1](#blobsidecarsbyrange-v1) - [Design decision rationale](#design-decision-rationale) - [Why are blobs relayed as a sidecar, separate from beacon blocks?](#why-are-blobs-relayed-as-a-sidecar-separate-from-beacon-blocks) @@ -311,11 +311,11 @@ Clients SHOULD include a sidecar in the response as soon as it passes the gossip Clients SHOULD NOT respond with sidecars related to blocks that fail gossip validation rules. Clients SHOULD NOT respond with sidecars related to blocks that fail the beacon chain state transition -###### Usage of `engine_getBlobsV1` engine API method -In addition to `BlobSidecarsByRoot` requests, recent blobs recovery MAY also be done by querying the Execution Layer via `engine_getBlobsV1`. +###### Blobs recovery via local execution layer client +In addition to `BlobSidecarsByRoot` requests, recent blobs recovery MAY also be done by querying the Execution Layer (i.e. via `engine_getBlobsV1`) Implementers are encouraged to leverage this method to increase the likelihood of incorporating and attesting to the last block when its proposer is not able to publish blobs on time. -Client MUST publish the corresponding `blob_sidecar` whenever successfully recovers blobs via `engine_getBlobsV1`. +Client MUST publish the corresponding `blob_sidecar` whenever successfully recovers blobs via local execution layer client. ##### BlobSidecarsByRange v1 From 97db65eef3d0bec0620d19f552453fcb5e73463a Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Wed, 2 Oct 2024 12:37:17 +0200 Subject: [PATCH 027/142] remove another reference --- specs/deneb/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 0c4658f0f2..3992828e3b 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -185,7 +185,7 @@ The following validations MUST pass before forwarding the `blob_sidecar` on the - _[REJECT]_ The sidecar's inclusion proof is valid as verified by `verify_blob_sidecar_inclusion_proof(blob_sidecar)`. - _[REJECT]_ The sidecar's blob is valid as verified by `verify_blob_kzg_proof(blob_sidecar.blob, blob_sidecar.kzg_commitment, blob_sidecar.kzg_proof)`. - _[IGNORE]_ The sidecar is the first sidecar for the tuple `(block_header.slot, block_header.proposer_index, blob_sidecar.index)` with valid header signature, sidecar inclusion proof, and kzg proof. -*Note:* If client obtains the blob via `engine_getBlobsV1` or any other method before receiving the corresponding sidecar via this topic, it MUST NOT consider the tuple as "seen" (i.e. it MUST participate in the dissemination of the blob_sidecar even if it already obtained the blob) +*Note:* If client obtains the blob via local execution layer client or any other method before receiving the corresponding sidecar via this topic, it MUST NOT consider the tuple as "seen" (i.e. it MUST participate in the dissemination of the blob_sidecar even if it already obtained the blob) - _[REJECT]_ The sidecar is proposed by the expected `proposer_index` for the block's slot in the context of the current shuffling (defined by `block_header.parent_root`/`block_header.slot`). If the `proposer_index` cannot immediately be verified against the expected shuffling, the sidecar MAY be queued for later processing while proposers for the block's branch are calculated -- in such a case _do not_ `REJECT`, instead `IGNORE` this message. From 911019c41f35aee666b07a558a46bbc033951f20 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Wed, 2 Oct 2024 13:33:44 +0200 Subject: [PATCH 028/142] constant for concurrent requests --- specs/phase0/p2p-interface.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 269981a807..2ede34e558 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -203,6 +203,7 @@ This section outlines configurations that are used in this spec. | `ATTESTATION_SUBNET_COUNT` | `2**6` (= 64) | The number of attestation subnets used in the gossipsub protocol. | | `ATTESTATION_SUBNET_EXTRA_BITS` | `0` | The number of extra bits of a NodeId to use when mapping to a subscribed subnet | | `ATTESTATION_SUBNET_PREFIX_BITS` | `int(ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS)` | | +| `MAX_CONCURRENT_REQUESTS` | `2` | Maximum number of concurrent requests per protocol ID that a client may issue | ### MetaData @@ -560,7 +561,7 @@ The request MUST be encoded according to the encoding strategy. The requester MUST close the write side of the stream once it finishes writing the request message. At this point, the stream will be half-closed. -The requester MUST NOT make more than two concurrent requests with the same ID. +The requester MUST NOT make more than `MAX_CONCURRENT_REQUESTS` concurrent requests with the same protocol ID. If a timeout happens on the requesting side, they SHOULD reset the stream. From 456372e550239f171c0e3f4388055d83b1aa42fa Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Thu, 3 Oct 2024 08:16:17 +0200 Subject: [PATCH 029/142] two streams again --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 2ede34e558..bc99c3ee63 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -625,7 +625,7 @@ The `ErrorMessage` schema is: *Note*: By convention, the `error_message` is a sequence of bytes that MAY be interpreted as a UTF-8 string (for debugging purposes). Clients MUST treat as valid any byte sequences. -The responder MAY penalise peers that concurrently open more than two streams for the same request type, for the protocol IDs defined in this specification. +The responder MAY penalise peers that concurrently open more than `MAX_CONCURRENT_REQUESTS` streams for the same request type, for the protocol IDs defined in this specification. #### Encoding strategies From edd7377b5f9e9b87cf4db279b97b7240bcdf21b7 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Thu, 3 Oct 2024 10:59:01 +0200 Subject: [PATCH 030/142] apply suggestions --- specs/deneb/p2p-interface.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 3992828e3b..19943d5347 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -185,7 +185,6 @@ The following validations MUST pass before forwarding the `blob_sidecar` on the - _[REJECT]_ The sidecar's inclusion proof is valid as verified by `verify_blob_sidecar_inclusion_proof(blob_sidecar)`. - _[REJECT]_ The sidecar's blob is valid as verified by `verify_blob_kzg_proof(blob_sidecar.blob, blob_sidecar.kzg_commitment, blob_sidecar.kzg_proof)`. - _[IGNORE]_ The sidecar is the first sidecar for the tuple `(block_header.slot, block_header.proposer_index, blob_sidecar.index)` with valid header signature, sidecar inclusion proof, and kzg proof. -*Note:* If client obtains the blob via local execution layer client or any other method before receiving the corresponding sidecar via this topic, it MUST NOT consider the tuple as "seen" (i.e. it MUST participate in the dissemination of the blob_sidecar even if it already obtained the blob) - _[REJECT]_ The sidecar is proposed by the expected `proposer_index` for the block's slot in the context of the current shuffling (defined by `block_header.parent_root`/`block_header.slot`). If the `proposer_index` cannot immediately be verified against the expected shuffling, the sidecar MAY be queued for later processing while proposers for the block's branch are calculated -- in such a case _do not_ `REJECT`, instead `IGNORE` this message. @@ -315,7 +314,7 @@ Clients SHOULD NOT respond with sidecars related to blocks that fail the beacon In addition to `BlobSidecarsByRoot` requests, recent blobs recovery MAY also be done by querying the Execution Layer (i.e. via `engine_getBlobsV1`) Implementers are encouraged to leverage this method to increase the likelihood of incorporating and attesting to the last block when its proposer is not able to publish blobs on time. -Client MUST publish the corresponding `blob_sidecar` whenever successfully recovers blobs via local execution layer client. +Clients MUST publish the corresponding `blob_sidecar` on the `blob_sidecar_{subnet_id}` subnet whenever it successfully recovers blobs via local execution layer client. In such case, clients MUST also consider the `blob_sidecar` as "seen" with regards to blob subnet gossip rules. ##### BlobSidecarsByRange v1 From 2612f7f278ffa2d0ed405683c767a7e000f746cb Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Thu, 3 Oct 2024 17:20:37 +0200 Subject: [PATCH 031/142] rephrasing --- specs/deneb/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 19943d5347..d68eb5e327 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -314,7 +314,7 @@ Clients SHOULD NOT respond with sidecars related to blocks that fail the beacon In addition to `BlobSidecarsByRoot` requests, recent blobs recovery MAY also be done by querying the Execution Layer (i.e. via `engine_getBlobsV1`) Implementers are encouraged to leverage this method to increase the likelihood of incorporating and attesting to the last block when its proposer is not able to publish blobs on time. -Clients MUST publish the corresponding `blob_sidecar` on the `blob_sidecar_{subnet_id}` subnet whenever it successfully recovers blobs via local execution layer client. In such case, clients MUST also consider the `blob_sidecar` as "seen" with regards to blob subnet gossip rules. +Clients MUST publish the corresponding `blob_sidecar` on the `blob_sidecar_{subnet_id}` subnet whenever it successfully recovers blobs via local execution layer client. In such case, clients MUST also treat the `blob_sidecar` as if it had been received via gossip (ie update the anti-equivocation cache) ##### BlobSidecarsByRange v1 From a96065c618f802d7fe018f1505fd7482c57b4c1f Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Thu, 3 Oct 2024 17:49:48 +0200 Subject: [PATCH 032/142] clarification --- specs/deneb/p2p-interface.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index d68eb5e327..69a8188558 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -314,7 +314,9 @@ Clients SHOULD NOT respond with sidecars related to blocks that fail the beacon In addition to `BlobSidecarsByRoot` requests, recent blobs recovery MAY also be done by querying the Execution Layer (i.e. via `engine_getBlobsV1`) Implementers are encouraged to leverage this method to increase the likelihood of incorporating and attesting to the last block when its proposer is not able to publish blobs on time. -Clients MUST publish the corresponding `blob_sidecar` on the `blob_sidecar_{subnet_id}` subnet whenever it successfully recovers blobs via local execution layer client. In such case, clients MUST also treat the `blob_sidecar` as if it had been received via gossip (ie update the anti-equivocation cache) +When clients uses local execution layer to retrieve blobs relative to the current slot they MUST behave as if the corresponding `blob_sidecar` had been received via gossip. In particular they MUST: +- publish the corresponding `blob_sidecar` on the `blob_sidecar_{subnet_id}` subnet. +- update gossip rule related data structures (ie update the anti-equivocation cache). ##### BlobSidecarsByRange v1 From 5b93d6a0418600384ff4ca270f2fb92e2251e220 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Thu, 3 Oct 2024 17:52:56 +0200 Subject: [PATCH 033/142] improvement --- specs/deneb/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 69a8188558..95d0e33ed1 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -314,7 +314,7 @@ Clients SHOULD NOT respond with sidecars related to blocks that fail the beacon In addition to `BlobSidecarsByRoot` requests, recent blobs recovery MAY also be done by querying the Execution Layer (i.e. via `engine_getBlobsV1`) Implementers are encouraged to leverage this method to increase the likelihood of incorporating and attesting to the last block when its proposer is not able to publish blobs on time. -When clients uses local execution layer to retrieve blobs relative to the current slot they MUST behave as if the corresponding `blob_sidecar` had been received via gossip. In particular they MUST: +When clients uses local execution layer to recover missing blobs relative to the current slot, they MUST behave as if the corresponding `blob_sidecar` had been received via gossip. In particular they MUST: - publish the corresponding `blob_sidecar` on the `blob_sidecar_{subnet_id}` subnet. - update gossip rule related data structures (ie update the anti-equivocation cache). From 7c31b3ee7a8963fca527972be39a7ce43e3f1fe9 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Fri, 4 Oct 2024 14:13:04 +0200 Subject: [PATCH 034/142] Update specs/deneb/p2p-interface.md Co-authored-by: Mehdi AOUADI --- specs/deneb/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 95d0e33ed1..c7ea310584 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -314,7 +314,7 @@ Clients SHOULD NOT respond with sidecars related to blocks that fail the beacon In addition to `BlobSidecarsByRoot` requests, recent blobs recovery MAY also be done by querying the Execution Layer (i.e. via `engine_getBlobsV1`) Implementers are encouraged to leverage this method to increase the likelihood of incorporating and attesting to the last block when its proposer is not able to publish blobs on time. -When clients uses local execution layer to recover missing blobs relative to the current slot, they MUST behave as if the corresponding `blob_sidecar` had been received via gossip. In particular they MUST: +When clients use local execution layer to recover missing blobs relative to the current slot, they MUST behave as if the corresponding `blob_sidecar` had been received via gossip. In particular they MUST: - publish the corresponding `blob_sidecar` on the `blob_sidecar_{subnet_id}` subnet. - update gossip rule related data structures (ie update the anti-equivocation cache). From 96fbfcb3ac7f958e7e2326c060e8249bfce8bfaf Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Mon, 7 Oct 2024 09:00:19 +0200 Subject: [PATCH 035/142] Update specs/electra/p2p-interface.md Co-authored-by: NC <17676176+ensi321@users.noreply.github.com> --- specs/electra/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/electra/p2p-interface.md b/specs/electra/p2p-interface.md index b8370c77f8..d5fc70cbc0 100644 --- a/specs/electra/p2p-interface.md +++ b/specs/electra/p2p-interface.md @@ -62,7 +62,7 @@ The following convenience variables are re-defined: The following validations are added: - _[REJECT]_ `attestation.data.index == 0` - _[REJECT]_ The attester is a member of the committee -- i.e. - `atestation.attester_index in get_beacon_committee(state, attestation.data.slot, index)`. + `attestation.attester_index in get_beacon_committee(state, attestation.data.slot, index)`. The following validations are removed: - _[REJECT]_ The attestation is unaggregated -- From 8f044abe0a08cb7980e4df85b431157e0fbd6dd6 Mon Sep 17 00:00:00 2001 From: Manu NALEPA Date: Tue, 8 Oct 2024 01:07:10 +0200 Subject: [PATCH 036/142] EIP-7594: Specify inclusion proof function to run. For by root and by range requests checks, the exact function to run is specified for KZG commitments, but not for the inclusion proof. This PR aligns both checks. --- specs/_features/eip7594/p2p-interface.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/_features/eip7594/p2p-interface.md b/specs/_features/eip7594/p2p-interface.md index 4c4ae83efb..98fb5dd732 100644 --- a/specs/_features/eip7594/p2p-interface.md +++ b/specs/_features/eip7594/p2p-interface.md @@ -223,7 +223,7 @@ Requests sidecars by block root and index. The response is a list of `DataColumnIdentifier` whose length is less than or equal to the number of requests. It may be less in the case that the responding peer is missing blocks or sidecars. -Before consuming the next response chunk, the response reader SHOULD verify the data column sidecar is well-formatted, has valid inclusion proof, and is correct w.r.t. the expected KZG commitments through `verify_data_column_sidecar_kzg_proofs`. +Before consuming the next response chunk, the response reader SHOULD verify the data column sidecar is well-formatted, has valid inclusion proof through `verify_data_column_sidecar_inclusion_proof`, and is correct w.r.t. the expected KZG commitments through `verify_data_column_sidecar_kzg_proofs`. No more than `MAX_REQUEST_DATA_COLUMN_SIDECARS` may be requested at a time. @@ -269,7 +269,7 @@ Response Content: Requests data column sidecars in the slot range `[start_slot, start_slot + count)` of the given `columns`, leading up to the current head block as selected by fork choice. -Before consuming the next response chunk, the response reader SHOULD verify the data column sidecar is well-formatted, has valid inclusion proof, and is correct w.r.t. the expected KZG commitments through `verify_data_column_sidecar_kzg_proofs`. +Before consuming the next response chunk, the response reader SHOULD verify the data column sidecar is well-formatted, has valid inclusion proof through `verify_data_column_sidecar_inclusion_proof`, and is correct w.r.t. the expected KZG commitments through `verify_data_column_sidecar_kzg_proofs`. `DataColumnSidecarsByRange` is primarily used to sync data columns that may have been missed on gossip and to sync within the `MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS` window. From a5d45778aa7f3b53d33e1ed07c92c3cd2b37619d Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 9 Oct 2024 07:41:17 -0500 Subject: [PATCH 037/142] In README, move outdated specs to new section --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index fa41dc6298..15c80a6520 100644 --- a/README.md +++ b/README.md @@ -24,9 +24,15 @@ Features are researched and developed in parallel, and then consolidated into se | 4 | **Deneb** | `269568` |
  • Core
    • [Beacon Chain changes](specs/deneb/beacon-chain.md)
    • [Deneb fork](specs/deneb/fork.md)
    • [Polynomial commitments](specs/deneb/polynomial-commitments.md)
    • [Fork choice changes](specs/deneb/fork-choice.md)
  • Additions
    • [Light client sync protocol changes](specs/deneb/light-client/sync-protocol.md) ([fork](specs/deneb/light-client/fork.md), [full node](specs/deneb/light-client/full-node.md), [networking](specs/deneb/light-client/p2p-interface.md))
    • [Honest validator guide changes](specs/deneb/validator.md)
    • [P2P networking](specs/deneb/p2p-interface.md)
| ### In-development Specifications + | Code Name or Topic | Specs | Notes | | - | - | - | | Electra |
  • Core
    • [Beacon Chain changes](specs/electra/beacon-chain.md)
    • [EIP-6110 fork](specs/electra/fork.md)
  • Additions
    • [Light client sync protocol changes](specs/electra/light-client/sync-protocol.md) ([fork](specs/electra/light-client/fork.md), [full node](specs/electra/light-client/full-node.md), [networking](specs/electra/light-client/p2p-interface.md))
    • [Honest validator guide changes](specs/electra/validator.md)
| + +### Outdated Specifications + +| Code Name or Topic | Specs | Notes | +| - | - | - | | Sharding (outdated) |
  • Core
    • [Beacon Chain changes](specs/_features/sharding/beacon-chain.md)
  • Additions
    • [P2P networking](specs/_features/sharding/p2p-interface.md)
| | Custody Game (outdated) |
  • Core
    • [Beacon Chain changes](specs/_features/custody_game/beacon-chain.md)
  • Additions
    • [Honest validator guide changes](specs/_features/custody_game/validator.md)
| Dependent on sharding | | Data Availability Sampling (outdated) |
  • Core
    • [Core types and functions](specs/_features/das/das-core.md)
    • [Fork choice changes](specs/_features/das/fork-choice.md)
  • Additions
    • [P2P Networking](specs/_features/das/p2p-interface.md)
    • [Sampling process](specs/_features/das/sampling.md)
|
  • Dependent on sharding
  • [Technical explainer](https://hackmd.io/@HWeNw8hNRimMm2m2GH56Cw/B1YJPGkpD)
| From afdf5633fd2294f77e1364f864923b4c1ad72498 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 9 Oct 2024 07:43:13 -0500 Subject: [PATCH 038/142] Remove "(outdated)" from names --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 15c80a6520..5598a44997 100644 --- a/README.md +++ b/README.md @@ -33,9 +33,9 @@ Features are researched and developed in parallel, and then consolidated into se | Code Name or Topic | Specs | Notes | | - | - | - | -| Sharding (outdated) |
  • Core
    • [Beacon Chain changes](specs/_features/sharding/beacon-chain.md)
  • Additions
    • [P2P networking](specs/_features/sharding/p2p-interface.md)
| -| Custody Game (outdated) |
  • Core
    • [Beacon Chain changes](specs/_features/custody_game/beacon-chain.md)
  • Additions
    • [Honest validator guide changes](specs/_features/custody_game/validator.md)
| Dependent on sharding | -| Data Availability Sampling (outdated) |
  • Core
    • [Core types and functions](specs/_features/das/das-core.md)
    • [Fork choice changes](specs/_features/das/fork-choice.md)
  • Additions
    • [P2P Networking](specs/_features/das/p2p-interface.md)
    • [Sampling process](specs/_features/das/sampling.md)
|
  • Dependent on sharding
  • [Technical explainer](https://hackmd.io/@HWeNw8hNRimMm2m2GH56Cw/B1YJPGkpD)
| +| Sharding |
  • Core
    • [Beacon Chain changes](specs/_features/sharding/beacon-chain.md)
  • Additions
    • [P2P networking](specs/_features/sharding/p2p-interface.md)
| +| Custody Game |
  • Core
    • [Beacon Chain changes](specs/_features/custody_game/beacon-chain.md)
  • Additions
    • [Honest validator guide changes](specs/_features/custody_game/validator.md)
| Dependent on sharding | +| Data Availability Sampling |
  • Core
    • [Core types and functions](specs/_features/das/das-core.md)
    • [Fork choice changes](specs/_features/das/fork-choice.md)
  • Additions
    • [P2P Networking](specs/_features/das/p2p-interface.md)
    • [Sampling process](specs/_features/das/sampling.md)
|
  • Dependent on sharding
  • [Technical explainer](https://hackmd.io/@HWeNw8hNRimMm2m2GH56Cw/B1YJPGkpD)
| ### Accompanying documents can be found in [specs](specs) and include: From d9661880398afb3bd80bc0db867d35b96017f789 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 9 Oct 2024 07:47:11 -0500 Subject: [PATCH 039/142] Use same table format for stable & in-development specs --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5598a44997..afbca3c5e6 100644 --- a/README.md +++ b/README.md @@ -25,9 +25,9 @@ Features are researched and developed in parallel, and then consolidated into se ### In-development Specifications -| Code Name or Topic | Specs | Notes | -| - | - | - | -| Electra |
  • Core
    • [Beacon Chain changes](specs/electra/beacon-chain.md)
    • [EIP-6110 fork](specs/electra/fork.md)
  • Additions
    • [Light client sync protocol changes](specs/electra/light-client/sync-protocol.md) ([fork](specs/electra/light-client/fork.md), [full node](specs/electra/light-client/full-node.md), [networking](specs/electra/light-client/p2p-interface.md))
    • [Honest validator guide changes](specs/electra/validator.md)
| +| Seq. | Code Name | Fork Epoch | Specs | +| - | - | - | - | +| 5 | Electra | TBD|
  • Core
    • [Beacon Chain changes](specs/electra/beacon-chain.md)
    • [EIP-6110 fork](specs/electra/fork.md)
  • Additions
    • [Light client sync protocol changes](specs/electra/light-client/sync-protocol.md) ([fork](specs/electra/light-client/fork.md), [full node](specs/electra/light-client/full-node.md), [networking](specs/electra/light-client/p2p-interface.md))
    • [Honest validator guide changes](specs/electra/validator.md)
| ### Outdated Specifications From d1bca1ef42a16584546cfde921f63f8ea5fa59ec Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 9 Oct 2024 07:49:50 -0500 Subject: [PATCH 040/142] Change "EIP-6110 fork" to "Electra fork" --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index afbca3c5e6..a0b6516078 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Features are researched and developed in parallel, and then consolidated into se | Seq. | Code Name | Fork Epoch | Specs | | - | - | - | - | -| 5 | Electra | TBD|
  • Core
    • [Beacon Chain changes](specs/electra/beacon-chain.md)
    • [EIP-6110 fork](specs/electra/fork.md)
  • Additions
    • [Light client sync protocol changes](specs/electra/light-client/sync-protocol.md) ([fork](specs/electra/light-client/fork.md), [full node](specs/electra/light-client/full-node.md), [networking](specs/electra/light-client/p2p-interface.md))
    • [Honest validator guide changes](specs/electra/validator.md)
| +| 5 | Electra | TBD|
  • Core
    • [Beacon Chain changes](specs/electra/beacon-chain.md)
    • [Electra fork](specs/electra/fork.md)
  • Additions
    • [Light client sync protocol changes](specs/electra/light-client/sync-protocol.md) ([fork](specs/electra/light-client/fork.md), [full node](specs/electra/light-client/full-node.md), [networking](specs/electra/light-client/p2p-interface.md))
    • [Honest validator guide changes](specs/electra/validator.md)
| ### Outdated Specifications From bf49f53f722bb0de76d2b45516b8a64ed689a482 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 9 Oct 2024 07:52:48 -0500 Subject: [PATCH 041/142] Add electra p2p networking changes to additions --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a0b6516078..979cce83df 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Features are researched and developed in parallel, and then consolidated into se | Seq. | Code Name | Fork Epoch | Specs | | - | - | - | - | -| 5 | Electra | TBD|
  • Core
    • [Beacon Chain changes](specs/electra/beacon-chain.md)
    • [Electra fork](specs/electra/fork.md)
  • Additions
    • [Light client sync protocol changes](specs/electra/light-client/sync-protocol.md) ([fork](specs/electra/light-client/fork.md), [full node](specs/electra/light-client/full-node.md), [networking](specs/electra/light-client/p2p-interface.md))
    • [Honest validator guide changes](specs/electra/validator.md)
| +| 5 | Electra | TBD|
  • Core
    • [Beacon Chain changes](specs/electra/beacon-chain.md)
    • [Electra fork](specs/electra/fork.md)
  • Additions
    • [Light client sync protocol changes](specs/electra/light-client/sync-protocol.md) ([fork](specs/electra/light-client/fork.md), [full node](specs/electra/light-client/full-node.md), [networking](specs/electra/light-client/p2p-interface.md))
    • [Honest validator guide changes](specs/electra/validator.md)
    • [P2P networking](specs/electra/p2p-interface.md)
| ### Outdated Specifications From 8630a44fd45d251dc93428e16d377bef57df0170 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 9 Oct 2024 08:32:26 -0500 Subject: [PATCH 042/142] Delete extra unordered list in capella/deneb/electra --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 979cce83df..75151cd453 100644 --- a/README.md +++ b/README.md @@ -20,14 +20,14 @@ Features are researched and developed in parallel, and then consolidated into se | 0 | **Phase0** |`0` |
  • Core
    • [The beacon chain](specs/phase0/beacon-chain.md)
    • [Deposit contract](specs/phase0/deposit-contract.md)
    • [Beacon chain fork choice](specs/phase0/fork-choice.md)
  • Additions
    • [Honest validator guide](specs/phase0/validator.md)
    • [P2P networking](specs/phase0/p2p-interface.md)
    • [Weak subjectivity](specs/phase0/weak-subjectivity.md)
| | 1 | **Altair** | `74240` |
  • Core
    • [Beacon chain changes](specs/altair/beacon-chain.md)
    • [Altair fork](specs/altair/fork.md)
  • Additions
    • [Light client sync protocol](specs/altair/light-client/sync-protocol.md) ([full node](specs/altair/light-client/full-node.md), [light client](specs/altair/light-client/light-client.md), [networking](specs/altair/light-client/p2p-interface.md))
    • [Honest validator guide changes](specs/altair/validator.md)
    • [P2P networking](specs/altair/p2p-interface.md)
| | 2 | **Bellatrix**
(["The Merge"](https://ethereum.org/en/upgrades/merge/)) | `144896` |
  • Core
    • [Beacon Chain changes](specs/bellatrix/beacon-chain.md)
    • [Bellatrix fork](specs/bellatrix/fork.md)
    • [Fork choice changes](specs/bellatrix/fork-choice.md)
  • Additions
    • [Honest validator guide changes](specs/bellatrix/validator.md)
    • [P2P networking](specs/bellatrix/p2p-interface.md)
| -| 3 | **Capella** | `194048` |
  • Core
    • [Beacon chain changes](specs/capella/beacon-chain.md)
    • [Capella fork](specs/capella/fork.md)
  • Additions
    • [Light client sync protocol changes](specs/capella/light-client/sync-protocol.md) ([fork](specs/capella/light-client/fork.md), [full node](specs/capella/light-client/full-node.md), [networking](specs/capella/light-client/p2p-interface.md))
    • [Validator additions](specs/capella/validator.md)
    • [P2P networking](specs/capella/p2p-interface.md)
| -| 4 | **Deneb** | `269568` |
  • Core
    • [Beacon Chain changes](specs/deneb/beacon-chain.md)
    • [Deneb fork](specs/deneb/fork.md)
    • [Polynomial commitments](specs/deneb/polynomial-commitments.md)
    • [Fork choice changes](specs/deneb/fork-choice.md)
  • Additions
    • [Light client sync protocol changes](specs/deneb/light-client/sync-protocol.md) ([fork](specs/deneb/light-client/fork.md), [full node](specs/deneb/light-client/full-node.md), [networking](specs/deneb/light-client/p2p-interface.md))
    • [Honest validator guide changes](specs/deneb/validator.md)
    • [P2P networking](specs/deneb/p2p-interface.md)
| +| 3 | **Capella** | `194048` |
  • Core
    • [Beacon chain changes](specs/capella/beacon-chain.md)
    • [Capella fork](specs/capella/fork.md)
  • Additions
    • [Light client sync protocol changes](specs/capella/light-client/sync-protocol.md) ([fork](specs/capella/light-client/fork.md), [full node](specs/capella/light-client/full-node.md), [networking](specs/capella/light-client/p2p-interface.md))
    • [Validator additions](specs/capella/validator.md)
    • [P2P networking](specs/capella/p2p-interface.md)
| +| 4 | **Deneb** | `269568` |
  • Core
    • [Beacon Chain changes](specs/deneb/beacon-chain.md)
    • [Deneb fork](specs/deneb/fork.md)
    • [Polynomial commitments](specs/deneb/polynomial-commitments.md)
    • [Fork choice changes](specs/deneb/fork-choice.md)
  • Additions
    • [Light client sync protocol changes](specs/deneb/light-client/sync-protocol.md) ([fork](specs/deneb/light-client/fork.md), [full node](specs/deneb/light-client/full-node.md), [networking](specs/deneb/light-client/p2p-interface.md))
    • [Honest validator guide changes](specs/deneb/validator.md)
    • [P2P networking](specs/deneb/p2p-interface.md)
| ### In-development Specifications | Seq. | Code Name | Fork Epoch | Specs | | - | - | - | - | -| 5 | Electra | TBD|
  • Core
    • [Beacon Chain changes](specs/electra/beacon-chain.md)
    • [Electra fork](specs/electra/fork.md)
  • Additions
    • [Light client sync protocol changes](specs/electra/light-client/sync-protocol.md) ([fork](specs/electra/light-client/fork.md), [full node](specs/electra/light-client/full-node.md), [networking](specs/electra/light-client/p2p-interface.md))
    • [Honest validator guide changes](specs/electra/validator.md)
    • [P2P networking](specs/electra/p2p-interface.md)
| +| 5 | Electra | TBD|
  • Core
    • [Beacon Chain changes](specs/electra/beacon-chain.md)
    • [Electra fork](specs/electra/fork.md)
  • Additions
    • [Light client sync protocol changes](specs/electra/light-client/sync-protocol.md) ([fork](specs/electra/light-client/fork.md), [full node](specs/electra/light-client/full-node.md), [networking](specs/electra/light-client/p2p-interface.md))
    • [Honest validator guide changes](specs/electra/validator.md)
    • [P2P networking](specs/electra/p2p-interface.md)
| ### Outdated Specifications From 03655aec93aeab950cfcf6f87ca4a2c4debdb006 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 9 Oct 2024 08:33:19 -0500 Subject: [PATCH 043/142] Make Electra bold --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 75151cd453..85843dd1de 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Features are researched and developed in parallel, and then consolidated into se | Seq. | Code Name | Fork Epoch | Specs | | - | - | - | - | -| 5 | Electra | TBD|
  • Core
    • [Beacon Chain changes](specs/electra/beacon-chain.md)
    • [Electra fork](specs/electra/fork.md)
  • Additions
    • [Light client sync protocol changes](specs/electra/light-client/sync-protocol.md) ([fork](specs/electra/light-client/fork.md), [full node](specs/electra/light-client/full-node.md), [networking](specs/electra/light-client/p2p-interface.md))
    • [Honest validator guide changes](specs/electra/validator.md)
    • [P2P networking](specs/electra/p2p-interface.md)
| +| 5 | **Electra** | TBD|
  • Core
    • [Beacon Chain changes](specs/electra/beacon-chain.md)
    • [Electra fork](specs/electra/fork.md)
  • Additions
    • [Light client sync protocol changes](specs/electra/light-client/sync-protocol.md) ([fork](specs/electra/light-client/fork.md), [full node](specs/electra/light-client/full-node.md), [networking](specs/electra/light-client/p2p-interface.md))
    • [Honest validator guide changes](specs/electra/validator.md)
    • [P2P networking](specs/electra/p2p-interface.md)
| ### Outdated Specifications From 6e18e704689c3bf5c88d0e64f24987b49863dca5 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 9 Oct 2024 09:41:59 -0500 Subject: [PATCH 044/142] Add missing space --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 85843dd1de..215e8b3925 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Features are researched and developed in parallel, and then consolidated into se | Seq. | Code Name | Fork Epoch | Specs | | - | - | - | - | -| 5 | **Electra** | TBD|
  • Core
    • [Beacon Chain changes](specs/electra/beacon-chain.md)
    • [Electra fork](specs/electra/fork.md)
  • Additions
    • [Light client sync protocol changes](specs/electra/light-client/sync-protocol.md) ([fork](specs/electra/light-client/fork.md), [full node](specs/electra/light-client/full-node.md), [networking](specs/electra/light-client/p2p-interface.md))
    • [Honest validator guide changes](specs/electra/validator.md)
    • [P2P networking](specs/electra/p2p-interface.md)
| +| 5 | **Electra** | TBD |
  • Core
    • [Beacon Chain changes](specs/electra/beacon-chain.md)
    • [Electra fork](specs/electra/fork.md)
  • Additions
    • [Light client sync protocol changes](specs/electra/light-client/sync-protocol.md) ([fork](specs/electra/light-client/fork.md), [full node](specs/electra/light-client/full-node.md), [networking](specs/electra/light-client/p2p-interface.md))
    • [Honest validator guide changes](specs/electra/validator.md)
    • [P2P networking](specs/electra/p2p-interface.md)
| ### Outdated Specifications From 72f51900a79f417683634f8f3c7d9b78c2336f9c Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 9 Oct 2024 11:55:49 -0500 Subject: [PATCH 045/142] Fix lint --- .../eth2spec/test/eip7594/unittests/test_networking.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_networking.py b/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_networking.py index 633508f9a2..931cc9c1d1 100644 --- a/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_networking.py +++ b/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_networking.py @@ -15,8 +15,8 @@ from eth2spec.test.helpers.execution_payload import ( compute_el_block_hash, ) -from eth2spec.test.helpers.sharding import ( - get_sample_opaque_tx, +from eth2spec.test.helpers.blob import ( + get_sample_blob_tx, ) @@ -25,7 +25,7 @@ def compute_data_column_sidecar(spec, state): rng = random.Random(5566) - opaque_tx, blobs, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=2) + opaque_tx, blobs, blob_kzg_commitments, _ = get_sample_blob_tx(spec, blob_count=2) block = get_random_ssz_object( rng, spec.BeaconBlock, From 81a0f435f63717bdf1f130e9ac722aa27a7e7d61 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 9 Oct 2024 14:31:08 -0500 Subject: [PATCH 046/142] Fix invalid file references in markdown files --- README.md | 2 +- specs/_features/custody_game/validator.md | 2 +- specs/_features/eip6800/fork.md | 2 +- specs/_features/eip6914/beacon-chain.md | 2 +- specs/electra/beacon-chain.md | 2 +- specs/electra/validator.md | 2 +- tests/formats/kzg_7594/README.md | 3 +-- 7 files changed, 7 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index fa41dc6298..361d1f13cd 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ This repository hosts the current Ethereum proof-of-stake specifications. Discus [![GitHub release](https://img.shields.io/github/v/release/ethereum/consensus-specs)](https://github.com/ethereum/consensus-specs/releases/) [![PyPI version](https://badge.fury.io/py/eth2spec.svg)](https://badge.fury.io/py/eth2spec) [![testgen](https://github.com/ethereum/consensus-specs/actions/workflows/generate_vectors.yml/badge.svg?branch=dev&event=schedule)](https://github.com/ethereum/consensus-specs/actions/workflows/generate_vectors.yml) -Core specifications for Ethereum proof-of-stake clients can be found in [specs](specs/). These are divided into features. +Core specifications for Ethereum proof-of-stake clients can be found in [specs](./specs). These are divided into features. Features are researched and developed in parallel, and then consolidated into sequential upgrades when ready. ### Stable Specifications diff --git a/specs/_features/custody_game/validator.md b/specs/_features/custody_game/validator.md index 37df35e882..e711d92a1c 100644 --- a/specs/_features/custody_game/validator.md +++ b/specs/_features/custody_game/validator.md @@ -1,7 +1,7 @@ # Custody Game -- Honest Validator **Notice**: This document is a work-in-progress for researchers and implementers. -This is an accompanying document to the [Custody Game](./), which describes the expected actions of a "validator" +This is an accompanying document to [Custody Game -- The Beacon Chain](./beacon-chain.md), which describes the expected actions of a "validator" participating in the shard data Custody Game. ## Table of contents diff --git a/specs/_features/eip6800/fork.md b/specs/_features/eip6800/fork.md index 74f143f597..14172c9f3e 100644 --- a/specs/_features/eip6800/fork.md +++ b/specs/_features/eip6800/fork.md @@ -68,7 +68,7 @@ If `state.slot % SLOTS_PER_EPOCH == 0` and `compute_epoch_at_slot(state.slot) == an irregular state change is made to upgrade to eip6800. The upgrade occurs after the completion of the inner loop of `process_slots` that sets `state.slot` equal to `EIP6800_FORK_EPOCH * SLOTS_PER_EPOCH`. -Care must be taken when transitioning through the fork boundary as implementations will need a modified [state transition function](../phase0/beacon-chain.md#beacon-chain-state-transition-function) that deviates from the Phase 0 document. +Care must be taken when transitioning through the fork boundary as implementations will need a modified [state transition function](../../phase0/beacon-chain.md#beacon-chain-state-transition-function) that deviates from the Phase 0 document. In particular, the outer `state_transition` function defined in the Phase 0 document will not expose the precise fork slot to execute the upgrade in the presence of skipped slots at the fork boundary. Instead, the logic must be within `process_slots`. ```python diff --git a/specs/_features/eip6914/beacon-chain.md b/specs/_features/eip6914/beacon-chain.md index 1e0b20747e..fcb7716f7e 100644 --- a/specs/_features/eip6914/beacon-chain.md +++ b/specs/_features/eip6914/beacon-chain.md @@ -23,7 +23,7 @@ This is the beacon chain specification to assign new deposits to existing validator records. Refers to [EIP-6914](https://github.com/ethereum/EIPs/pull/6914). -*Note:* This specification is built upon [Capella](../../capella/beacon_chain.md) and is under active development. +*Note:* This specification is built upon [Capella](../../capella/beacon-chain.md) and is under active development. ## Preset diff --git a/specs/electra/beacon-chain.md b/specs/electra/beacon-chain.md index 7c6d9fe1fa..ccc60082ed 100644 --- a/specs/electra/beacon-chain.md +++ b/specs/electra/beacon-chain.md @@ -118,7 +118,7 @@ Electra is a consensus-layer upgrade containing a number of features. Including: * [EIP-7251](https://eips.ethereum.org/EIPS/eip-7251): Increase the MAX_EFFECTIVE_BALANCE * [EIP-7549](https://eips.ethereum.org/EIPS/eip-7549): Move committee index outside Attestation -*Note:* This specification is built upon [Deneb](../deneb/beacon_chain.md) and is under active development. +*Note:* This specification is built upon [Deneb](../deneb/beacon-chain.md) and is under active development. ## Constants diff --git a/specs/electra/validator.md b/specs/electra/validator.md index f589e963c5..a9869d4e9b 100644 --- a/specs/electra/validator.md +++ b/specs/electra/validator.md @@ -32,7 +32,7 @@ This document represents the changes to be made in the code of an "honest valida ## Prerequisites -This document is an extension of the [Deneb -- Honest Validator](../../deneb/validator.md) guide. +This document is an extension of the [Deneb -- Honest Validator](../deneb/validator.md) guide. All behaviors and definitions defined in this document, and documents it extends, carry over unless explicitly noted or overridden. All terminology, constants, functions, and protocol mechanics defined in the updated Beacon Chain doc of [Electra](./beacon-chain.md) are requisite for this document and used throughout. diff --git a/tests/formats/kzg_7594/README.md b/tests/formats/kzg_7594/README.md index f03bc3707c..ce1d8ff54c 100644 --- a/tests/formats/kzg_7594/README.md +++ b/tests/formats/kzg_7594/README.md @@ -6,7 +6,6 @@ We do not recommend rolling your own crypto or using an untested KZG library. The KZG test suite runner has the following handlers: -- [`compute_cells`](./compute_cells.md) - [`compute_cells_and_kzg_proofs`](./compute_cells_and_kzg_proofs.md) - [`verify_cell_kzg_proof_batch`](./verify_cell_kzg_proof_batch.md) -- [`recover_all_cells`](./recover_all_cells.md) +- [`recover_cells_and_kzg_proofs`](./recover_cells_and_kzg_proofs.md) From 8cfe8fb7a6d961df3a3ccc4f6b454688b27545f1 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 9 Oct 2024 14:32:17 -0500 Subject: [PATCH 047/142] Re-order kzg tests --- tests/formats/kzg_7594/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/formats/kzg_7594/README.md b/tests/formats/kzg_7594/README.md index ce1d8ff54c..baa8589f8a 100644 --- a/tests/formats/kzg_7594/README.md +++ b/tests/formats/kzg_7594/README.md @@ -7,5 +7,5 @@ We do not recommend rolling your own crypto or using an untested KZG library. The KZG test suite runner has the following handlers: - [`compute_cells_and_kzg_proofs`](./compute_cells_and_kzg_proofs.md) -- [`verify_cell_kzg_proof_batch`](./verify_cell_kzg_proof_batch.md) - [`recover_cells_and_kzg_proofs`](./recover_cells_and_kzg_proofs.md) +- [`verify_cell_kzg_proof_batch`](./verify_cell_kzg_proof_batch.md) From 3011727214a70ce8ddb0158dfbf3769d718f7d11 Mon Sep 17 00:00:00 2001 From: Lucas Saldanha Date: Thu, 10 Oct 2024 12:25:18 +1300 Subject: [PATCH 048/142] Updated validator spec with rules for including execution requests in the beacon block body --- specs/electra/validator.md | 50 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/specs/electra/validator.md b/specs/electra/validator.md index f589e963c5..e6f0906d83 100644 --- a/specs/electra/validator.md +++ b/specs/electra/validator.md @@ -38,6 +38,19 @@ All behaviors and definitions defined in this document, and documents it extends All terminology, constants, functions, and protocol mechanics defined in the updated Beacon Chain doc of [Electra](./beacon-chain.md) are requisite for this document and used throughout. Please see related Beacon Chain doc before continuing and use them as a reference throughout. +## Helpers + +### Modified `GetPayloadResponse` + +```python +@dataclass +class GetPayloadResponse(object): + execution_payload: ExecutionPayload + block_value: uint256 + blobs_bundle: BlobsBundle + execution_requests: list[bytes] # [New in Electra] +``` + ## Containers ### Modified Containers @@ -59,6 +72,24 @@ class SignedAggregateAndProof(Container): signature: BLSSignature ``` +## Protocol + +### `ExecutionEngine` + +#### Modified `get_payload` + +Given the `payload_id`, `get_payload` returns the most recent version of the execution payload that +has been built since the corresponding call to `notify_forkchoice_updated` method. + +```python +def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> GetPayloadResponse: + """ + Return ExecutionPayload, uint256, BlobsBundle and list[bytes] objects. + """ + # pylint: disable=unused-argument + ... +``` + ## Block proposal ### Constructing the `BeaconBlockBody` @@ -148,6 +179,25 @@ def prepare_execution_payload(state: BeaconState, ) ``` +#### Execution Requests + +*[New in Electra]* + +1. The execution payload is obtained from the execution engine as defined above using `payload_id`. The response also includes a `execution_requests` entry containing a list of bytes. Each element on the list corresponds to one ssz list of requests as defined +in [EIP-7685](https://eips.ethereum.org/EIPS/eip-7685). The index of each element in the array determines the type of request. +2. Set `block.body.execution_requests = get_execution_requests(execution_requests)`, where: + +```python +def get_execution_requests(execution_requests: list[bytes]) -> ExecutionRequests: + requests = ExecutionRequests() + + requests.deposits = deserialize(execution_requests[0], DepositRequest) + requests.withdrawals = deserialize(execution_requests[0], WithdrawalRequest) + requests.consolidations = deserialize(execution_requests[0], ConsolidationRequest) + + return requests +``` + ## Attesting ### Construct attestation From 1a2ff476d9c30497df1d872c3257049761343c99 Mon Sep 17 00:00:00 2001 From: Lucas Saldanha Date: Thu, 10 Oct 2024 12:42:59 +1300 Subject: [PATCH 049/142] Fix indexes --- specs/electra/validator.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/electra/validator.md b/specs/electra/validator.md index e6f0906d83..02de8727e7 100644 --- a/specs/electra/validator.md +++ b/specs/electra/validator.md @@ -192,8 +192,8 @@ def get_execution_requests(execution_requests: list[bytes]) -> ExecutionRequests requests = ExecutionRequests() requests.deposits = deserialize(execution_requests[0], DepositRequest) - requests.withdrawals = deserialize(execution_requests[0], WithdrawalRequest) - requests.consolidations = deserialize(execution_requests[0], ConsolidationRequest) + requests.withdrawals = deserialize(execution_requests[1], WithdrawalRequest) + requests.consolidations = deserialize(execution_requests[2], ConsolidationRequest) return requests ``` From a3153a53ca12d6054a5f9c3ffebcab284e27b442 Mon Sep 17 00:00:00 2001 From: Lucas Saldanha Date: Thu, 10 Oct 2024 17:27:19 +1300 Subject: [PATCH 050/142] Add deserialize fn --- pysetup/spec_builders/electra.py | 2 +- specs/electra/validator.md | 10 ++++------ tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 ++ 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pysetup/spec_builders/electra.py b/pysetup/spec_builders/electra.py index 48082249b6..d08dd40248 100644 --- a/pysetup/spec_builders/electra.py +++ b/pysetup/spec_builders/electra.py @@ -10,7 +10,7 @@ class ElectraSpecBuilder(BaseSpecBuilder): def imports(cls, preset_name: str): return f''' from eth2spec.deneb import {preset_name} as deneb -from eth2spec.utils.ssz.ssz_impl import serialize +from eth2spec.utils.ssz.ssz_impl import serialize, deserialize ''' @classmethod diff --git a/specs/electra/validator.md b/specs/electra/validator.md index 02de8727e7..1c715668ca 100644 --- a/specs/electra/validator.md +++ b/specs/electra/validator.md @@ -189,13 +189,11 @@ in [EIP-7685](https://eips.ethereum.org/EIPS/eip-7685). The index of each elemen ```python def get_execution_requests(execution_requests: list[bytes]) -> ExecutionRequests: - requests = ExecutionRequests() + deposits = deserialize(DepositRequest, execution_requests[0]) + withdrawals = deserialize(WithdrawalRequest, execution_requests[1]) + consolidations = deserialize(ConsolidationRequest, execution_requests[2]) - requests.deposits = deserialize(execution_requests[0], DepositRequest) - requests.withdrawals = deserialize(execution_requests[1], WithdrawalRequest) - requests.consolidations = deserialize(execution_requests[2], ConsolidationRequest) - - return requests + return ExecutionRequests(deposits, withdrawals, consolidations) ``` ## Attesting diff --git a/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py b/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py index 65808038ea..52164f593c 100644 --- a/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -8,6 +8,8 @@ def serialize(obj: View) -> bytes: return obj.encode_bytes() +def deserialize(cls: View, data: bytes) -> object: + return cls.decode_bytes(data) def hash_tree_root(obj: View) -> Bytes32: return Bytes32(obj.get_backing().merkle_root()) From 5eb77c2655a4528ec3aa13548e94a5b577a6ef44 Mon Sep 17 00:00:00 2001 From: Lucas Saldanha Date: Thu, 10 Oct 2024 17:33:00 +1300 Subject: [PATCH 051/142] fix deserialize typing --- specs/electra/validator.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/electra/validator.md b/specs/electra/validator.md index 1c715668ca..a2577f0343 100644 --- a/specs/electra/validator.md +++ b/specs/electra/validator.md @@ -189,9 +189,9 @@ in [EIP-7685](https://eips.ethereum.org/EIPS/eip-7685). The index of each elemen ```python def get_execution_requests(execution_requests: list[bytes]) -> ExecutionRequests: - deposits = deserialize(DepositRequest, execution_requests[0]) - withdrawals = deserialize(WithdrawalRequest, execution_requests[1]) - consolidations = deserialize(ConsolidationRequest, execution_requests[2]) + deposits = deserialize(List[DepositRequest, MAX_DEPOSIT_REQUESTS_PER_PAYLOAD], execution_requests[0]) + withdrawals = deserialize(List[WithdrawalRequest, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD], execution_requests[1]) + consolidations = deserialize(List[ConsolidationRequest, MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD], execution_requests[2]) return ExecutionRequests(deposits, withdrawals, consolidations) ``` From cd51168bdfd7ae0bc1773ba20119c2e4b92d4d58 Mon Sep 17 00:00:00 2001 From: Lucas Saldanha Date: Thu, 10 Oct 2024 17:36:36 +1300 Subject: [PATCH 052/142] fix toc --- specs/electra/validator.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/specs/electra/validator.md b/specs/electra/validator.md index a2577f0343..b35ec2cbc5 100644 --- a/specs/electra/validator.md +++ b/specs/electra/validator.md @@ -8,16 +8,22 @@ - [Introduction](#introduction) - [Prerequisites](#prerequisites) +- [Helpers](#helpers) + - [Modified `GetPayloadResponse`](#modified-getpayloadresponse) - [Containers](#containers) - [Modified Containers](#modified-containers) - [`AggregateAndProof`](#aggregateandproof) - [`SignedAggregateAndProof`](#signedaggregateandproof) +- [Protocol](#protocol) + - [`ExecutionEngine`](#executionengine) + - [Modified `get_payload`](#modified-get_payload) - [Block proposal](#block-proposal) - [Constructing the `BeaconBlockBody`](#constructing-the-beaconblockbody) - [Attester slashings](#attester-slashings) - [Attestations](#attestations) - [Deposits](#deposits) - [Execution payload](#execution-payload) + - [Execution Requests](#execution-requests) - [Attesting](#attesting) - [Construct attestation](#construct-attestation) - [Attestation aggregation](#attestation-aggregation) From 16e6085a3c007af23577d827f2b5aaa02dd08dc8 Mon Sep 17 00:00:00 2001 From: Lucas Saldanha Date: Thu, 10 Oct 2024 18:06:57 +1300 Subject: [PATCH 053/142] Change list -> List --- specs/electra/validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/electra/validator.md b/specs/electra/validator.md index b35ec2cbc5..520862ed47 100644 --- a/specs/electra/validator.md +++ b/specs/electra/validator.md @@ -194,7 +194,7 @@ in [EIP-7685](https://eips.ethereum.org/EIPS/eip-7685). The index of each elemen 2. Set `block.body.execution_requests = get_execution_requests(execution_requests)`, where: ```python -def get_execution_requests(execution_requests: list[bytes]) -> ExecutionRequests: +def get_execution_requests(execution_requests: List[bytes]) -> ExecutionRequests: deposits = deserialize(List[DepositRequest, MAX_DEPOSIT_REQUESTS_PER_PAYLOAD], execution_requests[0]) withdrawals = deserialize(List[WithdrawalRequest, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD], execution_requests[1]) consolidations = deserialize(List[ConsolidationRequest, MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD], execution_requests[2]) From bc3ee13797c0c9f976784e8a2fc66ff7f66bd46b Mon Sep 17 00:00:00 2001 From: Lucas Saldanha Date: Thu, 10 Oct 2024 19:22:40 +1300 Subject: [PATCH 054/142] Yet another change list -> List --- specs/electra/validator.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/electra/validator.md b/specs/electra/validator.md index 520862ed47..016383c448 100644 --- a/specs/electra/validator.md +++ b/specs/electra/validator.md @@ -54,7 +54,7 @@ class GetPayloadResponse(object): execution_payload: ExecutionPayload block_value: uint256 blobs_bundle: BlobsBundle - execution_requests: list[bytes] # [New in Electra] + execution_requests: List[bytes] # [New in Electra] ``` ## Containers @@ -90,7 +90,7 @@ has been built since the corresponding call to `notify_forkchoice_updated` metho ```python def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> GetPayloadResponse: """ - Return ExecutionPayload, uint256, BlobsBundle and list[bytes] objects. + Return ExecutionPayload, uint256, BlobsBundle and List[bytes] objects. """ # pylint: disable=unused-argument ... From 2a163adda29e411b8a52a51fd7004b9392b6a974 Mon Sep 17 00:00:00 2001 From: Lucas Saldanha Date: Thu, 10 Oct 2024 19:30:10 +1300 Subject: [PATCH 055/142] Fix lint errors --- specs/electra/validator.md | 3 ++- tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/specs/electra/validator.md b/specs/electra/validator.md index 016383c448..169d3cdf5a 100644 --- a/specs/electra/validator.md +++ b/specs/electra/validator.md @@ -197,7 +197,8 @@ in [EIP-7685](https://eips.ethereum.org/EIPS/eip-7685). The index of each elemen def get_execution_requests(execution_requests: List[bytes]) -> ExecutionRequests: deposits = deserialize(List[DepositRequest, MAX_DEPOSIT_REQUESTS_PER_PAYLOAD], execution_requests[0]) withdrawals = deserialize(List[WithdrawalRequest, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD], execution_requests[1]) - consolidations = deserialize(List[ConsolidationRequest, MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD], execution_requests[2]) + consolidations = deserialize(List[ConsolidationRequest, MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD], + execution_requests[2]) return ExecutionRequests(deposits, withdrawals, consolidations) ``` diff --git a/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py b/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py index 52164f593c..1edeb05d29 100644 --- a/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -8,9 +8,11 @@ def serialize(obj: View) -> bytes: return obj.encode_bytes() + def deserialize(cls: View, data: bytes) -> object: return cls.decode_bytes(data) + def hash_tree_root(obj: View) -> Bytes32: return Bytes32(obj.get_backing().merkle_root()) From 04a40d2afbbce741c13f72feb98433879d2db291 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Thu, 10 Oct 2024 08:39:06 -0500 Subject: [PATCH 056/142] Use Sequence for serialized execution requests --- specs/electra/beacon-chain.md | 6 +++--- specs/electra/validator.md | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/specs/electra/beacon-chain.md b/specs/electra/beacon-chain.md index 7c6d9fe1fa..8dca8efa4e 100644 --- a/specs/electra/beacon-chain.md +++ b/specs/electra/beacon-chain.md @@ -992,9 +992,9 @@ class NewPayloadRequest(object): def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayload, parent_beacon_block_root: Root, - execution_requests_list: list[bytes]) -> bool: + execution_requests_list: Sequence[bytes]) -> bool: """ - Return ``True`` if and only if ``execution_payload`` and ``execution_requests`` + Return ``True`` if and only if ``execution_payload`` and ``execution_requests`` are valid with respect to ``self.execution_state``. """ ... @@ -1145,7 +1145,7 @@ def process_withdrawals(state: BeaconState, payload: ExecutionPayload) -> None: *Note*: Encodes execution requests as defined by [EIP-7685](https://eips.ethereum.org/EIPS/eip-7685). ```python -def get_execution_requests_list(execution_requests: ExecutionRequests) -> list[bytes]: +def get_execution_requests_list(execution_requests: ExecutionRequests) -> Sequence[bytes]: deposit_bytes = serialize(execution_requests.deposits) withdrawal_bytes = serialize(execution_requests.withdrawals) consolidation_bytes = serialize(execution_requests.consolidations) diff --git a/specs/electra/validator.md b/specs/electra/validator.md index 169d3cdf5a..fe9c143d02 100644 --- a/specs/electra/validator.md +++ b/specs/electra/validator.md @@ -54,7 +54,7 @@ class GetPayloadResponse(object): execution_payload: ExecutionPayload block_value: uint256 blobs_bundle: BlobsBundle - execution_requests: List[bytes] # [New in Electra] + execution_requests: Sequence[bytes] # [New in Electra] ``` ## Containers @@ -90,7 +90,7 @@ has been built since the corresponding call to `notify_forkchoice_updated` metho ```python def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> GetPayloadResponse: """ - Return ExecutionPayload, uint256, BlobsBundle and List[bytes] objects. + Return ExecutionPayload, uint256, BlobsBundle and Sequence[bytes] objects. """ # pylint: disable=unused-argument ... @@ -194,10 +194,10 @@ in [EIP-7685](https://eips.ethereum.org/EIPS/eip-7685). The index of each elemen 2. Set `block.body.execution_requests = get_execution_requests(execution_requests)`, where: ```python -def get_execution_requests(execution_requests: List[bytes]) -> ExecutionRequests: +def get_execution_requests(execution_requests: Sequence[bytes]) -> ExecutionRequests: deposits = deserialize(List[DepositRequest, MAX_DEPOSIT_REQUESTS_PER_PAYLOAD], execution_requests[0]) withdrawals = deserialize(List[WithdrawalRequest, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD], execution_requests[1]) - consolidations = deserialize(List[ConsolidationRequest, MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD], + consolidations = deserialize(List[ConsolidationRequest, MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD], execution_requests[2]) return ExecutionRequests(deposits, withdrawals, consolidations) From 5e9d27dc3232ed93bdd2d66e653edbed8bde5fee Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Thu, 10 Oct 2024 08:45:10 -0500 Subject: [PATCH 057/142] Use Sequence in NoopExecutionEngine too --- pysetup/spec_builders/electra.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysetup/spec_builders/electra.py b/pysetup/spec_builders/electra.py index d08dd40248..23237dce91 100644 --- a/pysetup/spec_builders/electra.py +++ b/pysetup/spec_builders/electra.py @@ -30,7 +30,7 @@ class NoopExecutionEngine(ExecutionEngine): def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayload, parent_beacon_block_root: Root, - execution_requests_list: list[bytes]) -> bool: + execution_requests_list: Sequence[bytes]) -> bool: return True def notify_forkchoice_updated(self: ExecutionEngine, From 0ecc87164b0b09f9659800ab45214408d00f8028 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Thu, 10 Oct 2024 09:43:50 -0500 Subject: [PATCH 058/142] Update deserialize() definition --- tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py b/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py index 1edeb05d29..a76cf80f90 100644 --- a/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from typing import TypeVar from remerkleable.basic import uint -from remerkleable.core import View +from remerkleable.core import Type, View from remerkleable.byte_arrays import Bytes32 @@ -9,8 +9,8 @@ def serialize(obj: View) -> bytes: return obj.encode_bytes() -def deserialize(cls: View, data: bytes) -> object: - return cls.decode_bytes(data) +def deserialize(typ: Type[View], data: bytes) -> View: + return typ.decode_bytes(data) def hash_tree_root(obj: View) -> Bytes32: From f9ae4c5501b7f1ddbcbca01c507f5e69dcf83a2c Mon Sep 17 00:00:00 2001 From: Justin Traglia <95511699+jtraglia@users.noreply.github.com> Date: Thu, 10 Oct 2024 13:29:12 -0500 Subject: [PATCH 059/142] Clarify execution requests return Co-authored-by: Alex Stokes --- specs/electra/validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/electra/validator.md b/specs/electra/validator.md index fe9c143d02..29293d301c 100644 --- a/specs/electra/validator.md +++ b/specs/electra/validator.md @@ -90,7 +90,7 @@ has been built since the corresponding call to `notify_forkchoice_updated` metho ```python def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> GetPayloadResponse: """ - Return ExecutionPayload, uint256, BlobsBundle and Sequence[bytes] objects. + Return ExecutionPayload, uint256, BlobsBundle and execution requests (as Sequence[bytes]) objects. """ # pylint: disable=unused-argument ... From 83c04b8469c6a9f930795313914353f4db9f6817 Mon Sep 17 00:00:00 2001 From: Justin Traglia <95511699+jtraglia@users.noreply.github.com> Date: Thu, 10 Oct 2024 13:29:27 -0500 Subject: [PATCH 060/142] Capitalize SSZ Co-authored-by: Alex Stokes --- specs/electra/validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/electra/validator.md b/specs/electra/validator.md index 29293d301c..4622ff4805 100644 --- a/specs/electra/validator.md +++ b/specs/electra/validator.md @@ -189,7 +189,7 @@ def prepare_execution_payload(state: BeaconState, *[New in Electra]* -1. The execution payload is obtained from the execution engine as defined above using `payload_id`. The response also includes a `execution_requests` entry containing a list of bytes. Each element on the list corresponds to one ssz list of requests as defined +1. The execution payload is obtained from the execution engine as defined above using `payload_id`. The response also includes a `execution_requests` entry containing a list of bytes. Each element on the list corresponds to one SSZ list of requests as defined in [EIP-7685](https://eips.ethereum.org/EIPS/eip-7685). The index of each element in the array determines the type of request. 2. Set `block.body.execution_requests = get_execution_requests(execution_requests)`, where: From 6416a562abb6085b87889e9c202cd3d2cb7d2b39 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Thu, 10 Oct 2024 13:41:02 -0500 Subject: [PATCH 061/142] Add ssz_{de,}serialize aliases --- pysetup/spec_builders/electra.py | 2 +- specs/electra/beacon-chain.md | 6 +++--- specs/electra/validator.md | 8 ++++---- tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py | 12 ++++++++++-- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/pysetup/spec_builders/electra.py b/pysetup/spec_builders/electra.py index 23237dce91..6746d9aada 100644 --- a/pysetup/spec_builders/electra.py +++ b/pysetup/spec_builders/electra.py @@ -10,7 +10,7 @@ class ElectraSpecBuilder(BaseSpecBuilder): def imports(cls, preset_name: str): return f''' from eth2spec.deneb import {preset_name} as deneb -from eth2spec.utils.ssz.ssz_impl import serialize, deserialize +from eth2spec.utils.ssz.ssz_impl import ssz_serialize, ssz_deserialize ''' @classmethod diff --git a/specs/electra/beacon-chain.md b/specs/electra/beacon-chain.md index 8dca8efa4e..aff2167cfc 100644 --- a/specs/electra/beacon-chain.md +++ b/specs/electra/beacon-chain.md @@ -1146,9 +1146,9 @@ def process_withdrawals(state: BeaconState, payload: ExecutionPayload) -> None: ```python def get_execution_requests_list(execution_requests: ExecutionRequests) -> Sequence[bytes]: - deposit_bytes = serialize(execution_requests.deposits) - withdrawal_bytes = serialize(execution_requests.withdrawals) - consolidation_bytes = serialize(execution_requests.consolidations) + deposit_bytes = ssz_serialize(execution_requests.deposits) + withdrawal_bytes = ssz_serialize(execution_requests.withdrawals) + consolidation_bytes = ssz_serialize(execution_requests.consolidations) return [deposit_bytes, withdrawal_bytes, consolidation_bytes] ``` diff --git a/specs/electra/validator.md b/specs/electra/validator.md index 4622ff4805..ae8d53c8f0 100644 --- a/specs/electra/validator.md +++ b/specs/electra/validator.md @@ -195,10 +195,10 @@ in [EIP-7685](https://eips.ethereum.org/EIPS/eip-7685). The index of each elemen ```python def get_execution_requests(execution_requests: Sequence[bytes]) -> ExecutionRequests: - deposits = deserialize(List[DepositRequest, MAX_DEPOSIT_REQUESTS_PER_PAYLOAD], execution_requests[0]) - withdrawals = deserialize(List[WithdrawalRequest, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD], execution_requests[1]) - consolidations = deserialize(List[ConsolidationRequest, MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD], - execution_requests[2]) + deposits = ssz_deserialize(List[DepositRequest, MAX_DEPOSIT_REQUESTS_PER_PAYLOAD], execution_requests[0]) + withdrawals = ssz_deserialize(List[WithdrawalRequest, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD], execution_requests[1]) + consolidations = ssz_deserialize(List[ConsolidationRequest, MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD], + execution_requests[2]) return ExecutionRequests(deposits, withdrawals, consolidations) ``` diff --git a/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py b/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py index a76cf80f90..645cbff775 100644 --- a/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -5,14 +5,22 @@ from remerkleable.byte_arrays import Bytes32 -def serialize(obj: View) -> bytes: +def ssz_serialize(obj: View) -> bytes: return obj.encode_bytes() -def deserialize(typ: Type[View], data: bytes) -> View: +def serialize(obj: View) -> bytes: + return ssz_serialize(obj) + + +def ssz_deserialize(typ: Type[View], data: bytes) -> View: return typ.decode_bytes(data) +def deserialize(typ: Type[View], data: bytes) -> View: + return ssz_deserialize(typ, data) + + def hash_tree_root(obj: View) -> Bytes32: return Bytes32(obj.get_backing().merkle_root()) From c051f4f02ae0beee31fff02492beb6f13a345199 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Fri, 11 Oct 2024 10:34:31 -0500 Subject: [PATCH 062/142] Fix validator fields in get_validator_from_deposit --- specs/electra/beacon-chain.md | 5 +++-- specs/phase0/beacon-chain.md | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/specs/electra/beacon-chain.md b/specs/electra/beacon-chain.md index 7c6d9fe1fa..94dce11a68 100644 --- a/specs/electra/beacon-chain.md +++ b/specs/electra/beacon-chain.md @@ -994,7 +994,7 @@ def notify_new_payload(self: ExecutionEngine, parent_beacon_block_root: Root, execution_requests_list: list[bytes]) -> bool: """ - Return ``True`` if and only if ``execution_payload`` and ``execution_requests`` + Return ``True`` if and only if ``execution_payload`` and ``execution_requests`` are valid with respect to ``self.execution_state``. """ ... @@ -1290,11 +1290,12 @@ def get_validator_from_deposit(pubkey: BLSPubkey, withdrawal_credentials: Bytes3 validator = Validator( pubkey=pubkey, withdrawal_credentials=withdrawal_credentials, + effective_balance=Gwei(0), + slashed=False, activation_eligibility_epoch=FAR_FUTURE_EPOCH, activation_epoch=FAR_FUTURE_EPOCH, exit_epoch=FAR_FUTURE_EPOCH, withdrawable_epoch=FAR_FUTURE_EPOCH, - effective_balance=Gwei(0), ) # [Modified in Electra:EIP7251] diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 9d5f8446a0..3d860d4a3e 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1854,11 +1854,12 @@ def get_validator_from_deposit(pubkey: BLSPubkey, withdrawal_credentials: Bytes3 return Validator( pubkey=pubkey, withdrawal_credentials=withdrawal_credentials, + effective_balance=effective_balance, + slashed=False, activation_eligibility_epoch=FAR_FUTURE_EPOCH, activation_epoch=FAR_FUTURE_EPOCH, exit_epoch=FAR_FUTURE_EPOCH, withdrawable_epoch=FAR_FUTURE_EPOCH, - effective_balance=effective_balance, ) ``` From 179cde6780c677a1293c4f075ac749787715aaca Mon Sep 17 00:00:00 2001 From: Justin Traglia <95511699+jtraglia@users.noreply.github.com> Date: Mon, 14 Oct 2024 10:42:57 -0500 Subject: [PATCH 063/142] Add new `check_mods` function for generators (#3970) * Add new check_mods function for generators * Use single qoutes for consistency * Add [ERROR] to the exception & update readme * Fix typos * Fix lint --- .../gen_helpers/gen_from_tests/gen.py | 67 +++++++++++++++++++ tests/generators/README.md | 1 + tests/generators/epoch_processing/main.py | 3 +- tests/generators/finality/main.py | 3 +- tests/generators/fork_choice/main.py | 3 +- tests/generators/genesis/main.py | 3 +- tests/generators/light_client/main.py | 3 +- tests/generators/merkle_proof/main.py | 3 +- tests/generators/networking/main.py | 3 +- tests/generators/operations/main.py | 3 +- tests/generators/random/main.py | 3 +- tests/generators/rewards/main.py | 3 +- tests/generators/sanity/main.py | 3 +- tests/generators/sync/main.py | 3 +- 14 files changed, 92 insertions(+), 12 deletions(-) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py index 153d6eee49..6f73f7155d 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py @@ -1,5 +1,6 @@ from importlib import import_module from inspect import getmembers, isfunction +from pkgutil import walk_packages from typing import Any, Callable, Dict, Iterable, Optional, List, Union from eth2spec.utils import bls @@ -134,3 +135,69 @@ def combine_mods(dict_1, dict_2): dict_3[key].append(dict_1[key]) return dict_3 + + +def check_mods(all_mods, pkg): + """ + Raise an exception if there is a missing/unexpected module in all_mods. + """ + def get_expected_modules(package, absolute=False): + """ + Return all modules (which are not packages) inside the given package. + """ + modules = [] + eth2spec = import_module('eth2spec') + prefix = eth2spec.__name__ + '.' + for _, modname, ispkg in walk_packages(eth2spec.__path__, prefix): + s = package if absolute else f'.{package}.' + if s in modname and not ispkg: + modules.append(modname) + return modules + + mods = [] + for fork in all_mods: + for mod in all_mods[fork].values(): + # If this key has a single value, normalize to list. + if isinstance(mod, str): + mod = [mod] + + # For each submodule, check if it is package. + # This is a "trick" we do to reuse a test format. + for sub in mod: + is_package = '.test_' not in sub + if is_package: + mods.extend(get_expected_modules(sub, absolute=True)) + else: + mods.append(sub) + + problems = [] + expected_mods = get_expected_modules(pkg) + if mods != expected_mods: + for e in expected_mods: + # Skip forks which are not in all_mods. + # The fork name is the 3rd item in the path. + fork = e.split('.')[2] + if fork not in all_mods: + continue + # Skip modules in the unittests package. + # These are not associated with generators. + if '.unittests.' in e: + continue + # The expected module is not in our list of modules. + # Add it to our list of problems. + if e not in mods: + problems.append('missing: ' + e) + + for t in mods: + # Skip helper modules. + # These do not define test functions. + if t.startswith('eth2spec.test.helpers'): + continue + # There is a module not defined in eth2spec. + # Add it to our list of problems. + if t not in expected_mods: + print('unexpected:', t) + problems.append('unexpected: ' + t) + + if problems: + raise Exception('[ERROR] module problems:\n ' + '\n '.join(problems)) diff --git a/tests/generators/README.md b/tests/generators/README.md index 0146ca35e8..148415f4de 100644 --- a/tests/generators/README.md +++ b/tests/generators/README.md @@ -185,6 +185,7 @@ if __name__ == "__main__": PHASE0: phase_0_mods, ALTAIR: altair_mods, } + check_mods(all_mods, "sanity") run_state_test_generators(runner_name="sanity", all_mods=all_mods) ``` diff --git a/tests/generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py index 1de3e84e3d..4c848a9d12 100644 --- a/tests/generators/epoch_processing/main.py +++ b/tests/generators/epoch_processing/main.py @@ -1,4 +1,4 @@ -from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods, check_mods from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA @@ -63,5 +63,6 @@ DENEB: deneb_mods, ELECTRA: electra_mods, } + check_mods(all_mods, "epoch_processing") run_state_test_generators(runner_name="epoch_processing", all_mods=all_mods) diff --git a/tests/generators/finality/main.py b/tests/generators/finality/main.py index 6d33ae6bdc..b31e949421 100644 --- a/tests/generators/finality/main.py +++ b/tests/generators/finality/main.py @@ -1,4 +1,4 @@ -from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, check_mods from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA @@ -18,5 +18,6 @@ DENEB: deneb_mods, ELECTRA: electra_mods, } + check_mods(all_mods, "finality") run_state_test_generators(runner_name="finality", all_mods=all_mods) diff --git a/tests/generators/fork_choice/main.py b/tests/generators/fork_choice/main.py index 92c67e4c0b..10a52fb954 100644 --- a/tests/generators/fork_choice/main.py +++ b/tests/generators/fork_choice/main.py @@ -1,4 +1,4 @@ -from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods, check_mods from eth2spec.test.helpers.constants import ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA @@ -37,5 +37,6 @@ DENEB: deneb_mods, ELECTRA: electra_mods, } + check_mods(all_mods, "fork_choice") run_state_test_generators(runner_name="fork_choice", all_mods=all_mods) diff --git a/tests/generators/genesis/main.py b/tests/generators/genesis/main.py index 57b680ebac..9e82d1c112 100644 --- a/tests/generators/genesis/main.py +++ b/tests/generators/genesis/main.py @@ -1,4 +1,4 @@ -from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods, check_mods from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA @@ -26,5 +26,6 @@ DENEB: deneb_mods, ELECTRA: electra_mods, } + check_mods(all_mods, "genesis") run_state_test_generators(runner_name="genesis", all_mods=all_mods) diff --git a/tests/generators/light_client/main.py b/tests/generators/light_client/main.py index a3cdfd62fd..23aed84775 100644 --- a/tests/generators/light_client/main.py +++ b/tests/generators/light_client/main.py @@ -1,5 +1,5 @@ from eth2spec.test.helpers.constants import ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA -from eth2spec.gen_helpers.gen_from_tests.gen import combine_mods, run_state_test_generators +from eth2spec.gen_helpers.gen_from_tests.gen import combine_mods, run_state_test_generators, check_mods if __name__ == "__main__": @@ -24,5 +24,6 @@ DENEB: deneb_mods, ELECTRA: electra_mods, } + check_mods(all_mods, "light_client") run_state_test_generators(runner_name="light_client", all_mods=all_mods) diff --git a/tests/generators/merkle_proof/main.py b/tests/generators/merkle_proof/main.py index 69500137ab..f05ddaa6e6 100644 --- a/tests/generators/merkle_proof/main.py +++ b/tests/generators/merkle_proof/main.py @@ -1,5 +1,5 @@ from eth2spec.test.helpers.constants import DENEB, ELECTRA, EIP7594 -from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods, check_mods if __name__ == "__main__": @@ -17,5 +17,6 @@ ELECTRA: electra_mods, EIP7594: eip_7594_mods, } + check_mods(all_mods, "merkle_proof") run_state_test_generators(runner_name="merkle_proof", all_mods=all_mods) diff --git a/tests/generators/networking/main.py b/tests/generators/networking/main.py index 2681daf68b..52b94929f7 100644 --- a/tests/generators/networking/main.py +++ b/tests/generators/networking/main.py @@ -1,6 +1,6 @@ from eth2spec.test.helpers.constants import EIP7594 -from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, check_mods if __name__ == "__main__": @@ -10,5 +10,6 @@ all_mods = { EIP7594: eip7594_mods } + check_mods(all_mods, "networking") run_state_test_generators(runner_name="networking", all_mods=all_mods) diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index ae66843f61..9e3a7c21a4 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -1,4 +1,4 @@ -from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods, check_mods from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA @@ -61,5 +61,6 @@ DENEB: deneb_mods, ELECTRA: electra_mods, } + check_mods(all_mods, "block_processing") run_state_test_generators(runner_name="operations", all_mods=all_mods) diff --git a/tests/generators/random/main.py b/tests/generators/random/main.py index 1c9472b5ad..1d176c03ca 100644 --- a/tests/generators/random/main.py +++ b/tests/generators/random/main.py @@ -1,7 +1,7 @@ from eth2spec.test.helpers.constants import ( PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA, ) -from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, check_mods if __name__ == "__main__": @@ -32,5 +32,6 @@ DENEB: deneb_mods, ELECTRA: electra_mods, } + check_mods(all_mods, "random") run_state_test_generators(runner_name="random", all_mods=all_mods) diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index dcf2c6ce23..f1b27133a9 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -1,4 +1,4 @@ -from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, check_mods from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA @@ -27,5 +27,6 @@ DENEB: deneb_mods, ELECTRA: electra_mods, } + check_mods(all_mods, "rewards") run_state_test_generators(runner_name="rewards", all_mods=all_mods) diff --git a/tests/generators/sanity/main.py b/tests/generators/sanity/main.py index 7b1eff0324..8039b82a44 100644 --- a/tests/generators/sanity/main.py +++ b/tests/generators/sanity/main.py @@ -1,5 +1,5 @@ from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA -from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods, check_mods if __name__ == "__main__": @@ -44,5 +44,6 @@ DENEB: deneb_mods, ELECTRA: electra_mods, } + check_mods(all_mods, "sanity") run_state_test_generators(runner_name="sanity", all_mods=all_mods) diff --git a/tests/generators/sync/main.py b/tests/generators/sync/main.py index 94e392b775..fd2f3f0209 100644 --- a/tests/generators/sync/main.py +++ b/tests/generators/sync/main.py @@ -1,4 +1,4 @@ -from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, check_mods from eth2spec.test.helpers.constants import BELLATRIX, CAPELLA, DENEB, ELECTRA @@ -16,5 +16,6 @@ DENEB: deneb_mods, ELECTRA: electra_mods, } + check_mods(all_mods, "sync") run_state_test_generators(runner_name="sync", all_mods=all_mods) From 8a2277660f4bf0116129e89014ec66570b82a90b Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Mon, 14 Oct 2024 11:12:32 -0500 Subject: [PATCH 064/142] Also install preinstallation requirements --- solidity_deposit_contract/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/solidity_deposit_contract/Makefile b/solidity_deposit_contract/Makefile index 164df20a99..a353d931bb 100644 --- a/solidity_deposit_contract/Makefile +++ b/solidity_deposit_contract/Makefile @@ -30,6 +30,7 @@ install_deposit_contract_web3_tester: @cd $(DEPOSIT_CONTRACT_TESTER_DIR); \ python3 -m venv venv; \ source venv/bin/activate; \ + python3 -m pip install -r ../../requirements_preinstallation.txt; \ python3 -m pip install -r requirements.txt test_deposit_contract_web3_tests: @@ -38,4 +39,4 @@ test_deposit_contract_web3_tests: python3 -m pytest . clean: - @git clean -fdx \ No newline at end of file + @git clean -fdx From dcdf30e2fe5a75a3f73ef4a87581a0af0bd0820f Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Wed, 16 Oct 2024 17:30:33 +0600 Subject: [PATCH 065/142] Bugfix and more withdrawal tests --- specs/electra/beacon-chain.md | 16 +- .../test_process_withdrawals.py | 353 +++++++++++++++++- .../eth2spec/test/helpers/withdrawals.py | 59 +-- 3 files changed, 387 insertions(+), 41 deletions(-) diff --git a/specs/electra/beacon-chain.md b/specs/electra/beacon-chain.md index 0c56491907..1bf78851c7 100644 --- a/specs/electra/beacon-chain.md +++ b/specs/electra/beacon-chain.md @@ -1056,7 +1056,7 @@ def get_expected_withdrawals(state: BeaconState) -> Tuple[Sequence[Withdrawal], withdrawal_index = state.next_withdrawal_index validator_index = state.next_withdrawal_validator_index withdrawals: List[Withdrawal] = [] - partial_withdrawals_count = 0 + processed_partial_withdrawals_count = 0 # [New in Electra:EIP7251] Consume pending partial withdrawals for withdrawal in state.pending_partial_withdrawals: @@ -1076,13 +1076,16 @@ def get_expected_withdrawals(state: BeaconState) -> Tuple[Sequence[Withdrawal], )) withdrawal_index += WithdrawalIndex(1) - partial_withdrawals_count += 1 + processed_partial_withdrawals_count += 1 # Sweep for remaining. bound = min(len(state.validators), MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP) for _ in range(bound): validator = state.validators[validator_index] - balance = state.balances[validator_index] + # [Modified in Electra:EIP7251] + partially_withdrawn_balance = sum( + withdrawal.amount for withdrawal in withdrawals if withdrawal.validator_index == validator_index) + balance = state.balances[validator_index] - partially_withdrawn_balance if is_fully_withdrawable_validator(validator, balance, epoch): withdrawals.append(Withdrawal( index=withdrawal_index, @@ -1102,7 +1105,7 @@ def get_expected_withdrawals(state: BeaconState) -> Tuple[Sequence[Withdrawal], if len(withdrawals) == MAX_WITHDRAWALS_PER_PAYLOAD: break validator_index = ValidatorIndex((validator_index + 1) % len(state.validators)) - return withdrawals, partial_withdrawals_count + return withdrawals, processed_partial_withdrawals_count ``` ##### Modified `process_withdrawals` @@ -1111,7 +1114,8 @@ def get_expected_withdrawals(state: BeaconState) -> Tuple[Sequence[Withdrawal], ```python def process_withdrawals(state: BeaconState, payload: ExecutionPayload) -> None: - expected_withdrawals, partial_withdrawals_count = get_expected_withdrawals(state) # [Modified in Electra:EIP7251] + # [Modified in Electra:EIP7251] + expected_withdrawals, processed_partial_withdrawals_count = get_expected_withdrawals(state) assert payload.withdrawals == expected_withdrawals @@ -1119,7 +1123,7 @@ def process_withdrawals(state: BeaconState, payload: ExecutionPayload) -> None: decrease_balance(state, withdrawal.validator_index, withdrawal.amount) # Update pending partial withdrawals [New in Electra:EIP7251] - state.pending_partial_withdrawals = state.pending_partial_withdrawals[partial_withdrawals_count:] + state.pending_partial_withdrawals = state.pending_partial_withdrawals[processed_partial_withdrawals_count:] # Update the next withdrawal index if this block contained withdrawals if len(expected_withdrawals) != 0: diff --git a/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_withdrawals.py b/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_withdrawals.py index 555eae85b5..c7ba00641b 100644 --- a/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_withdrawals.py +++ b/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_withdrawals.py @@ -11,7 +11,7 @@ next_slot, ) from eth2spec.test.helpers.withdrawals import ( - prepare_expected_withdrawals_compounding, + prepare_expected_withdrawals, run_withdrawals_processing, set_compounding_withdrawal_credential_with_balance, prepare_pending_withdrawal, @@ -23,11 +23,11 @@ def test_success_mixed_fully_and_partial_withdrawable_compounding(spec, state): num_full_withdrawals = spec.MAX_WITHDRAWALS_PER_PAYLOAD // 2 num_partial_withdrawals = spec.MAX_WITHDRAWALS_PER_PAYLOAD - num_full_withdrawals - fully_withdrawable_indices, partial_withdrawals_indices = prepare_expected_withdrawals_compounding( + fully_withdrawable_indices, partial_withdrawals_indices = prepare_expected_withdrawals( spec, state, rng=random.Random(42), - num_full_withdrawals=num_full_withdrawals, - num_partial_withdrawals_sweep=num_partial_withdrawals, + num_full_withdrawals_comp=num_full_withdrawals, + num_partial_withdrawals_comp=num_partial_withdrawals, ) next_slot(spec, state) @@ -94,14 +94,351 @@ def test_pending_withdrawals_one_skipped_one_effective(spec, state): index_0 = 3 index_1 = 5 - withdrawal_0 = prepare_pending_withdrawal(spec, state, index_0) - withdrawal_1 = prepare_pending_withdrawal(spec, state, index_1) + pending_withdrawal_0 = prepare_pending_withdrawal(spec, state, index_0) + pending_withdrawal_1 = prepare_pending_withdrawal(spec, state, index_1) # If validator doesn't have an excess balance pending withdrawal is skipped state.balances[index_0] = spec.MIN_ACTIVATION_BALANCE execution_payload = build_empty_execution_payload(spec, state) - assert state.pending_partial_withdrawals == [withdrawal_0, withdrawal_1] - yield from run_withdrawals_processing(spec, state, execution_payload, num_expected_withdrawals=1) + assert state.pending_partial_withdrawals == [pending_withdrawal_0, pending_withdrawal_1] + yield from run_withdrawals_processing( + spec, state, + execution_payload, + num_expected_withdrawals=1, + pending_withdrawal_requests=[pending_withdrawal_1] + ) + + assert state.pending_partial_withdrawals == [] + + +@with_electra_and_later +@spec_state_test +def test_pending_withdrawals_next_epoch(spec, state): + validator_index = len(state.validators) // 2 + next_epoch = spec.get_current_epoch(state) + 1 + + pending_withdrawal = prepare_pending_withdrawal(spec, state, validator_index, withdrawable_epoch=next_epoch) + + execution_payload = build_empty_execution_payload(spec, state) + yield from run_withdrawals_processing(spec, state, execution_payload, num_expected_withdrawals=0) + + assert state.pending_partial_withdrawals == [pending_withdrawal] + + +@with_electra_and_later +@spec_state_test +def test_pending_withdrawals_at_max(spec, state): + pending_withdrawal_requests = [] + # Create spec.MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP + 1 partial withdrawals + for i in range(0, spec.MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP + 1): + pending_withdrawal = prepare_pending_withdrawal(spec, state, i) + pending_withdrawal_requests.append(pending_withdrawal) + + assert len(state.pending_partial_withdrawals) == spec.MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP + 1 + + execution_payload = build_empty_execution_payload(spec, state) + yield from run_withdrawals_processing( + spec, state, + execution_payload, + num_expected_withdrawals=spec.MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP, + pending_withdrawal_requests=pending_withdrawal_requests[:spec.MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP] + ) + + withdrawals_exceeding_max = pending_withdrawal_requests[spec.MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP:] + assert state.pending_partial_withdrawals == withdrawals_exceeding_max + + +@with_electra_and_later +@spec_state_test +def test_pending_withdrawals_exiting_validator(spec, state): + validator_index = len(state.validators) // 2 + + pending_withdrawal = prepare_pending_withdrawal(spec, state, validator_index) + spec.initiate_validator_exit(state, pending_withdrawal.index) + + execution_payload = build_empty_execution_payload(spec, state) + yield from run_withdrawals_processing(spec, state, execution_payload, num_expected_withdrawals=0) + + assert state.pending_partial_withdrawals == [] + + +@with_electra_and_later +@spec_state_test +def test_pending_withdrawals_low_effective_balance(spec, state): + validator_index = len(state.validators) // 2 + + pending_withdrawal = prepare_pending_withdrawal(spec, state, validator_index) + state.validators[pending_withdrawal.index].effective_balance = ( + spec.MIN_ACTIVATION_BALANCE - spec.EFFECTIVE_BALANCE_INCREMENT + ) + + execution_payload = build_empty_execution_payload(spec, state) + yield from run_withdrawals_processing(spec, state, execution_payload, num_expected_withdrawals=0) + + assert state.pending_partial_withdrawals == [] + + +@with_electra_and_later +@spec_state_test +def test_pending_withdrawals_no_excess_balance(spec, state): + validator_index = len(state.validators) // 2 + + pending_withdrawal = prepare_pending_withdrawal(spec, state, validator_index) + state.balances[pending_withdrawal.index] = spec.MIN_ACTIVATION_BALANCE + + execution_payload = build_empty_execution_payload(spec, state) + yield from run_withdrawals_processing(spec, state, execution_payload, num_expected_withdrawals=0) + + assert state.pending_partial_withdrawals == [] + + +@with_electra_and_later +@spec_state_test +def test_pending_withdrawals_with_ineffective_sweep_on_top(spec, state): + # Ensure validator will be processed by the sweep + validator_index = min(len(state.validators), spec.MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP) // 2 + + pending_withdrawal = prepare_pending_withdrawal( + spec, state, + validator_index, + effective_balance=spec.MAX_EFFECTIVE_BALANCE_ELECTRA, + ) + + # Check that validator is partially withdrawable before pending withdrawal is processed + assert spec.is_partially_withdrawable_validator( + state.validators[validator_index], + state.balances[validator_index] + ) + # And is not partially withdrawable thereafter + assert not spec.is_partially_withdrawable_validator( + state.validators[validator_index], + state.balances[validator_index] - pending_withdrawal.amount + ) + + next_slot(spec, state) + execution_payload = build_empty_execution_payload(spec, state) + yield from run_withdrawals_processing( + spec, state, + execution_payload, + num_expected_withdrawals=1, + fully_withdrawable_indices=[], + partial_withdrawals_indices=[], + pending_withdrawal_requests=[pending_withdrawal] + ) + + assert state.pending_partial_withdrawals == [] + + +@with_electra_and_later +@spec_state_test +def test_pending_withdrawals_with_ineffective_sweep_on_top_2(spec, state): + # Ensure validator will be processed by the sweep + validator_index = min(len(state.validators), spec.MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP) // 2 + + pending_withdrawal_0 = prepare_pending_withdrawal( + spec, state, + validator_index, + effective_balance=spec.MAX_EFFECTIVE_BALANCE_ELECTRA, + amount=spec.EFFECTIVE_BALANCE_INCREMENT // 2 + ) + + pending_withdrawal_1 = prepare_pending_withdrawal( + spec, state, + validator_index, + effective_balance=spec.MAX_EFFECTIVE_BALANCE_ELECTRA, + amount=spec.EFFECTIVE_BALANCE_INCREMENT + ) + + # Set excess balance in a way that validator + # becomes not partially withdrawable only after the second pending withdrawal is processed + state.balances[validator_index] = spec.MAX_EFFECTIVE_BALANCE_ELECTRA + spec.EFFECTIVE_BALANCE_INCREMENT + assert spec.is_partially_withdrawable_validator( + state.validators[validator_index], + state.balances[validator_index] - pending_withdrawal_0.amount + ) + assert not spec.is_partially_withdrawable_validator( + state.validators[validator_index], + state.balances[validator_index] - pending_withdrawal_0.amount - pending_withdrawal_1.amount + ) + + next_slot(spec, state) + execution_payload = build_empty_execution_payload(spec, state) + yield from run_withdrawals_processing( + spec, state, + execution_payload, + num_expected_withdrawals=2, + fully_withdrawable_indices=[], + partial_withdrawals_indices=[], + pending_withdrawal_requests=[pending_withdrawal_0, pending_withdrawal_1] + ) + + assert state.pending_partial_withdrawals == [] + + +@with_electra_and_later +@spec_state_test +def test_pending_withdrawals_with_effective_sweep_on_top(spec, state): + # Ensure validator will be processed by the sweep + validator_index = min(len(state.validators), spec.MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP) // 2 + + pending_withdrawal_0 = prepare_pending_withdrawal( + spec, state, + validator_index, + effective_balance=spec.MAX_EFFECTIVE_BALANCE_ELECTRA, + amount=spec.EFFECTIVE_BALANCE_INCREMENT // 2 + ) + + pending_withdrawal_1 = prepare_pending_withdrawal( + spec, state, + validator_index, + effective_balance=spec.MAX_EFFECTIVE_BALANCE_ELECTRA, + amount=spec.EFFECTIVE_BALANCE_INCREMENT + ) + + # Set excess balance to requested amount times three, + # so the validator is partially withdrawable after pending withdrawal is processed + state.balances[validator_index] = spec.MAX_EFFECTIVE_BALANCE_ELECTRA + spec.EFFECTIVE_BALANCE_INCREMENT * 3 + assert spec.is_partially_withdrawable_validator( + state.validators[validator_index], + state.balances[validator_index] - pending_withdrawal_0.amount - pending_withdrawal_1.amount + ) + + next_slot(spec, state) + execution_payload = build_empty_execution_payload(spec, state) + yield from run_withdrawals_processing( + spec, state, + execution_payload, + num_expected_withdrawals=3, + fully_withdrawable_indices=[], + partial_withdrawals_indices=[validator_index], + pending_withdrawal_requests=[pending_withdrawal_0, pending_withdrawal_1] + ) + + assert state.pending_partial_withdrawals == [] + + +@with_electra_and_later +@spec_state_test +def test_pending_withdrawals_with_sweep_different_validator(spec, state): + # Ensure validator will be processed by the sweep + validator_index_0 = min(len(state.validators), spec.MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP) // 2 - 1 + validator_index_1 = min(len(state.validators), spec.MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP) // 2 + + # Initiate pending withdrawal for the first validator + pending_withdrawal_0 = prepare_pending_withdrawal( + spec, state, + validator_index_0, + effective_balance=spec.MAX_EFFECTIVE_BALANCE_ELECTRA, + amount=spec.EFFECTIVE_BALANCE_INCREMENT + ) + + # Make the second validator partially withdrawable by the sweep + set_compounding_withdrawal_credential_with_balance( + spec, state, validator_index_1, + effective_balance=spec.MAX_EFFECTIVE_BALANCE_ELECTRA, + balance=(spec.MAX_EFFECTIVE_BALANCE_ELECTRA + spec.EFFECTIVE_BALANCE_INCREMENT) + ) + + assert spec.is_partially_withdrawable_validator( + state.validators[validator_index_1], + state.balances[validator_index_1] + ) + + next_slot(spec, state) + execution_payload = build_empty_execution_payload(spec, state) + yield from run_withdrawals_processing( + spec, state, + execution_payload, + num_expected_withdrawals=2, + fully_withdrawable_indices=[], + partial_withdrawals_indices=[validator_index_1], + pending_withdrawal_requests=[pending_withdrawal_0] + ) + + assert state.pending_partial_withdrawals == [] + + +@with_electra_and_later +@spec_state_test +def test_pending_withdrawals_mixed_with_sweep_and_fully_withdrawable(spec, state): + num_full_withdrawals = spec.MAX_WITHDRAWALS_PER_PAYLOAD // 4 + num_partial_withdrawals = spec.MAX_WITHDRAWALS_PER_PAYLOAD // 4 + num_full_withdrawals_comp = spec.MAX_WITHDRAWALS_PER_PAYLOAD // 4 + num_partial_withdrawals_comp = spec.MAX_WITHDRAWALS_PER_PAYLOAD // 4 + num_pending_withdrawal_requests = spec.MAX_WITHDRAWALS_PER_PAYLOAD // 4 + + fully_withdrawable_indices, partial_withdrawals_indices = prepare_expected_withdrawals( + spec, state, + rng=random.Random(42), + num_full_withdrawals=num_full_withdrawals, + num_partial_withdrawals=num_partial_withdrawals, + num_full_withdrawals_comp=num_full_withdrawals_comp, + num_partial_withdrawals_comp=num_partial_withdrawals_comp, + ) + + pending_withdrawal_requests = [] + for index in range(0, len(state.validators)): + if len(pending_withdrawal_requests) >= num_pending_withdrawal_requests: + break + if index in (fully_withdrawable_indices + partial_withdrawals_indices): + continue + + pending_withdrawal = prepare_pending_withdrawal(spec, state, index) + pending_withdrawal_requests.append(pending_withdrawal) + + next_slot(spec, state) + execution_payload = build_empty_execution_payload(spec, state) + yield from run_withdrawals_processing( + spec, state, + execution_payload, + num_expected_withdrawals=spec.MAX_WITHDRAWALS_PER_PAYLOAD, + fully_withdrawable_indices=fully_withdrawable_indices, + partial_withdrawals_indices=partial_withdrawals_indices, + pending_withdrawal_requests=pending_withdrawal_requests + ) assert state.pending_partial_withdrawals == [] + + +@with_electra_and_later +@spec_state_test +def test_pending_withdrawals_at_max_mixed_with_sweep_and_fully_withdrawable(spec, state): + num_full_withdrawals = spec.MAX_WITHDRAWALS_PER_PAYLOAD // 4 + num_partial_withdrawals = spec.MAX_WITHDRAWALS_PER_PAYLOAD // 4 + num_full_withdrawals_comp = spec.MAX_WITHDRAWALS_PER_PAYLOAD // 4 + num_partial_withdrawals_comp = spec.MAX_WITHDRAWALS_PER_PAYLOAD // 4 + num_pending_withdrawal_requests = spec.MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP + 1 + + fully_withdrawable_indices, partial_withdrawals_indices = prepare_expected_withdrawals( + spec, state, + rng=random.Random(42), + num_full_withdrawals=num_full_withdrawals, + num_partial_withdrawals=num_partial_withdrawals, + num_full_withdrawals_comp=num_full_withdrawals_comp, + num_partial_withdrawals_comp=num_partial_withdrawals_comp, + ) + + pending_withdrawal_requests = [] + for index in range(0, len(state.validators)): + if len(pending_withdrawal_requests) >= num_pending_withdrawal_requests: + break + if index in (fully_withdrawable_indices + partial_withdrawals_indices): + continue + + pending_withdrawal = prepare_pending_withdrawal(spec, state, index) + pending_withdrawal_requests.append(pending_withdrawal) + + next_slot(spec, state) + execution_payload = build_empty_execution_payload(spec, state) + yield from run_withdrawals_processing( + spec, state, + execution_payload, + num_expected_withdrawals=spec.MAX_WITHDRAWALS_PER_PAYLOAD, + fully_withdrawable_indices=fully_withdrawable_indices, + partial_withdrawals_indices=partial_withdrawals_indices, + pending_withdrawal_requests=pending_withdrawal_requests[:spec.MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP] + ) + + withdrawals_exceeding_max = pending_withdrawal_requests[spec.MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP:] + assert state.pending_partial_withdrawals == withdrawals_exceeding_max diff --git a/tests/core/pyspec/eth2spec/test/helpers/withdrawals.py b/tests/core/pyspec/eth2spec/test/helpers/withdrawals.py index 518920aeb2..b1cfaf869d 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/withdrawals.py +++ b/tests/core/pyspec/eth2spec/test/helpers/withdrawals.py @@ -63,11 +63,21 @@ def sample_withdrawal_indices(spec, state, rng, num_full_withdrawals, num_partia def prepare_expected_withdrawals(spec, state, rng, - num_full_withdrawals=0, num_partial_withdrawals=0): + num_full_withdrawals=0, num_partial_withdrawals=0, + num_full_withdrawals_comp=0, num_partial_withdrawals_comp=0): fully_withdrawable_indices, partial_withdrawals_indices = sample_withdrawal_indices( - spec, state, rng, num_full_withdrawals, num_partial_withdrawals + spec, state, rng, + num_full_withdrawals + num_full_withdrawals_comp, + num_partial_withdrawals + num_partial_withdrawals_comp ) + fully_withdrawable_indices_comp = rng.sample(fully_withdrawable_indices, num_full_withdrawals_comp) + partial_withdrawals_indices_comp = rng.sample(partial_withdrawals_indices, num_partial_withdrawals_comp) + + for index in (fully_withdrawable_indices_comp + partial_withdrawals_indices_comp): + address = state.validators[index].withdrawal_credentials[12:] + set_compounding_withdrawal_credential_with_balance(spec, state, index, address=address) + for index in fully_withdrawable_indices: set_validator_fully_withdrawable(spec, state, index) for index in partial_withdrawals_indices: @@ -97,32 +107,13 @@ def set_compounding_withdrawal_credential_with_balance(spec, state, index, state.balances[index] = balance -def prepare_expected_withdrawals_compounding(spec, state, rng, - num_full_withdrawals=0, - num_partial_withdrawals_sweep=0, - excess_balance=1000000000): - assert is_post_electra(spec) - - fully_withdrawable_indices, partial_withdrawals_sweep_indices = sample_withdrawal_indices( - spec, state, rng, num_full_withdrawals, num_partial_withdrawals_sweep - ) - - for index in fully_withdrawable_indices + partial_withdrawals_sweep_indices: - address = state.validators[index].withdrawal_credentials[12:] - set_compounding_withdrawal_credential_with_balance(spec, state, index, address=address) - - for index in fully_withdrawable_indices: - set_validator_fully_withdrawable(spec, state, index) - for index in partial_withdrawals_sweep_indices: - set_validator_partially_withdrawable(spec, state, index) - - return fully_withdrawable_indices, partial_withdrawals_sweep_indices - - def prepare_pending_withdrawal(spec, state, validator_index, - effective_balance=32_000_000_000, amount=1_000_000_000): + effective_balance=32_000_000_000, amount=1_000_000_000, withdrawable_epoch=None): assert is_post_electra(spec) + if withdrawable_epoch is None: + withdrawable_epoch = spec.get_current_epoch(state) + balance = effective_balance + amount set_compounding_withdrawal_credential_with_balance( spec, state, validator_index, effective_balance, balance @@ -131,7 +122,7 @@ def prepare_pending_withdrawal(spec, state, validator_index, withdrawal = spec.PendingPartialWithdrawal( index=validator_index, amount=amount, - withdrawable_epoch=spec.get_current_epoch(state), + withdrawable_epoch=withdrawable_epoch, ) state.pending_partial_withdrawals.append(withdrawal) @@ -175,7 +166,8 @@ def verify_post_state(state, spec, expected_withdrawals, def run_withdrawals_processing(spec, state, execution_payload, num_expected_withdrawals=None, - fully_withdrawable_indices=None, partial_withdrawals_indices=None, valid=True): + fully_withdrawable_indices=None, partial_withdrawals_indices=None, + pending_withdrawal_requests=None, valid=True): """ Run ``process_withdrawals``, yielding: - pre-state ('pre') @@ -206,6 +198,11 @@ def run_withdrawals_processing(spec, state, execution_payload, num_expected_with yield 'post', state + # Check withdrawal indices + assert state.next_withdrawal_index == pre_state.next_withdrawal_index + len(expected_withdrawals) + for index, withdrawal in enumerate(execution_payload.withdrawals): + assert withdrawal.index == pre_state.next_withdrawal_index + index + if len(expected_withdrawals) == 0: next_withdrawal_validator_index = ( pre_state.next_withdrawal_validator_index + spec.MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP @@ -220,4 +217,12 @@ def run_withdrawals_processing(spec, state, execution_payload, num_expected_with if fully_withdrawable_indices is not None or partial_withdrawals_indices is not None: verify_post_state(state, spec, expected_withdrawals, fully_withdrawable_indices, partial_withdrawals_indices) + # Check withdrawal requests + if pending_withdrawal_requests is not None: + assert len(pending_withdrawal_requests) <= len(execution_payload.withdrawals) + for index, request in enumerate(pending_withdrawal_requests): + withdrawal = execution_payload.withdrawals[index] + assert withdrawal.validator_index == request.index + assert withdrawal.amount == request.amount + return expected_withdrawals From 5b55e95984710ef0f270e1210a5602fa1b5251ba Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Wed, 16 Oct 2024 18:24:19 +0200 Subject: [PATCH 066/142] Update specs/deneb/p2p-interface.md Co-authored-by: Justin Traglia <95511699+jtraglia@users.noreply.github.com> --- specs/deneb/p2p-interface.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index c7ea310584..05920d0099 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -310,7 +310,8 @@ Clients SHOULD include a sidecar in the response as soon as it passes the gossip Clients SHOULD NOT respond with sidecars related to blocks that fail gossip validation rules. Clients SHOULD NOT respond with sidecars related to blocks that fail the beacon chain state transition -###### Blobs recovery via local execution layer client +###### Blob retrieval via local execution layer client + In addition to `BlobSidecarsByRoot` requests, recent blobs recovery MAY also be done by querying the Execution Layer (i.e. via `engine_getBlobsV1`) Implementers are encouraged to leverage this method to increase the likelihood of incorporating and attesting to the last block when its proposer is not able to publish blobs on time. From 71b995cdd9d47f36c9405285866a79a3deadd739 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Wed, 16 Oct 2024 18:24:37 +0200 Subject: [PATCH 067/142] Update specs/deneb/p2p-interface.md Co-authored-by: Justin Traglia <95511699+jtraglia@users.noreply.github.com> --- specs/deneb/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 05920d0099..ec63c2f4b7 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -312,7 +312,7 @@ Clients SHOULD NOT respond with sidecars related to blocks that fail the beacon ###### Blob retrieval via local execution layer client -In addition to `BlobSidecarsByRoot` requests, recent blobs recovery MAY also be done by querying the Execution Layer (i.e. via `engine_getBlobsV1`) +In addition to `BlobSidecarsByRoot` requests, recent blobs MAY be retrieved by querying the Execution Layer (i.e. via `engine_getBlobsV1`). Implementers are encouraged to leverage this method to increase the likelihood of incorporating and attesting to the last block when its proposer is not able to publish blobs on time. When clients use local execution layer to recover missing blobs relative to the current slot, they MUST behave as if the corresponding `blob_sidecar` had been received via gossip. In particular they MUST: From cb837f27c30dcc2101fe312c5195fc4362bbf302 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Wed, 16 Oct 2024 18:24:59 +0200 Subject: [PATCH 068/142] Update specs/deneb/p2p-interface.md Co-authored-by: Justin Traglia <95511699+jtraglia@users.noreply.github.com> --- specs/deneb/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index ec63c2f4b7..8cebccb006 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -317,7 +317,7 @@ Implementers are encouraged to leverage this method to increase the likelihood o When clients use local execution layer to recover missing blobs relative to the current slot, they MUST behave as if the corresponding `blob_sidecar` had been received via gossip. In particular they MUST: - publish the corresponding `blob_sidecar` on the `blob_sidecar_{subnet_id}` subnet. -- update gossip rule related data structures (ie update the anti-equivocation cache). +- update gossip rule related data structures (i.e. update the anti-equivocation cache). ##### BlobSidecarsByRange v1 From 6c99b0bbf4293d63694112c5a22b0d1331be7a2d Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Wed, 16 Oct 2024 18:26:59 +0200 Subject: [PATCH 069/142] Update p2p-interface.md --- specs/deneb/p2p-interface.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 8cebccb006..a03532a146 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -316,8 +316,8 @@ In addition to `BlobSidecarsByRoot` requests, recent blobs MAY be retrieved by q Implementers are encouraged to leverage this method to increase the likelihood of incorporating and attesting to the last block when its proposer is not able to publish blobs on time. When clients use local execution layer to recover missing blobs relative to the current slot, they MUST behave as if the corresponding `blob_sidecar` had been received via gossip. In particular they MUST: -- publish the corresponding `blob_sidecar` on the `blob_sidecar_{subnet_id}` subnet. -- update gossip rule related data structures (i.e. update the anti-equivocation cache). +* publish the corresponding `blob_sidecar` on the `blob_sidecar_{subnet_id}` subnet. +* update gossip rule related data structures (i.e. update the anti-equivocation cache). ##### BlobSidecarsByRange v1 From 1767c344930eaa4e9f8df935c568a6016bf6ed5f Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Wed, 16 Oct 2024 18:27:42 +0200 Subject: [PATCH 070/142] Update specs/deneb/p2p-interface.md Co-authored-by: Justin Traglia <95511699+jtraglia@users.noreply.github.com> --- specs/deneb/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index a03532a146..6b6d2ea505 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -315,7 +315,7 @@ Clients SHOULD NOT respond with sidecars related to blocks that fail the beacon In addition to `BlobSidecarsByRoot` requests, recent blobs MAY be retrieved by querying the Execution Layer (i.e. via `engine_getBlobsV1`). Implementers are encouraged to leverage this method to increase the likelihood of incorporating and attesting to the last block when its proposer is not able to publish blobs on time. -When clients use local execution layer to recover missing blobs relative to the current slot, they MUST behave as if the corresponding `blob_sidecar` had been received via gossip. In particular they MUST: +When clients use the local execution layer to retrieve blobs, they MUST behave as if the corresponding `blob_sidecar` had been received via gossip. In particular they MUST: * publish the corresponding `blob_sidecar` on the `blob_sidecar_{subnet_id}` subnet. * update gossip rule related data structures (i.e. update the anti-equivocation cache). From 9787a61c881ec104b06db6c94092b6cbd5495fcd Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Wed, 16 Oct 2024 18:27:56 +0200 Subject: [PATCH 071/142] Update specs/deneb/p2p-interface.md Co-authored-by: Justin Traglia <95511699+jtraglia@users.noreply.github.com> --- specs/deneb/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 6b6d2ea505..aa63ebe87e 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -34,7 +34,7 @@ The specification of these changes continues in the same format as the network s - [BeaconBlocksByRange v2](#beaconblocksbyrange-v2) - [BeaconBlocksByRoot v2](#beaconblocksbyroot-v2) - [BlobSidecarsByRoot v1](#blobsidecarsbyroot-v1) - - [Blobs recovery via local execution layer client](#blobs-recovery-via-local-execution-layer-client) + - [Blob retrieval via local execution layer client](#blob-retrieval-via-local-execution-layer-client) - [BlobSidecarsByRange v1](#blobsidecarsbyrange-v1) - [Design decision rationale](#design-decision-rationale) - [Why are blobs relayed as a sidecar, separate from beacon blocks?](#why-are-blobs-relayed-as-a-sidecar-separate-from-beacon-blocks) From 62a3d1e77e1e2c4b61d081158321ed516ffbd581 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Thu, 17 Oct 2024 14:45:43 +0600 Subject: [PATCH 072/142] Apply suggestions by @jtraglia Co-authored-by: Justin Traglia <95511699+jtraglia@users.noreply.github.com> --- .../test/electra/block_processing/test_process_withdrawals.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_withdrawals.py b/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_withdrawals.py index c7ba00641b..1757c79994 100644 --- a/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_withdrawals.py +++ b/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_withdrawals.py @@ -298,7 +298,7 @@ def test_pending_withdrawals_with_effective_sweep_on_top(spec, state): # Set excess balance to requested amount times three, # so the validator is partially withdrawable after pending withdrawal is processed - state.balances[validator_index] = spec.MAX_EFFECTIVE_BALANCE_ELECTRA + spec.EFFECTIVE_BALANCE_INCREMENT * 3 + state.balances[validator_index] = spec.MAX_EFFECTIVE_BALANCE_ELECTRA + spec.EFFECTIVE_BALANCE_INCREMENT * 2 assert spec.is_partially_withdrawable_validator( state.validators[validator_index], state.balances[validator_index] - pending_withdrawal_0.amount - pending_withdrawal_1.amount @@ -366,7 +366,7 @@ def test_pending_withdrawals_mixed_with_sweep_and_fully_withdrawable(spec, state num_partial_withdrawals = spec.MAX_WITHDRAWALS_PER_PAYLOAD // 4 num_full_withdrawals_comp = spec.MAX_WITHDRAWALS_PER_PAYLOAD // 4 num_partial_withdrawals_comp = spec.MAX_WITHDRAWALS_PER_PAYLOAD // 4 - num_pending_withdrawal_requests = spec.MAX_WITHDRAWALS_PER_PAYLOAD // 4 + num_pending_withdrawal_requests = spec.MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP // 2 fully_withdrawable_indices, partial_withdrawals_indices = prepare_expected_withdrawals( spec, state, From 778a32ab9b002c029f810299743f8686bf74eb32 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Thu, 17 Oct 2024 16:21:24 +0600 Subject: [PATCH 073/142] Add missing withdrawal requests tests --- .../test_process_withdrawal_request.py | 48 ++++++++++++++----- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_withdrawal_request.py b/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_withdrawal_request.py index e3ebcae7e6..39626ee059 100644 --- a/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_withdrawal_request.py +++ b/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_withdrawal_request.py @@ -124,7 +124,7 @@ def test_basic_withdrawal_request_with_full_partial_withdrawal_queue(spec, state ) -# Invalid tests +# Tests that should fail @with_electra_and_later @@ -237,6 +237,31 @@ def test_activation_epoch_less_than_shard_committee_period(spec, state): ) +@with_electra_and_later +@spec_state_test +def test_unknown_pubkey(spec, state): + rng = random.Random(1344) + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + + current_epoch = spec.get_current_epoch(state) + validator_index = rng.choice(spec.get_active_validator_indices(state, current_epoch)) + address = b"\x22" * 20 + pubkey = spec.BLSPubkey(b"\x23" * 48) + set_eth1_withdrawal_credential_with_balance( + spec, state, validator_index, address=address + ) + withdrawal_request = spec.WithdrawalRequest( + source_address=address, + validator_pubkey=pubkey, + amount=spec.FULL_EXIT_REQUEST_AMOUNT, + ) + + yield from run_withdrawal_request_processing( + spec, state, withdrawal_request, success=False + ) + + # Partial withdrawals tests @with_electra_and_later @@ -896,10 +921,6 @@ def run_withdrawal_request_processing( If ``valid == False``, run expecting ``AssertionError`` If ``success == False``, it doesn't initiate exit successfully """ - validator_index = get_validator_index_by_pubkey( - state, withdrawal_request.validator_pubkey - ) - yield "pre", state yield "withdrawal_request", withdrawal_request @@ -912,14 +933,7 @@ def run_withdrawal_request_processing( yield "post", None return - pre_exit_epoch = state.validators[validator_index].exit_epoch - pre_pending_partial_withdrawals = state.pending_partial_withdrawals.copy() - pre_balance = state.balances[validator_index] - pre_effective_balance = state.validators[validator_index].effective_balance pre_state = state.copy() - expected_amount_to_withdraw = compute_amount_to_withdraw( - spec, state, validator_index, withdrawal_request.amount - ) spec.process_withdrawal_request( state, withdrawal_request @@ -931,6 +945,13 @@ def run_withdrawal_request_processing( # No-op assert pre_state == state else: + validator_index = get_validator_index_by_pubkey( + state, withdrawal_request.validator_pubkey + ) + pre_exit_epoch = pre_state.validators[validator_index].exit_epoch + pre_pending_partial_withdrawals = pre_state.pending_partial_withdrawals.copy() + pre_balance = pre_state.balances[validator_index] + pre_effective_balance = pre_state.validators[validator_index].effective_balance assert state.balances[validator_index] == pre_balance assert ( state.validators[validator_index].effective_balance == pre_effective_balance @@ -943,6 +964,9 @@ def run_withdrawal_request_processing( assert state.pending_partial_withdrawals == pre_pending_partial_withdrawals # Partial withdrawal request else: + expected_amount_to_withdraw = compute_amount_to_withdraw( + spec, pre_state, validator_index, withdrawal_request.amount + ) assert state.validators[validator_index].exit_epoch == spec.FAR_FUTURE_EPOCH expected_withdrawable_epoch = ( state.earliest_exit_epoch From 792cb18bd1bbb0d09a00830bb62ad098c5f854fd Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Thu, 17 Oct 2024 17:30:51 +0600 Subject: [PATCH 074/142] Add more process_attestation tests --- .../test_process_attestation.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_attestation.py index f268feb034..aca41a5a63 100644 --- a/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_attestation.py +++ b/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_attestation.py @@ -1,12 +1,16 @@ +from eth2spec.test.helpers.constants import MINIMAL from eth2spec.test.context import ( always_bls, spec_state_test, with_electra_and_later, + with_presets, ) from eth2spec.test.helpers.attestations import ( run_attestation_processing, get_valid_attestation, sign_attestation, + build_attestation_data, + fill_aggregate_attestation, ) from eth2spec.test.helpers.state import ( next_slots, @@ -79,3 +83,30 @@ def test_invalid_nonset_committe_bits(spec, state): attestation.committee_bits[committee_index] = 0 yield from run_attestation_processing(spec, state, attestation, valid=False) + + +@with_electra_and_later +@spec_state_test +@with_presets([MINIMAL], "need multiple committees per slot") +@always_bls +def test_multiple_committees(spec, state): + attestation_data = build_attestation_data(spec, state, slot=state.slot, index=0) + attestation = spec.Attestation(data=attestation_data) + + # fill the attestation with two committees and finally sign it + fill_aggregate_attestation(spec, state, attestation, signed=False, committee_index=0) + fill_aggregate_attestation(spec, state, attestation, signed=True, committee_index=1) + + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) + + yield from run_attestation_processing(spec, state, attestation) + + +@with_electra_and_later +@spec_state_test +@always_bls +def test_one_committee_with_gap(spec, state): + attestation = get_valid_attestation(spec, state, index=1, signed=True) + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) + + yield from run_attestation_processing(spec, state, attestation) From f4a7f02af0480d77ed204615c42656c0c7629c77 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Thu, 17 Oct 2024 20:31:29 +0600 Subject: [PATCH 075/142] Apply suggestions by @jtraglia --- .../test/electra/block_processing/test_process_attestation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_attestation.py index aca41a5a63..0922f4bd0c 100644 --- a/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_attestation.py +++ b/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_attestation.py @@ -93,8 +93,8 @@ def test_multiple_committees(spec, state): attestation_data = build_attestation_data(spec, state, slot=state.slot, index=0) attestation = spec.Attestation(data=attestation_data) - # fill the attestation with two committees and finally sign it - fill_aggregate_attestation(spec, state, attestation, signed=False, committee_index=0) + # fill the attestation with two committees + fill_aggregate_attestation(spec, state, attestation, signed=True, committee_index=0) fill_aggregate_attestation(spec, state, attestation, signed=True, committee_index=1) next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) From 93496e35a1b9d54c263c7d3478b63bb972b00743 Mon Sep 17 00:00:00 2001 From: Justin Traglia <95511699+jtraglia@users.noreply.github.com> Date: Thu, 17 Oct 2024 10:57:18 -0500 Subject: [PATCH 076/142] Add remark about verify_data_column_sidecar --- specs/_features/eip7594/p2p-interface.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/_features/eip7594/p2p-interface.md b/specs/_features/eip7594/p2p-interface.md index 98fb5dd732..c989711807 100644 --- a/specs/_features/eip7594/p2p-interface.md +++ b/specs/_features/eip7594/p2p-interface.md @@ -223,7 +223,7 @@ Requests sidecars by block root and index. The response is a list of `DataColumnIdentifier` whose length is less than or equal to the number of requests. It may be less in the case that the responding peer is missing blocks or sidecars. -Before consuming the next response chunk, the response reader SHOULD verify the data column sidecar is well-formatted, has valid inclusion proof through `verify_data_column_sidecar_inclusion_proof`, and is correct w.r.t. the expected KZG commitments through `verify_data_column_sidecar_kzg_proofs`. +Before consuming the next response chunk, the response reader SHOULD verify the data column sidecar is well-formatted through `verify_data_column_sidecar`, has valid inclusion proof through `verify_data_column_sidecar_inclusion_proof`, and is correct w.r.t. the expected KZG commitments through `verify_data_column_sidecar_kzg_proofs`. No more than `MAX_REQUEST_DATA_COLUMN_SIDECARS` may be requested at a time. @@ -269,7 +269,7 @@ Response Content: Requests data column sidecars in the slot range `[start_slot, start_slot + count)` of the given `columns`, leading up to the current head block as selected by fork choice. -Before consuming the next response chunk, the response reader SHOULD verify the data column sidecar is well-formatted, has valid inclusion proof through `verify_data_column_sidecar_inclusion_proof`, and is correct w.r.t. the expected KZG commitments through `verify_data_column_sidecar_kzg_proofs`. +Before consuming the next response chunk, the response reader SHOULD verify the data column sidecar is well-formatted through `verify_data_column_sidecar`, has valid inclusion proof through `verify_data_column_sidecar_inclusion_proof`, and is correct w.r.t. the expected KZG commitments through `verify_data_column_sidecar_kzg_proofs`. `DataColumnSidecarsByRange` is primarily used to sync data columns that may have been missed on gossip and to sync within the `MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS` window. From 8ce4418feb3c8ed2a7d19aba50873ced12af691c Mon Sep 17 00:00:00 2001 From: Justin Traglia <95511699+jtraglia@users.noreply.github.com> Date: Thu, 17 Oct 2024 11:13:11 -0500 Subject: [PATCH 077/142] Bump version to 1.5.0-alpha.9 --- tests/core/pyspec/eth2spec/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index 93244d44a1..e3af99e3c0 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -1.5.0-alpha.8 +1.5.0-alpha.9 From d64e537bd24c979f0a68d1d37b1f6cc468d23490 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Fri, 18 Oct 2024 09:31:51 +0600 Subject: [PATCH 078/142] Fix presets for process_attestation test --- .../test/electra/block_processing/test_process_attestation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_attestation.py index 0922f4bd0c..635d2472f4 100644 --- a/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_attestation.py +++ b/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_attestation.py @@ -104,6 +104,7 @@ def test_multiple_committees(spec, state): @with_electra_and_later @spec_state_test +@with_presets([MINIMAL], "need multiple committees per slot") @always_bls def test_one_committee_with_gap(spec, state): attestation = get_valid_attestation(spec, state, index=1, signed=True) From 17f64540359c410b1fddcf35051ac3439e831dd2 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Fri, 18 Oct 2024 13:52:11 +0600 Subject: [PATCH 079/142] Add tests for process_registry_updates --- specs/electra/beacon-chain.md | 8 +- .../test_process_registry_updates.py | 6 +- .../test_process_registry_updates.py | 92 +++++++++++++++++++ 3 files changed, 101 insertions(+), 5 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/electra/epoch_processing/test_process_registry_updates.py diff --git a/specs/electra/beacon-chain.md b/specs/electra/beacon-chain.md index 0c56491907..0d6fc0b4e3 100644 --- a/specs/electra/beacon-chain.md +++ b/specs/electra/beacon-chain.md @@ -778,23 +778,25 @@ def process_epoch(state: BeaconState) -> None: #### Modified `process_registry_updates` -*Note*: The function `process_registry_updates` is modified to use the updated definition of `initiate_validator_exit` +*Note*: The function `process_registry_updates` is modified to +use the updated definitions of `initiate_validator_exit` and `is_eligible_for_activation_queue` and changes how the activation epochs are computed for eligible validators. ```python def process_registry_updates(state: BeaconState) -> None: # Process activation eligibility and ejections for index, validator in enumerate(state.validators): - if is_eligible_for_activation_queue(validator): + if is_eligible_for_activation_queue(validator): # [Modified in Electra:EIP7251] validator.activation_eligibility_epoch = get_current_epoch(state) + 1 if ( is_active_validator(validator, get_current_epoch(state)) and validator.effective_balance <= EJECTION_BALANCE ): - initiate_validator_exit(state, ValidatorIndex(index)) + initiate_validator_exit(state, ValidatorIndex(index)) # [Modified in Electra:EIP7251] # Activate all eligible validators + # [Modified in Electra:EIP7251] activation_epoch = compute_activation_exit_epoch(get_current_epoch(state)) for validator in state.validators: if is_eligible_for_activation(state, validator): diff --git a/tests/core/pyspec/eth2spec/test/deneb/epoch_processing/test_process_registry_updates.py b/tests/core/pyspec/eth2spec/test/deneb/epoch_processing/test_process_registry_updates.py index 9d5a2e5d9c..0914bc1fb9 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/epoch_processing/test_process_registry_updates.py +++ b/tests/core/pyspec/eth2spec/test/deneb/epoch_processing/test_process_registry_updates.py @@ -23,6 +23,8 @@ def run_test_activation_churn_limit(spec, state): validator_count_0 = len(state.validators) + balance = spec.MIN_ACTIVATION_BALANCE if is_post_electra(spec) else spec.MAX_EFFECTIVE_BALANCE + for i in range(mock_activations): index = validator_count_0 + i validator = spec.Validator( @@ -32,10 +34,10 @@ def run_test_activation_churn_limit(spec, state): activation_epoch=spec.FAR_FUTURE_EPOCH, exit_epoch=spec.FAR_FUTURE_EPOCH, withdrawable_epoch=spec.FAR_FUTURE_EPOCH, - effective_balance=spec.MAX_EFFECTIVE_BALANCE, + effective_balance=balance, ) state.validators.append(validator) - state.balances.append(spec.MAX_EFFECTIVE_BALANCE) + state.balances.append(balance) state.previous_epoch_participation.append(spec.ParticipationFlags(0b0000_0000)) state.current_epoch_participation.append(spec.ParticipationFlags(0b0000_0000)) state.inactivity_scores.append(0) diff --git a/tests/core/pyspec/eth2spec/test/electra/epoch_processing/test_process_registry_updates.py b/tests/core/pyspec/eth2spec/test/electra/epoch_processing/test_process_registry_updates.py new file mode 100644 index 0000000000..ce5915269a --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/electra/epoch_processing/test_process_registry_updates.py @@ -0,0 +1,92 @@ +from eth2spec.test.helpers.deposits import mock_deposit +from eth2spec.test.helpers.state import next_epoch +from eth2spec.test.context import spec_state_test, with_electra_and_later +from eth2spec.test.helpers.epoch_processing import run_epoch_processing_with +from eth2spec.test.helpers.withdrawals import ( + set_eth1_withdrawal_credential_with_balance, + set_compounding_withdrawal_credential_with_balance +) + + +def run_test_activation_queue_eligibility(spec, state, validator_index, balance): + # move past first two irregular epochs wrt finality + next_epoch(spec, state) + next_epoch(spec, state) + + state.balances[validator_index] = balance + state.validators[validator_index].effective_balance = balance + + # ready for entrance into activation queue + mock_deposit(spec, state, validator_index) + + yield from run_epoch_processing_with(spec, state, 'process_registry_updates') + + # validator moved into activation queue if eligible + validator = state.validators[validator_index] + if validator.effective_balance < spec.MIN_ACTIVATION_BALANCE: + assert validator.activation_eligibility_epoch == spec.FAR_FUTURE_EPOCH + else: + assert validator.activation_eligibility_epoch < spec.FAR_FUTURE_EPOCH + + +@with_electra_and_later +@spec_state_test +def test_activation_queue_eligibility__less_than_min_activation_balance(spec, state): + # move past first two irregular epochs wrt finality + next_epoch(spec, state) + next_epoch(spec, state) + + index = 3 + balance = spec.MIN_ACTIVATION_BALANCE - spec.EFFECTIVE_BALANCE_INCREMENT + yield from run_test_activation_queue_eligibility(spec, state, index, balance) + + +@with_electra_and_later +@spec_state_test +def test_activation_queue_eligibility__min_activation_balance(spec, state): + # move past first two irregular epochs wrt finality + next_epoch(spec, state) + next_epoch(spec, state) + + index = 5 + balance = spec.MIN_ACTIVATION_BALANCE + yield from run_test_activation_queue_eligibility(spec, state, index, balance) + + +@with_electra_and_later +@spec_state_test +def test_activation_queue_eligibility__min_activation_balance_eth1_creds(spec, state): + # move past first two irregular epochs wrt finality + next_epoch(spec, state) + next_epoch(spec, state) + + index = 7 + balance = spec.MIN_ACTIVATION_BALANCE + set_eth1_withdrawal_credential_with_balance(spec, state, index) + yield from run_test_activation_queue_eligibility(spec, state, index, balance) + + +@with_electra_and_later +@spec_state_test +def test_activation_queue_eligibility__min_activation_balance_compounding_creds(spec, state): + # move past first two irregular epochs wrt finality + next_epoch(spec, state) + next_epoch(spec, state) + + index = 11 + balance = spec.MIN_ACTIVATION_BALANCE + set_compounding_withdrawal_credential_with_balance(spec, state, index) + yield from run_test_activation_queue_eligibility(spec, state, index, balance) + + +@with_electra_and_later +@spec_state_test +def test_activation_queue_eligibility__greater_than_min_activation_balance(spec, state): + # move past first two irregular epochs wrt finality + next_epoch(spec, state) + next_epoch(spec, state) + + index = 13 + balance = spec.MIN_ACTIVATION_BALANCE + spec.EFFECTIVE_BALANCE_INCREMENT + set_compounding_withdrawal_credential_with_balance(spec, state, index) + yield from run_test_activation_queue_eligibility(spec, state, index, balance) From 2a06d9165dee10de96b9cb0d7ffcba5898301aa6 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Fri, 18 Oct 2024 14:04:55 +0600 Subject: [PATCH 080/142] Add more process_pending_consolidations tests --- .../test_process_pending_consolidations.py | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/electra/epoch_processing/test_process_pending_consolidations.py b/tests/core/pyspec/eth2spec/test/electra/epoch_processing/test_process_pending_consolidations.py index 322224b78e..8da4028452 100644 --- a/tests/core/pyspec/eth2spec/test/electra/epoch_processing/test_process_pending_consolidations.py +++ b/tests/core/pyspec/eth2spec/test/electra/epoch_processing/test_process_pending_consolidations.py @@ -302,3 +302,67 @@ def test_pending_consolidation_with_pending_deposit(spec, state): # Pending deposit to the source was not processed. # It should only be processed in the next epoch transition assert state.pending_deposits == [pending_deposit] + + +@with_electra_and_later +@spec_state_test +def test_pending_consolidation_source_balance_less_than_effective(spec, state): + current_epoch = spec.get_current_epoch(state) + source_index = spec.get_active_validator_indices(state, current_epoch)[0] + target_index = spec.get_active_validator_indices(state, current_epoch)[1] + # append pending consolidation + state.pending_consolidations.append( + spec.PendingConsolidation(source_index=source_index, target_index=target_index) + ) + # Set withdrawable epoch to current epoch to allow processing + state.validators[source_index].withdrawable_epoch = current_epoch + # Set the target withdrawal credential to eth1 + eth1_withdrawal_credential = ( + spec.ETH1_ADDRESS_WITHDRAWAL_PREFIX + b"\x00" * 11 + b"\x11" * 20 + ) + state.validators[target_index].withdrawal_credentials = eth1_withdrawal_credential + # Set the source balance to be less than effective_balance + pre_balance_source = state.validators[source_index].effective_balance - spec.EFFECTIVE_BALANCE_INCREMENT // 8 + state.balances[source_index] = pre_balance_source + + pre_balance_target = state.balances[target_index] + + yield from run_epoch_processing_with(spec, state, "process_pending_consolidations") + + # Pending consolidation was successfully processed + assert state.balances[target_index] == pre_balance_target + pre_balance_source + assert state.balances[source_index] == 0 + assert state.pending_consolidations == [] + + +@with_electra_and_later +@spec_state_test +def test_pending_consolidation_source_balance_greater_than_effective(spec, state): + current_epoch = spec.get_current_epoch(state) + source_index = spec.get_active_validator_indices(state, current_epoch)[0] + target_index = spec.get_active_validator_indices(state, current_epoch)[1] + # append pending consolidation + state.pending_consolidations.append( + spec.PendingConsolidation(source_index=source_index, target_index=target_index) + ) + # Set withdrawable epoch to current epoch to allow processing + state.validators[source_index].withdrawable_epoch = current_epoch + # Set the target withdrawal credential to eth1 + eth1_withdrawal_credential = ( + spec.ETH1_ADDRESS_WITHDRAWAL_PREFIX + b"\x00" * 11 + b"\x11" * 20 + ) + state.validators[target_index].withdrawal_credentials = eth1_withdrawal_credential + # Set the source balance to be greater than effective_balance + pre_balance_source = state.validators[source_index].effective_balance + spec.EFFECTIVE_BALANCE_INCREMENT // 8 + state.balances[source_index] = pre_balance_source + + pre_balance_target = state.balances[target_index] + + yield from run_epoch_processing_with(spec, state, "process_pending_consolidations") + + # Pending consolidation was successfully processed + assert state.balances[target_index] == ( + pre_balance_target + spec.get_max_effective_balance(state.validators[source_index])) + assert state.balances[source_index] == ( + pre_balance_source - spec.get_max_effective_balance(state.validators[source_index])) + assert state.pending_consolidations == [] From 84d7ba25b69a3921b927acddcfada39519a1d209 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Fri, 18 Oct 2024 14:21:47 +0600 Subject: [PATCH 081/142] Fix linter --- .../electra/epoch_processing/test_process_registry_updates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/electra/epoch_processing/test_process_registry_updates.py b/tests/core/pyspec/eth2spec/test/electra/epoch_processing/test_process_registry_updates.py index ce5915269a..1f2666402e 100644 --- a/tests/core/pyspec/eth2spec/test/electra/epoch_processing/test_process_registry_updates.py +++ b/tests/core/pyspec/eth2spec/test/electra/epoch_processing/test_process_registry_updates.py @@ -18,7 +18,7 @@ def run_test_activation_queue_eligibility(spec, state, validator_index, balance) # ready for entrance into activation queue mock_deposit(spec, state, validator_index) - + yield from run_epoch_processing_with(spec, state, 'process_registry_updates') # validator moved into activation queue if eligible From a2f9b887ef214b1835733af8ce3c4b4e0e168c08 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Fri, 18 Oct 2024 15:51:18 +0600 Subject: [PATCH 082/142] Add more pending deposit tests --- .../test_process_pending_deposits.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/electra/epoch_processing/pending_deposits/test_process_pending_deposits.py b/tests/core/pyspec/eth2spec/test/electra/epoch_processing/pending_deposits/test_process_pending_deposits.py index e1f2544020..ee9ceccee7 100644 --- a/tests/core/pyspec/eth2spec/test/electra/epoch_processing/pending_deposits/test_process_pending_deposits.py +++ b/tests/core/pyspec/eth2spec/test/electra/epoch_processing/pending_deposits/test_process_pending_deposits.py @@ -2,6 +2,12 @@ from eth2spec.test.context import ( spec_state_test, with_electra_and_later, + with_presets, + spec_test, + single_phase, + with_custom_state, + scaled_churn_balances_exceed_activation_exit_churn_limit, + default_activation_threshold, ) from eth2spec.test.helpers.deposits import prepare_pending_deposit from eth2spec.test.helpers.state import ( @@ -9,6 +15,7 @@ advance_finality_to, set_full_participation, ) +from eth2spec.test.helpers.constants import MINIMAL def run_process_pending_deposits(spec, state): @@ -488,3 +495,26 @@ def test_process_pending_deposits_withdrawable_validator_not_churned(spec, state assert state.pending_deposits == [ prepare_pending_deposit(spec, validator_index=1, amount=amount) ] + + +@with_electra_and_later +@with_presets([MINIMAL], "need sufficient consolidation churn limit") +@with_custom_state( + balances_fn=scaled_churn_balances_exceed_activation_exit_churn_limit, + threshold_fn=default_activation_threshold, +) +@spec_test +@single_phase +def test_process_pending_deposits_scaled_churn(spec, state): + index = 0 + amount = spec.get_activation_exit_churn_limit(state) + state.pending_deposits.append( + prepare_pending_deposit(spec, index, amount) + ) + pre_balance = state.balances[index] + + yield from run_process_pending_deposits(spec, state) + + assert state.balances[index] == pre_balance + amount + assert state.deposit_balance_to_consume == 0 + assert state.pending_deposits == [] From ba8c0928766f76def10ea8db4705906f699c77d4 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Fri, 18 Oct 2024 16:49:26 +0600 Subject: [PATCH 083/142] Add more checks to electra fork transition tests --- specs/electra/fork.md | 3 +- .../electra/fork/test_electra_fork_basic.py | 29 +++++++++++++++---- .../eth2spec/test/helpers/electra/fork.py | 2 +- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/specs/electra/fork.md b/specs/electra/fork.md index 6ac5be5b03..af24f67749 100644 --- a/specs/electra/fork.md +++ b/specs/electra/fork.md @@ -72,7 +72,6 @@ an irregular state change is made to upgrade to Electra. ```python def upgrade_to_electra(pre: deneb.BeaconState) -> BeaconState: epoch = deneb.get_current_epoch(pre) - latest_execution_payload_header = pre.latest_execution_payload_header exit_epochs = [v.exit_epoch for v in pre.validators if v.exit_epoch != FAR_FUTURE_EPOCH] if not exit_epochs: @@ -119,7 +118,7 @@ def upgrade_to_electra(pre: deneb.BeaconState) -> BeaconState: current_sync_committee=pre.current_sync_committee, next_sync_committee=pre.next_sync_committee, # Execution-layer - latest_execution_payload_header=latest_execution_payload_header, # [Modified in Electra:EIP6110:EIP7002] + latest_execution_payload_header=pre.latest_execution_payload_header, # Withdrawals next_withdrawal_index=pre.next_withdrawal_index, next_withdrawal_validator_index=pre.next_withdrawal_validator_index, diff --git a/tests/core/pyspec/eth2spec/test/electra/fork/test_electra_fork_basic.py b/tests/core/pyspec/eth2spec/test/electra/fork/test_electra_fork_basic.py index e569be35e3..b70b961d78 100644 --- a/tests/core/pyspec/eth2spec/test/electra/fork/test_electra_fork_basic.py +++ b/tests/core/pyspec/eth2spec/test/electra/fork/test_electra_fork_basic.py @@ -87,11 +87,22 @@ def test_fork_random_large_validator_set(spec, phases, state): @with_state @with_meta_tags(ELECTRA_FORK_TEST_META_TAGS) def test_fork_pre_activation(spec, phases, state): + index = 0 post_spec = phases[ELECTRA] - state.validators[0].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[index].activation_epoch = spec.FAR_FUTURE_EPOCH post_state = yield from run_fork_test(post_spec, state) - assert len(post_state.pending_deposits) > 0 + validator = post_state.validators[index] + assert post_state.balances[index] == 0 + assert validator.effective_balance == 0 + assert validator.activation_eligibility_epoch == spec.FAR_FUTURE_EPOCH + assert post_state.pending_deposits == [post_spec.PendingDeposit( + pubkey=validator.pubkey, + withdrawal_credentials=validator.withdrawal_credentials, + amount=state.balances[index], + signature=spec.bls.G2_POINT_AT_INFINITY, + slot=spec.GENESIS_SLOT, + )] @with_phases(phases=[DENEB], other_phases=[ELECTRA]) @@ -99,10 +110,18 @@ def test_fork_pre_activation(spec, phases, state): @with_state @with_meta_tags(ELECTRA_FORK_TEST_META_TAGS) def test_fork_has_compounding_withdrawal_credential(spec, phases, state): + index = 0 post_spec = phases[ELECTRA] - validator = state.validators[0] - state.balances[0] = post_spec.MIN_ACTIVATION_BALANCE + 1 + validator = state.validators[index] + state.balances[index] = post_spec.MIN_ACTIVATION_BALANCE + 1 validator.withdrawal_credentials = post_spec.COMPOUNDING_WITHDRAWAL_PREFIX + validator.withdrawal_credentials[1:] post_state = yield from run_fork_test(post_spec, state) - assert len(post_state.pending_deposits) > 0 + assert post_state.balances[index] == post_spec.MIN_ACTIVATION_BALANCE + assert post_state.pending_deposits == [post_spec.PendingDeposit( + pubkey=validator.pubkey, + withdrawal_credentials=validator.withdrawal_credentials, + amount=state.balances[index] - post_spec.MIN_ACTIVATION_BALANCE, + signature=spec.bls.G2_POINT_AT_INFINITY, + slot=spec.GENESIS_SLOT, + )] diff --git a/tests/core/pyspec/eth2spec/test/helpers/electra/fork.py b/tests/core/pyspec/eth2spec/test/helpers/electra/fork.py index 886fc7cce0..2aeaaa995d 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/electra/fork.py +++ b/tests/core/pyspec/eth2spec/test/helpers/electra/fork.py @@ -54,7 +54,7 @@ def run_fork_test(post_spec, pre_state): stable_validator_fields = [ 'pubkey', 'withdrawal_credentials', 'slashed', - 'exit_epoch', 'withdrawable_epoch', + 'activation_epoch', 'exit_epoch', 'withdrawable_epoch', ] for field in stable_validator_fields: assert getattr(pre_validator, field) == getattr(post_validator, field) From 991ccd233f783015574c1871237385762640f2dc Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Fri, 18 Oct 2024 11:44:01 -0500 Subject: [PATCH 084/142] Add gen-collect-only CI check --- .github/workflows/run-tests.yml | 21 ++++++++++++++++--- Makefile | 5 ++++- .../gen_helpers/gen_base/gen_runner.py | 4 ++++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 83926c47c5..7b9cf51ab3 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -24,7 +24,7 @@ on: - cron: '0 0 * * *' jobs: - table_of_contents: + table-of-contents: runs-on: [self-hosted-ghr-custom, size-s-x64, profile-consensusSpecs] steps: - name: Checkout repository @@ -71,7 +71,7 @@ jobs: pyspec-tests: runs-on: [self-hosted-ghr-custom, size-xl-x64, profile-consensusSpecs] - needs: [lint,codespell,table_of_contents] + needs: [lint, codespell, table-of-contents] strategy: matrix: version: ["phase0", "altair", "bellatrix", "capella", "deneb", "electra", "whisk", "eip7594"] @@ -104,9 +104,24 @@ jobs: - name: Install pyspec requirements run: make install_test - name: test-${{ matrix.version }} - run: make citest fork=${{ matrix.version }} TEST_PRESET_TYPE=${{env.spec_test_preset_type}} + run: make citest fork=${{ matrix.version }} TEST_PRESET_TYPE=${{ env.spec_test_preset_type }} - uses: actions/upload-artifact@v4 if: always() with: name: test-reports-${{ matrix.version }} path: tests/core/pyspec/test-reports + + gen-collect-only: + runs-on: [self-hosted-ghr-custom, size-s-x64, profile-consensusSpecs] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12.4' + cache: '' + - name: Install pyspec requirements + run: make install_test + - name: Run generators with --collect-only + run: make generate_tests collect_only=true diff --git a/Makefile b/Makefile index 39d493645e..be83d1a302 100644 --- a/Makefile +++ b/Makefile @@ -159,6 +159,9 @@ lint_generators: pyspec . venv/bin/activate; cd $(TEST_GENERATORS_DIR); \ flake8 --config $(LINTER_CONFIG_FILE) +# If set to true, it will not run generator tests. +collect_only ?= false + # Runs a generator, identified by param 1 define run_generator # Started! @@ -177,7 +180,7 @@ define run_generator . venv/bin/activate; \ pip3 install ../../../dist/eth2spec-*.whl; \ pip3 install 'eth2spec[generator]'; \ - python3 main.py -o $(CURRENT_DIR)/$(TEST_VECTOR_DIR); \ + python3 main.py -o $(CURRENT_DIR)/$(TEST_VECTOR_DIR) $(if $(filter true,$(collect_only)),--collect-only); \ echo "generator $(1) finished" endef diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index 85ccec017a..aa60d6d92b 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -248,6 +248,10 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): print(f"Collected test at: {case_dir}") diagnostics_obj.collected_test_count += 1 + # Bail here if we do not want to generate. + if collect_only: + continue + is_skip, diagnostics_obj = should_skip_case_dir(case_dir, args.force, diagnostics_obj) if is_skip: continue From 57bd0ae2bfa5bd10c9cf3fc9d87c79068d84bf9a Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Fri, 18 Oct 2024 12:15:59 -0500 Subject: [PATCH 085/142] Remove --collect-only & rebrand to modcheck --- .github/workflows/run-tests.yml | 6 ++-- Makefile | 4 +-- .../gen_helpers/gen_base/gen_runner.py | 36 ++++++++----------- 3 files changed, 20 insertions(+), 26 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 7b9cf51ab3..0fe661eb98 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -111,7 +111,7 @@ jobs: name: test-reports-${{ matrix.version }} path: tests/core/pyspec/test-reports - gen-collect-only: + gen-modcheck: runs-on: [self-hosted-ghr-custom, size-s-x64, profile-consensusSpecs] steps: - name: Checkout repository @@ -123,5 +123,5 @@ jobs: cache: '' - name: Install pyspec requirements run: make install_test - - name: Run generators with --collect-only - run: make generate_tests collect_only=true + - name: Run generators with --modcheck + run: make generate_tests modcheck=true diff --git a/Makefile b/Makefile index be83d1a302..abb4ca21b1 100644 --- a/Makefile +++ b/Makefile @@ -160,7 +160,7 @@ lint_generators: pyspec flake8 --config $(LINTER_CONFIG_FILE) # If set to true, it will not run generator tests. -collect_only ?= false +modcheck ?= false # Runs a generator, identified by param 1 define run_generator @@ -180,7 +180,7 @@ define run_generator . venv/bin/activate; \ pip3 install ../../../dist/eth2spec-*.whl; \ pip3 install 'eth2spec[generator]'; \ - python3 main.py -o $(CURRENT_DIR)/$(TEST_VECTOR_DIR) $(if $(filter true,$(collect_only)),--collect-only); \ + python3 main.py -o $(CURRENT_DIR)/$(TEST_VECTOR_DIR) $(if $(filter true,$(modcheck)),--modcheck); \ echo "generator $(1) finished" endef diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index aa60d6d92b..7597475b22 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -187,14 +187,18 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): help="specify forks to run with. Allows all if no fork names are specified.", ) parser.add_argument( - "-c", - "--collect-only", + "--modcheck", action="store_true", default=False, - help="if set only print tests to generate, do not actually run the test and dump the target data", + help="check generator modules, do not run any tests.", ) - args = parser.parse_args() + + # Bail here if we are checking modules. + if args.modcheck: + print("Doing modcheck, returning early.") + return + output_dir = args.output_dir if not args.force: file_mode = "x" @@ -222,8 +226,6 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): if len(presets) != 0: print(f"Filtering test-generator runs to only include forks: {', '.join(forks)}") - collect_only = args.collect_only - diagnostics_obj = Diagnostics() provider_start = time.time() @@ -231,9 +233,8 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): all_test_case_params = [] for tprov in test_providers: - if not collect_only: - # runs anything that we don't want to repeat for every test case. - tprov.prepare() + # Runs anything that we don't want to repeat for every test case. + tprov.prepare() for test_case in tprov.make_cases(): # If preset list is assigned, filter by presets. @@ -248,10 +249,6 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): print(f"Collected test at: {case_dir}") diagnostics_obj.collected_test_count += 1 - # Bail here if we do not want to generate. - if collect_only: - continue - is_skip, diagnostics_obj = should_skip_case_dir(case_dir, args.force, diagnostics_obj) if is_skip: continue @@ -273,14 +270,11 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): provider_end = time.time() span = round(provider_end - provider_start, 2) - if collect_only: - print(f"Collected {diagnostics_obj.collected_test_count} tests in total") - else: - summary_message = f"completed generation of {generator_name} with {diagnostics_obj.generated_test_count} tests" - summary_message += f" ({diagnostics_obj.skipped_test_count} skipped tests)" - if span > TIME_THRESHOLD_TO_PRINT: - summary_message += f" in {span} seconds" - print(summary_message) + summary_message = f"completed generation of {generator_name} with {diagnostics_obj.generated_test_count} tests" + summary_message += f" ({diagnostics_obj.skipped_test_count} skipped tests)" + if span > TIME_THRESHOLD_TO_PRINT: + summary_message += f" in {span} seconds" + print(summary_message) diagnostics_output = { "collected_test_count": diagnostics_obj.collected_test_count, From c280e56957fb165c6cbbb13949983335e32f90b4 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Fri, 18 Oct 2024 12:26:52 -0500 Subject: [PATCH 086/142] Remove print statement --- tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index 7597475b22..b6883cc34d 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -196,7 +196,6 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): # Bail here if we are checking modules. if args.modcheck: - print("Doing modcheck, returning early.") return output_dir = args.output_dir From d2a9506e0446df2353b86dc61da915994e65cdfd Mon Sep 17 00:00:00 2001 From: NC <17676176+ensi321@users.noreply.github.com> Date: Fri, 18 Oct 2024 12:06:50 -0700 Subject: [PATCH 087/142] Clean up electra LC --- specs/electra/light-client/full-node.md | 58 +-------------------- specs/electra/light-client/sync-protocol.md | 26 --------- 2 files changed, 1 insertion(+), 83 deletions(-) diff --git a/specs/electra/light-client/full-node.md b/specs/electra/light-client/full-node.md index 0393aaec24..49b028a4a0 100644 --- a/specs/electra/light-client/full-node.md +++ b/specs/electra/light-client/full-node.md @@ -9,66 +9,10 @@ - [Introduction](#introduction) -- [Helper functions](#helper-functions) - - [Modified `block_to_light_client_header`](#modified-block_to_light_client_header) ## Introduction -Execution payload data is updated to account for the Electra upgrade. - -## Helper functions - -### Modified `block_to_light_client_header` - -```python -def block_to_light_client_header(block: SignedBeaconBlock) -> LightClientHeader: - epoch = compute_epoch_at_slot(block.message.slot) - - if epoch >= CAPELLA_FORK_EPOCH: - payload = block.message.body.execution_payload - execution_header = ExecutionPayloadHeader( - parent_hash=payload.parent_hash, - fee_recipient=payload.fee_recipient, - state_root=payload.state_root, - receipts_root=payload.receipts_root, - logs_bloom=payload.logs_bloom, - prev_randao=payload.prev_randao, - block_number=payload.block_number, - gas_limit=payload.gas_limit, - gas_used=payload.gas_used, - timestamp=payload.timestamp, - extra_data=payload.extra_data, - base_fee_per_gas=payload.base_fee_per_gas, - block_hash=payload.block_hash, - transactions_root=hash_tree_root(payload.transactions), - withdrawals_root=hash_tree_root(payload.withdrawals), - ) - if epoch >= DENEB_FORK_EPOCH: - execution_header.blob_gas_used = payload.blob_gas_used - execution_header.excess_blob_gas = payload.excess_blob_gas - - execution_branch = ExecutionBranch( - compute_merkle_proof(block.message.body, EXECUTION_PAYLOAD_GINDEX)) - else: - # Note that during fork transitions, `finalized_header` may still point to earlier forks. - # While Bellatrix blocks also contain an `ExecutionPayload` (minus `withdrawals_root`), - # it was not included in the corresponding light client data. To ensure compatibility - # with legacy data going through `upgrade_lc_header_to_capella`, leave out execution data. - execution_header = ExecutionPayloadHeader() - execution_branch = ExecutionBranch() - - return LightClientHeader( - beacon=BeaconBlockHeader( - slot=block.message.slot, - proposer_index=block.message.proposer_index, - parent_root=block.message.parent_root, - state_root=block.message.state_root, - body_root=hash_tree_root(block.message.body), - ), - execution=execution_header, - execution_branch=execution_branch, - ) -``` +No changes occurred in Electra upgrade. diff --git a/specs/electra/light-client/sync-protocol.md b/specs/electra/light-client/sync-protocol.md index 26da70ee34..f042119b00 100644 --- a/specs/electra/light-client/sync-protocol.md +++ b/specs/electra/light-client/sync-protocol.md @@ -18,7 +18,6 @@ - [Modified `current_sync_committee_gindex_at_slot`](#modified-current_sync_committee_gindex_at_slot) - [Modified `next_sync_committee_gindex_at_slot`](#modified-next_sync_committee_gindex_at_slot) - [Modified `get_lc_execution_root`](#modified-get_lc_execution_root) - - [Modified `is_valid_light_client_header`](#modified-is_valid_light_client_header) @@ -152,28 +151,3 @@ def get_lc_execution_root(header: LightClientHeader) -> Root: return Root() ``` - -### Modified `is_valid_light_client_header` - -```python -def is_valid_light_client_header(header: LightClientHeader) -> bool: - epoch = compute_epoch_at_slot(header.beacon.slot) - - if epoch < DENEB_FORK_EPOCH: - if header.execution.blob_gas_used != uint64(0) or header.execution.excess_blob_gas != uint64(0): - return False - - if epoch < CAPELLA_FORK_EPOCH: - return ( - header.execution == ExecutionPayloadHeader() - and header.execution_branch == ExecutionBranch() - ) - - return is_valid_merkle_branch( - leaf=get_lc_execution_root(header), - branch=header.execution_branch, - depth=floorlog2(EXECUTION_PAYLOAD_GINDEX), - index=get_subtree_index(EXECUTION_PAYLOAD_GINDEX), - root=header.beacon.body_root, - ) -``` From 3748ac1352a77f23f31c08de724c0ad3a43e7a73 Mon Sep 17 00:00:00 2001 From: NC <17676176+ensi321@users.noreply.github.com> Date: Fri, 18 Oct 2024 18:58:07 -0700 Subject: [PATCH 088/142] Remove full-node.md --- README.md | 2 +- specs/electra/light-client/full-node.md | 18 ------------------ 2 files changed, 1 insertion(+), 19 deletions(-) delete mode 100644 specs/electra/light-client/full-node.md diff --git a/README.md b/README.md index ae7e566ba8..c3a57c6256 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Features are researched and developed in parallel, and then consolidated into se | Seq. | Code Name | Fork Epoch | Specs | | - | - | - | - | -| 5 | **Electra** | TBD |
  • Core
    • [Beacon Chain changes](specs/electra/beacon-chain.md)
    • [Electra fork](specs/electra/fork.md)
  • Additions
    • [Light client sync protocol changes](specs/electra/light-client/sync-protocol.md) ([fork](specs/electra/light-client/fork.md), [full node](specs/electra/light-client/full-node.md), [networking](specs/electra/light-client/p2p-interface.md))
    • [Honest validator guide changes](specs/electra/validator.md)
    • [P2P networking](specs/electra/p2p-interface.md)
| +| 5 | **Electra** | TBD |
  • Core
    • [Beacon Chain changes](specs/electra/beacon-chain.md)
    • [Electra fork](specs/electra/fork.md)
  • Additions
    • [Light client sync protocol changes](specs/electra/light-client/sync-protocol.md) ([fork](specs/electra/light-client/fork.md), [networking](specs/electra/light-client/p2p-interface.md))
    • [Honest validator guide changes](specs/electra/validator.md)
    • [P2P networking](specs/electra/p2p-interface.md)
| ### Outdated Specifications diff --git a/specs/electra/light-client/full-node.md b/specs/electra/light-client/full-node.md deleted file mode 100644 index 49b028a4a0..0000000000 --- a/specs/electra/light-client/full-node.md +++ /dev/null @@ -1,18 +0,0 @@ -# Electra Light Client -- Full Node - -**Notice**: This document is a work-in-progress for researchers and implementers. - -## Table of contents - - - - - -- [Introduction](#introduction) - - - - -## Introduction - -No changes occurred in Electra upgrade. From f58873349bd9bdefc140392e735cabb0a7ef37d5 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Sat, 19 Oct 2024 10:35:10 -0500 Subject: [PATCH 089/142] Fix all pytest warnings --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f976778dc9..8d5aa213b7 100644 --- a/setup.py +++ b/setup.py @@ -572,7 +572,7 @@ def run(self): "py_ecc==6.0.0", "milagro_bls_binding==1.9.0", "remerkleable==0.1.28", - "trie==2.0.2", + "trie>=3,<4", RUAMEL_YAML_VERSION, "lru-dict==1.2.0", MARKO_VERSION, From cdfe808cebd1d099ee5064a5216aefd73502ebc0 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Sun, 20 Oct 2024 09:10:12 +0600 Subject: [PATCH 090/142] Add tests for operations on transition to electra --- .../test/electra/transition/__init__.py | 0 .../electra/transition/test_operations.py | 98 +++++++++++++++++++ .../eth2spec/test/helpers/consolidations.py | 18 ++++ .../pyspec/eth2spec/test/helpers/constants.py | 3 + .../eth2spec/test/helpers/fork_transition.py | 45 ++++++++- .../eth2spec/test/helpers/withdrawals.py | 19 ++++ tests/generators/transition/main.py | 4 + 7 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 tests/core/pyspec/eth2spec/test/electra/transition/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/electra/transition/test_operations.py create mode 100644 tests/core/pyspec/eth2spec/test/helpers/consolidations.py diff --git a/tests/core/pyspec/eth2spec/test/electra/transition/__init__.py b/tests/core/pyspec/eth2spec/test/electra/transition/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/core/pyspec/eth2spec/test/electra/transition/test_operations.py b/tests/core/pyspec/eth2spec/test/electra/transition/test_operations.py new file mode 100644 index 0000000000..b927a1bf1d --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/electra/transition/test_operations.py @@ -0,0 +1,98 @@ +from eth2spec.test.context import ( + ForkMeta, + always_bls, + with_fork_metas, + with_presets, +) +from eth2spec.test.helpers.constants import ( + AFTER_ELECTRA_PRE_POST_FORKS, + MINIMAL, +) +from eth2spec.test.helpers.fork_transition import ( + OperationType, + run_transition_with_operation, +) + + +# +# DepositRequest +# + +@with_fork_metas([ForkMeta(pre_fork_name=pre, post_fork_name=post, fork_epoch=2) + for pre, post in AFTER_ELECTRA_PRE_POST_FORKS]) +@always_bls +def test_transition_with_deposit_request_right_after_fork(state, fork_epoch, spec, post_spec, pre_tag, post_tag): + """ + Create a DEPOSIT_REQUEST right *after* the transition + """ + yield from run_transition_with_operation( + state, + fork_epoch, + spec, + post_spec, + pre_tag, + post_tag, + operation_type=OperationType.DEPOSIT_REQUEST, + operation_at_slot=fork_epoch * spec.SLOTS_PER_EPOCH, + ) + + +# +# WithdrawalRequest +# + +@with_fork_metas([ForkMeta(pre_fork_name=pre, post_fork_name=post, fork_epoch=66) + for pre, post in AFTER_ELECTRA_PRE_POST_FORKS]) +@with_presets([MINIMAL], reason="too slow") +@always_bls +def test_transition_with_full_withdrawal_request_right_after_fork( + state, + fork_epoch, + spec, + post_spec, + pre_tag, + post_tag +): + """ + Create a WITHDRAWAL_REQUEST right *after* the transition + """ + yield from run_transition_with_operation( + state, + fork_epoch, + spec, + post_spec, + pre_tag, + post_tag, + operation_type=OperationType.FULL_WITHDRAWAL_REQUEST, + operation_at_slot=fork_epoch * spec.SLOTS_PER_EPOCH, + ) + + +# +# ConsolidationRequest +# + +@with_fork_metas([ForkMeta(pre_fork_name=pre, post_fork_name=post, fork_epoch=2) + for pre, post in AFTER_ELECTRA_PRE_POST_FORKS]) +@always_bls +def test_transition_with_switch_to_compounding_request_right_after_fork( + state, + fork_epoch, + spec, + post_spec, + pre_tag, + post_tag +): + """ + Create a SWITCH_TO_COMPOUNDING_REQUEST right *after* the transition + """ + yield from run_transition_with_operation( + state, + fork_epoch, + spec, + post_spec, + pre_tag, + post_tag, + operation_type=OperationType.SWITCH_TO_COMPOUNDING_REQUEST, + operation_at_slot=fork_epoch * spec.SLOTS_PER_EPOCH, + ) diff --git a/tests/core/pyspec/eth2spec/test/helpers/consolidations.py b/tests/core/pyspec/eth2spec/test/helpers/consolidations.py new file mode 100644 index 0000000000..f47ea669fc --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/helpers/consolidations.py @@ -0,0 +1,18 @@ +from eth2spec.test.helpers.withdrawals import ( + set_eth1_withdrawal_credential_with_balance +) + + +def prepare_switch_to_compounding_request(spec, state, validator_index, address=None): + if address is not None: + set_eth1_withdrawal_credential_with_balance(spec, state, validator_index, address=address) + + validator = state.validators[validator_index] + if not spec.has_execution_withdrawal_credential(validator): + set_eth1_withdrawal_credential_with_balance(spec, state, validator_index) + + return spec.ConsolidationRequest( + source_address=state.validators[validator_index].withdrawal_credentials[12:], + source_pubkey=state.validators[validator_index].pubkey, + target_pubkey=state.validators[validator_index].pubkey, + ) diff --git a/tests/core/pyspec/eth2spec/test/helpers/constants.py b/tests/core/pyspec/eth2spec/test/helpers/constants.py index 7e5d0a0f4c..a19d1a75cd 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/constants.py +++ b/tests/core/pyspec/eth2spec/test/helpers/constants.py @@ -74,7 +74,10 @@ ALL_PRE_POST_FORKS = POST_FORK_OF.items() DENEB_TRANSITION_UPGRADES_AND_AFTER = {key: value for key, value in POST_FORK_OF.items() if key not in [PHASE0, ALTAIR, BELLATRIX]} +ELECTRA_TRANSITION_UPGRADES_AND_AFTER = {key: value for key, value in POST_FORK_OF.items() + if key not in [PHASE0, ALTAIR, BELLATRIX, CAPELLA]} AFTER_DENEB_PRE_POST_FORKS = DENEB_TRANSITION_UPGRADES_AND_AFTER.items() +AFTER_ELECTRA_PRE_POST_FORKS = ELECTRA_TRANSITION_UPGRADES_AND_AFTER.items() # # Config and Preset diff --git a/tests/core/pyspec/eth2spec/test/helpers/fork_transition.py b/tests/core/pyspec/eth2spec/test/helpers/fork_transition.py index 8eb72fdad5..41d159e6d8 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/fork_transition.py +++ b/tests/core/pyspec/eth2spec/test/helpers/fork_transition.py @@ -21,6 +21,7 @@ ) from eth2spec.test.helpers.deposits import ( prepare_state_and_deposit, + prepare_deposit_request, ) from eth2spec.test.helpers.proposer_slashings import ( get_valid_proposer_slashing, @@ -37,6 +38,12 @@ from eth2spec.test.helpers.voluntary_exits import ( prepare_signed_exits, ) +from eth2spec.test.helpers.withdrawals import ( + prepare_withdrawal_request, +) +from eth2spec.test.helpers.consolidations import ( + prepare_switch_to_compounding_request, +) class OperationType(Enum): @@ -45,11 +52,18 @@ class OperationType(Enum): DEPOSIT = auto() VOLUNTARY_EXIT = auto() BLS_TO_EXECUTION_CHANGE = auto() + DEPOSIT_REQUEST = auto() + FULL_WITHDRAWAL_REQUEST = auto() + SWITCH_TO_COMPOUNDING_REQUEST = auto() def _set_operations_by_dict(block, operation_dict): for key, value in operation_dict.items(): - setattr(block.body, key, value) + # to handle e.g. `execution_requests.deposits` and `deposits` + obj = block.body + for attr in key.split('.')[:-1]: + obj = getattr(obj, attr) + setattr(obj, key.split('.')[-1], value) def _state_transition_and_sign_block_at_slot(spec, @@ -328,6 +342,21 @@ def run_transition_with_operation(state, selected_validator_index = 0 bls_to_execution_changes = [get_signed_address_change(spec, state, selected_validator_index)] operation_dict = {'bls_to_execution_changes': bls_to_execution_changes} + elif operation_type == OperationType.DEPOSIT_REQUEST: + # create a new deposit request + selected_validator_index = len(state.validators) + amount = post_spec.MIN_ACTIVATION_BALANCE + deposit_request = prepare_deposit_request(post_spec, selected_validator_index, amount, signed=True) + operation_dict = {'execution_requests.deposits': [deposit_request]} + elif operation_type == OperationType.FULL_WITHDRAWAL_REQUEST: + selected_validator_index = 0 + withdrawal_request = prepare_withdrawal_request( + post_spec, state, selected_validator_index, amount=post_spec.FULL_EXIT_REQUEST_AMOUNT) + operation_dict = {'execution_requests.withdrawals': [withdrawal_request]} + elif operation_type == OperationType.SWITCH_TO_COMPOUNDING_REQUEST: + selected_validator_index = 0 + consolidation_request = prepare_switch_to_compounding_request(post_spec, state, selected_validator_index) + operation_dict = {'execution_requests.consolidations': [consolidation_request]} def _check_state(): if operation_type == OperationType.PROPOSER_SLASHING: @@ -352,6 +381,20 @@ def _check_state(): elif operation_type == OperationType.BLS_TO_EXECUTION_CHANGE: validator = state.validators[selected_validator_index] assert validator.withdrawal_credentials[:1] == spec.ETH1_ADDRESS_WITHDRAWAL_PREFIX + elif operation_type == OperationType.DEPOSIT_REQUEST: + assert state.pending_deposits == [post_spec.PendingDeposit( + pubkey=deposit_request.pubkey, + withdrawal_credentials=deposit_request.withdrawal_credentials, + amount=deposit_request.amount, + signature=deposit_request.signature, + slot=state.slot, + )] + elif operation_type == OperationType.FULL_WITHDRAWAL_REQUEST: + validator = state.validators[selected_validator_index] + assert validator.exit_epoch < post_spec.FAR_FUTURE_EPOCH + elif operation_type == OperationType.SWITCH_TO_COMPOUNDING_REQUEST: + validator = state.validators[selected_validator_index] + assert validator.withdrawal_credentials[:1] == post_spec.COMPOUNDING_WITHDRAWAL_PREFIX yield "pre", state diff --git a/tests/core/pyspec/eth2spec/test/helpers/withdrawals.py b/tests/core/pyspec/eth2spec/test/helpers/withdrawals.py index 518920aeb2..f590e81e9a 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/withdrawals.py +++ b/tests/core/pyspec/eth2spec/test/helpers/withdrawals.py @@ -137,6 +137,25 @@ def prepare_pending_withdrawal(spec, state, validator_index, return withdrawal + +def prepare_withdrawal_request(spec, state, validator_index, address=None, amount=None): + if address is not None: + set_eth1_withdrawal_credential_with_balance(spec, state, validator_index, address=address) + + validator = state.validators[validator_index] + if not spec.has_execution_withdrawal_credential(validator): + set_eth1_withdrawal_credential_with_balance(spec, state, validator_index) + + if amount is None: + amount = spec.FULL_EXIT_REQUEST_AMOUNT + + return spec.WithdrawalRequest( + source_address=state.validators[validator_index].withdrawal_credentials[12:], + validator_pubkey=state.validators[validator_index].pubkey, + amount=amount, + ) + + # # Run processing # diff --git a/tests/generators/transition/main.py b/tests/generators/transition/main.py index 75ce72b28c..7acef3335f 100644 --- a/tests/generators/transition/main.py +++ b/tests/generators/transition/main.py @@ -20,6 +20,9 @@ test_operations as test_deneb_operations, test_transition as test_deneb_transition, ) +from eth2spec.test.electra.transition import ( + test_operations as test_electra_operations, +) def create_provider(tests_src, preset_name: str, pre_fork_name: str, post_fork_name: str) -> gen_typing.TestProvider: @@ -49,6 +52,7 @@ def cases_fn() -> Iterable[gen_typing.TestCase]: test_altair_operations, test_deneb_operations, test_deneb_transition, + test_electra_operations, ) for transition_test_module in all_tests: for pre_fork, post_fork in ALL_PRE_POST_FORKS: From aa5ebb3825400f6844b82d350732316bb5fbb81c Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Sun, 20 Oct 2024 09:45:27 +0600 Subject: [PATCH 091/142] Apply suggestions by @jtraglia --- .../test_process_pending_consolidations.py | 18 ++++++++++------- .../test_process_registry_updates.py | 20 ------------------- tests/generators/epoch_processing/main.py | 1 + 3 files changed, 12 insertions(+), 27 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/electra/epoch_processing/test_process_pending_consolidations.py b/tests/core/pyspec/eth2spec/test/electra/epoch_processing/test_process_pending_consolidations.py index 8da4028452..1aca175896 100644 --- a/tests/core/pyspec/eth2spec/test/electra/epoch_processing/test_process_pending_consolidations.py +++ b/tests/core/pyspec/eth2spec/test/electra/epoch_processing/test_process_pending_consolidations.py @@ -306,7 +306,7 @@ def test_pending_consolidation_with_pending_deposit(spec, state): @with_electra_and_later @spec_state_test -def test_pending_consolidation_source_balance_less_than_effective(spec, state): +def test_pending_consolidation_source_balance_less_than_max_effective(spec, state): current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] target_index = spec.get_active_validator_indices(state, current_epoch)[1] @@ -327,6 +327,8 @@ def test_pending_consolidation_source_balance_less_than_effective(spec, state): pre_balance_target = state.balances[target_index] + assert state.balances[source_index] < spec.get_max_effective_balance(state.validators[source_index]) + yield from run_epoch_processing_with(spec, state, "process_pending_consolidations") # Pending consolidation was successfully processed @@ -337,7 +339,7 @@ def test_pending_consolidation_source_balance_less_than_effective(spec, state): @with_electra_and_later @spec_state_test -def test_pending_consolidation_source_balance_greater_than_effective(spec, state): +def test_pending_consolidation_source_balance_greater_than_max_effective(spec, state): current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] target_index = spec.get_active_validator_indices(state, current_epoch)[1] @@ -353,16 +355,18 @@ def test_pending_consolidation_source_balance_greater_than_effective(spec, state ) state.validators[target_index].withdrawal_credentials = eth1_withdrawal_credential # Set the source balance to be greater than effective_balance - pre_balance_source = state.validators[source_index].effective_balance + spec.EFFECTIVE_BALANCE_INCREMENT // 8 + excess_source_balance = spec.EFFECTIVE_BALANCE_INCREMENT // 8 + pre_balance_source = state.validators[source_index].effective_balance + excess_source_balance state.balances[source_index] = pre_balance_source pre_balance_target = state.balances[target_index] + source_max_effective_balance = spec.get_max_effective_balance(state.validators[source_index]) + assert state.balances[source_index] > source_max_effective_balance + yield from run_epoch_processing_with(spec, state, "process_pending_consolidations") # Pending consolidation was successfully processed - assert state.balances[target_index] == ( - pre_balance_target + spec.get_max_effective_balance(state.validators[source_index])) - assert state.balances[source_index] == ( - pre_balance_source - spec.get_max_effective_balance(state.validators[source_index])) + assert state.balances[target_index] == pre_balance_target + source_max_effective_balance + assert state.balances[source_index] == excess_source_balance assert state.pending_consolidations == [] diff --git a/tests/core/pyspec/eth2spec/test/electra/epoch_processing/test_process_registry_updates.py b/tests/core/pyspec/eth2spec/test/electra/epoch_processing/test_process_registry_updates.py index 1f2666402e..df7764befd 100644 --- a/tests/core/pyspec/eth2spec/test/electra/epoch_processing/test_process_registry_updates.py +++ b/tests/core/pyspec/eth2spec/test/electra/epoch_processing/test_process_registry_updates.py @@ -32,10 +32,6 @@ def run_test_activation_queue_eligibility(spec, state, validator_index, balance) @with_electra_and_later @spec_state_test def test_activation_queue_eligibility__less_than_min_activation_balance(spec, state): - # move past first two irregular epochs wrt finality - next_epoch(spec, state) - next_epoch(spec, state) - index = 3 balance = spec.MIN_ACTIVATION_BALANCE - spec.EFFECTIVE_BALANCE_INCREMENT yield from run_test_activation_queue_eligibility(spec, state, index, balance) @@ -44,10 +40,6 @@ def test_activation_queue_eligibility__less_than_min_activation_balance(spec, st @with_electra_and_later @spec_state_test def test_activation_queue_eligibility__min_activation_balance(spec, state): - # move past first two irregular epochs wrt finality - next_epoch(spec, state) - next_epoch(spec, state) - index = 5 balance = spec.MIN_ACTIVATION_BALANCE yield from run_test_activation_queue_eligibility(spec, state, index, balance) @@ -56,10 +48,6 @@ def test_activation_queue_eligibility__min_activation_balance(spec, state): @with_electra_and_later @spec_state_test def test_activation_queue_eligibility__min_activation_balance_eth1_creds(spec, state): - # move past first two irregular epochs wrt finality - next_epoch(spec, state) - next_epoch(spec, state) - index = 7 balance = spec.MIN_ACTIVATION_BALANCE set_eth1_withdrawal_credential_with_balance(spec, state, index) @@ -69,10 +57,6 @@ def test_activation_queue_eligibility__min_activation_balance_eth1_creds(spec, s @with_electra_and_later @spec_state_test def test_activation_queue_eligibility__min_activation_balance_compounding_creds(spec, state): - # move past first two irregular epochs wrt finality - next_epoch(spec, state) - next_epoch(spec, state) - index = 11 balance = spec.MIN_ACTIVATION_BALANCE set_compounding_withdrawal_credential_with_balance(spec, state, index) @@ -82,10 +66,6 @@ def test_activation_queue_eligibility__min_activation_balance_compounding_creds( @with_electra_and_later @spec_state_test def test_activation_queue_eligibility__greater_than_min_activation_balance(spec, state): - # move past first two irregular epochs wrt finality - next_epoch(spec, state) - next_epoch(spec, state) - index = 13 balance = spec.MIN_ACTIVATION_BALANCE + spec.EFFECTIVE_BALANCE_INCREMENT set_compounding_withdrawal_credential_with_balance(spec, state, index) diff --git a/tests/generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py index 4c848a9d12..62653e2fe8 100644 --- a/tests/generators/epoch_processing/main.py +++ b/tests/generators/epoch_processing/main.py @@ -40,6 +40,7 @@ _new_electra_mods_1 = {key: 'eth2spec.test.electra.epoch_processing.test_process_' + key for key in [ 'effective_balance_updates', 'pending_consolidations', + 'registry_updates', ]} # This is a trick to allow tests be split into multiple files and use the same test format. _new_electra_mods_2 = {key: 'eth2spec.test.electra.epoch_processing.' + key for key in [ From 44c1c3141f2d97c8570bf0b63e358b0a17231f69 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Mon, 21 Oct 2024 13:17:59 +0600 Subject: [PATCH 092/142] Add consolidation tests for comp creds --- .../test_process_pending_consolidations.py | 82 +++++++++++++++++-- 1 file changed, 73 insertions(+), 9 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/electra/epoch_processing/test_process_pending_consolidations.py b/tests/core/pyspec/eth2spec/test/electra/epoch_processing/test_process_pending_consolidations.py index 1aca175896..b061efee7b 100644 --- a/tests/core/pyspec/eth2spec/test/electra/epoch_processing/test_process_pending_consolidations.py +++ b/tests/core/pyspec/eth2spec/test/electra/epoch_processing/test_process_pending_consolidations.py @@ -9,6 +9,10 @@ from eth2spec.test.helpers.state import ( next_epoch_with_full_participation, ) +from eth2spec.test.helpers.withdrawals import ( + set_eth1_withdrawal_credential_with_balance, + set_compounding_withdrawal_credential_with_balance, +) # *********************** # * CONSOLIDATION TESTS * @@ -316,11 +320,9 @@ def test_pending_consolidation_source_balance_less_than_max_effective(spec, stat ) # Set withdrawable epoch to current epoch to allow processing state.validators[source_index].withdrawable_epoch = current_epoch - # Set the target withdrawal credential to eth1 - eth1_withdrawal_credential = ( - spec.ETH1_ADDRESS_WITHDRAWAL_PREFIX + b"\x00" * 11 + b"\x11" * 20 - ) - state.validators[target_index].withdrawal_credentials = eth1_withdrawal_credential + # Set source and target withdrawal credential to eth1 + set_eth1_withdrawal_credential_with_balance(spec, state, source_index) + set_eth1_withdrawal_credential_with_balance(spec, state, target_index) # Set the source balance to be less than effective_balance pre_balance_source = state.validators[source_index].effective_balance - spec.EFFECTIVE_BALANCE_INCREMENT // 8 state.balances[source_index] = pre_balance_source @@ -349,11 +351,73 @@ def test_pending_consolidation_source_balance_greater_than_max_effective(spec, s ) # Set withdrawable epoch to current epoch to allow processing state.validators[source_index].withdrawable_epoch = current_epoch - # Set the target withdrawal credential to eth1 - eth1_withdrawal_credential = ( - spec.ETH1_ADDRESS_WITHDRAWAL_PREFIX + b"\x00" * 11 + b"\x11" * 20 + # Set source and target withdrawal credential to eth1 + set_eth1_withdrawal_credential_with_balance(spec, state, source_index) + set_eth1_withdrawal_credential_with_balance(spec, state, target_index) + # Set the source balance to be greater than effective_balance + excess_source_balance = spec.EFFECTIVE_BALANCE_INCREMENT // 8 + pre_balance_source = state.validators[source_index].effective_balance + excess_source_balance + state.balances[source_index] = pre_balance_source + + pre_balance_target = state.balances[target_index] + + source_max_effective_balance = spec.get_max_effective_balance(state.validators[source_index]) + assert state.balances[source_index] > source_max_effective_balance + + yield from run_epoch_processing_with(spec, state, "process_pending_consolidations") + + # Pending consolidation was successfully processed + assert state.balances[target_index] == pre_balance_target + source_max_effective_balance + assert state.balances[source_index] == excess_source_balance + assert state.pending_consolidations == [] + + +@with_electra_and_later +@spec_state_test +def test_pending_consolidation_source_balance_less_than_max_effective_compounding(spec, state): + current_epoch = spec.get_current_epoch(state) + source_index = spec.get_active_validator_indices(state, current_epoch)[0] + target_index = spec.get_active_validator_indices(state, current_epoch)[1] + # append pending consolidation + state.pending_consolidations.append( + spec.PendingConsolidation(source_index=source_index, target_index=target_index) ) - state.validators[target_index].withdrawal_credentials = eth1_withdrawal_credential + # Set withdrawable epoch to current epoch to allow processing + state.validators[source_index].withdrawable_epoch = current_epoch + # Set source and target withdrawal credential to compounding + set_compounding_withdrawal_credential_with_balance(spec, state, source_index) + set_compounding_withdrawal_credential_with_balance(spec, state, target_index) + # Set the source balance to be less than effective_balance + pre_balance_source = state.validators[source_index].effective_balance - spec.EFFECTIVE_BALANCE_INCREMENT // 8 + state.balances[source_index] = pre_balance_source + + pre_balance_target = state.balances[target_index] + + assert state.balances[source_index] < spec.get_max_effective_balance(state.validators[source_index]) + + yield from run_epoch_processing_with(spec, state, "process_pending_consolidations") + + # Pending consolidation was successfully processed + assert state.balances[target_index] == pre_balance_target + pre_balance_source + assert state.balances[source_index] == 0 + assert state.pending_consolidations == [] + + +@with_electra_and_later +@spec_state_test +def test_pending_consolidation_source_balance_greater_than_max_effective_compounding(spec, state): + current_epoch = spec.get_current_epoch(state) + source_index = spec.get_active_validator_indices(state, current_epoch)[0] + target_index = spec.get_active_validator_indices(state, current_epoch)[1] + # append pending consolidation + state.pending_consolidations.append( + spec.PendingConsolidation(source_index=source_index, target_index=target_index) + ) + # Set withdrawable epoch to current epoch to allow processing + state.validators[source_index].withdrawable_epoch = current_epoch + # Set source and target withdrawal credential to compounding + set_compounding_withdrawal_credential_with_balance(spec, state, source_index) + set_compounding_withdrawal_credential_with_balance(spec, state, target_index) # Set the source balance to be greater than effective_balance excess_source_balance = spec.EFFECTIVE_BALANCE_INCREMENT // 8 pre_balance_source = state.validators[source_index].effective_balance + excess_source_balance From a232b25e7f1067b1301be37c9daab9b8685ee4dc Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Mon, 21 Oct 2024 09:17:42 +0200 Subject: [PATCH 093/142] language --- specs/phase0/p2p-interface.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index bc99c3ee63..0e1953946e 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -563,7 +563,7 @@ At this point, the stream will be half-closed. The requester MUST NOT make more than `MAX_CONCURRENT_REQUESTS` concurrent requests with the same protocol ID. -If a timeout happens on the requesting side, they SHOULD reset the stream. +If a timeout occurs or the response is no longer relevant, the requester SHOULD reset the stream. A requester SHOULD read from the stream until either: 1. An error result is received in one of the chunks (the error payload MAY be read before stopping). @@ -592,7 +592,7 @@ The responder MUST: If steps (1), (2), or (3) fail due to invalid, malformed, or inconsistent data, the responder MUST respond in error. Clients tracking peer reputation MAY record such failures, as well as unexpected events, e.g. early stream resets. -The responder MAY rate-limit chunks by withholding each chunk until capacity is available. The responding side MUST NOT respond with an error or close the stream when rate limiting. +The responder MAY rate-limit chunks by withholding each chunk until capacity is available. The responder MUST NOT respond with an error or close the stream when rate limiting. When rate limiting, the responder MUST send each `response_chunk` in full promptly but may introduce delays between each chunk. @@ -1458,15 +1458,15 @@ Req/Resp is used to avoid confusion with JSON-RPC and similar user-client intera #### What is a typical rate limiting strategy? -The serving side typically will want to rate limit requests to protect against spam and to manage resource consumption, while the requesting side will want to maximise performance based on its own resource allocation strategy. For the network, it is beneficial if available resources are used optimally. +The responder typically will want to rate limit requests to protect against spam and to manage resource consumption, while the requester will want to maximise performance based on its own resource allocation strategy. For the network, it is beneficial if available resources are used optimally. -Broadly, the requesting side does not know the capacity / limit of each server but can derive it from the rate of responses for the purpose of selecting the next peer for a request. +Broadly, the requester does not know the capacity / limit of each server but can derive it from the rate of responses for the purpose of selecting the next peer for a request. Because the server withholds the response until capacity is available, a client can optimistically send requests without risking running into negative scoring situations or sub-optimal rate polling. -A typical approach for the requesting side is to implement a timeout on the request that depends on the nature of the request and on connectivity parameters in general - for example when requesting blocks, a peer might choose to send a request to a second peer if the first peer does not respond within a reasonable time, and to reset the request to the first peer if the second peer responds faster. Clients may use past response performance to reward fast peers when implementing peer scoring. +A typical approach for the requester is to implement a timeout on the request that depends on the nature of the request and on connectivity parameters in general - for example when requesting blocks, a peer might choose to send a request to a second peer if the first peer does not respond within a reasonable time, and to reset the request to the first peer if the second peer responds faster. Clients may use past response performance to reward fast peers when implementing peer scoring. -A typical approach for the responding side is to implement a two-level token/leaky bucket with a per-peer limit and a global limit. The granularity of rate limiting may be based either on full requests or individual chunks with the latter being preferable. A token cost may be assigned to the request itself and separately each chunk in the response so as to remain protected both against large and frequent requests. +A typical approach for the responder is to implement a two-level token/leaky bucket with a per-peer limit and a global limit. The granularity of rate limiting may be based either on full requests or individual chunks with the latter being preferable. A token cost may be assigned to the request itself and separately each chunk in the response so as to remain protected both against large and frequent requests. For requesters, rate limiting is not distinguishable from other conditions causing slow responses (slow peers, congestion etc) and since the latter conditions must be handled anyway, including rate limiting in this strategy keeps the implementation simple. @@ -1501,10 +1501,10 @@ If a request for the parent fails, it's indicative of poor peer quality since pe When connecting, the `Status` message gives an idea about the sync status of a particular peer, but this changes over time. By the time a subsequent `BeaconBlockByRange` request is processed, the information may be stale, -and the responding side might have moved on to a new finalization point and pruned blocks around the previous head and finalized blocks. +and the responder might have moved on to a new finalization point and pruned blocks around the previous head and finalized blocks. -To avoid this race condition, we allow the responding side to choose which branch to send to the requesting client. -The requesting client then goes on to validate the blocks and incorporate them in their own database +To avoid this race condition, we allow the responder to choose which branch to send to the requester. +The requester then goes on to validate the blocks and incorporate them in their own database -- because they follow the same rules, they should at this point arrive at the same canonical chain. #### Why are `BlocksByRange` requests only required to be served for the latest `MIN_EPOCHS_FOR_BLOCK_REQUESTS` epochs? From a6095bf498243dfd2619a996c5c9f9fcbf79cfdc Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Mon, 21 Oct 2024 13:00:10 +0200 Subject: [PATCH 094/142] Update specs/deneb/beacon-chain.md Co-authored-by: Justin Traglia <95511699+jtraglia@users.noreply.github.com> --- specs/deneb/beacon-chain.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 1b7a322c2f..98a4b05bda 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -293,9 +293,8 @@ def verify_and_notify_new_payload(self: ExecutionEngine, """ Return ``True`` if and only if ``new_payload_request`` is valid with respect to ``self.execution_state``. """ - # [Modified in Deneb:EIP4788] execution_payload = new_payload_request.execution_payload - parent_beacon_block_root = new_payload_request.parent_beacon_block_root + parent_beacon_block_root = new_payload_request.parent_beacon_block_root # [Modified in Deneb:EIP4788] if b'' in execution_payload.transactions: return False From 44d5a1b49bffbddc79f9a49ca4245c7ef0e7c693 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Mon, 21 Oct 2024 13:01:30 +0200 Subject: [PATCH 095/142] Sync changes to Electra --- specs/electra/beacon-chain.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/specs/electra/beacon-chain.md b/specs/electra/beacon-chain.md index 0c56491907..89575d671c 100644 --- a/specs/electra/beacon-chain.md +++ b/specs/electra/beacon-chain.md @@ -1015,6 +1015,9 @@ def verify_and_notify_new_payload(self: ExecutionEngine, parent_beacon_block_root = new_payload_request.parent_beacon_block_root execution_requests_list = get_execution_requests_list(new_payload_request.execution_requests) # [New in Electra] + if b'' in execution_payload.transactions: + return False + if not self.is_valid_block_hash(execution_payload, parent_beacon_block_root): return False From 5270d7913dc3a18a49514036a881a5c33374c538 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Mon, 21 Oct 2024 17:20:14 +0600 Subject: [PATCH 096/142] Use generalized op types for Electra transition tests --- .../test/electra/transition/test_operations.py | 8 ++++---- .../pyspec/eth2spec/test/helpers/fork_transition.py | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/electra/transition/test_operations.py b/tests/core/pyspec/eth2spec/test/electra/transition/test_operations.py index b927a1bf1d..7f5a1af9f5 100644 --- a/tests/core/pyspec/eth2spec/test/electra/transition/test_operations.py +++ b/tests/core/pyspec/eth2spec/test/electra/transition/test_operations.py @@ -63,7 +63,7 @@ def test_transition_with_full_withdrawal_request_right_after_fork( post_spec, pre_tag, post_tag, - operation_type=OperationType.FULL_WITHDRAWAL_REQUEST, + operation_type=OperationType.WITHDRAWAL_REQUEST, operation_at_slot=fork_epoch * spec.SLOTS_PER_EPOCH, ) @@ -75,7 +75,7 @@ def test_transition_with_full_withdrawal_request_right_after_fork( @with_fork_metas([ForkMeta(pre_fork_name=pre, post_fork_name=post, fork_epoch=2) for pre, post in AFTER_ELECTRA_PRE_POST_FORKS]) @always_bls -def test_transition_with_switch_to_compounding_request_right_after_fork( +def test_transition_with_consolidation_request_right_after_fork( state, fork_epoch, spec, @@ -84,7 +84,7 @@ def test_transition_with_switch_to_compounding_request_right_after_fork( post_tag ): """ - Create a SWITCH_TO_COMPOUNDING_REQUEST right *after* the transition + Create a CONSOLIDATION_REQUEST right *after* the transition """ yield from run_transition_with_operation( state, @@ -93,6 +93,6 @@ def test_transition_with_switch_to_compounding_request_right_after_fork( post_spec, pre_tag, post_tag, - operation_type=OperationType.SWITCH_TO_COMPOUNDING_REQUEST, + operation_type=OperationType.CONSOLIDATION_REQUEST, operation_at_slot=fork_epoch * spec.SLOTS_PER_EPOCH, ) diff --git a/tests/core/pyspec/eth2spec/test/helpers/fork_transition.py b/tests/core/pyspec/eth2spec/test/helpers/fork_transition.py index 41d159e6d8..69e1be669b 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/fork_transition.py +++ b/tests/core/pyspec/eth2spec/test/helpers/fork_transition.py @@ -53,8 +53,8 @@ class OperationType(Enum): VOLUNTARY_EXIT = auto() BLS_TO_EXECUTION_CHANGE = auto() DEPOSIT_REQUEST = auto() - FULL_WITHDRAWAL_REQUEST = auto() - SWITCH_TO_COMPOUNDING_REQUEST = auto() + WITHDRAWAL_REQUEST = auto() + CONSOLIDATION_REQUEST = auto() def _set_operations_by_dict(block, operation_dict): @@ -348,12 +348,12 @@ def run_transition_with_operation(state, amount = post_spec.MIN_ACTIVATION_BALANCE deposit_request = prepare_deposit_request(post_spec, selected_validator_index, amount, signed=True) operation_dict = {'execution_requests.deposits': [deposit_request]} - elif operation_type == OperationType.FULL_WITHDRAWAL_REQUEST: + elif operation_type == OperationType.WITHDRAWAL_REQUEST: selected_validator_index = 0 withdrawal_request = prepare_withdrawal_request( post_spec, state, selected_validator_index, amount=post_spec.FULL_EXIT_REQUEST_AMOUNT) operation_dict = {'execution_requests.withdrawals': [withdrawal_request]} - elif operation_type == OperationType.SWITCH_TO_COMPOUNDING_REQUEST: + elif operation_type == OperationType.CONSOLIDATION_REQUEST: selected_validator_index = 0 consolidation_request = prepare_switch_to_compounding_request(post_spec, state, selected_validator_index) operation_dict = {'execution_requests.consolidations': [consolidation_request]} @@ -389,10 +389,10 @@ def _check_state(): signature=deposit_request.signature, slot=state.slot, )] - elif operation_type == OperationType.FULL_WITHDRAWAL_REQUEST: + elif operation_type == OperationType.WITHDRAWAL_REQUEST: validator = state.validators[selected_validator_index] assert validator.exit_epoch < post_spec.FAR_FUTURE_EPOCH - elif operation_type == OperationType.SWITCH_TO_COMPOUNDING_REQUEST: + elif operation_type == OperationType.CONSOLIDATION_REQUEST: validator = state.validators[selected_validator_index] assert validator.withdrawal_credentials[:1] == post_spec.COMPOUNDING_WITHDRAWAL_PREFIX From 0f964b04d288433fdf096e223031bd3e73a09807 Mon Sep 17 00:00:00 2001 From: Justin Traglia <95511699+jtraglia@users.noreply.github.com> Date: Mon, 21 Oct 2024 08:43:25 -0500 Subject: [PATCH 097/142] Change "modified" to "new" --- specs/deneb/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 98a4b05bda..1025af7b61 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -294,7 +294,7 @@ def verify_and_notify_new_payload(self: ExecutionEngine, Return ``True`` if and only if ``new_payload_request`` is valid with respect to ``self.execution_state``. """ execution_payload = new_payload_request.execution_payload - parent_beacon_block_root = new_payload_request.parent_beacon_block_root # [Modified in Deneb:EIP4788] + parent_beacon_block_root = new_payload_request.parent_beacon_block_root # [New in Deneb:EIP4788] if b'' in execution_payload.transactions: return False From 82093ddaacc61eab9f44bedda757650e998996c2 Mon Sep 17 00:00:00 2001 From: Justin Traglia <95511699+jtraglia@users.noreply.github.com> Date: Mon, 21 Oct 2024 14:11:37 -0500 Subject: [PATCH 098/142] Revert some changes --- .github/workflows/run-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 0fe661eb98..b8c2fa6e14 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -24,7 +24,7 @@ on: - cron: '0 0 * * *' jobs: - table-of-contents: + table_of_contents: runs-on: [self-hosted-ghr-custom, size-s-x64, profile-consensusSpecs] steps: - name: Checkout repository @@ -71,7 +71,7 @@ jobs: pyspec-tests: runs-on: [self-hosted-ghr-custom, size-xl-x64, profile-consensusSpecs] - needs: [lint, codespell, table-of-contents] + needs: [lint,codespell,table_of_contents] strategy: matrix: version: ["phase0", "altair", "bellatrix", "capella", "deneb", "electra", "whisk", "eip7594"] @@ -104,7 +104,7 @@ jobs: - name: Install pyspec requirements run: make install_test - name: test-${{ matrix.version }} - run: make citest fork=${{ matrix.version }} TEST_PRESET_TYPE=${{ env.spec_test_preset_type }} + run: make citest fork=${{ matrix.version }} TEST_PRESET_TYPE=${{env.spec_test_preset_type}} - uses: actions/upload-artifact@v4 if: always() with: From 6de2e297ea3db93f9882630de83e07c6d7693650 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Mon, 21 Oct 2024 14:25:17 -0500 Subject: [PATCH 099/142] Upgrade pylint & split config files --- Makefile | 9 ++++----- flake8.ini | 3 +++ linter.ini | 18 ------------------ mypy.ini | 7 +++++++ pylint.ini | 3 +++ setup.py | 2 +- 6 files changed, 18 insertions(+), 24 deletions(-) create mode 100644 flake8.ini delete mode 100644 linter.ini create mode 100644 mypy.ini create mode 100644 pylint.ini diff --git a/Makefile b/Makefile index 39d493645e..657a746586 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,6 @@ COV_HTML_OUT_DIR=$(PY_SPEC_DIR)/$(COV_HTML_OUT) COV_INDEX_FILE=$(COV_HTML_OUT_DIR)/index.html CURRENT_DIR = ${CURDIR} -LINTER_CONFIG_FILE = $(CURRENT_DIR)/linter.ini GENERATOR_ERROR_LOG_FILE = $(CURRENT_DIR)/$(TEST_VECTOR_DIR)/testgen_error_log.txt SCRIPTS_DIR = ${CURRENT_DIR}/scripts @@ -151,13 +150,13 @@ codespell: lint: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ - flake8 --config $(LINTER_CONFIG_FILE) ./eth2spec \ - && python -m pylint --rcfile $(LINTER_CONFIG_FILE) $(PYLINT_SCOPE) \ - && python -m mypy --config-file $(LINTER_CONFIG_FILE) $(MYPY_SCOPE) + flake8 --config $(CURRENT_DIR)/flake8.ini ./eth2spec \ + && python -m pylint --rcfile $(CURRENT_DIR)/pylint.ini $(PYLINT_SCOPE) \ + && python -m mypy --config-file $(CURRENT_DIR)/mypy.ini $(MYPY_SCOPE) lint_generators: pyspec . venv/bin/activate; cd $(TEST_GENERATORS_DIR); \ - flake8 --config $(LINTER_CONFIG_FILE) + flake8 --config $(CURRENT_DIR)/flake8.ini # Runs a generator, identified by param 1 define run_generator diff --git a/flake8.ini b/flake8.ini new file mode 100644 index 0000000000..f94b2ad47a --- /dev/null +++ b/flake8.ini @@ -0,0 +1,3 @@ +[flake8] +ignore = E252,W504,W503 +max-line-length = 120 \ No newline at end of file diff --git a/linter.ini b/linter.ini deleted file mode 100644 index 52a3aec0e0..0000000000 --- a/linter.ini +++ /dev/null @@ -1,18 +0,0 @@ -[flake8] -ignore = E252,W504,W503 -max-line-length = 120 - -[mypy] -disallow_incomplete_defs = True -disallow_untyped_defs = True - -warn_unused_ignores = True -warn_unused_configs = True -warn_redundant_casts = True - -ignore_missing_imports = True - -# pylint -[MESSAGES CONTROL] -disable = all -enable = unused-argument diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000000..6f16a21c7a --- /dev/null +++ b/mypy.ini @@ -0,0 +1,7 @@ +[mypy] +disallow_incomplete_defs = True +disallow_untyped_defs = True +warn_unused_ignores = True +warn_unused_configs = True +warn_redundant_casts = True +ignore_missing_imports = True \ No newline at end of file diff --git a/pylint.ini b/pylint.ini new file mode 100644 index 0000000000..a8242683a3 --- /dev/null +++ b/pylint.ini @@ -0,0 +1,3 @@ +[MESSAGES CONTROL] +disable = all +enable = unused-argument \ No newline at end of file diff --git a/setup.py b/setup.py index 8d5aa213b7..83ae7accb4 100644 --- a/setup.py +++ b/setup.py @@ -561,7 +561,7 @@ def run(self): python_requires=">=3.9, <4", extras_require={ "test": ["pytest>=4.4", "pytest-cov", "pytest-xdist"], - "lint": ["flake8==5.0.4", "mypy==0.981", "pylint==2.15.3"], + "lint": ["flake8==5.0.4", "mypy==0.981", "pylint==3.3.1"], "generator": ["setuptools>=72.0.0", "pytest>4.4", "python-snappy==0.7.3", "filelock", "pathos==0.3.0"], "docs": ["mkdocs==1.4.2", "mkdocs-material==9.1.5", "mdx-truly-sane-lists==1.3", "mkdocs-awesome-pages-plugin==2.8.0"] }, From f2e45fb774ace1ff73ef92c4570554979eed742f Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Tue, 22 Oct 2024 12:36:17 +0600 Subject: [PATCH 100/142] Simplify test helpers --- .../core/pyspec/eth2spec/test/helpers/consolidations.py | 9 ++++----- tests/core/pyspec/eth2spec/test/helpers/withdrawals.py | 9 ++++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/consolidations.py b/tests/core/pyspec/eth2spec/test/helpers/consolidations.py index f47ea669fc..94faf42757 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/consolidations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/consolidations.py @@ -4,12 +4,11 @@ def prepare_switch_to_compounding_request(spec, state, validator_index, address=None): - if address is not None: - set_eth1_withdrawal_credential_with_balance(spec, state, validator_index, address=address) - validator = state.validators[validator_index] - if not spec.has_execution_withdrawal_credential(validator): - set_eth1_withdrawal_credential_with_balance(spec, state, validator_index) + + # Set the supplied address or the default one if needed + if address is not None or not spec.has_execution_withdrawal_credential(validator): + set_eth1_withdrawal_credential_with_balance(spec, state, validator_index, address=address) return spec.ConsolidationRequest( source_address=state.validators[validator_index].withdrawal_credentials[12:], diff --git a/tests/core/pyspec/eth2spec/test/helpers/withdrawals.py b/tests/core/pyspec/eth2spec/test/helpers/withdrawals.py index f590e81e9a..afa0cc00e3 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/withdrawals.py +++ b/tests/core/pyspec/eth2spec/test/helpers/withdrawals.py @@ -139,12 +139,11 @@ def prepare_pending_withdrawal(spec, state, validator_index, def prepare_withdrawal_request(spec, state, validator_index, address=None, amount=None): - if address is not None: - set_eth1_withdrawal_credential_with_balance(spec, state, validator_index, address=address) - validator = state.validators[validator_index] - if not spec.has_execution_withdrawal_credential(validator): - set_eth1_withdrawal_credential_with_balance(spec, state, validator_index) + + # Set the supplied address or the default one if needed + if address is not None or not spec.has_execution_withdrawal_credential(validator): + set_eth1_withdrawal_credential_with_balance(spec, state, validator_index, address=address) if amount is None: amount = spec.FULL_EXIT_REQUEST_AMOUNT From d53419bffb622cbe01ff66ab818abfd0a8fb310c Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Tue, 22 Oct 2024 12:39:51 +0600 Subject: [PATCH 101/142] Simplify test helpers, take 2 --- tests/core/pyspec/eth2spec/test/helpers/consolidations.py | 4 +--- tests/core/pyspec/eth2spec/test/helpers/withdrawals.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/consolidations.py b/tests/core/pyspec/eth2spec/test/helpers/consolidations.py index 94faf42757..c16108b97a 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/consolidations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/consolidations.py @@ -5,9 +5,7 @@ def prepare_switch_to_compounding_request(spec, state, validator_index, address=None): validator = state.validators[validator_index] - - # Set the supplied address or the default one if needed - if address is not None or not spec.has_execution_withdrawal_credential(validator): + if not spec.has_execution_withdrawal_credential(validator): set_eth1_withdrawal_credential_with_balance(spec, state, validator_index, address=address) return spec.ConsolidationRequest( diff --git a/tests/core/pyspec/eth2spec/test/helpers/withdrawals.py b/tests/core/pyspec/eth2spec/test/helpers/withdrawals.py index afa0cc00e3..802cd65ec4 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/withdrawals.py +++ b/tests/core/pyspec/eth2spec/test/helpers/withdrawals.py @@ -140,9 +140,7 @@ def prepare_pending_withdrawal(spec, state, validator_index, def prepare_withdrawal_request(spec, state, validator_index, address=None, amount=None): validator = state.validators[validator_index] - - # Set the supplied address or the default one if needed - if address is not None or not spec.has_execution_withdrawal_credential(validator): + if not spec.has_execution_withdrawal_credential(validator): set_eth1_withdrawal_credential_with_balance(spec, state, validator_index, address=address) if amount is None: From dc772a6dca031ea41e8b8976822c3a84a49d907a Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Tue, 22 Oct 2024 13:33:00 -0500 Subject: [PATCH 102/142] Add test_fork_pending_deposits_are_sorted test --- .../electra/fork/test_electra_fork_basic.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/electra/fork/test_electra_fork_basic.py b/tests/core/pyspec/eth2spec/test/electra/fork/test_electra_fork_basic.py index e569be35e3..bf97746dbc 100644 --- a/tests/core/pyspec/eth2spec/test/electra/fork/test_electra_fork_basic.py +++ b/tests/core/pyspec/eth2spec/test/electra/fork/test_electra_fork_basic.py @@ -94,6 +94,27 @@ def test_fork_pre_activation(spec, phases, state): assert len(post_state.pending_deposits) > 0 +@with_phases(phases=[DENEB], other_phases=[ELECTRA]) +@spec_test +@with_state +@with_meta_tags(ELECTRA_FORK_TEST_META_TAGS) +def test_fork_pending_deposits_are_sorted(spec, phases, state): + post_spec = phases[ELECTRA] + state.validators[0].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[0].activation_eligibility_epoch = 2 + state.validators[1].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[1].activation_eligibility_epoch = 3 + state.validators[2].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[2].activation_eligibility_epoch = 2 + + post_state = yield from run_fork_test(post_spec, state) + + assert len(post_state.pending_deposits) == 3 + assert post_state.pending_deposits[0].pubkey == state.validators[0].pubkey + assert post_state.pending_deposits[1].pubkey == state.validators[2].pubkey + assert post_state.pending_deposits[2].pubkey == state.validators[1].pubkey + + @with_phases(phases=[DENEB], other_phases=[ELECTRA]) @spec_test @with_state From 9507b86bebb02e149f675b0fa1859aee91fe118e Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 23 Oct 2024 09:57:05 -0500 Subject: [PATCH 103/142] Add one more validator --- .../test/electra/fork/test_electra_fork_basic.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/electra/fork/test_electra_fork_basic.py b/tests/core/pyspec/eth2spec/test/electra/fork/test_electra_fork_basic.py index bf97746dbc..884bfcb4eb 100644 --- a/tests/core/pyspec/eth2spec/test/electra/fork/test_electra_fork_basic.py +++ b/tests/core/pyspec/eth2spec/test/electra/fork/test_electra_fork_basic.py @@ -106,13 +106,16 @@ def test_fork_pending_deposits_are_sorted(spec, phases, state): state.validators[1].activation_eligibility_epoch = 3 state.validators[2].activation_epoch = spec.FAR_FUTURE_EPOCH state.validators[2].activation_eligibility_epoch = 2 + state.validators[3].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[3].activation_eligibility_epoch = 1 post_state = yield from run_fork_test(post_spec, state) - assert len(post_state.pending_deposits) == 3 - assert post_state.pending_deposits[0].pubkey == state.validators[0].pubkey - assert post_state.pending_deposits[1].pubkey == state.validators[2].pubkey - assert post_state.pending_deposits[2].pubkey == state.validators[1].pubkey + assert len(post_state.pending_deposits) == 4 + assert post_state.pending_deposits[0].pubkey == state.validators[3].pubkey + assert post_state.pending_deposits[1].pubkey == state.validators[0].pubkey + assert post_state.pending_deposits[2].pubkey == state.validators[2].pubkey + assert post_state.pending_deposits[3].pubkey == state.validators[1].pubkey @with_phases(phases=[DENEB], other_phases=[ELECTRA]) From 3a4101b3e4a219498dccf96971cc526c8aafb531 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 23 Oct 2024 14:20:53 -0500 Subject: [PATCH 104/142] Rename index vars in get_attesting_indices() --- specs/electra/beacon-chain.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/specs/electra/beacon-chain.md b/specs/electra/beacon-chain.md index 0c56491907..ca6c588455 100644 --- a/specs/electra/beacon-chain.md +++ b/specs/electra/beacon-chain.md @@ -572,10 +572,12 @@ def get_attesting_indices(state: BeaconState, attestation: Attestation) -> Set[V output: Set[ValidatorIndex] = set() committee_indices = get_committee_indices(attestation.committee_bits) committee_offset = 0 - for index in committee_indices: - committee = get_beacon_committee(state, attestation.data.slot, index) + for committee_index in committee_indices: + committee = get_beacon_committee(state, attestation.data.slot, committee_index) committee_attesters = set( - index for i, index in enumerate(committee) if attestation.aggregation_bits[committee_offset + i]) + attester_index for i, attester_index in enumerate(committee) + if attestation.aggregation_bits[committee_offset + i] + ) output = output.union(committee_attesters) committee_offset += len(committee) From 90ae0a4ecfb36da82a7c353f15f66652f561aeeb Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Thu, 24 Oct 2024 14:48:49 -0500 Subject: [PATCH 105/142] Group pending types & request types --- specs/electra/beacon-chain.md | 51 ++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/specs/electra/beacon-chain.md b/specs/electra/beacon-chain.md index 0c56491907..3813a1f445 100644 --- a/specs/electra/beacon-chain.md +++ b/specs/electra/beacon-chain.md @@ -24,12 +24,12 @@ - [Validator cycle](#validator-cycle) - [Containers](#containers) - [New containers](#new-containers) - - [`DepositRequest`](#depositrequest) - [`PendingDeposit`](#pendingdeposit) - [`PendingPartialWithdrawal`](#pendingpartialwithdrawal) + - [`PendingConsolidation`](#pendingconsolidation) + - [`DepositRequest`](#depositrequest) - [`WithdrawalRequest`](#withdrawalrequest) - [`ConsolidationRequest`](#consolidationrequest) - - [`PendingConsolidation`](#pendingconsolidation) - [`ExecutionRequests`](#executionrequests) - [Modified Containers](#modified-containers) - [`AttesterSlashing`](#attesterslashing) @@ -201,19 +201,6 @@ The following values are (non-configurable) constants used throughout the specif ### New containers -#### `DepositRequest` - -*Note*: The container is new in EIP6110. - -```python -class DepositRequest(Container): - pubkey: BLSPubkey - withdrawal_credentials: Bytes32 - amount: Gwei - signature: BLSSignature - index: uint64 -``` - #### `PendingDeposit` *Note*: The container is new in EIP7251. @@ -237,6 +224,30 @@ class PendingPartialWithdrawal(Container): amount: Gwei withdrawable_epoch: Epoch ``` + +#### `PendingConsolidation` + +*Note*: The container is new in EIP7251. + +```python +class PendingConsolidation(Container): + source_index: ValidatorIndex + target_index: ValidatorIndex +``` + +#### `DepositRequest` + +*Note*: The container is new in EIP6110. + +```python +class DepositRequest(Container): + pubkey: BLSPubkey + withdrawal_credentials: Bytes32 + amount: Gwei + signature: BLSSignature + index: uint64 +``` + #### `WithdrawalRequest` *Note*: The container is new in EIP7251:EIP7002. @@ -259,16 +270,6 @@ class ConsolidationRequest(Container): target_pubkey: BLSPubkey ``` -#### `PendingConsolidation` - -*Note*: The container is new in EIP7251. - -```python -class PendingConsolidation(Container): - source_index: ValidatorIndex - target_index: ValidatorIndex -``` - #### `ExecutionRequests` *Note*: This container holds requests from the execution layer that are received in [ From 0c5a1362c983c7e49519b9a603e62f13c0b45963 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Tue, 29 Oct 2024 14:42:13 -0500 Subject: [PATCH 106/142] Exclude empty requests in requests list --- specs/electra/beacon-chain.md | 24 +++- specs/electra/validator.md | 43 +++++-- .../unittests/test_execution_requests.py | 115 ++++++++++++++++++ 3 files changed, 169 insertions(+), 13 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/electra/unittests/test_execution_requests.py diff --git a/specs/electra/beacon-chain.md b/specs/electra/beacon-chain.md index 0c56491907..5f0f77a2b8 100644 --- a/specs/electra/beacon-chain.md +++ b/specs/electra/beacon-chain.md @@ -137,6 +137,14 @@ The following values are (non-configurable) constants used throughout the specif | - | - | | `COMPOUNDING_WITHDRAWAL_PREFIX` | `Bytes1('0x02')` | +### Execution layer triggered requests + +| Name | Value | +| - | - | +| `DEPOSIT_REQUEST_TYPE` | `Bytes1('0x00')` | +| `WITHDRAWAL_REQUEST_TYPE` | `Bytes1('0x01')` | +| `CONSOLIDATION_REQUEST_TYPE` | `Bytes1('0x02')` | + ## Preset ### Gwei values @@ -1146,11 +1154,17 @@ def process_withdrawals(state: BeaconState, payload: ExecutionPayload) -> None: ```python def get_execution_requests_list(execution_requests: ExecutionRequests) -> Sequence[bytes]: - deposit_bytes = ssz_serialize(execution_requests.deposits) - withdrawal_bytes = ssz_serialize(execution_requests.withdrawals) - consolidation_bytes = ssz_serialize(execution_requests.consolidations) - - return [deposit_bytes, withdrawal_bytes, consolidation_bytes] + requests = [ + (DEPOSIT_REQUEST_TYPE, execution_requests.deposits), + (WITHDRAWAL_REQUEST_TYPE, execution_requests.withdrawals), + (CONSOLIDATION_REQUEST_TYPE, execution_requests.consolidations), + ] + + return [ + request_type + ssz_serialize(request_data) + for request_type, request_data in requests + if len(request_data) != 0 + ] ``` ##### Modified `process_execution_payload` diff --git a/specs/electra/validator.md b/specs/electra/validator.md index 553eeaa702..c896efe11b 100644 --- a/specs/electra/validator.md +++ b/specs/electra/validator.md @@ -189,18 +189,45 @@ def prepare_execution_payload(state: BeaconState, *[New in Electra]* -1. The execution payload is obtained from the execution engine as defined above using `payload_id`. The response also includes a `execution_requests` entry containing a list of bytes. Each element on the list corresponds to one SSZ list of requests as defined -in [EIP-7685](https://eips.ethereum.org/EIPS/eip-7685). The index of each element in the array determines the type of request. +1. The execution payload is obtained from the execution engine as defined above using `payload_id`. The response also includes a `execution_requests` entry containing a list of bytes. Each element on the list corresponds to one SSZ list of requests as defined in [EIP-7685](https://eips.ethereum.org/EIPS/eip-7685). The first byte of each request is used to determine the request type. There can only be one instance of each request type per execution requests. 2. Set `block.body.execution_requests = get_execution_requests(execution_requests)`, where: ```python def get_execution_requests(execution_requests: Sequence[bytes]) -> ExecutionRequests: - deposits = ssz_deserialize(List[DepositRequest, MAX_DEPOSIT_REQUESTS_PER_PAYLOAD], execution_requests[0]) - withdrawals = ssz_deserialize(List[WithdrawalRequest, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD], execution_requests[1]) - consolidations = ssz_deserialize(List[ConsolidationRequest, MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD], - execution_requests[2]) - - return ExecutionRequests(deposits, withdrawals, consolidations) + deposits = None + withdrawals = None + consolidations = None + + for request in execution_requests: + request_type, request_data = request[0:1], request[1:] + assert len(request_data) != 0, "empty request data" + + if request_type == DEPOSIT_REQUEST_TYPE: + assert deposits is None, "duplicate deposit request" + deposits = ssz_deserialize( + List[DepositRequest, MAX_DEPOSIT_REQUESTS_PER_PAYLOAD], + request_data + ) + elif request_type == WITHDRAWAL_REQUEST_TYPE: + assert withdrawals is None, "duplicate withdrawal request" + withdrawals = ssz_deserialize( + List[WithdrawalRequest, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD], + request_data + ) + elif request_type == CONSOLIDATION_REQUEST_TYPE: + assert consolidations is None, "duplicate consolidation request" + consolidations = ssz_deserialize( + List[ConsolidationRequest, MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD], + request_data + ) + else: + raise ValueError(f"unexpected request type: {repr(request_type)}") + + return ExecutionRequests( + deposits=deposits or [], + withdrawals=withdrawals or [], + consolidations=consolidations or [] + ) ``` ## Attesting diff --git a/tests/core/pyspec/eth2spec/test/electra/unittests/test_execution_requests.py b/tests/core/pyspec/eth2spec/test/electra/unittests/test_execution_requests.py new file mode 100644 index 0000000000..581b1a54a8 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/electra/unittests/test_execution_requests.py @@ -0,0 +1,115 @@ +from eth2spec.test.context import ( + single_phase, + spec_test, + with_electra_and_later, +) + + +@with_electra_and_later +@spec_test +@single_phase +def test_requests_serialization_round_trip__empty(spec): + execution_requests = spec.ExecutionRequests() + serialized_execution_requests = spec.get_execution_requests_list(execution_requests) + deserialized_execution_requests = spec.get_execution_requests(serialized_execution_requests) + assert deserialized_execution_requests == execution_requests + + +@with_electra_and_later +@spec_test +@single_phase +def test_requests_serialization_round_trip__one_request(spec): + execution_requests = spec.ExecutionRequests( + deposits=[spec.DepositRequest()], + ) + serialized_execution_requests = spec.get_execution_requests_list(execution_requests) + deserialized_execution_requests = spec.get_execution_requests(serialized_execution_requests) + assert deserialized_execution_requests == execution_requests + + +@with_electra_and_later +@spec_test +@single_phase +def test_requests_serialization_round_trip__multiple_requests(spec): + execution_requests = spec.ExecutionRequests( + deposits=[spec.DepositRequest()], + withdrawals=[spec.WithdrawalRequest()], + consolidations=[spec.ConsolidationRequest()], + ) + serialized_execution_requests = spec.get_execution_requests_list(execution_requests) + deserialized_execution_requests = spec.get_execution_requests(serialized_execution_requests) + assert deserialized_execution_requests == execution_requests + + +@with_electra_and_later +@spec_test +@single_phase +def test_requests_serialization_round_trip__one_request_with_real_data(spec): + execution_requests = spec.ExecutionRequests( + deposits=[ + spec.DepositRequest( + pubkey=spec.BLSPubkey(48 * "aa"), + withdrawal_credentials=spec.Bytes32(32 * "bb"), + amount=spec.Gwei(11111111), + signature=spec.BLSSignature(96 * "cc"), + index=spec.uint64(22222222), + ), + ] + ) + serialized_execution_requests = spec.get_execution_requests_list(execution_requests) + deserialized_execution_requests = spec.get_execution_requests(serialized_execution_requests) + assert deserialized_execution_requests == execution_requests + + +@with_electra_and_later +@spec_test +@single_phase +def test_requests_deserialize__allow_out_of_order_requests(spec): + serialized_execution_requests = [ + spec.WITHDRAWAL_REQUEST_TYPE + 76 * b"\x0a", + spec.DEPOSIT_REQUEST_TYPE + 192 * b"\x0b", + ] + assert int(serialized_execution_requests[0][0]) > int(serialized_execution_requests[1][0]) + spec.get_execution_requests(serialized_execution_requests) + + +@with_electra_and_later +@spec_test +@single_phase +def test_requests_deserialize__reject_empty_request(spec): + serialized_execution_requests = [b"\x01"] + try: + spec.get_execution_requests(serialized_execution_requests) + assert False, "expected exception" + except Exception as e: + assert "empty request data" in str(e) + + +@with_electra_and_later +@spec_test +@single_phase +def test_requests_deserialize__reject_duplicate_request(spec): + serialized_withdrawal = 76 * b"\x0a" + serialized_execution_requests = [ + spec.WITHDRAWAL_REQUEST_TYPE + serialized_withdrawal, + spec.WITHDRAWAL_REQUEST_TYPE + serialized_withdrawal, + ] + try: + spec.get_execution_requests(serialized_execution_requests) + assert False, "expected exception" + except Exception as e: + assert "duplicate withdrawal request" in str(e) + + +@with_electra_and_later +@spec_test +@single_phase +def test_requests_deserialize__reject_unexpected_request_type(spec): + serialized_execution_requests = [ + b"\x03\xff\xff\xff", + ] + try: + spec.get_execution_requests(serialized_execution_requests) + assert False, "expected exception" + except Exception as e: + assert "unexpected request type" in str(e) From 369b7e578072aa3c6640351db39fd696a4fd484c Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Tue, 29 Oct 2024 14:47:16 -0500 Subject: [PATCH 107/142] Fix table of contents --- specs/electra/beacon-chain.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/electra/beacon-chain.md b/specs/electra/beacon-chain.md index 5f0f77a2b8..940bba7b8b 100644 --- a/specs/electra/beacon-chain.md +++ b/specs/electra/beacon-chain.md @@ -12,6 +12,7 @@ - [Constants](#constants) - [Misc](#misc) - [Withdrawal prefixes](#withdrawal-prefixes) + - [Execution layer triggered requests](#execution-layer-triggered-requests) - [Preset](#preset) - [Gwei values](#gwei-values) - [Rewards and penalties](#rewards-and-penalties) From 2bd570c6be385d372ad877713e9d0c366b07ca2f Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Tue, 29 Oct 2024 14:58:21 -0500 Subject: [PATCH 108/142] Use assert instead of raise --- specs/electra/validator.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/specs/electra/validator.md b/specs/electra/validator.md index c896efe11b..6b544f56e4 100644 --- a/specs/electra/validator.md +++ b/specs/electra/validator.md @@ -199,7 +199,13 @@ def get_execution_requests(execution_requests: Sequence[bytes]) -> ExecutionRequ consolidations = None for request in execution_requests: + assert len(request) >= 1 request_type, request_data = request[0:1], request[1:] + assert request_type in [ + DEPOSIT_REQUEST_TYPE, + WITHDRAWAL_REQUEST_TYPE, + CONSOLIDATION_REQUEST_TYPE, + ], f"unexpected request type: {repr(request_type)}" assert len(request_data) != 0, "empty request data" if request_type == DEPOSIT_REQUEST_TYPE: @@ -220,8 +226,6 @@ def get_execution_requests(execution_requests: Sequence[bytes]) -> ExecutionRequ List[ConsolidationRequest, MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD], request_data ) - else: - raise ValueError(f"unexpected request type: {repr(request_type)}") return ExecutionRequests( deposits=deposits or [], From 8a8a3214cdaaedd6622329de1753d7484df46b4b Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Tue, 29 Oct 2024 15:10:22 -0500 Subject: [PATCH 109/142] Fix indentation --- specs/electra/validator.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/electra/validator.md b/specs/electra/validator.md index 6b544f56e4..028d4b659d 100644 --- a/specs/electra/validator.md +++ b/specs/electra/validator.md @@ -202,9 +202,9 @@ def get_execution_requests(execution_requests: Sequence[bytes]) -> ExecutionRequ assert len(request) >= 1 request_type, request_data = request[0:1], request[1:] assert request_type in [ - DEPOSIT_REQUEST_TYPE, - WITHDRAWAL_REQUEST_TYPE, - CONSOLIDATION_REQUEST_TYPE, + DEPOSIT_REQUEST_TYPE, + WITHDRAWAL_REQUEST_TYPE, + CONSOLIDATION_REQUEST_TYPE, ], f"unexpected request type: {repr(request_type)}" assert len(request_data) != 0, "empty request data" From 3ce29297ce1f954e006d60f926d9c16b008cce0f Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Tue, 29 Oct 2024 16:04:03 -0500 Subject: [PATCH 110/142] Simplify get_execution_requests --- specs/electra/validator.md | 63 ++++++++----------- .../unittests/test_execution_requests.py | 2 +- 2 files changed, 27 insertions(+), 38 deletions(-) diff --git a/specs/electra/validator.md b/specs/electra/validator.md index 028d4b659d..4348b0a320 100644 --- a/specs/electra/validator.md +++ b/specs/electra/validator.md @@ -193,45 +193,34 @@ def prepare_execution_payload(state: BeaconState, 2. Set `block.body.execution_requests = get_execution_requests(execution_requests)`, where: ```python -def get_execution_requests(execution_requests: Sequence[bytes]) -> ExecutionRequests: - deposits = None - withdrawals = None - consolidations = None - - for request in execution_requests: - assert len(request) >= 1 +def get_execution_requests(execution_requests_list: Sequence[bytes]) -> ExecutionRequests: + requests = { + DEPOSIT_REQUEST_TYPE: { + "value": None, + "field": "deposits", + "type": List[DepositRequest, MAX_DEPOSIT_REQUESTS_PER_PAYLOAD], + }, + WITHDRAWAL_REQUEST_TYPE: { + "value": None, + "field": "withdrawals", + "type": List[WithdrawalRequest, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD], + }, + CONSOLIDATION_REQUEST_TYPE: { + "value": None, + "field": "consolidations", + "type": List[ConsolidationRequest, MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD], + }, + } + + execution_requests = ExecutionRequests() + for request in execution_requests_list: request_type, request_data = request[0:1], request[1:] - assert request_type in [ - DEPOSIT_REQUEST_TYPE, - WITHDRAWAL_REQUEST_TYPE, - CONSOLIDATION_REQUEST_TYPE, - ], f"unexpected request type: {repr(request_type)}" + assert request_type in requests, "unexpected request type" assert len(request_data) != 0, "empty request data" - - if request_type == DEPOSIT_REQUEST_TYPE: - assert deposits is None, "duplicate deposit request" - deposits = ssz_deserialize( - List[DepositRequest, MAX_DEPOSIT_REQUESTS_PER_PAYLOAD], - request_data - ) - elif request_type == WITHDRAWAL_REQUEST_TYPE: - assert withdrawals is None, "duplicate withdrawal request" - withdrawals = ssz_deserialize( - List[WithdrawalRequest, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD], - request_data - ) - elif request_type == CONSOLIDATION_REQUEST_TYPE: - assert consolidations is None, "duplicate consolidation request" - consolidations = ssz_deserialize( - List[ConsolidationRequest, MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD], - request_data - ) - - return ExecutionRequests( - deposits=deposits or [], - withdrawals=withdrawals or [], - consolidations=consolidations or [] - ) + assert requests[request_type]["value"] is None, "duplicate request" + requests[request_type]["value"] = ssz_deserialize(requests[request_type]["type"], request_data) + setattr(execution_requests, requests[request_type]["field"], requests[request_type]["value"]) + return execution_requests ``` ## Attesting diff --git a/tests/core/pyspec/eth2spec/test/electra/unittests/test_execution_requests.py b/tests/core/pyspec/eth2spec/test/electra/unittests/test_execution_requests.py index 581b1a54a8..1204ffce17 100644 --- a/tests/core/pyspec/eth2spec/test/electra/unittests/test_execution_requests.py +++ b/tests/core/pyspec/eth2spec/test/electra/unittests/test_execution_requests.py @@ -98,7 +98,7 @@ def test_requests_deserialize__reject_duplicate_request(spec): spec.get_execution_requests(serialized_execution_requests) assert False, "expected exception" except Exception as e: - assert "duplicate withdrawal request" in str(e) + assert "duplicate request" in str(e) @with_electra_and_later From 4438166552c25536ed6c7d90712dfb2e85fc7d22 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Tue, 29 Oct 2024 19:22:27 -0500 Subject: [PATCH 111/142] Check for ascending order --- specs/electra/validator.md | 13 ++++--- .../unittests/test_execution_requests.py | 34 +++++++++++-------- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/specs/electra/validator.md b/specs/electra/validator.md index 4348b0a320..04e521ab84 100644 --- a/specs/electra/validator.md +++ b/specs/electra/validator.md @@ -189,37 +189,36 @@ def prepare_execution_payload(state: BeaconState, *[New in Electra]* -1. The execution payload is obtained from the execution engine as defined above using `payload_id`. The response also includes a `execution_requests` entry containing a list of bytes. Each element on the list corresponds to one SSZ list of requests as defined in [EIP-7685](https://eips.ethereum.org/EIPS/eip-7685). The first byte of each request is used to determine the request type. There can only be one instance of each request type per execution requests. +1. The execution payload is obtained from the execution engine as defined above using `payload_id`. The response also includes a `execution_requests` entry containing a list of bytes. Each element on the list corresponds to one SSZ list of requests as defined in [EIP-7685](https://eips.ethereum.org/EIPS/eip-7685). The first byte of each request is used to determine the request type. Requests must be order by request type in ascending order. As a result, there can only be one instance of each request type per execution requests. 2. Set `block.body.execution_requests = get_execution_requests(execution_requests)`, where: ```python def get_execution_requests(execution_requests_list: Sequence[bytes]) -> ExecutionRequests: requests = { DEPOSIT_REQUEST_TYPE: { - "value": None, "field": "deposits", "type": List[DepositRequest, MAX_DEPOSIT_REQUESTS_PER_PAYLOAD], }, WITHDRAWAL_REQUEST_TYPE: { - "value": None, "field": "withdrawals", "type": List[WithdrawalRequest, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD], }, CONSOLIDATION_REQUEST_TYPE: { - "value": None, "field": "consolidations", "type": List[ConsolidationRequest, MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD], }, } + prev_request_type = None execution_requests = ExecutionRequests() for request in execution_requests_list: request_type, request_data = request[0:1], request[1:] assert request_type in requests, "unexpected request type" assert len(request_data) != 0, "empty request data" - assert requests[request_type]["value"] is None, "duplicate request" - requests[request_type]["value"] = ssz_deserialize(requests[request_type]["type"], request_data) - setattr(execution_requests, requests[request_type]["field"], requests[request_type]["value"]) + assert prev_request_type is None or prev_request_type < request_type, "not ascending order" + prev_request_type = request_type + deserialized_request = ssz_deserialize(requests[request_type]["type"], request_data) + setattr(execution_requests, requests[request_type]["field"], deserialized_request) return execution_requests ``` diff --git a/tests/core/pyspec/eth2spec/test/electra/unittests/test_execution_requests.py b/tests/core/pyspec/eth2spec/test/electra/unittests/test_execution_requests.py index 1204ffce17..2a63c17a3f 100644 --- a/tests/core/pyspec/eth2spec/test/electra/unittests/test_execution_requests.py +++ b/tests/core/pyspec/eth2spec/test/electra/unittests/test_execution_requests.py @@ -64,41 +64,45 @@ def test_requests_serialization_round_trip__one_request_with_real_data(spec): @with_electra_and_later @spec_test @single_phase -def test_requests_deserialize__allow_out_of_order_requests(spec): +def test_requests_deserialize__reject_duplicate_request(spec): + serialized_withdrawal = 76 * b"\x0a" serialized_execution_requests = [ - spec.WITHDRAWAL_REQUEST_TYPE + 76 * b"\x0a", - spec.DEPOSIT_REQUEST_TYPE + 192 * b"\x0b", + spec.WITHDRAWAL_REQUEST_TYPE + serialized_withdrawal, + spec.WITHDRAWAL_REQUEST_TYPE + serialized_withdrawal, ] - assert int(serialized_execution_requests[0][0]) > int(serialized_execution_requests[1][0]) - spec.get_execution_requests(serialized_execution_requests) + try: + spec.get_execution_requests(serialized_execution_requests) + assert False, "expected exception" + except Exception as e: + assert "not ascending order" in str(e) @with_electra_and_later @spec_test @single_phase -def test_requests_deserialize__reject_empty_request(spec): - serialized_execution_requests = [b"\x01"] +def test_requests_deserialize__reject_out_of_order_requests(spec): + serialized_execution_requests = [ + spec.WITHDRAWAL_REQUEST_TYPE + 76 * b"\x0a", + spec.DEPOSIT_REQUEST_TYPE + 192 * b"\x0b", + ] + assert int(serialized_execution_requests[0][0]) > int(serialized_execution_requests[1][0]) try: spec.get_execution_requests(serialized_execution_requests) assert False, "expected exception" except Exception as e: - assert "empty request data" in str(e) + assert "not ascending order" in str(e) @with_electra_and_later @spec_test @single_phase -def test_requests_deserialize__reject_duplicate_request(spec): - serialized_withdrawal = 76 * b"\x0a" - serialized_execution_requests = [ - spec.WITHDRAWAL_REQUEST_TYPE + serialized_withdrawal, - spec.WITHDRAWAL_REQUEST_TYPE + serialized_withdrawal, - ] +def test_requests_deserialize__reject_empty_request(spec): + serialized_execution_requests = [b"\x01"] try: spec.get_execution_requests(serialized_execution_requests) assert False, "expected exception" except Exception as e: - assert "duplicate request" in str(e) + assert "empty request data" in str(e) @with_electra_and_later From 0ad0f4e3fdc7ba14725e75e11e4a5fef212244d2 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 30 Oct 2024 12:45:18 -0500 Subject: [PATCH 112/142] Append to pending deposits once in apply_deposit --- specs/electra/beacon-chain.md | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/specs/electra/beacon-chain.md b/specs/electra/beacon-chain.md index 4542b80754..28b281adca 100644 --- a/specs/electra/beacon-chain.md +++ b/specs/electra/beacon-chain.md @@ -1341,24 +1341,16 @@ def apply_deposit(state: BeaconState, # Verify the deposit signature (proof of possession) which is not checked by the deposit contract if is_valid_deposit_signature(pubkey, withdrawal_credentials, amount, signature): add_validator_to_registry(state, pubkey, withdrawal_credentials, Gwei(0)) # [Modified in Electra:EIP7251] - # [New in Electra:EIP7251] - state.pending_deposits.append(PendingDeposit( - pubkey=pubkey, - withdrawal_credentials=withdrawal_credentials, - amount=amount, - signature=signature, - slot=GENESIS_SLOT, # Use GENESIS_SLOT to distinguish from a pending deposit request - )) - else: - # Increase balance by deposit amount - # [Modified in Electra:EIP7251] - state.pending_deposits.append(PendingDeposit( - pubkey=pubkey, - withdrawal_credentials=withdrawal_credentials, - amount=amount, - signature=signature, - slot=GENESIS_SLOT # Use GENESIS_SLOT to distinguish from a pending deposit request - )) + + # Increase balance by deposit amount + # [Modified in Electra:EIP7251] + state.pending_deposits.append(PendingDeposit( + pubkey=pubkey, + withdrawal_credentials=withdrawal_credentials, + amount=amount, + signature=signature, + slot=GENESIS_SLOT # Use GENESIS_SLOT to distinguish from a pending deposit request + )) ``` ###### New `is_valid_deposit_signature` From a5a6939297629b8db34f2ffe01e3901283148920 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 30 Oct 2024 13:09:19 -0500 Subject: [PATCH 113/142] Delete useless comment --- specs/electra/beacon-chain.md | 1 - 1 file changed, 1 deletion(-) diff --git a/specs/electra/beacon-chain.md b/specs/electra/beacon-chain.md index 28b281adca..8bf793df87 100644 --- a/specs/electra/beacon-chain.md +++ b/specs/electra/beacon-chain.md @@ -845,7 +845,6 @@ def apply_pending_deposit(state: BeaconState, deposit: PendingDeposit) -> None: add_validator_to_registry(state, deposit.pubkey, deposit.withdrawal_credentials, deposit.amount) else: validator_index = ValidatorIndex(validator_pubkeys.index(deposit.pubkey)) - # Increase balance increase_balance(state, validator_index, deposit.amount) ``` From 076a0f154bf6c913b7e93e875961e92420aa300c Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 30 Oct 2024 13:52:01 -0500 Subject: [PATCH 114/142] Return without adding deposit if signature fails --- specs/electra/beacon-chain.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specs/electra/beacon-chain.md b/specs/electra/beacon-chain.md index 8bf793df87..f2eb19289b 100644 --- a/specs/electra/beacon-chain.md +++ b/specs/electra/beacon-chain.md @@ -1340,6 +1340,8 @@ def apply_deposit(state: BeaconState, # Verify the deposit signature (proof of possession) which is not checked by the deposit contract if is_valid_deposit_signature(pubkey, withdrawal_credentials, amount, signature): add_validator_to_registry(state, pubkey, withdrawal_credentials, Gwei(0)) # [Modified in Electra:EIP7251] + else: + return # Increase balance by deposit amount # [Modified in Electra:EIP7251] From 5a6121b2d26ab143fe883ac893abb7a3d1957d31 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 30 Oct 2024 13:54:56 -0500 Subject: [PATCH 115/142] Fix indentation --- specs/electra/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/electra/beacon-chain.md b/specs/electra/beacon-chain.md index f2eb19289b..59415b7c79 100644 --- a/specs/electra/beacon-chain.md +++ b/specs/electra/beacon-chain.md @@ -1341,7 +1341,7 @@ def apply_deposit(state: BeaconState, if is_valid_deposit_signature(pubkey, withdrawal_credentials, amount, signature): add_validator_to_registry(state, pubkey, withdrawal_credentials, Gwei(0)) # [Modified in Electra:EIP7251] else: - return + return # Increase balance by deposit amount # [Modified in Electra:EIP7251] From e90c792c5cf4412f6885f48f6a639a53ac570522 Mon Sep 17 00:00:00 2001 From: Justin Traglia <95511699+jtraglia@users.noreply.github.com> Date: Wed, 30 Oct 2024 15:51:49 -0500 Subject: [PATCH 116/142] Fix typo Co-authored-by: Alex Stokes --- specs/electra/validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/electra/validator.md b/specs/electra/validator.md index 04e521ab84..b79c262507 100644 --- a/specs/electra/validator.md +++ b/specs/electra/validator.md @@ -189,7 +189,7 @@ def prepare_execution_payload(state: BeaconState, *[New in Electra]* -1. The execution payload is obtained from the execution engine as defined above using `payload_id`. The response also includes a `execution_requests` entry containing a list of bytes. Each element on the list corresponds to one SSZ list of requests as defined in [EIP-7685](https://eips.ethereum.org/EIPS/eip-7685). The first byte of each request is used to determine the request type. Requests must be order by request type in ascending order. As a result, there can only be one instance of each request type per execution requests. +1. The execution payload is obtained from the execution engine as defined above using `payload_id`. The response also includes a `execution_requests` entry containing a list of bytes. Each element on the list corresponds to one SSZ list of requests as defined in [EIP-7685](https://eips.ethereum.org/EIPS/eip-7685). The first byte of each request is used to determine the request type. Requests must be ordered by request type in ascending order. As a result, there can only be one instance of each request type per execution requests. 2. Set `block.body.execution_requests = get_execution_requests(execution_requests)`, where: ```python From b3e77d2f575d9a03a3afd13269cfb6ca421cef78 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 30 Oct 2024 15:55:41 -0500 Subject: [PATCH 117/142] Apply other stokes suggestion --- specs/electra/validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/electra/validator.md b/specs/electra/validator.md index b79c262507..55f9f10273 100644 --- a/specs/electra/validator.md +++ b/specs/electra/validator.md @@ -189,7 +189,7 @@ def prepare_execution_payload(state: BeaconState, *[New in Electra]* -1. The execution payload is obtained from the execution engine as defined above using `payload_id`. The response also includes a `execution_requests` entry containing a list of bytes. Each element on the list corresponds to one SSZ list of requests as defined in [EIP-7685](https://eips.ethereum.org/EIPS/eip-7685). The first byte of each request is used to determine the request type. Requests must be ordered by request type in ascending order. As a result, there can only be one instance of each request type per execution requests. +1. The execution payload is obtained from the execution engine as defined above using `payload_id`. The response also includes a `execution_requests` entry containing a list of bytes. Each element on the list corresponds to one SSZ list of requests as defined in [EIP-7685](https://eips.ethereum.org/EIPS/eip-7685). The first byte of each request is used to determine the request type. Requests must be ordered by request type in ascending order. As a result, there can only be at most one instance of each request type. 2. Set `block.body.execution_requests = get_execution_requests(execution_requests)`, where: ```python From dfb3ca6c2a303d8902042ef8e2e3135ee4b64516 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 30 Oct 2024 16:41:41 -0500 Subject: [PATCH 118/142] Add new test --- .../block_processing/test_process_deposit.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py index a688952b49..ca86845f30 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py +++ b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py @@ -46,6 +46,24 @@ def test_new_deposit_over_max(spec, state): yield from run_deposit_processing(spec, state, deposit, validator_index) +@with_all_phases +@spec_state_test +def test_new_deposit_with_bad_signature(spec, state): + # fresh deposit = next validator index = validator appended to registry + validator_index = len(state.validators) + # a deposit that is not signed + amount = spec.EFFECTIVE_BALANCE_INCREMENT + deposit = prepare_state_and_deposit(spec, state, validator_index, amount, signed=False) + # save state so we can access pending deposits later + pre_state = state + + yield from run_deposit_processing(spec, state, deposit, validator_index, effective=False) + + if is_post_electra(spec): + # there should not be a new pending deposit + assert state.pending_deposits == pre_state.pending_deposits + + @with_all_phases @spec_state_test def test_new_deposit_eth1_withdrawal_credentials(spec, state): From 276c56201a56d9e343f500db13d4b8dd4482a1f7 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 30 Oct 2024 16:47:53 -0500 Subject: [PATCH 119/142] Use less python sugar --- specs/electra/validator.md | 51 ++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/specs/electra/validator.md b/specs/electra/validator.md index 55f9f10273..1cd4b91eca 100644 --- a/specs/electra/validator.md +++ b/specs/electra/validator.md @@ -194,32 +194,45 @@ def prepare_execution_payload(state: BeaconState, ```python def get_execution_requests(execution_requests_list: Sequence[bytes]) -> ExecutionRequests: - requests = { - DEPOSIT_REQUEST_TYPE: { - "field": "deposits", - "type": List[DepositRequest, MAX_DEPOSIT_REQUESTS_PER_PAYLOAD], - }, - WITHDRAWAL_REQUEST_TYPE: { - "field": "withdrawals", - "type": List[WithdrawalRequest, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD], - }, - CONSOLIDATION_REQUEST_TYPE: { - "field": "consolidations", - "type": List[ConsolidationRequest, MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD], - }, - } + deposits = [] + withdrawals = [] + consolidations = [] + + request_types = [ + DEPOSIT_REQUEST_TYPE, + WITHDRAWAL_REQUEST_TYPE, + CONSOLIDATION_REQUEST_TYPE, + ] prev_request_type = None - execution_requests = ExecutionRequests() for request in execution_requests_list: request_type, request_data = request[0:1], request[1:] - assert request_type in requests, "unexpected request type" + assert request_type in request_types, "unexpected request type" assert len(request_data) != 0, "empty request data" assert prev_request_type is None or prev_request_type < request_type, "not ascending order" prev_request_type = request_type - deserialized_request = ssz_deserialize(requests[request_type]["type"], request_data) - setattr(execution_requests, requests[request_type]["field"], deserialized_request) - return execution_requests + + if request_type == DEPOSIT_REQUEST_TYPE: + deposits = ssz_deserialize( + List[DepositRequest, MAX_DEPOSIT_REQUESTS_PER_PAYLOAD], + request_data + ) + elif request_type == WITHDRAWAL_REQUEST_TYPE: + withdrawals = ssz_deserialize( + List[WithdrawalRequest, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD], + request_data + ) + elif request_type == CONSOLIDATION_REQUEST_TYPE: + consolidations = ssz_deserialize( + List[ConsolidationRequest, MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD], + request_data + ) + + return ExecutionRequests( + deposits=deposits, + withdrawals=withdrawals, + consolidations=consolidations, + ) ``` ## Attesting From 2c59da7a33545691b70bbd50471e328285f8d5a5 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 30 Oct 2024 17:08:59 -0500 Subject: [PATCH 120/142] Delete new test & update run_deposit_processing --- .../pyspec/eth2spec/test/helpers/deposits.py | 16 +++++++++------- .../block_processing/test_process_deposit.py | 18 ------------------ 2 files changed, 9 insertions(+), 25 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/deposits.py b/tests/core/pyspec/eth2spec/test/helpers/deposits.py index bd0d670491..985b08c7c8 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/deposits.py +++ b/tests/core/pyspec/eth2spec/test/helpers/deposits.py @@ -271,7 +271,7 @@ def run_deposit_processing(spec, state, deposit, validator_index, valid=True, ef pre_effective_balance = state.validators[validator_index].effective_balance if is_post_electra(spec): - pre_pending_deposits = len(state.pending_deposits) + pre_pending_deposits_count = len(state.pending_deposits) yield 'pre', state yield 'deposit', deposit @@ -290,6 +290,8 @@ def run_deposit_processing(spec, state, deposit, validator_index, valid=True, ef assert len(state.balances) == pre_validator_count if is_top_up: assert get_balance(state, validator_index) == pre_balance + if is_post_electra(spec): + assert len(state.pending_deposits) == pre_pending_deposits_count else: if is_top_up: # Top-ups don't add validators @@ -313,13 +315,13 @@ def run_deposit_processing(spec, state, deposit, validator_index, valid=True, ef assert get_balance(state, validator_index) == pre_balance assert state.validators[validator_index].effective_balance == pre_effective_balance # new correct balance deposit queued up - assert len(state.pending_deposits) == pre_pending_deposits + 1 - assert state.pending_deposits[pre_pending_deposits].pubkey == deposit.data.pubkey + assert len(state.pending_deposits) == pre_pending_deposits_count + 1 + assert state.pending_deposits[pre_pending_deposits_count].pubkey == deposit.data.pubkey assert state.pending_deposits[ - pre_pending_deposits].withdrawal_credentials == deposit.data.withdrawal_credentials - assert state.pending_deposits[pre_pending_deposits].amount == deposit.data.amount - assert state.pending_deposits[pre_pending_deposits].signature == deposit.data.signature - assert state.pending_deposits[pre_pending_deposits].slot == spec.GENESIS_SLOT + pre_pending_deposits_count].withdrawal_credentials == deposit.data.withdrawal_credentials + assert state.pending_deposits[pre_pending_deposits_count].amount == deposit.data.amount + assert state.pending_deposits[pre_pending_deposits_count].signature == deposit.data.signature + assert state.pending_deposits[pre_pending_deposits_count].slot == spec.GENESIS_SLOT assert state.eth1_deposit_index == state.eth1_data.deposit_count diff --git a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py index ca86845f30..a688952b49 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py +++ b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py @@ -46,24 +46,6 @@ def test_new_deposit_over_max(spec, state): yield from run_deposit_processing(spec, state, deposit, validator_index) -@with_all_phases -@spec_state_test -def test_new_deposit_with_bad_signature(spec, state): - # fresh deposit = next validator index = validator appended to registry - validator_index = len(state.validators) - # a deposit that is not signed - amount = spec.EFFECTIVE_BALANCE_INCREMENT - deposit = prepare_state_and_deposit(spec, state, validator_index, amount, signed=False) - # save state so we can access pending deposits later - pre_state = state - - yield from run_deposit_processing(spec, state, deposit, validator_index, effective=False) - - if is_post_electra(spec): - # there should not be a new pending deposit - assert state.pending_deposits == pre_state.pending_deposits - - @with_all_phases @spec_state_test def test_new_deposit_eth1_withdrawal_credentials(spec, state): From 31d4974262b0aeaf3ae24f70f94fc02119c6137f Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Thu, 31 Oct 2024 20:06:32 +0600 Subject: [PATCH 121/142] Add more exit checks to consolidation processing --- specs/electra/beacon-chain.md | 6 + .../test_process_consolidation_request.py | 120 ++++++++++++++++++ 2 files changed, 126 insertions(+) diff --git a/specs/electra/beacon-chain.md b/specs/electra/beacon-chain.md index 29f1f807fe..3734b1cf61 100644 --- a/specs/electra/beacon-chain.md +++ b/specs/electra/beacon-chain.md @@ -1626,6 +1626,12 @@ def process_consolidation_request( return if target_validator.exit_epoch != FAR_FUTURE_EPOCH: return + # Verify the source has been active long enough + if current_epoch < source_validator.activation_epoch + SHARD_COMMITTEE_PERIOD: + return + # Verify the source has no pending withdrawals in the queue + if get_pending_balance_to_withdraw(state, source_index) > 0: + return # Initiate source validator exit and append pending consolidation source_validator.exit_epoch = compute_consolidation_epoch_and_update_churn( diff --git a/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_consolidation_request.py b/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_consolidation_request.py index 49744946f0..8fdbb8e2e5 100644 --- a/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_consolidation_request.py +++ b/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_consolidation_request.py @@ -28,6 +28,8 @@ @spec_test @single_phase def test_basic_consolidation_in_current_consolidation_epoch(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # This state has 256 validators each with 32 ETH in MINIMAL preset, 128 ETH consolidation churn current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -75,6 +77,8 @@ def test_basic_consolidation_in_current_consolidation_epoch(spec, state): @spec_test @single_phase def test_basic_consolidation_with_excess_target_balance(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # This state has 256 validators each with 32 ETH in MINIMAL preset, 128 ETH consolidation churn current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -125,6 +129,8 @@ def test_basic_consolidation_with_excess_target_balance(spec, state): @spec_test @single_phase def test_basic_consolidation_with_excess_target_balance_and_compounding_credentials(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # This state has 256 validators each with 32 ETH in MINIMAL preset, 128 ETH consolidation churn current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -175,6 +181,8 @@ def test_basic_consolidation_with_excess_target_balance_and_compounding_credenti @spec_test @single_phase def test_basic_consolidation_in_new_consolidation_epoch(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # This state has 256 validators each with 32 ETH in MINIMAL preset, 128 ETH consolidation churn # Set consolidation balance to consume to some arbitrary nonzero value below the churn limit state.consolidation_balance_to_consume = spec.EFFECTIVE_BALANCE_INCREMENT @@ -220,6 +228,8 @@ def test_basic_consolidation_in_new_consolidation_epoch(spec, state): @spec_test @single_phase def test_basic_consolidation_with_preexisting_churn(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # This state has 256 validators each with 32 ETH in MINIMAL preset, 128 ETH consolidation churn current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -267,6 +277,8 @@ def test_basic_consolidation_with_preexisting_churn(spec, state): @spec_test @single_phase def test_basic_consolidation_with_insufficient_preexisting_churn(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # This state has 256 validators each with 32 ETH in MINIMAL preset, 128 ETH consolidation churn current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -318,6 +330,8 @@ def test_basic_consolidation_with_insufficient_preexisting_churn(spec, state): @spec_test @single_phase def test_basic_consolidation_with_compounding_credentials(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # This state has 256 validators each with 32 ETH in MINIMAL preset, 128 ETH consolidation churn current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -363,6 +377,8 @@ def test_basic_consolidation_with_compounding_credentials(spec, state): @spec_test @single_phase def test_consolidation_churn_limit_balance(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # This state has 256 validators each with 32 ETH in MINIMAL preset, 128 ETH consolidation churn current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -411,6 +427,8 @@ def test_consolidation_churn_limit_balance(spec, state): @spec_test @single_phase def test_consolidation_balance_larger_than_churn_limit(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # This state has 256 validators each with 32 ETH in MINIMAL preset, 128 ETH consolidation churn current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -458,6 +476,8 @@ def test_consolidation_balance_larger_than_churn_limit(spec, state): @spec_test @single_phase def test_consolidation_balance_through_two_churn_epochs(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # This state has 256 validators each with 32 ETH in MINIMAL preset, 128 ETH consolidation churn current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -498,6 +518,8 @@ def test_consolidation_balance_through_two_churn_epochs(spec, state): @with_electra_and_later @spec_state_test def test_basic_switch_to_compounding(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -583,6 +605,9 @@ def test_switch_to_compounding_with_pending_consolidations_at_limit(spec, state) @spec_test @single_phase def test_incorrect_exceed_pending_consolidations_limit(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + state.pending_consolidations = [ spec.PendingConsolidation(source_index=0, target_index=1) ] * spec.PENDING_CONSOLIDATIONS_LIMIT @@ -614,6 +639,9 @@ def test_incorrect_exceed_pending_consolidations_limit(spec, state): @spec_state_test @single_phase def test_incorrect_not_enough_consolidation_churn_available(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + state.pending_consolidations = [ spec.PendingConsolidation(source_index=0, target_index=1) ] @@ -651,6 +679,8 @@ def test_incorrect_not_enough_consolidation_churn_available(spec, state): @spec_test @single_phase def test_incorrect_exited_source(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # Set up an otherwise correct consolidation current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -686,6 +716,8 @@ def test_incorrect_exited_source(spec, state): @spec_test @single_phase def test_incorrect_exited_target(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # Set up an otherwise correct consolidation current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -720,6 +752,8 @@ def test_incorrect_exited_target(spec, state): @spec_test @single_phase def test_incorrect_inactive_source(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # Set up an otherwise correct consolidation current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -755,6 +789,8 @@ def test_incorrect_inactive_source(spec, state): @spec_test @single_phase def test_incorrect_inactive_target(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # Set up an otherwise correct consolidation current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -790,6 +826,8 @@ def test_incorrect_inactive_target(spec, state): @spec_test @single_phase def test_incorrect_no_source_execution_withdrawal_credential(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # Set up a correct consolidation, but source does not have # an execution withdrawal credential current_epoch = spec.get_current_epoch(state) @@ -820,6 +858,8 @@ def test_incorrect_no_source_execution_withdrawal_credential(spec, state): @spec_test @single_phase def test_incorrect_no_target_execution_withdrawal_credential(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # Set up a correct consolidation, but target does not have # an execution withdrawal credential current_epoch = spec.get_current_epoch(state) @@ -852,6 +892,8 @@ def test_incorrect_no_target_execution_withdrawal_credential(spec, state): @spec_test @single_phase def test_incorrect_incorrect_source_address(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # Set up an otherwise correct consolidation current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -885,6 +927,8 @@ def test_incorrect_incorrect_source_address(spec, state): @spec_test @single_phase def test_incorrect_unknown_source_pubkey(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # Set up an otherwise correct consolidation current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -918,6 +962,8 @@ def test_incorrect_unknown_source_pubkey(spec, state): @spec_test @single_phase def test_incorrect_unknown_target_pubkey(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # Set up an otherwise correct consolidation current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -942,6 +988,80 @@ def test_incorrect_unknown_target_pubkey(spec, state): ) +@with_electra_and_later +@with_presets([MINIMAL], "need sufficient consolidation churn limit") +@with_custom_state( + balances_fn=scaled_churn_balances_exceed_activation_exit_churn_limit, + threshold_fn=default_activation_threshold, +) +@spec_test +@single_phase +def test_incorrect_source_has_pending_withdrawal(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # Set up an otherwise correct consolidation + current_epoch = spec.get_current_epoch(state) + source_index = spec.get_active_validator_indices(state, current_epoch)[0] + target_index = spec.get_active_validator_indices(state, current_epoch)[1] + source_address = b"\x22" * 20 + excess_balance = spec.EFFECTIVE_BALANCE_INCREMENT // 4 + set_eth1_withdrawal_credential_with_balance( + spec, state, source_index, address=source_address, balance=spec.MIN_ACTIVATION_BALANCE + excess_balance + ) + consolidation = spec.ConsolidationRequest( + source_address=source_address, + source_pubkey=state.validators[source_index].pubkey, + target_pubkey=state.validators[target_index].pubkey, + ) + set_eth1_withdrawal_credential_with_balance(spec, state, target_index) + + # Create pending withdrawal + pending_withdrawal = spec.PendingPartialWithdrawal( + index=0, amount=excess_balance, withdrawable_epoch=current_epoch + ) + state.pending_partial_withdrawals.append(pending_withdrawal) + + # Check the return condition + assert spec.get_pending_balance_to_withdraw(state, source_index) > 0 + + yield from run_consolidation_processing( + spec, state, consolidation, success=False + ) + + +@with_electra_and_later +@with_presets([MINIMAL], "need sufficient consolidation churn limit") +@with_custom_state( + balances_fn=scaled_churn_balances_exceed_activation_exit_churn_limit, + threshold_fn=default_activation_threshold, +) +@spec_test +@single_phase +def test_incorrect_source_not_active_long_enough(spec, state): + # Set up an otherwise correct consolidation + current_epoch = spec.get_current_epoch(state) + source_index = spec.get_active_validator_indices(state, current_epoch)[0] + target_index = spec.get_active_validator_indices(state, current_epoch)[1] + source_address = b"\x22" * 20 + excess_balance = spec.EFFECTIVE_BALANCE_INCREMENT // 4 + set_eth1_withdrawal_credential_with_balance( + spec, state, source_index, address=source_address, balance=spec.MIN_ACTIVATION_BALANCE + excess_balance + ) + consolidation = spec.ConsolidationRequest( + source_address=source_address, + source_pubkey=state.validators[source_index].pubkey, + target_pubkey=state.validators[target_index].pubkey, + ) + set_eth1_withdrawal_credential_with_balance(spec, state, target_index) + + # Check the return condition + assert current_epoch < state.validators[source_index].activation_epoch + spec.config.SHARD_COMMITTEE_PERIOD + + yield from run_consolidation_processing( + spec, state, consolidation, success=False + ) + + @with_electra_and_later @spec_state_test def test_switch_to_compounding_exited_source(spec, state): From 09754f5ca4864e1fd8c5ed08993d17f4ea17e0f3 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Thu, 31 Oct 2024 11:14:07 -0500 Subject: [PATCH 122/142] Apply suggestion from pop --- specs/electra/beacon-chain.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/specs/electra/beacon-chain.md b/specs/electra/beacon-chain.md index 59415b7c79..333d3b1e0d 100644 --- a/specs/electra/beacon-chain.md +++ b/specs/electra/beacon-chain.md @@ -1338,10 +1338,9 @@ def apply_deposit(state: BeaconState, validator_pubkeys = [v.pubkey for v in state.validators] if pubkey not in validator_pubkeys: # Verify the deposit signature (proof of possession) which is not checked by the deposit contract - if is_valid_deposit_signature(pubkey, withdrawal_credentials, amount, signature): - add_validator_to_registry(state, pubkey, withdrawal_credentials, Gwei(0)) # [Modified in Electra:EIP7251] - else: + if not is_valid_deposit_signature(pubkey, withdrawal_credentials, amount, signature): return + add_validator_to_registry(state, pubkey, withdrawal_credentials, Gwei(0)) # [Modified in Electra:EIP7251] # Increase balance by deposit amount # [Modified in Electra:EIP7251] From 08cb8286b7dba5dadfa2c9e9110c77112c37c30f Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Fri, 1 Nov 2024 12:45:09 +0600 Subject: [PATCH 123/142] Fix and add more attestation tests --- .../test_process_attestation.py | 51 ++++++++++++++++--- 1 file changed, 44 insertions(+), 7 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_attestation.py index 635d2472f4..b365bec1d6 100644 --- a/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_attestation.py +++ b/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_attestation.py @@ -10,7 +10,8 @@ get_valid_attestation, sign_attestation, build_attestation_data, - fill_aggregate_attestation, + get_valid_attestation_at_slot, + get_empty_eip7549_aggregation_bits, ) from eth2spec.test.helpers.state import ( next_slots, @@ -39,7 +40,7 @@ def test_invalid_attestation_data_index_not_zero(spec, state): @with_electra_and_later @spec_state_test @always_bls -def test_invalid_committe_index(spec, state): +def test_invalid_committee_index(spec, state): """ EIP-7549 test """ @@ -57,7 +58,7 @@ def test_invalid_committe_index(spec, state): @with_electra_and_later @spec_state_test -def test_invalid_too_many_committe_bits(spec, state): +def test_invalid_too_many_committee_bits(spec, state): """ EIP-7549 test """ @@ -72,7 +73,7 @@ def test_invalid_too_many_committe_bits(spec, state): @with_electra_and_later @spec_state_test -def test_invalid_nonset_committe_bits(spec, state): +def test_invalid_nonset_committee_bits(spec, state): """ EIP-7549 test """ @@ -85,18 +86,51 @@ def test_invalid_nonset_committe_bits(spec, state): yield from run_attestation_processing(spec, state, attestation, valid=False) +@with_electra_and_later +@spec_state_test +def test_invalid_nonset_multiple_committee_bits(spec, state): + """ + EIP-7549 test + """ + attestation_data = build_attestation_data(spec, state, slot=state.slot, index=0) + attestation = spec.Attestation(data=attestation_data) + + # a single attestation with all committees of a slot, but with unset aggregation_bits + committees_per_slot = spec.get_committee_count_per_slot(state, spec.get_current_epoch(state)) + for index in range(committees_per_slot): + attestation.committee_bits[index] = True + + attestation.aggregation_bits = get_empty_eip7549_aggregation_bits( + spec, state, attestation.committee_bits, attestation.data.slot + ) + + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) + + yield from run_attestation_processing(spec, state, attestation, valid=False) + + @with_electra_and_later @spec_state_test @with_presets([MINIMAL], "need multiple committees per slot") @always_bls def test_multiple_committees(spec, state): + """ + EIP-7549 test + """ attestation_data = build_attestation_data(spec, state, slot=state.slot, index=0) attestation = spec.Attestation(data=attestation_data) - # fill the attestation with two committees - fill_aggregate_attestation(spec, state, attestation, signed=True, committee_index=0) - fill_aggregate_attestation(spec, state, attestation, signed=True, committee_index=1) + # a single attestation with all committees of a slot + attestation = get_valid_attestation_at_slot(state, spec, state.slot) + + # check that all committees are presented in a single attestation + attesting_indices = set() + committees_per_slot = spec.get_committee_count_per_slot(state, spec.get_current_epoch(state)) + for index in range(committees_per_slot): + attesting_indices.update(spec.get_beacon_committee(state, state.slot, index)) + assert spec.get_attesting_indices(state, attestation) == attesting_indices + # advance a slot next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) yield from run_attestation_processing(spec, state, attestation) @@ -107,6 +141,9 @@ def test_multiple_committees(spec, state): @with_presets([MINIMAL], "need multiple committees per slot") @always_bls def test_one_committee_with_gap(spec, state): + """ + EIP-7549 test + """ attestation = get_valid_attestation(spec, state, index=1, signed=True) next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) From 13bff6209a582988984ad1237640244d6f10045d Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Fri, 1 Nov 2024 11:03:21 -0500 Subject: [PATCH 124/142] Update initial earliest_exit_epoch calculation & add tests --- specs/electra/fork.md | 10 ++-- .../electra/fork/test_electra_fork_basic.py | 58 +++++++++++++++++++ 2 files changed, 64 insertions(+), 4 deletions(-) diff --git a/specs/electra/fork.md b/specs/electra/fork.md index 6ac5be5b03..c9c2812f6c 100644 --- a/specs/electra/fork.md +++ b/specs/electra/fork.md @@ -74,10 +74,12 @@ def upgrade_to_electra(pre: deneb.BeaconState) -> BeaconState: epoch = deneb.get_current_epoch(pre) latest_execution_payload_header = pre.latest_execution_payload_header - exit_epochs = [v.exit_epoch for v in pre.validators if v.exit_epoch != FAR_FUTURE_EPOCH] - if not exit_epochs: - exit_epochs = [get_current_epoch(pre)] - earliest_exit_epoch = max(exit_epochs) + 1 + earliest_exit_epoch = compute_activation_exit_epoch(get_current_epoch(pre)) + for validator in pre.validators: + if validator.exit_epoch != FAR_FUTURE_EPOCH: + if validator.exit_epoch > earliest_exit_epoch: + earliest_exit_epoch = validator.exit_epoch + earliest_exit_epoch += 1 post = BeaconState( # Versioning diff --git a/tests/core/pyspec/eth2spec/test/electra/fork/test_electra_fork_basic.py b/tests/core/pyspec/eth2spec/test/electra/fork/test_electra_fork_basic.py index 884bfcb4eb..a27ed777fe 100644 --- a/tests/core/pyspec/eth2spec/test/electra/fork/test_electra_fork_basic.py +++ b/tests/core/pyspec/eth2spec/test/electra/fork/test_electra_fork_basic.py @@ -130,3 +130,61 @@ def test_fork_has_compounding_withdrawal_credential(spec, phases, state): post_state = yield from run_fork_test(post_spec, state) assert len(post_state.pending_deposits) > 0 + + +@with_phases(phases=[DENEB], other_phases=[ELECTRA]) +@spec_test +@with_state +@with_meta_tags(ELECTRA_FORK_TEST_META_TAGS) +def test_fork_earliest_exit_epoch_no_validator_exits(spec, phases, state): + # advance state so the current epoch is not zero + next_epoch(spec, state) + next_epoch(spec, state) + next_epoch(spec, state) + + post_spec = phases[ELECTRA] + post_state = yield from run_fork_test(post_spec, state) + + # the earliest exit epoch should be the compute_activation_exit_epoch + 1 + current_epoch = post_spec.compute_epoch_at_slot(post_state.slot) + expected_earliest_exit_epoch = post_spec.compute_activation_exit_epoch(current_epoch) + 1 + assert post_state.earliest_exit_epoch == expected_earliest_exit_epoch + + +@with_phases(phases=[DENEB], other_phases=[ELECTRA]) +@spec_test +@with_state +@with_meta_tags(ELECTRA_FORK_TEST_META_TAGS) +def test_fork_earliest_exit_epoch_is_max_validator_exit_epoch(spec, phases, state): + # assign some validators exit epochs + state.validators[0].exit_epoch = 20 + state.validators[1].exit_epoch = 30 + state.validators[2].exit_epoch = 10 + + post_state = yield from run_fork_test(phases[ELECTRA], state) + + # the earliest exit epoch should be the greatest validator exit epoch + 1 + expected_earliest_exit_epoch = post_state.validators[1].exit_epoch + 1 + assert post_state.earliest_exit_epoch == expected_earliest_exit_epoch + + +@with_phases(phases=[DENEB], other_phases=[ELECTRA]) +@spec_test +@with_state +@with_meta_tags(ELECTRA_FORK_TEST_META_TAGS) +def test_fork_earliest_exit_epoch_less_than_current_epoch(spec, phases, state): + # assign a validator an exit epoch + state.validators[0].exit_epoch = 1 + + # advance state so the current epoch is not zero + next_epoch(spec, state) + next_epoch(spec, state) + next_epoch(spec, state) + + post_spec = phases[ELECTRA] + post_state = yield from run_fork_test(post_spec, state) + + # the earliest exit epoch should be the compute_activation_exit_epoch + 1 + current_epoch = post_spec.compute_epoch_at_slot(post_state.slot) + expected_earliest_exit_epoch = post_spec.compute_activation_exit_epoch(current_epoch) + 1 + assert post_state.earliest_exit_epoch == expected_earliest_exit_epoch From 93d9ee65078bfe0a8326e9b70c1df169d3d78624 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Fri, 1 Nov 2024 11:15:24 -0500 Subject: [PATCH 125/142] Fix lint issue --- specs/electra/fork.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/electra/fork.md b/specs/electra/fork.md index c9c2812f6c..909556c666 100644 --- a/specs/electra/fork.md +++ b/specs/electra/fork.md @@ -79,7 +79,7 @@ def upgrade_to_electra(pre: deneb.BeaconState) -> BeaconState: if validator.exit_epoch != FAR_FUTURE_EPOCH: if validator.exit_epoch > earliest_exit_epoch: earliest_exit_epoch = validator.exit_epoch - earliest_exit_epoch += 1 + earliest_exit_epoch += Epoch(1) post = BeaconState( # Versioning From 410b14fa77d594bc11453ed3bbc4796ca46c2770 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Fri, 1 Nov 2024 11:31:57 -0500 Subject: [PATCH 126/142] Revert "Apply suggestion from pop" This reverts commit 09754f5ca4864e1fd8c5ed08993d17f4ea17e0f3. --- specs/electra/beacon-chain.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/electra/beacon-chain.md b/specs/electra/beacon-chain.md index 333d3b1e0d..59415b7c79 100644 --- a/specs/electra/beacon-chain.md +++ b/specs/electra/beacon-chain.md @@ -1338,9 +1338,10 @@ def apply_deposit(state: BeaconState, validator_pubkeys = [v.pubkey for v in state.validators] if pubkey not in validator_pubkeys: # Verify the deposit signature (proof of possession) which is not checked by the deposit contract - if not is_valid_deposit_signature(pubkey, withdrawal_credentials, amount, signature): + if is_valid_deposit_signature(pubkey, withdrawal_credentials, amount, signature): + add_validator_to_registry(state, pubkey, withdrawal_credentials, Gwei(0)) # [Modified in Electra:EIP7251] + else: return - add_validator_to_registry(state, pubkey, withdrawal_credentials, Gwei(0)) # [Modified in Electra:EIP7251] # Increase balance by deposit amount # [Modified in Electra:EIP7251] From a3d4dbaafaa5a3eb3a88cab11be8e903573f4346 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Mon, 4 Nov 2024 17:40:10 +0600 Subject: [PATCH 127/142] Ensure non-zero bits for each aggregate committee --- specs/electra/beacon-chain.md | 37 +++++++++++++++++-- .../test_process_attestation.py | 32 ++++++++++++++++ 2 files changed, 65 insertions(+), 4 deletions(-) diff --git a/specs/electra/beacon-chain.md b/specs/electra/beacon-chain.md index 3e12ab2171..f6743e967e 100644 --- a/specs/electra/beacon-chain.md +++ b/specs/electra/beacon-chain.md @@ -56,6 +56,7 @@ - [New `get_activation_exit_churn_limit`](#new-get_activation_exit_churn_limit) - [New `get_consolidation_churn_limit`](#new-get_consolidation_churn_limit) - [New `get_pending_balance_to_withdraw`](#new-get_pending_balance_to_withdraw) + - [Modified `get_indexed_attestation`](#modified-get_indexed_attestation) - [Modified `get_attesting_indices`](#modified-get_attesting_indices) - [Modified `get_next_sync_committee_indices`](#modified-get_next_sync_committee_indices) - [Beacon state mutators](#beacon-state-mutators) @@ -572,6 +573,25 @@ def get_pending_balance_to_withdraw(state: BeaconState, validator_index: Validat ) ``` +#### Modified `get_indexed_attestation` + +*Note*: The function is modified to use the new `get_attesting_indices`. + +```python +def get_indexed_attestation(state: BeaconState, attestation: Attestation) -> IndexedAttestation: + """ + Return the indexed attestation corresponding to ``attestation``. + """ + # [Modified in Electra:EIP7549] + attesting_indices = get_attesting_indices(state, attestation) + + return IndexedAttestation( + attesting_indices=sorted(attesting_indices), + data=attestation.data, + signature=attestation.signature, + ) +``` + #### Modified `get_attesting_indices` *Note*: The function `get_attesting_indices` is modified to support EIP7549. @@ -1254,6 +1274,8 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: ###### Modified `process_attestation` +*Note*: The function is modified to support EIP7549. + ```python def process_attestation(state: BeaconState, attestation: Attestation) -> None: data = attestation.data @@ -1264,18 +1286,24 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: # [Modified in Electra:EIP7549] assert data.index == 0 committee_indices = get_committee_indices(attestation.committee_bits) - participants_count = 0 + committee_offset = 0 for index in committee_indices: assert index < get_committee_count_per_slot(state, data.target.epoch) committee = get_beacon_committee(state, data.slot, index) - participants_count += len(committee) + committee_attesters = set( + attester_index for i, attester_index in enumerate(committee) + if attestation.aggregation_bits[committee_offset + i] + ) + assert len(committee_attesters) > 0 + committee_offset += len(committee) - assert len(attestation.aggregation_bits) == participants_count + # Bitfield length matches total number of participants + assert len(attestation.aggregation_bits) == committee_offset # Participation flag indices participation_flag_indices = get_attestation_participation_flag_indices(state, data, state.slot - data.slot) - # Verify signature + # Verify signature [Modified in Electra:EIP7549] assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) # Update epoch participation flags @@ -1285,6 +1313,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: epoch_participation = state.previous_epoch_participation proposer_reward_numerator = 0 + # [Modified in Electra:EIP7549] for index in get_attesting_indices(state, attestation): for flag_index, weight in enumerate(PARTICIPATION_FLAG_WEIGHTS): if flag_index in participation_flag_indices and not has_flag(epoch_participation[index], flag_index): diff --git a/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_attestation.py index f268feb034..f8d54b37f5 100644 --- a/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_attestation.py +++ b/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_attestation.py @@ -1,12 +1,15 @@ +from eth2spec.test.helpers.constants import MINIMAL from eth2spec.test.context import ( always_bls, spec_state_test, with_electra_and_later, + with_presets, ) from eth2spec.test.helpers.attestations import ( run_attestation_processing, get_valid_attestation, sign_attestation, + get_empty_eip7549_aggregation_bits, ) from eth2spec.test.helpers.state import ( next_slots, @@ -79,3 +82,32 @@ def test_invalid_nonset_committe_bits(spec, state): attestation.committee_bits[committee_index] = 0 yield from run_attestation_processing(spec, state, attestation, valid=False) + + +@with_electra_and_later +@spec_state_test +def test_invalid_nonset_bits_for_one_committee(spec, state): + """ + EIP-7549 test + """ + # Attestation with full committee participating + committee_0 = spec.get_beacon_committee(state, state.slot, 0) + attestation_1 = get_valid_attestation(spec, state, index=1, signed=True) + + # Create an on chain aggregate + aggregate = spec.Attestation(data=attestation_1.data, signature=attestation_1.signature) + aggregate.committee_bits[0] = True + aggregate.committee_bits[1] = True + aggregate.aggregation_bits = get_empty_eip7549_aggregation_bits( + spec, state, aggregate.committee_bits, aggregate.data.slot + ) + committee_offset = len(committee_0) + for i in range(len(attestation_1.aggregation_bits)): + aggregate.aggregation_bits[committee_offset + i] = attestation_1.aggregation_bits[i] + + # Check that only one committee is presented + assert spec.get_attesting_indices(state, aggregate) == spec.get_attesting_indices(state, attestation_1) + + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) + + yield from run_attestation_processing(spec, state, aggregate, valid=False) From 2222c24eeb0ab253fb0c9739e378479987ee03b4 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Mon, 4 Nov 2024 18:58:09 +0600 Subject: [PATCH 128/142] Run multiple committee tests with MINIMAL presets --- .../test/electra/block_processing/test_process_attestation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_attestation.py index f8d54b37f5..7102a46f8e 100644 --- a/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_attestation.py +++ b/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_attestation.py @@ -86,6 +86,7 @@ def test_invalid_nonset_committe_bits(spec, state): @with_electra_and_later @spec_state_test +@with_presets([MINIMAL], "need multiple committees per slot") def test_invalid_nonset_bits_for_one_committee(spec, state): """ EIP-7549 test From 1afabf7a0b05e20cdaebf5bf8658c265fdccc55b Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Tue, 5 Nov 2024 11:31:39 +0600 Subject: [PATCH 129/142] Rename index to committee_index in process_attestation --- specs/electra/beacon-chain.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/electra/beacon-chain.md b/specs/electra/beacon-chain.md index 306e304a78..861d8f0059 100644 --- a/specs/electra/beacon-chain.md +++ b/specs/electra/beacon-chain.md @@ -1288,9 +1288,9 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: assert data.index == 0 committee_indices = get_committee_indices(attestation.committee_bits) committee_offset = 0 - for index in committee_indices: - assert index < get_committee_count_per_slot(state, data.target.epoch) - committee = get_beacon_committee(state, data.slot, index) + for committee_index in committee_indices: + assert committee_index < get_committee_count_per_slot(state, data.target.epoch) + committee = get_beacon_committee(state, data.slot, committee_index) committee_attesters = set( attester_index for i, attester_index in enumerate(committee) if attestation.aggregation_bits[committee_offset + i] From ebbce03db569b81df9eefc2f7832252081b45690 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 6 Nov 2024 13:26:59 -0600 Subject: [PATCH 130/142] Replace assert messages with comments --- specs/electra/validator.md | 11 ++++++++--- .../electra/unittests/test_execution_requests.py | 16 ++++++++-------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/specs/electra/validator.md b/specs/electra/validator.md index 1cd4b91eca..ca92a1d955 100644 --- a/specs/electra/validator.md +++ b/specs/electra/validator.md @@ -207,9 +207,14 @@ def get_execution_requests(execution_requests_list: Sequence[bytes]) -> Executio prev_request_type = None for request in execution_requests_list: request_type, request_data = request[0:1], request[1:] - assert request_type in request_types, "unexpected request type" - assert len(request_data) != 0, "empty request data" - assert prev_request_type is None or prev_request_type < request_type, "not ascending order" + + # Check that the request type is valid + assert request_type in request_types + # Check that the request data is not empty + assert len(request_data) != 0 + # Check that requests are in strictly ascending order + # Each successive type must be greater than the last with no duplicates + assert prev_request_type is None or prev_request_type < request_type prev_request_type = request_type if request_type == DEPOSIT_REQUEST_TYPE: diff --git a/tests/core/pyspec/eth2spec/test/electra/unittests/test_execution_requests.py b/tests/core/pyspec/eth2spec/test/electra/unittests/test_execution_requests.py index 2a63c17a3f..d57e724312 100644 --- a/tests/core/pyspec/eth2spec/test/electra/unittests/test_execution_requests.py +++ b/tests/core/pyspec/eth2spec/test/electra/unittests/test_execution_requests.py @@ -73,8 +73,8 @@ def test_requests_deserialize__reject_duplicate_request(spec): try: spec.get_execution_requests(serialized_execution_requests) assert False, "expected exception" - except Exception as e: - assert "not ascending order" in str(e) + except Exception: + pass @with_electra_and_later @@ -89,8 +89,8 @@ def test_requests_deserialize__reject_out_of_order_requests(spec): try: spec.get_execution_requests(serialized_execution_requests) assert False, "expected exception" - except Exception as e: - assert "not ascending order" in str(e) + except Exception: + pass @with_electra_and_later @@ -101,8 +101,8 @@ def test_requests_deserialize__reject_empty_request(spec): try: spec.get_execution_requests(serialized_execution_requests) assert False, "expected exception" - except Exception as e: - assert "empty request data" in str(e) + except Exception: + pass @with_electra_and_later @@ -115,5 +115,5 @@ def test_requests_deserialize__reject_unexpected_request_type(spec): try: spec.get_execution_requests(serialized_execution_requests) assert False, "expected exception" - except Exception as e: - assert "unexpected request type" in str(e) + except Exception: + pass From 62e6a30ec1237db3f3d9d294a409d88cdc1fe1d8 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 13 Nov 2024 13:42:15 +0700 Subject: [PATCH 131/142] Add `BLOB_SIDECAR_SUBNET_COUNT_EIP7594`, `MAX_BLOBS_PER_BLOCK_EIP7594`, and `MAX_REQUEST_BLOB_SIDECARS_EIP7594` --- configs/mainnet.yaml | 4 + configs/minimal.yaml | 4 + specs/_features/eip7594/beacon-chain.md | 141 +++++++++++++++++++++++ specs/_features/eip7594/p2p-interface.md | 84 ++++++++++++++ specs/deneb/beacon-chain.md | 2 +- 5 files changed, 234 insertions(+), 1 deletion(-) create mode 100644 specs/_features/eip7594/beacon-chain.md diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 56c20a439c..36b4db4123 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -165,6 +165,10 @@ DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128 MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384 SAMPLES_PER_SLOT: 8 CUSTODY_REQUIREMENT: 4 +BLOB_SIDECAR_SUBNET_COUNT_EIP7594: 8 +MAX_BLOBS_PER_BLOCK_EIP7594: 8 +# `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK_EIP7594` +MAX_REQUEST_BLOB_SIDECARS_EIP7594: 1024 # [New in Electra:EIP7251] MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA: 128000000000 # 2**7 * 10**9 (= 128,000,000,000) diff --git a/configs/minimal.yaml b/configs/minimal.yaml index a2b4f2e736..ae19518af7 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -164,6 +164,10 @@ DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128 MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384 SAMPLES_PER_SLOT: 8 CUSTODY_REQUIREMENT: 4 +BLOB_SIDECAR_SUBNET_COUNT_EIP7594: 8 +MAX_BLOBS_PER_BLOCK_EIP7594: 8 +# `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK_EIP7594` +MAX_REQUEST_BLOB_SIDECARS_EIP7594: 1024 # [New in Electra:EIP7251] MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA: 64000000000 # 2**6 * 10**9 (= 64,000,000,000) diff --git a/specs/_features/eip7594/beacon-chain.md b/specs/_features/eip7594/beacon-chain.md new file mode 100644 index 0000000000..bf980b840d --- /dev/null +++ b/specs/_features/eip7594/beacon-chain.md @@ -0,0 +1,141 @@ +# EIP7594 -- The Beacon Chain + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + + + +- [Introduction](#introduction) +- [Configuration](#configuration) + - [Execution](#execution) + - [Execution payload](#execution-payload) + - [Modified `process_execution_payload`](#modified-process_execution_payload) +- [Testing](#testing) + + + + +## Introduction + +*Note:* This specification is built upon [Electra](../electra/beacon-chain.md) and is under active development. + +## Configuration + +### Execution + +| Name | Value | Description | +| - | - | - | +| `MAX_BLOBS_PER_BLOCK_EIP7594` | `uint64(8)` | *[New in EIP7494]* maximum number of blobs in a single block limited by `MAX_BLOB_COMMITMENTS_PER_BLOCK` | + + +#### Execution payload + +##### Modified `process_execution_payload` + +```python +def process_execution_payload(state: BeaconState, body: BeaconBlockBody, execution_engine: ExecutionEngine) -> None: + payload = body.execution_payload + + # Verify consistency of the parent hash with respect to the previous execution payload header + assert payload.parent_hash == state.latest_execution_payload_header.block_hash + # Verify prev_randao + assert payload.prev_randao == get_randao_mix(state, get_current_epoch(state)) + # Verify timestamp + assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) + # Verify commitments are under limit + assert len(body.blob_kzg_commitments) <= MAX_BLOBS_PER_BLOCK_EIP7594 # [Modified in EIP7594] + # Verify the execution payload is valid + versioned_hashes = [kzg_commitment_to_versioned_hash(commitment) for commitment in body.blob_kzg_commitments] + assert execution_engine.verify_and_notify_new_payload( + NewPayloadRequest( + execution_payload=payload, + versioned_hashes=versioned_hashes, + parent_beacon_block_root=state.latest_block_header.parent_root, + execution_requests=body.execution_requests, + ) + ) + # Cache execution payload header + state.latest_execution_payload_header = ExecutionPayloadHeader( + parent_hash=payload.parent_hash, + fee_recipient=payload.fee_recipient, + state_root=payload.state_root, + receipts_root=payload.receipts_root, + logs_bloom=payload.logs_bloom, + prev_randao=payload.prev_randao, + block_number=payload.block_number, + gas_limit=payload.gas_limit, + gas_used=payload.gas_used, + timestamp=payload.timestamp, + extra_data=payload.extra_data, + base_fee_per_gas=payload.base_fee_per_gas, + block_hash=payload.block_hash, + transactions_root=hash_tree_root(payload.transactions), + withdrawals_root=hash_tree_root(payload.withdrawals), + blob_gas_used=payload.blob_gas_used, + excess_blob_gas=payload.excess_blob_gas, + ) +``` + +## Testing + +*Note*: The function `initialize_beacon_state_from_eth1` is modified for pure EIP7594 testing only. + +```python +def initialize_beacon_state_from_eth1(eth1_block_hash: Hash32, + eth1_timestamp: uint64, + deposits: Sequence[Deposit], + execution_payload_header: ExecutionPayloadHeader=ExecutionPayloadHeader() + ) -> BeaconState: + fork = Fork( + previous_version=EIP7594_FORK_VERSION, # [Modified in EIP7594] for testing only + current_version=EIP7594_FORK_VERSION, # [Modified in EIP7594] + epoch=GENESIS_EPOCH, + ) + state = BeaconState( + genesis_time=eth1_timestamp + GENESIS_DELAY, + fork=fork, + eth1_data=Eth1Data(block_hash=eth1_block_hash, deposit_count=uint64(len(deposits))), + latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), + randao_mixes=[eth1_block_hash] * EPOCHS_PER_HISTORICAL_VECTOR, # Seed RANDAO with Eth1 entropy + deposit_requests_start_index=UNSET_DEPOSIT_REQUESTS_START_INDEX, + ) + + # Process deposits + leaves = list(map(lambda deposit: deposit.data, deposits)) + for index, deposit in enumerate(deposits): + deposit_data_list = List[DepositData, 2**DEPOSIT_CONTRACT_TREE_DEPTH](*leaves[:index + 1]) + state.eth1_data.deposit_root = hash_tree_root(deposit_data_list) + process_deposit(state, deposit) + + # Process deposit balance updates + validator_pubkeys = [v.pubkey for v in state.validators] + for deposit in state.pending_deposits: + validator_index = ValidatorIndex(validator_pubkeys.index(deposit.pubkey)) + increase_balance(state, validator_index, deposit.amount) + state.pending_deposits = [] + + # Process activations + for index, validator in enumerate(state.validators): + balance = state.balances[index] + validator.effective_balance = min( + balance - balance % EFFECTIVE_BALANCE_INCREMENT, get_max_effective_balance(validator)) + if validator.effective_balance >= MIN_ACTIVATION_BALANCE: + validator.activation_eligibility_epoch = GENESIS_EPOCH + validator.activation_epoch = GENESIS_EPOCH + + # Set genesis validators root for domain separation and chain versioning + state.genesis_validators_root = hash_tree_root(state.validators) + + # Fill in sync committees + # Note: A duplicate committee is assigned for the current and next committee at genesis + state.current_sync_committee = get_next_sync_committee(state) + state.next_sync_committee = get_next_sync_committee(state) + + # Initialize the execution payload header + state.latest_execution_payload_header = execution_payload_header + + return state +``` diff --git a/specs/_features/eip7594/p2p-interface.md b/specs/_features/eip7594/p2p-interface.md index c989711807..8fa1d2b82f 100644 --- a/specs/_features/eip7594/p2p-interface.md +++ b/specs/_features/eip7594/p2p-interface.md @@ -21,11 +21,15 @@ - [MetaData](#metadata) - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub) - [Topics and messages](#topics-and-messages) + - [Global topics](#global-topics) + - [`beacon_block`](#beacon_block) - [Blob subnets](#blob-subnets) - [Deprecated `blob_sidecar_{subnet_id}`](#deprecated-blob_sidecar_subnet_id) - [`data_column_sidecar_{subnet_id}`](#data_column_sidecar_subnet_id) - [The Req/Resp domain](#the-reqresp-domain) - [Messages](#messages) + - [BlobSidecarsByRoot v2](#blobsidecarsbyroot-v2) + - [BlobSidecarsByRange v2](#blobsidecarsbyrange-v2) - [DataColumnSidecarsByRoot v1](#datacolumnsidecarsbyroot-v1) - [DataColumnSidecarsByRange v1](#datacolumnsidecarsbyrange-v1) - [GetMetaData v3](#getmetadata-v3) @@ -52,6 +56,8 @@ |------------------------------------------------|------------------------------------------------|---------------------------------------------------------------------------| | `MAX_REQUEST_DATA_COLUMN_SIDECARS` | `MAX_REQUEST_BLOCKS_DENEB * NUMBER_OF_COLUMNS` | Maximum number of data column sidecars in a single request | | `MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS` | `2**12` (= 4096 epochs, ~18 days) | The minimum epoch range over which a node must serve data column sidecars | +| `MAX_REQUEST_BLOB_SIDECARS_EIP7594` | `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK_EIP7594` | Maximum number of blob sidecars in a single request | +| `BLOB_SIDECAR_SUBNET_COUNT_EIP7594` | `2**3` (= 8) | The number of blob sidecar subnets used in the gossipsub protocol. | ### Containers @@ -154,6 +160,16 @@ Some gossip meshes are upgraded in the EIP-7594 fork to support upgraded types. #### Topics and messages + +##### Global topics + +###### `beacon_block` + +*Updated validation* + +- _[REJECT]_ The length of KZG commitments is less than or equal to the limitation defined in Consensus Layer -- + i.e. validate that `len(body.signed_beacon_block.message.blob_kzg_commitments) <= MAX_BLOBS_PER_BLOCK_EIP7594` + ##### Blob subnets ###### Deprecated `blob_sidecar_{subnet_id}` @@ -189,6 +205,74 @@ The following validations MUST pass before forwarding the `sidecar: DataColumnSi #### Messages +##### BlobSidecarsByRoot v2 + +**Protocol ID:** `/eth2/beacon_chain/req/blob_sidecars_by_root/2/` + +*[Updated in EIP7594]* + +The `` field is calculated as `context = compute_fork_digest(fork_version, genesis_validators_root)`: + +[1]: # (eth2spec: skip) + +| `fork_version` | Chunk SSZ type | +|--------------------------|-------------------------------| +| `EIP7594_FORK_VERSION` | `eip7594.BlobSidecar` | + +Request Content: + +``` +( + List[BlobIdentifier, MAX_REQUEST_BLOB_SIDECARS_EIP7594] +) +``` + +Response Content: + +``` +( + List[BlobSidecar, MAX_REQUEST_BLOB_SIDECARS_EIP7594] +) +``` + +*Updated validation* + +No more than `MAX_REQUEST_BLOB_SIDECARS_EIP7594` may be requested at a time. + + +##### BlobSidecarsByRange v2 + +**Protocol ID:** `/eth2/beacon_chain/req/blob_sidecars_by_range/2/` + +*[Updated in EIP7594]* + +The `` field is calculated as `context = compute_fork_digest(fork_version, genesis_validators_root)`: + +[1]: # (eth2spec: skip) + +| `fork_version` | Chunk SSZ type | +|--------------------------|-------------------------------| +| `EIP7594_FORK_VERSION` | `eip7594.BlobSidecar` | + +Request Content: +``` +( + start_slot: Slot + count: uint64 +) +``` + +Response Content: +``` +( + List[BlobSidecar, MAX_REQUEST_BLOB_SIDECARS_EIP7594] +) +``` + +*Updated validation* + +Clients MUST respond with at least the blob sidecars of the first blob-carrying block that exists in the range, if they have it, and no more than `MAX_REQUEST_BLOB_SIDECARS_EIP7594` sidecars. + ##### DataColumnSidecarsByRoot v1 **Protocol ID:** `/eth2/beacon_chain/req/data_column_sidecars_by_root/1/` diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 0f6a8fc076..3404b11713 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -77,7 +77,7 @@ Deneb is a consensus-layer upgrade containing a number of features. Including: | Name | Value | Description | | - | - | - | -| `MAX_BLOB_COMMITMENTS_PER_BLOCK` | `uint64(2**12)` (= 4096) | *[New in Deneb:EIP4844]* hardfork independent fixed theoretical limit same as `LIMIT_BLOBS_PER_TX` (see EIP 4844) | +| `MAX_BLOB_COMMITMENTS_PER_BLOCK` | `uint64(2**12)` (= 4096) | *[New in Deneb:EIP4844]* hardfork independent fixed theoretical limit same as `TARGET_BLOB_GAS_PER_BLOCK` (see EIP 4844) | ## Configuration From f67e6cc359feb634b8a4d303891e774e15009e03 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 13 Nov 2024 14:23:23 +0700 Subject: [PATCH 132/142] update tests --- .../test_process_execution_payload.py | 3 ++- .../eth2spec/test/deneb/sanity/test_blocks.py | 11 ++++++----- .../eip7594/unittests/test_config_invariants.py | 13 +++++++++++++ tests/core/pyspec/eth2spec/test/helpers/blob.py | 11 +++++++++++ .../pyspec/eth2spec/test/helpers/fork_choice.py | 2 +- tests/core/pyspec/eth2spec/test/helpers/forks.py | 6 +++++- .../eth2spec/test/utils/randomized_block_tests.py | 1 + tests/formats/fork_choice/README.md | 2 +- 8 files changed, 40 insertions(+), 9 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py index d0c4a6f22a..e8ac314e10 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py @@ -12,6 +12,7 @@ ) from eth2spec.test.helpers.blob import ( get_sample_blob_tx, + get_max_blob_count, ) @@ -254,7 +255,7 @@ def test_invalid_correct_input__execution_invalid(spec, state): def test_invalid_exceed_max_blobs_per_block(spec, state): execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec, blob_count=spec.config.MAX_BLOBS_PER_BLOCK + 1) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec, blob_count=get_max_blob_count(spec) + 1) execution_payload.transactions = [opaque_tx] execution_payload.block_hash = compute_el_block_hash(spec, execution_payload, state) diff --git a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py index b019175369..19c0fcd0c5 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py @@ -16,6 +16,7 @@ ) from eth2spec.test.helpers.blob import ( get_sample_blob_tx, + get_max_blob_count, ) @@ -72,31 +73,31 @@ def test_one_blob_two_txs(spec, state): @with_deneb_and_later @spec_state_test def test_one_blob_max_txs(spec, state): - yield from run_block_with_blobs(spec, state, blob_count=1, tx_count=spec.MAX_BLOBS_PER_BLOCK) + yield from run_block_with_blobs(spec, state, blob_count=1, tx_count=get_max_blob_count(spec)) @with_deneb_and_later @spec_state_test def test_invalid_one_blob_max_plus_one_txs(spec, state): - yield from run_block_with_blobs(spec, state, blob_count=1, tx_count=spec.MAX_BLOBS_PER_BLOCK + 1, valid=False) + yield from run_block_with_blobs(spec, state, blob_count=1, tx_count=get_max_blob_count(spec) + 1, valid=False) @with_deneb_and_later @spec_state_test def test_max_blobs_per_block(spec, state): - yield from run_block_with_blobs(spec, state, blob_count=spec.MAX_BLOBS_PER_BLOCK) + yield from run_block_with_blobs(spec, state, blob_count=get_max_blob_count(spec)) @with_deneb_and_later @spec_state_test def test_invalid_max_blobs_per_block_two_txs(spec, state): - yield from run_block_with_blobs(spec, state, blob_count=spec.MAX_BLOBS_PER_BLOCK, tx_count=2, valid=False) + yield from run_block_with_blobs(spec, state, blob_count=get_max_blob_count(spec), tx_count=2, valid=False) @with_deneb_and_later @spec_state_test def test_invalid_exceed_max_blobs_per_block(spec, state): - yield from run_block_with_blobs(spec, state, blob_count=spec.MAX_BLOBS_PER_BLOCK + 1, valid=False) + yield from run_block_with_blobs(spec, state, blob_count=get_max_blob_count(spec) + 1, valid=False) @with_deneb_and_later diff --git a/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_config_invariants.py b/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_config_invariants.py index fc54cc3088..2c291890d4 100644 --- a/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_config_invariants.py +++ b/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_config_invariants.py @@ -25,3 +25,16 @@ def test_invariants(spec): @single_phase def test_polynomical_commitments_sampling(spec): assert spec.FIELD_ELEMENTS_PER_EXT_BLOB == 2 * spec.FIELD_ELEMENTS_PER_BLOB + + +@with_eip7594_and_later +@spec_test +@single_phase +def test_networking(spec): + assert spec.config.MAX_BLOBS_PER_BLOCK_EIP7594 < spec.MAX_BLOB_COMMITMENTS_PER_BLOCK + assert ( + spec.config.MAX_REQUEST_BLOB_SIDECARS_EIP7594 == + spec.config.MAX_REQUEST_BLOCKS_DENEB * spec.config.MAX_BLOBS_PER_BLOCK_EIP7594 + ) + # Start with the same size, but `BLOB_SIDECAR_SUBNET_COUNT` could potentially increase later. + assert spec.config.BLOB_SIDECAR_SUBNET_COUNT_EIP7594 == spec.config.MAX_BLOBS_PER_BLOCK_EIP7594 diff --git a/tests/core/pyspec/eth2spec/test/helpers/blob.py b/tests/core/pyspec/eth2spec/test/helpers/blob.py index fb2354a501..a31f032dcd 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/blob.py +++ b/tests/core/pyspec/eth2spec/test/helpers/blob.py @@ -2,6 +2,10 @@ from rlp import encode, Serializable from rlp.sedes import Binary, CountableList, List as RLPList, big_endian_int, binary +from eth2spec.test.helpers.forks import ( + is_post_eip7594, +) + class Eip4844RlpTransaction(Serializable): fields = ( @@ -99,3 +103,10 @@ def get_sample_blob_tx(spec, blob_count=1, rng=random.Random(5566), is_valid_blo ) opaque_tx = bytes([0x03]) + encode(signed_blob_tx) return opaque_tx, blobs, blob_kzg_commitments, blob_kzg_proofs + + +def get_max_blob_count(spec): + if is_post_eip7594: + return spec.config.MAX_BLOBS_PER_BLOCK_EIP7594 + else: + return spec.config.MAX_BLOBS_PER_BLOCK diff --git a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py index 8598870fb6..43555c268d 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py +++ b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py @@ -178,7 +178,7 @@ def add_block(spec, # Check blob_data if blob_data is not None: - blobs = spec.List[spec.Blob, spec.config.MAX_BLOBS_PER_BLOCK](blob_data.blobs) + blobs = spec.List[spec.Blob, spec.config.MAX_BLOB_COMMITMENTS_PER_BLOCK](blob_data.blobs) blobs_root = blobs.hash_tree_root() yield get_blobs_file_name(blobs_root=blobs_root), blobs diff --git a/tests/core/pyspec/eth2spec/test/helpers/forks.py b/tests/core/pyspec/eth2spec/test/helpers/forks.py index 288ad0d9e9..e261e3a754 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/forks.py +++ b/tests/core/pyspec/eth2spec/test/helpers/forks.py @@ -1,6 +1,6 @@ from .constants import ( PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, - ELECTRA, WHISK, EIP7732, + ELECTRA, WHISK, EIP7732, EIP7594, PREVIOUS_FORK_OF, ) @@ -45,6 +45,10 @@ def is_post_whisk(spec): return is_post_fork(spec.fork, WHISK) +def is_post_eip7594(spec): + return is_post_fork(spec.fork, EIP7594) + + def is_post_eip7732(spec): return is_post_fork(spec.fork, EIP7732) diff --git a/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py b/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py index 6e6cdd7683..93bb3b204c 100644 --- a/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py +++ b/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py @@ -250,6 +250,7 @@ def random_block_capella(spec, state, signed_blocks, scenario_state, rng=Random( def random_block_deneb(spec, state, signed_blocks, scenario_state, rng=Random(3456)): block = random_block_capella(spec, state, signed_blocks, scenario_state, rng=rng) # TODO: more commitments. blob_kzg_commitments: List[KZGCommitment, MAX_BLOBS_PER_BLOCK] + # TODO: add MAX_BLOBS_PER_BLOCK_EIP7594 at fulu opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx( spec, blob_count=rng.randint(0, spec.config.MAX_BLOBS_PER_BLOCK), rng=rng) block.body.execution_payload.transactions.append(opaque_tx) diff --git a/tests/formats/fork_choice/README.md b/tests/formats/fork_choice/README.md index 1258a66c06..258dfe433d 100644 --- a/tests/formats/fork_choice/README.md +++ b/tests/formats/fork_choice/README.md @@ -86,7 +86,7 @@ The parameter that is required for executing `on_block(store, block)`. block: string -- the name of the `block_<32-byte-root>.ssz_snappy` file. To execute `on_block(store, block)` with the given attestation. blobs: string -- optional, the name of the `blobs_<32-byte-root>.ssz_snappy` file. - The blobs file content is a `List[Blob, MAX_BLOBS_PER_BLOCK]` SSZ object. + The blobs file content is a `List[Blob, MAX_BLOB_COMMITMENTS_PER_BLOCK]` SSZ object. proofs: array of byte48 hex string -- optional, the proofs of blob commitments. valid: bool -- optional, default to `true`. If it's `false`, this execution step is expected to be invalid. From 34918e9744298a93153ecae6923fcb6cb8e109d3 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Mon, 18 Nov 2024 08:29:40 +0700 Subject: [PATCH 133/142] Remove unnecessary modification --- specs/electra/beacon-chain.md | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/specs/electra/beacon-chain.md b/specs/electra/beacon-chain.md index 861d8f0059..922294ce55 100644 --- a/specs/electra/beacon-chain.md +++ b/specs/electra/beacon-chain.md @@ -56,7 +56,6 @@ - [New `get_activation_exit_churn_limit`](#new-get_activation_exit_churn_limit) - [New `get_consolidation_churn_limit`](#new-get_consolidation_churn_limit) - [New `get_pending_balance_to_withdraw`](#new-get_pending_balance_to_withdraw) - - [Modified `get_indexed_attestation`](#modified-get_indexed_attestation) - [Modified `get_attesting_indices`](#modified-get_attesting_indices) - [Modified `get_next_sync_committee_indices`](#modified-get_next_sync_committee_indices) - [Beacon state mutators](#beacon-state-mutators) @@ -573,25 +572,6 @@ def get_pending_balance_to_withdraw(state: BeaconState, validator_index: Validat ) ``` -#### Modified `get_indexed_attestation` - -*Note*: The function is modified to use the new `get_attesting_indices`. - -```python -def get_indexed_attestation(state: BeaconState, attestation: Attestation) -> IndexedAttestation: - """ - Return the indexed attestation corresponding to ``attestation``. - """ - # [Modified in Electra:EIP7549] - attesting_indices = get_attesting_indices(state, attestation) - - return IndexedAttestation( - attesting_indices=sorted(attesting_indices), - data=attestation.data, - signature=attestation.signature, - ) -``` - #### Modified `get_attesting_indices` *Note*: The function `get_attesting_indices` is modified to support EIP7549. @@ -1304,7 +1284,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: # Participation flag indices participation_flag_indices = get_attestation_participation_flag_indices(state, data, state.slot - data.slot) - # Verify signature [Modified in Electra:EIP7549] + # Verify signature assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) # Update epoch participation flags @@ -1314,7 +1294,6 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: epoch_participation = state.previous_epoch_participation proposer_reward_numerator = 0 - # [Modified in Electra:EIP7549] for index in get_attesting_indices(state, attestation): for flag_index, weight in enumerate(PARTICIPATION_FLAG_WEIGHTS): if flag_index in participation_flag_indices and not has_flag(epoch_participation[index], flag_index): From ac53675d24915ffe1c4a81c8caf025d2dab5ed25 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Tue, 19 Nov 2024 14:26:08 -0600 Subject: [PATCH 134/142] Fix some minor nits --- specs/_features/eip7594/beacon-chain.md | 3 +- specs/_features/eip7594/p2p-interface.md | 30 ++++++++++--------- .../unittests/test_config_invariants.py | 2 +- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/specs/_features/eip7594/beacon-chain.md b/specs/_features/eip7594/beacon-chain.md index bf980b840d..28031178e1 100644 --- a/specs/_features/eip7594/beacon-chain.md +++ b/specs/_features/eip7594/beacon-chain.md @@ -28,8 +28,7 @@ | Name | Value | Description | | - | - | - | -| `MAX_BLOBS_PER_BLOCK_EIP7594` | `uint64(8)` | *[New in EIP7494]* maximum number of blobs in a single block limited by `MAX_BLOB_COMMITMENTS_PER_BLOCK` | - +| `MAX_BLOBS_PER_BLOCK_EIP7594` | `uint64(8)` | *[New in EIP7594]* Maximum number of blobs in a single block limited by `MAX_BLOB_COMMITMENTS_PER_BLOCK` | #### Execution payload diff --git a/specs/_features/eip7594/p2p-interface.md b/specs/_features/eip7594/p2p-interface.md index 8fa1d2b82f..de2d7e1f0b 100644 --- a/specs/_features/eip7594/p2p-interface.md +++ b/specs/_features/eip7594/p2p-interface.md @@ -52,12 +52,12 @@ *[New in EIP7594]* -| Name | Value | Description | -|------------------------------------------------|------------------------------------------------|---------------------------------------------------------------------------| -| `MAX_REQUEST_DATA_COLUMN_SIDECARS` | `MAX_REQUEST_BLOCKS_DENEB * NUMBER_OF_COLUMNS` | Maximum number of data column sidecars in a single request | -| `MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS` | `2**12` (= 4096 epochs, ~18 days) | The minimum epoch range over which a node must serve data column sidecars | -| `MAX_REQUEST_BLOB_SIDECARS_EIP7594` | `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK_EIP7594` | Maximum number of blob sidecars in a single request | -| `BLOB_SIDECAR_SUBNET_COUNT_EIP7594` | `2**3` (= 8) | The number of blob sidecar subnets used in the gossipsub protocol. | +| Name | Value | Description | +|------------------------------------------------|----------------------------------------------------------|---------------------------------------------------------------------------| +| `MAX_REQUEST_DATA_COLUMN_SIDECARS` | `MAX_REQUEST_BLOCKS_DENEB * NUMBER_OF_COLUMNS` | Maximum number of data column sidecars in a single request | +| `MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS` | `2**12` (= 4096 epochs, ~18 days) | The minimum epoch range over which a node must serve data column sidecars | +| `MAX_REQUEST_BLOB_SIDECARS_EIP7594` | `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK_EIP7594` | Maximum number of blob sidecars in a single request | +| `BLOB_SIDECAR_SUBNET_COUNT_EIP7594` | `2**3` (= 8) | The number of blob sidecar subnets used in the gossipsub protocol | ### Containers @@ -160,7 +160,6 @@ Some gossip meshes are upgraded in the EIP-7594 fork to support upgraded types. #### Topics and messages - ##### Global topics ###### `beacon_block` @@ -215,9 +214,9 @@ The `` field is calculated as `context = compute_fork_digest(fork [1]: # (eth2spec: skip) -| `fork_version` | Chunk SSZ type | -|--------------------------|-------------------------------| -| `EIP7594_FORK_VERSION` | `eip7594.BlobSidecar` | +| `fork_version` | Chunk SSZ type | +|------------------------|-----------------------| +| `EIP7594_FORK_VERSION` | `eip7594.BlobSidecar` | Request Content: @@ -239,7 +238,6 @@ Response Content: No more than `MAX_REQUEST_BLOB_SIDECARS_EIP7594` may be requested at a time. - ##### BlobSidecarsByRange v2 **Protocol ID:** `/eth2/beacon_chain/req/blob_sidecars_by_range/2/` @@ -250,11 +248,12 @@ The `` field is calculated as `context = compute_fork_digest(fork [1]: # (eth2spec: skip) -| `fork_version` | Chunk SSZ type | -|--------------------------|-------------------------------| -| `EIP7594_FORK_VERSION` | `eip7594.BlobSidecar` | +| `fork_version` | Chunk SSZ type | +|------------------------|-----------------------| +| `EIP7594_FORK_VERSION` | `eip7594.BlobSidecar` | Request Content: + ``` ( start_slot: Slot @@ -263,6 +262,7 @@ Request Content: ``` Response Content: + ``` ( List[BlobSidecar, MAX_REQUEST_BLOB_SIDECARS_EIP7594] @@ -336,6 +336,7 @@ The `` field is calculated as `context = compute_fork_digest(fork | `EIP7594_FORK_VERSION` | `eip7594.DataColumnSidecar` | Request Content: + ``` ( start_slot: Slot @@ -345,6 +346,7 @@ Request Content: ``` Response Content: + ``` ( List[DataColumnSidecar, MAX_REQUEST_DATA_COLUMN_SIDECARS] diff --git a/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_config_invariants.py b/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_config_invariants.py index 2c291890d4..8d14f4ae1c 100644 --- a/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_config_invariants.py +++ b/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_config_invariants.py @@ -31,7 +31,7 @@ def test_polynomical_commitments_sampling(spec): @spec_test @single_phase def test_networking(spec): - assert spec.config.MAX_BLOBS_PER_BLOCK_EIP7594 < spec.MAX_BLOB_COMMITMENTS_PER_BLOCK + assert spec.config.MAX_BLOBS_PER_BLOCK_EIP7594 <= spec.MAX_BLOB_COMMITMENTS_PER_BLOCK assert ( spec.config.MAX_REQUEST_BLOB_SIDECARS_EIP7594 == spec.config.MAX_REQUEST_BLOCKS_DENEB * spec.config.MAX_BLOBS_PER_BLOCK_EIP7594 From 7b0b7df0156edeb31b465a9a31fcfdbbc272c7b6 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Tue, 19 Nov 2024 14:33:16 -0600 Subject: [PATCH 135/142] Fix bug --- tests/core/pyspec/eth2spec/test/helpers/blob.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/blob.py b/tests/core/pyspec/eth2spec/test/helpers/blob.py index a31f032dcd..c65414b02b 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/blob.py +++ b/tests/core/pyspec/eth2spec/test/helpers/blob.py @@ -106,7 +106,7 @@ def get_sample_blob_tx(spec, blob_count=1, rng=random.Random(5566), is_valid_blo def get_max_blob_count(spec): - if is_post_eip7594: + if is_post_eip7594(spec): return spec.config.MAX_BLOBS_PER_BLOCK_EIP7594 else: return spec.config.MAX_BLOBS_PER_BLOCK From 5753be7e78cfc2e62ea988e155c5a7c74258fde9 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Tue, 19 Nov 2024 14:53:38 -0600 Subject: [PATCH 136/142] Fix blobs cast --- tests/core/pyspec/eth2spec/test/helpers/fork_choice.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py index 43555c268d..33e8535502 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py +++ b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py @@ -178,7 +178,7 @@ def add_block(spec, # Check blob_data if blob_data is not None: - blobs = spec.List[spec.Blob, spec.config.MAX_BLOB_COMMITMENTS_PER_BLOCK](blob_data.blobs) + blobs = spec.List[spec.Blob, spec.MAX_BLOB_COMMITMENTS_PER_BLOCK](blob_data.blobs) blobs_root = blobs.hash_tree_root() yield get_blobs_file_name(blobs_root=blobs_root), blobs From e472afd14407169d685723706842130162277942 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 20 Nov 2024 10:32:35 -0600 Subject: [PATCH 137/142] Include requests_root in block_hash computation --- .../test/helpers/execution_payload.py | 42 +++++++++++++++---- .../pyspec/eth2spec/test/helpers/genesis.py | 5 +++ 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py index 0766008b84..fa24309320 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py @@ -1,4 +1,5 @@ from eth_hash.auto import keccak +from hashlib import sha256 from trie import HexaryTrie from rlp import encode from rlp.sedes import big_endian_int, Binary, List @@ -7,7 +8,12 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.debug.random_value import get_random_bytes_list from eth2spec.test.helpers.withdrawals import get_expected_withdrawals -from eth2spec.test.helpers.forks import is_post_capella, is_post_deneb, is_post_eip7732 +from eth2spec.test.helpers.forks import ( + is_post_capella, + is_post_deneb, + is_post_electra, + is_post_eip7732, +) def get_execution_payload_header(spec, execution_payload): @@ -59,13 +65,22 @@ def compute_trie_root_from_indexed_data(data): return t.root_hash +def compute_requests_hash(block_requests): + m = sha256() + for r in block_requests: + if len(r) > 1: + m.update(sha256(r)) + return m.digest() + + # https://eips.ethereum.org/EIPS/eip-4895 # https://eips.ethereum.org/EIPS/eip-4844 def compute_el_header_block_hash(spec, payload_header, transactions_trie_root, withdrawals_trie_root=None, - parent_beacon_block_root=None): + parent_beacon_block_root=None, + requests_root=None): """ Computes the RLP execution block hash described by an `ExecutionPayloadHeader`. """ @@ -116,6 +131,9 @@ def compute_el_header_block_hash(spec, execution_payload_header_rlp.append((big_endian_int, payload_header.excess_blob_gas)) # parent_beacon_root execution_payload_header_rlp.append((Binary(32, 32), parent_beacon_block_root)) + if is_post_electra(spec): + # requests_root + execution_payload_header_rlp.append((Binary(32, 32), requests_root)) sedes = List([schema for schema, _ in execution_payload_header_rlp]) values = [value for _, value in execution_payload_header_rlp] @@ -191,7 +209,7 @@ def get_consolidation_request_rlp_bytes(consolidation_request): return b"\x02" + encode(values, sedes) -def compute_el_block_hash_with_parent_root(spec, payload, parent_beacon_block_root): +def compute_el_block_hash_with_new_fields(spec, payload, parent_beacon_block_root, requests_root): if payload == spec.ExecutionPayload(): return spec.Hash32() @@ -213,25 +231,35 @@ def compute_el_block_hash_with_parent_root(spec, payload, parent_beacon_block_ro transactions_trie_root, withdrawals_trie_root, parent_beacon_block_root, + requests_root, ) def compute_el_block_hash(spec, payload, pre_state): parent_beacon_block_root = None + requests_root = None if is_post_deneb(spec): previous_block_header = pre_state.latest_block_header.copy() if previous_block_header.state_root == spec.Root(): previous_block_header.state_root = pre_state.hash_tree_root() parent_beacon_block_root = previous_block_header.hash_tree_root() + if is_post_electra(spec): + requests_root = compute_requests_hash([]) - return compute_el_block_hash_with_parent_root( - spec, payload, parent_beacon_block_root) + return compute_el_block_hash_with_new_fields( + spec, payload, parent_beacon_block_root, requests_root) def compute_el_block_hash_for_block(spec, block): - return compute_el_block_hash_with_parent_root( - spec, block.body.execution_payload, block.parent_root) + requests_root = None + + if is_post_electra(spec): + requests_list = spec.get_execution_requests_list(block.body.execution_requests) + requests_root = compute_requests_hash(requests_list) + + return compute_el_block_hash_with_new_fields( + spec, block.body.execution_payload, block.parent_root, requests_root) def build_empty_post_eip7732_execution_payload_header(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/helpers/genesis.py b/tests/core/pyspec/eth2spec/test/helpers/genesis.py index bd4e5d3bf3..3213169145 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/genesis.py +++ b/tests/core/pyspec/eth2spec/test/helpers/genesis.py @@ -1,3 +1,4 @@ +from hashlib import sha256 from eth2spec.test.helpers.constants import ( PHASE0, PREVIOUS_FORK_OF, @@ -66,11 +67,14 @@ def get_sample_genesis_execution_payload_header(spec, transactions_trie_root = bytes.fromhex("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") withdrawals_trie_root = None parent_beacon_block_root = None + requests_root = None if is_post_capella(spec): withdrawals_trie_root = bytes.fromhex("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") if is_post_deneb(spec): parent_beacon_block_root = bytes.fromhex("0000000000000000000000000000000000000000000000000000000000000000") + if is_post_electra(spec): + requests_root = sha256(b"").digest() payload_header.block_hash = compute_el_header_block_hash( spec, @@ -78,6 +82,7 @@ def get_sample_genesis_execution_payload_header(spec, transactions_trie_root, withdrawals_trie_root, parent_beacon_block_root, + requests_root, ) return payload_header From b1c9d279bda21400caa9897c6552b6aa9ea9cec2 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 20 Nov 2024 10:34:36 -0600 Subject: [PATCH 138/142] Add comment to compute_requests_hash --- tests/core/pyspec/eth2spec/test/helpers/execution_payload.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py index fa24309320..e38a4c5045 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py @@ -65,6 +65,7 @@ def compute_trie_root_from_indexed_data(data): return t.root_hash +# https://eips.ethereum.org/EIPS/eip-7685 def compute_requests_hash(block_requests): m = sha256() for r in block_requests: From de52c76bd38a954c58270d402214ac716a9474a3 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Thu, 21 Nov 2024 23:58:31 +0100 Subject: [PATCH 139/142] Fix block hash computation for deposit transition tests Request hash is not considered in `compute_el_block_hash`, have to use one of the other overloads for this to work. --- .../test/electra/sanity/blocks/test_deposit_transition.py | 6 +++--- .../core/pyspec/eth2spec/test/helpers/execution_payload.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/electra/sanity/blocks/test_deposit_transition.py b/tests/core/pyspec/eth2spec/test/electra/sanity/blocks/test_deposit_transition.py index 9749c89ffd..a9c2c62814 100644 --- a/tests/core/pyspec/eth2spec/test/electra/sanity/blocks/test_deposit_transition.py +++ b/tests/core/pyspec/eth2spec/test/electra/sanity/blocks/test_deposit_transition.py @@ -12,7 +12,7 @@ prepare_deposit_request, ) from eth2spec.test.helpers.execution_payload import ( - compute_el_block_hash, + compute_el_block_hash_for_block, ) from eth2spec.test.helpers.keys import privkeys, pubkeys from eth2spec.test.helpers.state import ( @@ -134,7 +134,7 @@ def prepare_state_and_block(spec, # Assign deposits and deposit requests block.body.deposits = deposits block.body.execution_requests.deposits = deposit_requests - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload, state) + block.body.execution_payload.block_hash = compute_el_block_hash_for_block(spec, block) return state, block @@ -251,7 +251,7 @@ def test_deposit_transition__deposit_and_top_up_same_block(spec, state): # Artificially assign deposit's pubkey to a deposit request of the same block top_up_keys = [block.body.deposits[0].data.pubkey] block.body.execution_requests.deposits[0].pubkey = top_up_keys[0] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload, state) + block.body.execution_payload.block_hash = compute_el_block_hash_for_block(spec, block) pre_pending_deposits = len(state.pending_deposits) diff --git a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py index e38a4c5045..fe54b00b66 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py @@ -70,7 +70,7 @@ def compute_requests_hash(block_requests): m = sha256() for r in block_requests: if len(r) > 1: - m.update(sha256(r)) + m.update(sha256(r).digest()) return m.digest() From 15e3f151d7c7951a489404e20540b2049286d545 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 22 Nov 2024 09:03:02 +0100 Subject: [PATCH 140/142] Fix block hash computation for withdrawal sanity tests Request hash is not considered in `compute_el_block_hash`, have to use one of the other overloads for this to work. --- .../eth2spec/test/electra/sanity/blocks/test_blocks.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/electra/sanity/blocks/test_blocks.py b/tests/core/pyspec/eth2spec/test/electra/sanity/blocks/test_blocks.py index c3d2284610..5a4b98c3c8 100644 --- a/tests/core/pyspec/eth2spec/test/electra/sanity/blocks/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/electra/sanity/blocks/test_blocks.py @@ -9,7 +9,7 @@ get_signed_address_change, ) from eth2spec.test.helpers.execution_payload import ( - compute_el_block_hash, + compute_el_block_hash_for_block, ) from eth2spec.test.helpers.voluntary_exits import ( prepare_signed_exits, @@ -42,7 +42,7 @@ def test_basic_el_withdrawal_request(spec, state): ) block = build_empty_block_for_next_slot(spec, state) block.body.execution_requests.withdrawals = [withdrawal_request] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload, state) + block.body.execution_payload.block_hash = compute_el_block_hash_for_block(spec, block) signed_block = state_transition_and_sign_block(spec, state, block) yield 'blocks', [signed_block] @@ -80,7 +80,7 @@ def test_basic_btec_and_el_withdrawal_request_in_same_block(spec, state): ) block.body.execution_requests.withdrawals = [withdrawal_request] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload, state) + block.body.execution_payload.block_hash = compute_el_block_hash_for_block(spec, block) signed_block = state_transition_and_sign_block(spec, state, block) yield 'blocks', [signed_block] @@ -132,7 +132,7 @@ def test_basic_btec_before_el_withdrawal_request(spec, state): ) block_2 = build_empty_block_for_next_slot(spec, state) block_2.body.execution_requests.withdrawals = [withdrawal_request] - block_2.body.execution_payload.block_hash = compute_el_block_hash(spec, block_2.body.execution_payload, state) + block_2.body.execution_payload.block_hash = compute_el_block_hash_for_block(spec, block_2) signed_block_2 = state_transition_and_sign_block(spec, state, block_2) yield 'blocks', [signed_block_1, signed_block_2] @@ -165,7 +165,7 @@ def test_cl_exit_and_el_withdrawal_request_in_same_block(spec, state): block = build_empty_block_for_next_slot(spec, state) block.body.voluntary_exits = signed_voluntary_exits block.body.execution_requests.withdrawals = [withdrawal_request] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload, state) + block.body.execution_payload.block_hash = compute_el_block_hash_for_block(spec, block) signed_block = state_transition_and_sign_block(spec, state, block) yield 'blocks', [signed_block] From 6e9526360ad34c44f699b511e60fec61f323cb7c Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 22 Nov 2024 13:31:41 +0100 Subject: [PATCH 141/142] Clean up dead link and typo in LC docs for Electra Followup from #3987 to remove references to the deleted document. --- specs/electra/light-client/fork.md | 2 +- specs/electra/light-client/sync-protocol.md | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/specs/electra/light-client/fork.md b/specs/electra/light-client/fork.md index a315146b0e..902c1d6bf3 100644 --- a/specs/electra/light-client/fork.md +++ b/specs/electra/light-client/fork.md @@ -33,7 +33,7 @@ def normalize_merkle_branch(branch: Sequence[Bytes32], ## Upgrading light client data -A Electra `LightClientStore` can still process earlier light client data. In order to do so, that pre-Electra data needs to be locally upgraded to Electra before processing. +An Electra `LightClientStore` can still process earlier light client data. In order to do so, that pre-Electra data needs to be locally upgraded to Electra before processing. ```python def upgrade_lc_header_to_electra(pre: deneb.LightClientHeader) -> LightClientHeader: diff --git a/specs/electra/light-client/sync-protocol.md b/specs/electra/light-client/sync-protocol.md index f042119b00..e3e41cfb79 100644 --- a/specs/electra/light-client/sync-protocol.md +++ b/specs/electra/light-client/sync-protocol.md @@ -27,7 +27,6 @@ This upgrade updates light client data to include the Electra changes to the [`ExecutionPayload`](../beacon-chain.md) structure and to the generalized indices of surrounding containers. It extends the [Deneb Light Client specifications](../../deneb/light-client/sync-protocol.md). The [fork document](./fork.md) explains how to upgrade existing Deneb based deployments to Electra. Additional documents describes the impact of the upgrade on certain roles: -- [Full node](./full-node.md) - [Networking](./p2p-interface.md) ## Custom types From 4aad8ebcdc1cb8030e1f6b2a8d05a8adfb6b45ef Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Fri, 22 Nov 2024 07:02:29 -0600 Subject: [PATCH 142/142] Rename requests_root to requests_hash --- .../test/helpers/execution_payload.py | 22 +++++++++---------- .../pyspec/eth2spec/test/helpers/genesis.py | 6 ++--- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py index fe54b00b66..80684b9e6e 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py @@ -81,7 +81,7 @@ def compute_el_header_block_hash(spec, transactions_trie_root, withdrawals_trie_root=None, parent_beacon_block_root=None, - requests_root=None): + requests_hash=None): """ Computes the RLP execution block hash described by an `ExecutionPayloadHeader`. """ @@ -133,8 +133,8 @@ def compute_el_header_block_hash(spec, # parent_beacon_root execution_payload_header_rlp.append((Binary(32, 32), parent_beacon_block_root)) if is_post_electra(spec): - # requests_root - execution_payload_header_rlp.append((Binary(32, 32), requests_root)) + # requests_hash + execution_payload_header_rlp.append((Binary(32, 32), requests_hash)) sedes = List([schema for schema, _ in execution_payload_header_rlp]) values = [value for _, value in execution_payload_header_rlp] @@ -210,7 +210,7 @@ def get_consolidation_request_rlp_bytes(consolidation_request): return b"\x02" + encode(values, sedes) -def compute_el_block_hash_with_new_fields(spec, payload, parent_beacon_block_root, requests_root): +def compute_el_block_hash_with_new_fields(spec, payload, parent_beacon_block_root, requests_hash): if payload == spec.ExecutionPayload(): return spec.Hash32() @@ -232,13 +232,13 @@ def compute_el_block_hash_with_new_fields(spec, payload, parent_beacon_block_roo transactions_trie_root, withdrawals_trie_root, parent_beacon_block_root, - requests_root, + requests_hash, ) def compute_el_block_hash(spec, payload, pre_state): parent_beacon_block_root = None - requests_root = None + requests_hash = None if is_post_deneb(spec): previous_block_header = pre_state.latest_block_header.copy() @@ -246,21 +246,21 @@ def compute_el_block_hash(spec, payload, pre_state): previous_block_header.state_root = pre_state.hash_tree_root() parent_beacon_block_root = previous_block_header.hash_tree_root() if is_post_electra(spec): - requests_root = compute_requests_hash([]) + requests_hash = compute_requests_hash([]) return compute_el_block_hash_with_new_fields( - spec, payload, parent_beacon_block_root, requests_root) + spec, payload, parent_beacon_block_root, requests_hash) def compute_el_block_hash_for_block(spec, block): - requests_root = None + requests_hash = None if is_post_electra(spec): requests_list = spec.get_execution_requests_list(block.body.execution_requests) - requests_root = compute_requests_hash(requests_list) + requests_hash = compute_requests_hash(requests_list) return compute_el_block_hash_with_new_fields( - spec, block.body.execution_payload, block.parent_root, requests_root) + spec, block.body.execution_payload, block.parent_root, requests_hash) def build_empty_post_eip7732_execution_payload_header(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/helpers/genesis.py b/tests/core/pyspec/eth2spec/test/helpers/genesis.py index 3213169145..9c43676a41 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/genesis.py +++ b/tests/core/pyspec/eth2spec/test/helpers/genesis.py @@ -67,14 +67,14 @@ def get_sample_genesis_execution_payload_header(spec, transactions_trie_root = bytes.fromhex("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") withdrawals_trie_root = None parent_beacon_block_root = None - requests_root = None + requests_hash = None if is_post_capella(spec): withdrawals_trie_root = bytes.fromhex("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") if is_post_deneb(spec): parent_beacon_block_root = bytes.fromhex("0000000000000000000000000000000000000000000000000000000000000000") if is_post_electra(spec): - requests_root = sha256(b"").digest() + requests_hash = sha256(b"").digest() payload_header.block_hash = compute_el_header_block_hash( spec, @@ -82,7 +82,7 @@ def get_sample_genesis_execution_payload_header(spec, transactions_trie_root, withdrawals_trie_root, parent_beacon_block_root, - requests_root, + requests_hash, ) return payload_header