Merge commit 'v1.5.0-alpha.5' into eip-7495

.gitignore

@@ -25,6 +25,7 @@ tests/core/pyspec/eth2spec/electra/
 tests/core/pyspec/eth2spec/whisk/
 tests/core/pyspec/eth2spec/eip7594/
 tests/core/pyspec/eth2spec/eip6800/
+tests/core/pyspec/eth2spec/eip7732/
 
 # coverage reports
 .htmlcov

Makefile

@@ -14,7 +14,6 @@ SOLIDITY_FILE_NAME = deposit_contract.json
 DEPOSIT_CONTRACT_TESTER_DIR = ${SOLIDITY_DEPOSIT_CONTRACT_DIR}/web3_tester
 CONFIGS_DIR = ./configs
 TEST_PRESET_TYPE ?= minimal
-NUMBER_OF_CORES=16
 # Collect a list of generator names
 GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/.)))
 # Map this list of generator paths to "gen_{generator name}" entries
@@ -35,7 +34,7 @@ MARKDOWN_FILES = $(wildcard $(SPEC_DIR)/*/*.md) \
                  $(wildcard $(SPEC_DIR)/_features/*/*/*.md) \
                  $(wildcard $(SSZ_DIR)/*.md)
 
-ALL_EXECUTABLE_SPEC_NAMES = phase0 altair bellatrix capella deneb electra whisk eip6800
+ALL_EXECUTABLE_SPEC_NAMES = phase0 altair bellatrix capella deneb electra whisk eip6800 eip7732
 # The parameters for commands. Use `foreach` to avoid listing specs again.
 COVERAGE_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPEC_NAMES), --cov=eth2spec.$S.$(TEST_PRESET_TYPE))
 PYLINT_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPEC_NAMES), ./eth2spec/$S)
@@ -118,7 +117,7 @@ install_test: preinstallation
 # Testing against `minimal` or `mainnet` config by default
 test: pyspec
 	. venv/bin/activate; cd $(PY_SPEC_DIR); \
-	python3 -m pytest -n 4 --disable-bls $(COVERAGE_SCOPE) --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
+	python3 -m pytest -n auto --disable-bls $(COVERAGE_SCOPE) --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
 
 # Testing against `minimal` or `mainnet` config by default
 find_test: pyspec
@@ -129,10 +128,10 @@ citest: pyspec
 	mkdir -p $(TEST_REPORT_DIR);
 ifdef fork
 	. venv/bin/activate; cd $(PY_SPEC_DIR); \
-	python3 -m pytest -n $(NUMBER_OF_CORES) --bls-type=fastest --preset=$(TEST_PRESET_TYPE) --fork=$(fork) --junitxml=test-reports/test_results.xml eth2spec
+	python3 -m pytest -n auto --bls-type=fastest --preset=$(TEST_PRESET_TYPE) --fork=$(fork) --junitxml=test-reports/test_results.xml eth2spec
 else
 	. venv/bin/activate; cd $(PY_SPEC_DIR); \
-	python3 -m pytest -n $(NUMBER_OF_CORES) --bls-type=fastest --preset=$(TEST_PRESET_TYPE) --junitxml=test-reports/test_results.xml eth2spec
+	python3 -m pytest -n auto --bls-type=fastest --preset=$(TEST_PRESET_TYPE) --junitxml=test-reports/test_results.xml eth2spec
 endif
 
 

configs/mainnet.yaml

@@ -59,6 +59,9 @@ EIP7594_FORK_EPOCH: 18446744073709551615
 # WHISK
 WHISK_FORK_VERSION: 0x08000000  # temporary stub
 WHISK_FORK_EPOCH: 18446744073709551615
+# EIP7732
+EIP7732_FORK_VERSION: 0x09000000  # temporary stub
+EIP7732_FORK_EPOCH: 18446744073709551615
 
 # Time parameters
 # ---------------------------------------------------------------
@@ -158,12 +161,10 @@ WHISK_PROPOSER_SELECTION_GAP: 2
 
 # EIP7594
 NUMBER_OF_COLUMNS: 128
-MAX_CELLS_IN_EXTENDED_MATRIX: 768
-DATA_COLUMN_SIDECAR_SUBNET_COUNT: 32
+DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128
 MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384
 SAMPLES_PER_SLOT: 8
-CUSTODY_REQUIREMENT: 1
-TARGET_NUMBER_OF_PEERS: 70
+CUSTODY_REQUIREMENT: 4
 
 # [New in Electra:EIP7251]
 MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA: 128000000000 # 2**7 * 10**9 (= 128,000,000,000)

configs/minimal.yaml

@@ -58,6 +58,9 @@ EIP7594_FORK_EPOCH: 18446744073709551615
 # WHISK
 WHISK_FORK_VERSION: 0x08000001
 WHISK_FORK_EPOCH: 18446744073709551615
+# EIP7732
+EIP7732_FORK_VERSION: 0x09000001
+EIP7732_FORK_EPOCH: 18446744073709551615
 
 # Time parameters
 # ---------------------------------------------------------------
@@ -157,12 +160,10 @@ WHISK_PROPOSER_SELECTION_GAP: 1
 
 # EIP7594
 NUMBER_OF_COLUMNS: 128
-MAX_CELLS_IN_EXTENDED_MATRIX: 768
-DATA_COLUMN_SIDECAR_SUBNET_COUNT: 32
+DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128
 MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384
 SAMPLES_PER_SLOT: 8
-CUSTODY_REQUIREMENT: 1
-TARGET_NUMBER_OF_PEERS: 70
+CUSTODY_REQUIREMENT: 4
 
 # [New in Electra:EIP7251]
 MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA: 64000000000 # 2**6 * 10**9 (= 64,000,000,000)

docker/README.md

@@ -6,15 +6,15 @@ This dockerfile sets up the dependencies required to run consensus-spec tests. T
 
 Handy commands:
 - `docker run -it $IMAGE_NAME /bin/sh` will give you a shell inside the docker container to manually run any tests
-- `docker run  $IMAGE_NAME make citest` will run the make citest command inside the docker container
+- `docker run $IMAGE_NAME make citest` will run the make citest command inside the docker container
 
 Ideally manual running of docker containers is for advanced users, we recommend the script based approach described below for most users.
 
-The `scripts/build_run_docker_tests.sh` script will cover most usecases. The script allows the user to configure the fork(altair/bellatrix/capella..), `$IMAGE_NAME` (specifies the container to use), number of cores, preset type (mainnet/minimal), and test all forks flags. Ideally, this is the main way that users interact with the spec tests instead of running it locally with varying versions of dependencies.
+The `scripts/build_run_docker_tests.sh` script will cover most usecases. The script allows the user to configure the fork(altair/bellatrix/capella..), `$IMAGE_NAME` (specifies the container to use), preset type (mainnet/minimal), and test all forks flags. Ideally, this is the main way that users interact with the spec tests instead of running it locally with varying versions of dependencies.
 
 E.g:
-- `./build_run_test.sh --p mainnet --n 16` will run the mainnet preset tests with 16 threads
+- `./build_run_test.sh --p mainnet` will run the mainnet preset tests
 - `./build_run_test.sh --a` will run all the tests across all the forks
-- `./build_run_test.sh --f deneb --n 16` will only run deneb tests on 16 threads
+- `./build_run_test.sh --f deneb` will only run deneb tests
 
 Results are always placed in a folder called `./testResults`. The results are `.xml` files and contain the fork they represent and the date/time they were run at.

fork_choice/safe-block.md

@@ -7,7 +7,7 @@
 
 - [Introduction](#introduction)
 - [`get_safe_beacon_block_root`](#get_safe_beacon_block_root)
-- [`get_safe_execution_payload_hash`](#get_safe_execution_payload_hash)
+- [`get_safe_execution_block_hash`](#get_safe_execution_block_hash)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
 <!-- /TOC -->
@@ -31,10 +31,10 @@ def get_safe_beacon_block_root(store: Store) -> Root:
 *Note*: Currently safe block algorithm simply returns `store.justified_checkpoint.root`
 and is meant to be improved in the future.
 
-## `get_safe_execution_payload_hash`
+## `get_safe_execution_block_hash`
 
 ```python
-def get_safe_execution_payload_hash(store: Store) -> Hash32:
+def get_safe_execution_block_hash(store: Store) -> Hash32:
     safe_block_root = get_safe_beacon_block_root(store)
     safe_block = store.blocks[safe_block_root]
 

presets/mainnet/eip-7732.yaml

@@ -0,0 +1,9 @@
+# Mainnet preset - EIP7732
+
+# Execution
+# ---------------------------------------------------------------
+# 2**9 (= 512)
+PTC_SIZE: 512
+# 2**2 (= 4)
+MAX_PAYLOAD_ATTESTATIONS: 4
+KZG_COMMITMENT_INCLUSION_PROOF_DEPTH_EIP7732: 13

presets/minimal/eip-7732.yaml

@@ -0,0 +1,9 @@
+# Minimal preset - EIP7732
+
+# Execution
+# ---------------------------------------------------------------
+# 2**1(= 2)
+PTC_SIZE: 2
+# 2**2 (= 4)
+MAX_PAYLOAD_ATTESTATIONS: 4
+KZG_COMMITMENT_INCLUSION_PROOF_DEPTH_EIP7732: 13

pysetup/constants.py

@@ -8,6 +8,7 @@
 EIP7594 = 'eip7594'
 EIP6800 = 'eip6800'
 WHISK = 'whisk'
+EIP7732 = 'eip7732'
 
 
 # The helper functions that are used when defining constants

pysetup/helpers.py

@@ -124,13 +124,22 @@ def format_constant(name: str, vardef: VariableDefinition) -> str:
     # Keep engine from the most recent fork
     execution_engine_cls = reduce(lambda txt, builder: builder.execution_engine_cls() or txt, builders, "")
 
+    # Remove deprecated constants
+    deprecate_constants = reduce(lambda obj, builder: obj.union(builder.deprecate_constants()), builders, set())
+    # constant_vars = {k: v for k, v in spec_object.constant_vars.items() if k not in deprecate_constants}
+    filtered_ssz_dep_constants = {k: v for k, v in hardcoded_ssz_dep_constants.items() if k not in deprecate_constants}
+    # Remove deprecated presets
+    deprecate_presets = reduce(lambda obj, builder: obj.union(builder.deprecate_presets()), builders, set())
+    # preset_vars = {k: v for k, v in spec_object.constant_vars.items() if k not in deprecate_constants}
+    filtered_hardcoded_func_dep_presets = {k: v for k, v in hardcoded_func_dep_presets.items() if k not in deprecate_presets}
+
     constant_vars_spec = '# Constant vars\n' + '\n'.join(format_constant(k, v) for k, v in spec_object.constant_vars.items())
     preset_vars_spec = '# Preset vars\n' + '\n'.join(format_constant(k, v) for k, v in spec_object.preset_vars.items())
     ordered_class_objects_spec = '\n\n\n'.join(ordered_class_objects.values())
     ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, hardcoded_ssz_dep_constants[x]), hardcoded_ssz_dep_constants))
-    ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), hardcoded_ssz_dep_constants))
+    ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), filtered_ssz_dep_constants))
     custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, hardcoded_custom_type_dep_constants[x]), hardcoded_custom_type_dep_constants))
-    func_dep_presets_verification = '\n'.join(map(lambda x: 'assert %s == %s  # noqa: E501' % (x, spec_object.func_dep_presets[x]), hardcoded_func_dep_presets))
+    func_dep_presets_verification = '\n'.join(map(lambda x: 'assert %s == %s  # noqa: E501' % (x, spec_object.func_dep_presets[x]), filtered_hardcoded_func_dep_presets))
     spec_strs = [
         imports,
         preparations,

pysetup/md_doc_paths.py

@@ -10,6 +10,7 @@
     WHISK,
     EIP7594,
     EIP6800,
+    EIP7732,
 )
 
 
@@ -23,6 +24,7 @@
     WHISK: CAPELLA,
     EIP7594: DENEB,
     EIP6800: DENEB,
+    EIP7732: ELECTRA,
 }
 
 ALL_FORKS = list(PREVIOUS_FORK_OF.keys())

pysetup/spec_builders/__init__.py

@@ -7,12 +7,13 @@
 from .whisk import WhiskSpecBuilder
 from .eip7594 import EIP7594SpecBuilder
 from .eip6800 import EIP6800SpecBuilder
+from .eip7732 import EIP7732SpecBuilder
 
 
 spec_builders = {
     builder.fork: builder
     for builder in (
         Phase0SpecBuilder, AltairSpecBuilder, BellatrixSpecBuilder, CapellaSpecBuilder, DenebSpecBuilder,
-        ElectraSpecBuilder, WhiskSpecBuilder, EIP7594SpecBuilder, EIP6800SpecBuilder,
+        ElectraSpecBuilder, WhiskSpecBuilder, EIP7594SpecBuilder, EIP6800SpecBuilder, EIP7732SpecBuilder,
     )
 }

pysetup/spec_builders/altair.py

@@ -35,7 +35,7 @@ def get_generalized_index(ssz_class: Any, *path: PyUnion[int, SSZVariableName])
 
 
 def compute_merkle_proof(object: SSZObject,
-                         index: GeneralizedIndex) -> Sequence[Bytes32]:
+                         index: GeneralizedIndex) -> list[Bytes32]:
     return build_proof(object.get_backing(), index)'''
 
 

pysetup/spec_builders/base.py

@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-from typing import Sequence, Dict
+from typing import Sequence, Dict, Set
 from pathlib import Path
 
 class BaseSpecBuilder(ABC):
@@ -54,3 +54,11 @@ def hardcoded_func_dep_presets(cls, spec_object) -> Dict[str, str]:
     @classmethod
     def implement_optimizations(cls, functions: Dict[str, str]) -> Dict[str, str]:
         return functions
+
+    @classmethod
+    def deprecate_constants(cls) -> Set[str]:
+        return set()
+
+    @classmethod
+    def deprecate_presets(cls) -> Set[str]:
+        return set()

pysetup/spec_builders/eip7594.py

@@ -12,14 +12,21 @@ def imports(cls, preset_name: str):
         return f'''
 from eth2spec.deneb import {preset_name} as deneb
 '''
+
+
+    @classmethod
+    def sundry_functions(cls) -> str:
+        return """
+def retrieve_column_sidecars(beacon_block_root: Root) -> Sequence[DataColumnSidecar]:
+    return []
+"""
 
     @classmethod
     def hardcoded_custom_type_dep_constants(cls, spec_object) -> str:
         return {
             'FIELD_ELEMENTS_PER_CELL': spec_object.preset_vars['FIELD_ELEMENTS_PER_CELL'].value,
             'FIELD_ELEMENTS_PER_EXT_BLOB': spec_object.preset_vars['FIELD_ELEMENTS_PER_EXT_BLOB'].value,
             'NUMBER_OF_COLUMNS': spec_object.config_vars['NUMBER_OF_COLUMNS'].value,
-            'MAX_CELLS_IN_EXTENDED_MATRIX': spec_object.config_vars['MAX_CELLS_IN_EXTENDED_MATRIX'].value,
         }
 
     @classmethod

pysetup/spec_builders/eip7732.py

@@ -0,0 +1,45 @@
+from typing import Dict, Set
+
+from .base import BaseSpecBuilder
+from ..constants import EIP7732
+
+
+class EIP7732SpecBuilder(BaseSpecBuilder):
+    fork: str = EIP7732
+
+    @classmethod
+    def imports(cls, preset_name: str):
+        return f'''
+from eth2spec.electra import {preset_name} as electra
+'''
+
+    @classmethod
+    def sundry_functions(cls) -> str:
+        return '''
+def concat_generalized_indices(*indices: GeneralizedIndex) -> GeneralizedIndex:
+    o = GeneralizedIndex(1)
+    for i in indices:
+        o = GeneralizedIndex(o * bit_floor(i) + (i - bit_floor(i)))
+    return o'''
+
+
+    @classmethod
+    def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]:
+        return {
+            'PTC_SIZE': spec_object.preset_vars['PTC_SIZE'].value,
+            'MAX_PAYLOAD_ATTESTATIONS': spec_object.preset_vars['MAX_PAYLOAD_ATTESTATIONS'].value,
+            'KZG_COMMITMENT_INCLUSION_PROOF_DEPTH_EIP7732':
+                spec_object.preset_vars['KZG_COMMITMENT_INCLUSION_PROOF_DEPTH_EIP7732'].value,
+        }
+
+    @classmethod
+    def deprecate_constants(cls) -> Set[str]:
+        return set([
+            'EXECUTION_PAYLOAD_GINDEX',
+        ])
+
+    @classmethod
+    def deprecate_presets(cls) -> Set[str]:
+        return set([
+            'KZG_COMMITMENT_INCLUSION_PROOF_DEPTH',
+        ])

scripts/build_run_docker_tests.sh

@@ -13,25 +13,22 @@
 ALL_EXECUTABLE_SPECS=("phase0" "altair" "bellatrix" "capella" "deneb" "electra" "whisk")
 TEST_PRESET_TYPE=minimal
 FORK_TO_TEST=phase0
-NUMBER_OF_CORES=4
 WORKDIR="//consensus-specs//tests//core//pyspec"
 ETH2SPEC_FOLDER_NAME="eth2spec"
 CONTAINER_NAME="consensus-specs-tests"
 DATE=$(date +"%Y%m%d-%H-%M")
 # Default flag values
 version=$(git log --pretty=format:'%h' -n 1)
 IMAGE_NAME="consensus-specs:$version"
-number_of_core=4
 
 # displays the available options
 display_help() {
   echo "Run 'consensus-specs' tests from a container instance."
   echo "Be sure to launch Docker before running this script."
   echo
-  echo "Syntax: build_run_test.sh [--v TAG | --n NUMBER_OF_CORE | --f FORK_TO_TEST | --p PRESET_TYPE | --a | --h HELP]"
+  echo "Syntax: build_run_test.sh [--v TAG | --f FORK_TO_TEST | --p PRESET_TYPE | --a | --h HELP]"
     echo "  --f <fork>   Specify the fork to test"
     echo "  --i <image_name> Specify the docker image to use"
-    echo "  --n <number> Specify the number of cores"
     echo "  --p <type>   Specify the test preset type"
     echo "  --a          Test all forks"
     echo "  --h          Display this help and exit"
@@ -63,7 +60,6 @@ while [[ "$#" -gt 0 ]]; do
     case $1 in
         --f) FORK_TO_TEST="$2"; shift ;;
         --v) IMAGE_NAME="$2"; shift ;;
-        --n) NUMBER_OF_CORES="$2"; shift ;;
         --p) TEST_PRESET_TYPE="$2"; shift ;;
         --a) FORK_TO_TEST="all" ;;
         --h) display_help; exit 0 ;;
@@ -90,12 +86,12 @@ fi
 if [ "$FORK_TO_TEST" == "all" ]; then
   for fork in "${ALL_EXECUTABLE_SPECS[@]}"; do
     docker run --name $CONTAINER_NAME $IMAGE_NAME \
-      make citest fork=$fork TEST_PRESET_TYPE=$TEST_PRESET_TYPE NUMBER_OF_CORES=$NUMBER_OF_CORES
+      make citest fork=$fork TEST_PRESET_TYPE=$TEST_PRESET_TYPE
       copy_test_results $fork
   done
 else
   docker run --name $CONTAINER_NAME $IMAGE_NAME \
-      make citest fork=$FORK_TO_TEST TEST_PRESET_TYPE=$TEST_PRESET_TYPE NUMBER_OF_CORES=$NUMBER_OF_CORES
+      make citest fork=$FORK_TO_TEST TEST_PRESET_TYPE=$TEST_PRESET_TYPE
   copy_test_results $FORK_TO_TEST
 fi