diff --git a/ethereum/executionclient/src/integration-test/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClientTest.java b/ethereum/executionclient/src/integration-test/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClientTest.java index 146186e0a5f..61b01eda021 100644 --- a/ethereum/executionclient/src/integration-test/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClientTest.java +++ b/ethereum/executionclient/src/integration-test/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClientTest.java @@ -48,6 +48,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.TestTemplate; import tech.pegasys.teku.ethereum.events.ExecutionClientEventsChannel; +import tech.pegasys.teku.ethereum.executionclient.schema.BlobAndProofV1; import tech.pegasys.teku.ethereum.executionclient.schema.ClientVersionV1; import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV3; import tech.pegasys.teku.ethereum.executionclient.schema.ForkChoiceStateV1; @@ -65,6 +66,7 @@ import tech.pegasys.teku.spec.SpecMilestone; import tech.pegasys.teku.spec.TestSpecContext; import tech.pegasys.teku.spec.TestSpecInvocationContextProvider.SpecContext; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayload; import tech.pegasys.teku.spec.executionlayer.PayloadStatus; import tech.pegasys.teku.spec.logic.versions.deneb.types.VersionedHash; @@ -355,6 +357,50 @@ public void newPayloadV4_shouldBuildRequestAndResponseSuccessfully() { .isEqualTo(parentBeaconBlockRoot.toHexString()); } + @TestTemplate + @SuppressWarnings("unchecked") + public void getBlobsV1_shouldBuildRequestAndResponseSuccessfully() { + assumeThat(specMilestone).isGreaterThanOrEqualTo(DENEB); + final List blobSidecars = + dataStructureUtil.randomBlobSidecars(spec.getMaxBlobsPerBlock().orElseThrow()); + final List blobsAndProofsV1 = + blobSidecars.stream() + .map( + blobSidecar -> + new BlobAndProofV1( + blobSidecar.getBlob().getBytes(), + blobSidecar.getKZGProof().getBytesCompressed())) + .toList(); + final String blobsAndProofsJson = + blobSidecars.stream() + .map( + blobSidecar -> + String.format( + "{ \"blob\": \"%s\", \"proof\": \"%s\" }", + blobSidecar.getBlob().getBytes().toHexString(), + blobSidecar.getKZGProof().getBytesCompressed().toHexString())) + .collect(Collectors.joining(", ")); + final String bodyResponse = + "{\"jsonrpc\": \"2.0\", \"id\": 0, \"result\": [" + blobsAndProofsJson + "]}"; + + mockSuccessfulResponse(bodyResponse); + + final List blobVersionedHashes = dataStructureUtil.randomVersionedHashes(3); + + final SafeFuture>> futureResponse = + eeClient.getBlobsV1(blobVersionedHashes); + + assertThat(futureResponse) + .succeedsWithin(1, TimeUnit.SECONDS) + .matches(response -> response.getPayload().equals(blobsAndProofsV1)); + + final Map requestData = takeRequest(); + verifyJsonRpcMethodCall(requestData, "engine_getBlobsV1"); + assertThat(requestData.get("params")) + .asInstanceOf(LIST) + .containsExactly(blobVersionedHashes.stream().map(VersionedHash::toHexString).toList()); + } + private void mockSuccessfulResponse(final String responseBody) { mockWebServer.enqueue( new MockResponse() diff --git a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/ExecutionEngineClient.java b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/ExecutionEngineClient.java index e66b17999e2..ebbdd3d4286 100644 --- a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/ExecutionEngineClient.java +++ b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/ExecutionEngineClient.java @@ -16,6 +16,7 @@ import java.util.List; import java.util.Optional; import org.apache.tuweni.bytes.Bytes32; +import tech.pegasys.teku.ethereum.executionclient.schema.BlobAndProofV1; import tech.pegasys.teku.ethereum.executionclient.schema.ClientVersionV1; import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1; import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV2; @@ -77,4 +78,6 @@ SafeFuture> forkChoiceUpdatedV3( SafeFuture>> exchangeCapabilities(List capabilities); SafeFuture>> getClientVersionV1(ClientVersionV1 clientVersion); + + SafeFuture>> getBlobsV1(List blobVersionedHashes); } diff --git a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/ThrottlingExecutionEngineClient.java b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/ThrottlingExecutionEngineClient.java index ef0600b784a..8ee347e077b 100644 --- a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/ThrottlingExecutionEngineClient.java +++ b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/ThrottlingExecutionEngineClient.java @@ -17,6 +17,7 @@ import java.util.Optional; import org.apache.tuweni.bytes.Bytes32; import org.hyperledger.besu.plugin.services.MetricsSystem; +import tech.pegasys.teku.ethereum.executionclient.schema.BlobAndProofV1; import tech.pegasys.teku.ethereum.executionclient.schema.ClientVersionV1; import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1; import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV2; @@ -155,4 +156,10 @@ public SafeFuture>> getClientVersionV1( final ClientVersionV1 clientVersion) { return taskQueue.queueTask(() -> delegate.getClientVersionV1(clientVersion)); } + + @Override + public SafeFuture>> getBlobsV1( + final List blobVersionedHashes) { + return taskQueue.queueTask(() -> delegate.getBlobsV1(blobVersionedHashes)); + } } diff --git a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineApiMethod.java b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineApiMethod.java index 44ec33d7a4c..a47654c47fe 100644 --- a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineApiMethod.java +++ b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineApiMethod.java @@ -16,7 +16,8 @@ public enum EngineApiMethod { ENGINE_NEW_PAYLOAD("engine_newPayload"), ENGINE_GET_PAYLOAD("engine_getPayload"), - ENGINE_FORK_CHOICE_UPDATED("engine_forkchoiceUpdated"); + ENGINE_FORK_CHOICE_UPDATED("engine_forkchoiceUpdated"), + ENGINE_GET_BLOBS("engine_getBlobs"); private final String name; diff --git a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java new file mode 100644 index 00000000000..40426616b87 --- /dev/null +++ b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java @@ -0,0 +1,93 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.ethereum.executionclient.methods; + +import java.util.List; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import tech.pegasys.teku.ethereum.executionclient.ExecutionEngineClient; +import tech.pegasys.teku.ethereum.executionclient.response.ResponseUnwrapper; +import tech.pegasys.teku.infrastructure.async.SafeFuture; +import tech.pegasys.teku.infrastructure.unsigned.UInt64; +import tech.pegasys.teku.spec.Spec; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSchema; +import tech.pegasys.teku.spec.datastructures.execution.BlobAndProof; +import tech.pegasys.teku.spec.logic.versions.deneb.types.VersionedHash; +import tech.pegasys.teku.spec.schemas.SchemaDefinitions; +import tech.pegasys.teku.spec.schemas.SchemaDefinitionsDeneb; + +public class EngineGetBlobsV1 extends AbstractEngineJsonRpcMethod> { + + private static final Logger LOG = LogManager.getLogger(); + private final Spec spec; + + public EngineGetBlobsV1(final ExecutionEngineClient executionEngineClient, final Spec spec) { + super(executionEngineClient); + this.spec = spec; + } + + @Override + public String getName() { + return EngineApiMethod.ENGINE_GET_BLOBS.getName(); + } + + @Override + public int getVersion() { + return 1; + } + + @Override + public boolean isOptional() { + return true; + } + + @Override + public SafeFuture> execute(final JsonRpcRequestParams params) { + + final List blobVersionedHashes = + params.getRequiredListParameter(0, VersionedHash.class); + + final UInt64 slot = params.getRequiredParameter(1, UInt64.class); + + LOG.trace( + "Calling {}(blobVersionedHashes={}, slot={})", + getVersionedName(), + blobVersionedHashes, + slot); + + return executionEngineClient + .getBlobsV1(blobVersionedHashes) + .thenApply(ResponseUnwrapper::unwrapExecutionClientResponseOrThrow) + .thenApply( + response -> { + final SchemaDefinitions schemaDefinitions = spec.atSlot(slot).getSchemaDefinitions(); + final BlobSchema blobSchema = + SchemaDefinitionsDeneb.required(schemaDefinitions).getBlobSchema(); + return response.stream() + .map( + blobAndProofV1 -> + blobAndProofV1 == null + ? null + : blobAndProofV1.asInternalBlobsAndProofs(blobSchema)) + .toList(); + }) + .thenPeek( + blobsAndProofs -> + LOG.trace( + "Response {}(blobVersionedHashes={}) -> {}", + getVersionedName(), + blobVersionedHashes, + blobsAndProofs)); + } +} diff --git a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineJsonRpcMethod.java b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineJsonRpcMethod.java index ddc4b4f196e..8c41f711697 100644 --- a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineJsonRpcMethod.java +++ b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineJsonRpcMethod.java @@ -28,6 +28,12 @@ default boolean isDeprecated() { return false; } + // TODO should be remove once all ELs implement engine_getBlobsV1. It has been added only to + // better handle the use case when the method is missing in the EL side + default boolean isOptional() { + return false; + } + default String getVersionedName() { return getVersion() == 0 ? getName() : getName() + "V" + getVersion(); } diff --git a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/metrics/MetricRecordingExecutionEngineClient.java b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/metrics/MetricRecordingExecutionEngineClient.java index b7a17765912..4e6d59668a0 100644 --- a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/metrics/MetricRecordingExecutionEngineClient.java +++ b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/metrics/MetricRecordingExecutionEngineClient.java @@ -19,6 +19,7 @@ import org.apache.tuweni.bytes.Bytes32; import org.hyperledger.besu.plugin.services.MetricsSystem; import tech.pegasys.teku.ethereum.executionclient.ExecutionEngineClient; +import tech.pegasys.teku.ethereum.executionclient.schema.BlobAndProofV1; import tech.pegasys.teku.ethereum.executionclient.schema.ClientVersionV1; import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1; import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV2; @@ -65,6 +66,7 @@ public class MetricRecordingExecutionEngineClient extends MetricRecordingAbstrac public static final String NEW_PAYLOAD_V4_METHOD = "new_payloadV4"; public static final String EXCHANGE_CAPABILITIES_METHOD = "exchange_capabilities"; public static final String GET_CLIENT_VERSION_V1_METHOD = "get_client_versionV1"; + public static final String GET_BLOBS_V1_METHOD = "get_blobs_versionV1"; private final ExecutionEngineClient delegate; @@ -194,4 +196,10 @@ public SafeFuture>> getClientVersionV1( return countRequest( () -> delegate.getClientVersionV1(clientVersion), GET_CLIENT_VERSION_V1_METHOD); } + + @Override + public SafeFuture>> getBlobsV1( + final List blobVersionedHashes) { + return countRequest(() -> delegate.getBlobsV1(blobVersionedHashes), GET_BLOBS_V1_METHOD); + } } diff --git a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/schema/BlobAndProofV1.java b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/schema/BlobAndProofV1.java new file mode 100644 index 00000000000..1a73026a865 --- /dev/null +++ b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/schema/BlobAndProofV1.java @@ -0,0 +1,88 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.ethereum.executionclient.schema; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.google.common.base.MoreObjects; +import java.util.Objects; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes48; +import tech.pegasys.teku.ethereum.executionclient.serialization.Bytes48Deserializer; +import tech.pegasys.teku.ethereum.executionclient.serialization.BytesDeserializer; +import tech.pegasys.teku.ethereum.executionclient.serialization.BytesSerializer; +import tech.pegasys.teku.kzg.KZGProof; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.Blob; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSchema; +import tech.pegasys.teku.spec.datastructures.execution.BlobAndProof; + +public class BlobAndProofV1 { + + @JsonSerialize(using = BytesSerializer.class) + @JsonDeserialize(using = BytesDeserializer.class) + private final Bytes blob; + + @JsonSerialize(using = BytesSerializer.class) + @JsonDeserialize(using = Bytes48Deserializer.class) + private final Bytes48 proof; + + public BlobAndProofV1( + @JsonProperty("blob") final Bytes blob, @JsonProperty("proof") final Bytes48 proof) { + checkNotNull(blob, "blob"); + checkNotNull(proof, "proof"); + this.proof = proof; + this.blob = blob; + } + + public BlobAndProof asInternalBlobsAndProofs(final BlobSchema blobSchema) { + return new BlobAndProof(new Blob(blobSchema, blob), new KZGProof(proof)); + } + + public static BlobAndProofV1 fromInternalBlobsBundle(final BlobAndProof blobAndProof) { + return new BlobAndProofV1( + blobAndProof.blob().getBytes(), blobAndProof.proof().getBytesCompressed()); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final BlobAndProofV1 that = (BlobAndProofV1) o; + return Objects.equals(blob, that.blob) && Objects.equals(proof, that.proof); + } + + @Override + public int hashCode() { + return Objects.hash(blob, proof); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("blob", bytesToBriefString(blob)) + .add("proof", bytesToBriefString(proof)) + .toString(); + } + + private String bytesToBriefString(final Bytes bytes) { + return bytes.slice(0, 7).toUnprefixedHexString(); + } +} diff --git a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/web3j/DefaultExecutionWeb3jClientProvider.java b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/web3j/DefaultExecutionWeb3jClientProvider.java index b414b3756a9..dd3465735b6 100644 --- a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/web3j/DefaultExecutionWeb3jClientProvider.java +++ b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/web3j/DefaultExecutionWeb3jClientProvider.java @@ -58,7 +58,8 @@ private synchronized void buildClient() { .jwtConfigOpt(jwtConfig) .timeProvider(timeProvider) .executionClientEventsPublisher(executionClientEventsPublisher) - .nonCriticalMethods("engine_exchangeCapabilities", "engine_getClientVersionV1") + .nonCriticalMethods( + "engine_exchangeCapabilities", "engine_getClientVersionV1", "engine_getBlobsV1") .build(); this.alreadyBuilt = true; } diff --git a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClient.java b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClient.java index 0dd296f366c..137d99b3211 100644 --- a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClient.java +++ b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClient.java @@ -27,6 +27,7 @@ import org.web3j.protocol.core.Request; import org.web3j.protocol.core.methods.response.EthBlock; import tech.pegasys.teku.ethereum.executionclient.ExecutionEngineClient; +import tech.pegasys.teku.ethereum.executionclient.schema.BlobAndProofV1; import tech.pegasys.teku.ethereum.executionclient.schema.ClientVersionV1; import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1; import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV2; @@ -51,6 +52,7 @@ public class Web3JExecutionEngineClient implements ExecutionEngineClient { private static final Duration EXCHANGE_CAPABILITIES_TIMEOUT = Duration.ofSeconds(1); private static final Duration GET_CLIENT_VERSION_TIMEOUT = Duration.ofSeconds(1); + private static final Duration GET_BLOBS_TIMEOUT = Duration.ofSeconds(1); private final Web3JClient web3JClient; @@ -256,6 +258,20 @@ public SafeFuture>> getClientVersionV1( return web3JClient.doRequest(web3jRequest, GET_CLIENT_VERSION_TIMEOUT); } + @Override + public SafeFuture>> getBlobsV1( + final List blobVersionedHashes) { + final List expectedBlobVersionedHashes = + blobVersionedHashes.stream().map(VersionedHash::toHexString).toList(); + final Request web3jRequest = + new Request<>( + "engine_getBlobsV1", + list(expectedBlobVersionedHashes), + web3JClient.getWeb3jService(), + GetBlobsVersionV1Web3jResponse.class); + return web3JClient.doRequest(web3jRequest, GET_BLOBS_TIMEOUT); + } + static class ExecutionPayloadV1Web3jResponse extends org.web3j.protocol.core.Response {} @@ -280,6 +296,9 @@ static class ExchangeCapabilitiesWeb3jResponse static class GetClientVersionV1Web3jResponse extends org.web3j.protocol.core.Response> {} + static class GetBlobsVersionV1Web3jResponse + extends org.web3j.protocol.core.Response> {} + /** * Returns a list that supports null items. * diff --git a/ethereum/executionclient/src/test/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1Test.java b/ethereum/executionclient/src/test/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1Test.java new file mode 100644 index 00000000000..85243691452 --- /dev/null +++ b/ethereum/executionclient/src/test/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1Test.java @@ -0,0 +1,149 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.ethereum.executionclient.methods; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import tech.pegasys.teku.ethereum.executionclient.ExecutionEngineClient; +import tech.pegasys.teku.ethereum.executionclient.response.InvalidRemoteResponseException; +import tech.pegasys.teku.ethereum.executionclient.schema.BlobAndProofV1; +import tech.pegasys.teku.ethereum.executionclient.schema.Response; +import tech.pegasys.teku.infrastructure.async.SafeFuture; +import tech.pegasys.teku.infrastructure.unsigned.UInt64; +import tech.pegasys.teku.spec.Spec; +import tech.pegasys.teku.spec.TestSpecFactory; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; +import tech.pegasys.teku.spec.datastructures.execution.BlobAndProof; +import tech.pegasys.teku.spec.logic.versions.deneb.types.VersionedHash; +import tech.pegasys.teku.spec.util.DataStructureUtil; + +public class EngineGetBlobsV1Test { + + private final Spec spec = TestSpecFactory.createMinimalElectra(); + private final DataStructureUtil dataStructureUtil = new DataStructureUtil(spec); + private final ExecutionEngineClient executionEngineClient = mock(ExecutionEngineClient.class); + private EngineGetBlobsV1 jsonRpcMethod; + + @BeforeEach + public void setUp() { + jsonRpcMethod = new EngineGetBlobsV1(executionEngineClient, spec); + } + + @Test + public void shouldReturnExpectedNameAndVersion() { + assertThat(jsonRpcMethod.getName()).isEqualTo("engine_getBlobs"); + assertThat(jsonRpcMethod.isOptional()).isTrue(); + assertThat(jsonRpcMethod.getVersion()).isEqualTo(1); + assertThat(jsonRpcMethod.getVersionedName()).isEqualTo("engine_getBlobsV1"); + } + + @Test + public void blobVersionedHashesParamIsRequired() { + final JsonRpcRequestParams params = new JsonRpcRequestParams.Builder().build(); + + assertThatThrownBy(() -> jsonRpcMethod.execute(params)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Missing required parameter at index 0"); + + verifyNoInteractions(executionEngineClient); + } + + @Test + public void slotParamIsRequired() { + final List versionedHashes = dataStructureUtil.randomVersionedHashes(4); + + final JsonRpcRequestParams params = + new JsonRpcRequestParams.Builder().add(versionedHashes).build(); + + assertThatThrownBy(() -> jsonRpcMethod.execute(params)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Missing required parameter at index 1"); + + verifyNoInteractions(executionEngineClient); + } + + @Test + public void shouldReturnFailedExecutionWhenEngineClientRequestFails() { + final List versionedHashes = dataStructureUtil.randomVersionedHashes(4); + final String errorResponseFromClient = "error!"; + + when(executionEngineClient.getBlobsV1(any())) + .thenReturn(dummyFailedResponse(errorResponseFromClient)); + + final JsonRpcRequestParams params = + new JsonRpcRequestParams.Builder().add(versionedHashes).add(UInt64.ZERO).build(); + + assertThat(jsonRpcMethod.execute(params)) + .failsWithin(1, TimeUnit.SECONDS) + .withThrowableOfType(ExecutionException.class) + .withRootCauseInstanceOf(InvalidRemoteResponseException.class) + .withMessageContaining( + "Invalid remote response from the execution client: %s", errorResponseFromClient); + } + + @Test + public void shouldCallGetBlobsV1AndParseResponseSuccessfully() { + final List versionedHashes = dataStructureUtil.randomVersionedHashes(4); + final List blobSidecars = + dataStructureUtil.randomBlobSidecars(spec.getMaxBlobsPerBlock().orElseThrow()); + + when(executionEngineClient.getBlobsV1(eq(versionedHashes))) + .thenReturn(dummySuccessfulResponse(blobSidecars)); + + final JsonRpcRequestParams params = + new JsonRpcRequestParams.Builder().add(versionedHashes).add(UInt64.ZERO).build(); + + jsonRpcMethod = new EngineGetBlobsV1(executionEngineClient, spec); + + final List expectedResponse = + blobSidecars.stream() + .map(blobSidecar -> new BlobAndProof(blobSidecar.getBlob(), blobSidecar.getKZGProof())) + .toList(); + assertThat(jsonRpcMethod.execute(params)).isCompletedWithValue(expectedResponse); + + verify(executionEngineClient).getBlobsV1(eq(versionedHashes)); + verifyNoMoreInteractions(executionEngineClient); + } + + private SafeFuture>> dummySuccessfulResponse( + final List blobSidecars) { + return SafeFuture.completedFuture( + new Response<>( + blobSidecars.stream() + .map( + blobSidecar -> + new BlobAndProofV1( + blobSidecar.getBlob().getBytes(), + blobSidecar.getKZGProof().getBytesCompressed())) + .toList())); + } + + private SafeFuture>> dummyFailedResponse( + final String errorMessage) { + return SafeFuture.completedFuture(Response.withErrorMessage(errorMessage)); + } +} diff --git a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/EngineCapabilitiesMonitor.java b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/EngineCapabilitiesMonitor.java index 702835fe05a..8ecc3afbcce 100644 --- a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/EngineCapabilitiesMonitor.java +++ b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/EngineCapabilitiesMonitor.java @@ -18,6 +18,7 @@ import java.util.List; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; +import java.util.stream.Stream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import tech.pegasys.teku.ethereum.events.SlotEventsChannel; @@ -40,6 +41,7 @@ public class EngineCapabilitiesMonitor implements SlotEventsChannel { private final Spec spec; private final EventLogger eventLogger; private final Supplier> capabilitiesSupplier; + private final Supplier> optionalCapabilitiesSupplier; private final ExecutionEngineClient executionEngineClient; public EngineCapabilitiesMonitor( @@ -51,6 +53,8 @@ public EngineCapabilitiesMonitor( this.eventLogger = eventLogger; this.capabilitiesSupplier = Suppliers.memoize(() -> new ArrayList<>(engineMethodsResolver.getCapabilities())); + this.optionalCapabilitiesSupplier = + Suppliers.memoize(() -> new ArrayList<>(engineMethodsResolver.getOptionalCapabilities())); this.executionEngineClient = executionEngineClient; } @@ -79,18 +83,35 @@ private boolean slotIsApplicable(final UInt64 slot) { private SafeFuture monitor() { final List capabilities = capabilitiesSupplier.get(); + final List optionalCapabilities = optionalCapabilitiesSupplier.get(); return executionEngineClient - .exchangeCapabilities(capabilities) + .exchangeCapabilities( + Stream.concat(capabilities.stream(), optionalCapabilities.stream()).toList()) .thenApply(ResponseUnwrapper::unwrapExecutionClientResponseOrThrow) .thenAccept( engineCapabilities -> { LOG.debug("Engine API capabilities response: " + engineCapabilities); + final List missingEngineCapabilities = capabilities.stream() - .filter(capability -> !engineCapabilities.contains(capability)) + .filter( + capability -> + !engineCapabilities.contains(capability) + && !optionalCapabilities.contains(capability)) + .toList(); + + final List missingOptionalCapabilities = + optionalCapabilities.stream() + .filter( + optionalCapability -> !engineCapabilities.contains(optionalCapability)) .toList(); + if (!missingEngineCapabilities.isEmpty()) { - eventLogger.missingEngineApiCapabilities(missingEngineCapabilities); + eventLogger.missingEngineApiCapabilities(missingEngineCapabilities, false); + } + + if (!missingOptionalCapabilities.isEmpty()) { + eventLogger.missingEngineApiCapabilities(missingOptionalCapabilities, true); } }); } diff --git a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/EngineJsonRpcMethodsResolver.java b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/EngineJsonRpcMethodsResolver.java index bcf580c3659..ce182addfea 100644 --- a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/EngineJsonRpcMethodsResolver.java +++ b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/EngineJsonRpcMethodsResolver.java @@ -13,6 +13,7 @@ package tech.pegasys.teku.ethereum.executionlayer; +import java.util.List; import java.util.Set; import java.util.function.Supplier; import tech.pegasys.teku.ethereum.executionclient.methods.EngineApiMethod; @@ -24,10 +25,19 @@ public interface EngineJsonRpcMethodsResolver { EngineJsonRpcMethod getMethod( EngineApiMethod method, Supplier milestoneSupplier, Class resultType); + EngineJsonRpcMethod> getListMethod( + EngineApiMethod method, Supplier milestoneSupplier, Class resultType); + /** * Get CL capabilities required for the engine_exchangeCapabilities * request */ Set getCapabilities(); + + /** + * TODO this optionality notion should be removed once all ELs implement the engine_getBlobsV1 RPC + * method. It has been added to ensure a softer and better logging when the method is missing only + */ + Set getOptionalCapabilities(); } diff --git a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionClientHandler.java b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionClientHandler.java index 41bb13f2b66..4799f24bd10 100644 --- a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionClientHandler.java +++ b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionClientHandler.java @@ -18,6 +18,7 @@ import org.apache.tuweni.bytes.Bytes32; import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.unsigned.UInt64; +import tech.pegasys.teku.spec.datastructures.execution.BlobAndProof; import tech.pegasys.teku.spec.datastructures.execution.ClientVersion; import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayloadContext; import tech.pegasys.teku.spec.datastructures.execution.GetPayloadResponse; @@ -27,6 +28,7 @@ import tech.pegasys.teku.spec.executionlayer.ForkChoiceUpdatedResult; import tech.pegasys.teku.spec.executionlayer.PayloadBuildingAttributes; import tech.pegasys.teku.spec.executionlayer.PayloadStatus; +import tech.pegasys.teku.spec.logic.versions.deneb.types.VersionedHash; public interface ExecutionClientHandler { @@ -44,4 +46,7 @@ SafeFuture engineGetPayload( SafeFuture engineNewPayload(NewPayloadRequest newPayloadRequest, UInt64 slot); SafeFuture> engineGetClientVersion(ClientVersion clientVersion); + + SafeFuture> engineGetBlobs( + List blobVersionedHashes, UInt64 slot); } diff --git a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionClientHandlerImpl.java b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionClientHandlerImpl.java index a7e082cafd1..06cd24fd07e 100644 --- a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionClientHandlerImpl.java +++ b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionClientHandlerImpl.java @@ -24,6 +24,7 @@ import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.Spec; +import tech.pegasys.teku.spec.datastructures.execution.BlobAndProof; import tech.pegasys.teku.spec.datastructures.execution.ClientVersion; import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayload; import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayloadContext; @@ -34,6 +35,7 @@ import tech.pegasys.teku.spec.executionlayer.ForkChoiceUpdatedResult; import tech.pegasys.teku.spec.executionlayer.PayloadBuildingAttributes; import tech.pegasys.teku.spec.executionlayer.PayloadStatus; +import tech.pegasys.teku.spec.logic.versions.deneb.types.VersionedHash; public class ExecutionClientHandlerImpl implements ExecutionClientHandler { @@ -127,4 +129,17 @@ public SafeFuture> engineGetClientVersion(final ClientVersio clientVersions -> clientVersions.stream().map(ClientVersionV1::asInternalClientVersion).toList()); } + + @Override + public SafeFuture> engineGetBlobs( + final List blobVersionedHashes, final UInt64 slot) { + final JsonRpcRequestParams params = + new JsonRpcRequestParams.Builder().add(blobVersionedHashes).add(slot).build(); + return engineMethodsResolver + .getListMethod( + EngineApiMethod.ENGINE_GET_BLOBS, + () -> spec.atSlot(slot).getMilestone(), + BlobAndProof.class) + .execute(params); + } } diff --git a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerManagerImpl.java b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerManagerImpl.java index 1c44c9c5e48..1ef114e771f 100644 --- a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerManagerImpl.java +++ b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerManagerImpl.java @@ -47,6 +47,7 @@ import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.builder.SignedValidatorRegistration; +import tech.pegasys.teku.spec.datastructures.execution.BlobAndProof; import tech.pegasys.teku.spec.datastructures.execution.BuilderBidOrFallbackData; import tech.pegasys.teku.spec.datastructures.execution.BuilderPayloadOrFallbackData; import tech.pegasys.teku.spec.datastructures.execution.ClientVersion; @@ -61,6 +62,7 @@ import tech.pegasys.teku.spec.executionlayer.ForkChoiceUpdatedResult; import tech.pegasys.teku.spec.executionlayer.PayloadBuildingAttributes; import tech.pegasys.teku.spec.executionlayer.PayloadStatus; +import tech.pegasys.teku.spec.logic.versions.deneb.types.VersionedHash; public class ExecutionLayerManagerImpl implements ExecutionLayerManager { @@ -219,6 +221,15 @@ public SafeFuture> engineGetClientVersion(final ClientVersio return executionClientHandler.engineGetClientVersion(clientVersion); } + @Override + public SafeFuture>> engineGetBlobs( + final List blobVersionedHashes, final UInt64 slot) { + LOG.trace("calling engineGetBlobs(blobVersionedHashes={}, slot={})", blobVersionedHashes, slot); + return executionClientHandler + .engineGetBlobs(blobVersionedHashes, slot) + .thenApply(blobsAndProofs -> blobsAndProofs.stream().map(Optional::ofNullable).toList()); + } + @Override public SafeFuture builderRegisterValidators( final SszList signedValidatorRegistrations, final UInt64 slot) { diff --git a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolver.java b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolver.java index 992aa0e5095..0651980f143 100644 --- a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolver.java +++ b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolver.java @@ -14,12 +14,14 @@ package tech.pegasys.teku.ethereum.executionlayer; import static tech.pegasys.teku.ethereum.executionclient.methods.EngineApiMethod.ENGINE_FORK_CHOICE_UPDATED; +import static tech.pegasys.teku.ethereum.executionclient.methods.EngineApiMethod.ENGINE_GET_BLOBS; import static tech.pegasys.teku.ethereum.executionclient.methods.EngineApiMethod.ENGINE_GET_PAYLOAD; import static tech.pegasys.teku.ethereum.executionclient.methods.EngineApiMethod.ENGINE_NEW_PAYLOAD; import java.util.Collections; import java.util.EnumMap; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Supplier; @@ -29,6 +31,7 @@ import tech.pegasys.teku.ethereum.executionclient.methods.EngineForkChoiceUpdatedV1; import tech.pegasys.teku.ethereum.executionclient.methods.EngineForkChoiceUpdatedV2; import tech.pegasys.teku.ethereum.executionclient.methods.EngineForkChoiceUpdatedV3; +import tech.pegasys.teku.ethereum.executionclient.methods.EngineGetBlobsV1; import tech.pegasys.teku.ethereum.executionclient.methods.EngineGetPayloadV1; import tech.pegasys.teku.ethereum.executionclient.methods.EngineGetPayloadV2; import tech.pegasys.teku.ethereum.executionclient.methods.EngineGetPayloadV3; @@ -106,6 +109,7 @@ private Map> denebSupportedMethods() { methods.put(ENGINE_NEW_PAYLOAD, new EngineNewPayloadV3(executionEngineClient)); methods.put(ENGINE_GET_PAYLOAD, new EngineGetPayloadV3(executionEngineClient, spec)); methods.put(ENGINE_FORK_CHOICE_UPDATED, new EngineForkChoiceUpdatedV3(executionEngineClient)); + methods.put(ENGINE_GET_BLOBS, new EngineGetBlobsV1(executionEngineClient, spec)); return methods; } @@ -116,6 +120,7 @@ private Map> electraSupportedMethods() { methods.put(ENGINE_NEW_PAYLOAD, new EngineNewPayloadV4(executionEngineClient)); methods.put(ENGINE_GET_PAYLOAD, new EngineGetPayloadV4(executionEngineClient, spec)); methods.put(ENGINE_FORK_CHOICE_UPDATED, new EngineForkChoiceUpdatedV3(executionEngineClient)); + methods.put(ENGINE_GET_BLOBS, new EngineGetBlobsV1(executionEngineClient, spec)); return methods; } @@ -138,10 +143,39 @@ public EngineJsonRpcMethod getMethod( return foundMethod; } + @Override + @SuppressWarnings({"unchecked", "unused"}) + public EngineJsonRpcMethod> getListMethod( + final EngineApiMethod method, + final Supplier milestoneSupplier, + final Class resultType) { + final SpecMilestone milestone = milestoneSupplier.get(); + final Map> milestoneMethods = + methodsByMilestone.getOrDefault(milestone, Collections.emptyMap()); + final EngineJsonRpcMethod> foundMethod = + (EngineJsonRpcMethod>) milestoneMethods.get(method); + if (foundMethod == null) { + throw new IllegalArgumentException( + "Can't find method with name " + method.getName() + " for milestone " + milestone); + } + return foundMethod; + } + @Override public Set getCapabilities() { return methodsByMilestone.values().stream() .flatMap(methods -> methods.values().stream()) + .filter(method -> !method.isOptional()) + .filter(method -> !method.isDeprecated()) + .map(EngineJsonRpcMethod::getVersionedName) + .collect(Collectors.toSet()); + } + + @Override + public Set getOptionalCapabilities() { + return methodsByMilestone.values().stream() + .flatMap(methods -> methods.values().stream()) + .filter(EngineJsonRpcMethod::isOptional) .filter(method -> !method.isDeprecated()) .map(EngineJsonRpcMethod::getVersionedName) .collect(Collectors.toSet()); diff --git a/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/DenebExecutionClientHandlerTest.java b/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/DenebExecutionClientHandlerTest.java index 415875b9746..af480670b49 100644 --- a/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/DenebExecutionClientHandlerTest.java +++ b/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/DenebExecutionClientHandlerTest.java @@ -24,6 +24,7 @@ import org.apache.tuweni.units.bigints.UInt256; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import tech.pegasys.teku.ethereum.executionclient.schema.BlobAndProofV1; import tech.pegasys.teku.ethereum.executionclient.schema.BlobsBundleV1; import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV3; import tech.pegasys.teku.ethereum.executionclient.schema.ForkChoiceStateV1; @@ -35,6 +36,8 @@ import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.TestSpecFactory; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; +import tech.pegasys.teku.spec.datastructures.execution.BlobAndProof; import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayload; import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayloadContext; import tech.pegasys.teku.spec.datastructures.execution.GetPayloadResponse; @@ -135,6 +138,30 @@ void engineForkChoiceUpdated_shouldCallEngineForkChoiceUpdatedV3() { assertThat(future).isCompleted(); } + @Test + void engineGetBlobs_shouldCallGetBlobsV1() { + final ExecutionClientHandler handler = getHandler(); + final int maxBlobsPerBlock = spec.getMaxBlobsPerBlock().orElseThrow(); + final List versionedHashes = + dataStructureUtil.randomVersionedHashes(maxBlobsPerBlock); + final List blobSidecars = dataStructureUtil.randomBlobSidecars(maxBlobsPerBlock); + final UInt64 slot = dataStructureUtil.randomUInt64(1_000_000); + final SafeFuture>> dummyResponse = + SafeFuture.completedFuture( + new Response<>( + blobSidecars.stream() + .map( + blobSidecar -> + new BlobAndProofV1( + blobSidecar.getBlob().getBytes(), + blobSidecar.getKZGProof().getBytesCompressed())) + .toList())); + when(executionEngineClient.getBlobsV1(versionedHashes)).thenReturn(dummyResponse); + final SafeFuture> future = handler.engineGetBlobs(versionedHashes, slot); + verify(executionEngineClient).getBlobsV1(versionedHashes); + assertThat(future).isCompleted(); + } + private ExecutionPayloadContext randomContext() { return new ExecutionPayloadContext( dataStructureUtil.randomBytes8(), diff --git a/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/ElectraExecutionClientHandlerTest.java b/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/ElectraExecutionClientHandlerTest.java index f0502db539b..c4d85f558b0 100644 --- a/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/ElectraExecutionClientHandlerTest.java +++ b/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/ElectraExecutionClientHandlerTest.java @@ -25,6 +25,7 @@ import org.apache.tuweni.units.bigints.UInt256; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import tech.pegasys.teku.ethereum.executionclient.schema.BlobAndProofV1; import tech.pegasys.teku.ethereum.executionclient.schema.BlobsBundleV1; import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV3; import tech.pegasys.teku.ethereum.executionclient.schema.ForkChoiceStateV1; @@ -36,6 +37,8 @@ import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.TestSpecFactory; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; +import tech.pegasys.teku.spec.datastructures.execution.BlobAndProof; import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayload; import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayloadContext; import tech.pegasys.teku.spec.datastructures.execution.GetPayloadResponse; @@ -147,6 +150,30 @@ void engineForkChoiceUpdated_shouldCallEngineForkChoiceUpdatedV3() { assertThat(future).isCompleted(); } + @Test + void engineGetBlobs_shouldCallGetBlobsV1() { + final ExecutionClientHandler handler = getHandler(); + final int maxBlobsPerBlock = spec.getMaxBlobsPerBlock().orElseThrow(); + final List versionedHashes = + dataStructureUtil.randomVersionedHashes(maxBlobsPerBlock); + final List blobSidecars = dataStructureUtil.randomBlobSidecars(maxBlobsPerBlock); + final UInt64 slot = dataStructureUtil.randomUInt64(1_000_000); + final SafeFuture>> dummyResponse = + SafeFuture.completedFuture( + new Response<>( + blobSidecars.stream() + .map( + blobSidecar -> + new BlobAndProofV1( + blobSidecar.getBlob().getBytes(), + blobSidecar.getKZGProof().getBytesCompressed())) + .toList())); + when(executionEngineClient.getBlobsV1(versionedHashes)).thenReturn(dummyResponse); + final SafeFuture> future = handler.engineGetBlobs(versionedHashes, slot); + verify(executionEngineClient).getBlobsV1(versionedHashes); + assertThat(future).isCompleted(); + } + private ExecutionPayloadContext randomContext() { return new ExecutionPayloadContext( dataStructureUtil.randomBytes8(), diff --git a/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/EngineCapabilitiesMonitorTest.java b/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/EngineCapabilitiesMonitorTest.java index ff4d6c5343c..9849430c6f1 100644 --- a/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/EngineCapabilitiesMonitorTest.java +++ b/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/EngineCapabilitiesMonitorTest.java @@ -25,6 +25,7 @@ import java.util.HashSet; import java.util.List; +import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import tech.pegasys.teku.ethereum.executionclient.ExecutionEngineClient; @@ -43,14 +44,18 @@ public class EngineCapabilitiesMonitorTest { mock(EngineJsonRpcMethodsResolver.class); private final ExecutionEngineClient executionEngineClient = mock(ExecutionEngineClient.class); + private final List engineCapabilities = List.of("method1", "method2", "method3"); private final List capabilities = List.of("method1", "method2"); + private final List optionalCapabilities = List.of("method3"); private EngineCapabilitiesMonitor engineCapabilitiesMonitor; @BeforeEach public void setUp() { when(engineMethodsResolver.getCapabilities()).thenReturn(new HashSet<>(capabilities)); - mockEngineCapabilitiesResponse(capabilities); + when(engineMethodsResolver.getOptionalCapabilities()) + .thenReturn(new HashSet<>(optionalCapabilities)); + mockEngineCapabilitiesResponse(engineCapabilities); engineCapabilitiesMonitor = new EngineCapabilitiesMonitor( spec, eventLogger, engineMethodsResolver, executionEngineClient); @@ -64,7 +69,18 @@ public void logsWarningIfEngineDoesNotSupportCapabilities() { // 3rd slot in epoch engineCapabilitiesMonitor.onSlot(UInt64.valueOf(2)); - verify(eventLogger).missingEngineApiCapabilities(List.of("method2")); + verify(eventLogger).missingEngineApiCapabilities(List.of("method2"), false); + } + + @Test + public void logsWarningIfEngineDoesNotSupportOptionalCapabilities() { + // engine only supports one of the methods + mockEngineCapabilitiesResponse(List.of("method1", "method2")); + + // 3rd slot in epoch + engineCapabilitiesMonitor.onSlot(UInt64.valueOf(2)); + + verify(eventLogger).missingEngineApiCapabilities(List.of("method3"), true); } @Test @@ -129,7 +145,8 @@ public void doesNotRunMonitoringIfNotAtRequiredSlot() { } private void mockEngineCapabilitiesResponse(final List engineCapabilities) { - when(executionEngineClient.exchangeCapabilities(capabilities)) + when(executionEngineClient.exchangeCapabilities( + Stream.concat(capabilities.stream(), optionalCapabilities.stream()).toList())) .thenReturn(SafeFuture.completedFuture(new Response<>(engineCapabilities))); } diff --git a/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerManagerImplTest.java b/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerManagerImplTest.java index 6104c88d512..e6fd00930a9 100644 --- a/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerManagerImplTest.java +++ b/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerManagerImplTest.java @@ -44,9 +44,11 @@ import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.TestSpecFactory; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.builder.BuilderBid; import tech.pegasys.teku.spec.datastructures.builder.SignedBuilderBid; +import tech.pegasys.teku.spec.datastructures.execution.BlobAndProof; import tech.pegasys.teku.spec.datastructures.execution.BlobsBundle; import tech.pegasys.teku.spec.datastructures.execution.BuilderBidOrFallbackData; import tech.pegasys.teku.spec.datastructures.execution.BuilderPayloadOrFallbackData; @@ -58,6 +60,7 @@ import tech.pegasys.teku.spec.datastructures.execution.FallbackReason; import tech.pegasys.teku.spec.datastructures.execution.GetPayloadResponse; import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState; +import tech.pegasys.teku.spec.logic.versions.deneb.types.VersionedHash; import tech.pegasys.teku.spec.util.DataStructureUtil; class ExecutionLayerManagerImplTest { @@ -656,6 +659,18 @@ void onSlot_shouldCleanUpFallbackCache() { verifyNoMoreInteractions(executionClientHandler); } + @Test + public void engineGetBlobs_shouldReturnGetBlobsResponseViaEngine() { + setupDeneb(); + final List versionedHashes = + dataStructureUtil.randomVersionedHashes(spec.getMaxBlobsPerBlock().orElseThrow()); + final UInt64 slot = dataStructureUtil.randomSlot(); + final List getBlobsResponse = + prepareEngineGetBlobsResponse(versionedHashes, slot); + assertThat(executionLayerManager.engineGetBlobs(versionedHashes, slot)) + .isCompletedWithValue(getBlobsResponse.stream().map(Optional::ofNullable).toList()); + } + private void setupDeneb() { spec = TestSpecFactory.createMinimalDeneb(); dataStructureUtil = new DataStructureUtil(spec); @@ -786,6 +801,19 @@ private GetPayloadResponse prepareEngineGetPayloadResponseWithBlobs( return getPayloadResponse; } + private List prepareEngineGetBlobsResponse( + final List blobVersionedHashes, final UInt64 slot) { + final List blobSidecars = + dataStructureUtil.randomBlobSidecars(spec.getMaxBlobsPerBlock().orElseThrow()); + final List getBlobsResponse = + blobSidecars.stream() + .map(blobSidecar -> new BlobAndProof(blobSidecar.getBlob(), blobSidecar.getKZGProof())) + .toList(); + when(executionClientHandler.engineGetBlobs(blobVersionedHashes, slot)) + .thenReturn(SafeFuture.completedFuture(getBlobsResponse)); + return getBlobsResponse; + } + private ExecutionLayerManagerImpl createExecutionLayerChannelImpl( final boolean builderEnabled, final boolean builderValidatorEnabled) { return createExecutionLayerChannelImpl( diff --git a/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolverTest.java b/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolverTest.java index a98ad90fbb8..79d6577e40a 100644 --- a/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolverTest.java +++ b/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolverTest.java @@ -18,6 +18,7 @@ import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.mockito.Mockito.mock; import static tech.pegasys.teku.ethereum.executionclient.methods.EngineApiMethod.ENGINE_FORK_CHOICE_UPDATED; +import static tech.pegasys.teku.ethereum.executionclient.methods.EngineApiMethod.ENGINE_GET_BLOBS; import static tech.pegasys.teku.ethereum.executionclient.methods.EngineApiMethod.ENGINE_GET_PAYLOAD; import static tech.pegasys.teku.ethereum.executionclient.methods.EngineApiMethod.ENGINE_NEW_PAYLOAD; @@ -33,6 +34,7 @@ import tech.pegasys.teku.ethereum.executionclient.methods.EngineForkChoiceUpdatedV1; import tech.pegasys.teku.ethereum.executionclient.methods.EngineForkChoiceUpdatedV2; import tech.pegasys.teku.ethereum.executionclient.methods.EngineForkChoiceUpdatedV3; +import tech.pegasys.teku.ethereum.executionclient.methods.EngineGetBlobsV1; import tech.pegasys.teku.ethereum.executionclient.methods.EngineGetPayloadV1; import tech.pegasys.teku.ethereum.executionclient.methods.EngineGetPayloadV2; import tech.pegasys.teku.ethereum.executionclient.methods.EngineGetPayloadV3; @@ -161,7 +163,8 @@ private static Stream denebMethods() { return Stream.of( arguments(ENGINE_NEW_PAYLOAD, EngineNewPayloadV3.class), arguments(ENGINE_GET_PAYLOAD, EngineGetPayloadV3.class), - arguments(ENGINE_FORK_CHOICE_UPDATED, EngineForkChoiceUpdatedV3.class)); + arguments(ENGINE_FORK_CHOICE_UPDATED, EngineForkChoiceUpdatedV3.class), + arguments(ENGINE_GET_BLOBS, EngineGetBlobsV1.class)); } @Test @@ -197,7 +200,8 @@ private static Stream electraMethods() { return Stream.of( arguments(ENGINE_NEW_PAYLOAD, EngineNewPayloadV4.class), arguments(ENGINE_GET_PAYLOAD, EngineGetPayloadV4.class), - arguments(ENGINE_FORK_CHOICE_UPDATED, EngineForkChoiceUpdatedV3.class)); + arguments(ENGINE_FORK_CHOICE_UPDATED, EngineForkChoiceUpdatedV3.class), + arguments(ENGINE_GET_BLOBS, EngineGetBlobsV1.class)); } @Test @@ -225,4 +229,18 @@ void getsCapabilities() { "engine_newPayloadV4", "engine_getPayloadV4"); } + + @Test + void getsOptionalCapabilities() { + final Spec spec = + TestSpecFactory.createMinimalWithCapellaDenebAndElectraForkEpoch( + UInt64.ONE, UInt64.valueOf(2), UInt64.valueOf(3)); + + final MilestoneBasedEngineJsonRpcMethodsResolver engineMethodsResolver = + new MilestoneBasedEngineJsonRpcMethodsResolver(spec, executionEngineClient); + + final Set capabilities = engineMethodsResolver.getOptionalCapabilities(); + + assertThat(capabilities).containsExactlyInAnyOrder("engine_getBlobsV1"); + } } diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/execution/BlobAndProof.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/execution/BlobAndProof.java new file mode 100644 index 00000000000..8509e4be196 --- /dev/null +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/execution/BlobAndProof.java @@ -0,0 +1,19 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.spec.datastructures.execution; + +import tech.pegasys.teku.kzg.KZGProof; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.Blob; + +public record BlobAndProof(Blob blob, KZGProof proof) {} diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/executionlayer/ExecutionLayerChannel.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/executionlayer/ExecutionLayerChannel.java index 5daaceb026e..f4cfac0dba1 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/executionlayer/ExecutionLayerChannel.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/executionlayer/ExecutionLayerChannel.java @@ -25,6 +25,7 @@ import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.builder.SignedValidatorRegistration; +import tech.pegasys.teku.spec.datastructures.execution.BlobAndProof; import tech.pegasys.teku.spec.datastructures.execution.BuilderBidOrFallbackData; import tech.pegasys.teku.spec.datastructures.execution.BuilderPayloadOrFallbackData; import tech.pegasys.teku.spec.datastructures.execution.ClientVersion; @@ -34,6 +35,7 @@ import tech.pegasys.teku.spec.datastructures.execution.NewPayloadRequest; import tech.pegasys.teku.spec.datastructures.execution.PowBlock; import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState; +import tech.pegasys.teku.spec.logic.versions.deneb.types.VersionedHash; public interface ExecutionLayerChannel extends ChannelInterface { String PREVIOUS_STUB_ENDPOINT_PREFIX = "stub"; @@ -76,6 +78,13 @@ public SafeFuture> engineGetClientVersion( return SafeFuture.completedFuture(List.of()); } + @Override + public SafeFuture>> engineGetBlobs( + final List blobVersionedHashes, final UInt64 slot) { + return SafeFuture.completedFuture( + blobVersionedHashes.stream().map(e -> Optional.empty()).toList()); + } + @Override public SafeFuture builderRegisterValidators( final SszList signedValidatorRegistrations, @@ -116,6 +125,9 @@ SafeFuture engineNewPayload( SafeFuture> engineGetClientVersion(ClientVersion clientVersion); + SafeFuture>> engineGetBlobs( + List blobVersionedHashes, UInt64 slot); + /** * This is low level method, use {@link * ExecutionLayerBlockProductionManager#initiateBlockProduction(ExecutionPayloadContext, diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/executionlayer/ExecutionLayerChannelStub.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/executionlayer/ExecutionLayerChannelStub.java index 203e1a63222..dabacf4d8c5 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/executionlayer/ExecutionLayerChannelStub.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/executionlayer/ExecutionLayerChannelStub.java @@ -53,6 +53,7 @@ import tech.pegasys.teku.spec.datastructures.builder.BuilderBid; import tech.pegasys.teku.spec.datastructures.builder.BuilderPayload; import tech.pegasys.teku.spec.datastructures.builder.SignedValidatorRegistration; +import tech.pegasys.teku.spec.datastructures.execution.BlobAndProof; import tech.pegasys.teku.spec.datastructures.execution.BlobsBundle; import tech.pegasys.teku.spec.datastructures.execution.BuilderBidOrFallbackData; import tech.pegasys.teku.spec.datastructures.execution.BuilderPayloadOrFallbackData; @@ -69,6 +70,7 @@ import tech.pegasys.teku.spec.datastructures.type.SszKZGCommitment; import tech.pegasys.teku.spec.datastructures.util.BlobsUtil; import tech.pegasys.teku.spec.datastructures.util.DepositRequestsUtil; +import tech.pegasys.teku.spec.logic.versions.deneb.types.VersionedHash; import tech.pegasys.teku.spec.schemas.SchemaDefinitions; import tech.pegasys.teku.spec.schemas.SchemaDefinitionsBellatrix; import tech.pegasys.teku.spec.schemas.SchemaDefinitionsDeneb; @@ -349,6 +351,13 @@ public SafeFuture> engineGetClientVersion(final ClientVersio return SafeFuture.completedFuture(List.of(STUB_CLIENT_VERSION)); } + @Override + public SafeFuture>> engineGetBlobs( + final List blobVersionedHashes, final UInt64 slot) { + return SafeFuture.completedFuture( + blobVersionedHashes.stream().map(e -> Optional.empty()).toList()); + } + @Override public SafeFuture builderRegisterValidators( final SszList signedValidatorRegistrations, final UInt64 slot) { diff --git a/infrastructure/logging/src/main/java/tech/pegasys/teku/infrastructure/logging/EventLogger.java b/infrastructure/logging/src/main/java/tech/pegasys/teku/infrastructure/logging/EventLogger.java index 4893ec3e908..651eff10f2a 100644 --- a/infrastructure/logging/src/main/java/tech/pegasys/teku/infrastructure/logging/EventLogger.java +++ b/infrastructure/logging/src/main/java/tech/pegasys/teku/infrastructure/logging/EventLogger.java @@ -162,11 +162,13 @@ public void executionClientRecovered() { info("Execution Client is responding to requests again after a previous failure", Color.GREEN); } - public void missingEngineApiCapabilities(final List missingCapabilities) { + // TODO remove the isOptional param when all ELs implement the engine_getBlob + public void missingEngineApiCapabilities( + final List missingCapabilities, final boolean isOptional) { warn( String.format( - "Execution Client does not support required Engine API methods: %s. Make sure it is upgraded to a compatible version.", - missingCapabilities), + "Execution Client does not support %s Engine API methods: %s. Make sure it is upgraded to a compatible version.", + isOptional ? "optional" : "required", missingCapabilities), Color.YELLOW); }