From 59da0927339903508fc1a741bd44e68917249279 Mon Sep 17 00:00:00 2001 From: Matt Whitehead Date: Wed, 21 Feb 2024 15:25:05 +0000 Subject: [PATCH] Restrict downgrade (#6307) * Add Besu version to DB metadata. Check for downgrades and reject if version < version recorded in DB metadata. Signed-off-by: Matthew Whitehead * Add --allow-downgrade CLI arg. If set it allows the downgrade and updates the Besu version in the metadata file to the downgraded version. Signed-off-by: Matthew Whitehead * Update gradle verification XML Signed-off-by: Matthew Whitehead * Add and update tests Signed-off-by: Matthew Whitehead * Refactoring Signed-off-by: Matthew Whitehead * Remove versioning from RocksDB, now in separate VERSION_DATADATA.json Signed-off-by: Matthew Whitehead * Tidy up and tests for the new class Signed-off-by: Matthew Whitehead * Move downgrade logic into VersionMetadata as BesuCommand is already very big Signed-off-by: Matthew Whitehead * Add more tests Signed-off-by: Matthew Whitehead * Refactor the naming of the option to version-compatibility-protection Signed-off-by: Matthew Whitehead * Remove remaining references to allow-downgrade Signed-off-by: Matthew Whitehead * Rename test Signed-off-by: Matthew Whitehead * Update comments Signed-off-by: Matthew Whitehead * Metadata verification update Signed-off-by: Matthew Whitehead * gradle fix Signed-off-by: Matthew Whitehead * Enable version downgrade protection by default for non-named networks Signed-off-by: Matthew Whitehead * Fix default logic Signed-off-by: Matthew Whitehead * Update ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/VersionMetadata.java Co-authored-by: Fabio Di Fabio Signed-off-by: Matt Whitehead * Update ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/VersionMetadata.java Co-authored-by: Fabio Di Fabio Signed-off-by: Matt Whitehead * mock-maker-inline no longer needed Signed-off-by: Matthew Whitehead --------- Signed-off-by: Matthew Whitehead Signed-off-by: Matt Whitehead Signed-off-by: Matt Whitehead Co-authored-by: Fabio Di Fabio --- CHANGELOG.md | 2 + .../org/hyperledger/besu/cli/BesuCommand.java | 30 ++++ .../hyperledger/besu/cli/BesuCommandTest.java | 7 + .../src/test/resources/everything_config.toml | 2 + ethereum/core/build.gradle | 1 + .../besu/ethereum/core/VersionMetadata.java | 147 +++++++++++++++++ .../ethereum/core/VersionMetadataTest.java | 150 ++++++++++++++++++ gradle/verification-metadata.xml | 114 +++++++++++++ gradle/versions.gradle | 2 + 9 files changed, 455 insertions(+) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/VersionMetadata.java create mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/VersionMetadataTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 22205db4bf8..1ed67e00e36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,8 @@ - Upgrade reference tests to version 13.1 [#6574](https://github.com/hyperledger/besu/pull/6574) - Extend `BesuConfiguration` service [#6584](https://github.com/hyperledger/besu/pull/6584) - Add `ethereum_min_gas_price` and `ethereum_min_priority_fee` metrics to track runtime values of `min-gas-price` and `min-priority-fee` [#6587](https://github.com/hyperledger/besu/pull/6587) +- Option to perform version incompatibility checks when starting Besu. In this first release of the feature, if `--version-compatibility-protection` is set to true it checks that the version of Besu being started is the same or higher than the previous version. [6307](https://github.com/hyperledger/besu/pull/6307) + ### Bug fixes - Fix the way an advertised host configured with `--p2p-host` is treated when communicating with the originator of a PING packet [#6225](https://github.com/hyperledger/besu/pull/6225) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 1cde9ce506a..f4b08e4f688 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -120,6 +120,7 @@ import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.MiningParametersMetrics; import org.hyperledger.besu.ethereum.core.PrivacyParameters; +import org.hyperledger.besu.ethereum.core.VersionMetadata; import org.hyperledger.besu.ethereum.eth.sync.SyncMode; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration; @@ -562,6 +563,12 @@ private InetAddress autoDiscoverDefaultIP() { arity = "1") private final Path kzgTrustedSetupFile = null; + @Option( + names = {"--version-compatibility-protection"}, + description = + "Perform compatibility checks between the version of Besu being started and the version of Besu that last started with this data directory. (default: ${DEFAULT-VALUE})") + private Boolean versionCompatibilityProtection = null; + @CommandLine.ArgGroup(validate = false, heading = "@|bold GraphQL Options|@%n") GraphQlOptions graphQlOptions = new GraphQlOptions(); @@ -1078,7 +1085,13 @@ public void run() { vertx = createVertx(createVertxOptions(metricsSystem.get())); validateOptions(); + configure(); + + // If we're not running against a named network, or if version compat protection has been + // explicitly enabled, perform compatibility check + VersionMetadata.versionCompatibilityChecks(versionCompatibilityProtection, dataDir()); + configureNativeLibs(); besuController = buildController(); @@ -1648,6 +1661,7 @@ private void configure() throws Exception { checkPortClash(); checkIfRequiredPortsAreAvailable(); syncMode = getDefaultSyncModeIfNotSet(); + versionCompatibilityProtection = getDefaultVersionCompatibilityProtectionIfNotSet(); ethNetworkConfig = updateNetworkConfig(network); @@ -2556,6 +2570,16 @@ String getLogLevel() { return loggingLevelOption.getLogLevel(); } + /** + * Returns the flag indicating that version compatiblity checks will be made. + * + * @return true if compatibility checks should be made, otherwise false + */ + @VisibleForTesting + public Boolean getVersionCompatibilityProtection() { + return versionCompatibilityProtection; + } + private void instantiateSignatureAlgorithmFactory() { if (SignatureAlgorithmFactory.isInstanceSet()) { return; @@ -2664,6 +2688,12 @@ private SyncMode getDefaultSyncModeIfNotSet() { : SyncMode.FULL); } + private Boolean getDefaultVersionCompatibilityProtectionIfNotSet() { + // Version compatibility protection is enabled by default for non-named networks + return Optional.ofNullable(versionCompatibilityProtection) + .orElse(commandLine.getParseResult().hasMatchedOption("network") ? false : true); + } + private String generateConfigurationOverview() { final ConfigurationOverviewBuilder builder = new ConfigurationOverviewBuilder(logger); diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index 961d69c2a59..09171e012e5 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -1341,6 +1341,13 @@ public void dnsEnabledOptionIsParsedCorrectly() { assertThat(besuCommand.getEnodeDnsConfiguration().updateEnabled()).isFalse(); } + @Test + public void versionCompatibilityProtectionTrueOptionIsParsedCorrectly() { + final TestBesuCommand besuCommand = parseCommand("--version-compatibility-protection", "true"); + + assertThat(besuCommand.getVersionCompatibilityProtection()).isTrue(); + } + @Test public void dnsUpdateEnabledOptionIsParsedCorrectly() { final TestBesuCommand besuCommand = diff --git a/besu/src/test/resources/everything_config.toml b/besu/src/test/resources/everything_config.toml index 3c6792b7b95..2bcbd21b496 100644 --- a/besu/src/test/resources/everything_config.toml +++ b/besu/src/test/resources/everything_config.toml @@ -16,6 +16,8 @@ node-private-key-file="./path/to/privateKey" pid-path="~/.pid" reorg-logging-threshold=0 static-nodes-file="~/besudata/static-nodes.json" +version-compatibility-protection=true + profile="NONE" # Security Module plugin to use security-module="localfile" diff --git a/ethereum/core/build.gradle b/ethereum/core/build.gradle index b616413dd22..dd1c0a8bd1b 100644 --- a/ethereum/core/build.gradle +++ b/ethereum/core/build.gradle @@ -48,6 +48,7 @@ dependencies { implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'com.google.guava:guava' implementation 'com.google.dagger:dagger' + implementation 'org.apache.maven:maven-artifact' annotationProcessor 'com.google.dagger:dagger-compiler' implementation 'io.opentelemetry:opentelemetry-api' implementation 'io.vertx:vertx-core' diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/VersionMetadata.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/VersionMetadata.java new file mode 100644 index 00000000000..0723fda97a0 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/VersionMetadata.java @@ -0,0 +1,147 @@ +/* + * Copyright Hyperledger Besu contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.core; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Path; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.maven.artifact.versioning.ComparableVersion; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class VersionMetadata { + private static final Logger LOG = LoggerFactory.getLogger(VersionMetadata.class); + + /** Represents an unknown Besu version in the version metadata file */ + public static final String BESU_VERSION_UNKNOWN = "UNKNOWN"; + + private static final String METADATA_FILENAME = "VERSION_METADATA.json"; + private static final ObjectMapper MAPPER = new ObjectMapper(); + private final String besuVersion; + + /** + * Get the version of Besu that is running. + * + * @return the version of Besu + */ + public static String getRuntimeVersion() { + return VersionMetadata.class.getPackage().getImplementationVersion() == null + ? BESU_VERSION_UNKNOWN + : VersionMetadata.class.getPackage().getImplementationVersion(); + } + + @JsonCreator + public VersionMetadata(@JsonProperty("besuVersion") final String besuVersion) { + this.besuVersion = besuVersion; + } + + public String getBesuVersion() { + return besuVersion; + } + + public static VersionMetadata lookUpFrom(final Path dataDir) throws IOException { + LOG.info("Lookup version metadata file in data directory: {}", dataDir); + return resolveVersionMetadata(getDefaultMetadataFile(dataDir)); + } + + public void writeToDirectory(final Path dataDir) throws IOException { + MAPPER.writeValue(getDefaultMetadataFile(dataDir), this); + } + + private static File getDefaultMetadataFile(final Path dataDir) { + return dataDir.resolve(METADATA_FILENAME).toFile(); + } + + private static VersionMetadata resolveVersionMetadata(final File metadataFile) + throws IOException { + VersionMetadata versionMetadata; + try { + versionMetadata = MAPPER.readValue(metadataFile, VersionMetadata.class); + LOG.info("Existing version data detected. Besu version {}", versionMetadata.besuVersion); + } catch (FileNotFoundException fnfe) { + versionMetadata = new VersionMetadata(BESU_VERSION_UNKNOWN); + } catch (JsonProcessingException jpe) { + throw new IllegalStateException( + String.format("Invalid metadata file %s", metadataFile.getAbsolutePath()), jpe); + } + return versionMetadata; + } + + /** + * This function is designed to protect a Besu instance from being unintentionally started at a + * version of Besu that might be incompatible with the version that last modified the specified + * data directory. Currently this check is limited to checking that the version is >= the previous + * version, to avoid accidentally running a lower version of Besu and potentially corrupting data, + * but the method could be extended to perform any other version-to-version compatibility checks + * necessary. If the --version-compatibility-protection flag is set to true and the compatibilty + * checks pass, the version metadata is updated to the current version of Besu. + */ + public static void versionCompatibilityChecks( + final boolean enforceCompatibilityProtection, final Path dataDir) throws IOException { + final VersionMetadata versionMetaData = VersionMetadata.lookUpFrom(dataDir); + if (versionMetaData.getBesuVersion().equals(VersionMetadata.BESU_VERSION_UNKNOWN)) { + // The version isn't known, potentially because the file doesn't exist. Write the latest + // version to the metadata file. + LOG.info( + "No version data detected. Writing Besu version {} to metadata file", + VersionMetadata.getRuntimeVersion()); + new VersionMetadata(VersionMetadata.getRuntimeVersion()).writeToDirectory(dataDir); + } else { + // Check the runtime version against the most recent version as recorded in the version + // metadata file + final String installedVersion = VersionMetadata.getRuntimeVersion().split("-", 2)[0]; + final String metadataVersion = versionMetaData.getBesuVersion().split("-", 2)[0]; + final int versionComparison = + new ComparableVersion(installedVersion).compareTo(new ComparableVersion(metadataVersion)); + if (versionComparison == 0) { + // Versions match - no-op + } else if (versionComparison < 0) { + if (!enforceCompatibilityProtection) { + LOG.warn( + "Besu version {} is lower than version {} that last started. Allowing startup because --version-compatibility-protection has been disabled.", + installedVersion, + metadataVersion); + // We've allowed startup at an older version of Besu. Since the version in the metadata + // file records the latest version of + // Besu to write to the database we'll update the metadata version to this + // downgraded-version. + new VersionMetadata(VersionMetadata.getRuntimeVersion()).writeToDirectory(dataDir); + } else { + final String message = + "Besu version " + + installedVersion + + " is lower than version " + + metadataVersion + + " that last started. Remove --version-compatibility-protection option to allow Besu to start at " + + " the lower version (warning - this may have unrecoverable effects on the database)."; + LOG.error(message, installedVersion, metadataVersion); + throw new IllegalStateException(message); + } + } else { + LOG.info( + "Besu version {} is higher than version {} that last started. Updating version metadata.", + installedVersion, + metadataVersion); + new VersionMetadata(VersionMetadata.getRuntimeVersion()).writeToDirectory(dataDir); + } + } + } +} diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/VersionMetadataTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/VersionMetadataTest.java new file mode 100644 index 00000000000..a506be214a9 --- /dev/null +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/VersionMetadataTest.java @@ -0,0 +1,150 @@ +/* + * Copyright Hyperledger Besu contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.hyperledger.besu.ethereum.core; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatNoException; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +class VersionMetadataTest { + @TempDir public Path temporaryFolder; + + @Test + void getVersion() { + final VersionMetadata versionMetadata = new VersionMetadata("23.10.2"); + assertThat(versionMetadata).isNotNull(); + assertThat(versionMetadata.getBesuVersion()).isEqualTo("23.10.2"); + } + + @Test + void metaFileShouldContain() throws Exception { + final Path tempDataDir = + createAndWrite("data", "VERSION_METADATA.json", "{\"besuVersion\":\"23.10.3\"}"); + + final VersionMetadata versionMetadata = VersionMetadata.lookUpFrom(tempDataDir); + assertThat(versionMetadata).isNotNull(); + assertThat(versionMetadata.getBesuVersion()).isEqualTo("23.10.3"); + } + + @Test + void compatibilityCheckShouldThrowExceptionIfEnabled() throws Exception { + // The version file says the last version to start was 23.10.3 + final Path tempDataDir = + createAndWrite("data", "VERSION_METADATA.json", "{\"besuVersion\":\"23.10.3\"}"); + + // The runtime says the current version is 23.10.2 (i.e. a downgrade) + try (MockedStatic mocked = + Mockito.mockStatic(VersionMetadata.class, Mockito.CALLS_REAL_METHODS)) { + mocked.when(VersionMetadata::getRuntimeVersion).thenReturn("23.10.2"); + + final VersionMetadata versionMetadata = VersionMetadata.lookUpFrom(tempDataDir); + assertThat(versionMetadata).isNotNull(); + assertThat(versionMetadata.getBesuVersion()).isEqualTo("23.10.3"); + assertThatThrownBy(() -> VersionMetadata.versionCompatibilityChecks(true, tempDataDir)) + .isInstanceOf(IllegalStateException.class); + } + + // Check that the file hasn't been updated + final String updatedFileContents = + Files.readString(tempDataDir.resolve("VERSION_METADATA.json")); + VersionMetadata newVersionMetadata = + new ObjectMapper().readValue(updatedFileContents, VersionMetadata.class); + assertThat(newVersionMetadata.getBesuVersion()).isEqualTo("23.10.3"); + } + + @Test + void compatibilityCheckShouldNotThrowExceptionIfDisabled() throws Exception { + // The version file says the last version to start was 23.10.3 + final Path tempDataDir = + createAndWrite("data", "VERSION_METADATA.json", "{\"besuVersion\":\"23.10.3\"}"); + + // The runtime says the current version is 23.10.2 (i.e. a downgrade) but we're setting + // version-compatibility-protection = false so no exception should be thrown + try (MockedStatic mocked = + Mockito.mockStatic(VersionMetadata.class, Mockito.CALLS_REAL_METHODS)) { + mocked.when(VersionMetadata::getRuntimeVersion).thenReturn("23.10.2"); + + final VersionMetadata versionMetadata = VersionMetadata.lookUpFrom(tempDataDir); + assertThat(versionMetadata).isNotNull(); + assertThat(versionMetadata.getBesuVersion()).isEqualTo("23.10.3"); + + assertThatNoException() + .isThrownBy(() -> VersionMetadata.versionCompatibilityChecks(false, tempDataDir)); + } + + // Check that the file has been updated + final String updatedFileContents = + Files.readString(tempDataDir.resolve("VERSION_METADATA.json")); + VersionMetadata newVersionMetadata = + new ObjectMapper().readValue(updatedFileContents, VersionMetadata.class); + assertThat(newVersionMetadata.getBesuVersion()).isEqualTo("23.10.2"); + } + + @Test + void compatibilityCheckShouldNotThrowExceptionIfResultIsUpgrade() throws Exception { + final Path tempDataDir = + createAndWrite("data", "VERSION_METADATA.json", "{\"besuVersion\":\"23.10.3\"}"); + + // The runtime says the current version is 23.10.2 (i.e. a downgrade) + try (MockedStatic mocked = + Mockito.mockStatic(VersionMetadata.class, Mockito.CALLS_REAL_METHODS)) { + mocked.when(VersionMetadata::getRuntimeVersion).thenReturn("23.10.4"); + + final VersionMetadata versionMetadata = VersionMetadata.lookUpFrom(tempDataDir); + assertThat(versionMetadata).isNotNull(); + assertThat(versionMetadata.getBesuVersion()).isEqualTo("23.10.3"); + + assertThatNoException() + .isThrownBy(() -> VersionMetadata.versionCompatibilityChecks(true, tempDataDir)); + } + + // Check that the file has been updated + final String updatedFileContents = + Files.readString(tempDataDir.resolve("VERSION_METADATA.json")); + VersionMetadata newVersionMetadata = + new ObjectMapper().readValue(updatedFileContents, VersionMetadata.class); + assertThat(newVersionMetadata.getBesuVersion()).isEqualTo("23.10.4"); + } + + private Path createAndWrite(final String dir, final String file, final String content) + throws IOException { + return createAndWrite(temporaryFolder, dir, file, content); + } + + private Path createAndWrite( + final Path temporaryFolder, final String dir, final String file, final String content) + throws IOException { + final Path tmpDir = temporaryFolder.resolve(dir); + Files.createDirectories(tmpDir); + createAndWrite(tmpDir.resolve(file), content); + return tmpDir; + } + + private void createAndWrite(final Path path, final String content) throws IOException { + path.toFile().createNewFile(); + Files.writeString(path, content); + } +} diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 9a8866b5f1e..07b387d350a 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -542,6 +542,14 @@ + + + + + + + + @@ -550,6 +558,11 @@ + + + + + @@ -2932,6 +2945,14 @@ + + + + + + + + @@ -3381,6 +3402,37 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -3572,6 +3624,68 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/gradle/versions.gradle b/gradle/versions.gradle index 23731c443d8..7c2967ffc20 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -233,6 +233,8 @@ dependencyManagement { dependency 'org.yaml:snakeyaml:2.0' + dependency 'org.apache.maven:maven-artifact:3.9.6' + dependency 'tech.pegasys.discovery:discovery:22.12.0' } }