From b311c2ca5ae9c38cd9d0dd2d656169d22f9e96e6 Mon Sep 17 00:00:00 2001 From: garyschulte Date: Thu, 31 Aug 2023 08:44:17 -0700 Subject: [PATCH] Do not create ignorable segments on `revert storage-variables` (#5830) * fix the bug that creates the ignorable chain pruner segment, add rocks exception parsing to RocksDBColumnarKeyValueStorage subclasses * parse rocksdb error for unprintable column family id's Signed-off-by: garyschulte --- CHANGELOG.md | 15 ++++- .../org/hyperledger/besu/cli/BesuCommand.java | 3 +- .../storage/StorageSubCommand.java | 2 + ...imisticRocksDBColumnarKeyValueStorage.java | 2 +- .../RocksDBColumnarKeyValueStorage.java | 56 ++++++++++++++++++- ...ctionDBRocksDBColumnarKeyValueStorage.java | 2 +- .../RocksDBColumnarKeyValueStorageTest.java | 4 +- 7 files changed, 77 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e0fb1a29d22..7a11b5dcc31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## 23.7.3 +### Additions and Improvements + +### Breaking Changes +- Removed support for Kotti network (ETC) [#5816](https://github.com/hyperledger/besu/pull/5816) + +### Additions and Improvements + +### Bug Fixes +- do not create ignorable storage on revert storage-variables subcommand [#5830](https://github.com/hyperledger/besu/pull/5830) + +### Download Links + + ## 23.7.2 ### Additions and Improvements @@ -30,7 +44,6 @@ ### Breaking Changes - Removed deprecated GoQuorum permissioning interop [#5607](https://github.com/hyperledger/besu/pull/5607) - Removed support for version 0 of the database as it is no longer used by any active node. [#5698](https://github.com/hyperledger/besu/pull/5698) -- Removed support for Kotti network (ETC) [#5816](https://github.com/hyperledger/besu/pull/5816) ### Additions and Improvements - `evmtool` launcher binaries now ship as part of the standard distribution. [#5701](https://github.com/hyperledger/besu/pull/5701) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index e0330170b7f..9ccf10df966 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -3489,7 +3489,8 @@ private void setMergeConfigOptions() { getActualGenesisConfigOptions().getTerminalTotalDifficulty().isPresent()); } - private void setIgnorableStorageSegments() { + /** Set ignorable segments in RocksDB Storage Provider plugin. */ + public void setIgnorableStorageSegments() { if (!unstableChainPruningOptions.getChainDataPruningEnabled()) { rocksDBPlugin.addIgnorableSegmentIdentifier(KeyValueSegmentIdentifier.CHAIN_PRUNER_STATE); } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java index f7a1a6eca11..bd40a42a431 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java @@ -99,6 +99,8 @@ public void run() { } private StorageProvider getStorageProvider() { + // init collection of ignorable segments + parentCommand.parentCommand.setIgnorableStorageSegments(); return parentCommand.parentCommand.getStorageProvider(); } diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/OptimisticRocksDBColumnarKeyValueStorage.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/OptimisticRocksDBColumnarKeyValueStorage.java index 13f50da388e..fa5786e6b9a 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/OptimisticRocksDBColumnarKeyValueStorage.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/OptimisticRocksDBColumnarKeyValueStorage.java @@ -63,7 +63,7 @@ public OptimisticRocksDBColumnarKeyValueStorage( initColumnHandles(); } catch (final RocksDBException e) { - throw new StorageException(e); + throw parseRocksDBException(e, segments, ignorableSegments); } } diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java index 567b3f623b9..3d98be37523 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java @@ -40,7 +40,10 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import com.google.common.base.Splitter; +import com.google.common.collect.Streams; import org.apache.commons.lang3.tuple.Pair; +import org.apache.tuweni.bytes.Bytes; import org.rocksdb.BlockBasedTableConfig; import org.rocksdb.BloomFilter; import org.rocksdb.ColumnFamilyDescriptor; @@ -161,7 +164,52 @@ public RocksDBColumnarKeyValueStorage( txOptions = new TransactionDBOptions(); columnHandles = new ArrayList<>(columnDescriptors.size()); } catch (RocksDBException e) { - throw new StorageException(e); + throw parseRocksDBException(e, defaultSegments, ignorableSegments); + } + } + + /** + * Parse RocksDBException and wrap in StorageException + * + * @param ex RocksDBException + * @param defaultSegments segments requested to open + * @param ignorableSegments segments which are ignorable if not present + * @return StorageException wrapping the RocksDB Exception + */ + protected static StorageException parseRocksDBException( + final RocksDBException ex, + final List defaultSegments, + final List ignorableSegments) { + String message = ex.getMessage(); + List knownSegments = + Streams.concat(defaultSegments.stream(), ignorableSegments.stream()).distinct().toList(); + + // parse out unprintable segment names for a more useful exception: + String columnExceptionMessagePrefix = "Column families not opened: "; + if (message.contains(columnExceptionMessagePrefix)) { + String substring = message.substring(message.indexOf(": ") + 2); + + List unHandledSegments = new ArrayList<>(); + Splitter.on(", ") + .splitToStream(substring) + .forEach( + part -> { + byte[] bytes = part.getBytes(StandardCharsets.UTF_8); + unHandledSegments.add( + knownSegments.stream() + .filter(seg -> Arrays.equals(seg.getId(), bytes)) + .findFirst() + .map(seg -> new SegmentRecord(seg.getName(), seg.getId())) + .orElse(new SegmentRecord(part, bytes)) + .forDisplay()); + }); + + return new StorageException( + "RocksDBException: Unhandled column families: [" + + unHandledSegments.stream().collect(Collectors.joining(", ")) + + "]"); + } else { + return new StorageException(ex); } } @@ -356,4 +404,10 @@ void throwIfClosed() { } abstract RocksDB getDB(); + + record SegmentRecord(String name, byte[] id) { + public String forDisplay() { + return String.format("'%s'(%s)", name, Bytes.of(id).toHexString()); + } + } } diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/TransactionDBRocksDBColumnarKeyValueStorage.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/TransactionDBRocksDBColumnarKeyValueStorage.java index 6825a05063a..4825154561a 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/TransactionDBRocksDBColumnarKeyValueStorage.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/TransactionDBRocksDBColumnarKeyValueStorage.java @@ -66,7 +66,7 @@ public TransactionDBRocksDBColumnarKeyValueStorage( initColumnHandles(); } catch (final RocksDBException e) { - throw new StorageException(e); + throw parseRocksDBException(e, segments, ignorableSegments); } } diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorageTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorageTest.java index 827b18eb0c1..5462d239b91 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorageTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorageTest.java @@ -222,7 +222,7 @@ public void dbShouldNotIgnoreExperimentalSegmentsIfExisted(@TempDir final Path t createSegmentedStore(testPath, Arrays.asList(TestSegment.FOO, TestSegment.BAR), List.of()); fail("DB without knowledge of experimental column family should fail"); } catch (StorageException e) { - assertThat(e.getMessage()).contains("Column families not opened"); + assertThat(e.getMessage()).contains("Unhandled column families"); } // Even if the column family is marked as ignored, as long as it exists, it will not be ignored @@ -265,7 +265,7 @@ public void dbWillBeBackwardIncompatibleAfterExperimentalSegmentsAreAdded( createSegmentedStore(testPath, Arrays.asList(TestSegment.FOO, TestSegment.BAR), List.of()); fail("DB without knowledge of experimental column family should fail"); } catch (StorageException e) { - assertThat(e.getMessage()).contains("Column families not opened"); + assertThat(e.getMessage()).contains("Unhandled column families"); } }