Add RockDB Subcommand for printing usage per column family (hyperledg…

…er#6185)

* Add RockDB Subcommand for printing usage per column family
Signed-off-by: Simon Dudley <simon.dudley@consensys.net>

* changed output to follow a MD table notation.

Signed-off-by: Gabriel Fukushima <gabrielfukushima@gmail.com>

---------

Signed-off-by: Gabriel Fukushima <gabrielfukushima@gmail.com>
Co-authored-by: Gabriel Fukushima <gabrielfukushima@gmail.com>

CHANGELOG.md

@@ -16,6 +16,7 @@
 - Fix the unavailability of `address` field when returning an `Account` entity on GraphQL in case of unreachable world state [#6198](https://github.com/hyperledger/besu/pull/6198)
 - Update OpenJ9 Docker image to latest version [#6226](https://github.com/hyperledger/besu/pull/6226)
 - Add error messages on authentication failures with username and password [#6212](https://github.com/hyperledger/besu/pull/6212)
+- Add `rocksdb usage` to the `storage` subcommand to allow users and dev to check columns families usage [#6185](https://github.com/hyperledger/besu/pull/6185)
 
 ### Bug fixes
 - Fix Docker image name clash between Besu and evmtool [#6194](https://github.com/hyperledger/besu/pull/6194)

besu/build.gradle

@@ -77,6 +77,7 @@ dependencies {
   implementation 'org.springframework.security:spring-security-crypto'
   implementation 'org.xerial.snappy:snappy-java'
   implementation 'tech.pegasys:jc-kzg-4844'
+  implementation 'org.rocksdb:rocksdbjni'
 
   runtimeOnly 'org.apache.logging.log4j:log4j-jul'
   runtimeOnly 'com.splunk.logging:splunk-library-javalogging'

besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RocksDbSubCommand.java

@@ -0,0 +1,115 @@
+/*
+ * Copyright Hyperledger Besu Contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.hyperledger.besu.cli.subcommands.storage;
+
+import static org.hyperledger.besu.controller.BesuController.DATABASE_PATH;
+
+import org.hyperledger.besu.cli.util.VersionProvider;
+
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.rocksdb.ColumnFamilyDescriptor;
+import org.rocksdb.ColumnFamilyHandle;
+import org.rocksdb.Options;
+import org.rocksdb.RocksDB;
+import org.rocksdb.RocksDBException;
+import picocli.CommandLine;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.ParentCommand;
+
+/** The RocksDB subcommand. */
+@Command(
+    name = "rocksdb",
+    description = "Print RocksDB information",
+    mixinStandardHelpOptions = true,
+    versionProvider = VersionProvider.class,
+    subcommands = {RocksDbSubCommand.RocksDbUsage.class})
+public class RocksDbSubCommand implements Runnable {
+
+  @SuppressWarnings("unused")
+  @ParentCommand
+  private StorageSubCommand parentCommand;
+
+  @SuppressWarnings("unused")
+  @CommandLine.Spec
+  private CommandLine.Model.CommandSpec spec;
+
+  @Override
+  public void run() {
+    spec.commandLine().usage(System.out);
+  }
+
+  @Command(
+      name = "usage",
+      description = "Print disk usage",
+      mixinStandardHelpOptions = true,
+      versionProvider = VersionProvider.class)
+  static class RocksDbUsage implements Runnable {
+
+    @SuppressWarnings("unused")
+    @CommandLine.Spec
+    private CommandLine.Model.CommandSpec spec;
+
+    @SuppressWarnings("unused")
+    @ParentCommand
+    private RocksDbSubCommand parentCommand;
+
+    @Override
+    public void run() {
+
+      final PrintWriter out = spec.commandLine().getOut();
+
+      final String dbPath =
+          parentCommand
+              .parentCommand
+              .parentCommand
+              .dataDir()
+              .toString()
+              .concat("/")
+              .concat(DATABASE_PATH);
+
+      RocksDB.loadLibrary();
+      Options options = new Options();
+      options.setCreateIfMissing(true);
+
+      // Open the RocksDB database with multiple column families
+      List<byte[]> cfNames;
+      try {
+        cfNames = RocksDB.listColumnFamilies(options, dbPath);
+      } catch (RocksDBException e) {
+        throw new RuntimeException(e);
+      }
+      final List<ColumnFamilyHandle> cfHandles = new ArrayList<>();
+      final List<ColumnFamilyDescriptor> cfDescriptors = new ArrayList<>();
+      for (byte[] cfName : cfNames) {
+        cfDescriptors.add(new ColumnFamilyDescriptor(cfName));
+      }
+      RocksDbUsageHelper.printTableHeader(out);
+      try (final RocksDB rocksdb = RocksDB.openReadOnly(dbPath, cfDescriptors, cfHandles)) {
+        for (ColumnFamilyHandle cfHandle : cfHandles) {
+          RocksDbUsageHelper.printUsageForColumnFamily(rocksdb, cfHandle, out);
+        }
+      } catch (RocksDBException e) {
+        throw new RuntimeException(e);
+      } finally {
+        for (ColumnFamilyHandle cfHandle : cfHandles) {
+          cfHandle.close();
+        }
+      }
+    }
+  }
+}

besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RocksDbUsageHelper.java

@@ -0,0 +1,105 @@
+/*
+ * Copyright contributors to Hyperledger Besu.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.hyperledger.besu.cli.subcommands.storage;
+
+import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier;
+
+import java.io.PrintWriter;
+
+import org.bouncycastle.util.Arrays;
+import org.rocksdb.ColumnFamilyHandle;
+import org.rocksdb.RocksDB;
+import org.rocksdb.RocksDBException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** RocksDB Usage subcommand helper methods for formatting and printing. */
+public class RocksDbUsageHelper {
+  private static final Logger LOG = LoggerFactory.getLogger(RocksDbUsageHelper.class);
+
+  static void printUsageForColumnFamily(
+      final RocksDB rocksdb, final ColumnFamilyHandle cfHandle, final PrintWriter out)
+      throws RocksDBException, NumberFormatException {
+    final String size = rocksdb.getProperty(cfHandle, "rocksdb.estimate-live-data-size");
+    boolean emptyColumnFamily = false;
+    if (!size.isEmpty() && !size.isBlank()) {
+      try {
+        final long sizeLong = Long.parseLong(size);
+        final String totalSstFilesSize =
+            rocksdb.getProperty(cfHandle, "rocksdb.total-sst-files-size");
+        final long totalSstFilesSizeLong =
+            !totalSstFilesSize.isEmpty() && !totalSstFilesSize.isBlank()
+                ? Long.parseLong(totalSstFilesSize)
+                : 0;
+        if (sizeLong == 0) {
+          emptyColumnFamily = true;
+        }
+
+        if (!emptyColumnFamily) {
+          printLine(
+              out,
+              getNameById(cfHandle.getName()),
+              rocksdb.getProperty(cfHandle, "rocksdb.estimate-num-keys"),
+              formatOutputSize(sizeLong),
+              formatOutputSize(totalSstFilesSizeLong));
+        }
+      } catch (NumberFormatException e) {
+        LOG.error("Failed to parse string into long: " + e.getMessage());
+      }
+    }
+  }
+
+  private static String formatOutputSize(final long size) {
+    if (size > (1024 * 1024 * 1024)) {
+      long sizeInGiB = size / (1024 * 1024 * 1024);
+      return sizeInGiB + " GiB";
+    } else if (size > (1024 * 1024)) {
+      long sizeInMiB = size / (1024 * 1024);
+      return sizeInMiB + " MiB";
+    } else if (size > 1024) {
+      long sizeInKiB = size / 1024;
+      return sizeInKiB + " KiB";
+    } else {
+      return size + " B";
+    }
+  }
+
+  private static String getNameById(final byte[] id) {
+    for (KeyValueSegmentIdentifier segment : KeyValueSegmentIdentifier.values()) {
+      if (Arrays.areEqual(segment.getId(), id)) {
+        return segment.getName();
+      }
+    }
+    return null; // id not found
+  }
+
+  static void printTableHeader(final PrintWriter out) {
+    out.format(
+        "| Column Family                  | Keys            | Column Size  | SST Files Size  |\n");
+    out.format(
+        "|--------------------------------|-----------------|--------------|-----------------|\n");
+  }
+
+  static void printLine(
+      final PrintWriter out,
+      final String cfName,
+      final String keys,
+      final String columnSize,
+      final String sstFilesSize) {
+    final String format = "| %-30s | %-15s | %-12s | %-15s |\n";
+    out.format(format, cfName, keys, columnSize, sstFilesSize);
+  }
+}

besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java

@@ -45,7 +45,7 @@
     description = "This command provides storage related actions.",
     mixinStandardHelpOptions = true,
     versionProvider = VersionProvider.class,
-    subcommands = {StorageSubCommand.RevertVariablesStorage.class, TrieLogSubCommand.class})
+    subcommands = {StorageSubCommand.RevertVariablesStorage.class, RocksDbSubCommand.class, TrieLogSubCommand.class})
 public class StorageSubCommand implements Runnable {
 
   /** The constant COMMAND_NAME. */

crypto/algorithms/src/main/java/org/hyperledger/besu/crypto/KeyPairUtil.java

@@ -84,6 +84,7 @@ public static KeyPair loadKeyPair(final File keyFile) {
     final KeyPair key;
     if (keyFile.exists()) {
 
+      LOG.info("Attempting to load public key from {}", keyFile.getAbsolutePath());
       key = load(keyFile);
       LOG.info(
           "Loaded public key {} from {}", key.getPublicKey().toString(), keyFile.getAbsolutePath());