From 8526f2e81f73b45cc6701ccb750fcb42c8823025 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" <6454655+adoroszlai@users.noreply.github.com> Date: Fri, 6 Dec 2024 08:32:34 +0100 Subject: [PATCH] HDDS-11826. Interactive mode for ozone shell. (#7515) --- .../hdds/cli/ExtensibleParentCommand.java | 2 +- .../apache/hadoop/hdds/cli/GenericCli.java | 11 ++- hadoop-hdds/tools/pom.xml | 16 ++-- .../dist/src/main/license/bin/LICENSE.txt | 2 + .../dist/src/main/license/jar-report.txt | 2 + hadoop-ozone/tools/pom.xml | 27 ++++-- .../AbstractReconfigureSubCommand.java | 1 + .../apache/hadoop/ozone/shell/Handler.java | 2 +- .../apache/hadoop/ozone/shell/OzoneRatis.java | 3 +- .../org/apache/hadoop/ozone/shell/REPL.java | 90 +++++++++++++++++++ .../org/apache/hadoop/ozone/shell/Shell.java | 45 +++++++++- pom.xml | 11 +++ 12 files changed, 189 insertions(+), 23 deletions(-) create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/REPL.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/ExtensibleParentCommand.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/ExtensibleParentCommand.java index 45bbb440054..d4fde1b75cb 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/ExtensibleParentCommand.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/ExtensibleParentCommand.java @@ -42,7 +42,7 @@ static void addSubcommands(CommandLine cli) { ServiceLoader subcommands = ServiceLoader.load(parentCommand.subcommandType()); for (Object subcommand : subcommands) { final CommandLine.Command commandAnnotation = subcommand.getClass().getAnnotation(CommandLine.Command.class); - CommandLine subcommandCommandLine = new CommandLine(subcommand); + CommandLine subcommandCommandLine = new CommandLine(subcommand, cli.getFactory()); cli.addSubcommand(commandAnnotation.name(), subcommandCommandLine); } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/GenericCli.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/GenericCli.java index a8ff931b23f..a64c4bd84a5 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/GenericCli.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/GenericCli.java @@ -56,15 +56,20 @@ public GenericCli() { } public GenericCli(Class type) { - cmd = new CommandLine(this); + this(type, CommandLine.defaultFactory()); + } + + public GenericCli(Class type, CommandLine.IFactory factory) { + cmd = new CommandLine(this, factory); cmd.setExecutionExceptionHandler((ex, commandLine, parseResult) -> { printError(ex); return EXECUTION_ERROR_EXIT_CODE; }); if (type != null) { - addSubcommands(getCmd(), type); + addSubcommands(cmd, type); } + ExtensibleParentCommand.addSubcommands(cmd); } @@ -75,7 +80,7 @@ private void addSubcommands(CommandLine cli, Class type) { if (subcommand.getParentType().equals(type)) { final Command commandAnnotation = subcommand.getClass().getAnnotation(Command.class); - CommandLine subcommandCommandLine = new CommandLine(subcommand); + CommandLine subcommandCommandLine = new CommandLine(subcommand, cli.getFactory()); addSubcommands(subcommandCommandLine, subcommand.getClass()); cli.addSubcommand(commandAnnotation.name(), subcommandCommandLine); } diff --git a/hadoop-hdds/tools/pom.xml b/hadoop-hdds/tools/pom.xml index 583c801bcd4..5b77f394c96 100644 --- a/hadoop-hdds/tools/pom.xml +++ b/hadoop-hdds/tools/pom.xml @@ -179,20 +179,20 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> maven-compiler-plugin - - org.apache.ozone - hdds-config - ${hdds.version} - org.kohsuke.metainf-services metainf-services ${metainf-services.version} + + info.picocli + picocli-codegen + ${picocli.version} + - org.apache.hadoop.hdds.conf.ConfigFileGenerator org.kohsuke.metainf_services.AnnotationProcessorImpl + picocli.codegen.aot.graalvm.processor.NativeImageConfigGeneratorProcessor @@ -207,8 +207,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> Only selected annotation processors are enabled, see configuration of maven-compiler-plugin. - org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator + org.apache.hadoop.hdds.conf.Config + org.apache.hadoop.hdds.conf.ConfigGroup org.apache.hadoop.hdds.scm.metadata.Replicate + org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator diff --git a/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt b/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt index 1c70fa0401b..b291afc568a 100644 --- a/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt +++ b/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt @@ -315,6 +315,7 @@ Apache License 2.0 commons-validator:commons-validator commons-fileupload:commons-fileupload info.picocli:picocli + info.picocli:picocli-shell-jline3 io.dropwizard.metrics:metrics-core io.grpc:grpc-api io.grpc:grpc-context @@ -479,6 +480,7 @@ BSD 3-Clause com.google.re2j:re2j com.jcraft:jsch com.thoughtworks.paranamer:paranamer + org.jline:jline3 org.ow2.asm:asm org.ow2.asm:asm-analysis org.ow2.asm:asm-commons diff --git a/hadoop-ozone/dist/src/main/license/jar-report.txt b/hadoop-ozone/dist/src/main/license/jar-report.txt index e6d9a1d2a2e..be48c1d1fe2 100644 --- a/hadoop-ozone/dist/src/main/license/jar-report.txt +++ b/hadoop-ozone/dist/src/main/license/jar-report.txt @@ -151,6 +151,7 @@ share/ozone/lib/jgrapht-core.jar share/ozone/lib/jgrapht-ext.jar share/ozone/lib/jgraphx.jar share/ozone/lib/jheaps.jar +share/ozone/lib/jline.jar share/ozone/lib/jmespath-java.jar share/ozone/lib/jna.jar share/ozone/lib/jna-platform.jar @@ -236,6 +237,7 @@ share/ozone/lib/ozone-s3gateway.jar share/ozone/lib/ozone-tools.jar share/ozone/lib/perfmark-api.jar share/ozone/lib/picocli.jar +share/ozone/lib/picocli-shell-jline3.jar share/ozone/lib/protobuf-java.jar share/ozone/lib/protobuf-java.jar share/ozone/lib/protobuf-java-util.jar diff --git a/hadoop-ozone/tools/pom.xml b/hadoop-ozone/tools/pom.xml index 92440812259..8ea8ded01ce 100644 --- a/hadoop-ozone/tools/pom.xml +++ b/hadoop-ozone/tools/pom.xml @@ -173,6 +173,14 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> info.picocli picocli + + info.picocli + picocli-shell-jline3 + + + org.jline + jline + jakarta.xml.bind jakarta.xml.bind-api @@ -275,21 +283,24 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> maven-compiler-plugin - - org.apache.ozone - hdds-config - ${hdds.version} - org.kohsuke.metainf-services metainf-services ${metainf-services.version} + + info.picocli + picocli-codegen + ${picocli.version} + - org.apache.hadoop.hdds.conf.ConfigFileGenerator org.kohsuke.metainf_services.AnnotationProcessorImpl + picocli.codegen.aot.graalvm.processor.NativeImageConfigGeneratorProcessor + + -Aproject=${project.groupId}/${project.artifactId} + @@ -303,8 +314,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> Only selected annotation processors are enabled, see configuration of maven-compiler-plugin. - org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator + org.apache.hadoop.hdds.conf.Config + org.apache.hadoop.hdds.conf.ConfigGroup org.apache.hadoop.hdds.scm.metadata.Replicate + org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/reconfig/AbstractReconfigureSubCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/reconfig/AbstractReconfigureSubCommand.java index 0a2666d30ee..b8ea45898d7 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/reconfig/AbstractReconfigureSubCommand.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/reconfig/AbstractReconfigureSubCommand.java @@ -28,6 +28,7 @@ /** * An abstract Class use to ReconfigureSubCommand. */ +@CommandLine.Command public abstract class AbstractReconfigureSubCommand implements Callable { @CommandLine.ParentCommand private ReconfigureCommands parent; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Handler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Handler.java index 92484936f21..d1755a68806 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Handler.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Handler.java @@ -111,7 +111,7 @@ protected boolean securityEnabled() { if (!enabled) { err().printf("Error: '%s' operation works only when security is " + "enabled. To enable security set ozone.security.enabled to " + - "true.%n", spec.qualifiedName()); + "true.%n", spec.qualifiedName().trim()); } return enabled; } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/OzoneRatis.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/OzoneRatis.java index 5bc98268064..f471911c1f4 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/OzoneRatis.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/OzoneRatis.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.ozone.shell; +import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.tracing.TracingUtil; import org.apache.ratis.shell.cli.sh.RatisShell; @@ -30,7 +31,7 @@ description = "Shell for running Ratis commands", versionProvider = HddsVersionProvider.class, mixinStandardHelpOptions = true) -public class OzoneRatis extends Shell { +public class OzoneRatis extends GenericCli { public OzoneRatis() { super(OzoneRatis.class); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/REPL.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/REPL.java new file mode 100644 index 00000000000..14848846348 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/REPL.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.shell; + +import org.jline.console.SystemRegistry; +import org.jline.console.impl.SystemRegistryImpl; +import org.jline.reader.EndOfFileException; +import org.jline.reader.LineReader; +import org.jline.reader.LineReaderBuilder; +import org.jline.reader.MaskingCallback; +import org.jline.reader.Parser; +import org.jline.reader.UserInterruptException; +import org.jline.reader.impl.DefaultParser; +import org.jline.terminal.Terminal; +import org.jline.terminal.TerminalBuilder; +import org.jline.widget.TailTipWidgets; +import org.jline.widget.TailTipWidgets.TipType; +import picocli.CommandLine; +import picocli.shell.jline3.PicocliCommands; +import picocli.shell.jline3.PicocliCommands.PicocliCommandsFactory; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.function.Supplier; + +/** + * Interactive shell for Ozone commands. + * (REPL = Read-Eval-Print Loop) + */ +class REPL { + + REPL(Shell shell, CommandLine cmd, PicocliCommandsFactory factory) { + Parser parser = new DefaultParser(); + Supplier workDir = () -> Paths.get(System.getProperty("user.dir")); + TerminalBuilder terminalBuilder = TerminalBuilder.builder() + .dumb(true); + try (Terminal terminal = terminalBuilder.build()) { + factory.setTerminal(terminal); + + PicocliCommands picocliCommands = new PicocliCommands(cmd); + picocliCommands.name(shell.name()); + SystemRegistry registry = new SystemRegistryImpl(parser, terminal, workDir, null); + registry.setCommandRegistries(picocliCommands); + registry.register("help", picocliCommands); + + LineReader reader = LineReaderBuilder.builder() + .terminal(terminal) + .completer(registry.completer()) + .parser(parser) + .variable(LineReader.LIST_MAX, 50) + .build(); + + TailTipWidgets widgets = new TailTipWidgets(reader, registry::commandDescription, 5, TipType.COMPLETER); + widgets.enable(); + + String prompt = shell.prompt() + "> "; + + while (true) { + try { + registry.cleanUp(); + String line = reader.readLine(prompt, null, (MaskingCallback) null, null); + registry.execute(line); + } catch (UserInterruptException ignored) { + // ignore + } catch (EndOfFileException e) { + return; + } catch (Exception e) { + registry.trace(e); + } + } + } catch (Exception e) { + shell.printError(e); + } + } +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Shell.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Shell.java index 97e160651bb..44893f78e66 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Shell.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Shell.java @@ -20,6 +20,8 @@ import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.ozone.om.exceptions.OMException; +import picocli.CommandLine; +import picocli.shell.jline3.PicocliCommands.PicocliCommandsFactory; /** * Ozone user interface commands. @@ -27,6 +29,7 @@ * This class uses dispatch method to make calls * to appropriate handlers that execute the ozone functions. */ +@CommandLine.Command public abstract class Shell extends GenericCli { public static final String OZONE_URI_DESCRIPTION = @@ -46,15 +49,52 @@ public abstract class Shell extends GenericCli { "Any unspecified information will be identified from\n" + "the config files.\n"; + private String name; + + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; + + @CommandLine.Option(names = { "--interactive" }, description = "Run in interactive mode") + private boolean interactive; + public Shell() { + this(null); } public Shell(Class type) { - super(type); + super(type, new PicocliCommandsFactory()); + } + + public String name() { + return name; + } + + // override if custom prompt is needed + public String prompt() { + return name(); + } + + @Override + public void run(String[] argv) { + name = spec.name(); + + try { + // parse args to check if interactive mode is requested + getCmd().parseArgs(argv); + } catch (Exception ignored) { + // failure will be reported by regular, non-interactive run + } + + if (interactive) { + spec.name(""); // use short name (e.g. "token get" instead of "ozone sh token get") + new REPL(this, getCmd(), (PicocliCommandsFactory) getCmd().getFactory()); + } else { + super.run(argv); + } } @Override - protected void printError(Throwable errorArg) { + public void printError(Throwable errorArg) { OMException omException = null; if (errorArg instanceof OMException) { @@ -77,4 +117,3 @@ protected void printError(Throwable errorArg) { } } } - diff --git a/pom.xml b/pom.xml index 16023cb28bd..869afebf493 100644 --- a/pom.xml +++ b/pom.xml @@ -301,6 +301,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs 2.6.0 1.4.0 3.9.12 + 3.23.0 5.3.39 3.11.10 @@ -323,6 +324,16 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs picocli ${picocli.version} + + info.picocli + picocli-shell-jline3 + ${picocli.version} + + + org.jline + jline + ${jline.version} + org.apache.derby derby