diff --git a/airbyte-commons-cli/build.gradle b/airbyte-commons-cli/build.gradle new file mode 100644 index 000000000000..3dbb175d2ad9 --- /dev/null +++ b/airbyte-commons-cli/build.gradle @@ -0,0 +1,7 @@ +plugins { + id "java-library" +} + +dependencies { + implementation 'commons-cli:commons-cli:1.4' +} diff --git a/airbyte-commons-cli/readme.md b/airbyte-commons-cli/readme.md new file mode 100644 index 000000000000..bd4bd8aab272 --- /dev/null +++ b/airbyte-commons-cli/readme.md @@ -0,0 +1 @@ +This module houses utility functions for the `commons-cli` library. It is separate from `commons`, because it depends on external library `commons-cli` which we do not want to introduce as a dependency to every module. diff --git a/airbyte-commons-cli/src/main/java/io/airbyte/commons/cli/Clis.java b/airbyte-commons-cli/src/main/java/io/airbyte/commons/cli/Clis.java new file mode 100644 index 000000000000..1833d5ed5218 --- /dev/null +++ b/airbyte-commons-cli/src/main/java/io/airbyte/commons/cli/Clis.java @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.cli; + +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionGroup; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; + +public class Clis { + + /** + * Parse an options object + * + * @param args - command line args + * @param options - expected options + * @return object with parsed values. + */ + public static CommandLine parse(final String[] args, final Options options, final CommandLineParser parser, final String commandLineSyntax) { + final HelpFormatter helpFormatter = new HelpFormatter(); + + try { + return parser.parse(options, args); + } catch (final ParseException e) { + if (commandLineSyntax != null && !commandLineSyntax.isEmpty()) { + helpFormatter.printHelp(commandLineSyntax, options); + } + throw new IllegalArgumentException(e); + } + } + + public static CommandLine parse(final String[] args, final Options options, final String commandLineSyntax) { + return parse(args, options, new DefaultParser(), commandLineSyntax); + } + + public static CommandLine parse(final String[] args, final Options options, final CommandLineParser parser) { + return parse(args, options, parser, null); + } + + public static CommandLine parse(final String[] args, final Options options) { + return parse(args, options, new DefaultParser()); + } + + /** + * Provide a fluent interface for building an OptionsGroup. + * + * @param isRequired - is the option group required + * @param options - options in the option group + * @return the created option group. + */ + public static OptionGroup createOptionGroup(final boolean isRequired, final Option... options) { + final OptionGroup optionGroup = new OptionGroup(); + optionGroup.setRequired(isRequired); + for (final Option option : options) { + optionGroup.addOption(option); + } + return optionGroup; + } + + public static CommandLineParser getRelaxedParser() { + return new RelaxedParser(); + } + + // https://stackoverflow.com/questions/33874902/apache-commons-cli-1-3-1-how-to-ignore-unknown-arguments + private static class RelaxedParser extends DefaultParser { + + @Override + public CommandLine parse(final Options options, final String[] arguments) throws ParseException { + final List knownArgs = new ArrayList<>(); + for (int i = 0; i < arguments.length; i++) { + if (options.hasOption(arguments[i])) { + knownArgs.add(arguments[i]); + if (i + 1 < arguments.length && options.getOption(arguments[i]).hasArg()) { + knownArgs.add(arguments[i + 1]); + } + } + } + return super.parse(options, knownArgs.toArray(new String[0])); + } + + } + +} diff --git a/airbyte-commons-cli/src/test/java/io/airbyte/commons/cli/ClisTest.java b/airbyte-commons-cli/src/test/java/io/airbyte/commons/cli/ClisTest.java new file mode 100644 index 000000000000..a2587c9ca18a --- /dev/null +++ b/airbyte-commons-cli/src/test/java/io/airbyte/commons/cli/ClisTest.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.cli; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionGroup; +import org.apache.commons.cli.Options; +import org.junit.jupiter.api.Test; + +class ClisTest { + + @Test + void testCreateOptionGroup() { + final Option optionA = new Option("a", "alpha"); + final Option optionB = new Option("b", "beta"); + final OptionGroup optionGroupExpected = new OptionGroup(); + optionGroupExpected.addOption(optionA); + optionGroupExpected.addOption(optionB); + + final OptionGroup optionGroupActual = Clis.createOptionGroup( + false, + optionA, + optionB); + + // hack: OptionGroup does not define hashcode, so compare its string instead of the object itself. + assertEquals(optionGroupExpected.toString(), optionGroupActual.toString()); + } + + @Test + void testParse() { + final Option optionA = Option.builder("a").required(true).hasArg(true).build(); + final Option optionB = Option.builder("b").required(true).hasArg(true).build(); + final Options options = new Options().addOption(optionA).addOption(optionB); + final String[] args = {"-a", "alpha", "-b", "beta"}; + final CommandLine parsed = Clis.parse(args, options, new DefaultParser()); + assertEquals("alpha", parsed.getOptions()[0].getValue()); + assertEquals("beta", parsed.getOptions()[1].getValue()); + } + + @Test + void testParseNonConforming() { + final Option optionA = Option.builder("a").required(true).hasArg(true).build(); + final Option optionB = Option.builder("b").required(true).hasArg(true).build(); + final Options options = new Options().addOption(optionA).addOption(optionB); + final String[] args = {"-a", "alpha", "-b", "beta", "-c", "charlie"}; + assertThrows(IllegalArgumentException.class, () -> Clis.parse(args, options, new DefaultParser())); + } + + @Test + void testParseNonConformingWithSyntax() { + final Option optionA = Option.builder("a").required(true).hasArg(true).build(); + final Option optionB = Option.builder("b").required(true).hasArg(true).build(); + final Options options = new Options().addOption(optionA).addOption(optionB); + final String[] args = {"-a", "alpha", "-b", "beta", "-c", "charlie"}; + assertThrows(IllegalArgumentException.class, () -> Clis.parse(args, options, new DefaultParser(), "search")); + } + + @Test + void testRelaxedParser() { + final Option optionA = Option.builder("a").required(true).hasArg(true).build(); + final Option optionB = Option.builder("b").required(true).hasArg(true).build(); + final Options options = new Options().addOption(optionA).addOption(optionB); + final String[] args = {"-a", "alpha", "-b", "beta", "-c", "charlie"}; + final CommandLine parsed = Clis.parse(args, options, Clis.getRelaxedParser()); + assertEquals("alpha", parsed.getOptions()[0].getValue()); + assertEquals("beta", parsed.getOptions()[1].getValue()); + } + +} diff --git a/airbyte-integrations/bases/base-java/build.gradle b/airbyte-integrations/bases/base-java/build.gradle index ac80d3242e1c..136e0f8c19d4 100644 --- a/airbyte-integrations/bases/base-java/build.gradle +++ b/airbyte-integrations/bases/base-java/build.gradle @@ -4,6 +4,10 @@ plugins { } dependencies { + implementation project(':airbyte-protocol:models') + implementation project(':airbyte-commons-cli') + implementation project(':airbyte-json-validation') + implementation 'commons-cli:commons-cli:1.4' implementation 'org.apache.sshd:sshd-mina:2.7.0' // bouncycastle is pinned to version-match the transitive dependency from kubernetes client-java @@ -12,8 +16,6 @@ dependencies { implementation 'org.bouncycastle:bcpkix-jdk15on:1.66' implementation 'org.bouncycastle:bctls-jdk15on:1.66' - implementation project(':airbyte-protocol:models') - implementation project(":airbyte-json-validation") implementation "org.testcontainers:testcontainers:1.15.3" implementation "org.testcontainers:jdbc:1.15.3" diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationCliParser.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationCliParser.java index 3e1a24a31bb7..ca6f46afd82f 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationCliParser.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationCliParser.java @@ -5,19 +5,14 @@ package io.airbyte.integrations.base; import com.google.common.base.Preconditions; +import io.airbyte.commons.cli.Clis; import java.nio.file.Path; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.DefaultParser; -import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionGroup; import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,31 +25,28 @@ public class IntegrationCliParser { private static final Logger LOGGER = LoggerFactory.getLogger(IntegrationCliParser.class); - private static final OptionGroup COMMAND_GROUP = new OptionGroup(); - - static { - COMMAND_GROUP.setRequired(true); - COMMAND_GROUP.addOption(Option.builder() - .longOpt(Command.SPEC.toString().toLowerCase()) - .desc("outputs the json configuration specification") - .build()); - COMMAND_GROUP.addOption(Option.builder() - .longOpt(Command.CHECK.toString().toLowerCase()) - .desc("checks the config can be used to connect") - .build()); - COMMAND_GROUP.addOption(Option.builder() - .longOpt(Command.DISCOVER.toString().toLowerCase()) - .desc("outputs a catalog describing the source's catalog") - .build()); - COMMAND_GROUP.addOption(Option.builder() - .longOpt(Command.READ.toString().toLowerCase()) - .desc("reads the source and outputs messages to STDOUT") - .build()); - COMMAND_GROUP.addOption(Option.builder() - .longOpt(Command.WRITE.toString().toLowerCase()) - .desc("writes messages from STDIN to the integration") - .build()); - } + private static final OptionGroup COMMAND_GROUP = Clis.createOptionGroup( + true, + Option.builder() + .longOpt(Command.SPEC.toString().toLowerCase()) + .desc("outputs the json configuration specification") + .build(), + Option.builder() + .longOpt(Command.CHECK.toString().toLowerCase()) + .desc("checks the config can be used to connect") + .build(), + Option.builder() + .longOpt(Command.DISCOVER.toString().toLowerCase()) + .desc("outputs a catalog describing the source's catalog") + .build(), + Option.builder() + .longOpt(Command.READ.toString().toLowerCase()) + .desc("reads the source and outputs messages to STDOUT") + .build(), + Option.builder() + .longOpt(Command.WRITE.toString().toLowerCase()) + .desc("writes messages from STDIN to the integration") + .build()); public IntegrationConfig parse(final String[] args) { final Command command = parseCommand(args); @@ -62,20 +54,11 @@ public IntegrationConfig parse(final String[] args) { } private static Command parseCommand(final String[] args) { - final CommandLineParser parser = new RelaxedParser(); - final HelpFormatter helpFormatter = new HelpFormatter(); - final Options options = new Options(); options.addOptionGroup(COMMAND_GROUP); - try { - final CommandLine parsed = parser.parse(options, args); - return Command.valueOf(parsed.getOptions()[0].getLongOpt().toUpperCase()); - // if discover, then validate, etc... - } catch (final ParseException e) { - helpFormatter.printHelp("java-base", options); - throw new IllegalArgumentException(e); - } + final CommandLine parsed = Clis.parse(args, options, Clis.getRelaxedParser()); + return Command.valueOf(parsed.getOptions()[0].getLongOpt().toUpperCase()); } private static IntegrationConfig parseOptions(final String[] args, final Command command) { @@ -87,26 +70,46 @@ private static IntegrationConfig parseOptions(final String[] args, final Command case SPEC -> { // no args. } - case CHECK, DISCOVER -> options.addOption(Option - .builder().longOpt(JavaBaseConstants.ARGS_CONFIG_KEY).desc(JavaBaseConstants.ARGS_CONFIG_DESC).hasArg(true).required(true).build()); + case CHECK, DISCOVER -> options.addOption(Option.builder() + .longOpt(JavaBaseConstants.ARGS_CONFIG_KEY) + .desc(JavaBaseConstants.ARGS_CONFIG_DESC) + .hasArg(true) + .required(true) + .build()); case READ -> { - options.addOption(Option - .builder().longOpt(JavaBaseConstants.ARGS_CONFIG_KEY).desc(JavaBaseConstants.ARGS_CONFIG_DESC).hasArg(true).required(true).build()); - options.addOption(Option - .builder().longOpt(JavaBaseConstants.ARGS_CATALOG_KEY).desc(JavaBaseConstants.ARGS_CATALOG_DESC).hasArg(true).build()); - options.addOption(Option - .builder().longOpt(JavaBaseConstants.ARGS_STATE_KEY).desc(JavaBaseConstants.ARGS_PATH_DESC).hasArg(true).build()); + options.addOption(Option.builder() + .longOpt(JavaBaseConstants.ARGS_CONFIG_KEY) + .desc(JavaBaseConstants.ARGS_CONFIG_DESC) + .hasArg(true) + .required(true) + .build()); + options.addOption(Option.builder() + .longOpt(JavaBaseConstants.ARGS_CATALOG_KEY) + .desc(JavaBaseConstants.ARGS_CATALOG_DESC) + .hasArg(true) + .build()); + options.addOption(Option.builder() + .longOpt(JavaBaseConstants.ARGS_STATE_KEY) + .desc(JavaBaseConstants.ARGS_PATH_DESC) + .hasArg(true) + .build()); } case WRITE -> { - options.addOption(Option - .builder().longOpt(JavaBaseConstants.ARGS_CONFIG_KEY).desc(JavaBaseConstants.ARGS_CONFIG_DESC).hasArg(true).required(true).build()); - options.addOption(Option - .builder().longOpt(JavaBaseConstants.ARGS_CATALOG_KEY).desc(JavaBaseConstants.ARGS_CATALOG_DESC).hasArg(true).build()); + options.addOption(Option.builder() + .longOpt(JavaBaseConstants.ARGS_CONFIG_KEY) + .desc(JavaBaseConstants.ARGS_CONFIG_DESC) + .hasArg(true) + .required(true).build()); + options.addOption(Option.builder() + .longOpt(JavaBaseConstants.ARGS_CATALOG_KEY) + .desc(JavaBaseConstants.ARGS_CATALOG_DESC) + .hasArg(true) + .build()); } default -> throw new IllegalStateException("Unexpected value: " + command); } - final CommandLine parsed = runParse(options, args, command); + final CommandLine parsed = Clis.parse(args, options, command.toString().toLowerCase()); Preconditions.checkNotNull(parsed); final Map argsMap = new HashMap<>(); for (final Option option : parsed.getOptions()) { @@ -139,35 +142,4 @@ private static IntegrationConfig parseOptions(final String[] args, final Command } } - private static CommandLine runParse(final Options options, final String[] args, final Command command) { - final CommandLineParser parser = new DefaultParser(); - final HelpFormatter helpFormatter = new HelpFormatter(); - - try { - return parser.parse(options, args); - } catch (final ParseException e) { - helpFormatter.printHelp(command.toString().toLowerCase(), options); - throw new IllegalArgumentException(e); - } - } - - // https://stackoverflow.com/questions/33874902/apache-commons-cli-1-3-1-how-to-ignore-unknown-arguments - private static class RelaxedParser extends DefaultParser { - - @Override - public CommandLine parse(final Options options, final String[] arguments) throws ParseException { - final List knownArgs = new ArrayList<>(); - for (int i = 0; i < arguments.length; i++) { - if (options.hasOption(arguments[i])) { - knownArgs.add(arguments[i]); - if (i + 1 < arguments.length && options.getOption(arguments[i]).hasArg()) { - knownArgs.add(arguments[i + 1]); - } - } - } - return super.parse(options, knownArgs.toArray(new String[0])); - } - - } - } diff --git a/settings.gradle b/settings.gradle index 5ce056ee7804..d8328b4fe28a 100644 --- a/settings.gradle +++ b/settings.gradle @@ -28,6 +28,7 @@ if(!System.getenv().containsKey("SUB_BUILD")) { include ':airbyte-api' include ':airbyte-commons' include ':airbyte-commons-docker' +include ':airbyte-commons-cli' include ':airbyte-config:models' // reused by acceptance tests in connector base. include ':airbyte-db:lib' // reused by acceptance tests in connector base. include ':airbyte-json-validation'