From eea927f917d507c6cec68706e6296024c6539233 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Tue, 22 May 2018 01:09:31 -0400 Subject: [PATCH] Enable installing plugins from snapshots.elastic.co (#30765) We post snapshot builds to snapshots.elastic.co yet the official plugin installer will not let you install such plugins without manually downloading them and installing them from a file URL. This commit adds the ability for the plugin installer to use snapshots.elastic.co for installing official plugins if a es.plugins.staging is set and the current build is also a snapshot build. Otherwise, we continue to use staging.elastic.co if the current build is a release build and es.plugins.staging is set and, of course, use the release artifacts at artifacts.elastic.co for release builds with es.plugins.staging unset. --- .../plugins/InstallPluginCommand.java | 40 ++++++++--- .../plugins/InstallPluginCommandTests.java | 71 +++++++++++++++---- 2 files changed, 87 insertions(+), 24 deletions(-) diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java index d6f6e36b8c48e..6aa9f43936a74 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java @@ -30,7 +30,6 @@ import org.elasticsearch.cli.ExitCodes; import org.elasticsearch.cli.Terminal; import org.elasticsearch.cli.UserException; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.hash.MessageDigests; @@ -240,7 +239,7 @@ private static void handleInstallXPack(final Build.Flavor flavor) throws UserExc /** Downloads the plugin and returns the file it was downloaded to. */ private Path download(Terminal terminal, String pluginId, Path tmpDir) throws Exception { if (OFFICIAL_PLUGINS.contains(pluginId)) { - final String url = getElasticUrl(terminal, getStagingHash(), Version.CURRENT, pluginId, Platforms.PLATFORM_NAME); + final String url = getElasticUrl(terminal, getStagingHash(), Version.CURRENT, isSnapshot(), pluginId, Platforms.PLATFORM_NAME); terminal.println("-> Downloading " + pluginId + " from elastic"); return downloadZipAndChecksum(terminal, url, tmpDir, false); } @@ -272,22 +271,43 @@ String getStagingHash() { return System.getProperty(PROPERTY_STAGING_ID); } + boolean isSnapshot() { + return Build.CURRENT.isSnapshot(); + } + /** Returns the url for an official elasticsearch plugin. */ - private String getElasticUrl(Terminal terminal, String stagingHash, Version version, - String pluginId, String platform) throws IOException { + private String getElasticUrl( + final Terminal terminal, + final String stagingHash, + final Version version, + final boolean isSnapshot, + final String pluginId, + final String platform) throws IOException, UserException { final String baseUrl; + if (isSnapshot && stagingHash == null) { + throw new UserException( + ExitCodes.CONFIG, "attempted to install release build of official plugin on snapshot build of Elasticsearch"); + } if (stagingHash != null) { - baseUrl = String.format(Locale.ROOT, - "https://staging.elastic.co/%s-%s/downloads/elasticsearch-plugins/%s", version, stagingHash, pluginId); + if (isSnapshot) { + baseUrl = nonReleaseUrl("snapshots", version, stagingHash, pluginId); + } else { + baseUrl = nonReleaseUrl("staging", version, stagingHash, pluginId); + } } else { - baseUrl = String.format(Locale.ROOT, - "https://artifacts.elastic.co/downloads/elasticsearch-plugins/%s", pluginId); + baseUrl = String.format(Locale.ROOT, "https://artifacts.elastic.co/downloads/elasticsearch-plugins/%s", pluginId); } - final String platformUrl = String.format(Locale.ROOT, "%s/%s-%s-%s.zip", baseUrl, pluginId, platform, version); + final String platformUrl = + String.format(Locale.ROOT, "%s/%s-%s-%s.zip", baseUrl, pluginId, platform, Version.displayVersion(version, isSnapshot)); if (urlExists(terminal, platformUrl)) { return platformUrl; } - return String.format(Locale.ROOT, "%s/%s-%s.zip", baseUrl, pluginId, version); + return String.format(Locale.ROOT, "%s/%s-%s.zip", baseUrl, pluginId, Version.displayVersion(version, isSnapshot)); + } + + private String nonReleaseUrl(final String hostname, final Version version, final String stagingHash, final String pluginId) { + return String.format( + Locale.ROOT, "https://%s.elastic.co/%s-%s/downloads/elasticsearch-plugins/%s", hostname, version, stagingHash, pluginId); } /** Returns the url for an elasticsearch plugin in maven. */ diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java index bfeb3c0279b65..07fe4f5403ae6 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java @@ -800,7 +800,7 @@ private void installPlugin(MockTerminal terminal, boolean isBatch) throws Except skipJarHellCommand.execute(terminal, pluginZip, isBatch, env.v2()); } - void assertInstallPluginFromUrl(String pluginId, String name, String url, String stagingHash, + void assertInstallPluginFromUrl(String pluginId, String name, String url, String stagingHash, boolean isSnapshot, String shaExtension, Function shaCalculator) throws Exception { Tuple env = createEnv(fs, temp); Path pluginDir = createPluginDir(temp); @@ -834,6 +834,12 @@ boolean urlExists(Terminal terminal, String urlString) throws IOException { String getStagingHash() { return stagingHash; } + + @Override + boolean isSnapshot() { + return isSnapshot; + } + @Override void jarHellCheck(PluginInfo candidateInfo, Path candidate, Path pluginsDir, Path modulesDir) throws Exception { // no jarhell check @@ -843,48 +849,82 @@ void jarHellCheck(PluginInfo candidateInfo, Path candidate, Path pluginsDir, Pat assertPlugin(name, pluginDir, env.v2()); } - public void assertInstallPluginFromUrl(String pluginId, String name, String url, String stagingHash) throws Exception { + public void assertInstallPluginFromUrl( + final String pluginId, final String name, final String url, final String stagingHash, boolean isSnapshot) throws Exception { MessageDigest digest = MessageDigest.getInstance("SHA-512"); - assertInstallPluginFromUrl(pluginId, name, url, stagingHash, ".sha512", checksumAndFilename(digest, url)); + assertInstallPluginFromUrl(pluginId, name, url, stagingHash, isSnapshot, ".sha512", checksumAndFilename(digest, url)); } public void testOfficalPlugin() throws Exception { String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip"; - assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null); + assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, false); + } + + public void testOfficialPluginSnapshot() throws Exception { + String url = String.format( + Locale.ROOT, + "https://snapshots.elastic.co/%s-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-%s.zip", + Version.CURRENT, + Version.displayVersion(Version.CURRENT, true)); + assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123", true); + } + + public void testInstallReleaseBuildOfPluginOnSnapshotBuild() { + String url = String.format( + Locale.ROOT, + "https://snapshots.elastic.co/%s-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-%s.zip", + Version.CURRENT, + Version.displayVersion(Version.CURRENT, true)); + // attemping to install a release build of a plugin (no staging ID) on a snapshot build should throw a user exception + final UserException e = + expectThrows(UserException.class, () -> assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, true)); + assertThat(e.exitCode, equalTo(ExitCodes.CONFIG)); + assertThat( + e, hasToString(containsString("attempted to install release build of official plugin on snapshot build of Elasticsearch"))); } public void testOfficalPluginStaging() throws Exception { String url = "https://staging.elastic.co/" + Version.CURRENT + "-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip"; - assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123"); + assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123", false); } public void testOfficalPlatformPlugin() throws Exception { String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Platforms.PLATFORM_NAME + "-" + Version.CURRENT + ".zip"; - assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null); + assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, false); + } + + public void testOfficialPlatformPluginSnapshot() throws Exception { + String url = String.format( + Locale.ROOT, + "https://snapshots.elastic.co/%s-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-%s-%s.zip", + Version.CURRENT, + Platforms.PLATFORM_NAME, + Version.displayVersion(Version.CURRENT, true)); + assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123", true); } public void testOfficalPlatformPluginStaging() throws Exception { String url = "https://staging.elastic.co/" + Version.CURRENT + "-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Platforms.PLATFORM_NAME + "-"+ Version.CURRENT + ".zip"; - assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123"); + assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123", false); } public void testMavenPlugin() throws Exception { String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip"; - assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null); + assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, false); } public void testMavenPlatformPlugin() throws Exception { String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-" + Platforms.PLATFORM_NAME + "-1.0.0.zip"; - assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null); + assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, false); } public void testMavenSha1Backcompat() throws Exception { String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip"; MessageDigest digest = MessageDigest.getInstance("SHA-1"); - assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, ".sha1", checksum(digest)); + assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, false, ".sha1", checksum(digest)); assertTrue(terminal.getOutput(), terminal.getOutput().contains("sha512 not found, falling back to sha1")); } @@ -892,7 +932,7 @@ public void testOfficialShaMissing() throws Exception { String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip"; MessageDigest digest = MessageDigest.getInstance("SHA-1"); UserException e = expectThrows(UserException.class, () -> - assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, ".sha1", checksum(digest))); + assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, false, ".sha1", checksum(digest))); assertEquals(ExitCodes.IO_ERROR, e.exitCode); assertEquals("Plugin checksum missing: " + url + ".sha512", e.getMessage()); } @@ -900,7 +940,7 @@ public void testOfficialShaMissing() throws Exception { public void testMavenShaMissing() throws Exception { String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip"; UserException e = expectThrows(UserException.class, () -> - assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, ".dne", bytes -> null)); + assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, false, ".dne", bytes -> null)); assertEquals(ExitCodes.IO_ERROR, e.exitCode); assertEquals("Plugin checksum missing: " + url + ".sha1", e.getMessage()); } @@ -909,7 +949,7 @@ public void testInvalidShaFileMissingFilename() throws Exception { String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip"; MessageDigest digest = MessageDigest.getInstance("SHA-512"); UserException e = expectThrows(UserException.class, () -> - assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, ".sha512", checksum(digest))); + assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, false, ".sha512", checksum(digest))); assertEquals(ExitCodes.IO_ERROR, e.exitCode); assertTrue(e.getMessage(), e.getMessage().startsWith("Invalid checksum file")); } @@ -923,6 +963,7 @@ public void testInvalidShaFileMismatchFilename() throws Exception { "analysis-icu", url, null, + false, ".sha512", checksumAndString(digest, " repository-s3-" + Version.CURRENT + ".zip"))); assertEquals(ExitCodes.IO_ERROR, e.exitCode); @@ -938,6 +979,7 @@ public void testInvalidShaFileContainingExtraLine() throws Exception { "analysis-icu", url, null, + false, ".sha512", checksumAndString(digest, " analysis-icu-" + Version.CURRENT + ".zip\nfoobar"))); assertEquals(ExitCodes.IO_ERROR, e.exitCode); @@ -952,6 +994,7 @@ public void testSha512Mismatch() throws Exception { "analysis-icu", url, null, + false, ".sha512", bytes -> "foobar analysis-icu-" + Version.CURRENT + ".zip")); assertEquals(ExitCodes.IO_ERROR, e.exitCode); @@ -961,7 +1004,7 @@ public void testSha512Mismatch() throws Exception { public void testSha1Mismatch() throws Exception { String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip"; UserException e = expectThrows(UserException.class, () -> - assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, ".sha1", bytes -> "foobar")); + assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, false, ".sha1", bytes -> "foobar")); assertEquals(ExitCodes.IO_ERROR, e.exitCode); assertTrue(e.getMessage(), e.getMessage().contains("SHA-1 mismatch, expected foobar")); }