Enable installing plugins from snapshots.elastic.co (#30765)

We post snapshot builds to snapshots.elastic.co yet the official plugin
installer will not let you install such plugins without manually
downloading them and installing them from a file URL. This commit adds
the ability for the plugin installer to use snapshots.elastic.co for
installing official plugins if a es.plugins.staging is set and the
current build is also a snapshot build. Otherwise, we continue to use
staging.elastic.co if the current build is a release build and
es.plugins.staging is set and, of course, use the release artifacts at
artifacts.elastic.co for release builds with es.plugins.staging unset.

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java

@@ -30,7 +30,6 @@
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserException;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.hash.MessageDigests;
@@ -240,7 +239,7 @@ private static void handleInstallXPack(final Build.Flavor flavor) throws UserExc
     /** Downloads the plugin and returns the file it was downloaded to. */
     private Path download(Terminal terminal, String pluginId, Path tmpDir) throws Exception {
         if (OFFICIAL_PLUGINS.contains(pluginId)) {
-            final String url = getElasticUrl(terminal, getStagingHash(), Version.CURRENT, pluginId, Platforms.PLATFORM_NAME);
+            final String url = getElasticUrl(terminal, getStagingHash(), Version.CURRENT, isSnapshot(), pluginId, Platforms.PLATFORM_NAME);
             terminal.println("-> Downloading " + pluginId + " from elastic");
             return downloadZipAndChecksum(terminal, url, tmpDir, false);
         }
@@ -272,22 +271,43 @@ String getStagingHash() {
         return System.getProperty(PROPERTY_STAGING_ID);
     }
 
+    boolean isSnapshot() {
+        return Build.CURRENT.isSnapshot();
+    }
+
     /** Returns the url for an official elasticsearch plugin. */
-    private String getElasticUrl(Terminal terminal, String stagingHash, Version version,
-                                        String pluginId, String platform) throws IOException {
+    private String getElasticUrl(
+            final Terminal terminal,
+            final String stagingHash,
+            final Version version,
+            final boolean isSnapshot,
+            final String pluginId,
+            final String platform) throws IOException, UserException {
         final String baseUrl;
+        if (isSnapshot && stagingHash == null) {
+            throw new UserException(
+                    ExitCodes.CONFIG, "attempted to install release build of official plugin on snapshot build of Elasticsearch");
+        }
         if (stagingHash != null) {
-            baseUrl = String.format(Locale.ROOT,
-                "https://staging.elastic.co/%s-%s/downloads/elasticsearch-plugins/%s", version, stagingHash, pluginId);
+            if (isSnapshot) {
+                baseUrl = nonReleaseUrl("snapshots", version, stagingHash, pluginId);
+            } else {
+                baseUrl = nonReleaseUrl("staging", version, stagingHash, pluginId);
+            }
         } else {
-            baseUrl = String.format(Locale.ROOT,
-                "https://artifacts.elastic.co/downloads/elasticsearch-plugins/%s", pluginId);
+            baseUrl = String.format(Locale.ROOT, "https://artifacts.elastic.co/downloads/elasticsearch-plugins/%s", pluginId);
         }
-        final String platformUrl = String.format(Locale.ROOT, "%s/%s-%s-%s.zip", baseUrl, pluginId, platform, version);
+        final String platformUrl =
+                String.format(Locale.ROOT, "%s/%s-%s-%s.zip", baseUrl, pluginId, platform, Version.displayVersion(version, isSnapshot));
         if (urlExists(terminal, platformUrl)) {
             return platformUrl;
         }
-        return String.format(Locale.ROOT, "%s/%s-%s.zip", baseUrl, pluginId, version);
+        return String.format(Locale.ROOT, "%s/%s-%s.zip", baseUrl, pluginId, Version.displayVersion(version, isSnapshot));
+    }
+
+    private String nonReleaseUrl(final String hostname, final Version version, final String stagingHash, final String pluginId) {
+        return String.format(
+                Locale.ROOT, "https://%s.elastic.co/%s-%s/downloads/elasticsearch-plugins/%s", hostname, version, stagingHash, pluginId);
     }
 
     /** Returns the url for an elasticsearch plugin in maven. */

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java

@@ -800,7 +800,7 @@ private void installPlugin(MockTerminal terminal, boolean isBatch) throws Except
         skipJarHellCommand.execute(terminal, pluginZip, isBatch, env.v2());
     }
 
-    void assertInstallPluginFromUrl(String pluginId, String name, String url, String stagingHash,
+    void assertInstallPluginFromUrl(String pluginId, String name, String url, String stagingHash, boolean isSnapshot,
                                                    String shaExtension, Function<byte[], String> shaCalculator) throws Exception {
         Tuple<Path, Environment> env = createEnv(fs, temp);
         Path pluginDir = createPluginDir(temp);
@@ -834,6 +834,12 @@ boolean urlExists(Terminal terminal, String urlString) throws IOException {
             String getStagingHash() {
                 return stagingHash;
             }
+
+            @Override
+            boolean isSnapshot() {
+                return isSnapshot;
+            }
+
             @Override
             void jarHellCheck(PluginInfo candidateInfo, Path candidate, Path pluginsDir, Path modulesDir) throws Exception {
                 // no jarhell check
@@ -843,64 +849,98 @@ void jarHellCheck(PluginInfo candidateInfo, Path candidate, Path pluginsDir, Pat
         assertPlugin(name, pluginDir, env.v2());
     }
 
-    public void assertInstallPluginFromUrl(String pluginId, String name, String url, String stagingHash) throws Exception {
+    public void assertInstallPluginFromUrl(
+            final String pluginId, final String name, final String url, final String stagingHash, boolean isSnapshot) throws Exception {
         MessageDigest digest = MessageDigest.getInstance("SHA-512");
-        assertInstallPluginFromUrl(pluginId, name, url, stagingHash, ".sha512", checksumAndFilename(digest, url));
+        assertInstallPluginFromUrl(pluginId, name, url, stagingHash, isSnapshot, ".sha512", checksumAndFilename(digest, url));
     }
 
     public void testOfficalPlugin() throws Exception {
         String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip";
-        assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null);
+        assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, false);
+    }
+
+    public void testOfficialPluginSnapshot() throws Exception {
+        String url = String.format(
+                Locale.ROOT,
+                "https://snapshots.elastic.co/%s-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-%s.zip",
+                Version.CURRENT,
+                Version.displayVersion(Version.CURRENT, true));
+        assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123", true);
+    }
+
+    public void testInstallReleaseBuildOfPluginOnSnapshotBuild() {
+        String url = String.format(
+                Locale.ROOT,
+                "https://snapshots.elastic.co/%s-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-%s.zip",
+                Version.CURRENT,
+                Version.displayVersion(Version.CURRENT, true));
+        // attemping to install a release build of a plugin (no staging ID) on a snapshot build should throw a user exception
+        final UserException e =
+                expectThrows(UserException.class, () -> assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, true));
+        assertThat(e.exitCode, equalTo(ExitCodes.CONFIG));
+        assertThat(
+                e, hasToString(containsString("attempted to install release build of official plugin on snapshot build of Elasticsearch")));
     }
 
     public void testOfficalPluginStaging() throws Exception {
         String url = "https://staging.elastic.co/" + Version.CURRENT + "-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-"
             + Version.CURRENT + ".zip";
-        assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123");
+        assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123", false);
     }
 
     public void testOfficalPlatformPlugin() throws Exception {
         String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Platforms.PLATFORM_NAME +
             "-" + Version.CURRENT + ".zip";
-        assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null);
+        assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, false);
+    }
+
+    public void testOfficialPlatformPluginSnapshot() throws Exception {
+        String url = String.format(
+                Locale.ROOT,
+                "https://snapshots.elastic.co/%s-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-%s-%s.zip",
+                Version.CURRENT,
+                Platforms.PLATFORM_NAME,
+                Version.displayVersion(Version.CURRENT, true));
+        assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123", true);
     }
 
     public void testOfficalPlatformPluginStaging() throws Exception {
         String url = "https://staging.elastic.co/" + Version.CURRENT + "-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-"
             + Platforms.PLATFORM_NAME + "-"+ Version.CURRENT + ".zip";
-        assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123");
+        assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123", false);
     }
 
     public void testMavenPlugin() throws Exception {
         String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip";
-        assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null);
+        assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, false);
     }
 
     public void testMavenPlatformPlugin() throws Exception {
         String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-" + Platforms.PLATFORM_NAME + "-1.0.0.zip";
-        assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null);
+        assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, false);
     }
 
     public void testMavenSha1Backcompat() throws Exception {
         String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip";
         MessageDigest digest = MessageDigest.getInstance("SHA-1");
-        assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, ".sha1", checksum(digest));
+        assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, false, ".sha1", checksum(digest));
         assertTrue(terminal.getOutput(), terminal.getOutput().contains("sha512 not found, falling back to sha1"));
     }
 
     public void testOfficialShaMissing() throws Exception {
         String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip";
         MessageDigest digest = MessageDigest.getInstance("SHA-1");
         UserException e = expectThrows(UserException.class, () ->
-            assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, ".sha1", checksum(digest)));
+            assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, false, ".sha1", checksum(digest)));
         assertEquals(ExitCodes.IO_ERROR, e.exitCode);
         assertEquals("Plugin checksum missing: " + url + ".sha512", e.getMessage());
     }
 
     public void testMavenShaMissing() throws Exception {
         String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip";
         UserException e = expectThrows(UserException.class, () ->
-            assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, ".dne", bytes -> null));
+            assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, false, ".dne", bytes -> null));
         assertEquals(ExitCodes.IO_ERROR, e.exitCode);
         assertEquals("Plugin checksum missing: " + url + ".sha1", e.getMessage());
     }
@@ -909,7 +949,7 @@ public void testInvalidShaFileMissingFilename() throws Exception {
         String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip";
         MessageDigest digest = MessageDigest.getInstance("SHA-512");
         UserException e = expectThrows(UserException.class, () ->
-                assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, ".sha512", checksum(digest)));
+                assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, false, ".sha512", checksum(digest)));
         assertEquals(ExitCodes.IO_ERROR, e.exitCode);
         assertTrue(e.getMessage(), e.getMessage().startsWith("Invalid checksum file"));
     }
@@ -923,6 +963,7 @@ public void testInvalidShaFileMismatchFilename() throws Exception {
                         "analysis-icu",
                         url,
                         null,
+                        false,
                         ".sha512",
                         checksumAndString(digest, "  repository-s3-" + Version.CURRENT + ".zip")));
         assertEquals(ExitCodes.IO_ERROR, e.exitCode);
@@ -938,6 +979,7 @@ public void testInvalidShaFileContainingExtraLine() throws Exception {
                     "analysis-icu",
                     url,
                     null,
+                    false,
                     ".sha512",
                     checksumAndString(digest, "  analysis-icu-" + Version.CURRENT + ".zip\nfoobar")));
         assertEquals(ExitCodes.IO_ERROR, e.exitCode);
@@ -952,6 +994,7 @@ public void testSha512Mismatch() throws Exception {
                     "analysis-icu",
                     url,
                     null,
+                    false,
                     ".sha512",
                     bytes -> "foobar  analysis-icu-" + Version.CURRENT + ".zip"));
         assertEquals(ExitCodes.IO_ERROR, e.exitCode);
@@ -961,7 +1004,7 @@ public void testSha512Mismatch() throws Exception {
     public void testSha1Mismatch() throws Exception {
         String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip";
         UserException e = expectThrows(UserException.class, () ->
-            assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, ".sha1", bytes -> "foobar"));
+            assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, false, ".sha1", bytes -> "foobar"));
         assertEquals(ExitCodes.IO_ERROR, e.exitCode);
         assertTrue(e.getMessage(), e.getMessage().contains("SHA-1 mismatch, expected foobar"));
     }