Adds docker registry projects for localstack and minio (#5593)

This allows us to version the testcontainer images just like we version our other docker images. To ensure our local image names work, we need to inform testcontainers that they are in fact compatible images via org.testcontainers.utility.DockerImageName#asCompatibleSubstituteFor. A Docker#localImageName utility was added and propogated to locations where we previously manually constructing or hardcoding the "local-build" image names. Fixes #5592
deephaven · Jun 11, 2024 · 198e673 · 198e673
1 parent df2733c
commit 198e673
Show file tree

Hide file tree

Showing 29 changed files with 83 additions and 87 deletions.
diff --git a/Integrations/build.gradle b/Integrations/build.gradle
@@ -78,13 +78,13 @@ def runInDocker = { String name, String sourcePath, List<String> command, Closur
         }
         parentContainers = [project(':docker-server-jetty').tasks.findByName('buildDocker-server-jetty')] // deephaven/server-jetty
 
-        imageName = 'deephaven/py-integrations:local-build'
+        imageName = Docker.localImageName('py-integrations')
 
         addConfig(it)
 
         dockerfile {
             // set up the container, env vars - things that aren't likely to change
-            from 'deephaven/server-jetty:local-build'
+            from Docker.localImageName('server-jetty')
             runCommand '''set -eux; \\
                       pip3 install unittest-xml-reporting==3.0.4; \\
                       mkdir -p /out/report; \\

diff --git a/R/build.gradle b/R/build.gradle
@@ -46,7 +46,7 @@ def buildRClient = Docker.registerDockerTask(project, 'rClient') {
         }
     }
     dockerfile {
-        from('deephaven/cpp-client:local-build')
+        from(Docker.localImageName('cpp-client'))
         //
         // Build and install client.
         //
@@ -90,7 +90,7 @@ def testRClient = Docker.registerDockerTask(project, 'testRClient') {
         into layout.buildDirectory.dir('test-results')
     }
     dockerfile {
-        from('deephaven/r-client:local-build')
+        from(Docker.localImageName('r-client'))
         copyFile('r-tests.sh', "${prefix}/bin/rdeephaven")
         //
         // Setup for test run; we should be inheriting other env vars
@@ -118,7 +118,7 @@ def rClientDoc = Docker.registerDockerTask(project, 'rClientDoc') {
         into layout.projectDirectory.dir('rdeephaven/man')
     }
     dockerfile {
-        from('deephaven/r-client:local-build')
+        from(Docker.localImageName('r-client'))
         runCommand('''set -eux; \
                       rm -fr /out; \
                       mkdir -p /out; \
@@ -172,7 +172,7 @@ def rClientSite = Docker.registerDockerTask(project, 'rClientSite') {
         into layout.projectDirectory.dir('rdeephaven/docs')
     }
     dockerfile {
-        from('deephaven/r-client-doc:local-build')
+        from(Docker.localImageName('r-client-doc'))
         // We need the contents of 'man' to build the docsite
         copyFile('rdeephaven/man/**', "${prefix}/src/rdeephaven/man/")
         copyFile('rdeephaven/pkgdown/**', "${prefix}/src/rdeephaven/pkgdown/")

diff --git a/buildSrc/src/main/groovy/Docker.groovy b/buildSrc/src/main/groovy/Docker.groovy
@@ -213,7 +213,7 @@ class Docker {
     static TaskProvider<? extends Task> registerDockerTask(Project project, String taskName, Action<? super DockerTaskConfig> action) {
         // create instance, assign defaults
         DockerTaskConfig cfg = project.objects.newInstance(DockerTaskConfig);
-        cfg.imageName = "deephaven/${taskName.replaceAll(/\B[A-Z]/) { String str -> '-' + str }.toLowerCase()}:${LOCAL_BUILD_TAG}"
+        cfg.imageName = localImageName(taskName.replaceAll(/\B[A-Z]/) { String str -> '-' + str }.toLowerCase())
 
         // ask for more configuration
         action.execute(cfg)
@@ -582,14 +582,14 @@ class Docker {
             action.execute(buildImage)
             checkValidTwoPhase(buildImage)
             buildImage.target.set(intermediate)
-            buildImage.images.add("deephaven/${baseName}-${intermediate}:local-build".toString())
+            buildImage.images.add(localImageName("${baseName}-${intermediate}".toString()))
         }
 
         return registerDockerImage(project, "buildDocker-${baseName}") { DockerBuildImage buildImage ->
             action.execute(buildImage)
             checkValidTwoPhase(buildImage)
             buildImage.dependsOn(intermediateTask)
-            buildImage.images.add("deephaven/${baseName}:local-build".toString())
+            buildImage.images.add(localImageName(baseName))
         }
     }
 
@@ -643,37 +643,6 @@ class Docker {
         return makeImage;
     }
 
-    static TaskProvider<? extends Task> buildPyWheel(Project project, String taskName, String imgName, String sourcePath) {
-        project.evaluationDependsOn(registryProject('python'))
-        return registerDockerTask(project, taskName) { DockerTaskConfig config ->
-            config.copyIn { Sync sync ->
-                sync.from(sourcePath) { CopySpec copySpec ->
-                    copySpec.exclude 'build', 'dist'
-                    copySpec.into 'src'
-                }
-            }
-            config.imageName = "${imgName}:local-build"
-            config.dockerfile { Dockerfile action ->
-                // set up the container, env vars - things that aren't likely to change
-                action.from 'deephaven/python:local-build as sources'
-                action.arg 'DEEPHAVEN_VERSION'
-                action.environmentVariable 'DEEPHAVEN_VERSION', project.version.toString()
-                action.workingDir '/usr/src/app'
-                action.copyFile '/src', '.'
-                action.from 'sources as build'
-                action.runCommand '''set -eux; \\
-                      test -n "${DEEPHAVEN_VERSION}";\\
-                      python setup.py bdist_wheel'''
-            }
-            config.parentContainers = [ registryTask(project, 'python') ]
-            config.containerOutPath='/usr/src/app/dist'
-            config.copyOut { Sync sync ->
-                sync.into "build/wheel${taskName}"
-            }
-        }
-    }
-
-
     static TaskProvider<? extends DockerBuildImage> registryRegister(Project project) {
 
         String imageName = project.property('deephaven.registry.imageName')
@@ -765,7 +734,7 @@ class Docker {
         }
 
         def dockerfile = project.tasks.register('dockerfile', Dockerfile) { dockerfile ->
-            dockerfile.description = "Internal task: creates a dockerfile, to be (built) tagged as 'deephaven/${project.projectDir.name}:local-build'."
+            dockerfile.description = "Internal task: creates a dockerfile, to be (built) tagged as '${localImageName(project.projectDir.name)}'."
             dockerfile.from(imageId)
         }
 
@@ -784,10 +753,10 @@ class Docker {
             def dockerFileTask = dockerfile.get()
 
             build.group = 'Docker Registry'
-            build.description = "Creates 'deephaven/${project.projectDir.name}:local-build'."
+            build.description = "Creates '${localImageName(project.projectDir.name)}'."
             build.inputs.files dockerFileTask.outputs.files
             build.dockerFile.set dockerFileTask.outputs.files.singleFile
-            build.images.add("deephaven/${project.projectDir.name}:local-build".toString())
+            build.images.add(localImageName(project.projectDir.name))
             if (platform != null) {
                 build.platform.set platform
             }
@@ -802,6 +771,10 @@ class Docker {
         project.project(":docker-${name}").tasks.findByName('tagLocalBuild')
     }
 
+    static String localImageName(String name) {
+        return "deephaven/${name}:${LOCAL_BUILD_TAG}".toString()
+    }
+
     static FileCollection registryFiles(Project project, String name) {
         registryTask(project, name).outputs.files
     }

diff --git a/cpp-client/build.gradle b/cpp-client/build.gradle
@@ -77,7 +77,7 @@ def buildCppClientImage = Docker.registerDockerTask(project, 'cppClient') {
 
     dockerfile {
         // See comment at the beginning of this file for why we use this base image.
-        from('deephaven/cpp-clients-multi-base:local-build')
+        from(Docker.localImageName('cpp-clients-multi-base'))
         //
         // Build and install client.
         //
@@ -126,7 +126,7 @@ def testCppClient = Docker.registerDockerTask(project, 'testCppClient') {
         into layout.buildDirectory.dir('test-results')
     }
     dockerfile {
-        from('deephaven/cpp-client:local-build')
+        from(Docker.localImageName('cpp-client'))
         //
         // Setup for test run.
         //
@@ -152,7 +152,7 @@ def buildCppClientPyImage = Docker.registerDockerTask(project, 'cppClientPy') {
     }
 
     dockerfile {
-        from('deephaven/manylinux2014_x86_64:local-build')
+        from(Docker.localImageName('manylinux2014_x86_64'))
         runCommand("""mkdir -p \\
                         /out \\
                         ${prefix} \\

diff --git a/docker/registry/localstack/build.gradle b/docker/registry/localstack/build.gradle
@@ -0,0 +1,3 @@
+plugins {
+    id 'io.deephaven.project.register'
+}
diff --git a/docker/registry/localstack/gradle.properties b/docker/registry/localstack/gradle.properties
@@ -0,0 +1,3 @@
+io.deephaven.project.ProjectType=DOCKER_REGISTRY
+deephaven.registry.imageName=localstack/localstack:3
+deephaven.registry.imageId=localstack/localstack@sha256:54fcf172f6ff70909e1e26652c3bb4587282890aff0d02c20aa7695469476ac0
diff --git a/docker/registry/minio/build.gradle b/docker/registry/minio/build.gradle
@@ -0,0 +1,3 @@
+plugins {
+    id 'io.deephaven.project.register'
+}
diff --git a/docker/registry/minio/gradle.properties b/docker/registry/minio/gradle.properties
@@ -0,0 +1,3 @@
+io.deephaven.project.ProjectType=DOCKER_REGISTRY
+deephaven.registry.imageName=minio/minio:latest
+deephaven.registry.imageId=minio/minio@sha256:c97dbb0238dbd650ebe3f57dda68984993f466abad70d36c6e3ca306ceec3f58
diff --git a/docker/runtime-base/build.gradle b/docker/runtime-base/build.gradle
@@ -32,7 +32,7 @@ def buildDocker = Docker.registerDockerImage(project, 'buildDocker') {
 
     inputDir.set dockerContext
 
-    images.add('deephaven/runtime-base:local-build')
+    images.add(Docker.localImageName('runtime-base'))
 }
 
 assemble.dependsOn buildDocker
diff --git a/docker/server-jetty/build.gradle b/docker/server-jetty/build.gradle
@@ -67,7 +67,7 @@ def buildDockerClosure = { String base, String server ->
         dependsOn prepareDocker
         inputDir.set context
         inputs.files Docker.registryFiles(project, base)
-        buildArgs.put('BASE', "deephaven/${base}:local-build")
+        buildArgs.put('BASE', Docker.localImageName(base))
         buildArgs.put('SERVER', server)
         buildArgs.put('DEEPHAVEN_VERSION', project.version)
     }

diff --git a/docker/server/build.gradle b/docker/server/build.gradle
@@ -76,7 +76,7 @@ def buildDockerClosure = { String base, String server ->
         dependsOn prepareDocker
         inputDir.set context
         inputs.files Docker.registryFiles(project, base)
-        buildArgs.put('BASE', "deephaven/${base}:local-build")
+        buildArgs.put('BASE', Docker.localImageName(base))
         buildArgs.put('SERVER', server)
         buildArgs.put('DEEPHAVEN_VERSION', project.version)
     }

diff --git a/docker/web-plugin-packager/build.gradle b/docker/web-plugin-packager/build.gradle
@@ -21,7 +21,7 @@ Docker.registerDockerImage(project, 'buildDocker') {
     inputDir.set dockerContext
     inputs.files prepareDocker.get().outputs.files
     buildArgs.put('DEEPHAVEN_VERSION', project.version)
-    images.add('deephaven/web-plugin-packager:local-build')
+    images.add(Docker.localImageName('web-plugin-packager'))
 }
 
 assemble.dependsOn buildDocker
diff --git a/extensions/iceberg/s3/build.gradle b/extensions/iceberg/s3/build.gradle
@@ -3,6 +3,9 @@ plugins {
     id 'io.deephaven.project.register'
 }
 
+evaluationDependsOn Docker.registryProject('localstack')
+evaluationDependsOn Docker.registryProject('minio')
+
 description 'Iceberg: Support to read iceberg catalogs.'
 
 dependencies {
@@ -36,6 +39,10 @@ tasks.register('testOutOfBand', Test) {
     useJUnitPlatform {
         includeTags("testcontainers")
     }
-    systemProperty 'testcontainers.localstack.image', project.property('testcontainers.localstack.image')
-    systemProperty 'testcontainers.minio.image', project.property('testcontainers.minio.image')
+
+    dependsOn Docker.registryTask(project, 'localstack')
+    systemProperty 'testcontainers.localstack.image', Docker.localImageName('localstack')
+
+    dependsOn Docker.registryTask(project, 'minio')
+    systemProperty 'testcontainers.minio.image', Docker.localImageName('minio')
 }
diff --git a/extensions/iceberg/s3/gradle.properties b/extensions/iceberg/s3/gradle.properties
@@ -1,4 +1 @@
 io.deephaven.project.ProjectType=JAVA_PUBLIC
-
-testcontainers.localstack.image=localstack/localstack:3.1.0
-testcontainers.minio.image=minio/minio:RELEASE.2024-02-04T22-36-13Z
diff --git a/extensions/parquet/table/build.gradle b/extensions/parquet/table/build.gradle
@@ -4,6 +4,9 @@ plugins {
 }
 import io.deephaven.tools.docker.Architecture
 
+evaluationDependsOn Docker.registryProject('localstack')
+evaluationDependsOn Docker.registryProject('minio')
+
 description 'Parquet Table: Integrating Engine tables with Parquet'
 
 
@@ -74,5 +77,8 @@ if (Architecture.fromHost() == Architecture.AMD64) {
 
 TestTools.addEngineOutOfBandTest(project)
 
-testOutOfBand.systemProperty 'testcontainers.localstack.image', project.property('testcontainers.localstack.image')
-testOutOfBand.systemProperty 'testcontainers.minio.image', project.property('testcontainers.minio.image')
+testOutOfBand.dependsOn Docker.registryTask(project, 'localstack')
+testOutOfBand.systemProperty 'testcontainers.localstack.image', Docker.localImageName('localstack')
+
+testOutOfBand.dependsOn Docker.registryTask(project, 'minio')
+testOutOfBand.systemProperty 'testcontainers.minio.image', Docker.localImageName('minio')
diff --git a/extensions/parquet/table/gradle.properties b/extensions/parquet/table/gradle.properties
@@ -1,5 +1 @@
 io.deephaven.project.ProjectType=JAVA_PUBLIC
-
-# TODO(deephaven-core#5115): EPIC: Dependency management
-testcontainers.localstack.image=localstack/localstack:3.1.0
-testcontainers.minio.image=minio/minio:RELEASE.2024-02-04T22-36-13Z
diff --git a/extensions/s3/build.gradle b/extensions/s3/build.gradle
@@ -3,6 +3,9 @@ plugins {
     id 'io.deephaven.project.register'
 }
 
+evaluationDependsOn Docker.registryProject('localstack')
+evaluationDependsOn Docker.registryProject('minio')
+
 description 'Used to create a channel provider plugin for reading and writing files stored in S3.'
 
 dependencies {
@@ -47,7 +50,11 @@ tasks.register('testOutOfBand', Test) {
     useJUnitPlatform {
         includeTags("testcontainers")
     }
-    systemProperty 'testcontainers.localstack.image', project.property('testcontainers.localstack.image')
-    systemProperty 'testcontainers.minio.image', project.property('testcontainers.minio.image')
+
+    dependsOn Docker.registryTask(project, 'localstack')
+    systemProperty 'testcontainers.localstack.image', Docker.localImageName('localstack')
+
+    dependsOn Docker.registryTask(project, 'minio')
+    systemProperty 'testcontainers.minio.image', Docker.localImageName('minio')
 }
 
diff --git a/extensions/s3/gradle.properties b/extensions/s3/gradle.properties
@@ -1,5 +1 @@
 io.deephaven.project.ProjectType=JAVA_PUBLIC
-
-# TODO(deephaven-core#5115): EPIC: Dependency management
-testcontainers.localstack.image=localstack/localstack:3.1.0
-testcontainers.minio.image=minio/minio:RELEASE.2024-02-04T22-36-13Z
diff --git a/extensions/s3/src/test/java/io/deephaven/extensions/s3/testlib/SingletonContainers.java b/extensions/s3/src/test/java/io/deephaven/extensions/s3/testlib/SingletonContainers.java
@@ -26,7 +26,8 @@ public final class SingletonContainers {
 
     public static final class LocalStack {
         private static final LocalStackContainer LOCALSTACK_S3 =
-                new LocalStackContainer(DockerImageName.parse(System.getProperty("testcontainers.localstack.image")))
+                new LocalStackContainer(DockerImageName.parse(System.getProperty("testcontainers.localstack.image"))
+                        .asCompatibleSubstituteFor("localstack/localstack"), false)
                         .withServices(Service.S3);
         static {
             LOCALSTACK_S3.start();
@@ -59,7 +60,8 @@ public static final class MinIO {
         // comments in S3Instructions.
         // https://min.io/docs/minio/linux/reference/minio-server/settings/core.html#domain
         private static final MinIOContainer MINIO =
-                new MinIOContainer(DockerImageName.parse(System.getProperty("testcontainers.minio.image")))
+                new MinIOContainer(DockerImageName.parse(System.getProperty("testcontainers.minio.image"))
+                        .asCompatibleSubstituteFor("minio/minio"))
                         .withEnv("MINIO_DOMAIN", DockerClientFactory.instance().dockerHostIpAddress());
         static {
             MINIO.start();

diff --git a/go/build.gradle b/go/build.gradle
@@ -65,7 +65,7 @@ def testGoClient = Docker.registerDockerTask(project, 'testGoClient') {
         into layout.buildDirectory.dir('test-results')
     }
     dockerfile {
-        from('deephaven/go:local-build')
+        from(Docker.localImageName('go'))
         workingDir('/project')
         copyFile('go.*', '/project/')
         runCommand('''set -eux; \\

diff --git a/proto/proto-backplane-grpc/build.gradle b/proto/proto-backplane-grpc/build.gradle
@@ -67,7 +67,7 @@ TaskProvider<Task> generateProtobuf = Docker.registerDockerTask(project, 'genera
   platform = 'linux/amd64'
 
   containerOutPath = '/generated'
-  imageName = 'deephaven/proto-backplane-grpc:local-build'
+  imageName = Docker.localImageName('proto-backplane-grpc')
   copyOut {
     into('build/generated/source/proto/main')
   }

diff --git a/proto/raw-js-openapi/build.gradle b/proto/raw-js-openapi/build.gradle
@@ -39,7 +39,7 @@ def webpackSources = Docker.registerDockerTask(project, 'webpackSources') {
         from ('.npmrc')
     }
     parentContainers = [ Docker.registryTask(project, 'node') ]
-    imageName = 'deephaven/js-out:local-build'
+    imageName = Docker.localImageName('js-out')
     containerOutPath = '/usr/src/app/raw-js-openapi/build/js-out'
     copyOut {
         include 'dh-internal.js'

diff --git a/py/client-ticking/build.gradle b/py/client-ticking/build.gradle
@@ -49,7 +49,7 @@ def pyClientTickingWheel = { String pythonTag -> Docker.registerDockerTask(proje
         into layout.buildDirectory.dir("pyClientTickingWheel/${pythonTag}")
     }
     dockerfile {
-        from('deephaven/cpp-client-py:local-build')
+        from(Docker.localImageName('cpp-client-py'))
         runCommand("""mkdir -p \\
                         /out \\
                         '${prefix}/log' \\
@@ -101,7 +101,7 @@ def testCPythonClientTicking = { String pythonVersion, String image -> Docker.re
         }
     }
     dockerfile {
-        from("deephaven/${image}:local-build")
+        from(Docker.localImageName(image))
         runCommand(
                 '''set -eux; \
                    DNF=`type microdnf >/dev/null 2>&1 && echo 'microdnf --disableplugin=subscription-manager' || echo 'dnf -q'`; \

diff --git a/py/client/build.gradle b/py/client/build.gradle
@@ -106,7 +106,7 @@ def testPyClient = Docker.registerDockerTask(project, 'testPyClient') {
     containerDependencies.finalizedBy = deephavenDocker.endTask
     network = deephavenDocker.networkName.get()
     dockerfile {
-        from('deephaven/python:local-build')
+        from(Docker.localImageName('python'))
         copyFile('project', '/project')
         workingDir('/project')
         runCommand '''set -eux; \\