Adds docker registry projects for localstack and minio

Integrations/build.gradle

@@ -78,13 +78,13 @@ def runInDocker = { String name, String sourcePath, List<String> command, Closur
         }
         parentContainers = [project(':docker-server-jetty').tasks.findByName('buildDocker-server-jetty')] // deephaven/server-jetty
 
-        imageName = 'deephaven/py-integrations:local-build'
+        imageName = Docker.localImageName('py-integrations')
 
         addConfig(it)
 
         dockerfile {
             // set up the container, env vars - things that aren't likely to change
-            from 'deephaven/server-jetty:local-build'
+            from Docker.localImageName('server-jetty')
             runCommand '''set -eux; \\
                       pip3 install unittest-xml-reporting==3.0.4; \\
                       mkdir -p /out/report; \\

R/build.gradle

@@ -46,7 +46,7 @@ def buildRClient = Docker.registerDockerTask(project, 'rClient') {
         }
     }
     dockerfile {
-        from('deephaven/cpp-client:local-build')
+        from(Docker.localImageName('cpp-client'))
         //
         // Build and install client.
         //
@@ -90,7 +90,7 @@ def testRClient = Docker.registerDockerTask(project, 'testRClient') {
         into layout.buildDirectory.dir('test-results')
     }
     dockerfile {
-        from('deephaven/r-client:local-build')
+        from(Docker.localImageName('r-client'))
         copyFile('r-tests.sh', "${prefix}/bin/rdeephaven")
         //
         // Setup for test run; we should be inheriting other env vars
@@ -118,7 +118,7 @@ def rClientDoc = Docker.registerDockerTask(project, 'rClientDoc') {
         into layout.projectDirectory.dir('rdeephaven/man')
     }
     dockerfile {
-        from('deephaven/r-client:local-build')
+        from(Docker.localImageName('r-client'))
         runCommand('''set -eux; \
                       rm -fr /out; \
                       mkdir -p /out; \
@@ -172,7 +172,7 @@ def rClientSite = Docker.registerDockerTask(project, 'rClientSite') {
         into layout.projectDirectory.dir('rdeephaven/docs')
     }
     dockerfile {
-        from('deephaven/r-client-doc:local-build')
+        from(Docker.localImageName('r-client-doc'))
         // We need the contents of 'man' to build the docsite
         copyFile('rdeephaven/man/**', "${prefix}/src/rdeephaven/man/")
         copyFile('rdeephaven/pkgdown/**', "${prefix}/src/rdeephaven/pkgdown/")

buildSrc/src/main/groovy/Docker.groovy

@@ -213,7 +213,7 @@ class Docker {
     static TaskProvider<? extends Task> registerDockerTask(Project project, String taskName, Action<? super DockerTaskConfig> action) {
         // create instance, assign defaults
         DockerTaskConfig cfg = project.objects.newInstance(DockerTaskConfig);
-        cfg.imageName = "deephaven/${taskName.replaceAll(/\B[A-Z]/) { String str -> '-' + str }.toLowerCase()}:${LOCAL_BUILD_TAG}"
+        cfg.imageName = localImageName(taskName.replaceAll(/\B[A-Z]/) { String str -> '-' + str }.toLowerCase())
 
         // ask for more configuration
         action.execute(cfg)
@@ -582,14 +582,14 @@ class Docker {
             action.execute(buildImage)
             checkValidTwoPhase(buildImage)
             buildImage.target.set(intermediate)
-            buildImage.images.add("deephaven/${baseName}-${intermediate}:local-build".toString())
+            buildImage.images.add(localImageName("${baseName}-${intermediate}".toString()))
         }
 
         return registerDockerImage(project, "buildDocker-${baseName}") { DockerBuildImage buildImage ->
             action.execute(buildImage)
             checkValidTwoPhase(buildImage)
             buildImage.dependsOn(intermediateTask)
-            buildImage.images.add("deephaven/${baseName}:local-build".toString())
+            buildImage.images.add(localImageName(baseName))
         }
     }
 
@@ -652,10 +652,10 @@ class Docker {
                     copySpec.into 'src'
                 }
             }
-            config.imageName = "${imgName}:local-build"
+            config.imageName = "${imgName}:${LOCAL_BUILD_TAG}"
             config.dockerfile { Dockerfile action ->
                 // set up the container, env vars - things that aren't likely to change
-                action.from 'deephaven/python:local-build as sources'
+                action.from "${localImageName('python')} as sources".toString()
                 action.arg 'DEEPHAVEN_VERSION'
                 action.environmentVariable 'DEEPHAVEN_VERSION', project.version.toString()
                 action.workingDir '/usr/src/app'
@@ -765,7 +765,7 @@ class Docker {
         }
 
         def dockerfile = project.tasks.register('dockerfile', Dockerfile) { dockerfile ->
-            dockerfile.description = "Internal task: creates a dockerfile, to be (built) tagged as 'deephaven/${project.projectDir.name}:local-build'."
+            dockerfile.description = "Internal task: creates a dockerfile, to be (built) tagged as '${localImageName(project.projectDir.name)}'."
             dockerfile.from(imageId)
         }
 
@@ -784,10 +784,10 @@ class Docker {
             def dockerFileTask = dockerfile.get()
 
             build.group = 'Docker Registry'
-            build.description = "Creates 'deephaven/${project.projectDir.name}:local-build'."
+            build.description = "Creates '${localImageName(project.projectDir.name)}'."
             build.inputs.files dockerFileTask.outputs.files
             build.dockerFile.set dockerFileTask.outputs.files.singleFile
-            build.images.add("deephaven/${project.projectDir.name}:local-build".toString())
+            build.images.add(localImageName(project.projectDir.name))
             if (platform != null) {
                 build.platform.set platform
             }
@@ -802,6 +802,10 @@ class Docker {
         project.project(":docker-${name}").tasks.findByName('tagLocalBuild')
     }
 
+    static String localImageName(String name) {
+        return "deephaven/${name}:${LOCAL_BUILD_TAG}".toString()
+    }
+
     static FileCollection registryFiles(Project project, String name) {
         registryTask(project, name).outputs.files
     }

cpp-client/build.gradle

@@ -77,7 +77,7 @@ def buildCppClientImage = Docker.registerDockerTask(project, 'cppClient') {
 
     dockerfile {
         // See comment at the beginning of this file for why we use this base image.
-        from('deephaven/cpp-clients-multi-base:local-build')
+        from(Docker.localImageName('cpp-clients-multi-base'))
         //
         // Build and install client.
         //
@@ -126,7 +126,7 @@ def testCppClient = Docker.registerDockerTask(project, 'testCppClient') {
         into layout.buildDirectory.dir('test-results')
     }
     dockerfile {
-        from('deephaven/cpp-client:local-build')
+        from(Docker.localImageName('cpp-client'))
         //
         // Setup for test run.
         //
@@ -152,7 +152,7 @@ def buildCppClientPyImage = Docker.registerDockerTask(project, 'cppClientPy') {
     }
 
     dockerfile {
-        from('deephaven/manylinux2014_x86_64:local-build')
+        from(Docker.localImageName('manylinux2014_x86_64'))
         runCommand("""mkdir -p \\
                         /out \\
                         ${prefix} \\

docker/registry/localstack/build.gradle

@@ -0,0 +1,3 @@
+plugins {
+    id 'io.deephaven.project.register'
+}

docker/registry/localstack/gradle.properties

@@ -0,0 +1,3 @@
+io.deephaven.project.ProjectType=DOCKER_REGISTRY
+deephaven.registry.imageName=localstack/localstack:3
+deephaven.registry.imageId=localstack/localstack@sha256:54fcf172f6ff70909e1e26652c3bb4587282890aff0d02c20aa7695469476ac0

docker/registry/minio/build.gradle

@@ -0,0 +1,3 @@
+plugins {
+    id 'io.deephaven.project.register'
+}

docker/registry/minio/gradle.properties

@@ -0,0 +1,3 @@
+io.deephaven.project.ProjectType=DOCKER_REGISTRY
+deephaven.registry.imageName=minio/minio:latest
+deephaven.registry.imageId=minio/minio@sha256:c97dbb0238dbd650ebe3f57dda68984993f466abad70d36c6e3ca306ceec3f58

docker/runtime-base/build.gradle

@@ -32,7 +32,7 @@ def buildDocker = Docker.registerDockerImage(project, 'buildDocker') {
 
     inputDir.set dockerContext
 
-    images.add('deephaven/runtime-base:local-build')
+    images.add(Docker.localImageName('runtime-base'))
 }
 
 assemble.dependsOn buildDocker

docker/server-jetty/build.gradle

@@ -67,7 +67,7 @@ def buildDockerClosure = { String base, String server ->
         dependsOn prepareDocker
         inputDir.set context
         inputs.files Docker.registryFiles(project, base)
-        buildArgs.put('BASE', "deephaven/${base}:local-build")
+        buildArgs.put('BASE', Docker.localImageName(base))
         buildArgs.put('SERVER', server)
         buildArgs.put('DEEPHAVEN_VERSION', project.version)
     }

docker/server/build.gradle

@@ -76,7 +76,7 @@ def buildDockerClosure = { String base, String server ->
         dependsOn prepareDocker
         inputDir.set context
         inputs.files Docker.registryFiles(project, base)
-        buildArgs.put('BASE', "deephaven/${base}:local-build")
+        buildArgs.put('BASE', Docker.localImageName(base))
         buildArgs.put('SERVER', server)
         buildArgs.put('DEEPHAVEN_VERSION', project.version)
     }

docker/web-plugin-packager/build.gradle

@@ -21,7 +21,7 @@ Docker.registerDockerImage(project, 'buildDocker') {
     inputDir.set dockerContext
     inputs.files prepareDocker.get().outputs.files
     buildArgs.put('DEEPHAVEN_VERSION', project.version)
-    images.add('deephaven/web-plugin-packager:local-build')
+    images.add(Docker.localImageName('web-plugin-packager'))
 }
 
 assemble.dependsOn buildDocker

extensions/iceberg/s3/build.gradle

@@ -3,6 +3,9 @@ plugins {
     id 'io.deephaven.project.register'
 }
 
+evaluationDependsOn Docker.registryProject('localstack')
+evaluationDependsOn Docker.registryProject('minio')
+
 description 'Iceberg: Support to read iceberg catalogs.'
 
 dependencies {
@@ -36,6 +39,10 @@ tasks.register('testOutOfBand', Test) {
     useJUnitPlatform {
         includeTags("testcontainers")
     }
-    systemProperty 'testcontainers.localstack.image', project.property('testcontainers.localstack.image')
-    systemProperty 'testcontainers.minio.image', project.property('testcontainers.minio.image')
+
+    dependsOn Docker.registryTask(project, 'localstack')
+    systemProperty 'testcontainers.localstack.image', Docker.localImageName('localstack')
+
+    dependsOn Docker.registryTask(project, 'minio')
+    systemProperty 'testcontainers.minio.image', Docker.localImageName('minio')
 }

extensions/iceberg/s3/gradle.properties

@@ -1,4 +1 @@
 io.deephaven.project.ProjectType=JAVA_PUBLIC
-
-testcontainers.localstack.image=localstack/localstack:3.1.0
-testcontainers.minio.image=minio/minio:RELEASE.2024-02-04T22-36-13Z

extensions/parquet/table/build.gradle

@@ -4,6 +4,9 @@ plugins {
 }
 import io.deephaven.tools.docker.Architecture
 
+evaluationDependsOn Docker.registryProject('localstack')
+evaluationDependsOn Docker.registryProject('minio')
+
 description 'Parquet Table: Integrating Engine tables with Parquet'
 
 
@@ -74,5 +77,8 @@ if (Architecture.fromHost() == Architecture.AMD64) {
 
 TestTools.addEngineOutOfBandTest(project)
 
-testOutOfBand.systemProperty 'testcontainers.localstack.image', project.property('testcontainers.localstack.image')
-testOutOfBand.systemProperty 'testcontainers.minio.image', project.property('testcontainers.minio.image')
+testOutOfBand.dependsOn Docker.registryTask(project, 'localstack')
+testOutOfBand.systemProperty 'testcontainers.localstack.image', Docker.localImageName('localstack')
+
+testOutOfBand.dependsOn Docker.registryTask(project, 'minio')
+testOutOfBand.systemProperty 'testcontainers.minio.image', Docker.localImageName('minio')

extensions/parquet/table/gradle.properties

@@ -1,5 +1 @@
 io.deephaven.project.ProjectType=JAVA_PUBLIC
-
-# TODO(deephaven-core#5115): EPIC: Dependency management
-testcontainers.localstack.image=localstack/localstack:3.1.0
-testcontainers.minio.image=minio/minio:RELEASE.2024-02-04T22-36-13Z

extensions/s3/build.gradle

@@ -3,6 +3,9 @@ plugins {
     id 'io.deephaven.project.register'
 }
 
+evaluationDependsOn Docker.registryProject('localstack')
+evaluationDependsOn Docker.registryProject('minio')
+
 description 'Used to create a channel provider plugin for reading and writing files stored in S3.'
 
 dependencies {
@@ -47,7 +50,11 @@ tasks.register('testOutOfBand', Test) {
     useJUnitPlatform {
         includeTags("testcontainers")
     }
-    systemProperty 'testcontainers.localstack.image', project.property('testcontainers.localstack.image')
-    systemProperty 'testcontainers.minio.image', project.property('testcontainers.minio.image')
+
+    dependsOn Docker.registryTask(project, 'localstack')
+    systemProperty 'testcontainers.localstack.image', Docker.localImageName('localstack')
+
+    dependsOn Docker.registryTask(project, 'minio')
+    systemProperty 'testcontainers.minio.image', Docker.localImageName('minio')
 }
 

extensions/s3/gradle.properties

@@ -1,5 +1 @@
 io.deephaven.project.ProjectType=JAVA_PUBLIC
-
-# TODO(deephaven-core#5115): EPIC: Dependency management
-testcontainers.localstack.image=localstack/localstack:3.1.0
-testcontainers.minio.image=minio/minio:RELEASE.2024-02-04T22-36-13Z

extensions/s3/src/test/java/io/deephaven/extensions/s3/testlib/SingletonContainers.java

@@ -26,7 +26,8 @@ public final class SingletonContainers {
 
     public static final class LocalStack {
         private static final LocalStackContainer LOCALSTACK_S3 =
-                new LocalStackContainer(DockerImageName.parse(System.getProperty("testcontainers.localstack.image")))
+                new LocalStackContainer(DockerImageName.parse(System.getProperty("testcontainers.localstack.image"))
+                        .asCompatibleSubstituteFor("localstack/localstack"), false)
                         .withServices(Service.S3);
         static {
             LOCALSTACK_S3.start();
@@ -59,7 +60,8 @@ public static final class MinIO {
         // comments in S3Instructions.
         // https://min.io/docs/minio/linux/reference/minio-server/settings/core.html#domain
         private static final MinIOContainer MINIO =
-                new MinIOContainer(DockerImageName.parse(System.getProperty("testcontainers.minio.image")))
+                new MinIOContainer(DockerImageName.parse(System.getProperty("testcontainers.minio.image"))
+                        .asCompatibleSubstituteFor("minio/minio"))
                         .withEnv("MINIO_DOMAIN", DockerClientFactory.instance().dockerHostIpAddress());
         static {
             MINIO.start();

go/build.gradle

@@ -65,7 +65,7 @@ def testGoClient = Docker.registerDockerTask(project, 'testGoClient') {
         into layout.buildDirectory.dir('test-results')
     }
     dockerfile {
-        from('deephaven/go:local-build')
+        from(Docker.localImageName('go'))
         workingDir('/project')
         copyFile('go.*', '/project/')
         runCommand('''set -eux; \\

proto/proto-backplane-grpc/build.gradle

@@ -67,7 +67,7 @@ TaskProvider<Task> generateProtobuf = Docker.registerDockerTask(project, 'genera
   platform = 'linux/amd64'
 
   containerOutPath = '/generated'
-  imageName = 'deephaven/proto-backplane-grpc:local-build'
+  imageName = Docker.localImageName('proto-backplane-grpc')
   copyOut {
     into('build/generated/source/proto/main')
   }

proto/raw-js-openapi/build.gradle

@@ -39,7 +39,7 @@ def webpackSources = Docker.registerDockerTask(project, 'webpackSources') {
         from ('.npmrc')
     }
     parentContainers = [ Docker.registryTask(project, 'node') ]
-    imageName = 'deephaven/js-out:local-build'
+    imageName = Docker.localImageName('js-out')
     containerOutPath = '/usr/src/app/raw-js-openapi/build/js-out'
     copyOut {
         include 'dh-internal.js'

py/client-ticking/build.gradle

@@ -49,7 +49,7 @@ def pyClientTickingWheel = { String pythonTag -> Docker.registerDockerTask(proje
         into layout.buildDirectory.dir("pyClientTickingWheel/${pythonTag}")
     }
     dockerfile {
-        from('deephaven/cpp-client-py:local-build')
+        from(Docker.localImageName('cpp-client-py'))
         runCommand("""mkdir -p \\
                         /out \\
                         '${prefix}/log' \\
@@ -101,7 +101,7 @@ def testCPythonClientTicking = { String pythonVersion, String image -> Docker.re
         }
     }
     dockerfile {
-        from("deephaven/${image}:local-build")
+        from(Docker.localImageName(image))
         runCommand(
                 '''set -eux; \
                    DNF=`type microdnf >/dev/null 2>&1 && echo 'microdnf --disableplugin=subscription-manager' || echo 'dnf -q'`; \

py/client/build.gradle

@@ -106,7 +106,7 @@ def testPyClient = Docker.registerDockerTask(project, 'testPyClient') {
     containerDependencies.finalizedBy = deephavenDocker.endTask
     network = deephavenDocker.networkName.get()
     dockerfile {
-        from('deephaven/python:local-build')
+        from(Docker.localImageName('python'))
         copyFile('project', '/project')
         workingDir('/project')
         runCommand '''set -eux; \\

py/jpy-integration/build.gradle

@@ -142,7 +142,7 @@ Closure<TaskProvider<Task>> gradleTestInDocker = { String taskName, SourceSet so
     parentContainers = [project(':docker-runtime-base').tasks.findByName('buildDocker')] // deephaven/runtime-base
     dockerfile {
       // base image with default java, python, wheels
-      from 'deephaven/runtime-base:local-build'
+      from Docker.localImageName('runtime-base')
 
       // set up the project
       copyFile 'project', '/project'
@@ -172,9 +172,9 @@ Closure<TaskProvider<Task>> javaMainInDocker = { String taskName, String javaMai
       }
     }
     parentContainers = [project(':docker-runtime-base').tasks.findByName('buildDocker')] // deephaven/runtime-base
-    imageName = 'deephaven/jpy-integration-java-to-python-tests:local-build'
+    imageName = Docker.localImageName('jpy-integration-java-to-python-tests')
     dockerfile {
-      from 'deephaven/runtime-base:local-build'
+      from Docker.localImageName('runtime-base')
 
       copyFile 'classpath', '/classpath'
     }
@@ -222,10 +222,10 @@ Closure<TaskProvider<Task>> pyExec = { String taskName, List<String> command, bo
       }
     }
     parentContainers = [project(':docker-runtime-base').tasks.findByName('buildDocker')] // deephaven/runtime-base
-    imageName = 'deephaven/jpy-integration-python-to-java-tests:local-build'
+    imageName = Docker.localImageName('jpy-integration-python-to-java-tests')
     dockerfile {
       // set up the container, env vars - things that aren't likely to change
-      from 'deephaven/runtime-base:local-build'
+      from Docker.localImageName('runtime-base')
       runCommand '''set -eux; \\
                       pip3 install unittest-xml-reporting==3.0.4;\\
                       mkdir /out;'''