Add --all-tags copy of container image between different container registries

tests/jenkins/TestCopyContainer.groovy

@@ -30,6 +30,7 @@ class TestCopyContainer extends BuildPipelineTest {
         binding.setVariable('SOURCE_IMAGE', sourceImage)
         binding.setVariable('DESTINATION_IMAGE_REGISTRY', 'opensearchproject')
         binding.setVariable('DESTINATION_IMAGE', destinationImage)
+        binding.setVariable('RECURSIVE_COPY', true)
         helper.registerAllowedMethod('withAWS', [Map, Closure], null)
         super.setUp()
 
@@ -38,28 +39,29 @@ class TestCopyContainer extends BuildPipelineTest {
     @Test
     public void testCopyContainerDockerStagingToDockerProd_verifyShellCommand() {
         super.testPipeline("tests/jenkins/jobs/DockerCopy_Jenkinsfile")
-         def shellCommands = getCommandExecutions('sh', 'gcrane').findAll {
-            shCommand -> shCommand.contains('gcrane')
-         }
 
-         assertThat(shellCommands.size(), equalTo(1))
-         assertThat(shellCommands, hasItem("gcrane cp opensearchstaging/alpine:3.15.4 opensearchproject/alpine:3.15.4; docker logout".toString()))
+        String gcrane_str = '''\n        set +x\n\n        if [ false = true ]; then\n            echo \"Copying all image tags recursively from opensearchstaging/alpine to opensearchproject/alpine\"\n            for source_entry in `gcrane ls opensearchstaging/alpine`; do\n                image_tag=`echo $source_entry | cut -d/ -f3 | cut -d: -f2`\n                destination_entry=\"opensearchproject/alpine:$image_tag\"\n                gcrane cp $source_entry $destination_entry\n            done\n        else\n            echo \"Copying single image tag from opensearchstaging/alpine:3.15.4 to opensearchproject/alpine:3.15.4\"\n            gcrane cp opensearchstaging/alpine:3.15.4 opensearchproject/alpine:3.15.4\n        fi\n\n        docker logout\n        docker logout opensearchproject\n    '''
+        assertThat(getShellCommands('sh', 'gcrane'), hasItem(gcrane_str))
     }
 
-    def getCommandExecutions(methodName, command) {
-    def shCommands = helper.callStack.findAll {
-        call ->
+    @Test
+    public void testCopyContainerDockerStagingToDockerProdRecursive_verifyShellCommand() {
+        super.testPipeline("tests/jenkins/jobs/DockerCopyRecursive_Jenkinsfile")
+
+        String gcrane_recursive_str = '''\n        set +x\n\n        if [ true = true ]; then\n            echo \"Copying all image tags recursively from opensearchstaging/alpine to opensearchproject/alpine\"\n            for source_entry in `gcrane ls opensearchstaging/alpine`; do\n                image_tag=`echo $source_entry | cut -d/ -f3 | cut -d: -f2`\n                destination_entry=\"opensearchproject/alpine:$image_tag\"\n                gcrane cp $source_entry $destination_entry\n            done\n        else\n            echo \"Copying single image tag from opensearchstaging/alpine:3.15.4 to opensearchproject/alpine:3.15.4\"\n            gcrane cp opensearchstaging/alpine:3.15.4 opensearchproject/alpine:3.15.4\n        fi\n\n        docker logout\n        docker logout opensearchproject\n    '''
+
+        assertThat(getShellCommands('sh', 'gcrane'), hasItem(gcrane_recursive_str))
+    }
+
+    def getShellCommands(methodName, searchString) {
+        def shCommands = helper.callStack.findAll { call ->
             call.methodName == methodName
-    }.
-    collect {
-        call ->
+        }.collect { call ->
             callArgsToString(call)
-    }.findAll {
-        shCommand ->
-            shCommand.contains(command)
+        }.findAll { command ->
+            command.contains(searchString)
+        }
+        return shCommands
     }
 
-    return shCommands
-}
-
 }

tests/jenkins/jobs/DockerCopyRecursive_Jenkinsfile

@@ -0,0 +1,42 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+pipeline {
+    agent none
+    stages {
+        stage("Image Copy") {
+            stages {
+                stage('Parameters Check') {
+                    steps {
+                        script {
+                            currentBuild.description = "${SOURCE_IMAGE_REGISTRY}/${SOURCE_IMAGE} to ${DESTINATION_IMAGE_REGISTRY}/${DESTINATION_IMAGE}"
+                            if( SOURCE_IMAGE.isEmpty() || DESTINATION_IMAGE.isEmpty()) {
+                                        currentBuild.result = 'ABORTED'
+                                        error('Make sure all the parameters are passed in.')
+                            }
+                        }
+                    }
+                }
+                stage('Copy Image to ECR/DockerHub Recursively') {
+                    steps {
+                        script {
+                            copyContainer(
+                                recursiveCopy: "true",
+                                sourceImage: "${SOURCE_IMAGE}",
+                                sourceRegistry: "${SOURCE_IMAGE_REGISTRY}",
+                                destinationImage: "${DESTINATION_IMAGE}",
+                                destinationRegistry: "${DESTINATION_IMAGE_REGISTRY}"
+                            )
+                        }
+                    }
+                }
+            }    
+        }
+    }
+}

tests/jenkins/jobs/DockerCopyRecursive_Jenkinsfile.txt

@@ -0,0 +1,29 @@
+   DockerCopyRecursive_Jenkinsfile.run()
+      DockerCopyRecursive_Jenkinsfile.pipeline(groovy.lang.Closure)
+         DockerCopyRecursive_Jenkinsfile.echo(Executing on agent [label:none])
+         DockerCopyRecursive_Jenkinsfile.stage(Parameters Check, groovy.lang.Closure)
+            DockerCopyRecursive_Jenkinsfile.script(groovy.lang.Closure)
+         DockerCopyRecursive_Jenkinsfile.stage(Copy Image to ECR/DockerHub Recursively, groovy.lang.Closure)
+            DockerCopyRecursive_Jenkinsfile.script(groovy.lang.Closure)
+               DockerCopyRecursive_Jenkinsfile.copyContainer({recursiveCopy=true, sourceImage=alpine:3.15.4, sourceRegistry=opensearchstaging, destinationImage=alpine:3.15.4, destinationRegistry=opensearchproject})
+                  copyContainer.usernamePassword({credentialsId=jenkins-production-dockerhub-credential, usernameVariable=DOCKER_USERNAME, passwordVariable=DOCKER_PASSWORD})
+                  copyContainer.withCredentials([[DOCKER_USERNAME, DOCKER_PASSWORD]], groovy.lang.Closure)
+                     copyContainer.sh({returnStdout=true, script=set +x && echo DOCKER_PASSWORD | docker login --username DOCKER_USERNAME --password-stdin})
+                     copyContainer.sh(
+        set +x
+
+        if [ true = true ]; then
+            echo "Copying all image tags recursively from opensearchstaging/alpine to opensearchproject/alpine"
+            for source_entry in `gcrane ls opensearchstaging/alpine`; do
+                image_tag=`echo $source_entry | cut -d/ -f3 | cut -d: -f2`
+                destination_entry="opensearchproject/alpine:$image_tag"
+                gcrane cp $source_entry $destination_entry
+            done
+        else
+            echo "Copying single image tag from opensearchstaging/alpine:3.15.4 to opensearchproject/alpine:3.15.4"
+            gcrane cp opensearchstaging/alpine:3.15.4 opensearchproject/alpine:3.15.4
+        fi
+
+        docker logout
+        docker logout opensearchproject
+    )

tests/jenkins/jobs/DockerCopy_Jenkinsfile.txt

@@ -9,4 +9,21 @@
                   copyContainer.usernamePassword({credentialsId=jenkins-production-dockerhub-credential, usernameVariable=DOCKER_USERNAME, passwordVariable=DOCKER_PASSWORD})
                   copyContainer.withCredentials([[DOCKER_USERNAME, DOCKER_PASSWORD]], groovy.lang.Closure)
                      copyContainer.sh({returnStdout=true, script=set +x && echo DOCKER_PASSWORD | docker login --username DOCKER_USERNAME --password-stdin})
-                     copyContainer.sh(gcrane cp opensearchstaging/alpine:3.15.4 opensearchproject/alpine:3.15.4; docker logout)
+                     copyContainer.sh(
+        set +x
+
+        if [ false = true ]; then
+            echo "Copying all image tags recursively from opensearchstaging/alpine to opensearchproject/alpine"
+            for source_entry in `gcrane ls opensearchstaging/alpine`; do
+                image_tag=`echo $source_entry | cut -d/ -f3 | cut -d: -f2`
+                destination_entry="opensearchproject/alpine:$image_tag"
+                gcrane cp $source_entry $destination_entry
+            done
+        else
+            echo "Copying single image tag from opensearchstaging/alpine:3.15.4 to opensearchproject/alpine:3.15.4"
+            gcrane cp opensearchstaging/alpine:3.15.4 opensearchproject/alpine:3.15.4
+        fi
+
+        docker logout
+        docker logout opensearchproject
+    )

vars/copyContainer.groovy

@@ -9,35 +9,68 @@
 /**@
  * Copies a container from one docker registry to another
  *
- * @param args A map of the following parameters
- * @param args.sourceImage The Source Image name and tag <IMAGE_NAME>:<IMAGE_TAG> Eg: opensearch:1.3.2
- * @param args.sourceRegistry The source docker registry, currently supports 'DockerHub' or 'ECR'
- * @param args.destinationImage The Destination Image name and tag <IMAGE_NAME>:<IMAGE_TAG> Eg: opensearch:1.3.2
- * @param args.destinationRegistry The destination docker registry, currently supports 'DockerHub' or 'ECR'
+ * @param [Required] args A map of the following parameters
+ * @param [Required] args.recursiveCopy Copy all the tags of a sourceImage to destinationImage if 'true' and <IMAGE_TAG> is ignored in sourceImage/destinationImage, default to 'false'
+ * @param [Required] args.sourceImage The Source Image name and tag <IMAGE_NAME>:<IMAGE_TAG> Eg: opensearch:1.3.2
+ * @param [Required] args.sourceRegistry The source docker registry, currently supports 'DockerHub' or 'ECR'
+ * @param [Required] args.destinationImage The Destination Image name and tag <IMAGE_NAME>:<IMAGE_TAG> Eg: opensearch:1.3.2
+ * @param [Required] args.destinationRegistry The destination docker registry, currently supports 'DockerHub' or 'ECR'
  */
 void call(Map args = [:]) {
 
+    recursive_copy = args.recursiveCopy ?: false
+    source_image = args.sourceImage
+    source_image_no_tag = source_image.split(':')[0]
+    source_registry = args.sourceRegistry
+    destination_image = args.destinationImage
+    destination_image_no_tag = destination_image.split(':')[0]
+    destination_registry = args.destinationRegistry
 
     if (args.destinationRegistry == 'opensearchstaging' || args.destinationRegistry == 'opensearchproject') {
         def dockerJenkinsCredential = args.destinationRegistry == 'opensearchproject' ? "jenkins-production-dockerhub-credential" : "jenkins-staging-dockerhub-credential"
         withCredentials([usernamePassword(credentialsId: dockerJenkinsCredential, usernameVariable: 'DOCKER_USERNAME', passwordVariable: 'DOCKER_PASSWORD')]) {
             def dockerLogin = sh(returnStdout: true, script: "set +x && echo $DOCKER_PASSWORD | docker login --username $DOCKER_USERNAME --password-stdin").trim()
-            sh """gcrane cp ${args.sourceRegistry}/${args.sourceImage} ${args.destinationRegistry}/${args.destinationImage}; docker logout"""
+            gcraneCopy()
         }
     }
+
     if (args.destinationRegistry == 'public.ecr.aws/opensearchproject') {
         withCredentials([
             string(credentialsId: 'jenkins-artifact-promotion-role', variable: 'ARTIFACT_PROMOTION_ROLE_NAME'),
             string(credentialsId: 'jenkins-aws-production-account', variable: 'AWS_ACCOUNT_ARTIFACT')])
             {
                 withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') {
                     def ecrLogin = sh(returnStdout: true, script: "set +x && aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin ${args.destinationRegistry}").trim()
-                    sh """gcrane cp ${args.sourceRegistry}/${args.sourceImage} ${args.destinationRegistry}/${args.destinationImage}; docker logout ${args.destinationRegistry}"""
+                    gcraneCopy()
                 }
             }
     }
+
     if(args.destinationRegistry == 'public.ecr.aws/opensearchstaging') {
-            def ecrLogin = sh(returnStdout: true, script: "set +x && aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin ${args.destinationRegistry}").trim()
-            sh """gcrane cp ${args.sourceRegistry}/${args.sourceImage} ${args.destinationRegistry}/${args.destinationImage}; docker logout ${args.destinationRegistry}"""
+        def ecrLogin = sh(returnStdout: true, script: "set +x && aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin ${args.destinationRegistry}").trim()
+        gcraneCopy()
     }
+
+}
+
+def gcraneCopy() {
+    sh """
+        set +x
+
+        if [ ${recursive_copy} = true ]; then
+            echo "Copying all image tags recursively from ${source_registry}/${source_image_no_tag} to ${destination_registry}/${destination_image_no_tag}"
+            for source_entry in `gcrane ls ${source_registry}/${source_image_no_tag}`; do
+                image_tag=`echo \$source_entry | cut -d/ -f3 | cut -d: -f2`
+                destination_entry="${destination_registry}/${destination_image_no_tag}:\$image_tag"
+                gcrane cp \$source_entry \$destination_entry
+            done
+        else
+            echo "Copying single image tag from ${source_registry}/${source_image} to ${destination_registry}/${destination_image}"
+            gcrane cp ${source_registry}/${source_image} ${destination_registry}/${destination_image}
+        fi
+
+        docker logout
+        docker logout ${destination_registry}
+    """
+    return
 }