Add support to validate both docker and ECR as image source

jenkins/validate-artifacts/validate-artifacts.jenkinsfile

@@ -97,8 +97,8 @@ pipeline {
         )
         choice(
             name: 'DOCKER_SOURCE',
-            description: 'Enter the docker-source to pull the docker image, either DockerHub or ECR.',
-            choices: ["dockerhub", "ecr"]
+            description: 'Enter the docker-source to pull the docker image, either DockerHub or ECR or Both.',
+            choices: ["dockerhub", "ecr", "Both"]
         )
         choice(
             name: 'PROJECTS',
@@ -131,6 +131,10 @@ pipeline {
                         PROJECTS = "opensearch opensearch-dashboards"
                     }
 
+                    if (DOCKER_SOURCE == 'Both') {
+                        DOCKER_SOURCE = "dockerhub ecr"
+                    }
+
                     if (OPENSEARCH_DASHBOARDS_ARTIFACT_URL && OPENSEARCH_ARTIFACT_URL == "") {
                         currentBuild.result = 'ABORTED'
                         error("Provide OPENSEARCH_ARTIFACT_URL to validate")

src/run_validation.py

@@ -21,8 +21,16 @@ def main() -> int:
     logging.getLogger("urllib3").setLevel(logging.WARNING)
 
     with TemporaryDirectory() as work_dir:
-        test_result = ValidationTestRunner.dispatch(args, args.distribution, work_dir).run()
-        logging.info(f'final test_result = {test_result}')
+        if args.distribution == "docker":
+            docker_source = args.docker_source
+            for source in docker_source:
+                docker_args = args
+                docker_args.docker_source = source
+                test_result = ValidationTestRunner.dispatch(docker_args, args.distribution, work_dir).run()
+
+        else:
+            test_result = ValidationTestRunner.dispatch(args, args.distribution, work_dir).run()
+        logging.info(f'final test_result = {test_result}\n\n')
         return 0 if test_result else 1  # type: ignore
 
 

src/validation_workflow/validation_args.py

@@ -14,7 +14,6 @@
 
 class ValidationArgs:
     SUPPORTED_PLATFORMS = ["linux", "windows"]
-    DOCKER_SOURCE = ["dockerhub", "ecr"]
 
     def __init__(self) -> None:
         parser = argparse.ArgumentParser(
@@ -74,11 +73,11 @@ def __init__(self) -> None:
         )
         parser.add_argument(
             "--docker-source",
-            type=str,
+            nargs='+',
             required=False,
-            choices=self.DOCKER_SOURCE,
-            help="(optional) Where to pull the docker image from, either DockerHub or ECR\n",
-            default="dockerhub",
+            help="(optional) Where to pull the docker image from, either DockerHub or ECR or Both \n",
+            choices=["dockerhub", "ecr"],
+            default=["dockerhub"],
         )
         parser.add_argument(
             "-v",

tests/jenkins/TestValidateArtifacts.groovy

@@ -42,6 +42,7 @@ class TestValidateArtifacts extends BuildPipelineTest {
         binding.setVariable('ARCHITECTURE', "x64 arm64")
         binding.setVariable('PLATFORM', "linux windows")
         binding.setVariable('PROJECTS', "Both")
+        binding.setVariable('DOCKER_SOURCE', "Both")
         binding.setVariable('ARTIFACT_TYPE', "production")
         binding.setVariable('OPTIONAL_ARGS', "using-staging-artifact-only")
 
@@ -56,7 +57,7 @@ class TestValidateArtifacts extends BuildPipelineTest {
     public void testValidateArtifactsPipeline() {
         super.testPipeline("jenkins/validate-artifacts/validate-artifacts.jenkinsfile",
                 "tests/jenkins/jenkinsjob-regression-files/validate-artifacts/validate-artifacts.jenkinsfile")
-        assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('./validation.sh --version 2.3.0  --distribution docker  --arch x64 --projects opensearch opensearch-dashboards --docker-source dockerhub --os-build-number 6039 --osd-build-number 4104   --using-staging-artifact-only'))
+        assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('./validation.sh --version 2.3.0  --distribution docker  --arch x64 --projects opensearch opensearch-dashboards --docker-source dockerhub ecr --os-build-number 6039 --osd-build-number 4104   --using-staging-artifact-only'))
         assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('./validation.sh --version 2.3.0  --distribution tar --platform linux --arch x64 --projects opensearch opensearch-dashboards  --os-build-number 6039 --osd-build-number 4104 --artifact-type production'))
         assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('./validation.sh --version 2.3.0  --distribution yum --platform linux --arch x64 --projects opensearch opensearch-dashboards  --os-build-number 6039 --osd-build-number 4104 --artifact-type production'))
         assertThat(getCommandExecutions('sh', 'validation.sh'), hasItem('./validation.sh --version 2.3.0  --distribution rpm --platform linux --arch arm64 --projects opensearch opensearch-dashboards  --os-build-number 6039 --osd-build-number 4104 --artifact-type production'))

tests/jenkins/jenkinsjob-regression-files/validate-artifacts/validate-artifacts.jenkinsfile.txt

@@ -23,9 +23,9 @@
                   validate-artifacts.timeout({time=30}, groovy.lang.Closure)
                      validate-artifacts.node(Jenkins-Agent-AL2023-X64-C54xlarge-Single-Host, groovy.lang.Closure)
                         validate-artifacts.unstash(validate-artifacts-123)
-                        validate-artifacts.validateArtifacts({version=2.3.0, distribution=docker, arch=x64, os_build_number=6039, osd_build_number=4104, projects=opensearch opensearch-dashboards, docker_args=using-staging-artifact-only, docker_source=dockerhub})
+                        validate-artifacts.validateArtifacts({version=2.3.0, distribution=docker, arch=x64, os_build_number=6039, osd_build_number=4104, projects=opensearch opensearch-dashboards, docker_args=using-staging-artifact-only, docker_source=dockerhub ecr})
                            validateArtifacts.fileExists(/tmp/workspace/validation.sh)
-                           validateArtifacts.sh(./validation.sh --version 2.3.0  --distribution docker  --arch x64 --projects opensearch opensearch-dashboards --docker-source dockerhub --os-build-number 6039 --osd-build-number 4104   --using-staging-artifact-only)
+                           validateArtifacts.sh(./validation.sh --version 2.3.0  --distribution docker  --arch x64 --projects opensearch opensearch-dashboards --docker-source dockerhub ecr --os-build-number 6039 --osd-build-number 4104   --using-staging-artifact-only)
                         validate-artifacts.echo(Completed validation for docker-x64)
                         validate-artifacts.postCleanup()
                            postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true})
@@ -85,9 +85,9 @@
                   validate-artifacts.timeout({time=30}, groovy.lang.Closure)
                      validate-artifacts.node(Jenkins-Agent-AL2023-Arm64-C6g4xlarge-Single-Host, groovy.lang.Closure)
                         validate-artifacts.unstash(validate-artifacts-123)
-                        validate-artifacts.validateArtifacts({version=2.3.0, distribution=docker, arch=arm64, os_build_number=6039, osd_build_number=4104, projects=opensearch opensearch-dashboards, docker_args=using-staging-artifact-only, docker_source=dockerhub})
+                        validate-artifacts.validateArtifacts({version=2.3.0, distribution=docker, arch=arm64, os_build_number=6039, osd_build_number=4104, projects=opensearch opensearch-dashboards, docker_args=using-staging-artifact-only, docker_source=dockerhub ecr})
                            validateArtifacts.fileExists(/tmp/workspace/validation.sh)
-                           validateArtifacts.sh(./validation.sh --version 2.3.0  --distribution docker  --arch arm64 --projects opensearch opensearch-dashboards --docker-source dockerhub --os-build-number 6039 --osd-build-number 4104   --using-staging-artifact-only)
+                           validateArtifacts.sh(./validation.sh --version 2.3.0  --distribution docker  --arch arm64 --projects opensearch opensearch-dashboards --docker-source dockerhub ecr --os-build-number 6039 --osd-build-number 4104   --using-staging-artifact-only)
                         validate-artifacts.echo(Completed validation for docker-arm64)
                         validate-artifacts.postCleanup()
                            postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true})

tests/test_run_validation.py

@@ -36,3 +36,22 @@ def test_main(self, mock_tar: Mock, *mocks: Any) -> None:
 
         result = main()
         self.assertEqual(result, 0)
+
+    @patch("argparse._sys.argv",
+           ["run_validation.py", "--version", "2.1.0", "--distribution", "docker", "--docker_source",
+            "dockerhub,ecr"])
+    @patch('run_validation.ValidationTestRunner')
+    def test_main_docker(self, mock_docker: Mock, *mocks: Any) -> None:
+        mock_docker_instance = mock_docker.dispatch.return_value
+        mock_docker_instance.run.return_value = True
+
+        with patch('run_validation.ValidationArgs') as MockValidationArgs:
+            mock_args = Mock()
+            mock_args.docker_source = ["dockerhub", "ecr"]
+            mock_args.distribution = "docker"
+            MockValidationArgs.return_value = mock_args
+
+            result = main()
+            self.assertEqual(result, 0)
+            self.assertEqual(mock_docker.dispatch.call_count, 2)
+            self.assertTrue(mock_docker_instance.run.call_count, 2)

tests/tests_validation_workflow/test_validation_args.py

@@ -39,6 +39,13 @@ def test_docker_distribution(self) -> None:
         self.assertEqual(ValidationArgs().distribution, "docker")
         self.assertNotEqual(ValidationArgs().distribution, "yum")
 
+    @patch("argparse._sys.argv",
+           [VALIDATION_PY, "--version", "2.4.0", "--distribution", "docker", "--projects", "opensearch", "--docker-source", "dockerhub", "ecr",
+            "--using-staging-artifact-only"])
+    def test_docker_source(self) -> None:
+        self.assertEqual(ValidationArgs().distribution, "docker")
+        self.assertEqual(ValidationArgs().docker_source, ["dockerhub", "ecr"])
+
     @patch("argparse._sys.argv", [VALIDATION_PY, "--version", "1.3.6", "--platform", "linux"])
     def test_platform_default(self) -> None:
         self.assertEqual(ValidationArgs().platform, "linux")