Retry buildah if incorrect arch created

iib/workers/tasks/build.py

@@ -1,4 +1,5 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
+import json
 import logging
 import os
 import shutil
@@ -66,7 +67,7 @@
 @retry(
     before_sleep=before_sleep_log(log, logging.WARNING),
     reraise=True,
-    retry=retry_if_exception_type(ExternalServiceError),
+    retry=(retry_if_exception_type(ExternalServiceError) | retry_if_exception_type(IIBError)),
     stop=stop_after_attempt(worker_config.iib_total_attempts),
     wait=wait_incrementing(
         start=worker_config.iib_retry_delay,
@@ -117,6 +118,16 @@ def _build_image(dockerfile_dir: str, dockerfile_name: str, request_id: int, arc
         {'cwd': dockerfile_dir},
         exc_msg=f'Failed to build the container image on the arch {arch}',
     )
+    log.debug('Verifying that %s was built with expected arch %s', destination, arch)
+    buildah_inspect = json.loads(run_cmd(['buildah', 'inspect', destination]))
+    archmap = {"amd64": "x86_64", "arm64": "aarch64", "s390x": "s390x", "ppc64le": "ppc64le"}
+    destination_arch = buildah_inspect['Docker']['config']['Labels']['architecture']
+
+    if destination_arch not in archmap.values() or destination_arch != archmap.get(arch, None):
+        log.warning("Wrong arch created for %s", destination)
+        exp_message = f'Wrong arch created, for image {destination} expected arch {arch}, \
+            found {destination_arch}'
+        raise IIBError(exp_message)
 
 
 def _cleanup() -> None:

tests/test_workers/test_tasks/test_build.py

@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 import copy
+import json
 import os
 import re
 import stat
@@ -16,32 +17,101 @@
 worker_config = get_worker_config()
 
 
-@pytest.mark.parametrize('arch', ('amd64', 'ppc64le', 's390x', 'spam'))
+@pytest.mark.parametrize(
+    'arch, destination_image_arch',
+    (('amd64', 'x86_64'), ('ppc64le', 'ppc64le'), ('s390x', 's390x'), ('arm64', 'aarch64')),
+)
 @mock.patch('iib.workers.tasks.build.run_cmd')
-def test_build_image(mock_run_cmd, arch):
+def test_build_image(mock_run_cmd, arch, destination_image_arch):
+    buildah_inspect_output = json.dumps(
+        {"Docker": {"config": {"Labels": {"architecture": destination_image_arch}}}}
+    )
+    mock_run_cmd.side_effect = [
+        None,
+        buildah_inspect_output,
+        None,
+        buildah_inspect_output,
+        None,
+        buildah_inspect_output,
+        None,
+        buildah_inspect_output,
+    ]
     build._build_image('/some/dir', 'some.Dockerfile', 3, arch)
 
-    mock_run_cmd.assert_called_once_with(
+    mock_run_cmd.assert_has_calls(
         [
-            'buildah',
-            'bud',
-            '--no-cache',
-            '--format',
-            'docker',
-            '--override-arch',
-            arch,
-            '--arch',
-            arch,
-            '-t',
-            f'iib-build:3-{arch}',
-            '-f',
-            '/some/dir/some.Dockerfile',
-        ],
-        {'cwd': '/some/dir'},
-        exc_msg=mock.ANY,
+            mock.call(
+                [
+                    'buildah',
+                    'bud',
+                    '--no-cache',
+                    '--format',
+                    'docker',
+                    '--override-arch',
+                    arch,
+                    '--arch',
+                    arch,
+                    '-t',
+                    f'iib-build:3-{arch}',
+                    '-f',
+                    '/some/dir/some.Dockerfile',
+                ],
+                {'cwd': '/some/dir'},
+                exc_msg=f"Failed to build the container image on the arch {arch}",
+            ),
+            mock.call(
+                [
+                    'buildah',
+                    'inspect',
+                    f'iib-build:3-{arch}',
+                ]
+            ),
+        ]
     )
 
 
+@mock.patch('iib.workers.tasks.build.run_cmd')
+def test_build_image_incorrect_arch(mock_run_cmd):
+    buildah_inspect_incorrect_arch = json.dumps(
+        {"Docker": {"config": {"Labels": {"architecture": "x86_64"}}}}
+    )
+    buildah_inspect_correct_arch = json.dumps(
+        {"Docker": {"config": {"Labels": {"architecture": "s390x"}}}}
+    )
+    mock_run_cmd.side_effect = [
+        None,
+        buildah_inspect_incorrect_arch,
+        None,
+        buildah_inspect_correct_arch,
+    ]
+    build._build_image('/some/dir', 'some.Dockerfile', 3, "s390x")
+    # build_image retried once, hence 2 buildah commands ran twice
+    assert mock_run_cmd.call_count == 4
+
+
+@mock.patch('iib.workers.tasks.build.run_cmd')
+def test_build_image_incorrect_arch_failure(mock_run_cmd):
+    buildah_inspect_incorrect_arch = json.dumps(
+        {"Docker": {"config": {"Labels": {"architecture": "x86_64"}}}}
+    )
+    mock_run_cmd.side_effect = [
+        None,
+        buildah_inspect_incorrect_arch,
+        None,
+        buildah_inspect_incorrect_arch,
+        None,
+        buildah_inspect_incorrect_arch,
+        None,
+        buildah_inspect_incorrect_arch,
+        None,
+        buildah_inspect_incorrect_arch,
+    ]
+    with pytest.raises(IIBError):
+        build._build_image('/some/dir', 'some.Dockerfile', 3, "s390x")
+        # build_image retried multiple times as the incorrect arch was created for image always
+        assert mock_run_cmd.call_count == 10
+
+
 @mock.patch(
     'iib.workers.tasks.build.run_cmd',
     side_effect=ExternalServiceError(