Merge pull request #93 from bluesliverx/main

Allow specifying builders for specific platforms in global config

README.rst

@@ -188,6 +188,14 @@ they are used when put into the global configuration file:
   # the configuration in the buildrunner.yaml file.
   disable-multi-platform: true/false (defaults to false)
 
+  # Overrides the buildx builder used when doing multi-platform builds. Buildx
+  # does not provide the capability to auto-select the builder based on the platform
+  # and therefore this must be configured in buildrunner itself to perform builds
+  # across multiple builders for different platforms. Any platform not specified
+  # here will use the default configured buildx builder.
+  platform-builders:
+    platform1: builder1
+
 Configuration Locations
 -----------------------
 

buildrunner/__init__.py

@@ -587,6 +587,7 @@ def run(self):  # pylint: disable=too-many-statements,too-many-branches,too-many
                 keep_images=not self.cleanup_images,
                 temp_dir=self.global_config.get_temp_dir(),
                 disable_multi_platform=self.disable_multi_platform,
+                platform_builders=self.global_config.get('platform-builders'),
             ) as multi_platform:
                 if not os.path.exists(self.build_results_dir):
                     # create a new results dir

buildrunner/docker/multiplatform_image_builder.py

@@ -11,7 +11,7 @@
 from platform import machine, system
 import shutil
 import tempfile
-from typing import List, Optional
+from typing import Dict, List, Optional
 
 import python_on_whales
 from python_on_whales import docker
@@ -125,13 +125,15 @@ def __init__(
             keep_images: bool = False,
             temp_dir: str = os.getcwd(),
             disable_multi_platform: bool = False,
+            platform_builders: Optional[Dict[str, str]] = None,
     ):
         self._docker_registry = docker_registry
         self._mp_registry_info = None
         self._use_local_registry = use_local_registry
         self._keep_images = keep_images
         self._temp_dir = temp_dir
         self._disable_multi_platform = disable_multi_platform
+        self._platform_builders = platform_builders
 
         # key is destination image name, value is list of built images
         self._intermediate_built_images = {}
@@ -240,7 +242,9 @@ def _build_with_inject(
             push: bool,
             path: str,
             file: str,
-            build_args: dict,) -> None:
+            build_args: dict,
+            builder: Optional[str],
+    ) -> None:
 
         if not path or not os.path.isdir(path):
             logger.warning(f"Failed to inject {inject} for {tagged_names} since path {path} isn't a directory.")
@@ -271,7 +275,9 @@ def _build_with_inject(
                 platforms=[platform],
                 push=push,
                 file=file,
-                build_args=build_args)
+                builder=builder,
+                build_args=build_args
+            )
 
     # pylint: disable=too-many-arguments
     @retry(python_on_whales.exceptions.DockerException,
@@ -310,7 +316,8 @@ def _build_single_image(
             f"'{file}'({os.path.exists(f'{file}')}) does not exist!"
 
         tagged_names = [f"{name}:{tag}" for tag in tags]
-        logger.debug(f"Building {tagged_names} for {platform}")
+        builder = self._platform_builders.get(platform) if self._platform_builders else None
+        logger.debug(f"Building {tagged_names} for {platform} with {builder or 'default'} builder")
 
         if inject and isinstance(inject, dict):
             self._build_with_inject(
@@ -320,15 +327,19 @@ def _build_single_image(
                 push=push,
                 path=path,
                 file=file,
-                build_args=build_args)
+                build_args=build_args,
+                builder=builder,
+            )
         else:
             docker.buildx.build(
                 path,
                 tags=tagged_names,
                 platforms=[platform],
                 push=push,
                 file=file,
-                build_args=build_args)
+                build_args=build_args,
+                builder=builder,
+            )
 
         # Check that the images were built and in the registry
         # Docker search is not currently implemented in python-on-wheels
@@ -433,77 +444,58 @@ def get_path(file):
 
         # Updates name to be compatible with docker
         image_prefix = "buildrunner-mp"
-        santized_name = f"{image_prefix}-{mp_image_name.replace('/', '-').replace(':', '-')}"
-        base_image_name = f"{self._mp_registry_info.ip_addr}:{self._mp_registry_info.port}/{santized_name}"
+        sanitized_name = f"{image_prefix}-{mp_image_name.replace('/', '-').replace(':', '-')}"
+        base_image_name = f"{self._mp_registry_info.ip_addr}:{self._mp_registry_info.port}/{sanitized_name}"
 
         # Keeps track of the built images {name: [ImageInfo(image_names)]]}
         manager = Manager()
         self._intermediate_built_images[mp_image_name] = manager.list()
         line = "-----------------------------------------------------------------"
 
         if self._disable_multi_platform:
-            platform = self.get_single_platform_to_build(platforms)
-            platform_image_name = f"{base_image_name}-{platform.replace('/', '-')}"
+            platforms = [self.get_single_platform_to_build(platforms)]
             print(f"{line}\n"
                   f"Note: Disabling multi-platform build, "
                   "this will only build a single-platform image.\n"
-                  f"image: {santized_name} platform:{platform}\n"
+                  f"image: {sanitized_name} platform:{platforms[0]}\n"
                   f"{line}")
-            self._build_single_image(
-                platform_image_name,
-                platform,
-                push,
-                path,
-                dockerfile,
-                tags,
-                build_args,
-                mp_image_name,
-                inject
-            )
         else:
-            processes = []
             print(f"{line}\n"
                   f"Note: Building multi-platform images can take a long time, please be patient.\n"
                   "If you are running this locally, you can speed this up by using the "
                   "'--disable-multi-platform' CLI flag "
                   "or set the 'disable-multi-platform' flag in the global config file.\n"
                   f"{line}")
-            for platform in platforms:
-                platform_image_name = f"{base_image_name}-{platform.replace('/', '-')}"
-                logger.debug(f"Building {platform_image_name} for {platform}")
-                if do_multiprocessing:
-                    processes.append(Process(
-                        target=self._build_single_image,
-                        args=(
-                            platform_image_name,
-                            platform,
-                            push,
-                            path,
-                            dockerfile,
-                            tags,
-                            build_args,
-                            mp_image_name,
-                            inject
-                        )
-                    ))
-                else:
-                    self._build_single_image(
-                        platform_image_name,
-                        platform,
-                        push,
-                        path,
-                        dockerfile,
-                        tags,
-                        build_args,
-                        mp_image_name,
-                        inject
-                    )
-
-            for proc in processes:
-                proc.start()
-
-            for proc in processes:
-                proc.join()
+
+        processes = []
+        for platform in platforms:
+            platform_image_name = f"{base_image_name}-{platform.replace('/', '-')}"
+            logger.debug(f"Building {platform_image_name} for {platform}")
+            process = Process(
+                target=self._build_single_image,
+                args=(
+                    platform_image_name,
+                    platform,
+                    push,
+                    path,
+                    dockerfile,
+                    tags,
+                    build_args,
+                    mp_image_name,
+                    inject
+                )
+            )
+            if do_multiprocessing:
+                processes.append(process)
+            else:
+                process.start()
+                process.join()
+
+        # Start and join processes in parallel (if not already done)
+        for proc in processes:
+            proc.start()
+        for proc in processes:
+            proc.join()
 
         if cleanup_dockerfile and dockerfile and os.path.exists(dockerfile):
             os.remove(dockerfile)

buildrunner/validation/config.py

@@ -48,6 +48,7 @@ class SSHKey(BaseModel, extra='forbid'):
     docker_registry: Optional[str] = Field(alias='docker-registry', default=None)
     temp_dir: Optional[str] = Field(alias='temp-dir', default=None)
     disable_multi_platform: Optional[bool] = Field(alias='disable-multi-platform', default=None)
+    platform_builders: Optional[Dict[str, str]] = Field(alias='platform-builders', default=None)
 
     @field_validator('steps')
     @classmethod