From b4f7b24bdb363cea6d2b5a0b5928077dc1ddb85d Mon Sep 17 00:00:00 2001 From: saville Date: Wed, 8 Nov 2023 16:58:24 -0700 Subject: [PATCH 1/2] Use load instead of push to support remote builders --- .../docker/multiplatform_image_builder.py | 12 ++++--- tests/test_multiplatform.py | 31 +++++++++++++------ 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/buildrunner/docker/multiplatform_image_builder.py b/buildrunner/docker/multiplatform_image_builder.py index c162ad28..d169896e 100644 --- a/buildrunner/docker/multiplatform_image_builder.py +++ b/buildrunner/docker/multiplatform_image_builder.py @@ -11,6 +11,7 @@ from platform import machine, system import shutil import tempfile +import uuid from typing import Dict, List, Optional import python_on_whales @@ -239,7 +240,6 @@ def _build_with_inject( inject: dict, tagged_names: List[str], platform: str, - push: bool, path: str, file: str, build_args: dict, @@ -273,7 +273,7 @@ def _build_with_inject( context_dir, tags=tagged_names, platforms=[platform], - push=push, + load=True, file=file, builder=builder, build_args=build_args @@ -324,7 +324,6 @@ def _build_single_image( inject=inject, tagged_names=tagged_names, platform=platform, - push=push, path=path, file=file, build_args=build_args, @@ -335,11 +334,14 @@ def _build_single_image( path, tags=tagged_names, platforms=[platform], - push=push, + load=True, file=file, build_args=build_args, builder=builder, ) + # Push after the initial load to support remote builders that cannot access the local registry + if push: + docker.push(tagged_names) # Check that the images were built and in the registry # Docker search is not currently implemented in python-on-wheels @@ -444,7 +446,7 @@ def get_path(file): # Updates name to be compatible with docker image_prefix = "buildrunner-mp" - sanitized_name = f"{image_prefix}-{mp_image_name.replace('/', '-').replace(':', '-')}" + sanitized_name = f"{image_prefix}-{str(uuid.uuid4())}" base_image_name = f"{self._mp_registry_info.ip_addr}:{self._mp_registry_info.port}/{sanitized_name}" # Keeps track of the built images {name: [ImageInfo(image_names)]]} diff --git a/tests/test_multiplatform.py b/tests/test_multiplatform.py index 6e6b1fb9..29991288 100644 --- a/tests/test_multiplatform.py +++ b/tests/test_multiplatform.py @@ -14,6 +14,19 @@ # FIXME: These tests can be broken if a custom buildx builder is set as default # pylint: disable=fixme +@pytest.fixture(autouse=True) +def fixture_uuid_mock(): + with patch('buildrunner.docker.multiplatform_image_builder.uuid') as uuid_mock: + counter = 0 + def _get_uuid(): + nonlocal counter + counter += 1 + return f'uuid{counter}' + + uuid_mock.uuid4.side_effect = _get_uuid + yield uuid_mock + + def actual_images_match_expected(actual_images, expected_images) -> List[str]: missing_images = [] found = False @@ -187,7 +200,7 @@ def test_find_native_platform(mock_os, @pytest.mark.parametrize("name, platforms, expected_image_names",[ ('test-image-tag-2000', ['linux/arm64'], - ['test-image-tag-2000-linux-arm64'] + ['buildrunner-mp-uuid1-linux-arm64'] )]) def test_tag_single_platform(name, platforms, expected_image_names): tag='latest' @@ -221,7 +234,7 @@ def test_tag_single_platform(name, platforms, expected_image_names): @pytest.mark.parametrize("name, platforms, expected_image_names",[ ('test-image-tag-2000', ['linux/arm64'], - ['test-image-tag-2000-linux-arm64'] + ['buildrunner-mp-uuid1-linux-arm64'] )]) def test_tag_single_platform_multiple_tags(name, platforms, expected_image_names): tags=['latest', '0.1.0'] @@ -256,7 +269,7 @@ def test_tag_single_platform_multiple_tags(name, platforms, expected_image_names @pytest.mark.parametrize("name, platforms, expected_image_names",[ ('test-image-tag-2000', ['linux/arm64'], - ['test-image-tag-2000-linux-arm64'] + ['buildrunner-mp-uuid1-linux-arm64'] )]) def test_tag_single_platform_keep_images(name, platforms, expected_image_names): tag='latest' @@ -377,11 +390,11 @@ def test_push_with_dest_names(): @pytest.mark.parametrize("name, platforms, expected_image_names",[ ('test-build-image-2000', ['linux/arm64'], - ['test-build-image-2000-linux-arm64'] + ['buildrunner-mp-uuid1-linux-arm64'] ), ('test-build-image-2001', ['linux/amd64', 'linux/arm64'], - ['test-build-image-2001-linux-amd64', 'test-build-image-2001-linux-arm64'] + ['buildrunner-mp-uuid1-linux-amd64', 'buildrunner-mp-uuid1-linux-arm64'] ) ]) @patch('buildrunner.docker.multiplatform_image_builder.docker.image.remove') @@ -415,11 +428,11 @@ def test_build_multiple_builds(mock_build, mock_pull, mock_inspect, mock_remove) mock_inspect.return_value.id = 'myfakeimageid' name1 = 'test-build-multi-image-2001' platforms1 = ['linux/amd64', 'linux/arm64'] - expected_image_names1 = ['test-build-multi-image-2001-linux-amd64', 'test-build-multi-image-2001-linux-arm64'] + expected_image_names1 = ['buildrunner-mp-uuid1-linux-amd64', 'buildrunner-mp-uuid1-linux-arm64'] name2 = 'test-build-multi-image-2002' platforms2 = ['linux/amd64', 'linux/arm64'] - expected_image_names2 = ['test-build-multi-image-2002-linux-amd64', 'test-build-multi-image-2002-linux-arm64'] + expected_image_names2 = ['buildrunner-mp-uuid2-linux-amd64', 'buildrunner-mp-uuid2-linux-arm64'] test_path = f'{TEST_DIR}/test-files/multiplatform' with MultiplatformImageBuilder() as mp: @@ -454,12 +467,12 @@ def test_build_multiple_builds(mock_build, mock_pull, mock_inspect, mock_remove) ('test-build-tag-image-2000', ['latest', '0.1.0'], ['linux/arm64'], - ['test-build-tag-image-2000-linux-arm64'] + ['buildrunner-mp-uuid1-linux-arm64'] ), ('test-build-tag-image-2001', ['latest', '0.2.0'], ['linux/amd64', 'linux/arm64'], - ['test-build-tag-image-2001-linux-amd64', 'test-build-tag-image-2001-linux-arm64'] + ['buildrunner-mp-uuid1-linux-amd64', 'buildrunner-mp-uuid1-linux-arm64'] ) ]) def test_build_with_tags(name, tags, platforms, expected_image_names): From 17a090270a18dafbbb4a5db1842dbb57ea700010 Mon Sep 17 00:00:00 2001 From: saville Date: Thu, 9 Nov 2023 10:46:54 -0700 Subject: [PATCH 2/2] Mock push as well during tests --- tests/test_multiplatform.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_multiplatform.py b/tests/test_multiplatform.py index 29991288..3db3b63a 100644 --- a/tests/test_multiplatform.py +++ b/tests/test_multiplatform.py @@ -399,9 +399,10 @@ def test_push_with_dest_names(): ]) @patch('buildrunner.docker.multiplatform_image_builder.docker.image.remove') @patch('buildrunner.docker.multiplatform_image_builder.docker.image.inspect') +@patch('buildrunner.docker.multiplatform_image_builder.docker.push') @patch('buildrunner.docker.multiplatform_image_builder.docker.image.pull') @patch('buildrunner.docker.multiplatform_image_builder.docker.buildx.build') -def test_build(mock_build, mock_pull, mock_inspect, mock_remove, name, platforms, expected_image_names): +def test_build(mock_build, mock_pull, mock_push, mock_inspect, mock_remove, name, platforms, expected_image_names): mock_inspect.return_value = MagicMock() mock_inspect.return_value.id = 'myfakeimageid' test_path = f'{TEST_DIR}/test-files/multiplatform' @@ -421,9 +422,10 @@ def test_build(mock_build, mock_pull, mock_inspect, mock_remove, name, platforms @patch('buildrunner.docker.multiplatform_image_builder.docker.image.remove') @patch('buildrunner.docker.multiplatform_image_builder.docker.image.inspect') +@patch('buildrunner.docker.multiplatform_image_builder.docker.push') @patch('buildrunner.docker.multiplatform_image_builder.docker.image.pull') @patch('buildrunner.docker.multiplatform_image_builder.docker.buildx.build') -def test_build_multiple_builds(mock_build, mock_pull, mock_inspect, mock_remove): +def test_build_multiple_builds(mock_build, mock_pull, mock_push, mock_inspect, mock_remove): mock_inspect.return_value = MagicMock() mock_inspect.return_value.id = 'myfakeimageid' name1 = 'test-build-multi-image-2001'