From e54ad41ee5e3689130cfd5cd7518a9e2ab4b1957 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Thu, 8 Aug 2024 07:56:52 +0200 Subject: [PATCH] GH-43608: [CI][Archery] Prefer `docker compose` over `docker-compose` (#43586) ### Rationale for this change The `docker-compose` utility is legacy, while the Docker CLI client now includes a `compose` subcommand that attempts to be compatible with docker-compose YAML files. ### What changes are included in this PR? 1. Call `docker compose` by default when `archery docker` is invoked, not `docker-compose` 2. Add an option to switch back to `docker-compose`, in case of Docker CLI client is too old and doesn't support some options (such as `--file`). ### Are these changes tested? Yes, on CI. ### Are there any user-facing changes? No. * GitHub Issue: #43608 Authored-by: Antoine Pitrou Signed-off-by: Sutou Kouhei --- .github/workflows/archery.yml | 1 - .github/workflows/cpp.yml | 6 ++--- .github/workflows/dev.yml | 1 - .github/workflows/docs.yml | 1 - .github/workflows/docs_light.yml | 1 - .github/workflows/go.yml | 7 +++-- .github/workflows/integration.yml | 1 - .github/workflows/java.yml | 1 - .github/workflows/java_jni.yml | 1 - .github/workflows/js.yml | 1 - .github/workflows/python.yml | 1 - .github/workflows/r.yml | 1 - .github/workflows/ruby.yml | 1 - .github/workflows/swift.yml | 1 - dev/archery/archery/docker/cli.py | 27 ++++++++++++------- dev/archery/archery/docker/core.py | 8 +++--- .../archery/docker/tests/test_docker.py | 6 ++--- dev/archery/requirements-test.txt | 1 - dev/tasks/docker-tests/github.cuda.yml | 1 + dev/tasks/java-jars/github.yml | 4 +-- dev/tasks/linux-packages/github.linux.yml | 2 +- dev/tasks/macros.jinja | 1 - dev/tasks/python-wheels/github.linux.yml | 3 +-- dev/tasks/python-wheels/github.windows.yml | 4 +-- dev/tasks/tasks.yml | 13 +++++---- 25 files changed, 43 insertions(+), 52 deletions(-) diff --git a/.github/workflows/archery.yml b/.github/workflows/archery.yml index 87f365b9065c8..b016f7d11b9fa 100644 --- a/.github/workflows/archery.yml +++ b/.github/workflows/archery.yml @@ -34,7 +34,6 @@ on: env: ARCHERY_DEBUG: 1 ARCHERY_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} - ARCHERY_USE_DOCKER_CLI: 1 concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index fc7f3c5dded3c..a82e1eb76660b 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -99,7 +99,7 @@ jobs: cat <> "$GITHUB_OUTPUT" { "arch": "arm64v8", - "archery-use-docker-cli": "0", + "archery-use-legacy-docker-compose": "1", "clang-tools": "10", "image": "ubuntu-cpp", "llvm": "10", @@ -124,9 +124,9 @@ jobs: include: ${{ fromJson(needs.docker-targets.outputs.targets) }} env: ARCH: ${{ matrix.arch }} - # By default, use Docker CLI because docker-compose v1 is obsolete, + # By default, use `docker compose` because docker-compose v1 is obsolete, # except where the Docker client version is too old. - ARCHERY_USE_DOCKER_CLI: ${{ matrix.archery-use-docker-cli || '1' }} + ARCHERY_USE_LEGACY_DOCKER_COMPOSE: ${{ matrix.archery-use-legacy-docker-compose || '0' }} ARROW_SIMD_LEVEL: ${{ matrix.simd-level }} CLANG_TOOLS: ${{ matrix.clang-tools }} LLVM: ${{ matrix.llvm }} diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 49568102e11f8..cc3ff6330746d 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -31,7 +31,6 @@ permissions: env: ARCHERY_DEBUG: 1 - ARCHERY_USE_DOCKER_CLI: 1 jobs: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index b6075746ff40a..25db1c39ad89e 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -25,7 +25,6 @@ permissions: env: ARCHERY_DEBUG: 1 - ARCHERY_USE_DOCKER_CLI: 1 ARROW_ENABLE_TIMING_TESTS: OFF DOCKER_VOLUME_PREFIX: ".docker/" diff --git a/.github/workflows/docs_light.yml b/.github/workflows/docs_light.yml index f66e8473e2516..ea7fe5d02d7b8 100644 --- a/.github/workflows/docs_light.yml +++ b/.github/workflows/docs_light.yml @@ -34,7 +34,6 @@ permissions: env: ARCHERY_DEBUG: 1 - ARCHERY_USE_DOCKER_CLI: 1 ARROW_ENABLE_TIMING_TESTS: OFF DOCKER_VOLUME_PREFIX: ".docker/" diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 0d369d252b56c..20c78d86cb2a3 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -43,7 +43,6 @@ permissions: env: ARCHERY_DEBUG: 1 - ARCHERY_USE_DOCKER_CLI: 1 jobs: @@ -79,14 +78,14 @@ jobs: { "arch-label": "ARM64", "arch": "arm64v8", - "archery-use-docker-cli": "0", + "archery-use-legacy-docker-compose": "1", "go": "1.21", "runs-on": ["self-hosted", "arm", "linux"] }, { "arch-label": "ARM64", "arch": "arm64v8", - "archery-use-docker-cli": "0", + "archery-use-legacy-docker-compose": "1", "go": "1.22", "runs-on": ["self-hosted", "arm", "linux"] } @@ -109,7 +108,7 @@ jobs: ARCH: ${{ matrix.arch }} # By default, use Docker CLI because docker-compose v1 is obsolete, # except where the Docker client version is too old. - ARCHERY_USE_DOCKER_CLI: ${{ matrix.archery-use-docker-cli || '1' }} + ARCHERY_USE_LEGACY_DOCKER_COMPOSE: ${{ matrix.archery-use-legacy-docker-compose || '0' }} GO: ${{ matrix.go }} steps: - name: Checkout Arrow diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 46d422a53ae69..43f8af0a600d8 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -54,7 +54,6 @@ permissions: env: ARCHERY_DEBUG: 1 - ARCHERY_USE_DOCKER_CLI: 1 DOCKER_VOLUME_PREFIX: ".docker/" jobs: diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index d4211c2c81cb5..0317879b580ba 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -46,7 +46,6 @@ permissions: env: ARCHERY_DEBUG: 1 - ARCHERY_USE_DOCKER_CLI: 1 DOCKER_VOLUME_PREFIX: ".docker/" jobs: diff --git a/.github/workflows/java_jni.yml b/.github/workflows/java_jni.yml index 533da7c36be34..c2bc679e681a2 100644 --- a/.github/workflows/java_jni.yml +++ b/.github/workflows/java_jni.yml @@ -46,7 +46,6 @@ permissions: env: ARCHERY_DEBUG: 1 - ARCHERY_USE_DOCKER_CLI: 1 DOCKER_VOLUME_PREFIX: ".docker/" jobs: diff --git a/.github/workflows/js.yml b/.github/workflows/js.yml index ad22968a3a68b..630bef61105f6 100644 --- a/.github/workflows/js.yml +++ b/.github/workflows/js.yml @@ -40,7 +40,6 @@ permissions: env: ARCHERY_DEBUG: 1 - ARCHERY_USE_DOCKER_CLI: 1 jobs: diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index daadd971f8ac8..916db2580e371 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -42,7 +42,6 @@ permissions: env: ARCHERY_DEBUG: 1 - ARCHERY_USE_DOCKER_CLI: 1 DOCKER_VOLUME_PREFIX: ".docker/" jobs: diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 0ff7266860f29..544aa9b75c249 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -52,7 +52,6 @@ permissions: env: ARCHERY_DEBUG: 1 - ARCHERY_USE_DOCKER_CLI: 1 DOCKER_VOLUME_PREFIX: ".docker/" jobs: diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index ca2305a7f9357..e4d650e74a8ad 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -54,7 +54,6 @@ permissions: env: ARCHERY_DEBUG: 1 - ARCHERY_USE_DOCKER_CLI: 1 DOCKER_VOLUME_PREFIX: ".docker/" jobs: diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml index 3f039315b505a..1b3c9eca1814a 100644 --- a/.github/workflows/swift.yml +++ b/.github/workflows/swift.yml @@ -42,7 +42,6 @@ permissions: env: ARCHERY_DEBUG: 1 - ARCHERY_USE_DOCKER_CLI: 1 DOCKER_VOLUME_PREFIX: ".docker/" jobs: diff --git a/dev/archery/archery/docker/cli.py b/dev/archery/archery/docker/cli.py index 7053db2afccff..23c565f7780ff 100644 --- a/dev/archery/archery/docker/cli.py +++ b/dev/archery/archery/docker/cli.py @@ -28,17 +28,17 @@ def _mock_compose_calls(compose): from types import MethodType from subprocess import CompletedProcess - def _mock(compose, executable): + def _mock(compose, command_tuple): def _execute(self, *args, **kwargs): - params = ['{}={}'.format(k, v) + params = [f'{k}={v}' for k, v in self.config.params.items()] - command = ' '.join(params + [executable] + list(args)) + command = ' '.join(params + command_tuple + args) click.echo(command) return CompletedProcess([], 0) return MethodType(_execute, compose) - compose._execute_docker = _mock(compose, executable='docker') - compose._execute_compose = _mock(compose, executable='docker-compose') + compose._execute_docker = _mock(compose, command_tuple=('docker',)) + compose._execute_compose = _mock(compose, command_tuple=('docker', 'compose')) @click.group() @@ -47,18 +47,24 @@ def _execute(self, *args, **kwargs): help="Specify Arrow source directory.") @click.option('--dry-run/--execute', default=False, help="Display the docker commands instead of executing them.") +@click.option('--using-legacy-docker-compose', default=False, is_flag=True, + envvar='ARCHERY_USE_LEGACY_DOCKER_COMPOSE', + help="Use legacy docker-compose utility instead of the built-in " + "`docker compose` subcommand. This may be necessary if the " + "Docker client is too old for some options.") @click.option('--using-docker-cli', default=False, is_flag=True, envvar='ARCHERY_USE_DOCKER_CLI', help="Use docker CLI directly for building instead of calling " - "docker-compose. This may help to reuse cached layers.") + "`docker compose`. This may help to reuse cached layers.") @click.option('--using-docker-buildx', default=False, is_flag=True, envvar='ARCHERY_USE_DOCKER_BUILDX', help="Use buildx with docker CLI directly for building instead " - "of calling docker-compose or the plain docker build " + "of calling `docker compose` or the plain docker build " "command. This option makes the build cache reusable " "across hosts.") @click.pass_context -def docker(ctx, src, dry_run, using_docker_cli, using_docker_buildx): +def docker(ctx, src, dry_run, using_legacy_docker_compose, using_docker_cli, + using_docker_buildx): """ Interact with docker-compose based builds. """ @@ -74,12 +80,13 @@ def docker(ctx, src, dry_run, using_docker_cli, using_docker_buildx): # take the docker-compose parameters like PYTHON, PANDAS, UBUNTU from the # environment variables to keep the usage similar to docker-compose using_docker_cli |= using_docker_buildx + compose_bin = ("docker-compose" if using_legacy_docker_compose + else "docker compose") compose = DockerCompose(config_path, params=os.environ, using_docker=using_docker_cli, using_buildx=using_docker_buildx, debug=ctx.obj.get('debug', False), - compose_bin=("docker compose" if using_docker_cli - else "docker-compose")) + compose_bin=compose_bin) if dry_run: _mock_compose_calls(compose) ctx.obj['compose'] = compose diff --git a/dev/archery/archery/docker/core.py b/dev/archery/archery/docker/core.py index 5be4887ea4f63..1c486e7aae629 100644 --- a/dev/archery/archery/docker/core.py +++ b/dev/archery/archery/docker/core.py @@ -135,9 +135,9 @@ def _read_config(self, config_path, compose_bin): compose = Docker() args = ['compose'] else: - compose = Command('docker-compose') + compose = Command(compose_bin) args = [] - args += ['--file', str(config_path), 'config'] + args += [f'--file={config_path}', 'config'] result = compose.run(*args, env=self.env, check=False, stderr=subprocess.PIPE, stdout=subprocess.PIPE) @@ -180,7 +180,7 @@ class DockerCompose(Command): def __init__(self, config_path, dotenv_path=None, compose_bin=None, using_docker=False, using_buildx=False, params=None, debug=False): - compose_bin = default_bin(compose_bin, 'docker-compose') + compose_bin = default_bin(compose_bin, 'docker compose') self.config = ComposeConfig(config_path, dotenv_path, compose_bin, params=params, using_docker=using_docker, using_buildx=using_buildx, debug=debug) @@ -193,7 +193,7 @@ def clear_pull_memory(self): def _execute_compose(self, *args, **kwargs): # execute as a docker compose command try: - result = super().run('--file', str(self.config.path), *args, + result = super().run(f'--file={self.config.path}', *args, env=self.config.env, **kwargs) result.check_returncode() except subprocess.CalledProcessError as e: diff --git a/dev/archery/archery/docker/tests/test_docker.py b/dev/archery/archery/docker/tests/test_docker.py index 386b7c2bdae3d..0849d1e97c984 100644 --- a/dev/archery/archery/docker/tests/test_docker.py +++ b/dev/archery/archery/docker/tests/test_docker.py @@ -243,7 +243,7 @@ def assert_docker_calls(compose, expected_args): def assert_compose_calls(compose, expected_args, env=mock.ANY): - base_command = ['docker-compose', '--file', str(compose.config.path)] + base_command = ['docker', 'compose', f'--file={compose.config.path}'] expected_commands = [] for args in expected_args: if isinstance(args, str): @@ -482,7 +482,7 @@ def test_compose_push(arrow_compose_path): ] for image in ["conda-cpp", "conda-python", "conda-python-pandas"]: expected_calls.append( - mock.call(["docker-compose", "--file", str(compose.config.path), + mock.call(["docker", "compose", f"--file={compose.config.path}", "push", image], check=True, env=expected_env) ) with assert_subprocess_calls(expected_calls): @@ -514,7 +514,7 @@ def test_image_with_gpu(arrow_compose_path): "run", "--rm", "--gpus", "all", "-e", "CUDA_ENV=1", "-e", "OTHER_ENV=2", - "-v", "/host:/container:rw", + "-v", "/host:/container", "org/ubuntu-cuda", "/bin/bash", "-c", "echo 1 > /tmp/dummy && cat /tmp/dummy", ] diff --git a/dev/archery/requirements-test.txt b/dev/archery/requirements-test.txt index e3e62a993c2a2..208ec64cdf026 100644 --- a/dev/archery/requirements-test.txt +++ b/dev/archery/requirements-test.txt @@ -1,3 +1,2 @@ -docker-compose pytest responses diff --git a/dev/tasks/docker-tests/github.cuda.yml b/dev/tasks/docker-tests/github.cuda.yml index 30879042924c4..9c7adf53a6f70 100644 --- a/dev/tasks/docker-tests/github.cuda.yml +++ b/dev/tasks/docker-tests/github.cuda.yml @@ -34,6 +34,7 @@ jobs: - name: Execute Docker Build shell: bash env: + ARCHERY_USE_LEGACY_DOCKER_COMPOSE: 1 {{ macros.github_set_sccache_envvars()|indent(8) }} run: | archery docker run \ diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml index 77e8867652e65..7cbd5f05dab4a 100644 --- a/dev/tasks/java-jars/github.yml +++ b/dev/tasks/java-jars/github.yml @@ -30,7 +30,7 @@ jobs: ARCH: {{ '${{ matrix.platform.archery_arch }}' }} ARCH_ALIAS: {{ '${{ matrix.platform.archery_arch_alias }}' }} ARCH_SHORT: {{ '${{ matrix.platform.archery_arch_short }}' }} - ARCHERY_USE_DOCKER_CLI: {{ "${{matrix.platform.archery_use_docker_cli || '1'}}" }} + ARCHERY_USE_LEGACY_DOCKER_COMPOSE: {{ "${{matrix.platform.archery_use_legacy_docker_compose || '0'}}" }} strategy: fail-fast: false matrix: @@ -45,7 +45,7 @@ jobs: archery_arch: "arm64v8" archery_arch_alias: "aarch64" archery_arch_short: "arm64" - archery_use_docker_cli: "0" + archery_use_legacy_docker_compose: "1" steps: {{ macros.github_checkout_arrow()|indent }} {{ macros.github_free_space()|indent }} diff --git a/dev/tasks/linux-packages/github.linux.yml b/dev/tasks/linux-packages/github.linux.yml index 891682c4358d8..4bf2295ef3e95 100644 --- a/dev/tasks/linux-packages/github.linux.yml +++ b/dev/tasks/linux-packages/github.linux.yml @@ -29,7 +29,7 @@ jobs: {% endif %} env: ARCHITECTURE: {{ architecture }} - ARCHERY_USE_DOCKER_CLI: {{ '0' if architecture == 'arm64' else '1' }} + ARCHERY_USE_LEGACY_DOCKER_COMPOSE: {{ '1' if architecture == 'arm64' else '0' }} steps: {{ macros.github_checkout_arrow()|indent }} {{ macros.github_login_dockerhub()|indent }} diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index b225109050aae..6423ca0e9efda 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -26,7 +26,6 @@ on: env: ARCHERY_DEBUG: 1 - ARCHERY_USE_DOCKER_CLI: 1 {% endmacro %} {%- macro github_checkout_arrow(fetch_depth=1, submodules="recursive", action_v="4") -%} diff --git a/dev/tasks/python-wheels/github.linux.yml b/dev/tasks/python-wheels/github.linux.yml index 5c82bf74b30b7..968c5da21897b 100644 --- a/dev/tasks/python-wheels/github.linux.yml +++ b/dev/tasks/python-wheels/github.linux.yml @@ -31,10 +31,9 @@ jobs: # archery uses these environment variables {% if arch == "amd64" %} ARCH: amd64 - ARCHERY_USE_DOCKER_CLI: 1 {% else %} ARCH: arm64v8 - ARCHERY_USE_DOCKER_CLI: 0 + ARCHERY_USE_LEGACY_DOCKER_COMPOSE: 1 {% endif %} PYTHON: "{{ python_version }}" diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml index 01f4977a9b0b1..a40b9c0d65103 100644 --- a/dev/tasks/python-wheels/github.windows.yml +++ b/dev/tasks/python-wheels/github.windows.yml @@ -33,8 +33,8 @@ jobs: # note that we don't run docker build since there wouldn't be a cache hit # and rebuilding the dependencies takes a fair amount of time REPO: ghcr.io/ursacomputing/arrow - # prefer the docker cli over docker-compose - ARCHERY_USE_DOCKER_CLI: 1 + # BuildKit isn't really supported on Windows for now + DOCKER_BUILDKIT: 0 steps: {{ macros.github_checkout_arrow()|indent }} diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 07a4d638f1291..6e1f7609a980f 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1110,9 +1110,13 @@ tasks: template: docker-tests/github.linux.yml params: env: - ARCHERY_USE_DOCKER_CLI: 0 UBUNTU: 20.04 - flags: -e ARROW_SKYHOOK=ON + flags: >- + -e ARROW_AZURE=OFF + -e ARROW_GANDIVA=OFF + -e ARROW_GCS=OFF + -e ARROW_S3=OFF + -e ARROW_SKYHOOK=ON image: ubuntu-cpp {% for debian_version in ["12"] %} @@ -1494,16 +1498,12 @@ tasks: ci: github template: docker-tests/github.cuda.yml params: - env: - ARCHERY_USE_DOCKER_CLI: 0 image: ubuntu-cuda-cpp test-cuda-python: ci: github template: docker-tests/github.cuda.yml params: - env: - ARCHERY_USE_DOCKER_CLI: 0 image: ubuntu-cuda-python ############################## Fuzz tests ################################# @@ -1565,7 +1565,6 @@ tasks: template: docker-tests/github.linux.yml params: env: - ARCHERY_USE_DOCKER_CLI: 0 HDFS: "{{ hdfs_version }}" PYTHON: "3.10" image: conda-python-hdfs