apacheGH-43608: [CI][Archery] Prefer docker compose over `docker-co…

…mpose` (apache#43586)

### Rationale for this change

The `docker-compose` utility is legacy, while the Docker CLI client now includes a `compose` subcommand that attempts to be compatible with docker-compose YAML files.

### What changes are included in this PR?

1. Call `docker compose` by default when `archery docker` is invoked, not `docker-compose`
2. Add an option to switch back to `docker-compose`, in case of Docker CLI client is too old and doesn't support some options (such as `--file`).

### Are these changes tested?

Yes, on CI.

### Are there any user-facing changes?

No.
* GitHub Issue: apache#43608

Authored-by: Antoine Pitrou <antoine@python.org>
Signed-off-by: Sutou Kouhei <kou@clear-code.com>

.github/workflows/archery.yml

@@ -34,7 +34,6 @@ on:
 env:
   ARCHERY_DEBUG: 1
   ARCHERY_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
-  ARCHERY_USE_DOCKER_CLI: 1
 
 concurrency:
   group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}

.github/workflows/cpp.yml

@@ -99,7 +99,7 @@ jobs:
             cat <<JSON >> "$GITHUB_OUTPUT"
           {
             "arch": "arm64v8",
-            "archery-use-docker-cli": "0",
+            "archery-use-legacy-docker-compose": "1",
             "clang-tools": "10",
             "image": "ubuntu-cpp",
             "llvm": "10",
@@ -124,9 +124,9 @@ jobs:
         include: ${{ fromJson(needs.docker-targets.outputs.targets) }}
     env:
       ARCH: ${{ matrix.arch }}
-      # By default, use Docker CLI because docker-compose v1 is obsolete,
+      # By default, use `docker compose` because docker-compose v1 is obsolete,
       # except where the Docker client version is too old.
-      ARCHERY_USE_DOCKER_CLI: ${{ matrix.archery-use-docker-cli || '1' }}
+      ARCHERY_USE_LEGACY_DOCKER_COMPOSE: ${{ matrix.archery-use-legacy-docker-compose || '0' }}
       ARROW_SIMD_LEVEL: ${{ matrix.simd-level }}
       CLANG_TOOLS: ${{ matrix.clang-tools }}
       LLVM: ${{ matrix.llvm }}

.github/workflows/dev.yml

@@ -31,7 +31,6 @@ permissions:
 
 env:
   ARCHERY_DEBUG: 1
-  ARCHERY_USE_DOCKER_CLI: 1
 
 jobs:
 

.github/workflows/docs.yml

@@ -25,7 +25,6 @@ permissions:
 
 env:
   ARCHERY_DEBUG: 1
-  ARCHERY_USE_DOCKER_CLI: 1
   ARROW_ENABLE_TIMING_TESTS: OFF
   DOCKER_VOLUME_PREFIX: ".docker/"
 

.github/workflows/docs_light.yml

@@ -34,7 +34,6 @@ permissions:
 
 env:
   ARCHERY_DEBUG: 1
-  ARCHERY_USE_DOCKER_CLI: 1
   ARROW_ENABLE_TIMING_TESTS: OFF
   DOCKER_VOLUME_PREFIX: ".docker/"
 

.github/workflows/go.yml

@@ -43,7 +43,6 @@ permissions:
 
 env:
   ARCHERY_DEBUG: 1
-  ARCHERY_USE_DOCKER_CLI: 1
 
 jobs:
 
@@ -79,14 +78,14 @@ jobs:
           {
             "arch-label": "ARM64",
             "arch": "arm64v8",
-            "archery-use-docker-cli": "0",
+            "archery-use-legacy-docker-compose": "1",
             "go": "1.21",
             "runs-on": ["self-hosted", "arm", "linux"]
           },
           {
             "arch-label": "ARM64",
             "arch": "arm64v8",
-            "archery-use-docker-cli": "0",
+            "archery-use-legacy-docker-compose": "1",
             "go": "1.22",
             "runs-on": ["self-hosted", "arm", "linux"]
           }
@@ -109,7 +108,7 @@ jobs:
       ARCH: ${{ matrix.arch }}
       # By default, use Docker CLI because docker-compose v1 is obsolete,
       # except where the Docker client version is too old.
-      ARCHERY_USE_DOCKER_CLI: ${{ matrix.archery-use-docker-cli || '1' }}
+      ARCHERY_USE_LEGACY_DOCKER_COMPOSE: ${{ matrix.archery-use-legacy-docker-compose || '0' }}
       GO: ${{ matrix.go }}
     steps:
       - name: Checkout Arrow

.github/workflows/integration.yml

@@ -54,7 +54,6 @@ permissions:
 
 env:
   ARCHERY_DEBUG: 1
-  ARCHERY_USE_DOCKER_CLI: 1
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:

.github/workflows/java.yml

@@ -46,7 +46,6 @@ permissions:
 
 env:
   ARCHERY_DEBUG: 1
-  ARCHERY_USE_DOCKER_CLI: 1
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:

.github/workflows/java_jni.yml

@@ -46,7 +46,6 @@ permissions:
 
 env:
   ARCHERY_DEBUG: 1
-  ARCHERY_USE_DOCKER_CLI: 1
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:

.github/workflows/js.yml

@@ -40,7 +40,6 @@ permissions:
 
 env:
   ARCHERY_DEBUG: 1
-  ARCHERY_USE_DOCKER_CLI: 1
 
 jobs:
 

.github/workflows/python.yml

@@ -42,7 +42,6 @@ permissions:
 
 env:
   ARCHERY_DEBUG: 1
-  ARCHERY_USE_DOCKER_CLI: 1
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:

.github/workflows/r.yml

@@ -52,7 +52,6 @@ permissions:
 
 env:
   ARCHERY_DEBUG: 1
-  ARCHERY_USE_DOCKER_CLI: 1
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:

.github/workflows/ruby.yml

@@ -54,7 +54,6 @@ permissions:
 
 env:
   ARCHERY_DEBUG: 1
-  ARCHERY_USE_DOCKER_CLI: 1
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:

.github/workflows/swift.yml

@@ -42,7 +42,6 @@ permissions:
 
 env:
   ARCHERY_DEBUG: 1
-  ARCHERY_USE_DOCKER_CLI: 1
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:

dev/archery/archery/docker/cli.py

@@ -28,17 +28,17 @@ def _mock_compose_calls(compose):
     from types import MethodType
     from subprocess import CompletedProcess
 
-    def _mock(compose, executable):
+    def _mock(compose, command_tuple):
         def _execute(self, *args, **kwargs):
-            params = ['{}={}'.format(k, v)
+            params = [f'{k}={v}'
                       for k, v in self.config.params.items()]
-            command = ' '.join(params + [executable] + list(args))
+            command = ' '.join(params + command_tuple + args)
             click.echo(command)
             return CompletedProcess([], 0)
         return MethodType(_execute, compose)
 
-    compose._execute_docker = _mock(compose, executable='docker')
-    compose._execute_compose = _mock(compose, executable='docker-compose')
+    compose._execute_docker = _mock(compose, command_tuple=('docker',))
+    compose._execute_compose = _mock(compose, command_tuple=('docker', 'compose'))
 
 
 @click.group()
@@ -47,18 +47,24 @@ def _execute(self, *args, **kwargs):
               help="Specify Arrow source directory.")
 @click.option('--dry-run/--execute', default=False,
               help="Display the docker commands instead of executing them.")
+@click.option('--using-legacy-docker-compose', default=False, is_flag=True,
+              envvar='ARCHERY_USE_LEGACY_DOCKER_COMPOSE',
+              help="Use legacy docker-compose utility instead of the built-in "
+                   "`docker compose` subcommand. This may be necessary if the "
+                   "Docker client is too old for some options.")
 @click.option('--using-docker-cli', default=False, is_flag=True,
               envvar='ARCHERY_USE_DOCKER_CLI',
               help="Use docker CLI directly for building instead of calling "
-                   "docker-compose. This may help to reuse cached layers.")
+                   "`docker compose`. This may help to reuse cached layers.")
 @click.option('--using-docker-buildx', default=False, is_flag=True,
               envvar='ARCHERY_USE_DOCKER_BUILDX',
               help="Use buildx with docker CLI directly for building instead "
-                   "of calling docker-compose or the plain docker build "
+                   "of calling `docker compose` or the plain docker build "
                    "command. This option makes the build cache reusable "
                    "across hosts.")
 @click.pass_context
-def docker(ctx, src, dry_run, using_docker_cli, using_docker_buildx):
+def docker(ctx, src, dry_run, using_legacy_docker_compose, using_docker_cli,
+           using_docker_buildx):
     """
     Interact with docker-compose based builds.
     """
@@ -74,12 +80,13 @@ def docker(ctx, src, dry_run, using_docker_cli, using_docker_buildx):
     # take the docker-compose parameters like PYTHON, PANDAS, UBUNTU from the
     # environment variables to keep the usage similar to docker-compose
     using_docker_cli |= using_docker_buildx
+    compose_bin = ("docker-compose" if using_legacy_docker_compose
+                   else "docker compose")
     compose = DockerCompose(config_path, params=os.environ,
                             using_docker=using_docker_cli,
                             using_buildx=using_docker_buildx,
                             debug=ctx.obj.get('debug', False),
-                            compose_bin=("docker compose" if using_docker_cli
-                                         else "docker-compose"))
+                            compose_bin=compose_bin)
     if dry_run:
         _mock_compose_calls(compose)
     ctx.obj['compose'] = compose

dev/archery/archery/docker/core.py

@@ -135,9 +135,9 @@ def _read_config(self, config_path, compose_bin):
             compose = Docker()
             args = ['compose']
         else:
-            compose = Command('docker-compose')
+            compose = Command(compose_bin)
             args = []
-        args += ['--file', str(config_path), 'config']
+        args += [f'--file={config_path}', 'config']
         result = compose.run(*args, env=self.env, check=False,
                              stderr=subprocess.PIPE, stdout=subprocess.PIPE)
 
@@ -180,7 +180,7 @@ class DockerCompose(Command):
     def __init__(self, config_path, dotenv_path=None, compose_bin=None,
                  using_docker=False, using_buildx=False, params=None,
                  debug=False):
-        compose_bin = default_bin(compose_bin, 'docker-compose')
+        compose_bin = default_bin(compose_bin, 'docker compose')
         self.config = ComposeConfig(config_path, dotenv_path, compose_bin,
                                     params=params, using_docker=using_docker,
                                     using_buildx=using_buildx, debug=debug)
@@ -193,7 +193,7 @@ def clear_pull_memory(self):
     def _execute_compose(self, *args, **kwargs):
         # execute as a docker compose command
         try:
-            result = super().run('--file', str(self.config.path), *args,
+            result = super().run(f'--file={self.config.path}', *args,
                                  env=self.config.env, **kwargs)
             result.check_returncode()
         except subprocess.CalledProcessError as e:

dev/archery/archery/docker/tests/test_docker.py

@@ -243,7 +243,7 @@ def assert_docker_calls(compose, expected_args):
 
 
 def assert_compose_calls(compose, expected_args, env=mock.ANY):
-    base_command = ['docker-compose', '--file', str(compose.config.path)]
+    base_command = ['docker', 'compose', f'--file={compose.config.path}']
     expected_commands = []
     for args in expected_args:
         if isinstance(args, str):
@@ -482,7 +482,7 @@ def test_compose_push(arrow_compose_path):
     ]
     for image in ["conda-cpp", "conda-python", "conda-python-pandas"]:
         expected_calls.append(
-            mock.call(["docker-compose", "--file", str(compose.config.path),
+            mock.call(["docker", "compose", f"--file={compose.config.path}",
                        "push", image], check=True, env=expected_env)
         )
     with assert_subprocess_calls(expected_calls):
@@ -514,7 +514,7 @@ def test_image_with_gpu(arrow_compose_path):
             "run", "--rm", "--gpus", "all",
             "-e", "CUDA_ENV=1",
             "-e", "OTHER_ENV=2",
-            "-v", "/host:/container:rw",
+            "-v", "/host:/container",
             "org/ubuntu-cuda",
             "/bin/bash", "-c", "echo 1 > /tmp/dummy && cat /tmp/dummy",
         ]

dev/archery/requirements-test.txt

@@ -1,3 +1,2 @@
-docker-compose
 pytest
 responses

dev/tasks/docker-tests/github.cuda.yml

@@ -34,6 +34,7 @@ jobs:
       - name: Execute Docker Build
         shell: bash
         env:
+          ARCHERY_USE_LEGACY_DOCKER_COMPOSE: 1
         {{ macros.github_set_sccache_envvars()|indent(8) }}
         run: |
           archery docker run \

dev/tasks/java-jars/github.yml

@@ -30,7 +30,7 @@ jobs:
       ARCH: {{ '${{ matrix.platform.archery_arch }}' }}
       ARCH_ALIAS: {{ '${{ matrix.platform.archery_arch_alias }}' }}
       ARCH_SHORT: {{ '${{ matrix.platform.archery_arch_short }}' }}
-      ARCHERY_USE_DOCKER_CLI: {{ "${{matrix.platform.archery_use_docker_cli || '1'}}" }}
+      ARCHERY_USE_LEGACY_DOCKER_COMPOSE: {{ "${{matrix.platform.archery_use_legacy_docker_compose || '0'}}" }}
     strategy:
       fail-fast: false
       matrix:
@@ -45,7 +45,7 @@ jobs:
             archery_arch: "arm64v8"
             archery_arch_alias: "aarch64"
             archery_arch_short: "arm64"
-            archery_use_docker_cli: "0"
+            archery_use_legacy_docker_compose: "1"
     steps:
       {{ macros.github_checkout_arrow()|indent }}
       {{ macros.github_free_space()|indent }}

dev/tasks/linux-packages/github.linux.yml

@@ -29,7 +29,7 @@ jobs:
     {% endif %}
     env:
       ARCHITECTURE: {{ architecture }}
-      ARCHERY_USE_DOCKER_CLI: {{ '0' if architecture == 'arm64' else '1' }}
+      ARCHERY_USE_LEGACY_DOCKER_COMPOSE: {{ '1' if architecture == 'arm64' else '0' }}
     steps:
       {{ macros.github_checkout_arrow()|indent }}
       {{ macros.github_login_dockerhub()|indent }}

dev/tasks/macros.jinja

@@ -26,7 +26,6 @@ on:
 
 env:
   ARCHERY_DEBUG: 1
-  ARCHERY_USE_DOCKER_CLI: 1
 {% endmacro %}
 
 {%- macro github_checkout_arrow(fetch_depth=1, submodules="recursive", action_v="4") -%}

dev/tasks/python-wheels/github.linux.yml

@@ -31,10 +31,9 @@ jobs:
       # archery uses these environment variables
       {% if arch == "amd64" %}
       ARCH: amd64
-      ARCHERY_USE_DOCKER_CLI: 1
       {% else %}
       ARCH: arm64v8
-      ARCHERY_USE_DOCKER_CLI: 0
+      ARCHERY_USE_LEGACY_DOCKER_COMPOSE: 1
       {% endif %}
       PYTHON: "{{ python_version }}"
 

dev/tasks/python-wheels/github.windows.yml

@@ -33,8 +33,8 @@ jobs:
       # note that we don't run docker build since there wouldn't be a cache hit
       # and rebuilding the dependencies takes a fair amount of time
       REPO: ghcr.io/ursacomputing/arrow
-      # prefer the docker cli over docker-compose
-      ARCHERY_USE_DOCKER_CLI: 1
+      # BuildKit isn't really supported on Windows for now
+      DOCKER_BUILDKIT: 0
 
     steps:
       {{ macros.github_checkout_arrow()|indent }}

dev/tasks/tasks.yml

@@ -1110,9 +1110,13 @@ tasks:
     template: docker-tests/github.linux.yml
     params:
       env:
-        ARCHERY_USE_DOCKER_CLI: 0
         UBUNTU: 20.04
-      flags: -e ARROW_SKYHOOK=ON
+      flags: >-
+        -e ARROW_AZURE=OFF
+        -e ARROW_GANDIVA=OFF
+        -e ARROW_GCS=OFF
+        -e ARROW_S3=OFF
+        -e ARROW_SKYHOOK=ON
       image: ubuntu-cpp
 
 {% for debian_version in ["12"] %}
@@ -1494,16 +1498,12 @@ tasks:
     ci: github
     template: docker-tests/github.cuda.yml
     params:
-      env:
-        ARCHERY_USE_DOCKER_CLI: 0
       image: ubuntu-cuda-cpp
 
   test-cuda-python:
     ci: github
     template: docker-tests/github.cuda.yml
     params:
-      env:
-        ARCHERY_USE_DOCKER_CLI: 0
       image: ubuntu-cuda-python
 
   ############################## Fuzz tests #################################
@@ -1565,7 +1565,6 @@ tasks:
     template: docker-tests/github.linux.yml
     params:
       env:
-        ARCHERY_USE_DOCKER_CLI: 0
         HDFS: "{{ hdfs_version }}"
         PYTHON: "3.10"
       image: conda-python-hdfs