ci: make it easier to run on ARM64 machines (#9488)

Make it easier to run some builds on ARM64 machines. The Dockerfiles download a few pre-built binaries, and those need to be platform specific. There is no Spanner emulator for ARM64, in this change we just don't install it. Some builds do not need it, but I did not take any action (such as disabling the integration tests) beyond that.
googleapis · Jul 16, 2022 · 8f7f95b · 8f7f95b
1 parent 8aeda9b
commit 8f7f95b
Show file tree

Hide file tree

Showing 12 changed files with 60 additions and 19 deletions.
diff --git a/.github/renovate.json b/.github/renovate.json
@@ -12,7 +12,7 @@
       ],
       "matchStrings": [
         "https://github\\.com/(?<depName>.*?)/archive/(?<currentValue>.*?)\\.tar\\.gz",
-        "https://github\\.com/(?<depName>.*?)/releases/download/(?<currentValue>[^/]+)/bazelisk-linux-amd64"
+        "https://github\\.com/(?<depName>.*?)/releases/download/(?<currentValue>[^/]+)/bazelisk-linux-"
       ],
       "datasourceTemplate": "github-releases",
       "versioningTemplate": "loose"

diff --git a/ci/cloudbuild/build.sh b/ci/cloudbuild/build.sh
@@ -92,11 +92,12 @@ function die() {
 # Use getopt to parse and normalize all the args.
 PARSED="$(getopt -a \
   --options="t:c:ldsh" \
-  --longoptions="build:,distro:,trigger:,cloud:,local,docker,docker-shell,docker-clean,verbose,help" \
+  --longoptions="arch:,build:,distro:,trigger:,cloud:,local,docker,docker-shell,docker-clean,verbose,help" \
   --name="${PROGRAM_NAME}" \
   -- "$@")"
 eval set -- "${PARSED}"
 
+ARCH_FLAG=""
 BUILD_FLAG=""
 DISTRO_FLAG=""
 TRIGGER_FLAG=""
@@ -108,6 +109,10 @@ SHELL_FLAG="false"
 : "${VERBOSE_FLAG:=false}"
 while true; do
   case "$1" in
+    --arch)
+      ARCH_FLAG="$2"
+      shift 2
+      ;;
     --build)
       BUILD_FLAG="$2"
       shift 2
@@ -307,6 +312,9 @@ if [[ "${DOCKER_FLAG}" = "true" ]]; then
     "--build-arg=NCPU=$(nproc)"
     -f "ci/cloudbuild/dockerfiles/${DISTRO_FLAG}.Dockerfile"
   )
+  if [[ -n "${ARCH_FLAG}" ]]; then
+    build_flags+=("--build-arg=ARCH=${ARCH_FLAG}")
+  fi
   export DOCKER_BUILDKIT=1
   io::run docker build "${build_flags[@]}" ci
   io::log_h2 "Starting docker container: ${image}"

diff --git a/ci/cloudbuild/dockerfiles/checkers.Dockerfile b/ci/cloudbuild/dockerfiles/checkers.Dockerfile
@@ -19,6 +19,7 @@
 # `checkers.sh` build.
 FROM fedora:36
 ARG NCPU=4
+ARG ARCH=amd64
 
 RUN dnf makecache && \
     dnf install -y \
@@ -35,16 +36,16 @@ RUN dnf makecache && \
 
 RUN cargo install typos-cli --version 1.3.9 --root /usr/local
 
-RUN curl -L -o /usr/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/5.0.1/buildifier-linux-amd64 && \
+RUN curl -L -o /usr/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/5.0.1/buildifier-linux-${ARCH} && \
     chmod 755 /usr/bin/buildifier
 
-RUN curl -L -o /usr/local/bin/shfmt https://github.com/mvdan/sh/releases/download/v3.4.3/shfmt_v3.4.3_linux_amd64 && \
+RUN curl -L -o /usr/local/bin/shfmt https://github.com/mvdan/sh/releases/download/v3.4.3/shfmt_v3.4.3_linux_${ARCH} && \
     chmod 755 /usr/local/bin/shfmt
 
 RUN pip3 install --upgrade pip
 RUN pip3 install cmake_format==0.6.13
 RUN pip3 install black==22.3.0
 
-RUN curl -o /usr/bin/bazelisk -sSL "https://github.com/bazelbuild/bazelisk/releases/download/v1.12.0/bazelisk-linux-amd64" && \
+RUN curl -o /usr/bin/bazelisk -sSL "https://github.com/bazelbuild/bazelisk/releases/download/v1.12.0/bazelisk-linux-${ARCH}" && \
     chmod +x /usr/bin/bazelisk && \
     ln -s /usr/bin/bazelisk /usr/bin/bazel
diff --git a/ci/cloudbuild/dockerfiles/debian-github.Dockerfile b/ci/cloudbuild/dockerfiles/debian-github.Dockerfile
@@ -14,8 +14,9 @@
 
 FROM debian:bullseye-slim
 ARG NCPU=4
+ARG ARCH=amd64
 
 RUN apt-get update && apt-get install -y curl git
 WORKDIR /var/tmp
-RUN curl -sSL -o gh.deb https://github.com/cli/cli/releases/download/v1.9.1/gh_1.9.1_linux_amd64.deb && \
+RUN curl -sSL -o gh.deb https://github.com/cli/cli/releases/download/v1.9.1/gh_1.9.1_linux_${ARCH}.deb && \
     dpkg --install gh.deb
diff --git a/ci/cloudbuild/dockerfiles/fedora-35-cxx20.Dockerfile b/ci/cloudbuild/dockerfiles/fedora-35-cxx20.Dockerfile
@@ -14,6 +14,7 @@
 
 FROM fedora:35
 ARG NCPU=4
+ARG ARCH=amd64
 
 # Fedora includes packages for gRPC, libcurl, and OpenSSL that are recent enough
 # for `google-cloud-cpp`. Install these packages and additional development
@@ -203,6 +204,6 @@ RUN dnf makecache && dnf install -y java-latest-openjdk-devel
 # those library directories will be found.
 RUN ldconfig /usr/local/lib*
 
-RUN curl -o /usr/bin/bazelisk -sSL "https://github.com/bazelbuild/bazelisk/releases/download/v1.12.0/bazelisk-linux-amd64" && \
+RUN curl -o /usr/bin/bazelisk -sSL "https://github.com/bazelbuild/bazelisk/releases/download/v1.12.0/bazelisk-linux-${ARCH}" && \
     chmod +x /usr/bin/bazelisk && \
     ln -s /usr/bin/bazelisk /usr/bin/bazel
diff --git a/ci/cloudbuild/dockerfiles/fedora-35.Dockerfile b/ci/cloudbuild/dockerfiles/fedora-35.Dockerfile
@@ -14,6 +14,7 @@
 
 FROM fedora:35
 ARG NCPU=4
+ARG ARCH=amd64
 
 # Fedora includes packages for gRPC, libcurl, and OpenSSL that are recent enough
 # for `google-cloud-cpp`. Install these packages and additional development
@@ -197,6 +198,6 @@ RUN dnf makecache && dnf install -y java-latest-openjdk-devel
 # those library directories will be found.
 RUN ldconfig /usr/local/lib*
 
-RUN curl -o /usr/bin/bazelisk -sSL "https://github.com/bazelbuild/bazelisk/releases/download/v1.12.0/bazelisk-linux-amd64" && \
+RUN curl -o /usr/bin/bazelisk -sSL "https://github.com/bazelbuild/bazelisk/releases/download/v1.12.0/bazelisk-linux-${ARCH}" && \
     chmod +x /usr/bin/bazelisk && \
     ln -s /usr/bin/bazelisk /usr/bin/bazel
diff --git a/ci/cloudbuild/dockerfiles/fedora-36-cxx14.Dockerfile b/ci/cloudbuild/dockerfiles/fedora-36-cxx14.Dockerfile
@@ -14,6 +14,7 @@
 
 FROM fedora:36
 ARG NCPU=4
+ARG ARCH=amd64
 
 # Fedora includes packages for gRPC, libcurl, and OpenSSL that are recent enough
 # for `google-cloud-cpp`. Install these packages and additional development
@@ -176,6 +177,6 @@ RUN dnf makecache && dnf install -y java-latest-openjdk-devel
 # those library directories will be found.
 RUN ldconfig /usr/local/lib*
 
-RUN curl -o /usr/bin/bazelisk -sSL "https://github.com/bazelbuild/bazelisk/releases/download/v1.12.0/bazelisk-linux-amd64" && \
+RUN curl -o /usr/bin/bazelisk -sSL "https://github.com/bazelbuild/bazelisk/releases/download/v1.12.0/bazelisk-linux-${ARCH}" && \
     chmod +x /usr/bin/bazelisk && \
     ln -s /usr/bin/bazelisk /usr/bin/bazel
diff --git a/ci/cloudbuild/dockerfiles/fedora-36.Dockerfile b/ci/cloudbuild/dockerfiles/fedora-36.Dockerfile
@@ -14,6 +14,7 @@
 
 FROM fedora:36
 ARG NCPU=4
+ARG ARCH=amd64
 
 # Fedora includes packages for gRPC, libcurl, and OpenSSL that are recent enough
 # for `google-cloud-cpp`. Install these packages and additional development
@@ -186,6 +187,6 @@ RUN dnf makecache && dnf install -y java-latest-openjdk-devel
 # those library directories will be found.
 RUN ldconfig /usr/local/lib*
 
-RUN curl -o /usr/bin/bazelisk -sSL "https://github.com/bazelbuild/bazelisk/releases/download/v1.12.0/bazelisk-linux-amd64" && \
+RUN curl -o /usr/bin/bazelisk -sSL "https://github.com/bazelbuild/bazelisk/releases/download/v1.12.0/bazelisk-linux-${ARCH}" && \
     chmod +x /usr/bin/bazelisk && \
     ln -s /usr/bin/bazelisk /usr/bin/bazel
diff --git a/ci/cloudbuild/dockerfiles/fedora-msan.Dockerfile b/ci/cloudbuild/dockerfiles/fedora-msan.Dockerfile
@@ -14,6 +14,7 @@
 
 FROM fedora:35
 ARG NCPU=4
+ARG ARCH=amd64
 
 # Install the minimal packages needed to compile libcxx, install Bazel, and
 # then compile our code.
@@ -61,6 +62,6 @@ ENV PATH=${CLOUD_SDK_LOCATION}/bin:${PATH}
 # The Cloud Pub/Sub emulator needs Java :shrug:
 RUN dnf makecache && dnf install -y java-latest-openjdk
 
-RUN curl -o /usr/bin/bazelisk -sSL "https://github.com/bazelbuild/bazelisk/releases/download/v1.12.0/bazelisk-linux-amd64" && \
+RUN curl -o /usr/bin/bazelisk -sSL "https://github.com/bazelbuild/bazelisk/releases/download/v1.12.0/bazelisk-linux-${ARCH}" && \
     chmod +x /usr/bin/bazelisk && \
     ln -s /usr/bin/bazelisk /usr/bin/bazel
diff --git a/ci/cloudbuild/dockerfiles/ubuntu-bionic.Dockerfile b/ci/cloudbuild/dockerfiles/ubuntu-bionic.Dockerfile
@@ -14,6 +14,7 @@
 
 FROM ubuntu:bionic
 ARG NCPU=4
+ARG ARCH=amd64
 
 RUN apt-get update && \
     apt-get --no-install-recommends install -y \
@@ -65,6 +66,6 @@ ENV PATH=${CLOUD_SDK_LOCATION}/bin:${PATH}
 # The Cloud Pub/Sub emulator needs Java :shrug:
 RUN apt update && (apt install -y openjdk-11-jre || apt install -y openjdk-9-jre)
 
-RUN curl -o /usr/bin/bazelisk -sSL "https://github.com/bazelbuild/bazelisk/releases/download/v1.12.0/bazelisk-linux-amd64" && \
+RUN curl -o /usr/bin/bazelisk -sSL "https://github.com/bazelbuild/bazelisk/releases/download/v1.12.0/bazelisk-linux-${ARCH}" && \
     chmod +x /usr/bin/bazelisk && \
     ln -s /usr/bin/bazelisk /usr/bin/bazel
diff --git a/ci/cloudbuild/dockerfiles/ubuntu-focal.Dockerfile b/ci/cloudbuild/dockerfiles/ubuntu-focal.Dockerfile
@@ -14,6 +14,7 @@
 
 FROM ubuntu:focal
 ARG NCPU=4
+ARG ARCH=amd64
 
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && \
@@ -66,6 +67,6 @@ ENV PATH=${CLOUD_SDK_LOCATION}/bin:${PATH}
 # The Cloud Pub/Sub emulator needs Java :shrug:
 RUN apt update && (apt install -y openjdk-11-jre || apt install -y openjdk-9-jre)
 
-RUN curl -o /usr/bin/bazelisk -sSL "https://github.com/bazelbuild/bazelisk/releases/download/v1.12.0/bazelisk-linux-amd64" && \
+RUN curl -o /usr/bin/bazelisk -sSL "https://github.com/bazelbuild/bazelisk/releases/download/v1.12.0/bazelisk-linux-${ARCH}" && \
     chmod +x /usr/bin/bazelisk && \
     ln -s /usr/bin/bazelisk /usr/bin/bazel
diff --git a/ci/install-cloud-sdk.sh b/ci/install-cloud-sdk.sh
@@ -16,14 +16,38 @@
 
 set -euo pipefail
 
-readonly GOOGLE_CLOUD_CPP_CLOUD_SDK_VERSION="390.0.0"
-readonly GOOGLE_CLOUD_CPP_SDK_SHA256="5ce31b85013599dd842ca8a08539aaa6047aee63bb216d5b3988fb2666b06c07"
+readonly GOOGLE_CLOUD_CPP_CLOUD_SDK_VERSION="393.0.0"
+declare -A -r GOOGLE_CLOUD_CPP_SDK_SHA256=(
+  ["x86_64"]="6f3aadf553011f9a0478dc1ef029dd71d8ee1b34a6c534653faaca13392a971c"
+  ["arm"]="55d47d9650adccfc592577c052313dad9e54d4e3ea006280d1b35de2104ddaa4"
+)
+
+ARCH="$(uname -p)"
+if [[ "${ARCH}" == "aarch64" ]]; then
+  # The tarball uses this name
+  ARCH="arm"
+fi
+readonly ARCH
+
+components=(
+  # All the emulators are in the "beta" components
+  beta
+  # We need the Bigtable emulator and CLI tool
+  bigtable
+  cbt
+  # We use the Pub/Sub emulator in our tests
+  pubsub-emulator
+)
+if [[ "${ARCH}" == "x86_64" ]]; then
+  # The spanner emulator is not available for ARM64, but we do use it for testing on x86_64
+  components+=(cloud-spanner-emulator)
+fi
 
 readonly SITE="https://dl.google.com/dl/cloudsdk/channels/rapid/downloads"
-readonly TARBALL="google-cloud-cli-${GOOGLE_CLOUD_CPP_CLOUD_SDK_VERSION}-linux-x86_64.tar.gz"
-curl -L "${SITE}/${TARBALL}" -o "${TARBALL}"
+readonly TARBALL="google-cloud-cli-${GOOGLE_CLOUD_CPP_CLOUD_SDK_VERSION}-linux-${ARCH}.tar.gz"
 
-echo "${GOOGLE_CLOUD_CPP_SDK_SHA256} ${TARBALL}" | sha256sum --check -
+curl -L "${SITE}/${TARBALL}" -o "${TARBALL}"
+echo "${GOOGLE_CLOUD_CPP_SDK_SHA256[${ARCH}]} ${TARBALL}" | sha256sum --check -
 tar x -C /usr/local -f "${TARBALL}"
 /usr/local/google-cloud-sdk/bin/gcloud --quiet components install \
-  beta bigtable cbt cloud-spanner-emulator pubsub-emulator
+  "${components[@]}"