From 85923b88147e710a3d82dd13428e38650b349b8f Mon Sep 17 00:00:00 2001 From: Florian Weikert Date: Fri, 11 Nov 2022 14:49:57 +0100 Subject: [PATCH] Add experimental ARM64 Linux support (#1493) The basic idea is to publish multi-arch Docker images, and then run the ARM64 images on the newly added Tau T2A VMs. I'm adding separate build/push scripts to not affect the existing images, since this change needs more testing. That's also why it only adds machines to the testing org, and why I'm merging this commit into the testing branch for now. Progress towards #1112 and #1402 --- buildkite/create_images.py | 37 +++++++++++++++++++++-- buildkite/docker/build_arm64.sh | 39 ++++++++++++++++++++++++ buildkite/docker/centos7/Dockerfile | 14 ++++++--- buildkite/docker/debian10/Dockerfile | 10 +++---- buildkite/docker/debian11/Dockerfile | 10 +++---- buildkite/docker/push_arm64.sh | 39 ++++++++++++++++++++++++ buildkite/docker/ubuntu1604/Dockerfile | 10 +++---- buildkite/docker/ubuntu1804/Dockerfile | 10 +++---- buildkite/docker/ubuntu2004/Dockerfile | 10 +++---- buildkite/docker/ubuntu2204/Dockerfile | 10 +++---- buildkite/gcloud.py | 2 +- buildkite/gcloud_utils.py | 13 +++++++- buildkite/instances.yml | 6 ++++ buildkite/promote_images.py | 9 ++++++ buildkite/setup-docker.sh | 41 ++++++++++++++++++++++---- buildkite/startup-docker-pdssd.sh | 9 +++++- 16 files changed, 223 insertions(+), 46 deletions(-) create mode 100644 buildkite/docker/build_arm64.sh create mode 100644 buildkite/docker/push_arm64.sh diff --git a/buildkite/create_images.py b/buildkite/create_images.py index f79c082868..dbfd7c74fd 100755 --- a/buildkite/create_images.py +++ b/buildkite/create_images.py @@ -28,6 +28,9 @@ DEBUG = False +DEFAULT_MACHINE_TYPE = "c2-standard-8" +DEFAULT_BOOT_DISK_SIZE = "500GB" + IMAGE_CREATION_VMS = { "bk-testing-docker": { "project": "bazel-public", @@ -40,6 +43,20 @@ "https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx" ], }, + "bk-testing-docker-arm64": { + "project": "bazel-public", + "zone": "us-central1-f", + "source_image_project": "ubuntu-os-cloud", + "source_image_family": "ubuntu-2004-lts-arm64", + "setup_script": "setup-docker.sh", + "guest_os_features": ["VIRTIO_SCSI_MULTIQUEUE"], + "licenses": [ + "https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx" + ], + # https://cloud.google.com/compute/docs/instances/create-arm-vm-instance#armpublicimage + "machine_type": "t2a-standard-8", + "network_interface": "nic-type=GVNIC", + }, "bk-testing-windows": { "project": "bazel-public", "zone": "us-central1-f", @@ -97,16 +114,30 @@ def create_instance(instance_name, params): "image-family": params["source_image_family"], } + network_params = {} + if "network_interface" in params: + if "network" in params: + print( + f"Config error: {instance_name} has both 'network' and " + "'network_interface', which are mutually exclusive.", + file=sys.stdout, + ) + exit(1) + + network_params["network_interface"] = params["network_interface"] + else: + network_params["network"] = (params.get("network", "default"),) + gcloud.create_instance( instance_name, project=params["project"], zone=params["zone"], - machine_type="c2-standard-8", - network=params.get("network", "default"), + machine_type=params.get("machine_type", DEFAULT_MACHINE_TYPE), metadata_from_file=startup_script, boot_disk_type="pd-ssd", - boot_disk_size=params.get("boot_disk_size", "500GB"), + boot_disk_size=params.get("boot_disk_size", DEFAULT_BOOT_DISK_SIZE), **image, + **network_params, ) finally: os.remove(setup_script) diff --git a/buildkite/docker/build_arm64.sh b/buildkite/docker/build_arm64.sh new file mode 100644 index 0000000000..19c4b6e244 --- /dev/null +++ b/buildkite/docker/build_arm64.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# TODO(fweikert): merge this file into build.sh once ARM64 support is no longer experimental + +set -euxo pipefail + +case $(git symbolic-ref --short HEAD) in + master) + PREFIX="bazel-public" + ;; + testing) + PREFIX="bazel-public/testing" + ;; + *) + echo "You must build Docker images either from the master or the testing branch!" + exit 1 +esac + +docker buildx builder prune -a -f +docker buildx buildx create --name cibuilder --use + +# Containers used by Bazel CI +docker buildx build -f centos7/Dockerfile --target centos7 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/centos7" centos7 & +docker buildx build -f debian10/Dockerfile --target debian10-java11 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/debian10-java11" debian10 & +docker buildx build -f debian11/Dockerfile --target debian11-java17 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/debian11-java17" debian11 & +docker buildx build -f ubuntu1604/Dockerfile --target ubuntu1604-java8 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu1604-java8" ubuntu1604 & +docker buildx build -f ubuntu1804/Dockerfile --target ubuntu1804-java11 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu1804-java11" ubuntu1804 & +docker buildx build -f ubuntu2004/Dockerfile --target ubuntu2004-java11 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu2004-java11" ubuntu2004 & +docker buildx build -f ubuntu2204/Dockerfile --target ubuntu2204-java17 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu2204-java17" ubuntu2204 & +wait + +docker buildx build -f centos7/Dockerfile --target centos7-java8 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/centos7-java8" centos7 +docker buildx build -f centos7/Dockerfile --target centos7-java11 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/centos7-java11" centos7 +docker buildx build -f centos7/Dockerfile --target centos7-java11-devtoolset10 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/centos7-java11-devtoolset10" centos7 +docker buildx build -f centos7/Dockerfile --target centos7-releaser --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/centos7-releaser" centos7 +docker buildx build -f ubuntu1604/Dockerfile --target ubuntu1604-bazel-java8 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu1604-bazel-java8" ubuntu1604 +docker buildx build -f ubuntu1804/Dockerfile --target ubuntu1804-bazel-java11 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu1804-bazel-java11" ubuntu1804 +docker buildx build -f ubuntu2004/Dockerfile --target ubuntu2004-bazel-java11 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu2004-bazel-java11" ubuntu2004 +docker buildx build -f ubuntu2004/Dockerfile --target ubuntu2004-java11-kythe --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu2004-java11-kythe" ubuntu2004 +docker buildx build -f ubuntu2204/Dockerfile --target ubuntu2204-bazel-java17 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu2204-bazel-java17" ubuntu2204 diff --git a/buildkite/docker/centos7/Dockerfile b/buildkite/docker/centos7/Dockerfile index faee90a3d2..01560ebdc6 100644 --- a/buildkite/docker/centos7/Dockerfile +++ b/buildkite/docker/centos7/Dockerfile @@ -1,5 +1,5 @@ FROM centos:7 as centos7 -ARG BUILDARCH +ARG TARGETARCH # Install required packages. COPY google-cloud-sdk.repo /etc/yum.repos.d/google-cloud-sdk.repo @@ -48,13 +48,13 @@ RUN localedef -i en_US -f ISO-8859-1 en_US.ISO-8859-1 # Bazelisk RUN LATEST_BAZELISK=$(curl -sSI https://github.com/bazelbuild/bazelisk/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \ - curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${BUILDARCH} && \ + curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${TARGETARCH} && \ chown root:root /usr/local/bin/bazel && \ chmod 0755 /usr/local/bin/bazel # Buildifier RUN LATEST_BUILDIFIER=$(curl -sSI https://github.com/bazelbuild/buildtools/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \ - curl -Lo /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${BUILDARCH} && \ + curl -Lo /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${TARGETARCH} && \ chown root:root /usr/local/bin/buildifier && \ chmod 0755 /usr/local/bin/buildifier @@ -64,9 +64,15 @@ RUN yum install -y java-1.8.0-openjdk-devel && yum clean all FROM centos7 AS centos7-java11 # Unfortunately Azul doesn't publish an RPM package for zulu11 on aarch64, so we have to use the tar.gz version. +RUN RUN mkdir -p /usr/lib/jvm/zulu-11 && \ pushd /usr/lib/jvm/zulu-11 && \ - curl "https://cdn.azul.com/zulu/bin/zulu11.58.23-ca-jdk11.0.16.1-linux_x64.tar.gz" | tar xvz --strip-components=1 && \ + if [ "$TARGETARCH" = "arm64" ]; then \ + export DOWNLOAD_URL="https://cdn.azul.com/zulu-embedded/bin/zulu11.58.23-ca-jdk11.0.16.1-linux_aarch64.tar.gz" ; \ + else \ + export DOWNLOAD_URL="https://cdn.azul.com/zulu/bin/zulu11.58.23-ca-jdk11.0.16.1-linux_x64.tar.gz" ; \ + fi; \ + curl "$DOWNLOAD_URL" | tar xvz --strip-components=1 && \ update-alternatives \ --install /usr/bin/java java /usr/lib/jvm/zulu-11/bin/java 2115200 \ --slave /usr/bin/jaotc jaotc /usr/lib/jvm/zulu-11/bin/jaotc \ diff --git a/buildkite/docker/debian10/Dockerfile b/buildkite/docker/debian10/Dockerfile index 0c74ac0a21..af66afdb2a 100644 --- a/buildkite/docker/debian10/Dockerfile +++ b/buildkite/docker/debian10/Dockerfile @@ -1,5 +1,5 @@ FROM debian:10 as debian10-java11 -ARG BUILDARCH +ARG TARGETARCH ENV DEBIAN_FRONTEND="noninteractive" ENV LANG "C.UTF-8" @@ -56,7 +56,7 @@ RUN apt-get -y update && \ # Allow using sudo inside the container. RUN echo "ALL ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers -ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-${BUILDARCH} +ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-${TARGETARCH} ### Install Google Cloud SDK. ### https://cloud.google.com/sdk/docs/quickstart-debian-ubuntu @@ -69,19 +69,19 @@ RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.c RUN apt-get -y update && \ apt-get -y install apt-transport-https ca-certificates && \ curl -sSL https://download.docker.com/linux/debian/gpg | apt-key add - && \ - add-apt-repository "deb [arch=$BUILDARCH] https://download.docker.com/linux/debian $(lsb_release -cs) stable" && \ + add-apt-repository "deb [arch=$TARGETARCH] https://download.docker.com/linux/debian $(lsb_release -cs) stable" && \ apt-get -y update && \ apt-get -y install docker-ce && \ rm -rf /var/lib/apt/lists/* # Bazelisk RUN LATEST_BAZELISK=$(curl -sSI https://github.com/bazelbuild/bazelisk/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \ - curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${BUILDARCH} && \ + curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${TARGETARCH} && \ chown root:root /usr/local/bin/bazel && \ chmod 0755 /usr/local/bin/bazel # Buildifier RUN LATEST_BUILDIFIER=$(curl -sSI https://github.com/bazelbuild/buildtools/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \ - curl -Lo /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${BUILDARCH} && \ + curl -Lo /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${TARGETARCH} && \ chown root:root /usr/local/bin/buildifier && \ chmod 0755 /usr/local/bin/buildifier diff --git a/buildkite/docker/debian11/Dockerfile b/buildkite/docker/debian11/Dockerfile index cf59c2f4e9..60086ec8fb 100644 --- a/buildkite/docker/debian11/Dockerfile +++ b/buildkite/docker/debian11/Dockerfile @@ -1,5 +1,5 @@ FROM debian:11 as debian11-java17 -ARG BUILDARCH +ARG TARGETARCH ENV DEBIAN_FRONTEND="noninteractive" ENV LANG "C.UTF-8" @@ -56,7 +56,7 @@ RUN apt-get -y update && \ # Allow using sudo inside the container. RUN echo "ALL ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers -ENV JAVA_HOME /usr/lib/jvm/java-17-openjdk-${BUILDARCH} +ENV JAVA_HOME /usr/lib/jvm/java-17-openjdk-${TARGETARCH} ### Install Google Cloud SDK. ### https://cloud.google.com/sdk/docs/quickstart-debian-ubuntu @@ -69,19 +69,19 @@ RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.c RUN apt-get -y update && \ apt-get -y install apt-transport-https ca-certificates && \ curl -sSL https://download.docker.com/linux/debian/gpg | apt-key add - && \ - add-apt-repository "deb [arch=$BUILDARCH] https://download.docker.com/linux/debian $(lsb_release -cs) stable" && \ + add-apt-repository "deb [arch=$TARGETARCH] https://download.docker.com/linux/debian $(lsb_release -cs) stable" && \ apt-get -y update && \ apt-get -y install docker-ce && \ rm -rf /var/lib/apt/lists/* # Bazelisk RUN LATEST_BAZELISK=$(curl -sSI https://github.com/bazelbuild/bazelisk/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \ - curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${BUILDARCH} && \ + curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${TARGETARCH} && \ chown root:root /usr/local/bin/bazel && \ chmod 0755 /usr/local/bin/bazel # Buildifier RUN LATEST_BUILDIFIER=$(curl -sSI https://github.com/bazelbuild/buildtools/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \ - curl -Lo /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${BUILDARCH} && \ + curl -Lo /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${TARGETARCH} && \ chown root:root /usr/local/bin/buildifier && \ chmod 0755 /usr/local/bin/buildifier diff --git a/buildkite/docker/push_arm64.sh b/buildkite/docker/push_arm64.sh new file mode 100644 index 0000000000..f579512daf --- /dev/null +++ b/buildkite/docker/push_arm64.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# TODO(fweikert): merge this file into push.sh once ARM64 support is no longer experimental + +set -euxo pipefail + +case $(git symbolic-ref --short HEAD) in + master) + PREFIX="bazel-public" + ;; + testing) + PREFIX="bazel-public/testing" + ;; + *) + echo "You must build Docker images either from the master or the testing branch!" + exit 1 +esac + +docker buildx builder prune -a -f +docker buildx buildx create --name cibuilder --use + +# Containers used by Bazel CI +docker buildx build --push -f centos7/Dockerfile --target centos7 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/centos7" centos7 & +docker buildx build --push -f debian10/Dockerfile --target debian10-java11 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/debian10-java11" debian10 & +docker buildx build --push -f debian11/Dockerfile --target debian11-java17 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/debian11-java17" debian11 & +docker buildx build --push -f ubuntu1604/Dockerfile --target ubuntu1604-java8 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu1604-java8" ubuntu1604 & +docker buildx build --push -f ubuntu1804/Dockerfile --target ubuntu1804-java11 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu1804-java11" ubuntu1804 & +docker buildx build --push -f ubuntu2004/Dockerfile --target ubuntu2004-java11 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu2004-java11" ubuntu2004 & +docker buildx build --push -f ubuntu2204/Dockerfile --target ubuntu2204-java17 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu2204-java17" ubuntu2204 & +wait + +docker buildx build --push -f centos7/Dockerfile --target centos7-java8 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/centos7-java8" centos7 +docker buildx build --push -f centos7/Dockerfile --target centos7-java11 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/centos7-java11" centos7 +docker buildx build --push -f centos7/Dockerfile --target centos7-java11-devtoolset10 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/centos7-java11-devtoolset10" centos7 +docker buildx build --push -f centos7/Dockerfile --target centos7-releaser --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/centos7-releaser" centos7 +docker buildx build --push -f ubuntu1604/Dockerfile --target ubuntu1604-bazel-java8 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu1604-bazel-java8" ubuntu1604 +docker buildx build --push -f ubuntu1804/Dockerfile --target ubuntu1804-bazel-java11 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu1804-bazel-java11" ubuntu1804 +docker buildx build --push -f ubuntu2004/Dockerfile --target ubuntu2004-bazel-java11 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu2004-bazel-java11" ubuntu2004 +docker buildx build --push -f ubuntu2004/Dockerfile --target ubuntu2004-java11-kythe --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu2004-java11-kythe" ubuntu2004 +docker buildx build --push -f ubuntu2204/Dockerfile --target ubuntu2204-bazel-java17 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu2204-bazel-java17" ubuntu2204 diff --git a/buildkite/docker/ubuntu1604/Dockerfile b/buildkite/docker/ubuntu1604/Dockerfile index 0368587c29..74a4962620 100644 --- a/buildkite/docker/ubuntu1604/Dockerfile +++ b/buildkite/docker/ubuntu1604/Dockerfile @@ -1,5 +1,5 @@ FROM ubuntu:16.04 as ubuntu1604-bazel-java8 -ARG BUILDARCH +ARG TARGETARCH ENV DEBIAN_FRONTEND="noninteractive" ENV LANG "C.UTF-8" @@ -55,7 +55,7 @@ RUN apt-get -y update && \ # Allow using sudo inside the container. RUN echo "ALL ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers -ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-${BUILDARCH} +ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-${TARGETARCH} FROM ubuntu1604-bazel-java8 AS ubuntu1604-java8 @@ -87,19 +87,19 @@ RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.c RUN apt-get -y update && \ apt-get -y install apt-transport-https ca-certificates && \ curl -sSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ - add-apt-repository "deb [arch=$BUILDARCH] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \ + add-apt-repository "deb [arch=$TARGETARCH] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \ apt-get -y update && \ apt-get -y install docker-ce && \ rm -rf /var/lib/apt/lists/* # Bazelisk RUN LATEST_BAZELISK=$(curl -sSI https://github.com/bazelbuild/bazelisk/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \ - curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${BUILDARCH} && \ + curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${TARGETARCH} && \ chown root:root /usr/local/bin/bazel && \ chmod 0755 /usr/local/bin/bazel # Buildifier RUN LATEST_BUILDIFIER=$(curl -sSI https://github.com/bazelbuild/buildtools/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \ - curl -Lo /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${BUILDARCH} && \ + curl -Lo /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${TARGETARCH} && \ chown root:root /usr/local/bin/buildifier && \ chmod 0755 /usr/local/bin/buildifier diff --git a/buildkite/docker/ubuntu1804/Dockerfile b/buildkite/docker/ubuntu1804/Dockerfile index 8595ac39f3..0f7642a1b5 100644 --- a/buildkite/docker/ubuntu1804/Dockerfile +++ b/buildkite/docker/ubuntu1804/Dockerfile @@ -1,5 +1,5 @@ FROM ubuntu:18.04 as ubuntu1804-bazel-java11 -ARG BUILDARCH +ARG TARGETARCH ENV DEBIAN_FRONTEND="noninteractive" ENV LANG "C.UTF-8" @@ -56,7 +56,7 @@ RUN apt-get -y update && \ # Allow using sudo inside the container. RUN echo "ALL ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers -ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-${BUILDARCH} +ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-${TARGETARCH} FROM ubuntu1804-bazel-java11 AS ubuntu1804-java11 @@ -71,19 +71,19 @@ RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.c RUN apt-get -y update && \ apt-get -y install apt-transport-https ca-certificates && \ curl -sSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ - add-apt-repository "deb [arch=$BUILDARCH] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \ + add-apt-repository "deb [arch=$TARGETARCH] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \ apt-get -y update && \ apt-get -y install docker-ce && \ rm -rf /var/lib/apt/lists/* # Bazelisk RUN LATEST_BAZELISK=$(curl -sSI https://github.com/bazelbuild/bazelisk/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \ - curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${BUILDARCH} && \ + curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${TARGETARCH} && \ chown root:root /usr/local/bin/bazel && \ chmod 0755 /usr/local/bin/bazel # Buildifier RUN LATEST_BUILDIFIER=$(curl -sSI https://github.com/bazelbuild/buildtools/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \ - curl -Lo /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${BUILDARCH} && \ + curl -Lo /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${TARGETARCH} && \ chown root:root /usr/local/bin/buildifier && \ chmod 0755 /usr/local/bin/buildifier diff --git a/buildkite/docker/ubuntu2004/Dockerfile b/buildkite/docker/ubuntu2004/Dockerfile index 7d66febb35..7dd28fa9d3 100644 --- a/buildkite/docker/ubuntu2004/Dockerfile +++ b/buildkite/docker/ubuntu2004/Dockerfile @@ -1,5 +1,5 @@ FROM ubuntu:20.04 as ubuntu2004-bazel-java11 -ARG BUILDARCH +ARG TARGETARCH ENV DEBIAN_FRONTEND="noninteractive" ENV LANG "C.UTF-8" @@ -57,7 +57,7 @@ RUN apt-get -y update && \ # Allow using sudo inside the container. RUN echo "ALL ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers -ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-${BUILDARCH} +ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-${TARGETARCH} FROM ubuntu2004-bazel-java11 AS ubuntu2004-java11 @@ -72,20 +72,20 @@ RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.c RUN apt-get -y update && \ apt-get -y install apt-transport-https ca-certificates && \ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ - add-apt-repository "deb [arch=$BUILDARCH] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \ + add-apt-repository "deb [arch=$TARGETARCH] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \ apt-get -y update && \ apt-get -y install docker-ce && \ rm -rf /var/lib/apt/lists/* # Bazelisk RUN LATEST_BAZELISK=$(curl -sSI https://github.com/bazelbuild/bazelisk/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \ - curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${BUILDARCH} && \ + curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${TARGETARCH} && \ chown root:root /usr/local/bin/bazel && \ chmod 0755 /usr/local/bin/bazel # Buildifier RUN LATEST_BUILDIFIER=$(curl -sSI https://github.com/bazelbuild/buildtools/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \ - curl -Lo /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${BUILDARCH} && \ + curl -Lo /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${TARGETARCH} && \ chown root:root /usr/local/bin/buildifier && \ chmod 0755 /usr/local/bin/buildifier diff --git a/buildkite/docker/ubuntu2204/Dockerfile b/buildkite/docker/ubuntu2204/Dockerfile index 3e285137cb..69fd770a43 100644 --- a/buildkite/docker/ubuntu2204/Dockerfile +++ b/buildkite/docker/ubuntu2204/Dockerfile @@ -1,5 +1,5 @@ FROM ubuntu:22.04 as ubuntu2204-bazel-java17 -ARG BUILDARCH +ARG TARGETARCH ENV DEBIAN_FRONTEND="noninteractive" ENV LANG "C.UTF-8" @@ -57,7 +57,7 @@ RUN apt-get -y update && \ # Allow using sudo inside the container. RUN echo "ALL ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers -ENV JAVA_HOME /usr/lib/jvm/java-17-openjdk-${BUILDARCH} +ENV JAVA_HOME /usr/lib/jvm/java-17-openjdk-${TARGETARCH} FROM ubuntu2204-bazel-java17 AS ubuntu2204-java17 @@ -72,19 +72,19 @@ RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.c RUN apt-get -y update && \ apt-get -y install apt-transport-https ca-certificates && \ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ - add-apt-repository "deb [arch=$BUILDARCH] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \ + add-apt-repository "deb [arch=$TARGETARCH] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \ apt-get -y update && \ apt-get -y install docker-ce && \ rm -rf /var/lib/apt/lists/* # Bazelisk RUN LATEST_BAZELISK=$(curl -sSI https://github.com/bazelbuild/bazelisk/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \ - curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${BUILDARCH} && \ + curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${TARGETARCH} && \ chown root:root /usr/local/bin/bazel && \ chmod 0755 /usr/local/bin/bazel # Buildifier RUN LATEST_BUILDIFIER=$(curl -sSI https://github.com/bazelbuild/buildtools/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \ - curl -Lo /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${BUILDARCH} && \ + curl -Lo /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${TARGETARCH} && \ chown root:root /usr/local/bin/buildifier && \ chmod 0755 /usr/local/bin/buildifier diff --git a/buildkite/gcloud.py b/buildkite/gcloud.py index 2407e3110b..6d8b585c96 100755 --- a/buildkite/gcloud.py +++ b/buildkite/gcloud.py @@ -31,7 +31,7 @@ def debug(*args, **kwargs): def is_sequence(seq): - return isinstance(seq, collections.Sequence) and not isinstance(seq, str) + return isinstance(seq, collections.abc.Sequence) and not isinstance(seq, str) def gcloud(*args, **kwargs): diff --git a/buildkite/gcloud_utils.py b/buildkite/gcloud_utils.py index 9649c5bdd2..c381e059fb 100755 --- a/buildkite/gcloud_utils.py +++ b/buildkite/gcloud_utils.py @@ -72,18 +72,29 @@ def print_pretty_logs(instance_name, log): print(lines) +_MAX_ATTEMPTS = 10 + def tail_serial_console(instance_name, project, zone, start=None, until=None): next_start = start if start else "0" + attempt = 1 while True: try: result = gcloud.get_serial_port_output( instance_name, project=project, zone=zone, start=next_start ) + attempt = 1 except subprocess.CalledProcessError as e: if "Could not fetch serial port output: TIMEOUT" in e.stderr: gcloud.debug("tail_serial_console: Retrying after TIMEOUT") continue - gcloud.debug("tail_serial_console: Done, because got exception: {}".format(e)) + + if e.stderr and "The service is currently unavailable" in e.stderr: + if attempt < _MAX_ATTEMPTS: + attempt += 1 + time.sleep(10) + continue + + gcloud.debug("tail_serial_console: Done, because got exception: {} [after {} attempts]".format(e, attempt)) if e.stdout: gcloud.debug("stdout: " + e.stdout) if e.stderr: diff --git a/buildkite/instances.yml b/buildkite/instances.yml index 9e91c098c1..df62cf9412 100644 --- a/buildkite/instances.yml +++ b/buildkite/instances.yml @@ -37,6 +37,12 @@ instance_groups: service_account: buildkite-testing@bazel-untrusted.iam.gserviceaccount.com image_family: bk-testing-docker metadata_from_file: startup-script=startup-docker-pdssd.sh + - name: bk-testing-docker-arm64 + count: 2 + project: bazel-untrusted + service_account: buildkite-testing@bazel-untrusted.iam.gserviceaccount.com + image_family: bk-testing-docker-arm64 + metadata_from_file: startup-script=startup-docker-pdssd.sh - name: bk-trusted-docker count: 8 project: bazel-public diff --git a/buildkite/promote_images.py b/buildkite/promote_images.py index a016a20ebb..442a5f81c0 100755 --- a/buildkite/promote_images.py +++ b/buildkite/promote_images.py @@ -32,6 +32,15 @@ "https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx" ], }, + "bk-docker-arm64": { + "project": "bazel-public", + "source_image_project": "bazel-public", + "source_image_family": "bk-testing-docker-arm64", + "guest_os_features": ["VIRTIO_SCSI_MULTIQUEUE"], + "licenses": [ + "https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx" + ], + }, "bk-windows": { "project": "bazel-public", "source_image_project": "bazel-public", diff --git a/buildkite/setup-docker.sh b/buildkite/setup-docker.sh index b98917c6e4..ce76dfce1b 100755 --- a/buildkite/setup-docker.sh +++ b/buildkite/setup-docker.sh @@ -29,7 +29,7 @@ export DEBIAN_FRONTEND="noninteractive" { apt-get -y update apt-get -y dist-upgrade - apt-get -y install python-is-python3 openjdk-11-jdk-headless unzip + apt-get -y install python-is-python3 openjdk-11-jdk-headless unzip e2fsprogs } ### Disable automatic upgrades, as they can interfere with our startup scripts. @@ -49,7 +49,13 @@ EOF ### Patch the filesystem options to increase I/O performance { - tune2fs -o ^acl,journal_data_writeback,nobarrier /dev/sda1 + if [[ "$(uname -m)" == "aarch64" ]]; + then + PRIMARY_PARTITION="/dev/nvme0n1p1" + else + PRIMARY_PARTITION="/dev/sda1" + fi + tune2fs -o ^acl,journal_data_writeback,nobarrier "$PRIMARY_PARTITION" cat > /etc/fstab <<'EOF' LABEL=cloudimg-rootfs / ext4 defaults,noatime,commit=300,journal_async_commit 0 0 LABEL=UEFI /boot/efi vfat defaults,noatime 0 0 @@ -103,9 +109,16 @@ EOF { apt-get -y install apt-transport-https ca-certificates + if [[ "$(uname -m)" == "aarch64" ]]; + then + ARCH="arm64" + else + ARCH="amd64" + fi + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg echo \ - "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ + "deb [arch=${ARCH} signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null apt-get -y update @@ -184,26 +197,42 @@ EOF mv cmdline-tools latest yes | latest/bin/sdkmanager --licenses > /dev/null || true latest/bin/sdkmanager --update - latest/bin/sdkmanager \ + + # TODO: fix build-tools problem + if [[ "$(uname -m)" == "aarch64" ]]; + then + # Redirect stdout to avoid crashing google_metadata_script_runner. + # sdkmanager prints an animated progress bar, which turns into a single gigantic line on GCE, + # failing with "error while communicating with "startup-script" script: bufio.Scanner: token too long" + latest/bin/sdkmanager \ + "build-tools;31.0.0" > /dev/null + else + latest/bin/sdkmanager \ "build-tools;28.0.2" \ - "build-tools;30.0.3" \ + "build-tools;30.0.3" > /dev/null + fi + + latest/bin/sdkmanager \ "extras;android;m2repository" \ "platform-tools" \ "platforms;android-24" \ "platforms;android-28" \ "platforms;android-29" \ "platforms;android-30" \ - "platforms;android-31" + "platforms;android-31" > /dev/null } ### Fix permissions in /opt. { + echo "[*] Fixing permissions..." chown -R root:root /opt } ### Clean up and trim the filesystem (potentially reduces the final image size). { + echo "[*] Cleaning up..." rm -rf /var/lib/apt/lists/* + echo "[*] Trimming filesystem..." fstrim -v / sleep 3 } diff --git a/buildkite/startup-docker-pdssd.sh b/buildkite/startup-docker-pdssd.sh index c0c1dd9fab..55ce44fba3 100644 --- a/buildkite/startup-docker-pdssd.sh +++ b/buildkite/startup-docker-pdssd.sh @@ -58,11 +58,18 @@ systemctl start docker gcloud auth configure-docker --quiet sudo -H -u buildkite-agent gcloud auth configure-docker --quiet +if [[ "$(uname -m)" == "aarch64" ]]; +then + AGENT_QUEUE="linux_arm64" +else + AGENT_QUEUE="default" +fi + ### Write the Buildkite agent configuration. cat > /etc/buildkite-agent/buildkite-agent.cfg <