diff --git a/buildkite/create_images.py b/buildkite/create_images.py index f79c082868..dbfd7c74fd 100755 --- a/buildkite/create_images.py +++ b/buildkite/create_images.py @@ -28,6 +28,9 @@ DEBUG = False +DEFAULT_MACHINE_TYPE = "c2-standard-8" +DEFAULT_BOOT_DISK_SIZE = "500GB" + IMAGE_CREATION_VMS = { "bk-testing-docker": { "project": "bazel-public", @@ -40,6 +43,20 @@ "https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx" ], }, + "bk-testing-docker-arm64": { + "project": "bazel-public", + "zone": "us-central1-f", + "source_image_project": "ubuntu-os-cloud", + "source_image_family": "ubuntu-2004-lts-arm64", + "setup_script": "setup-docker.sh", + "guest_os_features": ["VIRTIO_SCSI_MULTIQUEUE"], + "licenses": [ + "https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx" + ], + # https://cloud.google.com/compute/docs/instances/create-arm-vm-instance#armpublicimage + "machine_type": "t2a-standard-8", + "network_interface": "nic-type=GVNIC", + }, "bk-testing-windows": { "project": "bazel-public", "zone": "us-central1-f", @@ -97,16 +114,30 @@ def create_instance(instance_name, params): "image-family": params["source_image_family"], } + network_params = {} + if "network_interface" in params: + if "network" in params: + print( + f"Config error: {instance_name} has both 'network' and " + "'network_interface', which are mutually exclusive.", + file=sys.stdout, + ) + exit(1) + + network_params["network_interface"] = params["network_interface"] + else: + network_params["network"] = (params.get("network", "default"),) + gcloud.create_instance( instance_name, project=params["project"], zone=params["zone"], - machine_type="c2-standard-8", - network=params.get("network", "default"), + machine_type=params.get("machine_type", DEFAULT_MACHINE_TYPE), metadata_from_file=startup_script, boot_disk_type="pd-ssd", - boot_disk_size=params.get("boot_disk_size", "500GB"), + boot_disk_size=params.get("boot_disk_size", DEFAULT_BOOT_DISK_SIZE), **image, + **network_params, ) finally: os.remove(setup_script) diff --git a/buildkite/docker/build_arm64.sh b/buildkite/docker/build_arm64.sh new file mode 100644 index 0000000000..19c4b6e244 --- /dev/null +++ b/buildkite/docker/build_arm64.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# TODO(fweikert): merge this file into build.sh once ARM64 support is no longer experimental + +set -euxo pipefail + +case $(git symbolic-ref --short HEAD) in + master) + PREFIX="bazel-public" + ;; + testing) + PREFIX="bazel-public/testing" + ;; + *) + echo "You must build Docker images either from the master or the testing branch!" + exit 1 +esac + +docker buildx builder prune -a -f +docker buildx buildx create --name cibuilder --use + +# Containers used by Bazel CI +docker buildx build -f centos7/Dockerfile --target centos7 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/centos7" centos7 & +docker buildx build -f debian10/Dockerfile --target debian10-java11 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/debian10-java11" debian10 & +docker buildx build -f debian11/Dockerfile --target debian11-java17 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/debian11-java17" debian11 & +docker buildx build -f ubuntu1604/Dockerfile --target ubuntu1604-java8 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu1604-java8" ubuntu1604 & +docker buildx build -f ubuntu1804/Dockerfile --target ubuntu1804-java11 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu1804-java11" ubuntu1804 & +docker buildx build -f ubuntu2004/Dockerfile --target ubuntu2004-java11 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu2004-java11" ubuntu2004 & +docker buildx build -f ubuntu2204/Dockerfile --target ubuntu2204-java17 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu2204-java17" ubuntu2204 & +wait + +docker buildx build -f centos7/Dockerfile --target centos7-java8 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/centos7-java8" centos7 +docker buildx build -f centos7/Dockerfile --target centos7-java11 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/centos7-java11" centos7 +docker buildx build -f centos7/Dockerfile --target centos7-java11-devtoolset10 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/centos7-java11-devtoolset10" centos7 +docker buildx build -f centos7/Dockerfile --target centos7-releaser --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/centos7-releaser" centos7 +docker buildx build -f ubuntu1604/Dockerfile --target ubuntu1604-bazel-java8 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu1604-bazel-java8" ubuntu1604 +docker buildx build -f ubuntu1804/Dockerfile --target ubuntu1804-bazel-java11 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu1804-bazel-java11" ubuntu1804 +docker buildx build -f ubuntu2004/Dockerfile --target ubuntu2004-bazel-java11 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu2004-bazel-java11" ubuntu2004 +docker buildx build -f ubuntu2004/Dockerfile --target ubuntu2004-java11-kythe --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu2004-java11-kythe" ubuntu2004 +docker buildx build -f ubuntu2204/Dockerfile --target ubuntu2204-bazel-java17 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu2204-bazel-java17" ubuntu2204 diff --git a/buildkite/docker/centos7/Dockerfile b/buildkite/docker/centos7/Dockerfile index faee90a3d2..01560ebdc6 100644 --- a/buildkite/docker/centos7/Dockerfile +++ b/buildkite/docker/centos7/Dockerfile @@ -1,5 +1,5 @@ FROM centos:7 as centos7 -ARG BUILDARCH +ARG TARGETARCH # Install required packages. COPY google-cloud-sdk.repo /etc/yum.repos.d/google-cloud-sdk.repo @@ -48,13 +48,13 @@ RUN localedef -i en_US -f ISO-8859-1 en_US.ISO-8859-1 # Bazelisk RUN LATEST_BAZELISK=$(curl -sSI https://github.com/bazelbuild/bazelisk/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \ - curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${BUILDARCH} && \ + curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${TARGETARCH} && \ chown root:root /usr/local/bin/bazel && \ chmod 0755 /usr/local/bin/bazel # Buildifier RUN LATEST_BUILDIFIER=$(curl -sSI https://github.com/bazelbuild/buildtools/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \ - curl -Lo /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${BUILDARCH} && \ + curl -Lo /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${TARGETARCH} && \ chown root:root /usr/local/bin/buildifier && \ chmod 0755 /usr/local/bin/buildifier @@ -64,9 +64,15 @@ RUN yum install -y java-1.8.0-openjdk-devel && yum clean all FROM centos7 AS centos7-java11 # Unfortunately Azul doesn't publish an RPM package for zulu11 on aarch64, so we have to use the tar.gz version. +RUN RUN mkdir -p /usr/lib/jvm/zulu-11 && \ pushd /usr/lib/jvm/zulu-11 && \ - curl "https://cdn.azul.com/zulu/bin/zulu11.58.23-ca-jdk11.0.16.1-linux_x64.tar.gz" | tar xvz --strip-components=1 && \ + if [ "$TARGETARCH" = "arm64" ]; then \ + export DOWNLOAD_URL="https://cdn.azul.com/zulu-embedded/bin/zulu11.58.23-ca-jdk11.0.16.1-linux_aarch64.tar.gz" ; \ + else \ + export DOWNLOAD_URL="https://cdn.azul.com/zulu/bin/zulu11.58.23-ca-jdk11.0.16.1-linux_x64.tar.gz" ; \ + fi; \ + curl "$DOWNLOAD_URL" | tar xvz --strip-components=1 && \ update-alternatives \ --install /usr/bin/java java /usr/lib/jvm/zulu-11/bin/java 2115200 \ --slave /usr/bin/jaotc jaotc /usr/lib/jvm/zulu-11/bin/jaotc \ diff --git a/buildkite/docker/debian10/Dockerfile b/buildkite/docker/debian10/Dockerfile index 0c74ac0a21..af66afdb2a 100644 --- a/buildkite/docker/debian10/Dockerfile +++ b/buildkite/docker/debian10/Dockerfile @@ -1,5 +1,5 @@ FROM debian:10 as debian10-java11 -ARG BUILDARCH +ARG TARGETARCH ENV DEBIAN_FRONTEND="noninteractive" ENV LANG "C.UTF-8" @@ -56,7 +56,7 @@ RUN apt-get -y update && \ # Allow using sudo inside the container. RUN echo "ALL ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers -ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-${BUILDARCH} +ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-${TARGETARCH} ### Install Google Cloud SDK. ### https://cloud.google.com/sdk/docs/quickstart-debian-ubuntu @@ -69,19 +69,19 @@ RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.c RUN apt-get -y update && \ apt-get -y install apt-transport-https ca-certificates && \ curl -sSL https://download.docker.com/linux/debian/gpg | apt-key add - && \ - add-apt-repository "deb [arch=$BUILDARCH] https://download.docker.com/linux/debian $(lsb_release -cs) stable" && \ + add-apt-repository "deb [arch=$TARGETARCH] https://download.docker.com/linux/debian $(lsb_release -cs) stable" && \ apt-get -y update && \ apt-get -y install docker-ce && \ rm -rf /var/lib/apt/lists/* # Bazelisk RUN LATEST_BAZELISK=$(curl -sSI https://github.com/bazelbuild/bazelisk/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \ - curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${BUILDARCH} && \ + curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${TARGETARCH} && \ chown root:root /usr/local/bin/bazel && \ chmod 0755 /usr/local/bin/bazel # Buildifier RUN LATEST_BUILDIFIER=$(curl -sSI https://github.com/bazelbuild/buildtools/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \ - curl -Lo /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${BUILDARCH} && \ + curl -Lo /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${TARGETARCH} && \ chown root:root /usr/local/bin/buildifier && \ chmod 0755 /usr/local/bin/buildifier diff --git a/buildkite/docker/debian11/Dockerfile b/buildkite/docker/debian11/Dockerfile index cf59c2f4e9..60086ec8fb 100644 --- a/buildkite/docker/debian11/Dockerfile +++ b/buildkite/docker/debian11/Dockerfile @@ -1,5 +1,5 @@ FROM debian:11 as debian11-java17 -ARG BUILDARCH +ARG TARGETARCH ENV DEBIAN_FRONTEND="noninteractive" ENV LANG "C.UTF-8" @@ -56,7 +56,7 @@ RUN apt-get -y update && \ # Allow using sudo inside the container. RUN echo "ALL ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers -ENV JAVA_HOME /usr/lib/jvm/java-17-openjdk-${BUILDARCH} +ENV JAVA_HOME /usr/lib/jvm/java-17-openjdk-${TARGETARCH} ### Install Google Cloud SDK. ### https://cloud.google.com/sdk/docs/quickstart-debian-ubuntu @@ -69,19 +69,19 @@ RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.c RUN apt-get -y update && \ apt-get -y install apt-transport-https ca-certificates && \ curl -sSL https://download.docker.com/linux/debian/gpg | apt-key add - && \ - add-apt-repository "deb [arch=$BUILDARCH] https://download.docker.com/linux/debian $(lsb_release -cs) stable" && \ + add-apt-repository "deb [arch=$TARGETARCH] https://download.docker.com/linux/debian $(lsb_release -cs) stable" && \ apt-get -y update && \ apt-get -y install docker-ce && \ rm -rf /var/lib/apt/lists/* # Bazelisk RUN LATEST_BAZELISK=$(curl -sSI https://github.com/bazelbuild/bazelisk/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \ - curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${BUILDARCH} && \ + curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${TARGETARCH} && \ chown root:root /usr/local/bin/bazel && \ chmod 0755 /usr/local/bin/bazel # Buildifier RUN LATEST_BUILDIFIER=$(curl -sSI https://github.com/bazelbuild/buildtools/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \ - curl -Lo /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${BUILDARCH} && \ + curl -Lo /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${TARGETARCH} && \ chown root:root /usr/local/bin/buildifier && \ chmod 0755 /usr/local/bin/buildifier diff --git a/buildkite/docker/push_arm64.sh b/buildkite/docker/push_arm64.sh new file mode 100644 index 0000000000..f579512daf --- /dev/null +++ b/buildkite/docker/push_arm64.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# TODO(fweikert): merge this file into push.sh once ARM64 support is no longer experimental + +set -euxo pipefail + +case $(git symbolic-ref --short HEAD) in + master) + PREFIX="bazel-public" + ;; + testing) + PREFIX="bazel-public/testing" + ;; + *) + echo "You must build Docker images either from the master or the testing branch!" + exit 1 +esac + +docker buildx builder prune -a -f +docker buildx buildx create --name cibuilder --use + +# Containers used by Bazel CI +docker buildx build --push -f centos7/Dockerfile --target centos7 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/centos7" centos7 & +docker buildx build --push -f debian10/Dockerfile --target debian10-java11 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/debian10-java11" debian10 & +docker buildx build --push -f debian11/Dockerfile --target debian11-java17 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/debian11-java17" debian11 & +docker buildx build --push -f ubuntu1604/Dockerfile --target ubuntu1604-java8 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu1604-java8" ubuntu1604 & +docker buildx build --push -f ubuntu1804/Dockerfile --target ubuntu1804-java11 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu1804-java11" ubuntu1804 & +docker buildx build --push -f ubuntu2004/Dockerfile --target ubuntu2004-java11 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu2004-java11" ubuntu2004 & +docker buildx build --push -f ubuntu2204/Dockerfile --target ubuntu2204-java17 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu2204-java17" ubuntu2204 & +wait + +docker buildx build --push -f centos7/Dockerfile --target centos7-java8 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/centos7-java8" centos7 +docker buildx build --push -f centos7/Dockerfile --target centos7-java11 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/centos7-java11" centos7 +docker buildx build --push -f centos7/Dockerfile --target centos7-java11-devtoolset10 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/centos7-java11-devtoolset10" centos7 +docker buildx build --push -f centos7/Dockerfile --target centos7-releaser --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/centos7-releaser" centos7 +docker buildx build --push -f ubuntu1604/Dockerfile --target ubuntu1604-bazel-java8 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu1604-bazel-java8" ubuntu1604 +docker buildx build --push -f ubuntu1804/Dockerfile --target ubuntu1804-bazel-java11 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu1804-bazel-java11" ubuntu1804 +docker buildx build --push -f ubuntu2004/Dockerfile --target ubuntu2004-bazel-java11 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu2004-bazel-java11" ubuntu2004 +docker buildx build --push -f ubuntu2004/Dockerfile --target ubuntu2004-java11-kythe --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu2004-java11-kythe" ubuntu2004 +docker buildx build --push -f ubuntu2204/Dockerfile --target ubuntu2204-bazel-java17 --platform linux/arm64,linux/amd64 -t "gcr.io/$PREFIX/ubuntu2204-bazel-java17" ubuntu2204 diff --git a/buildkite/docker/ubuntu1604/Dockerfile b/buildkite/docker/ubuntu1604/Dockerfile index 0368587c29..74a4962620 100644 --- a/buildkite/docker/ubuntu1604/Dockerfile +++ b/buildkite/docker/ubuntu1604/Dockerfile @@ -1,5 +1,5 @@ FROM ubuntu:16.04 as ubuntu1604-bazel-java8 -ARG BUILDARCH +ARG TARGETARCH ENV DEBIAN_FRONTEND="noninteractive" ENV LANG "C.UTF-8" @@ -55,7 +55,7 @@ RUN apt-get -y update && \ # Allow using sudo inside the container. RUN echo "ALL ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers -ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-${BUILDARCH} +ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-${TARGETARCH} FROM ubuntu1604-bazel-java8 AS ubuntu1604-java8 @@ -87,19 +87,19 @@ RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.c RUN apt-get -y update && \ apt-get -y install apt-transport-https ca-certificates && \ curl -sSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ - add-apt-repository "deb [arch=$BUILDARCH] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \ + add-apt-repository "deb [arch=$TARGETARCH] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \ apt-get -y update && \ apt-get -y install docker-ce && \ rm -rf /var/lib/apt/lists/* # Bazelisk RUN LATEST_BAZELISK=$(curl -sSI https://github.com/bazelbuild/bazelisk/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \ - curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${BUILDARCH} && \ + curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${TARGETARCH} && \ chown root:root /usr/local/bin/bazel && \ chmod 0755 /usr/local/bin/bazel # Buildifier RUN LATEST_BUILDIFIER=$(curl -sSI https://github.com/bazelbuild/buildtools/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \ - curl -Lo /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${BUILDARCH} && \ + curl -Lo /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${TARGETARCH} && \ chown root:root /usr/local/bin/buildifier && \ chmod 0755 /usr/local/bin/buildifier diff --git a/buildkite/docker/ubuntu1804/Dockerfile b/buildkite/docker/ubuntu1804/Dockerfile index 8595ac39f3..0f7642a1b5 100644 --- a/buildkite/docker/ubuntu1804/Dockerfile +++ b/buildkite/docker/ubuntu1804/Dockerfile @@ -1,5 +1,5 @@ FROM ubuntu:18.04 as ubuntu1804-bazel-java11 -ARG BUILDARCH +ARG TARGETARCH ENV DEBIAN_FRONTEND="noninteractive" ENV LANG "C.UTF-8" @@ -56,7 +56,7 @@ RUN apt-get -y update && \ # Allow using sudo inside the container. RUN echo "ALL ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers -ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-${BUILDARCH} +ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-${TARGETARCH} FROM ubuntu1804-bazel-java11 AS ubuntu1804-java11 @@ -71,19 +71,19 @@ RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.c RUN apt-get -y update && \ apt-get -y install apt-transport-https ca-certificates && \ curl -sSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ - add-apt-repository "deb [arch=$BUILDARCH] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \ + add-apt-repository "deb [arch=$TARGETARCH] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \ apt-get -y update && \ apt-get -y install docker-ce && \ rm -rf /var/lib/apt/lists/* # Bazelisk RUN LATEST_BAZELISK=$(curl -sSI https://github.com/bazelbuild/bazelisk/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \ - curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${BUILDARCH} && \ + curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${TARGETARCH} && \ chown root:root /usr/local/bin/bazel && \ chmod 0755 /usr/local/bin/bazel # Buildifier RUN LATEST_BUILDIFIER=$(curl -sSI https://github.com/bazelbuild/buildtools/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \ - curl -Lo /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${BUILDARCH} && \ + curl -Lo /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${TARGETARCH} && \ chown root:root /usr/local/bin/buildifier && \ chmod 0755 /usr/local/bin/buildifier diff --git a/buildkite/docker/ubuntu2004/Dockerfile b/buildkite/docker/ubuntu2004/Dockerfile index 7d66febb35..7dd28fa9d3 100644 --- a/buildkite/docker/ubuntu2004/Dockerfile +++ b/buildkite/docker/ubuntu2004/Dockerfile @@ -1,5 +1,5 @@ FROM ubuntu:20.04 as ubuntu2004-bazel-java11 -ARG BUILDARCH +ARG TARGETARCH ENV DEBIAN_FRONTEND="noninteractive" ENV LANG "C.UTF-8" @@ -57,7 +57,7 @@ RUN apt-get -y update && \ # Allow using sudo inside the container. RUN echo "ALL ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers -ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-${BUILDARCH} +ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-${TARGETARCH} FROM ubuntu2004-bazel-java11 AS ubuntu2004-java11 @@ -72,20 +72,20 @@ RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.c RUN apt-get -y update && \ apt-get -y install apt-transport-https ca-certificates && \ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ - add-apt-repository "deb [arch=$BUILDARCH] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \ + add-apt-repository "deb [arch=$TARGETARCH] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \ apt-get -y update && \ apt-get -y install docker-ce && \ rm -rf /var/lib/apt/lists/* # Bazelisk RUN LATEST_BAZELISK=$(curl -sSI https://github.com/bazelbuild/bazelisk/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \ - curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${BUILDARCH} && \ + curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${TARGETARCH} && \ chown root:root /usr/local/bin/bazel && \ chmod 0755 /usr/local/bin/bazel # Buildifier RUN LATEST_BUILDIFIER=$(curl -sSI https://github.com/bazelbuild/buildtools/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \ - curl -Lo /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${BUILDARCH} && \ + curl -Lo /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${TARGETARCH} && \ chown root:root /usr/local/bin/buildifier && \ chmod 0755 /usr/local/bin/buildifier diff --git a/buildkite/docker/ubuntu2204/Dockerfile b/buildkite/docker/ubuntu2204/Dockerfile index 3e285137cb..69fd770a43 100644 --- a/buildkite/docker/ubuntu2204/Dockerfile +++ b/buildkite/docker/ubuntu2204/Dockerfile @@ -1,5 +1,5 @@ FROM ubuntu:22.04 as ubuntu2204-bazel-java17 -ARG BUILDARCH +ARG TARGETARCH ENV DEBIAN_FRONTEND="noninteractive" ENV LANG "C.UTF-8" @@ -57,7 +57,7 @@ RUN apt-get -y update && \ # Allow using sudo inside the container. RUN echo "ALL ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers -ENV JAVA_HOME /usr/lib/jvm/java-17-openjdk-${BUILDARCH} +ENV JAVA_HOME /usr/lib/jvm/java-17-openjdk-${TARGETARCH} FROM ubuntu2204-bazel-java17 AS ubuntu2204-java17 @@ -72,19 +72,19 @@ RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.c RUN apt-get -y update && \ apt-get -y install apt-transport-https ca-certificates && \ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ - add-apt-repository "deb [arch=$BUILDARCH] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \ + add-apt-repository "deb [arch=$TARGETARCH] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \ apt-get -y update && \ apt-get -y install docker-ce && \ rm -rf /var/lib/apt/lists/* # Bazelisk RUN LATEST_BAZELISK=$(curl -sSI https://github.com/bazelbuild/bazelisk/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \ - curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${BUILDARCH} && \ + curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${TARGETARCH} && \ chown root:root /usr/local/bin/bazel && \ chmod 0755 /usr/local/bin/bazel # Buildifier RUN LATEST_BUILDIFIER=$(curl -sSI https://github.com/bazelbuild/buildtools/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \ - curl -Lo /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${BUILDARCH} && \ + curl -Lo /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${TARGETARCH} && \ chown root:root /usr/local/bin/buildifier && \ chmod 0755 /usr/local/bin/buildifier diff --git a/buildkite/gcloud.py b/buildkite/gcloud.py index 2407e3110b..6d8b585c96 100755 --- a/buildkite/gcloud.py +++ b/buildkite/gcloud.py @@ -31,7 +31,7 @@ def debug(*args, **kwargs): def is_sequence(seq): - return isinstance(seq, collections.Sequence) and not isinstance(seq, str) + return isinstance(seq, collections.abc.Sequence) and not isinstance(seq, str) def gcloud(*args, **kwargs): diff --git a/buildkite/gcloud_utils.py b/buildkite/gcloud_utils.py index 9649c5bdd2..c381e059fb 100755 --- a/buildkite/gcloud_utils.py +++ b/buildkite/gcloud_utils.py @@ -72,18 +72,29 @@ def print_pretty_logs(instance_name, log): print(lines) +_MAX_ATTEMPTS = 10 + def tail_serial_console(instance_name, project, zone, start=None, until=None): next_start = start if start else "0" + attempt = 1 while True: try: result = gcloud.get_serial_port_output( instance_name, project=project, zone=zone, start=next_start ) + attempt = 1 except subprocess.CalledProcessError as e: if "Could not fetch serial port output: TIMEOUT" in e.stderr: gcloud.debug("tail_serial_console: Retrying after TIMEOUT") continue - gcloud.debug("tail_serial_console: Done, because got exception: {}".format(e)) + + if e.stderr and "The service is currently unavailable" in e.stderr: + if attempt < _MAX_ATTEMPTS: + attempt += 1 + time.sleep(10) + continue + + gcloud.debug("tail_serial_console: Done, because got exception: {} [after {} attempts]".format(e, attempt)) if e.stdout: gcloud.debug("stdout: " + e.stdout) if e.stderr: diff --git a/buildkite/instances.yml b/buildkite/instances.yml index 9e91c098c1..df62cf9412 100644 --- a/buildkite/instances.yml +++ b/buildkite/instances.yml @@ -37,6 +37,12 @@ instance_groups: service_account: buildkite-testing@bazel-untrusted.iam.gserviceaccount.com image_family: bk-testing-docker metadata_from_file: startup-script=startup-docker-pdssd.sh + - name: bk-testing-docker-arm64 + count: 2 + project: bazel-untrusted + service_account: buildkite-testing@bazel-untrusted.iam.gserviceaccount.com + image_family: bk-testing-docker-arm64 + metadata_from_file: startup-script=startup-docker-pdssd.sh - name: bk-trusted-docker count: 8 project: bazel-public diff --git a/buildkite/promote_images.py b/buildkite/promote_images.py index a016a20ebb..442a5f81c0 100755 --- a/buildkite/promote_images.py +++ b/buildkite/promote_images.py @@ -32,6 +32,15 @@ "https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx" ], }, + "bk-docker-arm64": { + "project": "bazel-public", + "source_image_project": "bazel-public", + "source_image_family": "bk-testing-docker-arm64", + "guest_os_features": ["VIRTIO_SCSI_MULTIQUEUE"], + "licenses": [ + "https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx" + ], + }, "bk-windows": { "project": "bazel-public", "source_image_project": "bazel-public", diff --git a/buildkite/setup-docker.sh b/buildkite/setup-docker.sh index b98917c6e4..ce76dfce1b 100755 --- a/buildkite/setup-docker.sh +++ b/buildkite/setup-docker.sh @@ -29,7 +29,7 @@ export DEBIAN_FRONTEND="noninteractive" { apt-get -y update apt-get -y dist-upgrade - apt-get -y install python-is-python3 openjdk-11-jdk-headless unzip + apt-get -y install python-is-python3 openjdk-11-jdk-headless unzip e2fsprogs } ### Disable automatic upgrades, as they can interfere with our startup scripts. @@ -49,7 +49,13 @@ EOF ### Patch the filesystem options to increase I/O performance { - tune2fs -o ^acl,journal_data_writeback,nobarrier /dev/sda1 + if [[ "$(uname -m)" == "aarch64" ]]; + then + PRIMARY_PARTITION="/dev/nvme0n1p1" + else + PRIMARY_PARTITION="/dev/sda1" + fi + tune2fs -o ^acl,journal_data_writeback,nobarrier "$PRIMARY_PARTITION" cat > /etc/fstab <<'EOF' LABEL=cloudimg-rootfs / ext4 defaults,noatime,commit=300,journal_async_commit 0 0 LABEL=UEFI /boot/efi vfat defaults,noatime 0 0 @@ -103,9 +109,16 @@ EOF { apt-get -y install apt-transport-https ca-certificates + if [[ "$(uname -m)" == "aarch64" ]]; + then + ARCH="arm64" + else + ARCH="amd64" + fi + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg echo \ - "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ + "deb [arch=${ARCH} signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null apt-get -y update @@ -184,26 +197,42 @@ EOF mv cmdline-tools latest yes | latest/bin/sdkmanager --licenses > /dev/null || true latest/bin/sdkmanager --update - latest/bin/sdkmanager \ + + # TODO: fix build-tools problem + if [[ "$(uname -m)" == "aarch64" ]]; + then + # Redirect stdout to avoid crashing google_metadata_script_runner. + # sdkmanager prints an animated progress bar, which turns into a single gigantic line on GCE, + # failing with "error while communicating with "startup-script" script: bufio.Scanner: token too long" + latest/bin/sdkmanager \ + "build-tools;31.0.0" > /dev/null + else + latest/bin/sdkmanager \ "build-tools;28.0.2" \ - "build-tools;30.0.3" \ + "build-tools;30.0.3" > /dev/null + fi + + latest/bin/sdkmanager \ "extras;android;m2repository" \ "platform-tools" \ "platforms;android-24" \ "platforms;android-28" \ "platforms;android-29" \ "platforms;android-30" \ - "platforms;android-31" + "platforms;android-31" > /dev/null } ### Fix permissions in /opt. { + echo "[*] Fixing permissions..." chown -R root:root /opt } ### Clean up and trim the filesystem (potentially reduces the final image size). { + echo "[*] Cleaning up..." rm -rf /var/lib/apt/lists/* + echo "[*] Trimming filesystem..." fstrim -v / sleep 3 } diff --git a/buildkite/startup-docker-pdssd.sh b/buildkite/startup-docker-pdssd.sh index c0c1dd9fab..55ce44fba3 100644 --- a/buildkite/startup-docker-pdssd.sh +++ b/buildkite/startup-docker-pdssd.sh @@ -58,11 +58,18 @@ systemctl start docker gcloud auth configure-docker --quiet sudo -H -u buildkite-agent gcloud auth configure-docker --quiet +if [[ "$(uname -m)" == "aarch64" ]]; +then + AGENT_QUEUE="linux_arm64" +else + AGENT_QUEUE="default" +fi + ### Write the Buildkite agent configuration. cat > /etc/buildkite-agent/buildkite-agent.cfg <