From ddddc28986c12de2596edd29c8d730f3f86f7883 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Tue, 6 Jun 2023 17:29:00 +0200 Subject: [PATCH 01/75] feat: add scripts for building custom node distributions --- .editorconfig | 11 ++++++ .gitignore | 2 + README.md | 8 ++++ build-image-config/Dockerfile | 31 ++++++++++++++++ build-image-config/README.md | 49 +++++++++++++++++++++++++ build-image-config/entrypoint.sh | 34 +++++++++++++++++ build-image-config/re2_entrypoint.sh | 55 ++++++++++++++++++++++++++++ scripts/build_re2.sh | 22 +++++++++++ scripts/bulid_nodejs.sh | 26 +++++++++++++ scripts/create_bulid_images.sh | 25 +++++++++++++ 10 files changed, 263 insertions(+) create mode 100644 .editorconfig create mode 100644 .gitignore create mode 100644 build-image-config/Dockerfile create mode 100644 build-image-config/README.md create mode 100644 build-image-config/entrypoint.sh create mode 100644 build-image-config/re2_entrypoint.sh create mode 100644 scripts/build_re2.sh create mode 100755 scripts/bulid_nodejs.sh create mode 100644 scripts/create_bulid_images.sh diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..b58823e --- /dev/null +++ b/.editorconfig @@ -0,0 +1,11 @@ +# editorconfig.org +root = true + +[*] +indent_style = space +indent_size = 2 +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true +max_line_length = 120 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..aafda3a --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.vscode +.idea diff --git a/README.md b/README.md index bf59127..4491408 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,10 @@ # kibana-custom-nodejs-builds Contains configuration and sources to build node.js for custom platforms + +TODO: write about the repo structure and the call-chain + +TODOs: + - create buildkite pipeline + - add node.js distribution publish step + - check Docker build, whether it will need uploading + - parameterize tasks for node version diff --git a/build-image-config/Dockerfile b/build-image-config/Dockerfile new file mode 100644 index 0000000..a1666a0 --- /dev/null +++ b/build-image-config/Dockerfile @@ -0,0 +1,31 @@ +FROM centos:7 + +ARG GROUP_ID=1000 +ARG USER_ID=1000 + +RUN groupadd --force --gid $GROUP_ID node +RUN adduser --force --gid $GROUP_ID --uid $USER_ID node + +RUN ulimit -n 1024 \ + && yum install -y epel-release \ + && yum install -y centos-release-scl-rh \ + && yum upgrade -y \ + && yum install -y \ + git \ + curl \ + make \ + python2 \ + python3 \ + ccache \ + xz-utils \ + devtoolset-9 \ + glibc-devel + +COPY --chown=node:node entrypoint.sh /home/node/entrypoint.sh +COPY --chown=node:node re2_entrypoint.sh /home/node/re2_entrypoint.sh + +USER node + +VOLUME /home/node/workdir + +ENTRYPOINT [ "/home/node/entrypoint.sh" ] diff --git a/build-image-config/README.md b/build-image-config/README.md new file mode 100644 index 0000000..ffd14f1 --- /dev/null +++ b/build-image-config/README.md @@ -0,0 +1,49 @@ +## Build multi-platform images (~10-15 minutes) + +```bash +docker run --rm --privileged multiarch/qemu-user-static --reset -p yes +DOCKER_BUILDKIT=1 docker buildx build --progress=plain --push \ + --platform linux/amd64,linux/arm64 \ + --build-arg UID=1000 --build-arg GID=1000 \ + --tag azasypkin/nodejs-custom:18.15.0 . +docker pull --platform linux/amd64 azasypkin/nodejs-custom:18.15.0 +docker pull --platform linux/arm64 azasypkin/nodejs-custom:18.15.0 +``` + +## Build Node.js (x64 - ~20 minutes, arm64 - ~4 hours) + +```bash +curl --create-dirs --output-dir ./workdir/src -fsSLO --compressed \ + https://nodejs.org/download/release/v18.15.0/node-v18.15.0.tar.xz +tar -xf ./workdir/src/node-v18.15.0.tar.xz -C ./workdir/src +docker run --rm -it --platform linux/amd64 \ + -v ./workdir:/home/node/workdir \ + azasypkin/nodejs-custom:18.15.0 \ + https://unofficial-builds.nodejs.org/download/release/ \ + v18.15.0 + +docker run --rm -it --platform linux/arm64 \ + -v ./workdir:/home/node/workdir \ + azasypkin/nodejs-custom:18.15.0 \ + https://unofficial-builds.nodejs.org/download/release/ \ + v18.15.0 +``` + +## Build re2 + +```bash +docker run --rm -it --platform linux/amd64 --entrypoint /home/node/re2_entrypoint.sh \ + -v ./workdir:/home/node/workdir \ + -v ./re2-entrypoint.sh:/home/node/re2_entrypoint.sh \ + azasypkin/nodejs-custom:18.15.0 \ + 1.17.7 \ + 18.15.0 \ + https://github.com/azasypkin/kibana/releases/download/nodej-custom +docker run --rm -it --platform linux/arm64 --entrypoint /home/node/re2_entrypoint.sh \ + -v ./workdir:/home/node/workdir \ + -v ./re2-entrypoint.sh:/home/node/re2_entrypoint.sh \ + azasypkin/nodejs-custom:18.15.0 \ + 1.17.7 \ + 18.15.0 \ + https://github.com/azasypkin/kibana/releases/download/nodej-custom +``` \ No newline at end of file diff --git a/build-image-config/entrypoint.sh b/build-image-config/entrypoint.sh new file mode 100644 index 0000000..fffa7f4 --- /dev/null +++ b/build-image-config/entrypoint.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +set -e +set -x + +release_url_base="$1" +full_version="$2" +config_flags=${3:-""} #"--without-dtrace --without-npm --without-etw" + +if [[ $(arch) == x86_64 ]]; then + architecture="x64"; +else + architecture="arm64" +fi + +cd "/home/node/workdir/src/node-${full_version}" + +# Compile from source +export CCACHE_DIR="/home/node/workdir/.ccache-${architecture}" +export CC="ccache gcc" +export CXX="ccache g++" + +. /opt/rh/devtoolset-9/enable + +make -j"$(getconf _NPROCESSORS_ONLN)" binary V= \ + DESTCPU="$architecture" \ + ARCH="$architecture" \ + VARIATION="glibc-217" \ + DISTTYPE="release" \ + RELEASE_URLBASE="$release_url_base" \ + CONFIG_FLAGS="$config_flags" + +mkdir -p /home/node/workdir/dist/ +mv node-*.tar.?z /home/node/workdir/dist/ diff --git a/build-image-config/re2_entrypoint.sh b/build-image-config/re2_entrypoint.sh new file mode 100644 index 0000000..a8165d2 --- /dev/null +++ b/build-image-config/re2_entrypoint.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +set -e +set -x + +re2_full_version="$1" +node_full_version="$2" +node_download_base_url="$3" + +if [[ $(arch) == x86_64 ]]; then + architecture="x64"; +else + architecture="arm64" +fi + +cd /home/node/workdir +mkdir -p dist/ +mkdir -p src/ + +## Download and unpack Node.js binary if needed. +node_folder_name="node-${node_full_version}-linux-${architecture}" +npm_binary="/home/node/workdir/${node_folder_name}/bin/npm" +if [ ! -f "$npm_binary" ]; then + if [ ! -f "/home/node/workdir/$node_folder_name.tar.xz" ]; then + curl -fsSLO --compressed "${node_download_base_url}/${node_folder_name}.tar.xz" + fi + + tar -xf "/home/node/workdir/${node_folder_name}.tar.xz" +fi + +cd src + +## Download re2 source if needed. +re2_source_folder="/home/node/workdir/src/node-re2-${re2_full_version}" +if [ ! -d "$re2_source_folder" ]; then + git clone --recurse-submodules --depth 1 --branch 1.17.4 https://github.com/uhop/node-re2.git "${re2_source_folder}" +fi + +cd "$re2_source_folder" +export PATH="/home/node/workdir/${node_folder_name}/bin:$PATH" +export DEVELOPMENT_SKIP_GETTING_ASSET=true + +export CCACHE_DIR="/home/node/workdir/.ccache-re2-${architecture}" +export CC="ccache gcc" +export CXX="ccache g++" + +. /opt/rh/devtoolset-9/enable + +npm i --unsafe-perm=true +npm run build --if-present +npm test + +mkdir -p /home/node/workdir/dist/ +cp "${re2_source_folder}/build/Release/re2.node" "/home/node/workdir/dist/linux-${architecture}-108" +gzip -f "/home/node/workdir/dist/linux-${architecture}-108" diff --git a/scripts/build_re2.sh b/scripts/build_re2.sh new file mode 100644 index 0000000..30edc6a --- /dev/null +++ b/scripts/build_re2.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -euo pipefail + +BUILD_IMAGE_NAME="nodejs-custom:18.15.0" + +RE2_FULL_VERSION="1.17.7" # $1 +NODE_FULL_VERSION="v18.15.0" # $2 +NODE_DOWNLOAD_BASE_URL="https://github.com/azasypkin/kibana/releases/download/nodej-custom" # $3, TODO: adjust based on the publish URLs + +docker run --rm -it --platform linux/amd64 --entrypoint /home/node/re2_entrypoint.sh \ + -v ./workdir:/home/node/workdir \ + $BUILD_IMAGE_NAME \ + $RE2_FULL_VERSION \ + $NODE_FULL_VERSION \ + $NODE_DOWNLOAD_BASE_URL + +docker run --rm -it --platform linux/arm64 --entrypoint /home/node/re2_entrypoint.sh \ + -v ./workdir:/home/node/workdir \ + $BUILD_IMAGE_NAME \ + $RE2_FULL_VERSION \ + $NODE_FULL_VERSION \ + $NODE_DOWNLOAD_BASE_URL diff --git a/scripts/bulid_nodejs.sh b/scripts/bulid_nodejs.sh new file mode 100755 index 0000000..c90b05d --- /dev/null +++ b/scripts/bulid_nodejs.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -euo pipefail + +TARGET_NODE_VERSION="v18.15.0" +RELEASE_URL_BASE="https://unofficial-builds.nodejs.org/download/release/" +BUILD_IMAGE_NAME=nodejs-custom:18.15.0 + +echo '--- Downloading node source' +curl --create-dirs --output-dir ./workdir/src -fsSLO --compressed \ + https://nodejs.org/download/release/$TARGET_NODE_VERSION/node-$TARGET_NODE_VERSION.tar.xz +tar -xf ./workdir/src/node-$TARGET_NODE_VERSION.tar.xz -C ./workdir/src + + +echo '--- Buidling node for linux/amd64' +docker run --rm -it --platform linux/amd64 \ + -v ./workdir:/home/node/workdir \ + $BUILD_IMAGE_NAME \ + $RELEASE_URL_BASE \ + $FULL_VERSION + +echo '--- Buidling node for linux/arm64' +docker run --rm -it --platform linux/arm64 \ + -v ./workdir:/home/node/workdir \ + $BUILD_IMAGE_NAME \ + $RELEASE_URL_BASE \ + $FULL_VERSION diff --git a/scripts/create_bulid_images.sh b/scripts/create_bulid_images.sh new file mode 100644 index 0000000..3dbfd48 --- /dev/null +++ b/scripts/create_bulid_images.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +set -euo pipefail + +SCRIPT_DIR=$(dirname $0) +DOCKER_BUILD_CONTEXT_DIR="$SCRIPT_DIR/../build-image-config/" + +echo "--- Starting qemu image" +docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + + +echo "--- Building node.js build images" +DOCKER_BUILDKIT=1 docker buildx build --progress=plain \ + --platform linux/amd64,linux/arm64 \ + --build-arg GROUP_ID=1000 --build-arg USER_ID=1000 \ + --tag nodejs-custom:18.15.0 $DOCKER_BUILD_CONTEXT_DIR + +# echo "--- Building node.js build images" +# DOCKER_BUILDKIT=1 docker buildx build --progress=plain --push \ +# --platform linux/amd64,linux/arm64 \ +# --build-arg GROUP_ID=1000 --build-arg USER_ID=1000 \ +# --tag nodejs-custom:18.15.0 $DOCKER_BUILD_CONTEXT_DIR + +# docker pull --platform linux/amd64 nodejs-custom:18.15.0 +# docker pull --platform linux/arm64 nodejs-custom:18.15.0 From 7e1ef7a02429cbda15b6107e31924e3f79e08994 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 7 Jun 2023 16:46:10 +0200 Subject: [PATCH 02/75] feat: add pipeline definition for the build --- .buildkite/pipeline.yml | 24 ++++++++++++++++++++++++ README.md | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 .buildkite/pipeline.yml diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml new file mode 100644 index 0000000..8cb5e01 --- /dev/null +++ b/.buildkite/pipeline.yml @@ -0,0 +1,24 @@ +env: + GITHUB_COMMIT_STATUS_ENABLED: 'true' + GITHUB_COMMIT_STATUS_CONTEXT: 'buildkite/on-merge' +steps: + - command: scripts/create_build_images.sh + label: Create node.js build images + timeout_in_minutes: 10 + agents: + queue: kibana-default + retry: + automatic: + - exit_status: '*' + limit: 1 + + - command: scripts/build_nodejs.sh + label: Build node.js + agents: + queue: kibana-default + key: build + timeout_in_minutes: 60 + retry: + automatic: + - exit_status: '*' + limit: 1 diff --git a/README.md b/README.md index 4491408..3397676 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,6 @@ TODO: write about the repo structure and the call-chain TODOs: - create buildkite pipeline - - add node.js distribution publish step + - add node.js distribution publish step (should this be published on Github?) - check Docker build, whether it will need uploading - parameterize tasks for node version From b1fc304ea01cc16839eb1a7409a8ce5ca8daf4bc Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 7 Jun 2023 17:03:22 +0200 Subject: [PATCH 03/75] feat: fix typos --- scripts/{bulid_nodejs.sh => build_nodejs.sh} | 0 scripts/{create_bulid_images.sh => create_build_images.sh} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename scripts/{bulid_nodejs.sh => build_nodejs.sh} (100%) rename scripts/{create_bulid_images.sh => create_build_images.sh} (100%) mode change 100644 => 100755 diff --git a/scripts/bulid_nodejs.sh b/scripts/build_nodejs.sh similarity index 100% rename from scripts/bulid_nodejs.sh rename to scripts/build_nodejs.sh diff --git a/scripts/create_bulid_images.sh b/scripts/create_build_images.sh old mode 100644 new mode 100755 similarity index 100% rename from scripts/create_bulid_images.sh rename to scripts/create_build_images.sh From 36f5f9f8ab2cc41f98172bafa05be77c760a5e06 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 7 Jun 2023 17:18:10 +0200 Subject: [PATCH 04/75] feat: try fixing 'privileged mode is incompatible with user namespaces' error --- scripts/create_build_images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/create_build_images.sh b/scripts/create_build_images.sh index 3dbfd48..4a32b3c 100755 --- a/scripts/create_build_images.sh +++ b/scripts/create_build_images.sh @@ -6,7 +6,7 @@ SCRIPT_DIR=$(dirname $0) DOCKER_BUILD_CONTEXT_DIR="$SCRIPT_DIR/../build-image-config/" echo "--- Starting qemu image" -docker run --rm --privileged multiarch/qemu-user-static --reset -p yes +docker run --rm --privileged multiarch/qemu-user-static --reset -p yes --userns host echo "--- Building node.js build images" From 4dd173cae07a135c8efe1d64afd24245aa771ada Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 7 Jun 2023 17:37:25 +0200 Subject: [PATCH 05/75] try without qemu --- scripts/create_build_images.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/create_build_images.sh b/scripts/create_build_images.sh index 4a32b3c..588f898 100755 --- a/scripts/create_build_images.sh +++ b/scripts/create_build_images.sh @@ -5,8 +5,8 @@ set -euo pipefail SCRIPT_DIR=$(dirname $0) DOCKER_BUILD_CONTEXT_DIR="$SCRIPT_DIR/../build-image-config/" -echo "--- Starting qemu image" -docker run --rm --privileged multiarch/qemu-user-static --reset -p yes --userns host +# echo "--- Starting qemu image" +# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes --userns host echo "--- Building node.js build images" From 260f10c48353feec3cf3b3c5dd51d44035f7756b Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 7 Jun 2023 17:43:28 +0200 Subject: [PATCH 06/75] try creating a driver instead --- scripts/create_build_images.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/create_build_images.sh b/scripts/create_build_images.sh index 588f898..dd30b12 100755 --- a/scripts/create_build_images.sh +++ b/scripts/create_build_images.sh @@ -8,6 +8,9 @@ DOCKER_BUILD_CONTEXT_DIR="$SCRIPT_DIR/../build-image-config/" # echo "--- Starting qemu image" # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes --userns host +echo "--- Creating multiarch driver" +docker buildx create --name multiarch --driver docker-container --use + echo "--- Building node.js build images" DOCKER_BUILDKIT=1 docker buildx build --progress=plain \ From 84e6a42d64467f6bfea12af138b686533f32e30c Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 7 Jun 2023 18:01:48 +0200 Subject: [PATCH 07/75] feat: remove --force flag --- build-image-config/Dockerfile | 2 +- scripts/create_build_images.sh | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/build-image-config/Dockerfile b/build-image-config/Dockerfile index a1666a0..a1d6faa 100644 --- a/build-image-config/Dockerfile +++ b/build-image-config/Dockerfile @@ -4,7 +4,7 @@ ARG GROUP_ID=1000 ARG USER_ID=1000 RUN groupadd --force --gid $GROUP_ID node -RUN adduser --force --gid $GROUP_ID --uid $USER_ID node +RUN adduser --gid $GROUP_ID --uid $USER_ID node RUN ulimit -n 1024 \ && yum install -y epel-release \ diff --git a/scripts/create_build_images.sh b/scripts/create_build_images.sh index dd30b12..927d87c 100755 --- a/scripts/create_build_images.sh +++ b/scripts/create_build_images.sh @@ -9,7 +9,9 @@ DOCKER_BUILD_CONTEXT_DIR="$SCRIPT_DIR/../build-image-config/" # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes --userns host echo "--- Creating multiarch driver" -docker buildx create --name multiarch --driver docker-container --use +if [[ $(docker buildx ls | grep multiarch | wc -l) -lt 1 ]]; then + docker buildx create --name multiarch --driver docker-container --use +fi echo "--- Building node.js build images" From 8f86ed5b71cd5f4901d4ed1ca5a257199db6a77d Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 7 Jun 2023 18:23:16 +0200 Subject: [PATCH 08/75] feat: increase timeouts and add wait between stages --- .buildkite/pipeline.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 8cb5e01..3467d6e 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,10 +1,7 @@ -env: - GITHUB_COMMIT_STATUS_ENABLED: 'true' - GITHUB_COMMIT_STATUS_CONTEXT: 'buildkite/on-merge' steps: - command: scripts/create_build_images.sh label: Create node.js build images - timeout_in_minutes: 10 + timeout_in_minutes: 30 agents: queue: kibana-default retry: @@ -12,12 +9,14 @@ steps: - exit_status: '*' limit: 1 + - wait + - command: scripts/build_nodejs.sh label: Build node.js agents: queue: kibana-default key: build - timeout_in_minutes: 60 + timeout_in_minutes: 100 retry: automatic: - exit_status: '*' From 545d8d10a3b30189e5b26e7d1781288bc981e15c Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 7 Jun 2023 18:44:31 +0200 Subject: [PATCH 09/75] feat: fix bad param name in build script --- .gitignore | 2 ++ scripts/build_nodejs.sh | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index aafda3a..bd29c04 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ .vscode .idea + +workdir diff --git a/scripts/build_nodejs.sh b/scripts/build_nodejs.sh index c90b05d..6c7c27f 100755 --- a/scripts/build_nodejs.sh +++ b/scripts/build_nodejs.sh @@ -16,11 +16,11 @@ docker run --rm -it --platform linux/amd64 \ -v ./workdir:/home/node/workdir \ $BUILD_IMAGE_NAME \ $RELEASE_URL_BASE \ - $FULL_VERSION + $TARGET_NODE_VERSION echo '--- Buidling node for linux/arm64' docker run --rm -it --platform linux/arm64 \ -v ./workdir:/home/node/workdir \ $BUILD_IMAGE_NAME \ $RELEASE_URL_BASE \ - $FULL_VERSION + $TARGET_NODE_VERSION From 2104c0c60c3f2a8a87d2bb67fbb772dae2325314 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Thu, 8 Jun 2023 10:51:44 +0200 Subject: [PATCH 10/75] feat: push/pull images to docker.io/elastic --- scripts/build_nodejs.sh | 5 +++-- scripts/create_build_images.sh | 14 ++++---------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/scripts/build_nodejs.sh b/scripts/build_nodejs.sh index 6c7c27f..1757fb7 100755 --- a/scripts/build_nodejs.sh +++ b/scripts/build_nodejs.sh @@ -1,9 +1,10 @@ #!/bin/bash set -euo pipefail -TARGET_NODE_VERSION="v18.15.0" +TARGET_VERSION="18.15.0" +TARGET_NODE_VERSION="v$TARGET_VERSION" RELEASE_URL_BASE="https://unofficial-builds.nodejs.org/download/release/" -BUILD_IMAGE_NAME=nodejs-custom:18.15.0 +BUILD_IMAGE_NAME="docker.io/elastic/nodejs-custom:$TARGET_VERSION" echo '--- Downloading node source' curl --create-dirs --output-dir ./workdir/src -fsSLO --compressed \ diff --git a/scripts/create_build_images.sh b/scripts/create_build_images.sh index 927d87c..d3d6822 100755 --- a/scripts/create_build_images.sh +++ b/scripts/create_build_images.sh @@ -4,6 +4,8 @@ set -euo pipefail SCRIPT_DIR=$(dirname $0) DOCKER_BUILD_CONTEXT_DIR="$SCRIPT_DIR/../build-image-config/" +TARGET_VERSION="18.15.0" +IMAGE_NAME="docker.io/elastic/nodejs-custom:$TARGET_VERSION" # echo "--- Starting qemu image" # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes --userns host @@ -18,13 +20,5 @@ echo "--- Building node.js build images" DOCKER_BUILDKIT=1 docker buildx build --progress=plain \ --platform linux/amd64,linux/arm64 \ --build-arg GROUP_ID=1000 --build-arg USER_ID=1000 \ - --tag nodejs-custom:18.15.0 $DOCKER_BUILD_CONTEXT_DIR - -# echo "--- Building node.js build images" -# DOCKER_BUILDKIT=1 docker buildx build --progress=plain --push \ -# --platform linux/amd64,linux/arm64 \ -# --build-arg GROUP_ID=1000 --build-arg USER_ID=1000 \ -# --tag nodejs-custom:18.15.0 $DOCKER_BUILD_CONTEXT_DIR - -# docker pull --platform linux/amd64 nodejs-custom:18.15.0 -# docker pull --platform linux/arm64 nodejs-custom:18.15.0 + --tag $IMAGE_NAME --push \ + $DOCKER_BUILD_CONTEXT_DIR From d0ac37f1ac5b1ea6a11eac39604099a979349dea Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Thu, 8 Jun 2023 17:19:16 +0200 Subject: [PATCH 11/75] feat: use new images, branch in build script based on ARCH env variable --- .buildkite/pipeline.yml | 17 ++++++++++------- scripts/build_nodejs.sh | 37 ++++++++++++++++++++++++------------- 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 3467d6e..b9fd82c 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,20 +1,23 @@ steps: - - command: scripts/create_build_images.sh - label: Create node.js build images - timeout_in_minutes: 30 + - command: scripts/build_nodejs.sh + env: + ARCH: arm64 + label: Build node.js agents: - queue: kibana-default + image: docker.elastic.co/ci-agent-images/kibana/custom-nodejs-builder:amd64-0.1 + key: build + timeout_in_minutes: 100 retry: automatic: - exit_status: '*' limit: 1 - - wait - - command: scripts/build_nodejs.sh + env: + ARCH: arm64 label: Build node.js agents: - queue: kibana-default + image: docker.elastic.co/ci-agent-images/kibana/custom-nodejs-builder:arm64-0.1 key: build timeout_in_minutes: 100 retry: diff --git a/scripts/build_nodejs.sh b/scripts/build_nodejs.sh index 1757fb7..ca3594f 100755 --- a/scripts/build_nodejs.sh +++ b/scripts/build_nodejs.sh @@ -6,22 +6,33 @@ TARGET_NODE_VERSION="v$TARGET_VERSION" RELEASE_URL_BASE="https://unofficial-builds.nodejs.org/download/release/" BUILD_IMAGE_NAME="docker.io/elastic/nodejs-custom:$TARGET_VERSION" +if [[ "$ARCH" == "arm64" || "$ARCH" == "amd64" ]]; then + # we're good, supported architecture + echo "Building for architecture: $ARCH" +else + echo "ARCH (=$ARCH) env variable is not one of: arm64, amd64" + exit 1 +fi + + echo '--- Downloading node source' curl --create-dirs --output-dir ./workdir/src -fsSLO --compressed \ https://nodejs.org/download/release/$TARGET_NODE_VERSION/node-$TARGET_NODE_VERSION.tar.xz tar -xf ./workdir/src/node-$TARGET_NODE_VERSION.tar.xz -C ./workdir/src -echo '--- Buidling node for linux/amd64' -docker run --rm -it --platform linux/amd64 \ - -v ./workdir:/home/node/workdir \ - $BUILD_IMAGE_NAME \ - $RELEASE_URL_BASE \ - $TARGET_NODE_VERSION - -echo '--- Buidling node for linux/arm64' -docker run --rm -it --platform linux/arm64 \ - -v ./workdir:/home/node/workdir \ - $BUILD_IMAGE_NAME \ - $RELEASE_URL_BASE \ - $TARGET_NODE_VERSION +if [[ "$ARCH" == "arm64" ]]; then + echo '--- Buidling node for linux/amd64' + docker run --rm -it --platform linux/amd64 \ + -v ./workdir:/home/node/workdir \ + $BUILD_IMAGE_NAME \ + $RELEASE_URL_BASE \ + $TARGET_NODE_VERSION +elif [[ "$ARCH" == "amd64" ]]; then + echo '--- Buidling node for linux/arm64' + docker run --rm -it --platform linux/arm64 \ + -v ./workdir:/home/node/workdir \ + $BUILD_IMAGE_NAME \ + $RELEASE_URL_BASE \ + $TARGET_NODE_VERSION +fi From 3cca2492f47f7ce37a66003ef8730b0f74fb9292 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Fri, 9 Jun 2023 15:37:52 +0200 Subject: [PATCH 12/75] Try manual login/build/upload to elastic's docker repo --- .buildkite/pipeline.yml | 24 ++++++++++++++++++------ scripts/build_nodejs.sh | 2 +- scripts/create_build_images.sh | 5 +---- scripts/docker_login.sh | 9 +++++++++ 4 files changed, 29 insertions(+), 11 deletions(-) create mode 100644 scripts/docker_login.sh diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index b9fd82c..56f5424 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,24 +1,36 @@ steps: + - command: scripts/docker_login.sh + + - wait + + - command: + - scripts/create_build_images.sh + - docker image inspect docker.io/elastic/nodejs-custom:18.15.0 | grep Arch + label: Create build images + timeout_in_minutes: 100 + retry: + automatic: + - exit_status: '*' + limit: 1 + + - wait + - command: scripts/build_nodejs.sh env: ARCH: arm64 label: Build node.js - agents: - image: docker.elastic.co/ci-agent-images/kibana/custom-nodejs-builder:amd64-0.1 - key: build timeout_in_minutes: 100 retry: automatic: - exit_status: '*' limit: 1 + - wait + - command: scripts/build_nodejs.sh env: ARCH: arm64 label: Build node.js - agents: - image: docker.elastic.co/ci-agent-images/kibana/custom-nodejs-builder:arm64-0.1 - key: build timeout_in_minutes: 100 retry: automatic: diff --git a/scripts/build_nodejs.sh b/scripts/build_nodejs.sh index ca3594f..c06d3d2 100755 --- a/scripts/build_nodejs.sh +++ b/scripts/build_nodejs.sh @@ -4,7 +4,7 @@ set -euo pipefail TARGET_VERSION="18.15.0" TARGET_NODE_VERSION="v$TARGET_VERSION" RELEASE_URL_BASE="https://unofficial-builds.nodejs.org/download/release/" -BUILD_IMAGE_NAME="docker.io/elastic/nodejs-custom:$TARGET_VERSION" +BUILD_IMAGE_NAME="docker.elastic.co/elastic/nodejs-custom:$TARGET_VERSION" if [[ "$ARCH" == "arm64" || "$ARCH" == "amd64" ]]; then # we're good, supported architecture diff --git a/scripts/create_build_images.sh b/scripts/create_build_images.sh index d3d6822..010e79e 100755 --- a/scripts/create_build_images.sh +++ b/scripts/create_build_images.sh @@ -5,10 +5,7 @@ set -euo pipefail SCRIPT_DIR=$(dirname $0) DOCKER_BUILD_CONTEXT_DIR="$SCRIPT_DIR/../build-image-config/" TARGET_VERSION="18.15.0" -IMAGE_NAME="docker.io/elastic/nodejs-custom:$TARGET_VERSION" - -# echo "--- Starting qemu image" -# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes --userns host +IMAGE_NAME="docker.elastic.co/elastic/nodejs-custom:$TARGET_VERSION" echo "--- Creating multiarch driver" if [[ $(docker buildx ls | grep multiarch | wc -l) -lt 1 ]]; then diff --git a/scripts/docker_login.sh b/scripts/docker_login.sh new file mode 100644 index 0000000..e7fbdb0 --- /dev/null +++ b/scripts/docker_login.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -euo pipefail + +registry_user=$(vault read -field=username secret/ci/elastic-ci-agent-images/docker-registry) +registry_password=$(vault read -field=password secret/ci/elastic-ci-agent-images/docker-registry) + +echo -n "Logging in to docker.elastic.co as ${registry_user}... " +docker login --username="${registry_user}" --password="${registry_password}" docker.elastic.co From 5ac745d234f967ba9eeafb3641e0d74e4106f6e9 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Fri, 9 Jun 2023 15:59:51 +0200 Subject: [PATCH 13/75] add agent definition --- .buildkite/pipeline.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 56f5424..a88cd4a 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,3 +1,6 @@ +agents: + queue: kibana-default + steps: - command: scripts/docker_login.sh From ef3d7bc04cb9e56a41d0c56fe70c253ecd13cf1f Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Mon, 12 Jun 2023 10:11:44 +0200 Subject: [PATCH 14/75] Load docker image before progressing to next step --- .buildkite/pipeline.yml | 27 +++++++++++++-------------- scripts/create_build_images.sh | 7 +++++-- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index a88cd4a..37da9ca 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -2,13 +2,12 @@ agents: queue: kibana-default steps: - - command: scripts/docker_login.sh + # - command: scripts/docker_login.sh - - wait + # - wait - command: - scripts/create_build_images.sh - - docker image inspect docker.io/elastic/nodejs-custom:18.15.0 | grep Arch label: Create build images timeout_in_minutes: 100 retry: @@ -20,7 +19,7 @@ steps: - command: scripts/build_nodejs.sh env: - ARCH: arm64 + ARCH: amd64 label: Build node.js timeout_in_minutes: 100 retry: @@ -28,14 +27,14 @@ steps: - exit_status: '*' limit: 1 - - wait + # - wait - - command: scripts/build_nodejs.sh - env: - ARCH: arm64 - label: Build node.js - timeout_in_minutes: 100 - retry: - automatic: - - exit_status: '*' - limit: 1 + # - command: scripts/build_nodejs.sh + # env: + # ARCH: arm64 + # label: Build node.js + # timeout_in_minutes: 100 + # retry: + # automatic: + # - exit_status: '*' + # limit: 1 diff --git a/scripts/create_build_images.sh b/scripts/create_build_images.sh index 010e79e..8d6465f 100755 --- a/scripts/create_build_images.sh +++ b/scripts/create_build_images.sh @@ -12,10 +12,13 @@ if [[ $(docker buildx ls | grep multiarch | wc -l) -lt 1 ]]; then docker buildx create --name multiarch --driver docker-container --use fi +# TARGET_PLATFORMS="linux/amd64,linux/arm64" +TARGET_PLATFORMS="linux/amd64" echo "--- Building node.js build images" DOCKER_BUILDKIT=1 docker buildx build --progress=plain \ - --platform linux/amd64,linux/arm64 \ + --platform $TARGET_PLATFORMS \ --build-arg GROUP_ID=1000 --build-arg USER_ID=1000 \ - --tag $IMAGE_NAME --push \ + --load \ + --tag $IMAGE_NAME \ $DOCKER_BUILD_CONTEXT_DIR From 07a3796063417b9bc13c4432859efc3909f69717 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Mon, 12 Jun 2023 10:35:25 +0200 Subject: [PATCH 15/75] Try building in the same step --- .buildkite/pipeline.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 37da9ca..0a81ef7 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -8,19 +8,10 @@ steps: - command: - scripts/create_build_images.sh + - scripts/build_nodejs.sh label: Create build images - timeout_in_minutes: 100 - retry: - automatic: - - exit_status: '*' - limit: 1 - - - wait - - - command: scripts/build_nodejs.sh env: ARCH: amd64 - label: Build node.js timeout_in_minutes: 100 retry: automatic: From f375c10031c20acc350763d504fd04f8b0048618 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Mon, 12 Jun 2023 10:55:34 +0200 Subject: [PATCH 16/75] List images, to see why they're not available --- scripts/build_nodejs.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/build_nodejs.sh b/scripts/build_nodejs.sh index c06d3d2..1b7f1f7 100755 --- a/scripts/build_nodejs.sh +++ b/scripts/build_nodejs.sh @@ -14,6 +14,9 @@ else exit 1 fi +echo "--- Listing docker images" +docker image ls + echo '--- Downloading node source' curl --create-dirs --output-dir ./workdir/src -fsSLO --compressed \ From 10c26ebc75d931f9cc42ce71ddab199ea8a46e7f Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Mon, 12 Jun 2023 11:04:29 +0200 Subject: [PATCH 17/75] Fix arm-amd mixup --- scripts/build_nodejs.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/build_nodejs.sh b/scripts/build_nodejs.sh index 1b7f1f7..01b1a2b 100755 --- a/scripts/build_nodejs.sh +++ b/scripts/build_nodejs.sh @@ -25,15 +25,15 @@ tar -xf ./workdir/src/node-$TARGET_NODE_VERSION.tar.xz -C ./workdir/src if [[ "$ARCH" == "arm64" ]]; then - echo '--- Buidling node for linux/amd64' - docker run --rm -it --platform linux/amd64 \ + echo '--- Buidling node for linux/arm64' + docker run --rm -it --platform linux/arm64 \ -v ./workdir:/home/node/workdir \ $BUILD_IMAGE_NAME \ $RELEASE_URL_BASE \ $TARGET_NODE_VERSION elif [[ "$ARCH" == "amd64" ]]; then - echo '--- Buidling node for linux/arm64' - docker run --rm -it --platform linux/arm64 \ + echo '--- Buidling node for linux/amd64' + docker run --rm -it --platform linux/amd64 \ -v ./workdir:/home/node/workdir \ $BUILD_IMAGE_NAME \ $RELEASE_URL_BASE \ From 4859c070e5c7245d4cfb51752356af9ba2c18016 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Mon, 12 Jun 2023 11:27:49 +0200 Subject: [PATCH 18/75] Add +x flag to entrypoint scripts --- build-image-config/entrypoint.sh | 0 build-image-config/re2_entrypoint.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 build-image-config/entrypoint.sh mode change 100644 => 100755 build-image-config/re2_entrypoint.sh diff --git a/build-image-config/entrypoint.sh b/build-image-config/entrypoint.sh old mode 100644 new mode 100755 diff --git a/build-image-config/re2_entrypoint.sh b/build-image-config/re2_entrypoint.sh old mode 100644 new mode 100755 From 4e5374870f91b596e517f8f2f4cdeac4f55df4db Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Mon, 12 Jun 2023 11:53:52 +0200 Subject: [PATCH 19/75] List dirs, wonder why they're not accessible --- build-image-config/entrypoint.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-image-config/entrypoint.sh b/build-image-config/entrypoint.sh index fffa7f4..ae1544c 100755 --- a/build-image-config/entrypoint.sh +++ b/build-image-config/entrypoint.sh @@ -13,6 +13,9 @@ else architecture="arm64" fi +ls -la "/home/node/workdir/src/" +ls -la "/home/node/workdir/src/node-${full_version}" + cd "/home/node/workdir/src/node-${full_version}" # Compile from source From 17acf3579a9709bad7da67fed6f52ffa2f11085d Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Mon, 12 Jun 2023 12:27:40 +0200 Subject: [PATCH 20/75] Add Z flag, to access the directories --- scripts/build_nodejs.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/scripts/build_nodejs.sh b/scripts/build_nodejs.sh index 01b1a2b..a72da9a 100755 --- a/scripts/build_nodejs.sh +++ b/scripts/build_nodejs.sh @@ -14,27 +14,24 @@ else exit 1 fi -echo "--- Listing docker images" -docker image ls - - echo '--- Downloading node source' curl --create-dirs --output-dir ./workdir/src -fsSLO --compressed \ https://nodejs.org/download/release/$TARGET_NODE_VERSION/node-$TARGET_NODE_VERSION.tar.xz tar -xf ./workdir/src/node-$TARGET_NODE_VERSION.tar.xz -C ./workdir/src +chmod -R a+rwx ./workdir/ if [[ "$ARCH" == "arm64" ]]; then echo '--- Buidling node for linux/arm64' docker run --rm -it --platform linux/arm64 \ - -v ./workdir:/home/node/workdir \ + -v ./workdir:/home/node/workdir:Z \ $BUILD_IMAGE_NAME \ $RELEASE_URL_BASE \ $TARGET_NODE_VERSION elif [[ "$ARCH" == "amd64" ]]; then echo '--- Buidling node for linux/amd64' docker run --rm -it --platform linux/amd64 \ - -v ./workdir:/home/node/workdir \ + -v ./workdir:/home/node/workdir:Z \ $BUILD_IMAGE_NAME \ $RELEASE_URL_BASE \ $TARGET_NODE_VERSION From 4583258955b96b2255c3bc8e64b626555a8e237f Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Mon, 12 Jun 2023 14:48:21 +0200 Subject: [PATCH 21/75] Increase timeout, try build in different steps --- .buildkite/pipeline.yml | 27 ++++++++------------------- scripts/create_build_images.sh | 3 +-- 2 files changed, 9 insertions(+), 21 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 0a81ef7..75eb6f6 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -2,30 +2,19 @@ agents: queue: kibana-default steps: - # - command: scripts/docker_login.sh + - command: scripts/create_build_images.sh - # - wait + - wait - command: - - scripts/create_build_images.sh - scripts/build_nodejs.sh label: Create build images env: ARCH: amd64 - timeout_in_minutes: 100 - retry: - automatic: - - exit_status: '*' - limit: 1 + timeout_in_minutes: 300 - # - wait - - # - command: scripts/build_nodejs.sh - # env: - # ARCH: arm64 - # label: Build node.js - # timeout_in_minutes: 100 - # retry: - # automatic: - # - exit_status: '*' - # limit: 1 + - command: scripts/build_nodejs.sh + env: + ARCH: arm64 + label: Build node.js + timeout_in_minutes: 600 diff --git a/scripts/create_build_images.sh b/scripts/create_build_images.sh index 8d6465f..720b92f 100755 --- a/scripts/create_build_images.sh +++ b/scripts/create_build_images.sh @@ -12,8 +12,7 @@ if [[ $(docker buildx ls | grep multiarch | wc -l) -lt 1 ]]; then docker buildx create --name multiarch --driver docker-container --use fi -# TARGET_PLATFORMS="linux/amd64,linux/arm64" -TARGET_PLATFORMS="linux/amd64" +TARGET_PLATFORMS="linux/amd64,linux/arm64" echo "--- Building node.js build images" DOCKER_BUILDKIT=1 docker buildx build --progress=plain \ From b45822e3d1dc83c1c00bc3a6ab1e447666e3b439 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Mon, 12 Jun 2023 15:25:07 +0200 Subject: [PATCH 22/75] Separate builds because 'ERROR: docker exporter does not currently support exporting manifest lists' --- .buildkite/pipeline-kibana.yml | 23 +++++++++++++++++++++ .buildkite/pipeline.yml | 18 ++++++++--------- scripts/create_build_images.sh | 37 +++++++++++++++++++++++++--------- 3 files changed, 60 insertions(+), 18 deletions(-) create mode 100644 .buildkite/pipeline-kibana.yml diff --git a/.buildkite/pipeline-kibana.yml b/.buildkite/pipeline-kibana.yml new file mode 100644 index 0000000..efbd45d --- /dev/null +++ b/.buildkite/pipeline-kibana.yml @@ -0,0 +1,23 @@ +agents: + queue: kibana-default + +steps: + - command: scripts/create_build_images.sh + label: Creating build images + env: + ARCH: amd64 + + - wait + + - command: + - scripts/build_nodejs.sh + label: Building node.js + env: + ARCH: amd64 + timeout_in_minutes: 300 + + # - command: scripts/build_nodejs.sh + # env: + # ARCH: arm64 + # label: Build node.js + # timeout_in_minutes: 600 diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 75eb6f6..ae78848 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,20 +1,20 @@ -agents: - queue: kibana-default - steps: - command: scripts/create_build_images.sh + label: Creating build images + env: + ARCH: amd64 - wait - command: - scripts/build_nodejs.sh - label: Create build images + label: Building node.js env: ARCH: amd64 timeout_in_minutes: 300 - - command: scripts/build_nodejs.sh - env: - ARCH: arm64 - label: Build node.js - timeout_in_minutes: 600 + # - command: scripts/build_nodejs.sh + # env: + # ARCH: arm64 + # label: Build node.js + # timeout_in_minutes: 600 diff --git a/scripts/create_build_images.sh b/scripts/create_build_images.sh index 720b92f..2a83b67 100755 --- a/scripts/create_build_images.sh +++ b/scripts/create_build_images.sh @@ -5,19 +5,38 @@ set -euo pipefail SCRIPT_DIR=$(dirname $0) DOCKER_BUILD_CONTEXT_DIR="$SCRIPT_DIR/../build-image-config/" TARGET_VERSION="18.15.0" -IMAGE_NAME="docker.elastic.co/elastic/nodejs-custom:$TARGET_VERSION" + +if [[ "$ARCH" == "arm64" || "$ARCH" == "amd64" ]]; then + # we're good, supported architecture + echo "Building for architecture: $ARCH" +else + echo "ARCH (=$ARCH) env variable is not one of: arm64, amd64" + exit 1 +fi + +TARGET_PLATFORM="linux/$ARCH" +IMAGE_NAME="docker.elastic.co/elastic/nodejs-custom:$TARGET_VERSION-$ARCH" echo "--- Creating multiarch driver" if [[ $(docker buildx ls | grep multiarch | wc -l) -lt 1 ]]; then docker buildx create --name multiarch --driver docker-container --use fi -TARGET_PLATFORMS="linux/amd64,linux/arm64" -echo "--- Building node.js build images" -DOCKER_BUILDKIT=1 docker buildx build --progress=plain \ - --platform $TARGET_PLATFORMS \ - --build-arg GROUP_ID=1000 --build-arg USER_ID=1000 \ - --load \ - --tag $IMAGE_NAME \ - $DOCKER_BUILD_CONTEXT_DIR +if [[ "$ARCH" == "arm64" ]]; then + echo "--- Building node.js build images (arm64)" + DOCKER_BUILDKIT=1 docker buildx build --progress=plain \ + --platform $TARGET_PLATFORM \ + --build-arg GROUP_ID=1000 --build-arg USER_ID=1000 \ + --load \ + --tag $IMAGE_NAME \ + $DOCKER_BUILD_CONTEXT_DIR +elif [[ "$ARCH" == "arm64" ]]; then + echo "--- Building node.js build images (amd64)" + DOCKER_BUILDKIT=1 docker buildx build --progress=plain \ + --platform $TARGET_PLATFORM \ + --build-arg GROUP_ID=1000 --build-arg USER_ID=1000 \ + --load \ + --tag $IMAGE_NAME \ + $DOCKER_BUILD_CONTEXT_DIR +fi From 15e9c5934915456847aa96570eb72592e01d9ab5 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Mon, 12 Jun 2023 16:11:26 +0200 Subject: [PATCH 23/75] Fix bad image names and conditionals --- scripts/build_nodejs.sh | 3 ++- scripts/create_build_images.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/build_nodejs.sh b/scripts/build_nodejs.sh index a72da9a..71a3bfe 100755 --- a/scripts/build_nodejs.sh +++ b/scripts/build_nodejs.sh @@ -4,7 +4,6 @@ set -euo pipefail TARGET_VERSION="18.15.0" TARGET_NODE_VERSION="v$TARGET_VERSION" RELEASE_URL_BASE="https://unofficial-builds.nodejs.org/download/release/" -BUILD_IMAGE_NAME="docker.elastic.co/elastic/nodejs-custom:$TARGET_VERSION" if [[ "$ARCH" == "arm64" || "$ARCH" == "amd64" ]]; then # we're good, supported architecture @@ -14,6 +13,8 @@ else exit 1 fi +BUILD_IMAGE_NAME="docker.elastic.co/elastic/nodejs-custom:$TARGET_VERSION-$ARCH" + echo '--- Downloading node source' curl --create-dirs --output-dir ./workdir/src -fsSLO --compressed \ https://nodejs.org/download/release/$TARGET_NODE_VERSION/node-$TARGET_NODE_VERSION.tar.xz diff --git a/scripts/create_build_images.sh b/scripts/create_build_images.sh index 2a83b67..a443866 100755 --- a/scripts/create_build_images.sh +++ b/scripts/create_build_images.sh @@ -31,7 +31,7 @@ if [[ "$ARCH" == "arm64" ]]; then --load \ --tag $IMAGE_NAME \ $DOCKER_BUILD_CONTEXT_DIR -elif [[ "$ARCH" == "arm64" ]]; then +elif [[ "$ARCH" == "amd64" ]]; then echo "--- Building node.js build images (amd64)" DOCKER_BUILDKIT=1 docker buildx build --progress=plain \ --platform $TARGET_PLATFORM \ From 43d3007108d3bfadee42a574d4b8eb72caf9114c Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Tue, 13 Jun 2023 13:21:44 +0200 Subject: [PATCH 24/75] Try arm64 build --- .buildkite/pipeline-kibana.yml | 29 +++++++++++++++++------------ .buildkite/pipeline.yml | 29 +++++++++++++++++------------ 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/.buildkite/pipeline-kibana.yml b/.buildkite/pipeline-kibana.yml index efbd45d..311faa7 100644 --- a/.buildkite/pipeline-kibana.yml +++ b/.buildkite/pipeline-kibana.yml @@ -2,22 +2,27 @@ agents: queue: kibana-default steps: + # - command: scripts/create_build_images.sh + # label: Creating build images + # env: + # ARCH: amd64 + - command: scripts/create_build_images.sh label: Creating build images env: - ARCH: amd64 + ARCH: arm64 - wait - - command: - - scripts/build_nodejs.sh - label: Building node.js - env: - ARCH: amd64 - timeout_in_minutes: 300 - - # - command: scripts/build_nodejs.sh + # - command: + # - scripts/build_nodejs.sh + # label: Building node.js # env: - # ARCH: arm64 - # label: Build node.js - # timeout_in_minutes: 600 + # ARCH: amd64 + # timeout_in_minutes: 300 + + - command: scripts/build_nodejs.sh + env: + ARCH: arm64 + label: Build node.js + timeout_in_minutes: 600 diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index ae78848..88f1ed8 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,20 +1,25 @@ steps: + # - command: scripts/create_build_images.sh + # label: Creating build images + # env: + # ARCH: amd64 + - command: scripts/create_build_images.sh label: Creating build images env: - ARCH: amd64 + ARCH: arm64 - wait - - command: - - scripts/build_nodejs.sh - label: Building node.js - env: - ARCH: amd64 - timeout_in_minutes: 300 - - # - command: scripts/build_nodejs.sh + # - command: + # - scripts/build_nodejs.sh + # label: Building node.js # env: - # ARCH: arm64 - # label: Build node.js - # timeout_in_minutes: 600 + # ARCH: amd64 + # timeout_in_minutes: 300 + + - command: scripts/build_nodejs.sh + env: + ARCH: arm64 + label: Build node.js + timeout_in_minutes: 600 From 1e0e91632530737fb5423a66b8c4a83633bab12b Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 14 Jun 2023 10:17:53 +0200 Subject: [PATCH 25/75] use arm machines for arm build --- .buildkite/pipeline-kibana.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline-kibana.yml b/.buildkite/pipeline-kibana.yml index 311faa7..b85ae96 100644 --- a/.buildkite/pipeline-kibana.yml +++ b/.buildkite/pipeline-kibana.yml @@ -1,16 +1,17 @@ -agents: - queue: kibana-default - steps: # - command: scripts/create_build_images.sh # label: Creating build images # env: # ARCH: amd64 + # agents: + # queue: kibana-default - command: scripts/create_build_images.sh label: Creating build images env: ARCH: arm64 + agents: + queue: macos-arm - wait @@ -20,9 +21,13 @@ steps: # env: # ARCH: amd64 # timeout_in_minutes: 300 + # agents: + # queue: kibana-default - command: scripts/build_nodejs.sh env: ARCH: arm64 label: Build node.js timeout_in_minutes: 600 + agents: + queue: macos-arm From 046ef032f1a306feb037ada037502117925d5c62 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 14 Jun 2023 11:18:53 +0200 Subject: [PATCH 26/75] Add a pre-checkout hook to try to fix github urls --- .buildkite/hooks/post-checkout | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100755 .buildkite/hooks/post-checkout diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout new file mode 100755 index 0000000..9d7825a --- /dev/null +++ b/.buildkite/hooks/post-checkout @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + + +echo "Before:" +git config --list | cat + +git config --global --add url."git@github.com:".insteadOf "https://github.com/" +git config --global --add remote.origin.url."git@github.com:".insteadOf "https://github.com/" + +echo "After:" +git config --list | cat From 5d3db88d66e37a6fd8cd055c2f50235c596cc5dc Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 14 Jun 2023 11:47:23 +0200 Subject: [PATCH 27/75] Add a few commands for discovery --- .buildkite/hooks/post-checkout | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index 9d7825a..bd25c80 100755 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -1,12 +1,19 @@ #!/bin/bash set -euo pipefail +set -x + +uname -a + +whoami + +pwd -echo "Before:" git config --list | cat git config --global --add url."git@github.com:".insteadOf "https://github.com/" git config --global --add remote.origin.url."git@github.com:".insteadOf "https://github.com/" -echo "After:" git config --list | cat + +set +x From 6e887aa081ed4d8781bee5494fc42480551b3aaa Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 14 Jun 2023 12:09:05 +0200 Subject: [PATCH 28/75] Add post-checkout hook to cleanup git configs --- .buildkite/hooks/post-checkout | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index bd25c80..b88ae48 100755 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -1,19 +1,11 @@ #!/bin/bash set -euo pipefail -set -x - -uname -a - -whoami - -pwd - +echo Cleaning up config overrides git config --list | cat -git config --global --add url."git@github.com:".insteadOf "https://github.com/" -git config --global --add remote.origin.url."git@github.com:".insteadOf "https://github.com/" +git config --global --remove url.git@github.com: +git config --global --remove remote.origin.url.git@github.com: +echo After cleanup git config --list | cat - -set +x From a3c45fa15357890f5864bc5f9505046f3ed495b2 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 14 Jun 2023 13:57:14 +0200 Subject: [PATCH 29/75] No post-checkout - it changes the kibana executors for good, might mess up the state for others --- .buildkite/hooks/post-checkout | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100755 .buildkite/hooks/post-checkout diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout deleted file mode 100755 index b88ae48..0000000 --- a/.buildkite/hooks/post-checkout +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -set -euo pipefail - -echo Cleaning up config overrides -git config --list | cat - -git config --global --remove url.git@github.com: -git config --global --remove remote.origin.url.git@github.com: - -echo After cleanup -git config --list | cat From a7bac4e7774ea0738f6e341a9f4b0d3549b9f886 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 14 Jun 2023 16:18:57 +0200 Subject: [PATCH 30/75] Refactor, and prepare pipeline for upload --- .buildkite/pipeline-kibana.yml | 36 ++++++++++------------------- scripts/build_nodejs.sh | 39 ++++++++++++------------------- scripts/build_re2.sh | 21 +++++++++-------- scripts/common.sh | 12 ++++++++++ scripts/create_build_images.sh | 42 ++++++++++++---------------------- scripts/docker_login.sh | 0 scripts/upload_artifacts.sh | 10 ++++++++ 7 files changed, 73 insertions(+), 87 deletions(-) mode change 100644 => 100755 scripts/build_re2.sh create mode 100644 scripts/common.sh mode change 100644 => 100755 scripts/docker_login.sh create mode 100755 scripts/upload_artifacts.sh diff --git a/.buildkite/pipeline-kibana.yml b/.buildkite/pipeline-kibana.yml index b85ae96..3672a7d 100644 --- a/.buildkite/pipeline-kibana.yml +++ b/.buildkite/pipeline-kibana.yml @@ -1,33 +1,21 @@ steps: - # - command: scripts/create_build_images.sh - # label: Creating build images - # env: - # ARCH: amd64 - # agents: - # queue: kibana-default - - - command: scripts/create_build_images.sh + - command: + - which gsutil + - scripts/create_build_images.sh + - scripts/build_nodejs.sh + - scripts/upload_artifacts.sh label: Creating build images env: - ARCH: arm64 + ARCH: amd64 + TARGET_VERSION: 18.15.0 agents: - queue: macos-arm - - - wait + queue: kibana-default # - command: + # - scripts/create_build_images.sh # - scripts/build_nodejs.sh - # label: Building node.js + # label: Creating build images # env: - # ARCH: amd64 - # timeout_in_minutes: 300 + # ARCH: arm64 # agents: - # queue: kibana-default - - - command: scripts/build_nodejs.sh - env: - ARCH: arm64 - label: Build node.js - timeout_in_minutes: 600 - agents: - queue: macos-arm + # queue: macos-arm diff --git a/scripts/build_nodejs.sh b/scripts/build_nodejs.sh index 71a3bfe..8acb45a 100755 --- a/scripts/build_nodejs.sh +++ b/scripts/build_nodejs.sh @@ -1,19 +1,17 @@ #!/bin/bash set -euo pipefail -TARGET_VERSION="18.15.0" -TARGET_NODE_VERSION="v$TARGET_VERSION" -RELEASE_URL_BASE="https://unofficial-builds.nodejs.org/download/release/" +source ./scripts/common.sh -if [[ "$ARCH" == "arm64" || "$ARCH" == "amd64" ]]; then - # we're good, supported architecture - echo "Building for architecture: $ARCH" -else - echo "ARCH (=$ARCH) env variable is not one of: arm64, amd64" - exit 1 -fi +# TARGET_VERSION provided in env +# ARCH provided in env +assert_correct_arch $ARCH +TARGET_NODE_VERSION="v$TARGET_VERSION" BUILD_IMAGE_NAME="docker.elastic.co/elastic/nodejs-custom:$TARGET_VERSION-$ARCH" +TARGET_PLATFORM="linux/$ARCH" +RELEASE_URL_BASE="https://unofficial-builds.nodejs.org/download/release/" + echo '--- Downloading node source' curl --create-dirs --output-dir ./workdir/src -fsSLO --compressed \ @@ -22,18 +20,9 @@ tar -xf ./workdir/src/node-$TARGET_NODE_VERSION.tar.xz -C ./workdir/src chmod -R a+rwx ./workdir/ -if [[ "$ARCH" == "arm64" ]]; then - echo '--- Buidling node for linux/arm64' - docker run --rm -it --platform linux/arm64 \ - -v ./workdir:/home/node/workdir:Z \ - $BUILD_IMAGE_NAME \ - $RELEASE_URL_BASE \ - $TARGET_NODE_VERSION -elif [[ "$ARCH" == "amd64" ]]; then - echo '--- Buidling node for linux/amd64' - docker run --rm -it --platform linux/amd64 \ - -v ./workdir:/home/node/workdir:Z \ - $BUILD_IMAGE_NAME \ - $RELEASE_URL_BASE \ - $TARGET_NODE_VERSION -fi +echo "--- Buidling node for $TARGET_PLATFORM" +docker run --rm -it --platform $TARGET_PLATFORM \ + -v ./workdir:/home/node/workdir:Z \ + $BUILD_IMAGE_NAME \ + $RELEASE_URL_BASE \ + $TARGET_NODE_VERSION diff --git a/scripts/build_re2.sh b/scripts/build_re2.sh old mode 100644 new mode 100755 index 30edc6a..8b109f1 --- a/scripts/build_re2.sh +++ b/scripts/build_re2.sh @@ -1,20 +1,21 @@ #!/bin/bash set -euo pipefail -BUILD_IMAGE_NAME="nodejs-custom:18.15.0" +source ./scripts/common.sh +# TARGET_VERSION provided in env +# ARCH provided in env +assert_correct_arch $ARCH + +BUILD_IMAGE_NAME="nodejs-custom:$TARGET_VERSION" RE2_FULL_VERSION="1.17.7" # $1 -NODE_FULL_VERSION="v18.15.0" # $2 -NODE_DOWNLOAD_BASE_URL="https://github.com/azasypkin/kibana/releases/download/nodej-custom" # $3, TODO: adjust based on the publish URLs +NODE_FULL_VERSION="v$TARGET_VERSION" # $2 +NODE_DOWNLOAD_BASE_URL="https://storage.cloud.google.com/kibana-custom-node-artifacts/node-glibc-217/dist/v$TARGET_VERSION/" +TARGET_PLATFORM="linux/$ARCH" -docker run --rm -it --platform linux/amd64 --entrypoint /home/node/re2_entrypoint.sh \ - -v ./workdir:/home/node/workdir \ - $BUILD_IMAGE_NAME \ - $RE2_FULL_VERSION \ - $NODE_FULL_VERSION \ - $NODE_DOWNLOAD_BASE_URL -docker run --rm -it --platform linux/arm64 --entrypoint /home/node/re2_entrypoint.sh \ +echo "--- Building RE2 for $TARGET_PLATFORM" +docker run --rm -it --platform $TARGET_PLATFORM --entrypoint /home/node/re2_entrypoint.sh \ -v ./workdir:/home/node/workdir \ $BUILD_IMAGE_NAME \ $RE2_FULL_VERSION \ diff --git a/scripts/common.sh b/scripts/common.sh new file mode 100644 index 0000000..37f49db --- /dev/null +++ b/scripts/common.sh @@ -0,0 +1,12 @@ + +function assert_correct_arch() { + ARCH=$1 + + if [[ "$ARCH" == "arm64" || "$ARCH" == "amd64" ]]; then + # we're good, supported architecture + echo "Building for architecture: $ARCH" + else + echo "ARCH (=$ARCH) env variable is not one of: arm64, amd64" + exit 1 + fi +} diff --git a/scripts/create_build_images.sh b/scripts/create_build_images.sh index a443866..15f36fd 100755 --- a/scripts/create_build_images.sh +++ b/scripts/create_build_images.sh @@ -1,21 +1,17 @@ #!/bin/bash - set -euo pipefail -SCRIPT_DIR=$(dirname $0) -DOCKER_BUILD_CONTEXT_DIR="$SCRIPT_DIR/../build-image-config/" -TARGET_VERSION="18.15.0" +source ./scripts/common.sh -if [[ "$ARCH" == "arm64" || "$ARCH" == "amd64" ]]; then - # we're good, supported architecture - echo "Building for architecture: $ARCH" -else - echo "ARCH (=$ARCH) env variable is not one of: arm64, amd64" - exit 1 -fi +# TARGET_VERSION provided in env +# ARCH provided in env +assert_correct_arch $ARCH TARGET_PLATFORM="linux/$ARCH" IMAGE_NAME="docker.elastic.co/elastic/nodejs-custom:$TARGET_VERSION-$ARCH" +SCRIPT_DIR=$(dirname $0) +DOCKER_BUILD_CONTEXT_DIR="$SCRIPT_DIR/../build-image-config/" + echo "--- Creating multiarch driver" if [[ $(docker buildx ls | grep multiarch | wc -l) -lt 1 ]]; then @@ -23,20 +19,10 @@ if [[ $(docker buildx ls | grep multiarch | wc -l) -lt 1 ]]; then fi -if [[ "$ARCH" == "arm64" ]]; then - echo "--- Building node.js build images (arm64)" - DOCKER_BUILDKIT=1 docker buildx build --progress=plain \ - --platform $TARGET_PLATFORM \ - --build-arg GROUP_ID=1000 --build-arg USER_ID=1000 \ - --load \ - --tag $IMAGE_NAME \ - $DOCKER_BUILD_CONTEXT_DIR -elif [[ "$ARCH" == "amd64" ]]; then - echo "--- Building node.js build images (amd64)" - DOCKER_BUILDKIT=1 docker buildx build --progress=plain \ - --platform $TARGET_PLATFORM \ - --build-arg GROUP_ID=1000 --build-arg USER_ID=1000 \ - --load \ - --tag $IMAGE_NAME \ - $DOCKER_BUILD_CONTEXT_DIR -fi +echo "--- Building node.js build image ($ARCH)" +DOCKER_BUILDKIT=1 docker buildx build --progress=plain \ + --platform $TARGET_PLATFORM \ + --build-arg GROUP_ID=1000 --build-arg USER_ID=1000 \ + --load \ + --tag $IMAGE_NAME \ + $DOCKER_BUILD_CONTEXT_DIR diff --git a/scripts/docker_login.sh b/scripts/docker_login.sh old mode 100644 new mode 100755 diff --git a/scripts/upload_artifacts.sh b/scripts/upload_artifacts.sh new file mode 100755 index 0000000..9404bee --- /dev/null +++ b/scripts/upload_artifacts.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -euox pipefail + +# TARGET_VERSION provided in env + +BUCKET_NAME="kibana-custom-node-artifacts" +ARTIFACT_BASE_PATH="node-glibc-217/dist/v$TARGET_VERSION/" + +echo "--- Uploading build artifacts" +gsutil cp -r "./workdir/dist/*" gs://$BUCKET_NAME/$ARTIFACT_BASE_PATH From 7a08d5df1da2177455e7016cb1e15f84b03378e9 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 14 Jun 2023 19:28:26 +0200 Subject: [PATCH 31/75] Retry after fixing access rights --- .buildkite/pipeline-kibana.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline-kibana.yml b/.buildkite/pipeline-kibana.yml index 3672a7d..c5006cf 100644 --- a/.buildkite/pipeline-kibana.yml +++ b/.buildkite/pipeline-kibana.yml @@ -1,6 +1,6 @@ steps: - command: - - which gsutil + - gsutil cp -r ./README.md gs://kibana-custom-node-artifacts/ - scripts/create_build_images.sh - scripts/build_nodejs.sh - scripts/upload_artifacts.sh From c3dd54d61692c49550c14005aca48395dfc5741b Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 14 Jun 2023 22:29:59 +0200 Subject: [PATCH 32/75] Strip -glibc-217 variant from filenames to keep original naming --- scripts/upload_artifacts.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/upload_artifacts.sh b/scripts/upload_artifacts.sh index 9404bee..6fa8e8f 100755 --- a/scripts/upload_artifacts.sh +++ b/scripts/upload_artifacts.sh @@ -1,10 +1,17 @@ #!/bin/bash -set -euox pipefail +set -euo pipefail # TARGET_VERSION provided in env BUCKET_NAME="kibana-custom-node-artifacts" ARTIFACT_BASE_PATH="node-glibc-217/dist/v$TARGET_VERSION/" +ARTIFACT_DIST_DIR="./workdir/dist" + +echo "--- Removing variant tags from file names" +for name in $(ls $ARTIFACT_DIST_DIR | grep tar); do + NAME_STRIPPED=$(echo "$name" | sed s/-glibc-217//) + mv $ARTIFACT_DIST_DIR/$name $ARTIFACT_DIST_DIR/$NAME_STRIPPED +done echo "--- Uploading build artifacts" -gsutil cp -r "./workdir/dist/*" gs://$BUCKET_NAME/$ARTIFACT_BASE_PATH +gsutil cp -r "$ARTIFACT_DIST_DIR/*" gs://$BUCKET_NAME/$ARTIFACT_BASE_PATH From c63603bfa48b878f2d48f0d9f5f4aea660d0cce0 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 14 Jun 2023 22:43:22 +0200 Subject: [PATCH 33/75] Plug in ARM build, try a complete build --- .buildkite/pipeline-kibana.yml | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/.buildkite/pipeline-kibana.yml b/.buildkite/pipeline-kibana.yml index c5006cf..7051de2 100644 --- a/.buildkite/pipeline-kibana.yml +++ b/.buildkite/pipeline-kibana.yml @@ -1,6 +1,5 @@ steps: - command: - - gsutil cp -r ./README.md gs://kibana-custom-node-artifacts/ - scripts/create_build_images.sh - scripts/build_nodejs.sh - scripts/upload_artifacts.sh @@ -10,12 +9,19 @@ steps: TARGET_VERSION: 18.15.0 agents: queue: kibana-default + timeout_in_minutes: 300 # 5 hrs + + + - command: + - scripts/create_build_images.sh + - scripts/build_nodejs.sh + - scripts/upload_artifacts.sh + label: Creating build images + env: + ARCH: amd64 + TARGET_VERSION: 18.15.0 + agents: + # queue: macos-arm # disabled while the git cloning doesn't work + queue: kibana-default # cross-compiling takes a while + timeout_in_minutes: 800 # 12+ hrs - # - command: - # - scripts/create_build_images.sh - # - scripts/build_nodejs.sh - # label: Creating build images - # env: - # ARCH: arm64 - # agents: - # queue: macos-arm From fa1032648425dc3f2faee9ddb838cd697746bc0a Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Thu, 15 Jun 2023 12:59:13 +0200 Subject: [PATCH 34/75] Attempt build on the macos executor --- .buildkite/pipeline-kibana.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline-kibana.yml b/.buildkite/pipeline-kibana.yml index 7051de2..6b6f9bc 100644 --- a/.buildkite/pipeline-kibana.yml +++ b/.buildkite/pipeline-kibana.yml @@ -21,7 +21,7 @@ steps: ARCH: amd64 TARGET_VERSION: 18.15.0 agents: - # queue: macos-arm # disabled while the git cloning doesn't work - queue: kibana-default # cross-compiling takes a while + queue: macos-arm # disabled while the git cloning doesn't work + # queue: kibana-default # cross-compiling takes a while timeout_in_minutes: 800 # 12+ hrs From 9b0d9f917d85827d3da7c86a3949a3547b80a861 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Thu, 15 Jun 2023 13:42:23 +0200 Subject: [PATCH 35/75] Retreat back to cross-compiling, while the agent is set up --- .buildkite/pipeline-kibana.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline-kibana.yml b/.buildkite/pipeline-kibana.yml index 6b6f9bc..7051de2 100644 --- a/.buildkite/pipeline-kibana.yml +++ b/.buildkite/pipeline-kibana.yml @@ -21,7 +21,7 @@ steps: ARCH: amd64 TARGET_VERSION: 18.15.0 agents: - queue: macos-arm # disabled while the git cloning doesn't work - # queue: kibana-default # cross-compiling takes a while + # queue: macos-arm # disabled while the git cloning doesn't work + queue: kibana-default # cross-compiling takes a while timeout_in_minutes: 800 # 12+ hrs From a20eefcc9dbfbb022a224ce89d3ab28e505b74dc Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Mon, 19 Jun 2023 11:22:15 +0200 Subject: [PATCH 36/75] Fix arm64<->amd64 parameterization, names --- .buildkite/pipeline-kibana.yml | 6 +++--- .buildkite/pipeline.yml | 23 +++++++++++++---------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/.buildkite/pipeline-kibana.yml b/.buildkite/pipeline-kibana.yml index 7051de2..5f23604 100644 --- a/.buildkite/pipeline-kibana.yml +++ b/.buildkite/pipeline-kibana.yml @@ -3,7 +3,7 @@ steps: - scripts/create_build_images.sh - scripts/build_nodejs.sh - scripts/upload_artifacts.sh - label: Creating build images + label: Build node.js 18 with glibc 2.17 for x64 env: ARCH: amd64 TARGET_VERSION: 18.15.0 @@ -16,9 +16,9 @@ steps: - scripts/create_build_images.sh - scripts/build_nodejs.sh - scripts/upload_artifacts.sh - label: Creating build images + label: Build node.js 18 with glibc 2.17 for arm64 env: - ARCH: amd64 + ARCH: arm64 TARGET_VERSION: 18.15.0 agents: # queue: macos-arm # disabled while the git cloning doesn't work diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 88f1ed8..a155672 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,15 +1,18 @@ steps: + - command: echo "OK" + label: Blank command, placeholder + # - command: scripts/create_build_images.sh # label: Creating build images # env: # ARCH: amd64 - - command: scripts/create_build_images.sh - label: Creating build images - env: - ARCH: arm64 + # - command: scripts/create_build_images.sh + # label: Creating build images + # env: + # ARCH: arm64 - - wait + # - wait # - command: # - scripts/build_nodejs.sh @@ -18,8 +21,8 @@ steps: # ARCH: amd64 # timeout_in_minutes: 300 - - command: scripts/build_nodejs.sh - env: - ARCH: arm64 - label: Build node.js - timeout_in_minutes: 600 + # - command: scripts/build_nodejs.sh + # env: + # ARCH: arm64 + # label: Build node.js + # timeout_in_minutes: 600 From 000e638fbc7b2f8a3ff7a414148b2057d288c2e1 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Mon, 19 Jun 2023 11:22:39 +0200 Subject: [PATCH 37/75] Allow all users to rwx on created artifacts, so external scripts can rename them --- build-image-config/entrypoint.sh | 1 + build-image-config/re2_entrypoint.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/build-image-config/entrypoint.sh b/build-image-config/entrypoint.sh index ae1544c..bd2e78a 100755 --- a/build-image-config/entrypoint.sh +++ b/build-image-config/entrypoint.sh @@ -35,3 +35,4 @@ make -j"$(getconf _NPROCESSORS_ONLN)" binary V= \ mkdir -p /home/node/workdir/dist/ mv node-*.tar.?z /home/node/workdir/dist/ +chmod a+rwx /home/node/workdir/dist/* diff --git a/build-image-config/re2_entrypoint.sh b/build-image-config/re2_entrypoint.sh index a8165d2..68a96e5 100755 --- a/build-image-config/re2_entrypoint.sh +++ b/build-image-config/re2_entrypoint.sh @@ -53,3 +53,4 @@ npm test mkdir -p /home/node/workdir/dist/ cp "${re2_source_folder}/build/Release/re2.node" "/home/node/workdir/dist/linux-${architecture}-108" gzip -f "/home/node/workdir/dist/linux-${architecture}-108" +chmod a+rwx /home/node/workdir/dist/* From 805e7655d6e9ad3900414dbd8411fc893574f732 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Mon, 19 Jun 2023 13:51:02 +0200 Subject: [PATCH 38/75] Allow writing of the created dist folder, to allow for rename --- build-image-config/entrypoint.sh | 1 + build-image-config/re2_entrypoint.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/build-image-config/entrypoint.sh b/build-image-config/entrypoint.sh index bd2e78a..a2b0603 100755 --- a/build-image-config/entrypoint.sh +++ b/build-image-config/entrypoint.sh @@ -34,5 +34,6 @@ make -j"$(getconf _NPROCESSORS_ONLN)" binary V= \ CONFIG_FLAGS="$config_flags" mkdir -p /home/node/workdir/dist/ +chmod a+w /home/node/workdir/dist mv node-*.tar.?z /home/node/workdir/dist/ chmod a+rwx /home/node/workdir/dist/* diff --git a/build-image-config/re2_entrypoint.sh b/build-image-config/re2_entrypoint.sh index 68a96e5..4067a26 100755 --- a/build-image-config/re2_entrypoint.sh +++ b/build-image-config/re2_entrypoint.sh @@ -51,6 +51,7 @@ npm run build --if-present npm test mkdir -p /home/node/workdir/dist/ +chmod a+w /home/node/workdir/dist cp "${re2_source_folder}/build/Release/re2.node" "/home/node/workdir/dist/linux-${architecture}-108" gzip -f "/home/node/workdir/dist/linux-${architecture}-108" chmod a+rwx /home/node/workdir/dist/* From d3e62845d3b14a2d68eb00dd243a062284ff6bef Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Tue, 20 Jun 2023 13:37:59 +0200 Subject: [PATCH 39/75] Try macos executor again --- .buildkite/pipeline-kibana.yml | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/.buildkite/pipeline-kibana.yml b/.buildkite/pipeline-kibana.yml index 5f23604..f20f001 100644 --- a/.buildkite/pipeline-kibana.yml +++ b/.buildkite/pipeline-kibana.yml @@ -1,15 +1,15 @@ steps: - - command: - - scripts/create_build_images.sh - - scripts/build_nodejs.sh - - scripts/upload_artifacts.sh - label: Build node.js 18 with glibc 2.17 for x64 - env: - ARCH: amd64 - TARGET_VERSION: 18.15.0 - agents: - queue: kibana-default - timeout_in_minutes: 300 # 5 hrs + # - command: + # - scripts/create_build_images.sh + # - scripts/build_nodejs.sh + # - scripts/upload_artifacts.sh + # label: Build node.js 18 with glibc 2.17 for x64 + # env: + # ARCH: amd64 + # TARGET_VERSION: 18.15.0 + # agents: + # queue: kibana-default + # timeout_in_minutes: 300 # 5 hrs - command: @@ -21,7 +21,6 @@ steps: ARCH: arm64 TARGET_VERSION: 18.15.0 agents: - # queue: macos-arm # disabled while the git cloning doesn't work - queue: kibana-default # cross-compiling takes a while + queue: macos-arm # disabled while the git cloning doesn't work + # queue: kibana-default # cross-compiling takes a while, 13 hours is not enough :/ timeout_in_minutes: 800 # 12+ hrs - From 75354d2cc698c66893517a020be3cc439f8d4806 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Tue, 20 Jun 2023 15:57:19 +0200 Subject: [PATCH 40/75] Re-add x64 build --- .buildkite/pipeline-kibana.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.buildkite/pipeline-kibana.yml b/.buildkite/pipeline-kibana.yml index f20f001..acd29ce 100644 --- a/.buildkite/pipeline-kibana.yml +++ b/.buildkite/pipeline-kibana.yml @@ -1,15 +1,15 @@ steps: - # - command: - # - scripts/create_build_images.sh - # - scripts/build_nodejs.sh - # - scripts/upload_artifacts.sh - # label: Build node.js 18 with glibc 2.17 for x64 - # env: - # ARCH: amd64 - # TARGET_VERSION: 18.15.0 - # agents: - # queue: kibana-default - # timeout_in_minutes: 300 # 5 hrs + - command: + - scripts/create_build_images.sh + - scripts/build_nodejs.sh + - scripts/upload_artifacts.sh + label: Build node.js 18 with glibc 2.17 for x64 + env: + ARCH: amd64 + TARGET_VERSION: 18.15.0 + agents: + queue: kibana-default + timeout_in_minutes: 300 # 5 hrs - command: From 6b71765a56e2add5a5cbeb2603551c5984004c70 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Tue, 20 Jun 2023 17:11:11 +0200 Subject: [PATCH 41/75] Bump to build node 18.16.0 --- .buildkite/pipeline-kibana.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline-kibana.yml b/.buildkite/pipeline-kibana.yml index acd29ce..dfba873 100644 --- a/.buildkite/pipeline-kibana.yml +++ b/.buildkite/pipeline-kibana.yml @@ -6,7 +6,7 @@ steps: label: Build node.js 18 with glibc 2.17 for x64 env: ARCH: amd64 - TARGET_VERSION: 18.15.0 + TARGET_VERSION: 18.16.0 agents: queue: kibana-default timeout_in_minutes: 300 # 5 hrs @@ -19,7 +19,7 @@ steps: label: Build node.js 18 with glibc 2.17 for arm64 env: ARCH: arm64 - TARGET_VERSION: 18.15.0 + TARGET_VERSION: 18.16.0 agents: queue: macos-arm # disabled while the git cloning doesn't work # queue: kibana-default # cross-compiling takes a while, 13 hours is not enough :/ From 3eb59559c6fe6d89a977e32efdbaf56de2ffbab4 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Tue, 20 Jun 2023 17:47:00 +0200 Subject: [PATCH 42/75] Find where the node builds are running, for debugging --- scripts/build_nodejs.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/build_nodejs.sh b/scripts/build_nodejs.sh index 8acb45a..33e52af 100755 --- a/scripts/build_nodejs.sh +++ b/scripts/build_nodejs.sh @@ -12,12 +12,14 @@ BUILD_IMAGE_NAME="docker.elastic.co/elastic/nodejs-custom:$TARGET_VERSION-$ARCH" TARGET_PLATFORM="linux/$ARCH" RELEASE_URL_BASE="https://unofficial-builds.nodejs.org/download/release/" +echo "Running node.js build in folder: `pwd`" echo '--- Downloading node source' curl --create-dirs --output-dir ./workdir/src -fsSLO --compressed \ https://nodejs.org/download/release/$TARGET_NODE_VERSION/node-$TARGET_NODE_VERSION.tar.xz tar -xf ./workdir/src/node-$TARGET_NODE_VERSION.tar.xz -C ./workdir/src chmod -R a+rwx ./workdir/ +chmod -R a+rwx ./workdir/src echo "--- Buidling node for $TARGET_PLATFORM" From 2e589f533eccfdb0a03b358f908f8d78b8f9bb2c Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Tue, 20 Jun 2023 17:55:27 +0200 Subject: [PATCH 43/75] More debug --- build-image-config/entrypoint.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build-image-config/entrypoint.sh b/build-image-config/entrypoint.sh index a2b0603..d533224 100755 --- a/build-image-config/entrypoint.sh +++ b/build-image-config/entrypoint.sh @@ -13,7 +13,11 @@ else architecture="arm64" fi -ls -la "/home/node/workdir/src/" +ls -la "/" +ls -la "/home/" +ls -la "/home/node/" +ls -la "/home/node/workdir/" +ls -la "/home/node/workdir/src" ls -la "/home/node/workdir/src/node-${full_version}" cd "/home/node/workdir/src/node-${full_version}" From 3eb345df2cd61f8bdb6d29880eb5225eaa3b9b76 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Tue, 20 Jun 2023 18:25:50 +0200 Subject: [PATCH 44/75] Try fast agents with cross-compilation --- .buildkite/pipeline-kibana.yml | 6 +++--- build-image-config/entrypoint.sh | 4 ---- scripts/build_nodejs.sh | 1 - 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.buildkite/pipeline-kibana.yml b/.buildkite/pipeline-kibana.yml index dfba873..0ab9314 100644 --- a/.buildkite/pipeline-kibana.yml +++ b/.buildkite/pipeline-kibana.yml @@ -8,7 +8,7 @@ steps: ARCH: amd64 TARGET_VERSION: 18.16.0 agents: - queue: kibana-default + queue: c2-60 timeout_in_minutes: 300 # 5 hrs @@ -21,6 +21,6 @@ steps: ARCH: arm64 TARGET_VERSION: 18.16.0 agents: - queue: macos-arm # disabled while the git cloning doesn't work - # queue: kibana-default # cross-compiling takes a while, 13 hours is not enough :/ + # queue: macos-arm # disabled while the git cloning doesn't work + queue: c2-60 # cross-compiling takes a while, 13 hours is not enough :/ timeout_in_minutes: 800 # 12+ hrs diff --git a/build-image-config/entrypoint.sh b/build-image-config/entrypoint.sh index d533224..e0e1389 100755 --- a/build-image-config/entrypoint.sh +++ b/build-image-config/entrypoint.sh @@ -13,10 +13,6 @@ else architecture="arm64" fi -ls -la "/" -ls -la "/home/" -ls -la "/home/node/" -ls -la "/home/node/workdir/" ls -la "/home/node/workdir/src" ls -la "/home/node/workdir/src/node-${full_version}" diff --git a/scripts/build_nodejs.sh b/scripts/build_nodejs.sh index 33e52af..7290807 100755 --- a/scripts/build_nodejs.sh +++ b/scripts/build_nodejs.sh @@ -19,7 +19,6 @@ curl --create-dirs --output-dir ./workdir/src -fsSLO --compressed \ https://nodejs.org/download/release/$TARGET_NODE_VERSION/node-$TARGET_NODE_VERSION.tar.xz tar -xf ./workdir/src/node-$TARGET_NODE_VERSION.tar.xz -C ./workdir/src chmod -R a+rwx ./workdir/ -chmod -R a+rwx ./workdir/src echo "--- Buidling node for $TARGET_PLATFORM" From b9916d15abd626ac7b2f79b6e09ea3fb40bbd90e Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 21 Jun 2023 12:27:32 +0200 Subject: [PATCH 45/75] Try building re2 --- .buildkite/pipeline-kibana.yml | 39 +++++++++++-------- scripts/build_re2.sh | 4 +- ...rtifacts.sh => upload_nodejs_artifacts.sh} | 0 scripts/upload_re2_artifacts.sh | 16 ++++++++ 4 files changed, 40 insertions(+), 19 deletions(-) rename scripts/{upload_artifacts.sh => upload_nodejs_artifacts.sh} (100%) create mode 100755 scripts/upload_re2_artifacts.sh diff --git a/.buildkite/pipeline-kibana.yml b/.buildkite/pipeline-kibana.yml index 0ab9314..f676885 100644 --- a/.buildkite/pipeline-kibana.yml +++ b/.buildkite/pipeline-kibana.yml @@ -1,26 +1,31 @@ steps: - - command: + - label: Build node.js 18 with glibc 2.17 for x64 + + command: - scripts/create_build_images.sh - - scripts/build_nodejs.sh - - scripts/upload_artifacts.sh - label: Build node.js 18 with glibc 2.17 for x64 + # - scripts/build_nodejs.sh + # - scripts/upload_nodejs_artifacts.sh + - scripts/build_re2.sh + - scripts/upload_re2_artifacts.sh env: ARCH: amd64 TARGET_VERSION: 18.16.0 + RE2_VERSION: 1.17.7 agents: - queue: c2-60 + queue: c2-16 timeout_in_minutes: 300 # 5 hrs - - command: - - scripts/create_build_images.sh - - scripts/build_nodejs.sh - - scripts/upload_artifacts.sh - label: Build node.js 18 with glibc 2.17 for arm64 - env: - ARCH: arm64 - TARGET_VERSION: 18.16.0 - agents: - # queue: macos-arm # disabled while the git cloning doesn't work - queue: c2-60 # cross-compiling takes a while, 13 hours is not enough :/ - timeout_in_minutes: 800 # 12+ hrs + # - command: + # - scripts/create_build_images.sh + # - scripts/build_nodejs.sh + # - scripts/upload_artifacts.sh + # label: Build node.js 18 with glibc 2.17 for arm64 + # env: + # ARCH: arm64 + # TARGET_VERSION: 18.16.0 + # RE2_VERSION: 1.17.7 + # agents: + # # queue: macos-arm # disabled while the git cloning doesn't work + # queue: c2-60 # cross-compiling takes a while, 13 hours is not enough :/ + # timeout_in_minutes: 800 # 12+ hrs diff --git a/scripts/build_re2.sh b/scripts/build_re2.sh index 8b109f1..740064d 100755 --- a/scripts/build_re2.sh +++ b/scripts/build_re2.sh @@ -8,13 +8,13 @@ source ./scripts/common.sh assert_correct_arch $ARCH BUILD_IMAGE_NAME="nodejs-custom:$TARGET_VERSION" -RE2_FULL_VERSION="1.17.7" # $1 +RE2_FULL_VERSION=${RE2_VERSION:-1.17.7} # $1 NODE_FULL_VERSION="v$TARGET_VERSION" # $2 NODE_DOWNLOAD_BASE_URL="https://storage.cloud.google.com/kibana-custom-node-artifacts/node-glibc-217/dist/v$TARGET_VERSION/" TARGET_PLATFORM="linux/$ARCH" - echo "--- Building RE2 for $TARGET_PLATFORM" +chmod -R a+rwx ./workdir/ docker run --rm -it --platform $TARGET_PLATFORM --entrypoint /home/node/re2_entrypoint.sh \ -v ./workdir:/home/node/workdir \ $BUILD_IMAGE_NAME \ diff --git a/scripts/upload_artifacts.sh b/scripts/upload_nodejs_artifacts.sh similarity index 100% rename from scripts/upload_artifacts.sh rename to scripts/upload_nodejs_artifacts.sh diff --git a/scripts/upload_re2_artifacts.sh b/scripts/upload_re2_artifacts.sh new file mode 100755 index 0000000..20efa7f --- /dev/null +++ b/scripts/upload_re2_artifacts.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -euo pipefail + +BUCKET_NAME="kibana-custom-node-artifacts" +RE2_FULL_VERSION=${RE2_VERSION:-1.17.7} # $1 +ARTIFACT_BASE_PATH="re2-glibc-217/v$RE2_FULL_VERSION/" +ARTIFACT_DIST_DIR="./workdir/dist" + +# echo "--- Removing variant tags from file names" +# for name in $(ls $ARTIFACT_DIST_DIR | grep tar); do +# NAME_STRIPPED=$(echo "$name" | sed s/-glibc-217//) +# mv $ARTIFACT_DIST_DIR/$name $ARTIFACT_DIST_DIR/$NAME_STRIPPED +# done + +echo "--- Uploading build artifacts" +gsutil cp -r "$ARTIFACT_DIST_DIR/*" gs://$BUCKET_NAME/$ARTIFACT_BASE_PATH From 2785d8371b74b9f5bffb52a6936e968d1ee441e5 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 21 Jun 2023 12:38:41 +0200 Subject: [PATCH 46/75] Mkdir & new folder names to avoid clash --- scripts/build_re2.sh | 5 +++-- scripts/upload_re2_artifacts.sh | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/build_re2.sh b/scripts/build_re2.sh index 740064d..05b6ca4 100755 --- a/scripts/build_re2.sh +++ b/scripts/build_re2.sh @@ -14,9 +14,10 @@ NODE_DOWNLOAD_BASE_URL="https://storage.cloud.google.com/kibana-custom-node-arti TARGET_PLATFORM="linux/$ARCH" echo "--- Building RE2 for $TARGET_PLATFORM" -chmod -R a+rwx ./workdir/ +mkdir -p ./workdir_re2/ +chmod -R a+rwx ./workdir_re2/ docker run --rm -it --platform $TARGET_PLATFORM --entrypoint /home/node/re2_entrypoint.sh \ - -v ./workdir:/home/node/workdir \ + -v ./workdir_re2:/home/node/workdir \ $BUILD_IMAGE_NAME \ $RE2_FULL_VERSION \ $NODE_FULL_VERSION \ diff --git a/scripts/upload_re2_artifacts.sh b/scripts/upload_re2_artifacts.sh index 20efa7f..5f2ee4e 100755 --- a/scripts/upload_re2_artifacts.sh +++ b/scripts/upload_re2_artifacts.sh @@ -4,7 +4,7 @@ set -euo pipefail BUCKET_NAME="kibana-custom-node-artifacts" RE2_FULL_VERSION=${RE2_VERSION:-1.17.7} # $1 ARTIFACT_BASE_PATH="re2-glibc-217/v$RE2_FULL_VERSION/" -ARTIFACT_DIST_DIR="./workdir/dist" +ARTIFACT_DIST_DIR="./workdir_re2/dist" # echo "--- Removing variant tags from file names" # for name in $(ls $ARTIFACT_DIST_DIR | grep tar); do From c037b402b14f776da401093f21d7c7344db9feb9 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 21 Jun 2023 12:43:42 +0200 Subject: [PATCH 47/75] Fix docker image name --- scripts/build_re2.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_re2.sh b/scripts/build_re2.sh index 05b6ca4..cda55fa 100755 --- a/scripts/build_re2.sh +++ b/scripts/build_re2.sh @@ -7,7 +7,7 @@ source ./scripts/common.sh # ARCH provided in env assert_correct_arch $ARCH -BUILD_IMAGE_NAME="nodejs-custom:$TARGET_VERSION" +BUILD_IMAGE_NAME="docker.elastic.co/elastic/nodejs-custom:$TARGET_VERSION" RE2_FULL_VERSION=${RE2_VERSION:-1.17.7} # $1 NODE_FULL_VERSION="v$TARGET_VERSION" # $2 NODE_DOWNLOAD_BASE_URL="https://storage.cloud.google.com/kibana-custom-node-artifacts/node-glibc-217/dist/v$TARGET_VERSION/" From 133ccffb2c96e6f99c3cc8db58d52a3e2b1ceecf Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 21 Jun 2023 12:59:08 +0200 Subject: [PATCH 48/75] Centralize build image name, so there are no typos --- scripts/build_nodejs.sh | 2 +- scripts/build_re2.sh | 2 +- scripts/common.sh | 7 +++++++ scripts/create_build_images.sh | 2 +- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/build_nodejs.sh b/scripts/build_nodejs.sh index 7290807..10d85cd 100755 --- a/scripts/build_nodejs.sh +++ b/scripts/build_nodejs.sh @@ -8,7 +8,7 @@ source ./scripts/common.sh assert_correct_arch $ARCH TARGET_NODE_VERSION="v$TARGET_VERSION" -BUILD_IMAGE_NAME="docker.elastic.co/elastic/nodejs-custom:$TARGET_VERSION-$ARCH" +BUILD_IMAGE_NAME=$(get_build_image_name) TARGET_PLATFORM="linux/$ARCH" RELEASE_URL_BASE="https://unofficial-builds.nodejs.org/download/release/" diff --git a/scripts/build_re2.sh b/scripts/build_re2.sh index cda55fa..181792a 100755 --- a/scripts/build_re2.sh +++ b/scripts/build_re2.sh @@ -7,7 +7,7 @@ source ./scripts/common.sh # ARCH provided in env assert_correct_arch $ARCH -BUILD_IMAGE_NAME="docker.elastic.co/elastic/nodejs-custom:$TARGET_VERSION" +BUILD_IMAGE_NAME=$(get_build_image_name) RE2_FULL_VERSION=${RE2_VERSION:-1.17.7} # $1 NODE_FULL_VERSION="v$TARGET_VERSION" # $2 NODE_DOWNLOAD_BASE_URL="https://storage.cloud.google.com/kibana-custom-node-artifacts/node-glibc-217/dist/v$TARGET_VERSION/" diff --git a/scripts/common.sh b/scripts/common.sh index 37f49db..12ab750 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -10,3 +10,10 @@ function assert_correct_arch() { exit 1 fi } + +function get_build_image_name() { + NODE_VERSION=${1:-$TARGET_VERSION} + PLATFORM=${2:-$ARCH} + + echo "docker.elastic.co/elastic/nodejs-custom:$NODE_VERSION-$PLATFORM" +} diff --git a/scripts/create_build_images.sh b/scripts/create_build_images.sh index 15f36fd..1798eb4 100755 --- a/scripts/create_build_images.sh +++ b/scripts/create_build_images.sh @@ -8,7 +8,7 @@ source ./scripts/common.sh assert_correct_arch $ARCH TARGET_PLATFORM="linux/$ARCH" -IMAGE_NAME="docker.elastic.co/elastic/nodejs-custom:$TARGET_VERSION-$ARCH" +IMAGE_NAME=$(get_build_image_name) SCRIPT_DIR=$(dirname $0) DOCKER_BUILD_CONTEXT_DIR="$SCRIPT_DIR/../build-image-config/" From 1813dece28f825ed003f4a403eab3fa2742ebc79 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 21 Jun 2023 13:25:50 +0200 Subject: [PATCH 49/75] Fix double slash typo in path --- scripts/build_re2.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_re2.sh b/scripts/build_re2.sh index 181792a..7179818 100755 --- a/scripts/build_re2.sh +++ b/scripts/build_re2.sh @@ -10,7 +10,7 @@ assert_correct_arch $ARCH BUILD_IMAGE_NAME=$(get_build_image_name) RE2_FULL_VERSION=${RE2_VERSION:-1.17.7} # $1 NODE_FULL_VERSION="v$TARGET_VERSION" # $2 -NODE_DOWNLOAD_BASE_URL="https://storage.cloud.google.com/kibana-custom-node-artifacts/node-glibc-217/dist/v$TARGET_VERSION/" +NODE_DOWNLOAD_BASE_URL="https://storage.cloud.google.com/kibana-custom-node-artifacts/node-glibc-217/dist/v$TARGET_VERSION" TARGET_PLATFORM="linux/$ARCH" echo "--- Building RE2 for $TARGET_PLATFORM" From 922dec35141af6a4c27c25a5cb20841001ec75f5 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 21 Jun 2023 13:42:15 +0200 Subject: [PATCH 50/75] Use public URLs for the node distro --- scripts/build_re2.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_re2.sh b/scripts/build_re2.sh index 7179818..50f32f8 100755 --- a/scripts/build_re2.sh +++ b/scripts/build_re2.sh @@ -10,7 +10,7 @@ assert_correct_arch $ARCH BUILD_IMAGE_NAME=$(get_build_image_name) RE2_FULL_VERSION=${RE2_VERSION:-1.17.7} # $1 NODE_FULL_VERSION="v$TARGET_VERSION" # $2 -NODE_DOWNLOAD_BASE_URL="https://storage.cloud.google.com/kibana-custom-node-artifacts/node-glibc-217/dist/v$TARGET_VERSION" +NODE_DOWNLOAD_BASE_URL="https://storage.googleapis.com/kibana-custom-node-artifacts/node-glibc-217/dist/v$TARGET_VERSION" TARGET_PLATFORM="linux/$ARCH" echo "--- Building RE2 for $TARGET_PLATFORM" From 42b4990d68c50c49ae84db14ea3f7642eb149c93 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 21 Jun 2023 15:48:31 +0200 Subject: [PATCH 51/75] Use npm binary extracted locally --- build-image-config/re2_entrypoint.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build-image-config/re2_entrypoint.sh b/build-image-config/re2_entrypoint.sh index 4067a26..177affa 100755 --- a/build-image-config/re2_entrypoint.sh +++ b/build-image-config/re2_entrypoint.sh @@ -46,9 +46,9 @@ export CXX="ccache g++" . /opt/rh/devtoolset-9/enable -npm i --unsafe-perm=true -npm run build --if-present -npm test +$npm_binary i --unsafe-perm=true +$npm_binary run build --if-present +$npm_binary test mkdir -p /home/node/workdir/dist/ chmod a+w /home/node/workdir/dist From c13eb228f061e9e3d2925ea79592ed2bdb60fb10 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 21 Jun 2023 16:29:29 +0200 Subject: [PATCH 52/75] Remove variant from extracted node folder if needed --- build-image-config/re2_entrypoint.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/build-image-config/re2_entrypoint.sh b/build-image-config/re2_entrypoint.sh index 177affa..ddf71cb 100755 --- a/build-image-config/re2_entrypoint.sh +++ b/build-image-config/re2_entrypoint.sh @@ -7,6 +7,8 @@ re2_full_version="$1" node_full_version="$2" node_download_base_url="$3" +VARIATION="glibc-217" + if [[ $(arch) == x86_64 ]]; then architecture="x64"; else @@ -26,6 +28,16 @@ if [ ! -f "$npm_binary" ]; then fi tar -xf "/home/node/workdir/${node_folder_name}.tar.xz" + + EXTRACTED_DIR_NAME=$(ls /home/node/workdir | grep $node_folder_name | grep -v "tar") + + if [[ "$EXTRACTED_DIR_NAME" == "$node_folder_name" ]]; then + # OK, no variation name to remove + echo "Extracted folder $node_folder_name" + else + mv /home/node/workdir/$EXTRACTED_DIR_NAME /home/node/workdir/$node_folder_name + echo "Extracted folder $EXTRACTED_DIR_NAME, renamed to $node_folder_name" + fi fi cd src From 07f47dfc1ff6f815149df10a2516cb237085f4c8 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 21 Jun 2023 19:58:10 +0200 Subject: [PATCH 53/75] Add SHA256 file fixing,remove variation, fix re2 artifact building --- .buildkite/pipeline-kibana.yml | 12 ++++--- build-image-config/entrypoint.sh | 1 - build-image-config/re2_entrypoint.sh | 16 ++++----- scripts/common.sh | 51 ++++++++++++++++++++++++++++ scripts/upload_nodejs_artifacts.sh | 22 ++++++++---- scripts/upload_re2_artifacts.sh | 6 ---- 6 files changed, 81 insertions(+), 27 deletions(-) diff --git a/.buildkite/pipeline-kibana.yml b/.buildkite/pipeline-kibana.yml index f676885..214c0fe 100644 --- a/.buildkite/pipeline-kibana.yml +++ b/.buildkite/pipeline-kibana.yml @@ -3,8 +3,8 @@ steps: command: - scripts/create_build_images.sh - # - scripts/build_nodejs.sh - # - scripts/upload_nodejs_artifacts.sh + - scripts/build_nodejs.sh + - scripts/upload_nodejs_artifacts.sh - scripts/build_re2.sh - scripts/upload_re2_artifacts.sh env: @@ -16,11 +16,13 @@ steps: timeout_in_minutes: 300 # 5 hrs - # - command: + # - label: Build node.js 18 with glibc 2.17 for arm64 + # command: # - scripts/create_build_images.sh # - scripts/build_nodejs.sh - # - scripts/upload_artifacts.sh - # label: Build node.js 18 with glibc 2.17 for arm64 + # - scripts/upload_nodejs_artifacts.sh + # - scripts/build_re2.sh + # - scripts/upload_re2_artifacts.sh # env: # ARCH: arm64 # TARGET_VERSION: 18.16.0 diff --git a/build-image-config/entrypoint.sh b/build-image-config/entrypoint.sh index e0e1389..db4fde1 100755 --- a/build-image-config/entrypoint.sh +++ b/build-image-config/entrypoint.sh @@ -28,7 +28,6 @@ export CXX="ccache g++" make -j"$(getconf _NPROCESSORS_ONLN)" binary V= \ DESTCPU="$architecture" \ ARCH="$architecture" \ - VARIATION="glibc-217" \ DISTTYPE="release" \ RELEASE_URLBASE="$release_url_base" \ CONFIG_FLAGS="$config_flags" diff --git a/build-image-config/re2_entrypoint.sh b/build-image-config/re2_entrypoint.sh index ddf71cb..ec3d7fa 100755 --- a/build-image-config/re2_entrypoint.sh +++ b/build-image-config/re2_entrypoint.sh @@ -7,7 +7,7 @@ re2_full_version="$1" node_full_version="$2" node_download_base_url="$3" -VARIATION="glibc-217" +# VARIATION="glibc-217" if [[ $(arch) == x86_64 ]]; then architecture="x64"; @@ -31,13 +31,13 @@ if [ ! -f "$npm_binary" ]; then EXTRACTED_DIR_NAME=$(ls /home/node/workdir | grep $node_folder_name | grep -v "tar") - if [[ "$EXTRACTED_DIR_NAME" == "$node_folder_name" ]]; then - # OK, no variation name to remove - echo "Extracted folder $node_folder_name" - else - mv /home/node/workdir/$EXTRACTED_DIR_NAME /home/node/workdir/$node_folder_name - echo "Extracted folder $EXTRACTED_DIR_NAME, renamed to $node_folder_name" - fi + # if [[ "$EXTRACTED_DIR_NAME" == "$node_folder_name" ]]; then + # # OK, no variation name to remove + # echo "Extracted folder $node_folder_name" + # else + # mv /home/node/workdir/$EXTRACTED_DIR_NAME /home/node/workdir/$node_folder_name + # echo "Extracted folder $EXTRACTED_DIR_NAME, renamed to $node_folder_name" + # fi fi cd src diff --git a/scripts/common.sh b/scripts/common.sh index 12ab750..6d7c2d3 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -17,3 +17,54 @@ function get_build_image_name() { echo "docker.elastic.co/elastic/nodejs-custom:$NODE_VERSION-$PLATFORM" } + +function retry() { + local retries=$1; shift + local delay=$1; shift + local attempts=1 + + until "$@"; do + retry_exit_status=$? + echo "Exited with $retry_exit_status" >&2 + if (( retries == "0" )); then + return $retry_exit_status + elif (( attempts == retries )); then + echo "Failed $attempts retries" >&2 + return $retry_exit_status + else + echo "Retrying $((retries - attempts)) more times..." >&2 + attempts=$((attempts + 1)) + sleep "$delay" + fi + done +} + +function replace_shasums_in_folder() { + local working_directory=$1 + + cd $working_directory + + # Check if SHASUMS256.txt file exists + if [ ! -f "./SHASUMS256.txt" ]; then + echo "SHASUMS256.txt file does not exist in folder: $working_directory" + exit 1 + fi + + # Loop through files in folder + for file in *; do + if [ "$file" != "SHASUMS256.txt" ] && [ -f "$file" ]; then + # Calculate SHA256 hash + new_sha256=$(shasum -a 256 "$file" | cut -d' ' -f1) + # Replace hashes in SHASUMS256.txt + node -e """ + lines = fs.readFileSync('SHASUMS256.txt').toString().split('\n') + output = lines.map(l => l.endsWith('$file') ? '$new_sha256\t$file' : l) + fs.writeFileSync('SHASUMS256.txt', output.join('\n')) + """ + fi + done + + cd - + + echo "SHASUM256.txt updated." +} diff --git a/scripts/upload_nodejs_artifacts.sh b/scripts/upload_nodejs_artifacts.sh index 6fa8e8f..582859f 100755 --- a/scripts/upload_nodejs_artifacts.sh +++ b/scripts/upload_nodejs_artifacts.sh @@ -1,17 +1,25 @@ #!/bin/bash set -euo pipefail +source ./scripts/common.sh + # TARGET_VERSION provided in env BUCKET_NAME="kibana-custom-node-artifacts" ARTIFACT_BASE_PATH="node-glibc-217/dist/v$TARGET_VERSION/" ARTIFACT_DIST_DIR="./workdir/dist" +SHASUMS_LOCATION="https://nodejs.org/dist/v$TARGET_VERSION/SHASUMS256.txt" + +# echo "--- Removing variant tags from file names" +# for name in $(ls $ARTIFACT_DIST_DIR | grep tar); do +# NAME_STRIPPED=$(echo "$name" | sed s/-glibc-217//) +# mv $ARTIFACT_DIST_DIR/$name $ARTIFACT_DIST_DIR/$NAME_STRIPPED +# done -echo "--- Removing variant tags from file names" -for name in $(ls $ARTIFACT_DIST_DIR | grep tar); do - NAME_STRIPPED=$(echo "$name" | sed s/-glibc-217//) - mv $ARTIFACT_DIST_DIR/$name $ARTIFACT_DIST_DIR/$NAME_STRIPPED -done +echo "--- Replacing SHASUMS256.txt entries" +retry 5 10 curl -fsSLO $SHASUMS_LOCATION +mv SHASUMS256.txt ./workdir/SHASUMS256.txt +replace_shasums_in_folder ./workdir -echo "--- Uploading build artifacts" -gsutil cp -r "$ARTIFACT_DIST_DIR/*" gs://$BUCKET_NAME/$ARTIFACT_BASE_PATH +# echo "--- Uploading build artifacts" +# gsutil cp -r "$ARTIFACT_DIST_DIR/*" gs://$BUCKET_NAME/$ARTIFACT_BASE_PATH diff --git a/scripts/upload_re2_artifacts.sh b/scripts/upload_re2_artifacts.sh index 5f2ee4e..b01f383 100755 --- a/scripts/upload_re2_artifacts.sh +++ b/scripts/upload_re2_artifacts.sh @@ -6,11 +6,5 @@ RE2_FULL_VERSION=${RE2_VERSION:-1.17.7} # $1 ARTIFACT_BASE_PATH="re2-glibc-217/v$RE2_FULL_VERSION/" ARTIFACT_DIST_DIR="./workdir_re2/dist" -# echo "--- Removing variant tags from file names" -# for name in $(ls $ARTIFACT_DIST_DIR | grep tar); do -# NAME_STRIPPED=$(echo "$name" | sed s/-glibc-217//) -# mv $ARTIFACT_DIST_DIR/$name $ARTIFACT_DIST_DIR/$NAME_STRIPPED -# done - echo "--- Uploading build artifacts" gsutil cp -r "$ARTIFACT_DIST_DIR/*" gs://$BUCKET_NAME/$ARTIFACT_BASE_PATH From edd9232d284f580d40febdd87668e7173c9c4735 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Thu, 22 Jun 2023 10:02:46 +0200 Subject: [PATCH 54/75] Re-add upload section --- build-image-config/re2_entrypoint.sh | 2 +- scripts/upload_nodejs_artifacts.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build-image-config/re2_entrypoint.sh b/build-image-config/re2_entrypoint.sh index ec3d7fa..61ee318 100755 --- a/build-image-config/re2_entrypoint.sh +++ b/build-image-config/re2_entrypoint.sh @@ -45,7 +45,7 @@ cd src ## Download re2 source if needed. re2_source_folder="/home/node/workdir/src/node-re2-${re2_full_version}" if [ ! -d "$re2_source_folder" ]; then - git clone --recurse-submodules --depth 1 --branch 1.17.4 https://github.com/uhop/node-re2.git "${re2_source_folder}" + git clone --recurse-submodules --depth 1 --branch $re2_full_version https://github.com/uhop/node-re2.git "${re2_source_folder}" fi cd "$re2_source_folder" diff --git a/scripts/upload_nodejs_artifacts.sh b/scripts/upload_nodejs_artifacts.sh index 582859f..a54c1db 100755 --- a/scripts/upload_nodejs_artifacts.sh +++ b/scripts/upload_nodejs_artifacts.sh @@ -21,5 +21,5 @@ retry 5 10 curl -fsSLO $SHASUMS_LOCATION mv SHASUMS256.txt ./workdir/SHASUMS256.txt replace_shasums_in_folder ./workdir -# echo "--- Uploading build artifacts" -# gsutil cp -r "$ARTIFACT_DIST_DIR/*" gs://$BUCKET_NAME/$ARTIFACT_BASE_PATH +echo "--- Uploading build artifacts" +gsutil cp -r "$ARTIFACT_DIST_DIR/*" gs://$BUCKET_NAME/$ARTIFACT_BASE_PATH From bdd64e86ae3c7b82f48843554aa9262faa0d82f1 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Thu, 22 Jun 2023 10:36:32 +0200 Subject: [PATCH 55/75] Use correct path for SHASUMS256.txt --- .gitignore | 3 +- scripts/common.sh | 4 +-- scripts/replace_sha_hashes.js | 53 ++++++++++++++++++++++++++++++ scripts/upload_nodejs_artifacts.sh | 4 +-- 4 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 scripts/replace_sha_hashes.js diff --git a/.gitignore b/.gitignore index bd29c04..4c6bf8a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .vscode .idea -workdir +workdir/ +workdir_re2/ diff --git a/scripts/common.sh b/scripts/common.sh index 6d7c2d3..33d6858 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -64,7 +64,7 @@ function replace_shasums_in_folder() { fi done - cd - + echo "`pwd`/SHASUMS256.txt updated" - echo "SHASUM256.txt updated." + cd - } diff --git a/scripts/replace_sha_hashes.js b/scripts/replace_sha_hashes.js new file mode 100644 index 0000000..109478e --- /dev/null +++ b/scripts/replace_sha_hashes.js @@ -0,0 +1,53 @@ +const fs = require("fs"); +const { spawnSync } = require("child_process"); + +const [exec, scriptName, workingDir] = process.argv; + +if (!fs.existsSync(workingDir)) { + throw new Error("Working directory doesn't exist: " + workingDir); +} +process.chdir(workingDir); +if (!fs.existsSync("SHASUMS256.txt")) { + throw new Error("No SHASUMS256.txt to override in folder: " + workingDir); +} + +// 1. read shasums file +const shaSums = fs + .readFileSync("SHASUMS256.txt") + .toString() + .trim() + .split("\n") + .reduce((hashes, line) => { + const [file, hash] = line.split(/\s+/); + hashes[file] = hash; + return hashes; + }); + +// 2. generate new shas +const files = fs + .readdirSync(".", { withFileTypes: true }) + .filter((e) => e.isFile() && e.name !== "SHASUMS256.txt") + .map((e) => e.name); + +const proc = spawnSync("shasum", ["-a", "256", ...files]); + +if (proc.stderr.length) { + console.error(proc.stderr.toString()); + process.exit(1); +} + +const shasumOutput = proc.stdout.toString(); +newHashes = shasumOutput.trim().split("\n"); + +// 2. replace old ones +newHashes.forEach(([fileName, hash]) => { + shaSums[fileName] = hash; +}); + +// 3. write shasums256.txt +const fileContent = Object.keys(shaSums).reduce((output, fileName) => { + output += `${shaSums[fileName]}\t${fileName}`; + return output; +}, ""); + +fs.writeFileSync("SHASUMS256.txt", fileContent); diff --git a/scripts/upload_nodejs_artifacts.sh b/scripts/upload_nodejs_artifacts.sh index a54c1db..7c1f57a 100755 --- a/scripts/upload_nodejs_artifacts.sh +++ b/scripts/upload_nodejs_artifacts.sh @@ -18,8 +18,8 @@ SHASUMS_LOCATION="https://nodejs.org/dist/v$TARGET_VERSION/SHASUMS256.txt" echo "--- Replacing SHASUMS256.txt entries" retry 5 10 curl -fsSLO $SHASUMS_LOCATION -mv SHASUMS256.txt ./workdir/SHASUMS256.txt -replace_shasums_in_folder ./workdir +mv SHASUMS256.txt $ARTIFACT_DIST_DIR/SHASUMS256.txt +replace_shasums_in_folder $ARTIFACT_DIST_DIR echo "--- Uploading build artifacts" gsutil cp -r "$ARTIFACT_DIST_DIR/*" gs://$BUCKET_NAME/$ARTIFACT_BASE_PATH From 7a61c2e1bfe9f422c7fa1cbd5b5fc59c1672950e Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Thu, 22 Jun 2023 11:02:11 +0200 Subject: [PATCH 56/75] Use spaces in shasum file, re-enable mac build, bump node version --- .buildkite/pipeline-kibana.yml | 32 ++++++++++++++++---------------- scripts/common.sh | 2 +- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.buildkite/pipeline-kibana.yml b/.buildkite/pipeline-kibana.yml index 214c0fe..127a415 100644 --- a/.buildkite/pipeline-kibana.yml +++ b/.buildkite/pipeline-kibana.yml @@ -9,25 +9,25 @@ steps: - scripts/upload_re2_artifacts.sh env: ARCH: amd64 - TARGET_VERSION: 18.16.0 + TARGET_VERSION: 18.16.1 RE2_VERSION: 1.17.7 agents: queue: c2-16 timeout_in_minutes: 300 # 5 hrs - # - label: Build node.js 18 with glibc 2.17 for arm64 - # command: - # - scripts/create_build_images.sh - # - scripts/build_nodejs.sh - # - scripts/upload_nodejs_artifacts.sh - # - scripts/build_re2.sh - # - scripts/upload_re2_artifacts.sh - # env: - # ARCH: arm64 - # TARGET_VERSION: 18.16.0 - # RE2_VERSION: 1.17.7 - # agents: - # # queue: macos-arm # disabled while the git cloning doesn't work - # queue: c2-60 # cross-compiling takes a while, 13 hours is not enough :/ - # timeout_in_minutes: 800 # 12+ hrs + - label: Build node.js 18 with glibc 2.17 for arm64 + command: + - scripts/create_build_images.sh + - scripts/build_nodejs.sh + - scripts/upload_nodejs_artifacts.sh + - scripts/build_re2.sh + - scripts/upload_re2_artifacts.sh + env: + ARCH: arm64 + TARGET_VERSION: 18.16.1 + RE2_VERSION: 1.17.7 + agents: + # queue: macos-arm # disabled while the git cloning doesn't work + queue: c2-60 # cross-compiling takes a while + timeout_in_minutes: 720 # 12 hrs diff --git a/scripts/common.sh b/scripts/common.sh index 33d6858..faf3723 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -58,7 +58,7 @@ function replace_shasums_in_folder() { # Replace hashes in SHASUMS256.txt node -e """ lines = fs.readFileSync('SHASUMS256.txt').toString().split('\n') - output = lines.map(l => l.endsWith('$file') ? '$new_sha256\t$file' : l) + output = lines.map(l => l.endsWith('$file') ? '$new_sha256 $file' : l) fs.writeFileSync('SHASUMS256.txt', output.join('\n')) """ fi From 702ca66622d4765352b09ce07fe08c66943dc9f7 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Thu, 22 Jun 2023 13:20:05 +0200 Subject: [PATCH 57/75] Add SHASUM fixing as a last step after both agents finished --- .buildkite/pipeline-kibana.yml | 17 ++++++---- scripts/replace_sha_hashes.js | 53 ------------------------------ scripts/replace_sha_hashes.sh | 27 +++++++++++++++ scripts/upload_nodejs_artifacts.sh | 12 ------- 4 files changed, 38 insertions(+), 71 deletions(-) delete mode 100644 scripts/replace_sha_hashes.js create mode 100755 scripts/replace_sha_hashes.sh diff --git a/.buildkite/pipeline-kibana.yml b/.buildkite/pipeline-kibana.yml index 127a415..4a9df2e 100644 --- a/.buildkite/pipeline-kibana.yml +++ b/.buildkite/pipeline-kibana.yml @@ -1,6 +1,9 @@ +env: + TARGET_VERSION: 18.16.1 + RE2_VERSION: 1.17.7 + steps: - label: Build node.js 18 with glibc 2.17 for x64 - command: - scripts/create_build_images.sh - scripts/build_nodejs.sh @@ -9,13 +12,10 @@ steps: - scripts/upload_re2_artifacts.sh env: ARCH: amd64 - TARGET_VERSION: 18.16.1 - RE2_VERSION: 1.17.7 agents: queue: c2-16 timeout_in_minutes: 300 # 5 hrs - - label: Build node.js 18 with glibc 2.17 for arm64 command: - scripts/create_build_images.sh @@ -25,9 +25,14 @@ steps: - scripts/upload_re2_artifacts.sh env: ARCH: arm64 - TARGET_VERSION: 18.16.1 - RE2_VERSION: 1.17.7 agents: # queue: macos-arm # disabled while the git cloning doesn't work queue: c2-60 # cross-compiling takes a while timeout_in_minutes: 720 # 12 hrs + + - wait + + - label: Fix SHASUMS256.txt with newly built files' hashes + command: node replace_sha_hashes.js + agents: + queue: kibana-default diff --git a/scripts/replace_sha_hashes.js b/scripts/replace_sha_hashes.js deleted file mode 100644 index 109478e..0000000 --- a/scripts/replace_sha_hashes.js +++ /dev/null @@ -1,53 +0,0 @@ -const fs = require("fs"); -const { spawnSync } = require("child_process"); - -const [exec, scriptName, workingDir] = process.argv; - -if (!fs.existsSync(workingDir)) { - throw new Error("Working directory doesn't exist: " + workingDir); -} -process.chdir(workingDir); -if (!fs.existsSync("SHASUMS256.txt")) { - throw new Error("No SHASUMS256.txt to override in folder: " + workingDir); -} - -// 1. read shasums file -const shaSums = fs - .readFileSync("SHASUMS256.txt") - .toString() - .trim() - .split("\n") - .reduce((hashes, line) => { - const [file, hash] = line.split(/\s+/); - hashes[file] = hash; - return hashes; - }); - -// 2. generate new shas -const files = fs - .readdirSync(".", { withFileTypes: true }) - .filter((e) => e.isFile() && e.name !== "SHASUMS256.txt") - .map((e) => e.name); - -const proc = spawnSync("shasum", ["-a", "256", ...files]); - -if (proc.stderr.length) { - console.error(proc.stderr.toString()); - process.exit(1); -} - -const shasumOutput = proc.stdout.toString(); -newHashes = shasumOutput.trim().split("\n"); - -// 2. replace old ones -newHashes.forEach(([fileName, hash]) => { - shaSums[fileName] = hash; -}); - -// 3. write shasums256.txt -const fileContent = Object.keys(shaSums).reduce((output, fileName) => { - output += `${shaSums[fileName]}\t${fileName}`; - return output; -}, ""); - -fs.writeFileSync("SHASUMS256.txt", fileContent); diff --git a/scripts/replace_sha_hashes.sh b/scripts/replace_sha_hashes.sh new file mode 100755 index 0000000..cfa1ac6 --- /dev/null +++ b/scripts/replace_sha_hashes.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -euo pipefail + +source ./scripts/common.sh + +# TARGET_VERSION provided in env + +BUCKET_NAME="kibana-custom-node-artifacts" +ARTIFACT_BASE_PATH="node-glibc-217/dist/v$TARGET_VERSION" +SHASUMS_LOCATION="https://nodejs.org/dist/v$TARGET_VERSION/SHASUMS256.txt" + +ARTIFACT_DIST_DIR="./workdir/download" + +echo "--- Downloading node.js $TARGET_VERSION glibc-217 artifacts" +mkdir -p $ARTIFACT_DIST_DIR + +gsutil cp -r gs://$BUCKET_NAME/$ARTIFACT_BASE_PATH/* "$ARTIFACT_DIST_DIR/" + +echo "--- Downloading SHASUMS256.txt" +retry 5 10 curl -fsSLO $SHASUMS_LOCATION +mv SHASUMS256.txt $ARTIFACT_DIST_DIR/SHASUMS256.txt + +echo "--- Replacing checksums with local file hashes" +replace_shasums_in_folder $ARTIFACT_DIST_DIR + +echo "--- Uploading SHASUMS256.txt to $BUCKET_NAME" +gsutil cp -r "$ARTIFACT_DIST_DIR/SHASUMS256.txt" gs://$BUCKET_NAME/$ARTIFACT_BASE_PATH/ diff --git a/scripts/upload_nodejs_artifacts.sh b/scripts/upload_nodejs_artifacts.sh index 7c1f57a..c8f10f9 100755 --- a/scripts/upload_nodejs_artifacts.sh +++ b/scripts/upload_nodejs_artifacts.sh @@ -8,18 +8,6 @@ source ./scripts/common.sh BUCKET_NAME="kibana-custom-node-artifacts" ARTIFACT_BASE_PATH="node-glibc-217/dist/v$TARGET_VERSION/" ARTIFACT_DIST_DIR="./workdir/dist" -SHASUMS_LOCATION="https://nodejs.org/dist/v$TARGET_VERSION/SHASUMS256.txt" - -# echo "--- Removing variant tags from file names" -# for name in $(ls $ARTIFACT_DIST_DIR | grep tar); do -# NAME_STRIPPED=$(echo "$name" | sed s/-glibc-217//) -# mv $ARTIFACT_DIST_DIR/$name $ARTIFACT_DIST_DIR/$NAME_STRIPPED -# done - -echo "--- Replacing SHASUMS256.txt entries" -retry 5 10 curl -fsSLO $SHASUMS_LOCATION -mv SHASUMS256.txt $ARTIFACT_DIST_DIR/SHASUMS256.txt -replace_shasums_in_folder $ARTIFACT_DIST_DIR echo "--- Uploading build artifacts" gsutil cp -r "$ARTIFACT_DIST_DIR/*" gs://$BUCKET_NAME/$ARTIFACT_BASE_PATH From 58f0c0a08dd97ad80e47aaab3fa5ae6a73f4c862 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Thu, 22 Jun 2023 13:45:29 +0200 Subject: [PATCH 58/75] Add retry around node download.. node.js org is flaky --- scripts/build_nodejs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_nodejs.sh b/scripts/build_nodejs.sh index 10d85cd..8b80773 100755 --- a/scripts/build_nodejs.sh +++ b/scripts/build_nodejs.sh @@ -15,7 +15,7 @@ RELEASE_URL_BASE="https://unofficial-builds.nodejs.org/download/release/" echo "Running node.js build in folder: `pwd`" echo '--- Downloading node source' -curl --create-dirs --output-dir ./workdir/src -fsSLO --compressed \ +retry 5 15 curl --create-dirs --output-dir ./workdir/src -fsSLO --compressed \ https://nodejs.org/download/release/$TARGET_NODE_VERSION/node-$TARGET_NODE_VERSION.tar.xz tar -xf ./workdir/src/node-$TARGET_NODE_VERSION.tar.xz -C ./workdir/src chmod -R a+rwx ./workdir/ From e85ade7f9bf46af26414660e3a9e894faedaf869 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Thu, 22 Jun 2023 15:50:13 +0200 Subject: [PATCH 59/75] Fix typo, temporarily disable build steps --- .buildkite/pipeline-kibana.yml | 54 +++++++++++++++++----------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/.buildkite/pipeline-kibana.yml b/.buildkite/pipeline-kibana.yml index 4a9df2e..d131a3d 100644 --- a/.buildkite/pipeline-kibana.yml +++ b/.buildkite/pipeline-kibana.yml @@ -3,36 +3,36 @@ env: RE2_VERSION: 1.17.7 steps: - - label: Build node.js 18 with glibc 2.17 for x64 - command: - - scripts/create_build_images.sh - - scripts/build_nodejs.sh - - scripts/upload_nodejs_artifacts.sh - - scripts/build_re2.sh - - scripts/upload_re2_artifacts.sh - env: - ARCH: amd64 - agents: - queue: c2-16 - timeout_in_minutes: 300 # 5 hrs + # - label: Build node.js 18 with glibc 2.17 for x64 + # command: + # - scripts/create_build_images.sh + # - scripts/build_nodejs.sh + # - scripts/upload_nodejs_artifacts.sh + # - scripts/build_re2.sh + # - scripts/upload_re2_artifacts.sh + # env: + # ARCH: amd64 + # agents: + # queue: c2-16 + # timeout_in_minutes: 300 # 5 hrs - - label: Build node.js 18 with glibc 2.17 for arm64 - command: - - scripts/create_build_images.sh - - scripts/build_nodejs.sh - - scripts/upload_nodejs_artifacts.sh - - scripts/build_re2.sh - - scripts/upload_re2_artifacts.sh - env: - ARCH: arm64 - agents: - # queue: macos-arm # disabled while the git cloning doesn't work - queue: c2-60 # cross-compiling takes a while - timeout_in_minutes: 720 # 12 hrs + # - label: Build node.js 18 with glibc 2.17 for arm64 + # command: + # - scripts/create_build_images.sh + # - scripts/build_nodejs.sh + # - scripts/upload_nodejs_artifacts.sh + # - scripts/build_re2.sh + # - scripts/upload_re2_artifacts.sh + # env: + # ARCH: arm64 + # agents: + # # queue: macos-arm # disabled while the git cloning doesn't work + # queue: c2-60 # cross-compiling takes a while + # timeout_in_minutes: 720 # 12 hrs - - wait + # - wait - label: Fix SHASUMS256.txt with newly built files' hashes - command: node replace_sha_hashes.js + command: scripts/replace_sha_hashes.sh agents: queue: kibana-default From 7dde65d6a5954a6b404b64f0cae04cd90863a476 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Thu, 22 Jun 2023 15:52:06 +0200 Subject: [PATCH 60/75] Re-enable build steps --- .buildkite/pipeline-kibana.yml | 52 +++++++++++++++++----------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/.buildkite/pipeline-kibana.yml b/.buildkite/pipeline-kibana.yml index d131a3d..1407b13 100644 --- a/.buildkite/pipeline-kibana.yml +++ b/.buildkite/pipeline-kibana.yml @@ -3,34 +3,34 @@ env: RE2_VERSION: 1.17.7 steps: - # - label: Build node.js 18 with glibc 2.17 for x64 - # command: - # - scripts/create_build_images.sh - # - scripts/build_nodejs.sh - # - scripts/upload_nodejs_artifacts.sh - # - scripts/build_re2.sh - # - scripts/upload_re2_artifacts.sh - # env: - # ARCH: amd64 - # agents: - # queue: c2-16 - # timeout_in_minutes: 300 # 5 hrs + - label: Build node.js 18 with glibc 2.17 for x64 + command: + - scripts/create_build_images.sh + - scripts/build_nodejs.sh + - scripts/upload_nodejs_artifacts.sh + - scripts/build_re2.sh + - scripts/upload_re2_artifacts.sh + env: + ARCH: amd64 + agents: + queue: c2-16 + timeout_in_minutes: 300 # 5 hrs - # - label: Build node.js 18 with glibc 2.17 for arm64 - # command: - # - scripts/create_build_images.sh - # - scripts/build_nodejs.sh - # - scripts/upload_nodejs_artifacts.sh - # - scripts/build_re2.sh - # - scripts/upload_re2_artifacts.sh - # env: - # ARCH: arm64 - # agents: - # # queue: macos-arm # disabled while the git cloning doesn't work - # queue: c2-60 # cross-compiling takes a while - # timeout_in_minutes: 720 # 12 hrs + - label: Build node.js 18 with glibc 2.17 for arm64 + command: + - scripts/create_build_images.sh + - scripts/build_nodejs.sh + - scripts/upload_nodejs_artifacts.sh + - scripts/build_re2.sh + - scripts/upload_re2_artifacts.sh + env: + ARCH: arm64 + agents: + # queue: macos-arm # disabled while the git cloning doesn't work + queue: c2-60 # cross-compiling takes a while + timeout_in_minutes: 720 # 12 hrs - # - wait + - wait - label: Fix SHASUMS256.txt with newly built files' hashes command: scripts/replace_sha_hashes.sh From fecc714ab61d36ca17ed36f271792100c841698a Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Thu, 22 Jun 2023 16:18:01 +0200 Subject: [PATCH 61/75] Remove unnecessary files, add README/Context --- README.md | 51 ++++++++++++++++++++++++---- build-image-config/README.md | 49 -------------------------- build-image-config/re2_entrypoint.sh | 12 ------- scripts/docker_login.sh | 9 ----- 4 files changed, 44 insertions(+), 77 deletions(-) delete mode 100644 build-image-config/README.md delete mode 100755 scripts/docker_login.sh diff --git a/README.md b/README.md index 3397676..421be77 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,47 @@ # kibana-custom-nodejs-builds -Contains configuration and sources to build node.js for custom platforms +Contains configuration and sources to build node.js -TODO: write about the repo structure and the call-chain +The main usecase right now is building `node.js@18+` with `glibc@2.17`, which is required for some older platforms. (More context: https://github.com/nodejs/unofficial-builds/pull/69) -TODOs: - - create buildkite pipeline - - add node.js distribution publish step (should this be published on Github?) - - check Docker build, whether it will need uploading - - parameterize tasks for node version +## Running locally +You can run most scripts locally on Mac/Linux. You'll need a few of the build/infra tools: + - Docker + - node.js + - gsutil (`brew install google-cloud-sdk`) + +Export some env variables required for the builds +```sh + export ARCH="arm64" + export TARGET_VERSION="18.16.1" + export RE2_VERSION="1.17.7" +``` + +Then run individual scripts locally: +```sh + ./scripts/create_build_images.sh + ./scripts/build_nodejs.sh +``` + +## Docker image for node.js builds +One of the main components we need to create in this step is a docker image for an environment that's set up for building `node.js`. + +The bits for this component are in the [build-image-config](./build-image-config/) folder. + +The docker image uses mounted directories as working directories, as well as outputting the artifacts in these directories. + + +## Scripts for running the builds +Most of the buildkite logic is sheltered in the [scripts](./scripts/) directory. + + + +## Context +During development, we found some more information that can be helpful as context, should anyone find this repo again + + - This repository is only needed until + - centos:7 / RHEL7 is supported by Elastic, and we ship node.js with Kibana + - the unofficial-builds repo accepts a linux/arm64 build (https://github.com/nodejs/unofficial-builds/pull/83) + - The created Docker images needn't be pushed + - they can be used once for the build, then rebuilt in case we need to run it again + - I decided to remove the `VARIATION` attribute on the node.js: + - build would result in the variation showing up in the file and folder names, making the logistics more difficult, if we want to keep this mostly transparent for Kibana diff --git a/build-image-config/README.md b/build-image-config/README.md deleted file mode 100644 index ffd14f1..0000000 --- a/build-image-config/README.md +++ /dev/null @@ -1,49 +0,0 @@ -## Build multi-platform images (~10-15 minutes) - -```bash -docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -DOCKER_BUILDKIT=1 docker buildx build --progress=plain --push \ - --platform linux/amd64,linux/arm64 \ - --build-arg UID=1000 --build-arg GID=1000 \ - --tag azasypkin/nodejs-custom:18.15.0 . -docker pull --platform linux/amd64 azasypkin/nodejs-custom:18.15.0 -docker pull --platform linux/arm64 azasypkin/nodejs-custom:18.15.0 -``` - -## Build Node.js (x64 - ~20 minutes, arm64 - ~4 hours) - -```bash -curl --create-dirs --output-dir ./workdir/src -fsSLO --compressed \ - https://nodejs.org/download/release/v18.15.0/node-v18.15.0.tar.xz -tar -xf ./workdir/src/node-v18.15.0.tar.xz -C ./workdir/src -docker run --rm -it --platform linux/amd64 \ - -v ./workdir:/home/node/workdir \ - azasypkin/nodejs-custom:18.15.0 \ - https://unofficial-builds.nodejs.org/download/release/ \ - v18.15.0 - -docker run --rm -it --platform linux/arm64 \ - -v ./workdir:/home/node/workdir \ - azasypkin/nodejs-custom:18.15.0 \ - https://unofficial-builds.nodejs.org/download/release/ \ - v18.15.0 -``` - -## Build re2 - -```bash -docker run --rm -it --platform linux/amd64 --entrypoint /home/node/re2_entrypoint.sh \ - -v ./workdir:/home/node/workdir \ - -v ./re2-entrypoint.sh:/home/node/re2_entrypoint.sh \ - azasypkin/nodejs-custom:18.15.0 \ - 1.17.7 \ - 18.15.0 \ - https://github.com/azasypkin/kibana/releases/download/nodej-custom -docker run --rm -it --platform linux/arm64 --entrypoint /home/node/re2_entrypoint.sh \ - -v ./workdir:/home/node/workdir \ - -v ./re2-entrypoint.sh:/home/node/re2_entrypoint.sh \ - azasypkin/nodejs-custom:18.15.0 \ - 1.17.7 \ - 18.15.0 \ - https://github.com/azasypkin/kibana/releases/download/nodej-custom -``` \ No newline at end of file diff --git a/build-image-config/re2_entrypoint.sh b/build-image-config/re2_entrypoint.sh index 61ee318..eaddba2 100755 --- a/build-image-config/re2_entrypoint.sh +++ b/build-image-config/re2_entrypoint.sh @@ -7,8 +7,6 @@ re2_full_version="$1" node_full_version="$2" node_download_base_url="$3" -# VARIATION="glibc-217" - if [[ $(arch) == x86_64 ]]; then architecture="x64"; else @@ -28,16 +26,6 @@ if [ ! -f "$npm_binary" ]; then fi tar -xf "/home/node/workdir/${node_folder_name}.tar.xz" - - EXTRACTED_DIR_NAME=$(ls /home/node/workdir | grep $node_folder_name | grep -v "tar") - - # if [[ "$EXTRACTED_DIR_NAME" == "$node_folder_name" ]]; then - # # OK, no variation name to remove - # echo "Extracted folder $node_folder_name" - # else - # mv /home/node/workdir/$EXTRACTED_DIR_NAME /home/node/workdir/$node_folder_name - # echo "Extracted folder $EXTRACTED_DIR_NAME, renamed to $node_folder_name" - # fi fi cd src diff --git a/scripts/docker_login.sh b/scripts/docker_login.sh deleted file mode 100755 index e7fbdb0..0000000 --- a/scripts/docker_login.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -registry_user=$(vault read -field=username secret/ci/elastic-ci-agent-images/docker-registry) -registry_password=$(vault read -field=password secret/ci/elastic-ci-agent-images/docker-registry) - -echo -n "Logging in to docker.elastic.co as ${registry_user}... " -docker login --username="${registry_user}" --password="${registry_password}" docker.elastic.co From 1656a11c3e9590ff5c4670d271b8289c2ef92d52 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Fri, 23 Jun 2023 09:49:17 +0200 Subject: [PATCH 62/75] Update README.md Co-authored-by: Jon --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 421be77..c55f34c 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ Most of the buildkite logic is sheltered in the [scripts](./scripts/) directory. ## Context During development, we found some more information that can be helpful as context, should anyone find this repo again - - This repository is only needed until + - This repository is only needed while - centos:7 / RHEL7 is supported by Elastic, and we ship node.js with Kibana - the unofficial-builds repo accepts a linux/arm64 build (https://github.com/nodejs/unofficial-builds/pull/83) - The created Docker images needn't be pushed From 21f14c0e6f6da00069972079542bad13c06a4bf9 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Mon, 26 Jun 2023 10:50:30 +0200 Subject: [PATCH 63/75] Centralize the artifacts' upload bucket name --- README.md | 2 +- scripts/build_re2.sh | 2 +- scripts/common.sh | 2 ++ scripts/replace_sha_hashes.sh | 1 - scripts/upload_nodejs_artifacts.sh | 1 - scripts/upload_re2_artifacts.sh | 3 ++- 6 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c55f34c..1f50b3d 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ The docker image uses mounted directories as working directories, as well as out ## Scripts for running the builds -Most of the buildkite logic is sheltered in the [scripts](./scripts/) directory. +Most of the `buildkite` logic is sheltered in the [scripts](./scripts/) directory. diff --git a/scripts/build_re2.sh b/scripts/build_re2.sh index 50f32f8..01613b4 100755 --- a/scripts/build_re2.sh +++ b/scripts/build_re2.sh @@ -10,7 +10,7 @@ assert_correct_arch $ARCH BUILD_IMAGE_NAME=$(get_build_image_name) RE2_FULL_VERSION=${RE2_VERSION:-1.17.7} # $1 NODE_FULL_VERSION="v$TARGET_VERSION" # $2 -NODE_DOWNLOAD_BASE_URL="https://storage.googleapis.com/kibana-custom-node-artifacts/node-glibc-217/dist/v$TARGET_VERSION" +NODE_DOWNLOAD_BASE_URL="https://storage.googleapis.com/$BUCKET_NAME/node-glibc-217/dist/v$TARGET_VERSION" TARGET_PLATFORM="linux/$ARCH" echo "--- Building RE2 for $TARGET_PLATFORM" diff --git a/scripts/common.sh b/scripts/common.sh index faf3723..eb7bc2b 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -1,4 +1,6 @@ +BUCKET_NAME="kibana-custom-node-artifacts" + function assert_correct_arch() { ARCH=$1 diff --git a/scripts/replace_sha_hashes.sh b/scripts/replace_sha_hashes.sh index cfa1ac6..a77c759 100755 --- a/scripts/replace_sha_hashes.sh +++ b/scripts/replace_sha_hashes.sh @@ -5,7 +5,6 @@ source ./scripts/common.sh # TARGET_VERSION provided in env -BUCKET_NAME="kibana-custom-node-artifacts" ARTIFACT_BASE_PATH="node-glibc-217/dist/v$TARGET_VERSION" SHASUMS_LOCATION="https://nodejs.org/dist/v$TARGET_VERSION/SHASUMS256.txt" diff --git a/scripts/upload_nodejs_artifacts.sh b/scripts/upload_nodejs_artifacts.sh index c8f10f9..4295aeb 100755 --- a/scripts/upload_nodejs_artifacts.sh +++ b/scripts/upload_nodejs_artifacts.sh @@ -5,7 +5,6 @@ source ./scripts/common.sh # TARGET_VERSION provided in env -BUCKET_NAME="kibana-custom-node-artifacts" ARTIFACT_BASE_PATH="node-glibc-217/dist/v$TARGET_VERSION/" ARTIFACT_DIST_DIR="./workdir/dist" diff --git a/scripts/upload_re2_artifacts.sh b/scripts/upload_re2_artifacts.sh index b01f383..8ff77d2 100755 --- a/scripts/upload_re2_artifacts.sh +++ b/scripts/upload_re2_artifacts.sh @@ -1,7 +1,8 @@ #!/bin/bash set -euo pipefail -BUCKET_NAME="kibana-custom-node-artifacts" +source ./scripts/common.sh + RE2_FULL_VERSION=${RE2_VERSION:-1.17.7} # $1 ARTIFACT_BASE_PATH="re2-glibc-217/v$RE2_FULL_VERSION/" ARTIFACT_DIST_DIR="./workdir_re2/dist" From 68d258e03a40b4ffb95571f1a31b1e1d39d63ba0 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Tue, 27 Jun 2023 12:00:17 +0200 Subject: [PATCH 64/75] Remove RE2 aspects of the build --- .buildkite/pipeline-kibana.yml | 5 --- .gitignore | 1 - README.md | 1 - build-image-config/Dockerfile | 1 - build-image-config/entrypoint.sh | 2 +- build-image-config/re2_entrypoint.sh | 57 ---------------------------- scripts/build_re2.sh | 24 ------------ scripts/upload_re2_artifacts.sh | 11 ------ 8 files changed, 1 insertion(+), 101 deletions(-) delete mode 100755 build-image-config/re2_entrypoint.sh delete mode 100755 scripts/build_re2.sh delete mode 100755 scripts/upload_re2_artifacts.sh diff --git a/.buildkite/pipeline-kibana.yml b/.buildkite/pipeline-kibana.yml index 1407b13..0c0b900 100644 --- a/.buildkite/pipeline-kibana.yml +++ b/.buildkite/pipeline-kibana.yml @@ -1,6 +1,5 @@ env: TARGET_VERSION: 18.16.1 - RE2_VERSION: 1.17.7 steps: - label: Build node.js 18 with glibc 2.17 for x64 @@ -8,8 +7,6 @@ steps: - scripts/create_build_images.sh - scripts/build_nodejs.sh - scripts/upload_nodejs_artifacts.sh - - scripts/build_re2.sh - - scripts/upload_re2_artifacts.sh env: ARCH: amd64 agents: @@ -21,8 +18,6 @@ steps: - scripts/create_build_images.sh - scripts/build_nodejs.sh - scripts/upload_nodejs_artifacts.sh - - scripts/build_re2.sh - - scripts/upload_re2_artifacts.sh env: ARCH: arm64 agents: diff --git a/.gitignore b/.gitignore index 5a06f8f..61b15c1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,3 @@ .idea workdir/ -workdir_re2/ \ No newline at end of file diff --git a/README.md b/README.md index 1f50b3d..019d76b 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,6 @@ Export some env variables required for the builds ```sh export ARCH="arm64" export TARGET_VERSION="18.16.1" - export RE2_VERSION="1.17.7" ``` Then run individual scripts locally: diff --git a/build-image-config/Dockerfile b/build-image-config/Dockerfile index a1d6faa..34ece57 100644 --- a/build-image-config/Dockerfile +++ b/build-image-config/Dockerfile @@ -22,7 +22,6 @@ RUN ulimit -n 1024 \ glibc-devel COPY --chown=node:node entrypoint.sh /home/node/entrypoint.sh -COPY --chown=node:node re2_entrypoint.sh /home/node/re2_entrypoint.sh USER node diff --git a/build-image-config/entrypoint.sh b/build-image-config/entrypoint.sh index db4fde1..90e0398 100755 --- a/build-image-config/entrypoint.sh +++ b/build-image-config/entrypoint.sh @@ -5,7 +5,7 @@ set -x release_url_base="$1" full_version="$2" -config_flags=${3:-""} #"--without-dtrace --without-npm --without-etw" +config_flags=${3:-""} if [[ $(arch) == x86_64 ]]; then architecture="x64"; diff --git a/build-image-config/re2_entrypoint.sh b/build-image-config/re2_entrypoint.sh deleted file mode 100755 index eaddba2..0000000 --- a/build-image-config/re2_entrypoint.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -x - -re2_full_version="$1" -node_full_version="$2" -node_download_base_url="$3" - -if [[ $(arch) == x86_64 ]]; then - architecture="x64"; -else - architecture="arm64" -fi - -cd /home/node/workdir -mkdir -p dist/ -mkdir -p src/ - -## Download and unpack Node.js binary if needed. -node_folder_name="node-${node_full_version}-linux-${architecture}" -npm_binary="/home/node/workdir/${node_folder_name}/bin/npm" -if [ ! -f "$npm_binary" ]; then - if [ ! -f "/home/node/workdir/$node_folder_name.tar.xz" ]; then - curl -fsSLO --compressed "${node_download_base_url}/${node_folder_name}.tar.xz" - fi - - tar -xf "/home/node/workdir/${node_folder_name}.tar.xz" -fi - -cd src - -## Download re2 source if needed. -re2_source_folder="/home/node/workdir/src/node-re2-${re2_full_version}" -if [ ! -d "$re2_source_folder" ]; then - git clone --recurse-submodules --depth 1 --branch $re2_full_version https://github.com/uhop/node-re2.git "${re2_source_folder}" -fi - -cd "$re2_source_folder" -export PATH="/home/node/workdir/${node_folder_name}/bin:$PATH" -export DEVELOPMENT_SKIP_GETTING_ASSET=true - -export CCACHE_DIR="/home/node/workdir/.ccache-re2-${architecture}" -export CC="ccache gcc" -export CXX="ccache g++" - -. /opt/rh/devtoolset-9/enable - -$npm_binary i --unsafe-perm=true -$npm_binary run build --if-present -$npm_binary test - -mkdir -p /home/node/workdir/dist/ -chmod a+w /home/node/workdir/dist -cp "${re2_source_folder}/build/Release/re2.node" "/home/node/workdir/dist/linux-${architecture}-108" -gzip -f "/home/node/workdir/dist/linux-${architecture}-108" -chmod a+rwx /home/node/workdir/dist/* diff --git a/scripts/build_re2.sh b/scripts/build_re2.sh deleted file mode 100755 index 01613b4..0000000 --- a/scripts/build_re2.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -set -euo pipefail - -source ./scripts/common.sh - -# TARGET_VERSION provided in env -# ARCH provided in env -assert_correct_arch $ARCH - -BUILD_IMAGE_NAME=$(get_build_image_name) -RE2_FULL_VERSION=${RE2_VERSION:-1.17.7} # $1 -NODE_FULL_VERSION="v$TARGET_VERSION" # $2 -NODE_DOWNLOAD_BASE_URL="https://storage.googleapis.com/$BUCKET_NAME/node-glibc-217/dist/v$TARGET_VERSION" -TARGET_PLATFORM="linux/$ARCH" - -echo "--- Building RE2 for $TARGET_PLATFORM" -mkdir -p ./workdir_re2/ -chmod -R a+rwx ./workdir_re2/ -docker run --rm -it --platform $TARGET_PLATFORM --entrypoint /home/node/re2_entrypoint.sh \ - -v ./workdir_re2:/home/node/workdir \ - $BUILD_IMAGE_NAME \ - $RE2_FULL_VERSION \ - $NODE_FULL_VERSION \ - $NODE_DOWNLOAD_BASE_URL diff --git a/scripts/upload_re2_artifacts.sh b/scripts/upload_re2_artifacts.sh deleted file mode 100755 index 8ff77d2..0000000 --- a/scripts/upload_re2_artifacts.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -set -euo pipefail - -source ./scripts/common.sh - -RE2_FULL_VERSION=${RE2_VERSION:-1.17.7} # $1 -ARTIFACT_BASE_PATH="re2-glibc-217/v$RE2_FULL_VERSION/" -ARTIFACT_DIST_DIR="./workdir_re2/dist" - -echo "--- Uploading build artifacts" -gsutil cp -r "$ARTIFACT_DIST_DIR/*" gs://$BUCKET_NAME/$ARTIFACT_BASE_PATH From ab58662b78d4a2fe62c0a829543266313676570a Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Tue, 27 Jun 2023 12:22:26 +0200 Subject: [PATCH 65/75] Remove traces of the ci-systems pipeline --- .buildkite/pipeline.yml | 3 - catalog-info.yaml | 18 ------ .../custom-nodejs-builder-amd64/Dockerfile | 31 ----------- .../custom-nodejs-builder-amd64/drivah.toml | 13 ----- .../custom-nodejs-builder-amd64/entrypoint.sh | 34 ------------ .../re2_entrypoint.sh | 55 ------------------- .../custom-nodejs-builder-arm64/Dockerfile | 31 ----------- .../custom-nodejs-builder-arm64/drivah.toml | 15 ----- .../custom-nodejs-builder-arm64/entrypoint.sh | 34 ------------ .../re2_entrypoint.sh | 55 ------------------- 10 files changed, 289 deletions(-) delete mode 100644 .buildkite/pipeline.yml delete mode 100644 containers/custom-nodejs-builder-amd64/Dockerfile delete mode 100644 containers/custom-nodejs-builder-amd64/drivah.toml delete mode 100644 containers/custom-nodejs-builder-amd64/entrypoint.sh delete mode 100644 containers/custom-nodejs-builder-amd64/re2_entrypoint.sh delete mode 100644 containers/custom-nodejs-builder-arm64/Dockerfile delete mode 100644 containers/custom-nodejs-builder-arm64/drivah.toml delete mode 100644 containers/custom-nodejs-builder-arm64/entrypoint.sh delete mode 100644 containers/custom-nodejs-builder-arm64/re2_entrypoint.sh diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml deleted file mode 100644 index ba34741..0000000 --- a/.buildkite/pipeline.yml +++ /dev/null @@ -1,3 +0,0 @@ -steps: - - command: echo "OK" - label: Blank command, placeholder \ No newline at end of file diff --git a/catalog-info.yaml b/catalog-info.yaml index 7800189..2c815f7 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -8,21 +8,3 @@ metadata: links: - title: Pipeline url: https://buildkite.com/elastic/kibana-custom-nodejs-builds - -spec: - type: buildkite-pipeline - owner: group:kibana-operations - system: buildkite - implementation: - apiVersion: buildkite.elastic.dev/v1 - kind: Pipeline - metadata: - name: kibana-custom-nodejs-builds - spec: - repository: elastic/kibana-custom-nodejs-builds - pipeline_file: ".buildkite/pipeline.yml" - teams: - kibana-operations: - access_level: MANAGE_BUILD_AND_READ - everyone: - access_level: READ_ONLY diff --git a/containers/custom-nodejs-builder-amd64/Dockerfile b/containers/custom-nodejs-builder-amd64/Dockerfile deleted file mode 100644 index a1d6faa..0000000 --- a/containers/custom-nodejs-builder-amd64/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -FROM centos:7 - -ARG GROUP_ID=1000 -ARG USER_ID=1000 - -RUN groupadd --force --gid $GROUP_ID node -RUN adduser --gid $GROUP_ID --uid $USER_ID node - -RUN ulimit -n 1024 \ - && yum install -y epel-release \ - && yum install -y centos-release-scl-rh \ - && yum upgrade -y \ - && yum install -y \ - git \ - curl \ - make \ - python2 \ - python3 \ - ccache \ - xz-utils \ - devtoolset-9 \ - glibc-devel - -COPY --chown=node:node entrypoint.sh /home/node/entrypoint.sh -COPY --chown=node:node re2_entrypoint.sh /home/node/re2_entrypoint.sh - -USER node - -VOLUME /home/node/workdir - -ENTRYPOINT [ "/home/node/entrypoint.sh" ] diff --git a/containers/custom-nodejs-builder-amd64/drivah.toml b/containers/custom-nodejs-builder-amd64/drivah.toml deleted file mode 100644 index 60292f6..0000000 --- a/containers/custom-nodejs-builder-amd64/drivah.toml +++ /dev/null @@ -1,13 +0,0 @@ -[container.image] -names = ["docker.elastic.co/ci-agent-images/kibana/custom-nodejs-builder"] -tags = ["amd64-0.1"] - -[container.image.build_args] -GROUP_ID = 1000 -USER_ID = 1000 - -[docker] -build_flags = [ - "--progress=plain", - "--platform=linux/amd64" -] diff --git a/containers/custom-nodejs-builder-amd64/entrypoint.sh b/containers/custom-nodejs-builder-amd64/entrypoint.sh deleted file mode 100644 index fffa7f4..0000000 --- a/containers/custom-nodejs-builder-amd64/entrypoint.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -x - -release_url_base="$1" -full_version="$2" -config_flags=${3:-""} #"--without-dtrace --without-npm --without-etw" - -if [[ $(arch) == x86_64 ]]; then - architecture="x64"; -else - architecture="arm64" -fi - -cd "/home/node/workdir/src/node-${full_version}" - -# Compile from source -export CCACHE_DIR="/home/node/workdir/.ccache-${architecture}" -export CC="ccache gcc" -export CXX="ccache g++" - -. /opt/rh/devtoolset-9/enable - -make -j"$(getconf _NPROCESSORS_ONLN)" binary V= \ - DESTCPU="$architecture" \ - ARCH="$architecture" \ - VARIATION="glibc-217" \ - DISTTYPE="release" \ - RELEASE_URLBASE="$release_url_base" \ - CONFIG_FLAGS="$config_flags" - -mkdir -p /home/node/workdir/dist/ -mv node-*.tar.?z /home/node/workdir/dist/ diff --git a/containers/custom-nodejs-builder-amd64/re2_entrypoint.sh b/containers/custom-nodejs-builder-amd64/re2_entrypoint.sh deleted file mode 100644 index a8165d2..0000000 --- a/containers/custom-nodejs-builder-amd64/re2_entrypoint.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -x - -re2_full_version="$1" -node_full_version="$2" -node_download_base_url="$3" - -if [[ $(arch) == x86_64 ]]; then - architecture="x64"; -else - architecture="arm64" -fi - -cd /home/node/workdir -mkdir -p dist/ -mkdir -p src/ - -## Download and unpack Node.js binary if needed. -node_folder_name="node-${node_full_version}-linux-${architecture}" -npm_binary="/home/node/workdir/${node_folder_name}/bin/npm" -if [ ! -f "$npm_binary" ]; then - if [ ! -f "/home/node/workdir/$node_folder_name.tar.xz" ]; then - curl -fsSLO --compressed "${node_download_base_url}/${node_folder_name}.tar.xz" - fi - - tar -xf "/home/node/workdir/${node_folder_name}.tar.xz" -fi - -cd src - -## Download re2 source if needed. -re2_source_folder="/home/node/workdir/src/node-re2-${re2_full_version}" -if [ ! -d "$re2_source_folder" ]; then - git clone --recurse-submodules --depth 1 --branch 1.17.4 https://github.com/uhop/node-re2.git "${re2_source_folder}" -fi - -cd "$re2_source_folder" -export PATH="/home/node/workdir/${node_folder_name}/bin:$PATH" -export DEVELOPMENT_SKIP_GETTING_ASSET=true - -export CCACHE_DIR="/home/node/workdir/.ccache-re2-${architecture}" -export CC="ccache gcc" -export CXX="ccache g++" - -. /opt/rh/devtoolset-9/enable - -npm i --unsafe-perm=true -npm run build --if-present -npm test - -mkdir -p /home/node/workdir/dist/ -cp "${re2_source_folder}/build/Release/re2.node" "/home/node/workdir/dist/linux-${architecture}-108" -gzip -f "/home/node/workdir/dist/linux-${architecture}-108" diff --git a/containers/custom-nodejs-builder-arm64/Dockerfile b/containers/custom-nodejs-builder-arm64/Dockerfile deleted file mode 100644 index a1d6faa..0000000 --- a/containers/custom-nodejs-builder-arm64/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -FROM centos:7 - -ARG GROUP_ID=1000 -ARG USER_ID=1000 - -RUN groupadd --force --gid $GROUP_ID node -RUN adduser --gid $GROUP_ID --uid $USER_ID node - -RUN ulimit -n 1024 \ - && yum install -y epel-release \ - && yum install -y centos-release-scl-rh \ - && yum upgrade -y \ - && yum install -y \ - git \ - curl \ - make \ - python2 \ - python3 \ - ccache \ - xz-utils \ - devtoolset-9 \ - glibc-devel - -COPY --chown=node:node entrypoint.sh /home/node/entrypoint.sh -COPY --chown=node:node re2_entrypoint.sh /home/node/re2_entrypoint.sh - -USER node - -VOLUME /home/node/workdir - -ENTRYPOINT [ "/home/node/entrypoint.sh" ] diff --git a/containers/custom-nodejs-builder-arm64/drivah.toml b/containers/custom-nodejs-builder-arm64/drivah.toml deleted file mode 100644 index 1b040fb..0000000 --- a/containers/custom-nodejs-builder-arm64/drivah.toml +++ /dev/null @@ -1,15 +0,0 @@ -[container.image] -names = ["docker.elastic.co/ci-agent-images/kibana/custom-nodejs-builder"] -tags = ["arm64-0.1"] - -[container.image.build_args] -GROUP_ID = 1000 -USER_ID = 1000 - -[docker] -build_flags = [ - "--progress=plain", - "--platform=linux/arm64", - "--build-arg=\"GROUP_ID=1000\"", - "--build-arg=\"USER_ID=1000\"" -] diff --git a/containers/custom-nodejs-builder-arm64/entrypoint.sh b/containers/custom-nodejs-builder-arm64/entrypoint.sh deleted file mode 100644 index fffa7f4..0000000 --- a/containers/custom-nodejs-builder-arm64/entrypoint.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -x - -release_url_base="$1" -full_version="$2" -config_flags=${3:-""} #"--without-dtrace --without-npm --without-etw" - -if [[ $(arch) == x86_64 ]]; then - architecture="x64"; -else - architecture="arm64" -fi - -cd "/home/node/workdir/src/node-${full_version}" - -# Compile from source -export CCACHE_DIR="/home/node/workdir/.ccache-${architecture}" -export CC="ccache gcc" -export CXX="ccache g++" - -. /opt/rh/devtoolset-9/enable - -make -j"$(getconf _NPROCESSORS_ONLN)" binary V= \ - DESTCPU="$architecture" \ - ARCH="$architecture" \ - VARIATION="glibc-217" \ - DISTTYPE="release" \ - RELEASE_URLBASE="$release_url_base" \ - CONFIG_FLAGS="$config_flags" - -mkdir -p /home/node/workdir/dist/ -mv node-*.tar.?z /home/node/workdir/dist/ diff --git a/containers/custom-nodejs-builder-arm64/re2_entrypoint.sh b/containers/custom-nodejs-builder-arm64/re2_entrypoint.sh deleted file mode 100644 index a8165d2..0000000 --- a/containers/custom-nodejs-builder-arm64/re2_entrypoint.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -x - -re2_full_version="$1" -node_full_version="$2" -node_download_base_url="$3" - -if [[ $(arch) == x86_64 ]]; then - architecture="x64"; -else - architecture="arm64" -fi - -cd /home/node/workdir -mkdir -p dist/ -mkdir -p src/ - -## Download and unpack Node.js binary if needed. -node_folder_name="node-${node_full_version}-linux-${architecture}" -npm_binary="/home/node/workdir/${node_folder_name}/bin/npm" -if [ ! -f "$npm_binary" ]; then - if [ ! -f "/home/node/workdir/$node_folder_name.tar.xz" ]; then - curl -fsSLO --compressed "${node_download_base_url}/${node_folder_name}.tar.xz" - fi - - tar -xf "/home/node/workdir/${node_folder_name}.tar.xz" -fi - -cd src - -## Download re2 source if needed. -re2_source_folder="/home/node/workdir/src/node-re2-${re2_full_version}" -if [ ! -d "$re2_source_folder" ]; then - git clone --recurse-submodules --depth 1 --branch 1.17.4 https://github.com/uhop/node-re2.git "${re2_source_folder}" -fi - -cd "$re2_source_folder" -export PATH="/home/node/workdir/${node_folder_name}/bin:$PATH" -export DEVELOPMENT_SKIP_GETTING_ASSET=true - -export CCACHE_DIR="/home/node/workdir/.ccache-re2-${architecture}" -export CC="ccache gcc" -export CXX="ccache g++" - -. /opt/rh/devtoolset-9/enable - -npm i --unsafe-perm=true -npm run build --if-present -npm test - -mkdir -p /home/node/workdir/dist/ -cp "${re2_source_folder}/build/Release/re2.node" "/home/node/workdir/dist/linux-${architecture}-108" -gzip -f "/home/node/workdir/dist/linux-${architecture}-108" From 9957258896b305f3290de1794a2898345e3fdb45 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Tue, 27 Jun 2023 12:25:03 +0200 Subject: [PATCH 66/75] Rationalize timeouts for beefier runner instances --- .buildkite/pipeline-kibana.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline-kibana.yml b/.buildkite/pipeline-kibana.yml index 0c0b900..8f244f0 100644 --- a/.buildkite/pipeline-kibana.yml +++ b/.buildkite/pipeline-kibana.yml @@ -11,7 +11,7 @@ steps: ARCH: amd64 agents: queue: c2-16 - timeout_in_minutes: 300 # 5 hrs + timeout_in_minutes: 60 # ideally runs in ~30m - label: Build node.js 18 with glibc 2.17 for arm64 command: @@ -21,9 +21,8 @@ steps: env: ARCH: arm64 agents: - # queue: macos-arm # disabled while the git cloning doesn't work queue: c2-60 # cross-compiling takes a while - timeout_in_minutes: 720 # 12 hrs + timeout_in_minutes: 180 # ideally runs in ~2hr - wait From 12ed8d82979a1a8cda31d3f2ba06a4e9ca13a692 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Tue, 27 Jun 2023 12:30:23 +0200 Subject: [PATCH 67/75] Rename pipeline-kibana to pipeline, as the default --- .buildkite/{pipeline-kibana.yml => pipeline.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .buildkite/{pipeline-kibana.yml => pipeline.yml} (100%) diff --git a/.buildkite/pipeline-kibana.yml b/.buildkite/pipeline.yml similarity index 100% rename from .buildkite/pipeline-kibana.yml rename to .buildkite/pipeline.yml From 00a8b2f39cec6d26d55f253700fadf87e1968bc5 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Thu, 29 Jun 2023 10:33:42 +0200 Subject: [PATCH 68/75] Update .buildkite/pipeline.yml Co-authored-by: Tiago Costa --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 8f244f0..181f6d9 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -13,7 +13,7 @@ steps: queue: c2-16 timeout_in_minutes: 60 # ideally runs in ~30m - - label: Build node.js 18 with glibc 2.17 for arm64 + - label: Build custom node.js artifacts with glibc 2.17 for arm64 command: - scripts/create_build_images.sh - scripts/build_nodejs.sh From 8997a62a421d1afc8103fdfc7f9c38297faa05aa Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Thu, 29 Jun 2023 10:33:51 +0200 Subject: [PATCH 69/75] Update .buildkite/pipeline.yml Co-authored-by: Tiago Costa --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 181f6d9..f7731b6 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -2,7 +2,7 @@ env: TARGET_VERSION: 18.16.1 steps: - - label: Build node.js 18 with glibc 2.17 for x64 + - label: Build custom node.js artifacts with glibc 2.17 for x64 command: - scripts/create_build_images.sh - scripts/build_nodejs.sh From 983da3eaa169025ab38e77614b112a3987deb15e Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Thu, 29 Jun 2023 10:56:20 +0200 Subject: [PATCH 70/75] try overriding node target version in command --- .buildkite/pipeline.yml | 2 ++ scripts/create_build_images.sh | 3 +++ 2 files changed, 5 insertions(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index f7731b6..c34e980 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -4,6 +4,7 @@ env: steps: - label: Build custom node.js artifacts with glibc 2.17 for x64 command: + - export TARGET_VERSION=$${OVERRIDE_TARGET_VERSION:TARGET_VERSION} - scripts/create_build_images.sh - scripts/build_nodejs.sh - scripts/upload_nodejs_artifacts.sh @@ -15,6 +16,7 @@ steps: - label: Build custom node.js artifacts with glibc 2.17 for arm64 command: + - export TARGET_VERSION=$${OVERRIDE_TARGET_VERSION:TARGET_VERSION} - scripts/create_build_images.sh - scripts/build_nodejs.sh - scripts/upload_nodejs_artifacts.sh diff --git a/scripts/create_build_images.sh b/scripts/create_build_images.sh index 1798eb4..0151791 100755 --- a/scripts/create_build_images.sh +++ b/scripts/create_build_images.sh @@ -7,11 +7,14 @@ source ./scripts/common.sh # ARCH provided in env assert_correct_arch $ARCH +set -x TARGET_PLATFORM="linux/$ARCH" IMAGE_NAME=$(get_build_image_name) SCRIPT_DIR=$(dirname $0) DOCKER_BUILD_CONTEXT_DIR="$SCRIPT_DIR/../build-image-config/" +set +x +exit 1 echo "--- Creating multiarch driver" if [[ $(docker buildx ls | grep multiarch | wc -l) -lt 1 ]]; then From c18809e085b8a0137d941807516e546bdacff962 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Thu, 29 Jun 2023 11:21:15 +0200 Subject: [PATCH 71/75] Fix default value expression --- .buildkite/pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index c34e980..f9e08ee 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -4,7 +4,7 @@ env: steps: - label: Build custom node.js artifacts with glibc 2.17 for x64 command: - - export TARGET_VERSION=$${OVERRIDE_TARGET_VERSION:TARGET_VERSION} + - export TARGET_VERSION=$${OVERRIDE_TARGET_VERSION:-TARGET_VERSION} - scripts/create_build_images.sh - scripts/build_nodejs.sh - scripts/upload_nodejs_artifacts.sh @@ -16,7 +16,7 @@ steps: - label: Build custom node.js artifacts with glibc 2.17 for arm64 command: - - export TARGET_VERSION=$${OVERRIDE_TARGET_VERSION:TARGET_VERSION} + - export TARGET_VERSION=$${OVERRIDE_TARGET_VERSION:-TARGET_VERSION} - scripts/create_build_images.sh - scripts/build_nodejs.sh - scripts/upload_nodejs_artifacts.sh From 8e0ae33efa9f9c6744c2d18cf4a5df9e4d9f43d7 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Thu, 29 Jun 2023 11:29:37 +0200 Subject: [PATCH 72/75] Remove debug / exit statements --- scripts/create_build_images.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/create_build_images.sh b/scripts/create_build_images.sh index 0151791..1798eb4 100755 --- a/scripts/create_build_images.sh +++ b/scripts/create_build_images.sh @@ -7,14 +7,11 @@ source ./scripts/common.sh # ARCH provided in env assert_correct_arch $ARCH -set -x TARGET_PLATFORM="linux/$ARCH" IMAGE_NAME=$(get_build_image_name) SCRIPT_DIR=$(dirname $0) DOCKER_BUILD_CONTEXT_DIR="$SCRIPT_DIR/../build-image-config/" -set +x -exit 1 echo "--- Creating multiarch driver" if [[ $(docker buildx ls | grep multiarch | wc -l) -lt 1 ]]; then From 5c1d41e7383ede2db9989561b082ff04db5ea4f8 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Thu, 29 Jun 2023 11:47:25 +0200 Subject: [PATCH 73/75] Centralize version override, add annotation for built version --- .buildkite/pipeline.yml | 10 +++++++--- scripts/common.sh | 2 ++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index f9e08ee..7efebbe 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -2,9 +2,13 @@ env: TARGET_VERSION: 18.16.1 steps: + - label: Annotate build with node version + command: buildkite-agent annotate "Building node.js v$${OVERRIDE_TARGET_VERSION:-TARGET_VERSION} with glibc 2.17" -- style 'info' + agents: + queue: kibana-default + - label: Build custom node.js artifacts with glibc 2.17 for x64 command: - - export TARGET_VERSION=$${OVERRIDE_TARGET_VERSION:-TARGET_VERSION} - scripts/create_build_images.sh - scripts/build_nodejs.sh - scripts/upload_nodejs_artifacts.sh @@ -16,7 +20,6 @@ steps: - label: Build custom node.js artifacts with glibc 2.17 for arm64 command: - - export TARGET_VERSION=$${OVERRIDE_TARGET_VERSION:-TARGET_VERSION} - scripts/create_build_images.sh - scripts/build_nodejs.sh - scripts/upload_nodejs_artifacts.sh @@ -29,6 +32,7 @@ steps: - wait - label: Fix SHASUMS256.txt with newly built files' hashes - command: scripts/replace_sha_hashes.sh + command: + - scripts/replace_sha_hashes.sh agents: queue: kibana-default diff --git a/scripts/common.sh b/scripts/common.sh index eb7bc2b..ff8ca58 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -1,6 +1,8 @@ BUCKET_NAME="kibana-custom-node-artifacts" +export TARGET_VERSION=${OVERRIDE_TARGET_VERSION:-TARGET_VERSION} + function assert_correct_arch() { ARCH=$1 From 8c345004fabb5b2378d6cdee4f792e1e2a7f558c Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Thu, 29 Jun 2023 11:50:32 +0200 Subject: [PATCH 74/75] Fix typo for --style --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 7efebbe..386304e 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -3,7 +3,7 @@ env: steps: - label: Annotate build with node version - command: buildkite-agent annotate "Building node.js v$${OVERRIDE_TARGET_VERSION:-TARGET_VERSION} with glibc 2.17" -- style 'info' + command: buildkite-agent annotate "node.js v$${OVERRIDE_TARGET_VERSION:-TARGET_VERSION} with glibc 2.17" --style 'info' agents: queue: kibana-default From 7ce4d9b04ed31aeb2aa8650628d226cea9a13b62 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Thu, 29 Jun 2023 15:12:55 +0200 Subject: [PATCH 75/75] Fix defaulting logic --- .buildkite/pipeline.yml | 2 +- scripts/common.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 386304e..108556a 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -3,7 +3,7 @@ env: steps: - label: Annotate build with node version - command: buildkite-agent annotate "node.js v$${OVERRIDE_TARGET_VERSION:-TARGET_VERSION} with glibc 2.17" --style 'info' + command: buildkite-agent annotate "node.js v$${OVERRIDE_TARGET_VERSION:-$$TARGET_VERSION} with glibc 2.17" --style 'info' agents: queue: kibana-default diff --git a/scripts/common.sh b/scripts/common.sh index ff8ca58..c324ec2 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -1,7 +1,7 @@ BUCKET_NAME="kibana-custom-node-artifacts" -export TARGET_VERSION=${OVERRIDE_TARGET_VERSION:-TARGET_VERSION} +export TARGET_VERSION=${OVERRIDE_TARGET_VERSION:-$TARGET_VERSION} function assert_correct_arch() { ARCH=$1