From c08fa565466cd27df6cd0c4eb0b0dae451a40886 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Tue, 15 Oct 2024 19:22:38 -0400 Subject: [PATCH] chore(NODE-6401): migrate node download script to drivers-tools (#718) --- .evergreen/config.yml | 5 +- .evergreen/init-node-and-npm-env.sh | 21 ---- .evergreen/install-dependencies.sh | 109 ++---------------- ...{setup-environment.sh => prepare-shell.sh} | 20 +++- .evergreen/run-big-endian-test.sh | 2 +- .evergreen/run-bundling-test.sh | 2 +- .evergreen/run-checks.sh | 2 +- .evergreen/run-custom-benchmarks.sh | 2 +- .evergreen/run-eslint-plugin-test.sh | 2 +- .evergreen/run-granular-benchmarks.sh | 2 +- .evergreen/run-spec-benchmarks.sh | 2 +- .evergreen/run-tests.sh | 2 +- .evergreen/run-typescript.sh | 2 +- .gitignore | 3 + 14 files changed, 44 insertions(+), 132 deletions(-) delete mode 100644 .evergreen/init-node-and-npm-env.sh rename .evergreen/{setup-environment.sh => prepare-shell.sh} (51%) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index a043c638f..dd6d751b3 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -25,7 +25,7 @@ functions: binary: bash add_expansions_to_env: true args: - - .evergreen/setup-environment.sh + - '.evergreen/prepare-shell.sh' - command: expansions.update params: file: src/expansion.yml @@ -76,6 +76,7 @@ functions: params: working_dir: src timeout_secs: 60 + add_expansions_to_env: true env: PROJECT_DIRECTORY: ${PROJECT_DIRECTORY} TS_VERSION: "${TS_VERSION}" @@ -89,6 +90,7 @@ functions: params: working_dir: src timeout_secs: 60 + add_expansions_to_env: true env: PROJECT_DIRECTORY: ${PROJECT_DIRECTORY} binary: bash @@ -100,6 +102,7 @@ functions: params: working_dir: src binary: bash + add_expansions_to_env: true env: PROJECT_DIRECTORY: ${PROJECT_DIRECTORY} args: diff --git a/.evergreen/init-node-and-npm-env.sh b/.evergreen/init-node-and-npm-env.sh deleted file mode 100644 index b3cecf54e..000000000 --- a/.evergreen/init-node-and-npm-env.sh +++ /dev/null @@ -1,21 +0,0 @@ -#! /usr/bin/env bash -## -## This script add the location of `npm` and `node` to the path. -## This is necessary because evergreen uses separate bash scripts for -## different functions in a given CI run but doesn't persist the environment -## across them. So we manually invoke this script everywhere we need -## access to `npm`, `node`, or need to install something globally from -## npm. - -NODE_ARTIFACTS_PATH="${PROJECT_DIRECTORY}/node-artifacts" -if [[ "$OS" == "Windows_NT" ]]; then - NODE_ARTIFACTS_PATH=$(cygpath --unix "$NODE_ARTIFACTS_PATH") -fi - -export NODE_ARTIFACTS_PATH -# npm uses this environment variable to determine where to install global packages -export npm_global_prefix=$NODE_ARTIFACTS_PATH/npm_global -export PATH="$npm_global_prefix/bin:$NODE_ARTIFACTS_PATH/nodejs/bin:$PATH" -hash -r - -export NODE_OPTIONS="--trace-deprecation --trace-warnings" diff --git a/.evergreen/install-dependencies.sh b/.evergreen/install-dependencies.sh index 66ad117fd..d36fb1fab 100755 --- a/.evergreen/install-dependencies.sh +++ b/.evergreen/install-dependencies.sh @@ -1,108 +1,17 @@ #!/usr/bin/env bash set -o errexit # Exit the script with error if any of the commands fail -NODE_LTS_VERSION=${NODE_LTS_VERSION:-16} +# allowed values: +## a nodejs major version (i.e., 16) +## 'latest' +## a full nodejs version, in the format v..patch +export NODE_LTS_VERSION=${NODE_LTS_VERSION:-16} # npm version can be defined in the environment for cases where we need to install # a version lower than latest to support EOL Node versions. -NPM_VERSION=${NPM_VERSION:-latest} +export NPM_VERSION=${NPM_VERSION:-latest} -source "${PROJECT_DIRECTORY}/.evergreen/init-node-and-npm-env.sh" - -if [[ -z "${npm_global_prefix}" ]]; then echo "npm_global_prefix is unset" && exit 1; fi -if [[ -z "${NODE_ARTIFACTS_PATH}" ]]; then echo "NODE_ARTIFACTS_PATH is unset" && exit 1; fi - -CURL_FLAGS=( - --fail # Exit code 1 if request fails - --compressed # Request a compressed response should keep fetching fast - --location # Follow a redirect - --retry 8 # Retry HTTP 408, 429, 500, 502, 503 or 504, 8 times - --silent # Do not print a progress bar - --show-error # Despite the silent flag still print out errors - --max-time 900 # 900 seconds is 15 minutes, evergreen times out at 20 - --continue-at - # If a download is interrupted it can figure out where to resume -) - -mkdir -p "$NODE_ARTIFACTS_PATH/npm_global" - -# Comparisons are all case insensitive -shopt -s nocasematch - -# index.tab is a sorted tab separated values file with the following headers -# 0 1 2 3 4 5 6 7 8 9 10 -# version date files npm v8 uv zlib openssl modules lts security -curl "${CURL_FLAGS[@]}" "https://nodejs.org/dist/index.tab" --output node_index.tab - -while IFS=$'\t' read -r -a row; do - node_index_version="${row[0]}" - node_index_major_version=$(echo $node_index_version | sed -E 's/^v([0-9]+).*$/\1/') - node_index_date="${row[1]}" - node_index_lts="${row[9]}" - [[ "$node_index_version" = "version" ]] && continue # skip tsv header - [[ "$NODE_LTS_VERSION" = "latest" ]] && break # first line is latest - [[ "$NODE_LTS_VERSION" = "$node_index_version" ]] && break # match full version if specified - [[ "$NODE_LTS_VERSION" = "$node_index_major_version" ]] && break # case insensitive compare -done < node_index.tab - -if [[ "$OS" = "Windows_NT" ]]; then - operating_system="win" -elif [[ $(uname) = "darwin" ]]; then - operating_system="darwin" -elif [[ $(uname) = "linux" ]]; then - operating_system="linux" -else - echo "Unable to determine operating system: $operating_system" - exit 1 -fi - -architecture=$(uname -m) -if [[ $architecture = "x86_64" ]]; then - architecture="x64" -elif [[ $architecture = "arm64" ]]; then - architecture="arm64" -elif [[ $architecture = "aarch64" ]]; then - architecture="arm64" -elif [[ $architecture == s390* ]]; then - architecture="s390x" -elif [[ $architecture == ppc* ]]; then - architecture="ppc64le" -else - echo "Unable to determine operating system: $architecture" - exit 1 -fi - -file_extension="tar.gz" -if [[ "$OS" = "Windows_NT" ]]; then file_extension="zip"; fi - -node_directory="node-${node_index_version}-${operating_system}-${architecture}" -node_archive="${node_directory}.${file_extension}" -node_archive_path="$NODE_ARTIFACTS_PATH/${node_archive}" -node_download_url="https://nodejs.org/dist/${node_index_version}/${node_archive}" - -echo "Node.js ${node_index_version} for ${operating_system}-${architecture} released on ${node_index_date}" - -set -o xtrace - -curl "${CURL_FLAGS[@]}" "${node_download_url}" --output "$node_archive_path" - -if [[ "$file_extension" = "zip" ]]; then - unzip -q "$node_archive_path" -d "${NODE_ARTIFACTS_PATH}" - mkdir -p "${NODE_ARTIFACTS_PATH}/nodejs" - # Windows "bins" are at the top level - mv "${NODE_ARTIFACTS_PATH}/${node_directory}" "${NODE_ARTIFACTS_PATH}/nodejs/bin" - # Need to add executable flag ourselves - chmod +x "${NODE_ARTIFACTS_PATH}/nodejs/bin/node.exe" - chmod +x "${NODE_ARTIFACTS_PATH}/nodejs/bin/npm" -else - tar -xf "$node_archive_path" -C "${NODE_ARTIFACTS_PATH}" - mv "${NODE_ARTIFACTS_PATH}/${node_directory}" "${NODE_ARTIFACTS_PATH}/nodejs" -fi - -if [[ $operating_system != "win" ]]; then - npm install --global npm@$NPM_VERSION - hash -r -fi - -echo "npm location: $(which npm)" -echo "npm version: $(npm -v)" +source $DRIVERS_TOOLS/.evergreen/install-node.sh npm install "${NPM_OPTIONS}" + +npm ls diff --git a/.evergreen/setup-environment.sh b/.evergreen/prepare-shell.sh similarity index 51% rename from .evergreen/setup-environment.sh rename to .evergreen/prepare-shell.sh index 241839df6..094becbf0 100644 --- a/.evergreen/setup-environment.sh +++ b/.evergreen/prepare-shell.sh @@ -1,5 +1,22 @@ #! /usr/bin/env bash +# Only set errexit and xtrace if shell is NOT interactive +[[ $- == *i* ]] || set -o xtrace +[[ $- == *i* ]] || set -o errexit + +PROJECT_DIRECTORY="$(pwd)" +DRIVERS_TOOLS=$(cd .. && echo "$(pwd)/drivers-tools") +export PROJECT_DIRECTORY +export DRIVERS_TOOLS + + +if [ ! -d "$DRIVERS_TOOLS" ]; then + # Only clone driver tools if it does not exist + git clone --depth=1 "https://github.com/mongodb-labs/drivers-evergreen-tools.git" "${DRIVERS_TOOLS}" +fi + +echo "installed DRIVERS_TOOLS from commit $(git -C "${DRIVERS_TOOLS}" rev-parse HEAD)" + if [ -z "$NODE_LTS_VERSION" ]; then echo "NODE_LTS_VERSION environment variable must be specified" exit 1 @@ -11,17 +28,18 @@ if [ "${is_patch}" = "true" ]; then else CURRENT_VERSION=latest fi -export PROJECT_DIRECTORY="$(pwd)" cat < expansion.yml CURRENT_VERSION: "$CURRENT_VERSION" PROJECT_DIRECTORY: "$PROJECT_DIRECTORY" NODE_LTS_VERSION: "$NODE_LTS_VERSION" +DRIVERS_TOOLS: "$DRIVERS_TOOLS" PREPARE_SHELL: | set -o errexit set -o xtrace export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" export NODE_LTS_VERSION="$NODE_LTS_VERSION" + export DRIVERS_TOOLS="$DRIVERS_TOOLS" EOT # See what we've done cat expansion.yml diff --git a/.evergreen/run-big-endian-test.sh b/.evergreen/run-big-endian-test.sh index 3b37d132e..a1cc82404 100644 --- a/.evergreen/run-big-endian-test.sh +++ b/.evergreen/run-big-endian-test.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -source "${PROJECT_DIRECTORY}/.evergreen/init-node-and-npm-env.sh" +source $DRIVERS_TOOLS/.evergreen/init-node-and-npm-env.sh npx mocha test/s390x/big_endian.test.ts diff --git a/.evergreen/run-bundling-test.sh b/.evergreen/run-bundling-test.sh index f0b61e631..85bb52491 100644 --- a/.evergreen/run-bundling-test.sh +++ b/.evergreen/run-bundling-test.sh @@ -1,6 +1,6 @@ #! /usr/bin/env bash -source "${PROJECT_DIRECTORY}/.evergreen/init-node-and-npm-env.sh" +source $DRIVERS_TOOLS/.evergreen/init-node-and-npm-env.sh set -o xtrace set -o errexit diff --git a/.evergreen/run-checks.sh b/.evergreen/run-checks.sh index 65443ed1f..44930932a 100644 --- a/.evergreen/run-checks.sh +++ b/.evergreen/run-checks.sh @@ -1,6 +1,6 @@ #!/bin/bash set -o errexit # Exit the script with error if any of the commands fail -source "${PROJECT_DIRECTORY}/.evergreen/init-node-and-npm-env.sh" +source $DRIVERS_TOOLS/.evergreen/init-node-and-npm-env.sh npm run check:lint diff --git a/.evergreen/run-custom-benchmarks.sh b/.evergreen/run-custom-benchmarks.sh index 2984cb356..92920ee32 100644 --- a/.evergreen/run-custom-benchmarks.sh +++ b/.evergreen/run-custom-benchmarks.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -source "${PROJECT_DIRECTORY}/.evergreen/init-node-and-npm-env.sh" +source $DRIVERS_TOOLS/.evergreen/init-node-and-npm-env.sh set -o xtrace npm run check:custom-bench diff --git a/.evergreen/run-eslint-plugin-test.sh b/.evergreen/run-eslint-plugin-test.sh index c0c6693ff..10c085157 100644 --- a/.evergreen/run-eslint-plugin-test.sh +++ b/.evergreen/run-eslint-plugin-test.sh @@ -2,7 +2,7 @@ set -o errexit -source "${PROJECT_DIRECTORY}/.evergreen/init-node-and-npm-env.sh" +source $DRIVERS_TOOLS/.evergreen/init-node-and-npm-env.sh cd etc/eslint/no-bigint-usage npm install diff --git a/.evergreen/run-granular-benchmarks.sh b/.evergreen/run-granular-benchmarks.sh index f13097f24..ae8399bc2 100644 --- a/.evergreen/run-granular-benchmarks.sh +++ b/.evergreen/run-granular-benchmarks.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -source "${PROJECT_DIRECTORY}/.evergreen/init-node-and-npm-env.sh" +source $DRIVERS_TOOLS/.evergreen/init-node-and-npm-env.sh set -o xtrace WARMUP=$WARMUP ITERATIONS=$ITERATIONS diff --git a/.evergreen/run-spec-benchmarks.sh b/.evergreen/run-spec-benchmarks.sh index 5ce908336..dd85250b4 100644 --- a/.evergreen/run-spec-benchmarks.sh +++ b/.evergreen/run-spec-benchmarks.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -source "${PROJECT_DIRECTORY}/.evergreen/init-node-and-npm-env.sh" +source $DRIVERS_TOOLS/.evergreen/init-node-and-npm-env.sh npm run check:spec-bench diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index e834056c2..4cacd41e9 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -source "${PROJECT_DIRECTORY}/.evergreen/init-node-and-npm-env.sh" +source $DRIVERS_TOOLS/.evergreen/init-node-and-npm-env.sh case "${TEST_TARGET}" in "node") diff --git a/.evergreen/run-typescript.sh b/.evergreen/run-typescript.sh index 30f3ca463..d814fd17a 100644 --- a/.evergreen/run-typescript.sh +++ b/.evergreen/run-typescript.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -o errexit # Exit the script with error if any of the commands fail -source "${PROJECT_DIRECTORY}/.evergreen/init-node-and-npm-env.sh" +source $DRIVERS_TOOLS/.evergreen/init-node-and-npm-env.sh set -o xtrace diff --git a/.gitignore b/.gitignore index b2ee050b8..edd0d9b74 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,6 @@ docs/public benchmarks.json customBenchmarkResults.json + +expansion.yml +.drivers-tools/