diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs index 98ae7b692bb3c..6f8a630874570 100644 --- a/src/bootstrap/util.rs +++ b/src/bootstrap/util.rs @@ -262,6 +262,8 @@ pub enum CiEnv { None, /// The Azure Pipelines environment, for Linux (including Docker), Windows, and macOS builds. AzurePipelines, + /// The GitHub Actions environment, for Linux (including Docker), Windows and macOS builds. + GitHubActions, } impl CiEnv { @@ -269,6 +271,8 @@ impl CiEnv { pub fn current() -> CiEnv { if env::var("TF_BUILD").ok().map_or(false, |e| &*e == "True") { CiEnv::AzurePipelines + } else if env::var("GITHUB_ACTIONS").ok().map_or(false, |e| &*e == "true") { + CiEnv::GitHubActions } else { CiEnv::None } diff --git a/src/ci/azure-pipelines/auto.yml b/src/ci/azure-pipelines/auto.yml index 946eb483c2946..bfe5174bf1e57 100644 --- a/src/ci/azure-pipelines/auto.yml +++ b/src/ci/azure-pipelines/auto.yml @@ -19,136 +19,46 @@ jobs: strategy: matrix: x86_64-gnu-llvm-6.0: - IMAGE: x86_64-gnu-llvm-6.0 RUST_BACKTRACE: 1 - - dist-x86_64-linux: - IMAGE: dist-x86_64-linux - DEPLOY: 1 - - # "alternate" deployments, these are "nightlies" but have LLVM assertions - # turned on, they're deployed to a different location primarily for - # additional testing. + dist-x86_64-linux: {} dist-x86_64-linux-alt: IMAGE: dist-x86_64-linux - DEPLOY_ALT: 1 - - # Linux builders, remaining docker images - arm-android: - IMAGE: arm-android - - armhf-gnu: - IMAGE: armhf-gnu - - dist-various-1: - IMAGE: dist-various-1 - DEPLOY: 1 - - dist-various-2: - IMAGE: dist-various-2 - DEPLOY: 1 - - dist-aarch64-linux: - IMAGE: dist-aarch64-linux - DEPLOY: 1 - - dist-android: - IMAGE: dist-android - DEPLOY: 1 - - dist-arm-linux: - IMAGE: dist-arm-linux - DEPLOY: 1 - - dist-armhf-linux: - IMAGE: dist-armhf-linux - DEPLOY: 1 - - dist-armv7-linux: - IMAGE: dist-armv7-linux - DEPLOY: 1 - - dist-i586-gnu-i586-i686-musl: - IMAGE: dist-i586-gnu-i586-i686-musl - DEPLOY: 1 - - dist-i686-freebsd: - IMAGE: dist-i686-freebsd - DEPLOY: 1 - - dist-i686-linux: - IMAGE: dist-i686-linux - DEPLOY: 1 - - dist-mips-linux: - IMAGE: dist-mips-linux - DEPLOY: 1 - - dist-mips64-linux: - IMAGE: dist-mips64-linux - DEPLOY: 1 - - dist-mips64el-linux: - IMAGE: dist-mips64el-linux - DEPLOY: 1 - - dist-mipsel-linux: - IMAGE: dist-mipsel-linux - DEPLOY: 1 - - dist-powerpc-linux: - IMAGE: dist-powerpc-linux - DEPLOY: 1 - - dist-powerpc64-linux: - IMAGE: dist-powerpc64-linux - DEPLOY: 1 - - dist-powerpc64le-linux: - IMAGE: dist-powerpc64le-linux - DEPLOY: 1 - - dist-s390x-linux: - IMAGE: dist-s390x-linux - DEPLOY: 1 - - dist-x86_64-freebsd: - IMAGE: dist-x86_64-freebsd - DEPLOY: 1 - - dist-x86_64-musl: - IMAGE: dist-x86_64-musl - DEPLOY: 1 - - dist-x86_64-netbsd: - IMAGE: dist-x86_64-netbsd - DEPLOY: 1 - - i686-gnu: - IMAGE: i686-gnu - i686-gnu-nopt: - IMAGE: i686-gnu-nopt - test-various: - IMAGE: test-various - wasm32: - IMAGE: wasm32 - x86_64-gnu: - IMAGE: x86_64-gnu - x86_64-gnu-full-bootstrap: - IMAGE: x86_64-gnu-full-bootstrap - x86_64-gnu-aux: - IMAGE: x86_64-gnu-aux + arm-android: {} + armhf-gnu: {} + dist-various-1: {} + dist-various-2: {} + dist-aarch64-linux: {} + dist-android: {} + dist-arm-linux: {} + dist-armhf-linux: {} + dist-armv7-linux: {} + dist-i586-gnu-i586-i686-musl: {} + dist-i686-freebsd: {} + dist-i686-linux: {} + dist-mips-linux: {} + dist-mips64-linux: {} + dist-mips64el-linux: {} + dist-mipsel-linux: {} + dist-powerpc-linux: {} + dist-powerpc64-linux: {} + dist-powerpc64le-linux: {} + dist-s390x-linux: {} + dist-x86_64-freebsd: {} + dist-x86_64-musl: {} + dist-x86_64-netbsd: {} + i686-gnu: {} + i686-gnu-nopt: {} + test-various: {} + wasm32: {} + x86_64-gnu: {} + x86_64-gnu-full-bootstrap: {} + x86_64-gnu-aux: {} x86_64-gnu-tools: - IMAGE: x86_64-gnu-tools DEPLOY_TOOLSTATES_JSON: toolstates-linux.json - x86_64-gnu-debug: - IMAGE: x86_64-gnu-debug - x86_64-gnu-nopt: - IMAGE: x86_64-gnu-nopt - x86_64-gnu-distcheck: - IMAGE: x86_64-gnu-distcheck - mingw-check: - IMAGE: mingw-check + x86_64-gnu-debug: {} + x86_64-gnu-nopt: {} + x86_64-gnu-distcheck: {} + mingw-check: {} - job: macOS timeoutInMinutes: 600 @@ -176,7 +86,6 @@ jobs: dist-x86_64-apple: SCRIPT: ./x.py dist RUST_CONFIGURE_ARGS: --target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc - DEPLOY: 1 RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 NO_LLVM_ASSERTIONS: 1 @@ -186,7 +95,6 @@ jobs: dist-x86_64-apple-alt: SCRIPT: ./x.py dist RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --set rust.jemalloc - DEPLOY_ALT: 1 RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 NO_LLVM_ASSERTIONS: 1 @@ -204,7 +112,6 @@ jobs: dist-i686-apple: SCRIPT: ./x.py dist RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --enable-full-tools --enable-profiler --set rust.jemalloc - DEPLOY: 1 RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 NO_LLVM_ASSERTIONS: 1 @@ -304,7 +211,6 @@ jobs: --enable-profiler SCRIPT: python x.py dist DIST_REQUIRE_ALL_TOOLS: 1 - DEPLOY: 1 dist-i686-msvc: RUST_CONFIGURE_ARGS: >- --build=i686-pc-windows-msvc @@ -313,22 +219,18 @@ jobs: --enable-profiler SCRIPT: python x.py dist DIST_REQUIRE_ALL_TOOLS: 1 - DEPLOY: 1 dist-i686-mingw: RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools --enable-profiler SCRIPT: python x.py dist CUSTOM_MINGW: 1 DIST_REQUIRE_ALL_TOOLS: 1 - DEPLOY: 1 dist-x86_64-mingw: SCRIPT: python x.py dist RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler CUSTOM_MINGW: 1 DIST_REQUIRE_ALL_TOOLS: 1 - DEPLOY: 1 # "alternate" deployment, see .travis.yml for more info dist-x86_64-msvc-alt: RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler SCRIPT: python x.py dist - DEPLOY_ALT: 1 diff --git a/src/ci/azure-pipelines/pr.yml b/src/ci/azure-pipelines/pr.yml index 566e654fdb3f0..aee4d8d5136aa 100644 --- a/src/ci/azure-pipelines/pr.yml +++ b/src/ci/azure-pipelines/pr.yml @@ -18,10 +18,7 @@ jobs: - template: steps/run.yml strategy: matrix: - x86_64-gnu-llvm-6.0: - IMAGE: x86_64-gnu-llvm-6.0 - mingw-check: - IMAGE: mingw-check + x86_64-gnu-llvm-6.0: {} + mingw-check: {} x86_64-gnu-tools: - IMAGE: x86_64-gnu-tools CI_ONLY_WHEN_SUBMODULES_CHANGED: 1 diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index 214c11fd69024..f536388b25b96 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -28,6 +28,9 @@ steps: - checkout: self fetchDepth: 2 +- bash: src/ci/scripts/setup-environment.sh + displayName: Setup environment + - bash: src/ci/scripts/should-skip-this.sh displayName: Decide whether to run this job @@ -38,38 +41,26 @@ steps: displayName: Show the current environment - bash: src/ci/scripts/install-sccache.sh - env: - AGENT_OS: $(Agent.OS) displayName: Install sccache condition: and(succeeded(), not(variables.SKIP_JOB)) - bash: src/ci/scripts/install-clang.sh - env: - AGENT_OS: $(Agent.OS) displayName: Install clang condition: and(succeeded(), not(variables.SKIP_JOB)) - bash: src/ci/scripts/switch-xcode.sh - env: - AGENT_OS: $(Agent.OS) displayName: Switch to Xcode 9.3 condition: and(succeeded(), not(variables.SKIP_JOB)) - bash: src/ci/scripts/install-wix.sh - env: - AGENT_OS: $(Agent.OS) displayName: Install wix condition: and(succeeded(), not(variables.SKIP_JOB)) - bash: src/ci/scripts/install-innosetup.sh - env: - AGENT_OS: $(Agent.OS) displayName: Install InnoSetup condition: and(succeeded(), not(variables.SKIP_JOB)) - bash: src/ci/scripts/windows-symlink-build-dir.sh - env: - AGENT_OS: $(Agent.OS) displayName: Ensure the build happens on C:\ instead of D:\ condition: and(succeeded(), not(variables.SKIP_JOB)) @@ -78,35 +69,22 @@ steps: condition: and(succeeded(), not(variables.SKIP_JOB)) - bash: src/ci/scripts/install-msys2.sh - env: - AGENT_OS: $(Agent.OS) - SYSTEM_WORKFOLDER: $(System.Workfolder) displayName: Install msys2 condition: and(succeeded(), not(variables.SKIP_JOB)) - bash: src/ci/scripts/install-msys2-packages.sh - env: - AGENT_OS: $(Agent.OS) - SYSTEM_WORKFOLDER: $(System.Workfolder) displayName: Install msys2 packages condition: and(succeeded(), not(variables.SKIP_JOB)) - bash: src/ci/scripts/install-mingw.sh - env: - AGENT_OS: $(Agent.OS) - SYSTEM_WORKFOLDER: $(System.Workfolder) displayName: Install MinGW condition: and(succeeded(), not(variables.SKIP_JOB)) - bash: src/ci/scripts/install-ninja.sh - env: - AGENT_OS: $(Agent.OS) displayName: Install ninja condition: and(succeeded(), not(variables.SKIP_JOB)) - bash: src/ci/scripts/enable-docker-ipv6.sh - env: - AGENT_OS: $(Agent.OS) displayName: Enable IPv6 on Docker condition: and(succeeded(), not(variables.SKIP_JOB)) @@ -120,22 +98,16 @@ steps: condition: and(succeeded(), not(variables.SKIP_JOB)) - bash: src/ci/scripts/checkout-submodules.sh - env: - AGENT_OS: $(Agent.OS) displayName: Checkout submodules condition: and(succeeded(), not(variables.SKIP_JOB)) - bash: src/ci/scripts/verify-line-endings.sh - env: - AGENT_OS: $(Agent.OS) displayName: Verify line endings condition: and(succeeded(), not(variables.SKIP_JOB)) # Ensure the `aws` CLI is installed so we can deploy later on, cache docker # images, etc. - bash: src/ci/scripts/install-awscli.sh - env: - AGENT_OS: $(Agent.OS) condition: and(succeeded(), not(variables.SKIP_JOB)) displayName: Install awscli diff --git a/src/ci/azure-pipelines/try.yml b/src/ci/azure-pipelines/try.yml index fe39ce3e24116..b6177b2cc9b25 100644 --- a/src/ci/azure-pipelines/try.yml +++ b/src/ci/azure-pipelines/try.yml @@ -14,13 +14,9 @@ jobs: - template: steps/run.yml strategy: matrix: - dist-x86_64-linux: - IMAGE: dist-x86_64-linux - DEPLOY: 1 - + dist-x86_64-linux: {} dist-x86_64-linux-alt: IMAGE: dist-x86_64-linux - DEPLOY_ALT: 1 # The macOS and Windows builds here are currently disabled due to them not being # overly necessary on `try` builds. We also don't actually have anything that diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index cdafcbadc9ec7..f29f9f3bf1c45 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -172,6 +172,8 @@ docker \ --env CI \ --env TF_BUILD \ --env BUILD_SOURCEBRANCHNAME \ + --env GITHUB_ACTIONS \ + --env GITHUB_REF \ --env TOOLSTATE_REPO_ACCESS_TOKEN \ --env TOOLSTATE_REPO \ --env TOOLSTATE_PUBLISH \ diff --git a/src/ci/run.sh b/src/ci/run.sh index bce35670c8d46..ae5b22493ab07 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -23,9 +23,7 @@ fi ci_dir=`cd $(dirname $0) && pwd` source "$ci_dir/shared.sh" -branch_name=$(getCIBranch) - -if [ ! isCI ] || [ "$branch_name" = "auto" ] || [ "$branch_name" = "try" ]; then +if [ ! isCI ] || isCiBranch auto || isCiBranch beta; then RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set build.print-step-timings --enable-verbose-tests" fi diff --git a/src/ci/scripts/install-mingw.sh b/src/ci/scripts/install-mingw.sh index 8b579587b9e1f..98373df7fce50 100755 --- a/src/ci/scripts/install-mingw.sh +++ b/src/ci/scripts/install-mingw.sh @@ -52,7 +52,7 @@ if isWindows; then if [[ "${CUSTOM_MINGW-0}" -ne 1 ]]; then pacman -S --noconfirm --needed mingw-w64-$arch-toolchain mingw-w64-$arch-cmake \ mingw-w64-$arch-gcc mingw-w64-$arch-python2 - ciCommandAddPath "${SYSTEM_WORKFOLDER}/msys2/mingw${bits}/bin" + ciCommandAddPath "$(ciCheckoutPath)/msys2/mingw${bits}/bin" else mingw_dir="mingw${bits}" diff --git a/src/ci/scripts/install-msys2.sh b/src/ci/scripts/install-msys2.sh index ce37c3b146977..c9fafc7fe6b41 100755 --- a/src/ci/scripts/install-msys2.sh +++ b/src/ci/scripts/install-msys2.sh @@ -12,8 +12,8 @@ IFS=$'\n\t' source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" if isWindows; then - choco install msys2 --params="/InstallDir:${SYSTEM_WORKFOLDER}/msys2 /NoPath" -y --no-progress - mkdir -p "${SYSTEM_WORKFOLDER}/msys2/home/${USERNAME}" + choco install msys2 --params="/InstallDir:$(ciCheckoutPath)/msys2 /NoPath" -y --no-progress + mkdir -p "$(ciCheckoutPath)/msys2/home/${USERNAME}" - ciCommandAddPath "${SYSTEM_WORKFOLDER}/msys2/usr/bin" + ciCommandAddPath "$(ciCheckoutPath)/msys2/usr/bin" fi diff --git a/src/ci/scripts/setup-environment.sh b/src/ci/scripts/setup-environment.sh new file mode 100755 index 0000000000000..e126a06edab73 --- /dev/null +++ b/src/ci/scripts/setup-environment.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# This script guesses some environment variables based on the builder name and +# the current platform, to reduce the amount of variables defined in the CI +# configuration. + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +# Builders starting with `dist-` are dist builders, but if they also end with +# `-alt` they are alternate dist builders. +if [[ "${CI_JOB_NAME}" = dist-* ]]; then + if [[ "${CI_JOB_NAME}" = *-alt ]]; then + echo "alternate dist builder detected, setting DEPLOY_ALT=1" + ciCommandSetEnv DEPLOY_ALT 1 + else + echo "normal dist builder detected, setting DEPLOY=1" + ciCommandSetEnv DEPLOY 1 + fi +fi + +# All the Linux builds happen inside Docker. +if isLinux; then + if [[ -z "${IMAGE+x}" ]]; then + echo "linux builder detected, using docker to run the build" + ciCommandSetEnv IMAGE "${CI_JOB_NAME}" + else + echo "a custom docker image is already set" + fi +fi diff --git a/src/ci/shared.sh b/src/ci/shared.sh index 862ded0d5dbf0..394a57158873c 100644 --- a/src/ci/shared.sh +++ b/src/ci/shared.sh @@ -27,27 +27,66 @@ function retry { } function isCI { - [ "$CI" = "true" ] || [ "$TF_BUILD" = "True" ] + [[ "${CI-false}" = "true" ]] || isAzurePipelines || isGitHubActions +} + +function isAzurePipelines { + [[ "${TF_BUILD-False}" = "True" ]] +} + +function isGitHubActions { + [[ "${GITHUB_ACTIONS-false}" = "true" ]] } function isMacOS { - [ "$AGENT_OS" = "Darwin" ] + [[ "${OSTYPE}" = "darwin"* ]] } function isWindows { - [ "$AGENT_OS" = "Windows_NT" ] + [[ "${OSTYPE}" = "cygwin" ]] || [[ "${OSTYPE}" = "msys" ]] } function isLinux { - [ "$AGENT_OS" = "Linux" ] + [[ "${OSTYPE}" = "linux-gnu" ]] } -function getCIBranch { - echo "$BUILD_SOURCEBRANCHNAME" +function isCiBranch { + if [[ $# -ne 1 ]]; then + echo "usage: $0 " + exit 1 + fi + name="$1" + + if isAzurePipelines; then + [[ "${BUILD_SOURCEBRANCHNAME}" = "${name}" ]] + elif isGitHubActions; then + [[ "${GITHUB_REF}" = "refs/heads/${name}" ]] + else + echo "isCiBranch only works inside CI!" + exit 1 + fi } function ciCommit { - echo "${BUILD_SOURCEVERSION}" + if isAzurePipelines; then + echo "${BUILD_SOURCEVERSION}" + elif isGitHubActions; then + echo "${GITHUB_SHA}" + else + echo "ciCommit only works inside CI!" + exit 1 + fi +} + +function ciCheckoutPath { + if isAzurePipelines; then + echo "${BUILD_WORKFOLDER}" + elif isGitHubActions; then + echo "${GITHUB_WORKSPACE}" + else + echo "ciCheckoutPath only works inside CI!" + exit 1 + fi } function ciCommandAddPath { @@ -57,7 +96,14 @@ function ciCommandAddPath { fi path="$1" - echo "##vso[task.prependpath]${path}" + if isAzurePipelines; then + echo "##vso[task.prependpath]${path}" + elif isGitHubActions; then + echo "::add-path::${value}" + else + echo "ciCommandAddPath only works inside CI!" + exit 1 + fi } function ciCommandSetEnv { @@ -68,5 +114,12 @@ function ciCommandSetEnv { name="$1" value="$2" - echo "##vso[task.setvariable variable=${name}]${value}" + if isAzurePipelines; then + echo "##vso[task.setvariable variable=${name}]${value}" + elif isGitHubActions; then + echo "::set-env name=${name}::${value}" + else + echo "ciCommandSetEnv only works inside CI!" + exit 1 + fi }