Backport ci changes

azure-pipelines.yml

@@ -49,6 +49,7 @@ jobs:
   - publish: $(Pipeline.Workspace)/opentitan-repo.tar.gz
     artifact: opentitan-repo
     displayName: Upload repository
+
 - job: lint
   displayName: Quality (quick lint)
   # Run code quality checks (quick lint)
@@ -182,23 +183,13 @@ jobs:
   dependsOn: lint
   condition: and(succeeded(), eq(dependencies.lint.outputs['DetermineBuildType.onlyDocChanges'], '0'), eq(dependencies.lint.outputs['DetermineBuildType.onlyCdcChanges'], '0'))
   pool: ci-public
-  variables:
-    - name: bazelCacheGcpKeyPath
-      value: ''
   steps:
   - template: ci/checkout-template.yml
   - template: ci/install-package-dependencies.yml
-  - task: DownloadSecureFile@1
-    condition: eq(variables['Build.SourceBranchName'], 'master')
-    name: bazelCacheGcpKey
-    inputs:
-      secureFile: "bazel_cache_gcp_key.json"
-    # Set the remote cache GCP key path
-  - bash: echo "##vso[task.setvariable variable=bazelCacheGcpKeyPath]$(bazelCacheGcpKey.secureFilePath)"
-    condition: eq(variables['Build.SourceBranchName'], 'master')
-    displayName: GCP key path
+  - template: ci/load-bazel-cache-write-creds.yml
   - bash: |
       set -x -e
+
       # Check the entire build graph for conflicts in loading or analysis
       # phases. For context, see issue #18726.
       # First, test with an empty bitstream cache entry.
@@ -217,14 +208,13 @@ jobs:
       #   shallow exclusion; tests deeper under //hw will still be found.
       # * It excludes targets that depend on bitstream_splice rules, since the
       #   environment does not have access to Vivado.
-      export GCP_BAZEL_CACHE_KEY=$(bazelCacheGcpKeyPath)
       TARGET_PATTERN_FILE=target_pattern.txt
       echo //... > "${TARGET_PATTERN_FILE}"
       echo -//quality/... >> "${TARGET_PATTERN_FILE}"
       echo -//sw/otbn/crypto/... >> "${TARGET_PATTERN_FILE}"
       echo -//third_party/riscv-compliance/... >> "${TARGET_PATTERN_FILE}"
       echo -//hw:all >> "${TARGET_PATTERN_FILE}"
-      ./bazelisk.sh cquery \
+      ci/bazelisk.sh cquery \
         --noinclude_aspects \
         --output=starlark \
         --starlark:expr='"-{}".format(target.label)' \
@@ -250,8 +240,8 @@ jobs:
       ROM_REAL_TARGETS="//sw/device/silicon_creator/rom:package_real"
       ROM_FAKE_TARGETS="//sw/device/silicon_creator/rom:package_fake"
       QUERY_CMD_ARGS=(outquery-all --noinclude_aspects --noimplicit_deps)
-      ROM_REAL_FILES=($(./bazelisk.sh "${QUERY_CMD_ARGS[@]}" "${ROM_REAL_TARGETS}" | sort | uniq))
-      ROM_FAKE_FILES=($(./bazelisk.sh "${QUERY_CMD_ARGS[@]}" "${ROM_FAKE_TARGETS}" | sort | uniq))
+      ROM_REAL_FILES=($(ci/bazelisk.sh "${QUERY_CMD_ARGS[@]}" "${ROM_REAL_TARGETS}" | sort | uniq))
+      ROM_FAKE_FILES=($(ci/bazelisk.sh "${QUERY_CMD_ARGS[@]}" "${ROM_FAKE_TARGETS}" | sort | uniq))
       cp -Lvt "${ROM_TARGET}" "${ROM_FAKE_FILES[@]}" "${ROM_REAL_FILES[@]}"
   - template: ci/upload-artifacts-template.yml
     parameters:
@@ -263,21 +253,10 @@ jobs:
   timeoutInMinutes: 120
   dependsOn: sw_build
   pool: ci-public
-  variables:
-    - name: bazelCacheGcpKeyPath
-      value: ''
   steps:
   - template: ci/checkout-template.yml
   - template: ci/install-package-dependencies.yml
-  - task: DownloadSecureFile@1
-    condition: eq(variables['Build.SourceBranchName'], 'master')
-    name: bazelCacheGcpKey
-    inputs:
-      secureFile: "bazel_cache_gcp_key.json"
-    # Set the remote cache GCP key path
-  - bash: echo "##vso[task.setvariable variable=bazelCacheGcpKeyPath]$(bazelCacheGcpKey.secureFilePath)"
-    condition: eq(variables['Build.SourceBranchName'], 'master')
-    displayName: GCP key path
+  - template: ci/load-bazel-cache-write-creds.yml
   - download: current
     artifact: target_pattern_file
   - bash: |
@@ -320,24 +299,12 @@ jobs:
   pool: ci-public
   timeoutInMinutes: 240
   dependsOn: lint
-  variables:
-    - name: bazelCacheGcpKeyPath
-      value: ''
   steps:
   - template: ci/checkout-template.yml
   - template: ci/install-package-dependencies.yml
-  - task: DownloadSecureFile@1
-    condition: eq(variables['Build.SourceBranchName'], 'master')
-    name: bazelCacheGcpKey
-    inputs:
-      secureFile: "bazel_cache_gcp_key.json"
-  - bash: echo "##vso[task.setvariable variable=bazelCacheGcpKeyPath]$(bazelCacheGcpKey.secureFilePath)"
-    condition: eq(variables['Build.SourceBranchName'], 'master')
-    displayName: GCP key path
-    # Set the remote cache GCP key path
+  - template: ci/load-bazel-cache-write-creds.yml
   - bash: |
       set -x -e
-      export GCP_BAZEL_CACHE_KEY=$(bazelCacheGcpKeyPath)
       ci/scripts/run-verilator-tests.sh
     displayName: Build & execute tests
   - template: ci/publish-bazel-test-results.yml
@@ -350,7 +317,7 @@ jobs:
   - bash: |
       . util/build_consts.sh
       mkdir -p "$BIN_DIR/hw/top_earlgrey/"
-      cp $(./bazelisk.sh outquery-build.verilator //sw/device/tests:uart_smoketest_sim_verilator) \
+      cp $(ci/bazelisk.sh outquery-build.verilator //sw/device/tests:uart_smoketest_sim_verilator) \
         "$BIN_DIR/hw/top_earlgrey/Vchip_earlgrey_verilator"
     displayName: Copy verilated binary to $BIN_DIR
   - template: ci/upload-artifacts-template.yml
@@ -377,6 +344,7 @@ jobs:
     parameters:
       downloadPartialBuildBinFrom:
         - chip_englishbreakfast_verilator
+  - template: ci/load-bazel-cache-write-creds.yml
   - bash: |
       . util/build_consts.sh
       ci/scripts/run-english-breakfast-verilator-tests.sh
@@ -438,15 +406,7 @@ jobs:
   steps:
   - template: ci/checkout-template.yml
   - template: ci/install-package-dependencies.yml
-  - task: DownloadSecureFile@1
-    condition: eq(variables['Build.SourceBranchName'], 'master')
-    name: bazelCacheGcpKey
-    inputs:
-      secureFile: "bazel_cache_gcp_key.json"
-  - bash: echo "##vso[task.setvariable variable=bazelCacheGcpKeyPath]$(bazelCacheGcpKey.secureFilePath)"
-    condition: eq(variables['Build.SourceBranchName'], 'master')
-    displayName: GCP key path
-    # Set the remote cache GCP key path
+  - template: ci/load-bazel-cache-write-creds.yml
   - bash: |
       ci/bazelisk.sh test --test_tag_filters=-nightly //sw/otbn/crypto/...
     displayName: Execute tests
@@ -546,7 +506,7 @@ jobs:
         bucketURI: "gs://opentitan-bitstreams/earlgrey_es_sival"
 
 - job: execute_sival_fpga_tests_cw310
-  displayName: CW310 SiVal Tests
+  displayName: CW310 SiVal non-ROM_EXT Tests
   pool:
     name: $(fpga_pool)
     demands: BOARD -equals cw310
@@ -561,23 +521,38 @@ jobs:
   - template: ci/download-artifacts-template.yml
     parameters:
       downloadPartialBuildBinFrom:
-        - chip_earlgrey_cw310
         - chip_earlgrey_cw310_hyperdebug
         - sw_build
+  - template: ci/load-bazel-cache-write-creds.yml
+  # We run the update command twice to workaround an issue with udev on the container,
+  # where rusb cannot dynamically update its device list in CI (udev is not completely
+  # functional). If the device is in normal mode, the first thing that opentitantool
+  # does is to switch it to DFU mode and wait until it reconnects. This reconnection is
+  # never detected. But if we run the tool another time, the device list is queried again
+  # and opentitantool can finish the update. The device will now reboot in normal mode
+  # and work for the hyperdebug job.
+  - bash: |
+      ci/bazelisk.sh run \
+        //sw/host/opentitantool:opentitantool -- \
+        --interface=hyperdebug_dfu transport update-firmware \
+      || ci/bazelisk.sh run \
+        //sw/host/opentitantool:opentitantool -- \
+        --interface=hyperdebug_dfu transport update-firmware || true
+    displayName: "Update the hyperdebug firmware"
   - bash: |
       set -e
       . util/build_consts.sh
       module load "xilinx/vivado/$(VIVADO_VERSION)"
-      ci/scripts/run-fpga-tests.sh hyper310 cw310_sival,-broken || { res=$?; echo "To reproduce failures locally, follow the instructions at https://opentitan.org/book/doc/getting_started/setup_fpga.html#reproducing-fpga-ci-failures-locally"; exit "${res}"; }
+      ci/scripts/run-fpga-tests.sh hyper310 cw310_sival_but_not_rom_ext_tests || { res=$?; echo "To reproduce failures locally, follow the instructions at https://opentitan.org/book/doc/getting_started/setup_fpga.html#reproducing-fpga-ci-failures-locally"; exit "${res}"; }
     displayName: Execute tests
   - template: ci/publish-bazel-test-results.yml
 
-- job: execute_hyperdebug_tests_cw310
-  displayName: Hyperdebug CW310 Tests (Experimental)
+- job: execute_sival_rom_ext_fpga_tests_cw310
+  displayName: CW310 SiVal ROM_EXT Tests
   pool:
     name: $(fpga_pool)
     demands: BOARD -equals cw310
-  timeoutInMinutes: 60
+  timeoutInMinutes: 45
   dependsOn:
     - chip_earlgrey_cw310_hyperdebug
     - sw_build
@@ -590,8 +565,9 @@ jobs:
       downloadPartialBuildBinFrom:
         - chip_earlgrey_cw310_hyperdebug
         - sw_build
-  # We run the update command twice to workaround an issue with udev on the container.
-  # Where rusb cannot dynamically update its device list in CI (udev is not completely
+  - template: ci/load-bazel-cache-write-creds.yml
+  # We run the update command twice to workaround an issue with udev on the container,
+  # where rusb cannot dynamically update its device list in CI (udev is not completely
   # functional). If the device is in normal mode, the first thing that opentitantool
   # does is to switch it to DFU mode and wait until it reconnects. This reconnection is
   # never detected. But if we run the tool another time, the device list is queried again
@@ -609,7 +585,7 @@ jobs:
       set -e
       . util/build_consts.sh
       module load "xilinx/vivado/$(VIVADO_VERSION)"
-      ci/scripts/run-fpga-tests.sh hyper310 hyper310 || { res=$?; echo "To reproduce failures locally, follow the instructions at https://opentitan.org/book/doc/getting_started/setup_fpga.html#reproducing-fpga-ci-failures-locally"; exit "${res}"; }
+      ci/scripts/run-fpga-tests.sh hyper310 cw310_sival_rom_ext || { res=$?; echo "To reproduce failures locally, follow the instructions at https://opentitan.org/book/doc/getting_started/setup_fpga.html#reproducing-fpga-ci-failures-locally"; exit "${res}"; }
     displayName: Execute tests
   - template: ci/publish-bazel-test-results.yml
 
@@ -629,6 +605,7 @@ jobs:
       downloadPartialBuildBinFrom:
         - chip_earlgrey_cw340
         - sw_build
+  - template: ci/load-bazel-cache-write-creds.yml
   - bash: |
       set -e
       . util/build_consts.sh
@@ -655,6 +632,7 @@ jobs:
       downloadPartialBuildBinFrom:
         - chip_earlgrey_cw310
         - sw_build
+  - template: ci/load-bazel-cache-write-creds.yml
   - bash: |
       set -e
       . util/build_consts.sh

bazelisk.sh

@@ -93,10 +93,15 @@ function outquery_starlark_expr() {
     esac
 }
 
+# Arguments:
+# $qexpr: starlark expression - see `outquery_starlark_expr`
+# $name: name of an array containing Bazel arguments that should come _before_
+#        the subcommand (e.g. `--bazelrc=...`).
 function do_outquery() {
     local qexpr="$1"
     shift
-    "$file" cquery "$@" \
+
+    "$file" "${pre_cmd_args[@]}" cquery "$@" \
         --output=starlark --starlark:expr="$qexpr" \
         --ui_event_filters=-info --noshow_progress
 }
@@ -121,6 +126,14 @@ function main() {
         fi
     fi
 
+    # Shift all flags (starting with `-`) that come before the subcommand
+    # into an array.
+    pre_cmd_args=()
+    while [[ "${1-}" == -* ]]; do
+        pre_cmd_args+=("$1")
+        shift
+    done
+
     case "${1-}" in
         outquery*)
             # The custom 'outquery' command can be used to query bazel for the
@@ -150,13 +163,13 @@ function main() {
             local qexpr outfile
             qexpr="$(outquery_starlark_expr outquery)"
             outfile=$(do_outquery "$qexpr" "$@")
-            "$file" build "$@"
+            "$file" "${pre_cmd_args[@]}" build "$@"
             # shellcheck disable=SC2059
             # We are intentionally using $command_template as a format string.
             eval "$(printf "$command_template" "$outfile")"
             ;;
         *)
-            exec "$file" "$@"
+            exec "$file" "${pre_cmd_args[@]}" "$@"
             ;;
     esac
 }

ci/azure-pipelines-nightly.yml

@@ -82,7 +82,7 @@ jobs:
         --build_tests_only \
         --test_tag_filters=-broken \
         --test_output=errors \
-        $(bash ./bazelisk.sh query 'attr("tags", "hyper310", tests(//...))')
+        $(bash ./bazelisk.sh query 'attr("tags", "cw310_sival", tests(//...))')
     displayName: "Run all the hyperdebug tests"
 
 - job: slow_otbn_crypto_tests

ci/bazelisk.sh

@@ -11,18 +11,19 @@ if [[ -n "${PWD_OVERRIDE}" ]]; then
     cd "${PWD_OVERRIDE}" || exit
 fi
 
-echo "Running bazelisk in $(pwd)."
+echo "Running bazelisk in $(pwd)." >&2
 
 # An additional bazelrc must be synthesized to specify precisely how to use the
 # GCP bazel cache.
+GCP_CREDS_FILE="$GCP_BAZEL_CACHE_KEY_SECUREFILEPATH"
 GCP_BAZELRC="$(mktemp /tmp/XXXXXX.bazelrc)"
 trap 'rm ${GCP_BAZELRC}' EXIT
 
-if [[ -n "${GCP_BAZEL_CACHE_KEY}" && -f "${GCP_BAZEL_CACHE_KEY}" ]]; then
-    echo "Applying GCP cache key; will upload to the cache."
-    echo "build --google_credentials=${GCP_BAZEL_CACHE_KEY}" >> "${GCP_BAZELRC}"
+if [[ -n "$GCP_CREDS_FILE" && -f "$GCP_CREDS_FILE" ]]; then
+    echo "Applying GCP cache key; will upload to the cache." >&2
+    echo "build --google_credentials=${GCP_CREDS_FILE}" >> "${GCP_BAZELRC}"
 else
-    echo "No key/invalid path to key. Download from cache only."
+    echo "No key/invalid path to key. Download from cache only." >&2
     echo "build --remote_upload_local_results=false" >> "${GCP_BAZELRC}"
 fi
 

ci/load-bazel-cache-write-creds.yml

@@ -0,0 +1,17 @@
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+# Azure Pipelines template for downloading a "secure file" containing the write
+# credentials for a GCP bucket where we store a Bazel cache.
+#
+# The path to the downloaded file is automatically stored in an environment
+# variable `GCP_BAZEL_CACHE_KEY_SECUREFILEPATH`. This file is loaded by the
+# `ci/bazelisk.sh` script.
+
+steps:
+  - task: DownloadSecureFile@1
+    condition: eq(variables['Build.SourceBranchName'], 'master')
+    name: GCP_BAZEL_CACHE_KEY
+    inputs:
+      secureFile: "bazel_cache_gcp_key.json"

ci/scripts/check-bazel-banned-rules.sh

@@ -6,7 +6,7 @@
 
 set -e
 
-GIT_REPOS=$(./bazelisk.sh query "kind('(new_)?git_repository', //external:*)")
+GIT_REPOS=$(ci/bazelisk.sh query "kind('(new_)?git_repository', //external:*)")
 if [[ ${GIT_REPOS} ]]; then
   echo "Bazel's 'git_repository' rule is insecure and incompatible with OpenTitan's airgapping strategy."
   echo "Please replace $GIT_REPOS with our 'http_archive_or_local' rule and set a sha256 so it can be canonically reproducible."

ci/scripts/check-bazel-tags.sh

@@ -25,7 +25,7 @@ check_empty () {
 
 # This check ensures OpenTitan software can be built with a wildcard without
 # waiting for Verilator using --build_tag_filters=-verilator
-untagged=$(./bazelisk.sh query \
+untagged=$(ci/bazelisk.sh query \
   "rdeps(
       //...,
       //hw:verilator

ci/scripts/check-licence-headers.sh

@@ -30,7 +30,7 @@ set -o pipefail
 (for F in $(git diff --name-only --diff-filter=ACMRTUXB "$merge_base"); do
     echo "--test_arg=\"$F\""
 done)| \
-    xargs -r ./bazelisk.sh test //quality:license_check --test_output=streamed || {
+    xargs -r ci/bazelisk.sh test //quality:license_check --test_output=streamed || {
 
     echo >&2 -n "##vso[task.logissue type=error]"
     echo >&2 "Licence header check failed."

ci/scripts/check-module-ids.sh

@@ -24,7 +24,7 @@ query_elfs='
     )'
 target_file=$(mktemp)
 trap 'rm -f "$target_file"' EXIT
-./bazelisk.sh query "$query_elfs" >"$target_file"
+ci/bazelisk.sh query "$query_elfs" >"$target_file"
 # We now ask bazel to build all targets but we also add the module ID checker aspect
 # and we query the corresponding output group to force bazel to run the checks.
 ./ci/bazelisk.sh build \

ci/scripts/clang-format.sh

@@ -28,4 +28,4 @@ set -o pipefail
 (for F in $(git diff --name-only "$merge_base" -- "*.cpp" "*.cc" "*.c" "*.h" ':!*/vendor/*'); do
     echo "--test_arg=\"$F\""
 done) | \
-    xargs -r ./bazelisk.sh test //quality:clang_format_check --test_output=streamed
+    xargs -r ci/bazelisk.sh test //quality:clang_format_check --test_output=streamed

ci/scripts/get-bitstream-strategy.sh

@@ -33,12 +33,12 @@ else
 fi
 
 # Retrieve the most recent bitstream at or before HEAD.
-BITSTREAM="--refresh HEAD" ./bazelisk.sh build @bitstreams//:manifest
+BITSTREAM="--refresh HEAD" ci/bazelisk.sh build @bitstreams//:manifest
 
 # Find the bitstream commit from the manifest file.
 manifest_file=$(ci/scripts/target-location.sh @bitstreams//:manifest)
 
-bitstream_commit=$(./bazelisk.sh run //util/py/scripts:get_bitstream_build_id \
+bitstream_commit=$(ci/bazelisk.sh run //util/py/scripts:get_bitstream_build_id \
   -- --manifest ${manifest_file} \
   --schema ${REPO_TOP}/rules/scripts/bitstreams_manifest.schema.json \
   --design ${bitstream_design})

ci/scripts/lib/bazel_query.py

@@ -108,7 +108,7 @@ def query(self, query: str) -> List[str]:
             return self._backend(query)
 
         bazel = subprocess.run(
-            ["./bazelisk.sh", "query", "--output=label", query],
+            ["ci/bazelisk.sh", "query", "--output=label", query],
             stdout=subprocess.PIPE,
             stderr=subprocess.DEVNULL,
             encoding='utf-8',