[ci] Produce and upload bazel execution logs

azure-pipelines.yml

@@ -18,6 +18,10 @@ variables:
   # This controls where builds happen, and gets picked up by build_consts.sh.
   BUILD_ROOT: $(Build.ArtifactStagingDirectory)
   VIVADO_VERSION: "2021.1"
+  # This controls where the bazel execution logs go
+  BAZEL_EXEC_LOG_ROOT: $(Build.ArtifactStagingDirectory)/bazel_exec_logs
+
+  PYTHONDONTWRITEBYTECODE: 1
 
 trigger:
   batch: true
@@ -48,6 +52,7 @@ jobs:
     displayName: Upload repository
 - job: lint
   displayName: Quality (quick lint)
+  condition: false
   # Run code quality checks (quick lint)
   dependsOn: checkout
   pool: ci-public
@@ -121,7 +126,7 @@ jobs:
   displayName: Test an airgapped Bazel build
   timeoutInMinutes: 120
   dependsOn: checkout
-  condition: eq(variables['Build.Reason'], 'PullRequest')
+  condition: false
   pool:
     vmImage: ubuntu-20.04
   steps:
@@ -131,6 +136,8 @@ jobs:
 
 - job: slow_lints
   displayName: Quality (in-depth lint)
+  # Disable stage for experiments
+  condition: false
   # Run code quality checks (in-depth lint)
   dependsOn: lint
   pool:
@@ -176,8 +183,8 @@ jobs:
   displayName: Earl Grey SW Build
   # Build software tests for the Earl Grey toplevel design
   timeoutInMinutes: 120
-  dependsOn: lint
-  condition: and(succeeded(), eq(dependencies.lint.outputs['DetermineBuildType.onlyDocChanges'], '0'), eq(dependencies.lint.outputs['DetermineBuildType.onlyCdcChanges'], '0'))
+  dependsOn: checkout
+  #condition: and(succeeded(), eq(dependencies.lint.outputs['DetermineBuildType.onlyDocChanges'], '0'), eq(dependencies.lint.outputs['DetermineBuildType.onlyCdcChanges'], '0'))
   pool: ci-public
   variables:
     - name: bazelCacheGcpKeyPath
@@ -216,7 +223,8 @@ jobs:
       #   environment does not have access to Vivado.
       export GCP_BAZEL_CACHE_KEY=$(bazelCacheGcpKeyPath)
       TARGET_PATTERN_FILE=target_pattern.txt
-      echo //... > "${TARGET_PATTERN_FILE}"
+      mkdir "$BAZEL_EXEC_LOG_ROOT"
+      echo //sw/device/lib/... > "${TARGET_PATTERN_FILE}"
       echo -//quality/... >> "${TARGET_PATTERN_FILE}"
       echo -//sw/otbn/crypto/... >> "${TARGET_PATTERN_FILE}"
       echo -//third_party/riscv-compliance/... >> "${TARGET_PATTERN_FILE}"
@@ -234,31 +242,35 @@ jobs:
         --build_tests_only=false \
         --define DISABLE_VERILATOR_BUILD=true \
         --test_tag_filters=-broken,-cw310,-verilator,-dv \
-        --target_pattern_file="${TARGET_PATTERN_FILE}"
+        --target_pattern_file="${TARGET_PATTERN_FILE}" \
+        --execution_log_binary_file="$BAZEL_EXEC_LOG_ROOT/build.log" \
+        --noexecution_log_sort
     displayName: Build SW
+  - template: ci/publish-bazel-exec-logs.yml
   - publish: target_pattern.txt
     artifact: target_pattern_file
-  - bash: |
-      set -x -e
-      . util/build_consts.sh
-      # copy the rom to a specific location
-      ROM_TARGET="${BIN_DIR}/sw/device/silicon_creator/rom"
-      mkdir -p "${ROM_TARGET}"
-      ROM_REAL_TARGETS="//sw/device/silicon_creator/rom:package_real"
-      ROM_FAKE_TARGETS="//sw/device/silicon_creator/rom:package_fake"
-      QUERY_CMD_ARGS=(outquery-all --noinclude_aspects --noimplicit_deps)
-      ROM_REAL_FILES=($(./bazelisk.sh "${QUERY_CMD_ARGS[@]}" "${ROM_REAL_TARGETS}" | sort | uniq))
-      ROM_FAKE_FILES=($(./bazelisk.sh "${QUERY_CMD_ARGS[@]}" "${ROM_FAKE_TARGETS}" | sort | uniq))
-      cp -Lvt "${ROM_TARGET}" "${ROM_FAKE_FILES[@]}" "${ROM_REAL_FILES[@]}"
-  - template: ci/upload-artifacts-template.yml
-    parameters:
-      includePatterns:
-        - "/sw/***"
+  # - bash: |
+  #     set -x -e
+  #     . util/build_consts.sh
+  #     # copy the rom to a specific location
+  #     ROM_TARGET="${BIN_DIR}/sw/device/silicon_creator/rom"
+  #     mkdir -p "${ROM_TARGET}"
+  #     ROM_REAL_TARGETS="//sw/device/silicon_creator/rom:package_real"
+  #     ROM_FAKE_TARGETS="//sw/device/silicon_creator/rom:package_fake"
+  #     QUERY_CMD_ARGS=(outquery-all --noinclude_aspects --noimplicit_deps)
+  #     ROM_REAL_FILES=($(./bazelisk.sh "${QUERY_CMD_ARGS[@]}" "${ROM_REAL_TARGETS}" | sort | uniq))
+  #     ROM_FAKE_FILES=($(./bazelisk.sh "${QUERY_CMD_ARGS[@]}" "${ROM_FAKE_TARGETS}" | sort | uniq))
+  #     cp -Lvt "${ROM_TARGET}" "${ROM_FAKE_FILES[@]}" "${ROM_REAL_FILES[@]}"
+  # - template: ci/upload-artifacts-template.yml
+  #   parameters:
+  #     includePatterns:
+  #       - "/sw/***"
 
 - job: sw_test
   displayName: Earl Grey SW Test
   timeoutInMinutes: 120
   dependsOn: sw_build
+  condition: false
   pool: ci-public
   variables:
     - name: bazelCacheGcpKeyPath
@@ -279,17 +291,22 @@ jobs:
     artifact: target_pattern_file
   - bash: |
       TARGET_PATTERN_FILE="$(Pipeline.Workspace)/target_pattern_file/target_pattern.txt"
+      mkdir "$BAZEL_EXEC_LOG_ROOT"
       ci/bazelisk.sh test \
         --build_tests_only=false \
         --test_output=errors \
         --define DISABLE_VERILATOR_BUILD=true \
         --test_tag_filters=-broken,-cw310,-verilator,-dv,-silicon \
-        --target_pattern_file="${TARGET_PATTERN_FILE}"
+        --target_pattern_file="${TARGET_PATTERN_FILE}" \
+        --execution_log_binary_file="$BAZEL_EXEC_LOG_ROOT/test.log" \
+        --noexecution_log_sort
     displayName: Build & test SW
+  - template: ci/publish-bazel-exec-logs.yml
   - template: ci/publish-bazel-test-results.yml
 
 - job: chip_englishbreakfast_verilator
   displayName: Verilated English Breakfast (Build)
+  # Disable stage for experiments
   # Build Verilator simulation of the English Breakfast toplevel design
   dependsOn: lint
   condition: and(succeeded(), eq(dependencies.lint.outputs['DetermineBuildType.onlyDocChanges'], '0'), eq(dependencies.lint.outputs['DetermineBuildType.onlyCdcChanges'], '0'))
@@ -313,6 +330,7 @@ jobs:
 
 - job: execute_verilated_tests
   displayName: Fast Verilated Earl Grey tests
+  # Disable stage for experiments
   # Build and run fast tests on sim_verilator
   pool: ci-public
   timeoutInMinutes: 240
@@ -363,6 +381,7 @@ jobs:
 # Work towards this goal is tracked in issue #4669.
 - job: build_and_execute_verilated_tests_englishbreakfast
   displayName: Verilated English Breakfast (Execute)
+  # Disable stage for experiments
   # Build and execute tests on the Verilated English Breakfast toplevel design with Bazel
   pool:
     vmImage: ubuntu-20.04
@@ -394,6 +413,7 @@ jobs:
 
 - job: otbn_standalone_tests
   displayName: Run OTBN Smoke Test
+  # Disable stage for experiments
   dependsOn: lint
   condition: and(succeeded(), eq(dependencies.lint.outputs['DetermineBuildType.onlyCdcChanges'], '0'))
   pool:
@@ -427,6 +447,7 @@ jobs:
 
 - job: otbn_crypto_tests
   displayName: Run OTBN crypto tests
+  # Disable stage for experiments
   dependsOn: lint
   condition: and(succeeded(), eq(dependencies.lint.outputs['DetermineBuildType.onlyCdcChanges'], '0'))
   pool:
@@ -450,6 +471,7 @@ jobs:
 
 - job: chip_earlgrey_cw310
   displayName: CW310's Earl Grey Bitstream
+  # Disable stage for experiments
   # Build CW310 variant of the Earl Grey toplevel design using Vivado
   dependsOn:
     - lint
@@ -464,6 +486,7 @@ jobs:
 
 - job: chip_earlgrey_cw310_hyperdebug
   displayName: CW310's Earl Grey Bitstream for Hyperdebug
+  # Disable stage for experiments
   # Build CW310-hyperdebug variant of the Earl Grey toplevel design using Vivado
   dependsOn:
     - lint
@@ -478,6 +501,7 @@ jobs:
 
 - job: chip_earlgrey_cw340
   displayName: CW340's Earl Grey Bitstream
+  # Disable stage for experiments
   # Build CW340 variant of the Earl Grey toplevel design using Vivado
   dependsOn:
     - lint
@@ -492,6 +516,7 @@ jobs:
 
 - job: chip_englishbreakfast_cw305
   displayName: CW305's Bitstream
+  # Disable stage for experiments
   # Build CW305 variant of the English Breakfast toplevel design using Vivado
   dependsOn: build_and_execute_verilated_tests_englishbreakfast
   condition: and(succeeded(), eq(dependencies.lint.outputs['DetermineBuildType.onlyDocChanges'], '0'), eq(dependencies.lint.outputs['DetermineBuildType.onlyDvChanges'], '0'), eq(dependencies.lint.outputs['DetermineBuildType.onlyCdcChanges'], '0'))
@@ -516,6 +541,7 @@ jobs:
 
 - job: cache_bitstreams
   displayName: Cache bitstreams to GCP
+  # Disable stage for experiments
   pool:
     vmImage: ubuntu-20.04
   dependsOn:
@@ -544,6 +570,7 @@ jobs:
 
 - job: execute_test_rom_fpga_tests_cw310
   displayName: CW310 Test ROM Tests
+  # Disable stage for experiments
   pool:
     name: $(fpga_pool)
     demands: BOARD -equals cw310
@@ -570,6 +597,7 @@ jobs:
 
 - job: execute_rom_fpga_tests_cw310
   displayName: CW310 ROM Tests
+  # Disable stage for experiments
   pool:
     name: $(fpga_pool)
     demands: BOARD -equals cw310
@@ -596,6 +624,7 @@ jobs:
 
 - job: execute_sival_fpga_tests_cw310
   displayName: CW310 SiVal Tests
+  # Disable stage for experiments
   pool:
     name: $(fpga_pool)
     demands: BOARD -equals cw310
@@ -623,6 +652,7 @@ jobs:
 
 - job: execute_hyperdebug_tests_cw310
   displayName: Hyperdebug CW310 Tests (Experimental)
+  # Disable stage for experiments
   pool:
     name: $(fpga_pool)
     demands: BOARD -equals cw310
@@ -664,6 +694,7 @@ jobs:
 
 - job: execute_rom_fpga_tests_cw340
   displayName: CW340 ROM Tests
+  # Disable stage for experiments
   pool: FPGA CW340
   timeoutInMinutes: 60
   dependsOn:
@@ -688,6 +719,7 @@ jobs:
 
 - job: execute_fpga_manuf_tests_cw310
   displayName: CW310 Manufacturing Tests
+  # Disable stage for experiments
   pool:
     name: $(fpga_pool)
     demands: BOARD -equals cw310
@@ -715,6 +747,7 @@ jobs:
 
 - job: deploy_release_artifacts
   displayName: Package & deploy release
+  # Disable stage for experiments
   pool:
     vmImage: ubuntu-20.04
   dependsOn:
@@ -757,6 +790,7 @@ jobs:
 
 - job: build_docker_containers
   displayName: "Build Docker Containers"
+  # Disable stage for experiments
   pool:
     vmImage: ubuntu-20.04
   dependsOn:

ci/bazelisk.sh

@@ -31,6 +31,11 @@ fi
 # See #14695 for more information.
 echo "build --remote_default_exec_properties=OSVersion=\"$(lsb_release -ds)\"" >> "${GCP_BAZELRC}"
 
+# Override workspace status in CI.
+# NOTE: it seems the first instance of --workspace_status_command has precedence so this
+# will override the one in .bazelrc, but this behaviour does not seem to be documented.
+echo "build --workspace_status_command=ci/scripts/get_workspace_status.sh" >> "${GCP_BAZELRC}"
+
 "$(dirname $0)"/../bazelisk.sh \
   --bazelrc="${GCP_BAZELRC}" \
   --bazelrc="$(dirname $0)"/.bazelrc \

ci/publish-bazel-exec-logs.yml

@@ -0,0 +1,21 @@
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+# Azure template for publishing bazel execution logs so they can be analyzed
+# offline.
+#
+# This template expects that a variable $BAZEL_EXEC_LOG_ROOT is set and it
+# will upload a zip file containing all the files in this directory.
+
+steps:
+  - bash: |
+      set -e
+      cd $BAZEL_EXEC_LOG_ROOT
+      zip $(Build.ArtifactStagingDirectory)/exec_logs.zip *
+    displayName: Create zip with execution logs
+  - publish: "$(Build.ArtifactStagingDirectory)/exec_logs.zip"
+    # The PhaseName is the string after the "job" key in the build description,
+    # e.g. "job: my_phase_name".
+    artifact: exec-logs-$(System.PhaseName)
+    displayName: Upload execution logs

ci/scripts/get_workspace_status.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+# This script is passed to bazel's `--workspace_status_command` to generate
+# the workspace status. To ensure reproducible builds, we output constant
+# values.
+
+echo "BUILD_GIT_VERSION OpenTitanCI"
+echo "BUILD_SCM_REVISION 123456789"
+echo "BUILD_SCM_STATUS modified"
+echo "BUILD_TIMESTAMP 315532800"

third_party/python/pip.bzl

@@ -46,6 +46,10 @@ def _pip_wheel_impl(rctx):
     rctx.report_progress("Installing the Python wheel package")
     result = rctx.execute(
         args,
+        environment = {
+            "SOURCE_DATE_EPOCH": "315532800",
+            "PYTHONHASHSEED": "0",
+        },
         timeout = rctx.attr.timeout,
         quiet = rctx.attr.quiet,
     )
@@ -66,6 +70,10 @@ def _pip_wheel_impl(rctx):
     rctx.report_progress("Pre-building Python wheels")
     result = rctx.execute(
         args,
+        environment = {
+            "SOURCE_DATE_EPOCH": "315532800",
+            "PYTHONHASHSEED": "0",
+        },
         timeout = rctx.attr.timeout,
         quiet = rctx.attr.quiet,
     )
@@ -81,6 +89,10 @@ def _pip_wheel_impl(rctx):
     rctx.report_progress("Generating sanitzed requirements file")
     result = rctx.execute(
         args,
+        environment = {
+            "SOURCE_DATE_EPOCH": "315532800",
+            "PYTHONHASHSEED": "0",
+        },
         timeout = rctx.attr.timeout,
         quiet = rctx.attr.quiet,
         working_directory = "./",