[ci] Avoid running duplicate hyperdebug-based tests

At the moment, the sival job runs only `cw310_sival` jobs but not
`cw310_sival_rom_ext` which is not right. On the other hand,
the hyperdebug job runs all `hyper310` tests and that includes a
number of `cw310_sival` and `cw310_sival_rom_ext` tests so we run
some tests twice.

This commit changes removes the existing hyper310 job and split the
sival job into two subsets:
- sival_rom_ext tests,
- sival tests that do not have a corresponding sival_rom_ext tests.

Signed-off-by: Amaury Pouly <amaury.pouly@lowrisc.org>

azure-pipelines.yml

@@ -571,7 +571,7 @@ jobs:
   - template: ci/publish-bazel-test-results.yml
 
 - job: execute_sival_fpga_tests_cw310
-  displayName: CW310 SiVal Tests
+  displayName: CW310 SiVal non-ROM_EXT Tests
   pool:
     name: $(fpga_pool)
     demands: BOARD -equals cw310
@@ -586,24 +586,38 @@ jobs:
   - template: ci/download-artifacts-template.yml
     parameters:
       downloadPartialBuildBinFrom:
-        - chip_earlgrey_cw310
         - chip_earlgrey_cw310_hyperdebug
         - sw_build
   - template: ci/load-bazel-cache-write-creds.yml
+  # We run the update command twice to workaround an issue with udev on the container,
+  # where rusb cannot dynamically update its device list in CI (udev is not completely
+  # functional). If the device is in normal mode, the first thing that opentitantool
+  # does is to switch it to DFU mode and wait until it reconnects. This reconnection is
+  # never detected. But if we run the tool another time, the device list is queried again
+  # and opentitantool can finish the update. The device will now reboot in normal mode
+  # and work for the hyperdebug job.
+  - bash: |
+      ci/bazelisk.sh run \
+        //sw/host/opentitantool:opentitantool -- \
+        --interface=hyperdebug_dfu transport update-firmware \
+      || ci/bazelisk.sh run \
+        //sw/host/opentitantool:opentitantool -- \
+        --interface=hyperdebug_dfu transport update-firmware || true
+    displayName: "Update the hyperdebug firmware"
   - bash: |
       set -e
       . util/build_consts.sh
       module load "xilinx/vivado/$(VIVADO_VERSION)"
-      ci/scripts/run-fpga-tests.sh hyper310 cw310_sival,-broken || { res=$?; echo "To reproduce failures locally, follow the instructions at https://opentitan.org/book/doc/getting_started/setup_fpga.html#reproducing-fpga-ci-failures-locally"; exit "${res}"; }
+      ci/scripts/run-fpga-tests.sh hyper310 cw310_sival_but_not_rom_ext_tests || { res=$?; echo "To reproduce failures locally, follow the instructions at https://opentitan.org/book/doc/getting_started/setup_fpga.html#reproducing-fpga-ci-failures-locally"; exit "${res}"; }
     displayName: Execute tests
   - template: ci/publish-bazel-test-results.yml
 
-- job: execute_hyperdebug_tests_cw310
-  displayName: Hyperdebug CW310 Tests (Experimental)
+- job: execute_sival_rom_ext_fpga_tests_cw310
+  displayName: CW310 SiVal ROM_EXT Tests
   pool:
     name: $(fpga_pool)
     demands: BOARD -equals cw310
-  timeoutInMinutes: 60
+  timeoutInMinutes: 45
   dependsOn:
     - chip_earlgrey_cw310_hyperdebug
     - sw_build
@@ -617,8 +631,8 @@ jobs:
         - chip_earlgrey_cw310_hyperdebug
         - sw_build
   - template: ci/load-bazel-cache-write-creds.yml
-  # We run the update command twice to workaround an issue with udev on the container.
-  # Where rusb cannot dynamically update its device list in CI (udev is not completely
+  # We run the update command twice to workaround an issue with udev on the container,
+  # where rusb cannot dynamically update its device list in CI (udev is not completely
   # functional). If the device is in normal mode, the first thing that opentitantool
   # does is to switch it to DFU mode and wait until it reconnects. This reconnection is
   # never detected. But if we run the tool another time, the device list is queried again
@@ -636,7 +650,7 @@ jobs:
       set -e
       . util/build_consts.sh
       module load "xilinx/vivado/$(VIVADO_VERSION)"
-      ci/scripts/run-fpga-tests.sh hyper310 hyper310 || { res=$?; echo "To reproduce failures locally, follow the instructions at https://opentitan.org/book/doc/getting_started/setup_fpga.html#reproducing-fpga-ci-failures-locally"; exit "${res}"; }
+      ci/scripts/run-fpga-tests.sh hyper310 cw310_sival_rom_ext || { res=$?; echo "To reproduce failures locally, follow the instructions at https://opentitan.org/book/doc/getting_started/setup_fpga.html#reproducing-fpga-ci-failures-locally"; exit "${res}"; }
     displayName: Execute tests
   - template: ci/publish-bazel-test-results.yml
 

ci/scripts/run-fpga-tests.sh

@@ -9,14 +9,13 @@ set -e
 . util/build_consts.sh
 
 if [ $# == 0 ]; then
-    echo >&2 "Usage: run-fpga-tests.sh <fpga> <tags>"
-    echo >&2 "E.g. ./run-fpga-tests.sh cw310 cw310_rom"
-    echo >&2 "E.g. ./run-fpga-tests.sh cw340 cw340_rom cw340_test_rom"
+    echo >&2 "Usage: run-fpga-tests.sh <fpga> <tags or test set name>"
+    echo >&2 "E.g. ./run-fpga-tests.sh cw310 manuf"
+    echo >&2 "E.g. ./run-fpga-tests.sh cw310 cw310_rom_tests"
     exit 1
 fi
 fpga="$1"
-shift
-fpga_tags=("$@")
+fpga_tags="$2"
 
 # Copy bitstreams and related files into the cache directory so Bazel will have
 # the corresponding targets in the @bitstreams workspace.
@@ -45,14 +44,60 @@ trap 'ci/bazelisk.sh run //sw/host/opentitantool -- --rcfile= --interface=${fpga
 ci/bazelisk.sh run //sw/host/opentitantool -- --rcfile= --interface="$fpga" --logging debug fpga set-pll || true
 ci/bazelisk.sh run //sw/host/opentitantool -- --rcfile= --interface="$fpga" fpga clear-bitstream
 
-for tag in "${fpga_tags[@]}"; do
-    ci/bazelisk.sh test //... @manufacturer_test_hooks//...\
-        --define DISABLE_VERILATOR_BUILD=true \
-        --nokeep_going \
-        --test_tag_filters="${tag}",-broken,-skip_in_ci \
-        --test_timeout_filters=short,moderate \
-        --test_output=all \
-        --build_tests_only \
-        --define "$fpga"=lowrisc \
-        --flaky_test_attempts=2
-done
+pattern_file=$(mktemp)
+# Recognize special test set names, otherwise we interpret it as a list of tags.
+test_args=""
+echo "tags: ${fpga_tags}"
+if [ "${fpga_tags}" == "cw310_sival_but_not_rom_ext_tests" ]
+then
+    # Only consider tests that are tagged `cw310_sival` but not tagged `cw310_sival_rom_ext`.
+    # The difficulty is that, technically, they are different tests since `opentitan_test` creates
+    # one target for each execution environment. The following query relies on the existence
+    # of the test suite created by `opentitan_test` that depends on all per-exec-env tests.
+    ci/bazelisk.sh query \
+    "
+        `# Find all test tagged cw310_sival that are dependencies of the test suite identified`
+        attr(
+            \"tags\",
+            \"cw310_sival([^_]|$)\",
+            deps(
+                `# Find all test suites depending on a test tagged cw310_sival` \
+                kind(
+                    \"test_suite\",
+                    rdeps(
+                        //...,
+                        `# Find all tests tagged cw310_sival`
+                        attr(\"tags\",\"cw310_sival([^_]|$)\", //...),
+                        1
+                    )
+                )
+                except
+                `# Remove all test suites depending on a test tagged cw310_sival_rom_ext`
+                rdeps(
+                    //...,
+                    `# Find all tests tagged cw310_sival_rom_ext`
+                    attr(\"tags\",\"cw310_sival_rom_ext\", //...),
+                    1
+                ),
+                1
+            )
+        )
+    " \
+    > "${pattern_file}"
+    test_args="${test_args} --test_tag_filters=-broken,-skip_in_ci"
+else
+    test_args="${test_args} --test_tag_filters=${fpga_tags},-broken,-skip_in_ci"
+    echo "//..." > "${pattern_file}"
+    echo "@manufacturer_test_hooks//..." >> "${pattern_file}"
+fi
+
+ci/bazelisk.sh test \
+    --define DISABLE_VERILATOR_BUILD=true \
+    --nokeep_going \
+    --test_timeout_filters=short,moderate \
+    --test_output=all \
+    --build_tests_only \
+    --define "$fpga"=lowrisc \
+    --flaky_test_attempts=2 \
+    --target_pattern_file="${pattern_file}" \
+    ${test_args}