Remove zap checkout for darwin builds in CI, assume CIPD provides this (

#24677) * Remove zap checkout for darwin builds in CI, assume CIPD provides this. If this works, we can work on modifying the dockerfiles for linux too * Make zap-cli NOT available for xcodebuild * Add --version to zap-cli to not try to run more things * The zap test should be under the build environment, to get the path working * Run in build environment when checking for zap --------- Co-authored-by: Andrei Litvin <andreilitvin@google.com>
project-chip · Nov 14, 2023 · 7933028 · 7933028
1 parent 42a8c61
commit 7933028
Show file tree

Hide file tree

Showing 6 changed files with 28 additions and 120 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -352,12 +352,6 @@ jobs:
         timeout-minutes: 200
         runs-on: macos-latest
         if: github.actor != 'restyled-io[bot]'
-
-        env:
-            # NOTE: Generally kept in sync within the repo using
-            #       scripts/tools/zap/version_update.py
-            ZAP_VERSION: v2023.01.19-nightly
-            ZAP_INSTALL_PATH: /usr/local/zap/zap-v2023.01.19-nightly
 
         steps:
             - uses: Wandalen/wretry.action@v1.0.36
@@ -388,16 +382,6 @@ jobs:
                   ls -la /usr/local/Cellar/openssl@1.1
                   OPEN_SSL_VERSION=`ls -la /usr/local/Cellar/openssl@1.1 | cat | tail -n1 | awk '{print $NF}'`
                   ln -s /usr/local/Cellar/openssl@1.1/$OPEN_SSL_VERSION/lib/pkgconfig/* .
-            - name: Install zap
-              run: |
-                 sudo mkdir -p $ZAP_INSTALL_PATH
-                 sudo chown `whoami` $ZAP_INSTALL_PATH
-                 curl -L https://github.com/project-chip/zap/releases/download/${ZAP_VERSION}/zap-mac.zip \
-                      --output $ZAP_INSTALL_PATH/zap-mac.zip
-                 cd $ZAP_INSTALL_PATH
-                 unzip zap-mac.zip zap-cli
-                 rm zap-mac.zip
-                 ./zap-cli --version
             - name: Bootstrap
               timeout-minutes: 25
               run: scripts/build/gn_bootstrap.sh

diff --git a/.github/workflows/darwin-tests.yaml b/.github/workflows/darwin-tests.yaml
@@ -39,10 +39,6 @@ jobs:
         env:
             BUILD_VARIANT: ${{matrix.build_variant}}
             LSAN_OPTIONS: detect_leaks=1 malloc_context_size=40 suppressions=scripts/tests/chiptest/lsan-mac-suppressions.txt
-            # NOTE: Generally kept in sync within the repo using
-            #       scripts/tools/zap/version_update.py
-            ZAP_VERSION: v2023.01.19-nightly
-            ZAP_INSTALL_PATH: /usr/local/zap/zap-v2023.01.19-nightly
 
         if: github.actor != 'restyled-io[bot]'
         runs-on: macos-latest
@@ -76,16 +72,6 @@ jobs:
                   ls -la /usr/local/Cellar/openssl@1.1
                   OPEN_SSL_VERSION=`ls -la /usr/local/Cellar/openssl@1.1 | cat | tail -n1 | awk '{print $NF}'`
                   ln -s /usr/local/Cellar/openssl@1.1/$OPEN_SSL_VERSION/lib/pkgconfig/* .
-            - name: Install zap
-              run: |
-                 sudo mkdir -p $ZAP_INSTALL_PATH
-                 sudo chown `whoami` $ZAP_INSTALL_PATH
-                 curl -L https://github.com/project-chip/zap/releases/download/${ZAP_VERSION}/zap-mac.zip \
-                      --output $ZAP_INSTALL_PATH/zap-mac.zip
-                 cd $ZAP_INSTALL_PATH
-                 unzip zap-mac.zip zap-cli
-                 rm zap-mac.zip
-                 ./zap-cli --version
             - name: Bootstrap
               timeout-minutes: 25
               run: scripts/build/gn_bootstrap.sh

diff --git a/.github/workflows/darwin.yaml b/.github/workflows/darwin.yaml
@@ -52,22 +52,6 @@ jobs:
                   ls -la /usr/local/Cellar/openssl@1.1
                   OPEN_SSL_VERSION=`ls -la /usr/local/Cellar/openssl@1.1 | cat | tail -n1 | awk '{print $NF}'`
                   ln -s /usr/local/Cellar/openssl@1.1/$OPEN_SSL_VERSION/lib/pkgconfig/* .
-            - name: Install zap
-              run: |
-                 sudo mkdir -p $ZAP_INSTALL_PATH
-                 sudo chown `whoami` $ZAP_INSTALL_PATH
-                 curl -L https://github.com/project-chip/zap/releases/download/${ZAP_VERSION}/zap-mac.zip \
-                      --output $ZAP_INSTALL_PATH/zap-mac.zip
-                 cd $ZAP_INSTALL_PATH
-                 unzip zap-mac.zip zap-cli
-                 rm zap-mac.zip
-                 ./zap-cli --version
-              env:
-                  # NOTE: Generally kept in sync within the repo using
-                  #       scripts/tools/zap/version_update.py
-                  # This is scoped to only the steps that don't use xcodebuild.
-                  ZAP_VERSION: v2023.01.19-nightly
-                  ZAP_INSTALL_PATH: /usr/local/zap/zap-v2023.01.19-nightly
             - name: Bootstrap
               timeout-minutes: 25
               run: scripts/build/gn_bootstrap.sh
@@ -79,6 +63,12 @@ jobs:
                   path: |
                       .environment/gn_out/.ninja_log
                       .environment/pigweed-venv/*.log
+            - name: Block zap-cli from being used
+              # xcodebuild is NOT expected to require zap-cli
+              run: scripts/run_in_build_env.sh 'mv $PW_ENVIRONMENT_ROOT/cipd/packages/zap/zap-cli $PW_ENVIRONMENT_ROOT/cipd/packages/zap/zap-cli.moved'
+            - name: Validate zap-cli is NOT available
+              # run_in_build_env.sh is used to ensure PATH is set to something that would otherwise find zap-cli
+              run: scripts/run_in_build_env.sh '(zap-cli --version && exit 1) || exit 0'
             - name: Run iOS Build Debug
               timeout-minutes: 50
               working-directory: src/darwin/Framework
@@ -118,36 +108,22 @@ jobs:
             - name: Clean Build
               run: xcodebuild clean
               working-directory: src/darwin/Framework
+            - name: Make zap-cli work again
+              run: scripts/run_in_build_env.sh 'mv $PW_ENVIRONMENT_ROOT/cipd/packages/zap/zap-cli.moved $PW_ENVIRONMENT_ROOT/cipd/packages/zap/zap-cli'
+            - name: Validate zap-cli is again available
+              run: scripts/run_in_build_env.sh 'zap-cli --version'
             - name: Build example All Clusters Server
               timeout-minutes: 15
               run: |
                   scripts/examples/gn_build_example.sh examples/all-clusters-app/linux out/debug chip_config_network_layer_ble=false
-              env:
-                  # NOTE: Generally kept in sync within the repo using
-                  #       scripts/tools/zap/version_update.py
-                  # This is scoped to only the steps that don't use xcodebuild.
-                  ZAP_VERSION: v2023.01.19-nightly
-                  ZAP_INSTALL_PATH: /usr/local/zap/zap-v2023.01.19-nightly
             - name: Build example OTA Provider
               timeout-minutes: 10
               run: |
                   scripts/examples/gn_build_example.sh examples/ota-provider-app/linux out/debug chip_config_network_layer_ble=false
-              env:
-                  # NOTE: Generally kept in sync within the repo using
-                  #       scripts/tools/zap/version_update.py
-                  # This is scoped to only the steps that don't use xcodebuild.
-                  ZAP_VERSION: v2023.01.19-nightly
-                  ZAP_INSTALL_PATH: /usr/local/zap/zap-v2023.01.19-nightly
             - name: Build example OTA Requestor
               timeout-minutes: 10
               run: |
                   scripts/examples/gn_build_example.sh examples/ota-requestor-app/linux out/debug chip_config_network_layer_ble=false
-              env:
-                  # NOTE: Generally kept in sync within the repo using
-                  #       scripts/tools/zap/version_update.py
-                  # This is scoped to only the steps that don't use xcodebuild.
-                  ZAP_VERSION: v2023.01.19-nightly
-                  ZAP_INSTALL_PATH: /usr/local/zap/zap-v2023.01.19-nightly
             - name: Delete Defaults
               run: defaults delete com.apple.dt.xctest.tool
               continue-on-error: true

diff --git a/.github/workflows/fuzzing-build.yaml b/.github/workflows/fuzzing-build.yaml
@@ -88,12 +88,6 @@ jobs:
         runs-on: macos-latest
         if: github.actor != 'restyled-io[bot]'
 
-        env:
-            # NOTE: Generally kept in sync within the repo using
-            #       scripts/tools/zap/version_update.py
-            ZAP_VERSION: v2023.01.19-nightly
-            ZAP_INSTALL_PATH: /usr/local/zap/zap-v2023.01.19-nightly
-
         steps:
             - uses: Wandalen/wretry.action@v1.0.36
               name: Checkout
@@ -117,16 +111,6 @@ jobs:
                   ls -la /usr/local/Cellar/openssl@1.1
                   OPEN_SSL_VERSION=`ls -la /usr/local/Cellar/openssl@1.1 | cat | tail -n1 | awk '{print $NF}'`
                   ln -s /usr/local/Cellar/openssl@1.1/$OPEN_SSL_VERSION/lib/pkgconfig/* .
-            - name: Install zap
-              run: |
-                 sudo mkdir -p $ZAP_INSTALL_PATH
-                 sudo chown `whoami` $ZAP_INSTALL_PATH
-                 curl -L https://github.com/project-chip/zap/releases/download/${ZAP_VERSION}/zap-mac.zip \
-                      --output $ZAP_INSTALL_PATH/zap-mac.zip
-                 cd $ZAP_INSTALL_PATH
-                 unzip zap-mac.zip zap-cli
-                 rm zap-mac.zip
-                 ./zap-cli --version
             - name: Bootstrap
               timeout-minutes: 25
               run: scripts/build/gn_bootstrap.sh

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -260,10 +260,6 @@ jobs:
             CHIP_TOOL_VARIANT: ${{matrix.chip_tool}}
             TSAN_OPTIONS: "halt_on_error=1"
             LSAN_OPTIONS: detect_leaks=1 suppressions=scripts/tests/chiptest/lsan-mac-suppressions.txt
-            # NOTE: Generally kept in sync within the repo using
-            #       scripts/tools/zap/version_update.py
-            ZAP_VERSION: v2023.01.19-nightly
-            ZAP_INSTALL_PATH: /usr/local/zap/zap-v2023.01.19-nightly
 
         if: github.actor != 'restyled-io[bot]'
         runs-on: macos-latest
@@ -297,15 +293,6 @@ jobs:
                   ls -la /usr/local/Cellar/openssl@1.1
                   OPEN_SSL_VERSION=`ls -la /usr/local/Cellar/openssl@1.1 | cat | tail -n1 | awk '{print $NF}'`
                   ln -s /usr/local/Cellar/openssl@1.1/$OPEN_SSL_VERSION/lib/pkgconfig/* .
-            - name: Install zap
-              run: |
-                 sudo mkdir -p $ZAP_INSTALL_PATH
-                 sudo chown `whoami` $ZAP_INSTALL_PATH
-                 curl -L https://github.com/project-chip/zap/releases/download/${ZAP_VERSION}/zap-mac.zip \
-                      --output $ZAP_INSTALL_PATH/zap-mac.zip
-                 cd $ZAP_INSTALL_PATH
-                 unzip zap-mac.zip zap-cli
-                 rm zap-mac.zip
             - name: Bootstrap
               timeout-minutes: 25
               run: scripts/build/gn_bootstrap.sh
@@ -546,10 +533,6 @@ jobs:
         env:
             BUILD_VARIANT: ${{matrix.build_variant}}
             TSAN_OPTIONS: "halt_on_error=1"
-            # NOTE: Generally kept in sync within the repo using
-            #       scripts/tools/zap/version_update.py
-            ZAP_VERSION: v2023.01.19-nightly
-            ZAP_INSTALL_PATH: /usr/local/zap/zap-v2023.01.19-nightly
 
         if: github.actor != 'restyled-io[bot]' && false
         runs-on: macos-latest
@@ -577,15 +560,6 @@ jobs:
                   ls -la /usr/local/Cellar/openssl@1.1
                   OPEN_SSL_VERSION=`ls -la /usr/local/Cellar/openssl@1.1 | cat | tail -n1 | awk '{print $NF}'`
                   ln -s /usr/local/Cellar/openssl@1.1/$OPEN_SSL_VERSION/lib/pkgconfig/* .
-            - name: Install zap
-              run: |
-                 sudo mkdir -p $ZAP_INSTALL_PATH
-                 sudo chown `whoami` $ZAP_INSTALL_PATH
-                 curl -L https://github.com/project-chip/zap/releases/download/${ZAP_VERSION}/zap-mac.zip \
-                      --output $ZAP_INSTALL_PATH/zap-mac.zip
-                 cd $ZAP_INSTALL_PATH
-                 unzip zap-mac.zip zap-cli
-                 rm zap-mac.zip
             - name: Bootstrap
               timeout-minutes: 25
               run: scripts/build/gn_bootstrap.sh

diff --git a/scripts/tools/zap/version_update.py b/scripts/tools/zap/version_update.py
@@ -49,11 +49,6 @@
 # Set as a separate list to not pay the price of a full grep as the list of
 # files is not likely to change often
 USAGE_FILES_DEPENDING_ON_ZAP_VERSION = [
-    '.github/workflows/build.yaml',
-    '.github/workflows/darwin-tests.yaml',
-    '.github/workflows/darwin.yaml',
-    '.github/workflows/fuzzing-build.yaml',
-    '.github/workflows/tests.yaml',
     'integrations/docker/images/chip-cert-bins/Dockerfile',
     'scripts/zap.json',
 ]
@@ -96,9 +91,11 @@ class UpdateChoice(Flag):
 #
 # That line is of the form "MIN_ZAP_VERSION = '2021.1.9'"
 ZAP_EXECUTION_SCRIPT = 'scripts/tools/zap/zap_execution.py'
-ZAP_EXECUTION_MIN_RE = re.compile(r'(MIN_ZAP_VERSION = .)(\d\d\d\d\.\d\d?\.\d\d?)(.)')
+ZAP_EXECUTION_MIN_RE = re.compile(
+    r'(MIN_ZAP_VERSION = .)(\d\d\d\d\.\d\d?\.\d\d?)(.)')
 
-CHIP_ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..'))
+CHIP_ROOT_DIR = os.path.abspath(os.path.join(
+    os.path.dirname(__file__), '..', '..', '..'))
 
 
 @click.command()
@@ -117,14 +114,16 @@ class UpdateChoice(Flag):
     default=None,
     help='What version of ZAP to update to (like "v2023.01.09-nightly". If not set, versions will just be printed.')
 def version_update(log_level, update, new_version):
-    coloredlogs.install(level=__LOG_LEVELS__[log_level], fmt='%(asctime)s %(levelname)-7s %(message)s')
+    coloredlogs.install(level=__LOG_LEVELS__[
+                        log_level], fmt='%(asctime)s %(levelname)-7s %(message)s')
 
     update = __UPDATE_CHOICES__[update]
 
     if new_version:
         parsed = ZAP_VERSION_RE.match(new_version)
         if not parsed:
-            logging.error(f"Version '{new_version}' does not seem to parse as a ZAP VERSION")
+            logging.error(
+                f"Version '{new_version}' does not seem to parse as a ZAP VERSION")
             sys.exit(1)
 
         # get the numeric version for zap_execution
@@ -161,15 +160,18 @@ def version_update(log_level, update, new_version):
             while m:
                 version = file_data[m.start():m.end()]
                 if version == new_version:
-                    logging.warning("Nothing to replace. Version already %s", version)
+                    logging.warning(
+                        "Nothing to replace. Version already %s", version)
                     break
-                file_data = file_data[:m.start()] + new_version + file_data[m.end():]
+                file_data = file_data[:m.start()] + \
+                    new_version + file_data[m.end():]
                 need_replace = True
                 search_pos = m.end()  # generally ok since our versions are fixed length
                 m = ZAP_VERSION_RE.search(file_data, search_pos)
 
             if need_replace:
-                logging.info('Replacing with version %s in %s', new_version, name)
+                logging.info('Replacing with version %s in %s',
+                             new_version, name)
 
                 with open(os.path.join(CHIP_ROOT_DIR, name), 'wt') as f:
                     f.write(file_data)
@@ -183,8 +185,10 @@ def version_update(log_level, update, new_version):
         logging.info("Min version %s in %s", m.group(2), ZAP_EXECUTION_SCRIPT)
         if new_version:
             new_min_version = ("%d.%d.%d" % zap_min_version)
-            file_data = file_data[:m.start()] + m.group(1) + new_min_version + m.group(3) + file_data[m.end():]
-            logging.info('Updating min version to %s in %s', new_min_version, ZAP_EXECUTION_SCRIPT)
+            file_data = file_data[:m.start()] + m.group(1) + \
+                new_min_version + m.group(3) + file_data[m.end():]
+            logging.info('Updating min version to %s in %s',
+                         new_min_version, ZAP_EXECUTION_SCRIPT)
 
             with open(os.path.join(CHIP_ROOT_DIR, ZAP_EXECUTION_SCRIPT), 'wt') as f:
                 f.write(file_data)