Add compile-time code generation using zap for app-templates

.github/workflows/build.yaml

@@ -251,6 +251,14 @@ jobs:
                        --pregen-dir ./zzz_pregenerated \
                        build \
                     "
+            - name: Check no code generation in output
+              run: |
+                  CNT=$(find -name "CHIPClusters.h" out | wc -l)
+                  if [ "${CNT}" != "0" ]; then
+                     echo "ERROR: found unexpected generated files:"
+                     find -name "CHIPClusters.h" out
+                     exit 1
+                  fi
             - name: Undo code pre-generation changes (make compile time codegen work again)
               run: |
                   rm -rf ./zzz_pregenerated
@@ -355,6 +363,12 @@ jobs:
         timeout-minutes: 200
         runs-on: macos-latest
         if: github.actor != 'restyled-io[bot]'
+
+        env:
+            # NOTE: zap version should be kept in sync with the image used by 
+            # Docker, currently connectedhomeip/chip-build:0.6.17 
+            ZAP_VERSION: v2022.11.29-nightly
+            ZAP_INSTALL_PATH: /usr/local/zap/zap-v2022.11.29-nightly
 
         steps:
             - uses: Wandalen/wretry.action@v1.0.36
@@ -385,6 +399,16 @@ jobs:
                   ls -la /usr/local/Cellar/openssl@1.1
                   OPEN_SSL_VERSION=`ls -la /usr/local/Cellar/openssl@1.1 | cat | tail -n1 | awk '{print $NF}'`
                   ln -s /usr/local/Cellar/openssl@1.1/$OPEN_SSL_VERSION/lib/pkgconfig/* .
+            - name: Install zap
+              run: |
+                 sudo mkdir -p $ZAP_INSTALL_PATH
+                 sudo chown `whoami` $ZAP_INSTALL_PATH
+                 curl -L https://github.com/project-chip/zap/releases/download/${ZAP_VERSION}/zap-mac.zip \
+                      --output $ZAP_INSTALL_PATH/zap-mac.zip
+                 cd $ZAP_INSTALL_PATH
+                 unzip zap-mac.zip zap-cli
+                 rm zap-mac.zip
+                 ./zap-cli --version
             - name: Bootstrap
               timeout-minutes: 25
               run: scripts/build/gn_bootstrap.sh

.github/workflows/darwin-tests.yaml

@@ -39,6 +39,10 @@ jobs:
         env:
             BUILD_VARIANT: ${{matrix.build_variant}}
             LSAN_OPTIONS: detect_leaks=1 malloc_context_size=40 suppressions=scripts/tests/chiptest/lsan-mac-suppressions.txt
+            # NOTE: zap version should be kept in sync with the image used by 
+            # Docker, currently connectedhomeip/chip-build:0.6.17 
+            ZAP_VERSION: v2022.11.29-nightly
+            ZAP_INSTALL_PATH: /usr/local/zap/zap-v2022.11.29-nightly
 
         if: github.actor != 'restyled-io[bot]'
         runs-on: macos-latest
@@ -72,6 +76,16 @@ jobs:
                   ls -la /usr/local/Cellar/openssl@1.1
                   OPEN_SSL_VERSION=`ls -la /usr/local/Cellar/openssl@1.1 | cat | tail -n1 | awk '{print $NF}'`
                   ln -s /usr/local/Cellar/openssl@1.1/$OPEN_SSL_VERSION/lib/pkgconfig/* .
+            - name: Install zap
+              run: |
+                 sudo mkdir -p $ZAP_INSTALL_PATH
+                 sudo chown `whoami` $ZAP_INSTALL_PATH
+                 curl -L https://github.com/project-chip/zap/releases/download/${ZAP_VERSION}/zap-mac.zip \
+                      --output $ZAP_INSTALL_PATH/zap-mac.zip
+                 cd $ZAP_INSTALL_PATH
+                 unzip zap-mac.zip zap-cli
+                 rm zap-mac.zip
+                 ./zap-cli --version
             - name: Bootstrap
               timeout-minutes: 25
               run: scripts/build/gn_bootstrap.sh

.github/workflows/darwin.yaml

@@ -31,6 +31,12 @@ jobs:
         if: github.actor != 'restyled-io[bot]'
         runs-on: macos-latest
 
+        env:
+          # NOTE: zap version should be kept in sync with the image used by 
+            # Docker, currently connectedhomeip/chip-build:0.6.17 
+            ZAP_VERSION: v2022.11.29-nightly
+            ZAP_INSTALL_PATH: /usr/local/zap/zap-v2022.11.29-nightly
+
         steps:
             - uses: Wandalen/wretry.action@v1.0.36
               name: Checkout
@@ -52,6 +58,16 @@ jobs:
                   ls -la /usr/local/Cellar/openssl@1.1
                   OPEN_SSL_VERSION=`ls -la /usr/local/Cellar/openssl@1.1 | cat | tail -n1 | awk '{print $NF}'`
                   ln -s /usr/local/Cellar/openssl@1.1/$OPEN_SSL_VERSION/lib/pkgconfig/* .
+            - name: Install zap
+              run: |
+                 sudo mkdir -p $ZAP_INSTALL_PATH
+                 sudo chown `whoami` $ZAP_INSTALL_PATH
+                 curl -L https://github.com/project-chip/zap/releases/download/${ZAP_VERSION}/zap-mac.zip \
+                      --output $ZAP_INSTALL_PATH/zap-mac.zip
+                 cd $ZAP_INSTALL_PATH
+                 unzip zap-mac.zip zap-cli
+                 rm zap-mac.zip
+                 ./zap-cli --version
             - name: Bootstrap
               timeout-minutes: 25
               run: scripts/build/gn_bootstrap.sh

.github/workflows/examples-esp32.yaml

@@ -93,6 +93,14 @@ jobs:
                         build \
                         --copy-artifacts-to out/artifacts \
                      "
+            - name: Check no code generation in output
+              run: |
+                  CNT=$(find -name "CHIPClusters.h" out | wc -l)
+                  if [ "${CNT}" != "0" ]; then
+                     echo "ERROR: found unexpected generated files:"
+                     find -name "CHIPClusters.h" out
+                     exit 1
+                  fi
             - name: Undo code pregeneration changes
               run: |
                   rm -rf ./zzz_pregenerated

.github/workflows/fuzzing-build.yaml

@@ -88,6 +88,12 @@ jobs:
         runs-on: macos-latest
         if: github.actor != 'restyled-io[bot]'
 
+        env:
+          # NOTE: zap version should be kept in sync with the image used by 
+            # Docker, currently connectedhomeip/chip-build:0.6.17 
+            ZAP_VERSION: v2022.11.29-nightly
+            ZAP_INSTALL_PATH: /usr/local/zap/zap-v2022.11.29-nightly
+
         steps:
             - uses: Wandalen/wretry.action@v1.0.36
               name: Checkout
@@ -111,6 +117,16 @@ jobs:
                   ls -la /usr/local/Cellar/openssl@1.1
                   OPEN_SSL_VERSION=`ls -la /usr/local/Cellar/openssl@1.1 | cat | tail -n1 | awk '{print $NF}'`
                   ln -s /usr/local/Cellar/openssl@1.1/$OPEN_SSL_VERSION/lib/pkgconfig/* .
+            - name: Install zap
+              run: |
+                 sudo mkdir -p $ZAP_INSTALL_PATH
+                 sudo chown `whoami` $ZAP_INSTALL_PATH
+                 curl -L https://github.com/project-chip/zap/releases/download/${ZAP_VERSION}/zap-mac.zip \
+                      --output $ZAP_INSTALL_PATH/zap-mac.zip
+                 cd $ZAP_INSTALL_PATH
+                 unzip zap-mac.zip zap-cli
+                 rm zap-mac.zip
+                 ./zap-cli --version
             - name: Bootstrap
               timeout-minutes: 25
               run: scripts/build/gn_bootstrap.sh

.github/workflows/tests.yaml

@@ -242,6 +242,10 @@ jobs:
             CHIP_TOOL_VARIANT: ${{matrix.chip_tool}}
             TSAN_OPTIONS: "halt_on_error=1"
             LSAN_OPTIONS: detect_leaks=1 suppressions=scripts/tests/chiptest/lsan-mac-suppressions.txt
+            # NOTE: zap version should be kept in sync with the image used by 
+            # Docker, currently connectedhomeip/chip-build:0.6.17 
+            ZAP_VERSION: v2022.11.29-nightly
+            ZAP_INSTALL_PATH: /usr/local/zap/zap-v2022.11.29-nightly
 
         if: github.actor != 'restyled-io[bot]'
         runs-on: macos-latest
@@ -275,6 +279,15 @@ jobs:
                   ls -la /usr/local/Cellar/openssl@1.1
                   OPEN_SSL_VERSION=`ls -la /usr/local/Cellar/openssl@1.1 | cat | tail -n1 | awk '{print $NF}'`
                   ln -s /usr/local/Cellar/openssl@1.1/$OPEN_SSL_VERSION/lib/pkgconfig/* .
+            - name: Install zap
+              run: |
+                 sudo mkdir -p $ZAP_INSTALL_PATH
+                 sudo chown `whoami` $ZAP_INSTALL_PATH
+                 curl -L https://github.com/project-chip/zap/releases/download/${ZAP_VERSION}/zap-mac.zip \
+                      --output $ZAP_INSTALL_PATH/zap-mac.zip
+                 cd $ZAP_INSTALL_PATH
+                 unzip zap-mac.zip zap-cli
+                 rm zap-mac.zip
             - name: Bootstrap
               timeout-minutes: 25
               run: scripts/build/gn_bootstrap.sh
@@ -287,7 +300,7 @@ jobs:
                       .environment/gn_out/.ninja_log
                       .environment/pigweed-venv/*.log
             - name: Build Apps
-              timeout-minutes: 60
+              timeout-minutes: 90
               run: |
                   ./scripts/run_in_build_env.sh \
                      "./scripts/build/build_examples.py \
@@ -501,6 +514,10 @@ jobs:
         env:
             BUILD_VARIANT: ${{matrix.build_variant}}
             TSAN_OPTIONS: "halt_on_error=1"
+            # NOTE: zap version should be kept in sync with the image used by 
+            # Docker, currently connectedhomeip/chip-build:0.6.17 
+            ZAP_VERSION: v2022.11.29-nightly
+            ZAP_INSTALL_PATH: /usr/local/zap/zap-v2022.11.29-nightly
 
         if: github.actor != 'restyled-io[bot]' && false
         runs-on: macos-latest
@@ -528,6 +545,15 @@ jobs:
                   ls -la /usr/local/Cellar/openssl@1.1
                   OPEN_SSL_VERSION=`ls -la /usr/local/Cellar/openssl@1.1 | cat | tail -n1 | awk '{print $NF}'`
                   ln -s /usr/local/Cellar/openssl@1.1/$OPEN_SSL_VERSION/lib/pkgconfig/* .
+            - name: Install zap
+              run: |
+                 sudo mkdir -p $ZAP_INSTALL_PATH
+                 sudo chown `whoami` $ZAP_INSTALL_PATH
+                 curl -L https://github.com/project-chip/zap/releases/download/${ZAP_VERSION}/zap-mac.zip \
+                      --output $ZAP_INSTALL_PATH/zap-mac.zip
+                 cd $ZAP_INSTALL_PATH
+                 unzip zap-mac.zip zap-cli
+                 rm zap-mac.zip
             - name: Bootstrap
               timeout-minutes: 25
               run: scripts/build/gn_bootstrap.sh

BUILD.gn

@@ -263,6 +263,10 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") {
         "//scripts/idl:idl.tests",
         "//src:tests_run",
       ]
+
+      if (current_os == "linux") {
+        deps += [ "${chip_root}/scripts/tools/zap:tests" ]
+      }
     }
   }
 } else {

build/chip/chip_codegen.cmake

@@ -92,6 +92,140 @@ function(chip_codegen TARGET_NAME)
         endforeach()
 
 
+        set(${ARG_OUTPUT_FILES} "${OUT_NAMES}" PARENT_SCOPE)
+        set(${ARG_OUTPUT_PATH} "${GEN_FOLDER}" PARENT_SCOPE)
+
+        # allow adding dependencies to a phony target since no codegen is done
+        add_custom_target(${TARGET_NAME})
+    endif()
+endfunction()
+
+# Run chip code generation using zap
+#
+# Example usage:
+#   chip_codegen("app"
+#      INPUT     "some_file.zap"
+#      GENERATOR "app-templates"
+#      OUTPUTS
+#            "zap-generated/access.h",
+#            "zap-generated/CHIPClientCallbacks.h",
+#            "zap-generated/CHIPClusters.h"
+#            "zap-generated/endpoint_config.h",
+#            "zap-generated/gen_config.h",
+#            "zap-generated/IMClusterCommandHandler.cpp"
+#     OUTPUT_PATH  DIR_NAME_VAR
+#     OUTPUT_FILES  FILE_NAMES_VAR
+#   )
+#
+# Arguments:
+#   INPUT     - the name of the ".zap" file to use for generation
+#   GENERATOR - generator to use, like "app-templates"
+#   OUTPUTS   - EXPECTED output names
+#
+#   OUTPUT_PATH  - [OUT] output variable will contain the directory where the
+#                  files will be generated
+#
+#   OUTPUT_FILES - [OUT] output variable will contain the path of generated files.
+#                  suitable to be added within a build target
+#
+function(chip_zapgen TARGET_NAME)
+    cmake_parse_arguments(ARG
+         ""
+         "INPUT;GENERATOR;OUTPUT_PATH;OUTPUT_FILES"
+         "OUTPUTS"
+         ${ARGN}
+    )
+
+    set(CHIP_CODEGEN_PREGEN_DIR "" CACHE PATH "Pre-generated directory to use instead of compile-time code generation.")
+
+    if ("${CHIP_CODEGEN_PREGEN_DIR}" STREQUAL "")
+        set(GEN_FOLDER "${CMAKE_BINARY_DIR}/gen/${TARGET_NAME}/zapgen/${ARG_GENERATOR}")
+
+        string(REPLACE ";" "\n" OUTPUT_AS_NEWLINES "${ARG_OUTPUTS}")
+
+        file(MAKE_DIRECTORY "${GEN_FOLDER}")
+        file(GENERATE
+            OUTPUT "${GEN_FOLDER}/expected.outputs"
+            CONTENT "${OUTPUT_AS_NEWLINES}"
+        )
+
+
+        set(OUT_NAMES)
+        foreach(NAME IN LISTS ARG_OUTPUTS)
+            list(APPEND OUT_NAMES "${GEN_FOLDER}/${NAME}")
+        endforeach()
+
+        if ("${ARG_GENERATOR}" STREQUAL "app-templates")
+            SET(TEMPLATE_PATH "${CHIP_ROOT}/src/app/zap-templates/app-templates.json")
+
+            # TODO: unclear how to maintain these: there is no parser that can figure
+            #       out links of template files and zap files and such
+            SET(EXTRA_DEPENDENCIES
+                "${CHIP_ROOT}/src/app/zap-templates/partials/header.zapt"
+                "${CHIP_ROOT}/src/app/zap-templates/partials/idl/command_request_response.zapt"
+                "${CHIP_ROOT}/src/app/zap-templates/partials/idl/command_request_struct.zapt"
+                "${CHIP_ROOT}/src/app/zap-templates/partials/idl/command_response_struct.zapt"
+                "${CHIP_ROOT}/src/app/zap-templates/partials/idl/structure_definition.zapt"
+                "${CHIP_ROOT}/src/app/zap-templates/partials/idl/structure_member.zapt"
+                "${CHIP_ROOT}/src/app/zap-templates/partials/im_command_handler_cluster_commands.zapt"
+                "${CHIP_ROOT}/src/app/zap-templates/templates/app/access.zapt"
+                "${CHIP_ROOT}/src/app/zap-templates/templates/app/CHIPClientCallbacks.zapt"
+                "${CHIP_ROOT}/src/app/zap-templates/templates/app/CHIPClusters.zapt"
+                "${CHIP_ROOT}/src/app/zap-templates/templates/app/endpoint_config.zapt"
+                "${CHIP_ROOT}/src/app/zap-templates/templates/app/gen_config.zapt"
+                "${CHIP_ROOT}/src/app/zap-templates/templates/app/im-cluster-command-handler.zapt"
+                "${CHIP_ROOT}/src/app/zap-templates/templates/app/MatterIDL.zapt"
+           )
+           SET(OUTPUT_SUBDIR "zap-generated")
+        else()
+            message(SEND_ERROR "Unsupported zap generator: ${ARG_GENERATOR}")
+        endif()
+
+        # Python is expected to be in the path
+        #
+        # find_package(Python3 REQUIRED)
+        add_custom_command(
+            OUTPUT ${OUT_NAMES}
+            COMMAND "${CHIP_ROOT}/scripts/tools/zap/generate.py"
+            ARGS
+                "--no-prettify-output"
+                "--templates" "${TEMPLATE_PATH}"
+                "--output-dir" "${GEN_FOLDER}/${OUTPUT_SUBDIR}"
+                "--parallel"
+                "${ARG_INPUT}"
+            DEPENDS
+                "${ARG_INPUT}"
+                ${EXTRA_DEPENDENCIES}
+            VERBATIM
+        )
+
+        add_custom_target(${TARGET_NAME} DEPENDS "${OUT_NAMES}")
+
+        # Forward outputs to the parent
+        set(${ARG_OUTPUT_FILES} "${OUT_NAMES}" PARENT_SCOPE)
+        set(${ARG_OUTPUT_PATH} "${GEN_FOLDER}" PARENT_SCOPE)
+    else()
+        # Gets a path such as:
+        #    examples/lock-app/lock-common/lock-app.zap
+        file(RELATIVE_PATH MATTER_FILE_PATH "${CHIP_ROOT}" ${ARG_INPUT})
+
+        # Removes the trailing file extension to get something like:
+        #    examples/lock-app/lock-common/lock-app
+        string(REGEX REPLACE "\.zap$" "" CODEGEN_DIR_PATH "${MATTER_FILE_PATH}")
+
+
+        # Build the final location within the pregen directory
+        set(GEN_FOLDER "${CHIP_CODEGEN_PREGEN_DIR}/${CODEGEN_DIR_PATH}/zap/${ARG_GENERATOR}")
+
+        # TODO: build a fake target of ${TARGET_NAME}
+
+        # Here we have ${CHIP_CODEGEN_PREGEN_DIR}
+        set(OUT_NAMES)
+        foreach(NAME IN LISTS ARG_OUTPUTS)
+            list(APPEND OUT_NAMES "${GEN_FOLDER}/${NAME}")
+        endforeach()
+
+
         set(${ARG_OUTPUT_FILES} "${OUT_NAMES}" PARENT_SCOPE)
         set(${ARG_OUTPUT_PATH} "${GEN_FOLDER}" PARENT_SCOPE)