Cherry pick - Chef changes (06-27:07-01). RPC logging, Linux cross pl…

…atform building. (#20248)

* Put software version string as <branch>:<commit_id> on esp32 while doing automation testing (#19943)

* RPC: Support pw logging for linux rpc examples (#20057)

Add a config option CHIP_USE_PW_LOGGING, which uses pw log output
instead of printf on linux. This allows the logs to correctly get
HDLC encoded and be piped over the socket for linux RPC builds.

* Chef - Continue instead of breaking on build error (#19988)

* Move chef .matter files back to /chef/devices (#20142)

* Remove matter files from root zzz

* Move .matter regen for chef

* Update bundle for changed .matter location

* Chef - Support cross platform Linux builds for arm64 (#20232)

* Chef - Use array for passing linux args to args.gni

* Chef - Add support for cross compiling linux on arm64

Co-authored-by: MtTsai <7930801+MtTsai@users.noreply.github.com>
Co-authored-by: rgoliver <rgoliver@google.com>
Co-authored-by: Austin Bozowski <bozowski@google.com>

examples/chef/chef.py

@@ -150,10 +150,7 @@ def bundle(platform: str, device_name: str) -> None:
         flush_print(f"No bundle function for {platform}!")
         exit(1)
     flush_print(f"Copying {matter_file}")
-    src_item = os.path.join(_REPO_BASE_PATH,
-                            "zzz_generated",
-                            "chef-"+device_name,
-                            "zap-generated",
+    src_item = os.path.join(_DEVICE_FOLDER,
                             matter_file)
     dest_item = os.path.join(_CD_STAGING_DIR, matter_file)
     shutil.copy(src_item, dest_item)
@@ -322,8 +319,15 @@ def main(argv: Sequence[str]) -> None:
                       dest="use_zzz", action="store_true")
     parser.add_option("", "--build_all", help="For use in CD only. Builds and bundles all chef examples for the specified platform. Uses --use_zzz. Chef exits after completion.",
                       dest="build_all", action="store_true")
+    parser.add_option("-k", "--keep_going", help="For use in CD only. Continues building all sample apps in the event of an error.",
+                      dest="keep_going", action="store_true")
     parser.add_option(
         "", "--ci", help="Builds Chef examples defined in cicd_config. Uses --use_zzz. Uses specified target from -t. Chef exits after completion.", dest="ci", action="store_true")
+    parser.add_option(
+        "", "--ipv6only", help="Compile build which only supports ipv6. Linux only.",
+        action="store_true")
+    parser.add_option(
+        "", "--cpu_type", help="CPU type to compile for. Linux only.", choices=["arm64", "x64"])
 
     options, _ = parser.parse_args(argv)
 
@@ -387,13 +391,17 @@ def main(argv: Sequence[str]) -> None:
                 except RuntimeError as build_fail_error:
                     failed_builds.append((device_name, platform, "build"))
                     flush_print(str(build_fail_error))
-                    break
+                    if not options.keep_going:
+                        exit(1)
+                    continue
                 try:
                     bundle(platform, device_name)
                 except FileNotFoundError as bundle_fail_error:
                     failed_builds.append((device_name, platform, "bundle"))
                     flush_print(str(bundle_fail_error))
-                    break
+                    if not options.keep_going:
+                        exit(1)
+                    continue
                 archive_name = f"{label}-{device_name}"
                 archive_full_name = archive_prefix + archive_name + archive_suffix
                 flush_print(f"Adding build output to archive {archive_full_name}")
@@ -501,6 +509,7 @@ def main(argv: Sequence[str]) -> None:
     #
 
     if options.do_build:
+        sw_ver_string = ""
         if options.do_automated_test_stamp:
             branch = ""
             for branch_text in shell.run_cmd("git branch", return_cmd_output=True).split("\n"):
@@ -557,7 +566,8 @@ def main(argv: Sequence[str]) -> None:
                         set(CONFIG_DEVICE_VENDOR_ID {options.vid})
                         set(CONFIG_DEVICE_PRODUCT_ID {options.pid})
                         set(CONFIG_ENABLE_PW_RPC {"1" if options.do_rpc else "0"})
-                        set(SAMPLE_NAME {options.sample_device_type_name})"""))
+                        set(SAMPLE_NAME {options.sample_device_type_name})
+                        set(CHIP_DEVICE_CONFIG_DEVICE_SOFTWARE_VERSION_STRING \"{sw_ver_string}\")"""))
 
         if options.build_target == "esp32":
             shell.run_cmd(f"cd {_CHEF_SCRIPT_PATH}/esp32")
@@ -591,29 +601,56 @@ def main(argv: Sequence[str]) -> None:
 
         elif options.build_target == "linux":
             shell.run_cmd(f"cd {_CHEF_SCRIPT_PATH}/linux")
+
+            linux_args = []
+            if options.do_rpc:
+                linux_args.append('import("//with_pw_rpc.gni")')
+            linux_args.extend([
+                'import("//build_overrides/chip.gni")',
+                'import("${chip_root}/config/standalone/args.gni")',
+                'chip_shell_cmd_server = false',
+                'chip_build_libshell = true',
+                'chip_config_network_layer_ble = false',
+                f'target_defines = ["CHIP_DEVICE_CONFIG_DEVICE_VENDOR_ID={options.vid}", "CHIP_DEVICE_CONFIG_DEVICE_PRODUCT_ID={options.pid}", "CONFIG_ENABLE_PW_RPC={int(options.do_rpc)}"]',
+            ])
+            if options.cpu_type == "arm64":
+                uname_resp = shell.run_cmd("uname -m", return_cmd_output=True)
+                if "aarch" not in uname_resp and "arm" not in uname_resp:
+                    if (
+                            "aarch" not in uname_resp and
+                            "arm" not in uname_resp and
+                            "SYSROOT_AARCH64" not in shell.env):
+                        flush_print(
+                            "SYSROOT_AARCH64 env variable not set. "
+                            "AARCH64 toolchain needed for cross-compiling for arm64.")
+                        exit(1)
+                    shell.env["PKG_CONFIG_PATH"] = (
+                        f'{shell.env["SYSROOT_AARCH64"]}/lib/aarch64-linux-gnu/pkgconfig')
+                linux_args.append('target_cpu="arm64"')
+                linux_args.append('is_clang=true')
+                linux_args.append('chip_crypto="mbedtls"')
+                linux_args.append(f'sysroot="{shell.env["SYSROOT_AARCH64"]}"')
+            elif options.cpu_type == "x64":
+                uname_resp = shell.run_cmd("uname -m", return_cmd_output=True)
+                if "x64" not in uname_resp and "x86_64" not in uname_resp:
+                    flush_print(f"Unable to cross compile for x64 on {uname_resp}")
+                    exit(1)
+            if options.ipv6only:
+                linux_args.append("chip_inet_config_enable_ipv4=false")
+
+            if sw_ver_string:
+                linux_args.append(
+                    f'chip_device_config_device_software_version_string = "{sw_ver_string}"')
             with open(f"{_CHEF_SCRIPT_PATH}/linux/args.gni", "w") as f:
-                sw_ver_string_config_text = f"chip_device_config_device_software_version_string = \"{sw_ver_string}\"" if options.do_automated_test_stamp else ""
-                f.write(textwrap.dedent(f"""\
-                        import("//build_overrides/chip.gni")
-                        import("${{chip_root}}/config/standalone/args.gni")
-                        chip_shell_cmd_server = false
-                        chip_build_libshell = true
-                        chip_config_network_layer_ble = false
-                        target_defines = ["CHIP_DEVICE_CONFIG_DEVICE_VENDOR_ID={options.vid}", "CHIP_DEVICE_CONFIG_DEVICE_PRODUCT_ID={options.pid}", "CONFIG_ENABLE_PW_RPC={'1' if options.do_rpc else '0'}"]
-                        {sw_ver_string_config_text}
-                        """))
+                f.write("\n".join(linux_args))
             with open(f"{_CHEF_SCRIPT_PATH}/linux/sample.gni", "w") as f:
                 f.write(textwrap.dedent(f"""\
                         sample_zap_file = "{options.sample_device_type_name}.zap"
                         sample_name = "{options.sample_device_type_name}"
                         """))
             if options.do_clean:
                 shell.run_cmd(f"rm -rf out")
-            if options.do_rpc:
-                shell.run_cmd(
-                    "gn gen out --args='import(\"//with_pw_rpc.gni\")'")
-            else:
-                shell.run_cmd("gn gen out --args=''")
+            shell.run_cmd("gn gen out")
             shell.run_cmd("ninja -C out")
 
     #

examples/chef/esp32/CMakeLists.txt

@@ -37,8 +37,12 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 include(${CMAKE_CURRENT_LIST_DIR}/../project_include.cmake)
 message(STATUS "Product ID " ${CONFIG_DEVICE_PRODUCT_ID})
 message(STATUS "Vendor ID " ${CONFIG_DEVICE_VENDOR_ID})
+message(STATUS "SW Version String" ${CHIP_DEVICE_CONFIG_DEVICE_SOFTWARE_VERSION_STRING})
 idf_build_set_property(COMPILE_OPTIONS "-DCHIP_DEVICE_CONFIG_DEVICE_PRODUCT_ID=${CONFIG_DEVICE_PRODUCT_ID}" APPEND)
 idf_build_set_property(COMPILE_OPTIONS "-DCHIP_DEVICE_CONFIG_DEVICE_VENDOR_ID=${CONFIG_DEVICE_VENDOR_ID}" APPEND)
+if(NOT ${CHIP_DEVICE_CONFIG_DEVICE_SOFTWARE_VERSION_STRING} STREQUAL "")
+    idf_build_set_property(COMPILE_OPTIONS "-DCHIP_DEVICE_CONFIG_DEVICE_SOFTWARE_VERSION_STRING=\"${CHIP_DEVICE_CONFIG_DEVICE_SOFTWARE_VERSION_STRING}\"" APPEND)
+endif()
 idf_build_set_property(COMPILE_OPTIONS "-DCHIP_PLATFORM_ESP32=1" APPEND)
 
 project(chip-shell)

examples/chef/linux/with_pw_rpc.gni

@@ -41,3 +41,4 @@ pw_build_LINK_DEPS = [
 
 chip_enable_pw_rpc = true
 chip_build_pw_trace_lib = true
+chip_use_pw_logging = true

examples/common/pigweed/rpc_console/py/chip_rpc/console.py

@@ -253,7 +253,8 @@ def write_to_output(data: bytes,
                      "E": logging.ERROR, "F": logging.FATAL, "V": logging.DEBUG, "D": logging.DEBUG,
                      "<inf>": logging.INFO, "<dbg>": logging.DEBUG, "<err>": logging.ERROR,
                      "<info  >": logging.INFO, "<warn  >": logging.WARNING,
-                     "<error >": logging.ERROR, "<detail>": logging.DEBUG}
+                     "<error >": logging.ERROR, "<detail>": logging.DEBUG,
+                     "ERR": logging.ERROR, "DBG": logging.DEBUG, "INF": logging.INFO}
 
     ESP_CHIP_REGEX = r"(?P<level>[IWEFV]) \((?P<time>\d+)\) (?P<mod>chip\[[a-zA-Z]+\]):\s(?P<msg>.*)"
     ESP_APP_REGEX = r"(?P<level>[IWEFVD]) \((?P<time>\d+)\) (?P<mod>[a-z\-_A-Z]+):\s(?P<msg>.*)"
@@ -267,14 +268,17 @@ def write_to_output(data: bytes,
     NXP_CHIP_REGEX = r"\[(?P<time>\d+)\]\[(?P<level>[EPDF])\]\[(?P<mod>[a-z\-A-Z]+)\](?P<msg>.*)"
     NXP_APP_REGEX = r"\[(?P<time>\d+)\]\[(?P<mod>[a-z\-A-Z]+)\](?P<msg>.*)"
 
+    LINUX_REGEX = r".*(?P<level>INF|DBG|ERR).*\s+\[(?P<time>[0-9]+\.?[0-9]*)\]\[(?P<pid>\d+)\:(?P<tid>\d+)\] CHIP:(?P<mod>[a-z\-A-Z]+)\: (?P<msg>.*)"
+
     LogRegexes = [RegexStruct("ESP", "CHIP", re.compile(ESP_CHIP_REGEX), 4),
                   RegexStruct("ESP", "APP", re.compile(ESP_APP_REGEX), 4),
                   RegexStruct("EFR", "CHIP", re.compile(EFR_CHIP_REGEX), 3),
                   RegexStruct("EFR", "APP", re.compile(EFR_APP_REGEX), 1),
                   RegexStruct("NRF", "CHIP", re.compile(NRF_CHIP_REGEX), 4),
                   RegexStruct("NRF", "APP", re.compile(NRF_APP_REGEX), 3),
                   RegexStruct("NXP", "CHIP", re.compile(NXP_CHIP_REGEX), 4),
-                  RegexStruct("NXP", "APP", re.compile(NXP_APP_REGEX), 3)
+                  RegexStruct("NXP", "APP", re.compile(NXP_APP_REGEX), 3),
+                  RegexStruct("LINUX", "CHIP", re.compile(LINUX_REGEX), 6)
                   ]
     for line in log_line.decode(errors="surrogateescape").splitlines():
         fields = {'level': logging.INFO, "time": "",

examples/lighting-app/linux/with_pw_rpc.gni

@@ -41,3 +41,4 @@ pw_build_LINK_DEPS = [
 
 chip_enable_pw_rpc = true
 chip_build_pw_trace_lib = true
+chip_use_pw_logging = true

integrations/cloudbuild/chef.yaml

@@ -17,7 +17,7 @@ steps:
           - PW_ENVIRONMENT_ROOT=/pwenv
       args:
           - >-
-              ./examples/chef/chef.py --build_all
+              ./examples/chef/chef.py --build_all --keep_going
       id: CompileAll
       waitFor:
           - Bootstrap

scripts/tools/zap_regen_all.py

@@ -71,7 +71,10 @@ def generate(self):
             with open(af_gen_event, "w+"):  # Empty file needed for linux
                 pass
             idl_path = self.zap_config.replace(".zap", ".matter")
-            target_path = os.path.join(self.output_dir, os.path.basename(idl_path))
+            target_path = os.path.join("examples",
+                                       "chef",
+                                       "devices",
+                                       os.path.basename(idl_path))
             os.rename(idl_path, target_path)
 
 

src/lib/core/BUILD.gn

@@ -46,6 +46,7 @@ buildconfig_header("chip_buildconfig") {
     "CHIP_DETAIL_LOGGING=${chip_detail_logging}",
     "CHIP_AUTOMATION_LOGGING=${chip_automation_logging}",
     "CHIP_PW_TOKENIZER_LOGGING=${chip_pw_tokenizer_logging}",
+    "CHIP_USE_PW_LOGGING=${chip_use_pw_logging}",
     "CHIP_CONFIG_SHORT_ERROR_STR=${chip_config_short_error_str}",
     "CHIP_CONFIG_ENABLE_ARG_PARSER=${chip_config_enable_arg_parser}",
     "CHIP_TARGET_STYLE_UNIX=${chip_target_style_unix}",

src/lib/core/core.gni

@@ -36,6 +36,9 @@ declare_args() {
   # Enable pigweed tokenizer logging.
   chip_pw_tokenizer_logging = false
 
+  # Configure chip logging to output through pigweed logging.
+  chip_use_pw_logging = false
+
   # Enable short error strings.
   chip_config_short_error_str = false
 

src/platform/Linux/BUILD.gn

@@ -17,10 +17,15 @@ import("//build_overrides/chip.gni")
 
 import("${build_root}/config/linux/pkg_config.gni")
 
+import("${chip_root}/src/lib/core/core.gni")
 import("${chip_root}/src/platform/device.gni")
 
 assert(chip_device_platform == "linux")
 
+if (chip_use_pw_logging) {
+  import("//build_overrides/pigweed.gni")
+}
+
 if (chip_enable_openthread) {
   import("//build_overrides/openthread.gni")
   import("//build_overrides/ot_br_posix.gni")
@@ -122,6 +127,10 @@ static_library("Linux") {
     public_deps += [ "dbus/openthread" ]
   }
 
+  if (chip_use_pw_logging) {
+    deps += [ "$dir_pw_log" ]
+  }
+
   if (chip_enable_wifi) {
     public_deps += [ "dbus/wpa" ]
   }

src/platform/Linux/Logging.cpp

@@ -1,15 +1,21 @@
 /* See Project CHIP LICENSE file for licensing information. */
 
+#include <lib/core/CHIPConfig.h>
 #include <lib/support/EnforceFormat.h>
 #include <lib/support/logging/Constants.h>
 #include <platform/logging/LogV.h>
 
 #include <cinttypes>
 #include <cstdio>
+#include <cstring>
 #include <sys/syscall.h>
 #include <sys/time.h>
 #include <unistd.h>
 
+#if CHIP_USE_PW_LOGGING
+#include <pw_log/log.h>
+#endif // CHIP_USE_PW_LOGGING
+
 namespace chip {
 namespace DeviceLayer {
 
@@ -37,6 +43,7 @@ void ENFORCE_FORMAT(3, 0) LogV(const char * module, uint8_t category, const char
     // indicate the error occurred during getting time.
     gettimeofday(&tv, nullptr);
 
+#if !CHIP_USE_PW_LOGGING
     // Lock standard output, so a single log line will not be corrupted in case
     // where multiple threads are using logging subsystem at the same time.
     flockfile(stdout);
@@ -48,6 +55,30 @@ void ENFORCE_FORMAT(3, 0) LogV(const char * module, uint8_t category, const char
     fflush(stdout);
 
     funlockfile(stdout);
+#else  // !CHIP_USE_PW_LOGGING
+    char formattedMsg[CHIP_CONFIG_LOG_MESSAGE_MAX_SIZE];
+    snprintf(formattedMsg, sizeof(formattedMsg),
+             "[%" PRIu64 ".%06" PRIu64 "][%lld:%lld] CHIP:%s: ", static_cast<uint64_t>(tv.tv_sec),
+             static_cast<uint64_t>(tv.tv_usec), static_cast<long long>(syscall(SYS_getpid)),
+             static_cast<long long>(syscall(SYS_gettid)), module);
+    size_t len = strnlen(formattedMsg, sizeof(formattedMsg));
+    vsnprintf(formattedMsg + len, sizeof(formattedMsg) - len, msg, v);
+
+    switch (static_cast<LogCategory>(category))
+    {
+    case kLogCategory_Error:
+        PW_LOG_ERROR("%s", formattedMsg);
+        break;
+    case kLogCategory_Progress:
+        PW_LOG_INFO("%s", formattedMsg);
+        break;
+    case kLogCategory_Detail:
+    case kLogCategory_None:
+    case kLogCategory_Automation:
+        PW_LOG_DEBUG("%s", formattedMsg);
+        break;
+    }
+#endif // !CHIP_USE_PW_LOGGING
 
     // Let the application know that a log message has been emitted.
     DeviceLayer::OnLogOutput();