Move ClusterWriteMapping.java to be generated from matter.idl (#25773)

* Some initial changes

* Parser support for timed writes

* Small doc update

* Matter idl support for timed write

* Fix indent and codegen all

* Remove extra line from readme

* Some fixes

* Fix conditional in requires_timed_write

* Most codegen looks ok. Java boxing logic is suspect still

* More updates, output idential EXCEPT types for boxing

* Increase 1000 to 10000 to match original template

* Fix byte count comparison when long starts to take effect

* Fix length of underlying bitmap type sizing

* Fixed files, they are IDENTICAL

* Integrate java-jni and java-class since build rules are different between cpp and java files

* Fix python syntax

* Switch default to not have underscore

* Add java codegen via jinja to zap_regen_all

* Restyle, fix restyle logic

* Fix duplicated generation target

* Do not attempt to zap-generate Cluster write mapping

* Add golden image unit test for java codegen

* Add prettyfy for java output ... makes the input/output files more obviously identical

* Remove unused variable

* Fix upper/lowercase of acronyms, to be fully backwards compatible

* Add license blurb since we checkin generated file (and maybe jinja files should also have licenses

* Restyle

* Fix unit test

---------

Co-authored-by: Andrei Litvin <andreilitvin@google.com>

.restyled.yaml

@@ -78,6 +78,7 @@ exclude:
     - "scripts/run_codegen_targets.sh" # shellharden breaks for loops over command outputs
     - "src/darwin/Framework/CHIP/zap-generated/*" # already clang-formatted by our zap tooling
     - "zzz_generated/**/*" # already clang-formatted by our zap tooling
+    - "src/controller/java/generated/java/**/*" # not formatted: generated files
 
 
 changed_paths:

build/chip/chip_codegen.gni

@@ -263,7 +263,7 @@ template("_chip_build_time_zapgen") {
 #     The ".matter" file to use to start the code generation
 #
 #   generator
-#     Name of the generator to use (e.g. java, cpp-app)
+#     Name of the generator to use (e.g. java-jni, java-class, cpp-app)
 #
 #   outputs
 #     Explicit names of the expected outputs. Enforced to validate that
@@ -296,7 +296,7 @@ template("_chip_build_time_zapgen") {
 #
 #  chip_codegen("java-jni-generate") {
 #    input = "controller-clusters.matter"
-#    generator = "java"
+#    generator = "java-jni"
 #
 #    outputs = [
 #       "jni/IdentifyClient-ReadImpl.cpp",
@@ -358,7 +358,7 @@ template("chip_codegen") {
 #     The ".matter" file to use to start the code generation
 #
 #   generator
-#     Name of the generator to use (e.g. java, cpp-app)
+#     Name of the generator to use (e.g. java-jni, java-class, cpp-app)
 #
 #   outputs
 #     Explicit names of the expected outputs. Enforced to validate that
@@ -391,7 +391,7 @@ template("chip_codegen") {
 #
 #  chip_codegen("java-jni-generate") {
 #    input = "controller-clusters.matter"
-#    generator = "java"
+#    generator = "java-jni"
 #
 #    outputs = [
 #       "jni/IdentifyClient-ReadImpl.cpp",

scripts/codegen.py

@@ -69,7 +69,7 @@ def write_new_data(self, relative_path: str, content: str):
     help='Determines the verbosity of script output')
 @click.option(
     '--generator',
-    default='JAVA',
+    default='java-jni',
     help='What code generator to run.  The choices are: '+'|'.join(GENERATORS.keys())+'. ' +
          'When using custom, provide the plugin path using `--generator custom:<path_to_plugin>:<plugin_module_name>` syntax. ' +
     'For example, `--generator custom:./my_plugin:my_plugin_module` will load `./my_plugin/my_plugin_module/__init.py__` ' +

scripts/pregenerate/__init__.py

@@ -20,7 +20,8 @@
 from typing import Iterator, List, Optional
 
 from .types import IdlFileType, InputIdlFile
-from .using_codegen import CodegenBridgePregenerator, CodegenCppAppPregenerator, CodegenJavaPregenerator
+from .using_codegen import (CodegenBridgePregenerator, CodegenCppAppPregenerator, CodegenJavaClassPregenerator,
+                            CodegenJavaJNIPregenerator)
 from .using_zap import ZapApplicationPregenerator
 
 
@@ -77,7 +78,8 @@ def FindPregenerationTargets(sdk_root: str, filter: TargetFilter, runner):
     generators = [
         # Jinja-based codegen
         CodegenBridgePregenerator(sdk_root),
-        CodegenJavaPregenerator(sdk_root),
+        CodegenJavaJNIPregenerator(sdk_root),
+        CodegenJavaClassPregenerator(sdk_root),
         CodegenCppAppPregenerator(sdk_root),
 
         # ZAP codegen

scripts/pregenerate/using_codegen.py

@@ -73,7 +73,7 @@ def CreateTarget(self, idl: InputIdlFile, runner):
         return CodegenTarget(sdk_root=self.sdk_root, idl=idl, generator="bridge", runner=runner)
 
 
-class CodegenJavaPregenerator:
+class CodegenJavaJNIPregenerator:
     """Pregeneration logic for "java" codegen.py outputs"""
 
     def __init__(self, sdk_root):
@@ -85,7 +85,22 @@ def Accept(self, idl: InputIdlFile):
         return idl.relative_path == "src/controller/data_model/controller-clusters.matter"
 
     def CreateTarget(self, idl: InputIdlFile, runner):
-        return CodegenTarget(sdk_root=self.sdk_root, idl=idl, generator="java", runner=runner)
+        return CodegenTarget(sdk_root=self.sdk_root, idl=idl, generator="java-jni", runner=runner)
+
+
+class CodegenJavaClassPregenerator:
+    """Pregeneration logic for "java" codegen.py outputs"""
+
+    def __init__(self, sdk_root):
+        self.sdk_root = sdk_root
+
+    def Accept(self, idl: InputIdlFile):
+        # Java is highly specific, a single path is acceptable for dynamic
+        # bridge codegen
+        return idl.relative_path == "src/controller/data_model/controller-clusters.matter"
+
+    def CreateTarget(self, idl: InputIdlFile, runner):
+        return CodegenTarget(sdk_root=self.sdk_root, idl=idl, generator="java-class", runner=runner)
 
 
 class CodegenCppAppPregenerator:

scripts/py_matter_idl/BUILD.gn

@@ -56,6 +56,7 @@ pw_python_package("matter_idl") {
     "matter_idl/tests/outputs/several_clusters/bridge/SecondServer.h",
     "matter_idl/tests/outputs/several_clusters/bridge/Third.h",
     "matter_idl/tests/outputs/several_clusters/bridge/ThirdServer.h",
+    "matter_idl/tests/outputs/several_clusters/java/ClusterWriteMapping.java",
     "matter_idl/tests/outputs/several_clusters/jni/FirstClient-ReadImpl.cpp",
     "matter_idl/tests/outputs/several_clusters/jni/SecondClient-ReadImpl.cpp",
     "matter_idl/tests/outputs/several_clusters/jni/ThirdClient-ReadImpl.cpp",

scripts/py_matter_idl/files.gni

@@ -8,6 +8,7 @@ matter_idl_generator_templates = [
   "${chip_root}/scripts/py_matter_idl/matter_idl/generators/bridge/BridgeClustersGlobalStructs.jinja",
   "${chip_root}/scripts/py_matter_idl/matter_idl/generators/java/ChipClustersCpp.jinja",
   "${chip_root}/scripts/py_matter_idl/matter_idl/generators/java/ChipClustersRead.jinja",
+  "${chip_root}/scripts/py_matter_idl/matter_idl/generators/java/ClusterWriteMapping.jinja",
   "${chip_root}/scripts/py_matter_idl/matter_idl/generators/cpp/application/CallbackStubSource.jinja",
   "${chip_root}/scripts/py_matter_idl/matter_idl/generators/cpp/application/PluginApplicationCallbacksHeader.jinja",
 ]

scripts/py_matter_idl/matter_idl/generators/filters.py

@@ -24,6 +24,31 @@ def normalize_acronyms(s: str) -> str:
     return s.replace('WiFi', 'Wifi').replace('WI_FI', 'WIFI')
 
 
+def lowfirst(s: str) -> str:
+    """Make the first letter lowercase. """
+    return s[0].lower() + s[1:]
+
+
+def upfirst(s: str) -> str:
+    """Make the first letter uppercase """
+    return s[0].upper() + s[1:]
+
+
+def lowfirst_except_acronym(s: str) -> str:
+    """Make the first letter lowercase assuming the string is already in
+       CamelCase.
+
+       Differs from lowfirst because it checks the string for starting with
+       several uppercase, which is the case for acronyms (HVAC, ACL, WIFI),
+       in which case it will NOT lowercase first
+    """
+    if len(s) >= 2:
+        if s[1].isupper():
+            return s
+
+    return lowfirst(s)
+
+
 def RegisterCommonFilters(filtermap):
     """
     Register filters that are NOT considered platform-generator specific.
@@ -42,3 +67,6 @@ def RegisterCommonFilters(filtermap):
     filtermap['spinalcase'] = stringcase.spinalcase
 
     filtermap['normalize_acronyms'] = normalize_acronyms
+    filtermap['lowfirst'] = lowfirst
+    filtermap['lowfirst_except_acronym'] = lowfirst_except_acronym
+    filtermap['upfirst'] = upfirst

scripts/py_matter_idl/matter_idl/generators/java/ClusterWriteMapping.jinja

@@ -0,0 +1,67 @@
+/*
+ *
+ *    Copyright (c) 2023 Project CHIP Authors
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+package chip.devicecontroller;
+
+import chip.clusterinfo.CommandParameterInfo;
+import chip.clusterinfo.InteractionInfo;
+import chip.devicecontroller.ChipClusters.DefaultClusterCallback;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class ClusterWriteMapping {
+  public Map<String, Map<String, InteractionInfo>> getWriteAttributeMap() {
+    Map<String, Map<String, InteractionInfo>> writeAttributeMap = new HashMap<>();
+
+    {%- for cluster in clientClusters | sort(attribute='code') %}
+      {%-  set typeLookup = idl | createLookupContext(cluster) %}
+    Map<String, InteractionInfo> write{{cluster.name}}InteractionInfo = new LinkedHashMap<>();
+      {%- for attribute in cluster.attributes | sort(attribute='name') | attributesWithCallback(typeLookup) %}
+        {#- TODO: add support for struct-typed attributes -#}
+        {% if not attribute.definition.is_list and attribute.is_writable %}
+    Map<String, CommandParameterInfo> write{{cluster.name}}{{attribute.definition.name | upfirst}}CommandParams = new LinkedHashMap<String, CommandParameterInfo>();
+        {%-  set encodable = attribute.definition | asEncodable(typeLookup) %}
+    CommandParameterInfo {{cluster.name | lowfirst_except_acronym}}{{attribute.definition.name | lowfirst_except_acronym}}CommandParameterInfo =
+        new CommandParameterInfo(
+            "value", 
+            {{ encodable.boxed_java_type }}.class, {# {{asJavaType type null parent.parent.name removeGenericType=true}}.class, #}
+            {{ encodable.boxed_java_type }}.class {# {{asJavaType type null parent.parent.name underlyingType=true}}.class #}
+        );
+    write{{cluster.name}}{{attribute.definition.name | upfirst}}CommandParams.put(
+        "value",
+        {{cluster.name | lowfirst_except_acronym}}{{attribute.definition.name | lowfirst_except_acronym}}CommandParameterInfo
+    );
+    InteractionInfo write{{cluster.name}}{{attribute.definition.name | upfirst}}AttributeInteractionInfo = new InteractionInfo(
+      (cluster, callback, commandArguments) -> {
+        ((ChipClusters.{{cluster.name}}Cluster) cluster).write{{attribute.definition.name | upfirst}}Attribute(
+          (DefaultClusterCallback) callback,
+          ({{ encodable.boxed_java_type }}) commandArguments.get("value")
+          {%- if attribute.requires_timed_write -%}, 10000 {% endif %}
+        );
+      },
+      () -> new ClusterInfoMapping.DelegatedDefaultClusterCallback(),
+      write{{cluster.name}}{{attribute.definition.name | upfirst}}CommandParams
+    );
+    write{{cluster.name}}InteractionInfo.put("write{{attribute.definition.name | upfirst}}Attribute", write{{cluster.name}}{{attribute.definition.name | upfirst}}AttributeInteractionInfo);
+        {%- endif %}
+      {%- endfor %}
+    writeAttributeMap.put("{{cluster.name | lowfirst_except_acronym}}", write{{cluster.name}}InteractionInfo);
+    {%-  endfor -%}
+
+    return writeAttributeMap;
+  }
+}

scripts/py_matter_idl/matter_idl/generators/java/__init__.py

@@ -356,9 +356,11 @@ def CanGenerateSubscribe(attr: Attribute, lookup: TypeLookupContext) -> bool:
     return not lookup.is_struct_type(attr.definition.data_type.name)
 
 
-class JavaGenerator(CodeGenerator):
+class __JavaCodeGenerator(CodeGenerator):
     """
-    Generation of java code for matter.
+    Code generation for java-specific files.
+
+    Registers filters used by all java generators.
     """
 
     def __init__(self, storage: GeneratorStorage, idl: Idl, **kargs):
@@ -378,6 +380,13 @@ def __init__(self, storage: GeneratorStorage, idl: Idl, **kargs):
         self.jinja_env.filters['createLookupContext'] = CreateLookupContext
         self.jinja_env.filters['canGenerateSubscribe'] = CanGenerateSubscribe
 
+
+class JavaJNIGenerator(__JavaCodeGenerator):
+    """Generates JNI java files (i.e. C++ source/headers)."""
+
+    def __init__(self, *args, **kargs):
+        super().__init__(*args, **kargs)
+
     def internal_render_all(self):
         """
         Renders .CPP files required for JNI support.
@@ -406,3 +415,24 @@ def internal_render_all(self):
                     'typeLookup': TypeLookupContext(self.idl, cluster),
                 }
             )
+
+
+class JavaClassGenerator(__JavaCodeGenerator):
+    """Generates .java files """
+
+    def __init__(self, *args, **kargs):
+        super().__init__(*args, **kargs)
+
+    def internal_render_all(self):
+        """
+        Renders .java files required for java matter support
+        """
+
+        self.internal_render_one_output(
+            template_path="java/ClusterWriteMapping.jinja",
+            output_file_name="java/chip/devicecontroller/ClusterWriteMapping.java",
+            vars={
+                'idl': self.idl,
+                'clientClusters': [c for c in self.idl.clusters if c.side == ClusterSide.CLIENT],
+            }
+        )

scripts/py_matter_idl/matter_idl/generators/registry.py

@@ -17,7 +17,7 @@
 
 from matter_idl.generators.bridge import BridgeGenerator
 from matter_idl.generators.cpp.application import CppApplicationGenerator
-from matter_idl.generators.java import JavaGenerator
+from matter_idl.generators.java import JavaClassGenerator, JavaJNIGenerator
 
 
 class CodeGenerator(enum.Enum):
@@ -26,14 +26,17 @@ class CodeGenerator(enum.Enum):
     the simple enum value (user friendly and can be a command line input)
     into underlying generators.
     """
-    JAVA = enum.auto()
+    JAVA_JNI = enum.auto()
+    JAVA_CLASS = enum.auto()
     BRIDGE = enum.auto()
     CPP_APPLICATION = enum.auto()
     CUSTOM = enum.auto()
 
     def Create(self, *args, **kargs):
-        if self == CodeGenerator.JAVA:
-            return JavaGenerator(*args, **kargs)
+        if self == CodeGenerator.JAVA_JNI:
+            return JavaJNIGenerator(*args, **kargs)
+        elif self == CodeGenerator.JAVA_CLASS:
+            return JavaClassGenerator(*args, **kargs)
         elif self == CodeGenerator.BRIDGE:
             return BridgeGenerator(*args, **kargs)
         elif self == CodeGenerator.CPP_APPLICATION:
@@ -63,7 +66,8 @@ def FromString(name):
 # to uniquely identify them when running command line tools or
 # executing tests
 GENERATORS = {
-    'java': CodeGenerator.JAVA,
+    'java-jni': CodeGenerator.JAVA_JNI,
+    'java-class': CodeGenerator.JAVA_CLASS,
     'bridge': CodeGenerator.BRIDGE,
     'cpp-app': CodeGenerator.CPP_APPLICATION,
     'custom': CodeGenerator.CUSTOM,

scripts/py_matter_idl/matter_idl/test_generators.py

@@ -33,7 +33,7 @@
 from matter_idl.generators import GeneratorStorage
 from matter_idl.generators.bridge import BridgeGenerator
 from matter_idl.generators.cpp.application import CppApplicationGenerator
-from matter_idl.generators.java import JavaGenerator
+from matter_idl.generators.java import JavaClassGenerator, JavaJNIGenerator
 from matter_idl.matter_idl_types import Idl
 
 TESTS_DIR = os.path.join(os.path.dirname(__file__), "tests")
@@ -116,8 +116,10 @@ def add_test_cases(self, yaml_test_case_dict):
             self.test_cases.append(test_case)
 
     def _create_generator(self, storage: GeneratorStorage, idl: Idl):
-        if self.generator_name.lower() == 'java':
-            return JavaGenerator(storage, idl)
+        if self.generator_name.lower() == 'java-jni':
+            return JavaJNIGenerator(storage, idl)
+        if self.generator_name.lower() == 'java-class':
+            return JavaClassGenerator(storage, idl)
         if self.generator_name.lower() == 'bridge':
             return BridgeGenerator(storage, idl)
         if self.generator_name.lower() == 'cpp-app':

scripts/py_matter_idl/matter_idl/tests/available_tests.yaml

@@ -10,7 +10,7 @@
 #   - input_file is the input IDL
 #   - output_file/golden_path are the expected output file names
 #     and the expected content for those output files.
-java:
+java-jni:
     inputs/simple_attribute.matter:
         jni/MyClusterClient-ReadImpl.cpp: outputs/simple_attribute/jni/MyClusterClient-ReadImpl.cpp
         jni/MyClusterClient-InvokeSubscribeImpl.cpp: outputs/simple_attribute/jni/MyClusterClient-InvokeSubscribeImpl.cpp
@@ -35,6 +35,10 @@ java:
         jni/MyClusterClient-ReadImpl.cpp: outputs/optional_argument/jni/MyClusterClient-ReadImpl.cpp
         jni/MyClusterClient-InvokeSubscribeImpl.cpp: outputs/optional_argument/jni/MyClusterClient-InvokeSubscribeImpl.cpp
 
+java-class:
+    inputs/several_clusters.matter:
+        java/chip/devicecontroller/ClusterWriteMapping.java: outputs/several_clusters/java/ClusterWriteMapping.java
+
 bridge:
     inputs/simple_attribute.matter:
         bridge/BridgeClustersImpl.h: outputs/simple_attribute/bridge/BridgeClustersImpl.h