Setup keep_trailing_newline=True in python jinja templates (#23235)

* Keep trailing newlins in codegen

* Add option to auto-regenerate golden data

* Re-generate test outputs after skip whitespace changes

* Restyle

scripts/idl/generators/__init__.py

@@ -113,7 +113,8 @@ def __init__(self, storage: GeneratorStorage, idl: Idl):
         self.storage = storage
         self.idl = idl
         self.jinja_env = jinja2.Environment(
-            loader=jinja2.FileSystemLoader(searchpath=os.path.dirname(__file__)))
+            loader=jinja2.FileSystemLoader(searchpath=os.path.dirname(__file__)),
+            keep_trailing_newline=True)
         self.dry_run = False
 
         RegisterCommonFilters(self.jinja_env.filters)

scripts/idl/test_generators.py

@@ -37,6 +37,7 @@
 
 
 TESTS_DIR = os.path.join(os.path.dirname(__file__), "tests")
+REGENERATE_GOLDEN_IMAGES = False
 
 
 @dataclass
@@ -63,19 +64,33 @@ def __init__(self, test_case: GeneratorTestCase, checker: unittest.TestCase):
         self.checker = checker
         self.checked_files = set()
 
+    def get_existing_data_path(self, relative_path: str):
+        for expected in self.test_case.outputs:
+            if expected.file_name == relative_path:
+                return os.path.join(TESTS_DIR, expected.golden_path)
+
+        self.checker.fail("Expected output %s not found" % relative_path)
+        return None
+
     def get_existing_data(self, relative_path: str):
         self.checked_files.add(relative_path)
 
-        for expected in self.test_case.outputs:
-            if expected.file_name == relative_path:
-                with open(os.path.join(TESTS_DIR, expected.golden_path), 'rt') as golden:
-                    return golden.read()
+        path = self.get_existing_data_path(relative_path)
+        if path:
+            with open(path, 'rt') as golden:
+                return golden.read()
 
         # This will attempt a new write, causing a unit test failure
         self.checker.fail("Expected output %s not found" % relative_path)
         return None
 
     def write_new_data(self, relative_path: str, content: str):
+        if REGENERATE_GOLDEN_IMAGES:
+            # Expect writing only on regeneration
+            with open(self.get_existing_data_path(relative_path), 'wt') as golden:
+                golden.write(content)
+                return
+
         # This is a unit test failure: we do NOT expect
         # to write any new data
 
@@ -148,4 +163,8 @@ def test_generators(self):
 
 
 if __name__ == '__main__':
+    if 'IDL_GOLDEN_REGENERATE' in os.environ:
+        # run with `IDL_GOLDEN_REGENERATE=1` to cause a regeneration of test
+        # data. Then one can use `git diff` to see if the deltas make sense
+        REGENERATE_GOLDEN_IMAGES = True
     unittest.main()

scripts/idl/tests/outputs/cluster_struct_attribute/bridge/BridgeClustersImpl.h

@@ -23,4 +23,4 @@ struct ClusterInfo
   },
 };
 
-}
+}

scripts/idl/tests/outputs/cluster_struct_attribute/bridge/BridgeGlobalStructs.h

@@ -11,4 +11,4 @@ namespace clusters {
 
 
 
-}
+}

scripts/idl/tests/outputs/cluster_struct_attribute/bridge/DemoCluster.h

@@ -29,4 +29,4 @@ struct DemoClusterCluster : public GeneratedCluster
   ListAttribute<std::vector<ArmFailSafeRequest>> mArmFailsafes;
 };
 
-}
+}

scripts/idl/tests/outputs/cluster_struct_attribute/jni/DemoClusterClient-InvokeSubscribeImpl.cpp

@@ -53,4 +53,4 @@ JNI_METHOD(void, DemoClusterCluster, subscribeArmFailsafesAttribute)(JNIEnv * en
 
     onSuccess.release();
     onFailure.release();
-}
+}

scripts/idl/tests/outputs/cluster_struct_attribute/jni/DemoClusterClient-ReadImpl.cpp

@@ -35,3 +35,4 @@ JNI_METHOD(void, DemoClusterCluster, readArmFailsafesAttribute)(JNIEnv * env, jo
     onFailure.release();
 }
 
+

scripts/idl/tests/outputs/global_struct_attribute/bridge/BridgeClustersImpl.h

@@ -23,4 +23,4 @@ struct ClusterInfo
   },
 };
 
-}
+}

scripts/idl/tests/outputs/global_struct_attribute/bridge/BridgeGlobalStructs.h

@@ -34,4 +34,4 @@ struct LabelStruct
   std::string value;
 };
 
-}
+}

scripts/idl/tests/outputs/global_struct_attribute/bridge/DemoCluster.h

@@ -29,4 +29,4 @@ struct DemoClusterCluster : public GeneratedCluster
   ListAttribute<std::vector<LabelStruct>> mSomeLabels;
 };
 
-}
+}

scripts/idl/tests/outputs/global_struct_attribute/jni/DemoClusterClient-InvokeSubscribeImpl.cpp

@@ -53,4 +53,4 @@ JNI_METHOD(void, DemoClusterCluster, subscribeSomeLabelsAttribute)(JNIEnv * env,
 
     onSuccess.release();
     onFailure.release();
-}
+}

scripts/idl/tests/outputs/global_struct_attribute/jni/DemoClusterClient-ReadImpl.cpp

@@ -35,3 +35,4 @@ JNI_METHOD(void, DemoClusterCluster, readSomeLabelsAttribute)(JNIEnv * env, jobj
     onFailure.release();
 }
 
+

scripts/idl/tests/outputs/optional_argument/jni/MyClusterClient-InvokeSubscribeImpl.cpp

@@ -76,3 +76,4 @@ JNI_METHOD(void, MyClusterCluster,
     onSuccess.release();
     onFailure.release();
 }
+

scripts/idl/tests/outputs/optional_argument/jni/MyClusterClient-ReadImpl.cpp

@@ -11,4 +11,4 @@
 #include <platform/PlatformManager.h>
 
 #define JNI_METHOD(RETURN, CLASS_NAME, METHOD_NAME)                                                                                \
-    extern "C" JNIEXPORT RETURN JNICALL Java_chip_devicecontroller_ChipClusters_00024##CLASS_NAME##_##METHOD_NAME
+    extern "C" JNIEXPORT RETURN JNICALL Java_chip_devicecontroller_ChipClusters_00024##CLASS_NAME##_##METHOD_NAME

scripts/idl/tests/outputs/several_clusters/bridge/BridgeClustersImpl.h

@@ -41,4 +41,4 @@ struct ClusterInfo
   },
 };
 
-}
+}

scripts/idl/tests/outputs/several_clusters/bridge/BridgeGlobalStructs.h

@@ -11,4 +11,4 @@ namespace clusters {
 
 
 
-}
+}

scripts/idl/tests/outputs/several_clusters/bridge/First.h

@@ -26,4 +26,4 @@ struct FirstCluster : public GeneratedCluster
   Attribute<uint16_t> mSomeInteger;
 };
 
-}
+}

scripts/idl/tests/outputs/several_clusters/bridge/Second.h

@@ -26,4 +26,4 @@ struct SecondCluster : public GeneratedCluster
   Attribute<std::string> mSomeBytes;
 };
 
-}
+}

scripts/idl/tests/outputs/several_clusters/bridge/Third.h

@@ -26,4 +26,4 @@ struct ThirdCluster : public GeneratedCluster
   Attribute<uint8_t> mSomeEnum;
 };
 
-}
+}

scripts/idl/tests/outputs/several_clusters/jni/FirstClient-InvokeSubscribeImpl.cpp

@@ -53,4 +53,4 @@ JNI_METHOD(void, FirstCluster, subscribeSomeIntegerAttribute)(JNIEnv * env, jobj
 
     onSuccess.release();
     onFailure.release();
-}
+}

scripts/idl/tests/outputs/several_clusters/jni/FirstClient-ReadImpl.cpp

@@ -35,3 +35,4 @@ JNI_METHOD(void, FirstCluster, readSomeIntegerAttribute)(JNIEnv * env, jobject s
     onFailure.release();
 }
 
+

scripts/idl/tests/outputs/several_clusters/jni/SecondClient-InvokeSubscribeImpl.cpp

@@ -53,4 +53,4 @@ JNI_METHOD(void, SecondCluster, subscribeSomeBytesAttribute)(JNIEnv * env, jobje
 
     onSuccess.release();
     onFailure.release();
-}
+}

scripts/idl/tests/outputs/several_clusters/jni/SecondClient-ReadImpl.cpp

@@ -35,3 +35,4 @@ JNI_METHOD(void, SecondCluster, readSomeBytesAttribute)(JNIEnv * env, jobject se
     onFailure.release();
 }
 
+

scripts/idl/tests/outputs/several_clusters/jni/ThirdClient-InvokeSubscribeImpl.cpp

@@ -53,4 +53,4 @@ JNI_METHOD(void, ThirdCluster, subscribeSomeEnumAttribute)(JNIEnv * env, jobject
 
     onSuccess.release();
     onFailure.release();
-}
+}

scripts/idl/tests/outputs/several_clusters/jni/ThirdClient-ReadImpl.cpp

@@ -35,3 +35,4 @@ JNI_METHOD(void, ThirdCluster, readSomeEnumAttribute)(JNIEnv * env, jobject self
     onFailure.release();
 }
 
+

scripts/idl/tests/outputs/simple_attribute/bridge/BridgeClustersImpl.h

@@ -23,4 +23,4 @@ struct ClusterInfo
   },
 };
 
-}
+}

scripts/idl/tests/outputs/simple_attribute/bridge/BridgeGlobalStructs.h

@@ -11,4 +11,4 @@ namespace clusters {
 
 
 
-}
+}

scripts/idl/tests/outputs/simple_attribute/bridge/MyCluster.h

@@ -26,4 +26,4 @@ struct MyClusterCluster : public GeneratedCluster
   Attribute<uint16_t> mClusterAttr;
 };
 
-}
+}

scripts/idl/tests/outputs/simple_attribute/jni/MyClusterClient-InvokeSubscribeImpl.cpp

@@ -53,4 +53,4 @@ JNI_METHOD(void, MyClusterCluster, subscribeClusterAttrAttribute)(JNIEnv * env,
 
     onSuccess.release();
     onFailure.release();
-}
+}

scripts/idl/tests/outputs/simple_attribute/jni/MyClusterClient-ReadImpl.cpp

@@ -35,3 +35,4 @@ JNI_METHOD(void, MyClusterCluster, readClusterAttrAttribute)(JNIEnv * env, jobje
     onFailure.release();
 }
 
+