Updates for version, bitflags, bitfield and default tags.

codegen.py

@@ -10,20 +10,13 @@
 from codegen.Enum import Enum
 from codegen.Bitfield import Bitfield
 from codegen.Versions import Versions
+from codegen.Module import Module
 from codegen.naming_conventions import clean_comment_str
 
 logging.basicConfig(level=logging.DEBUG)
 
 FIELD_TYPES = ("add", "field")
-VER = "stream.version"
-
-
-def write_file(filename: str, contents: str):
-    file_dir = os.path.dirname(filename)
-    if not os.path.exists(file_dir):
-        os.makedirs(file_dir)
-    with open(filename, 'w', encoding='utf-8') as file:
-        file.write(contents)
+VER = "self.context.version"
 
 
 class XmlParser:
@@ -34,6 +27,8 @@ def __init__(self, format_name):
         """Set up the xml parser."""
 
         self.format_name = format_name
+        # which encoding to use for the output files
+        self.encoding='utf-8'
 
         # elements for versions
         self.version_string = None
@@ -53,19 +48,27 @@ def generate_module_paths(self, root):
         """preprocessing - generate module paths for imports relative to the output dir"""
         for child in root:
             # only check stuff that has a name - ignore version tags
-            if child.tag not in ("version", "module", "token"):
-                class_name = convention.name_class(child.attrib["name"])
-                out_segments = ["formats", self.format_name, child.tag, ]
-                if child.tag == "niobject":
-                    out_segments.append(child.attrib["module"])
-                out_segments.append(class_name)
+            if child.tag not in ("version", "token"):
+                base_segments = os.path.join("formats", self.format_name)
+                if child.tag == "module":
+                    # for modules, set the path to base/module_name
+                    class_name = convention.name_module(child.attrib["name"])
+                    class_segments = [class_name]
+                else:
+                    # for classes, set the path to module_path/tag/class_name or
+                    # base/tag/class_name if it's not part of a module
+                    class_name = convention.name_class(child.attrib["name"])
+                    if child.attrib.get("module"):
+                        base_segments = self.path_dict[convention.name_module(child.attrib["module"])]
+                    class_segments = [child.tag, class_name, ]
                 # store the final relative module path for this class
-                self.path_dict[class_name] = os.path.join(*out_segments)
+                self.path_dict[class_name] = os.path.join(base_segments, *class_segments)
                 self.tag_dict[class_name.lower()] = child.tag
 
         self.path_dict["Array"] = "array"
         self.path_dict["BasicBitfield"] = "bitfield"
         self.path_dict["BitfieldMember"] = "bitfield"
+        self.path_dict["ContextReference"] = "context"
         self.path_dict["UbyteEnum"] = "base_enum"
         self.path_dict["UshortEnum"] = "base_enum"
         self.path_dict["UintEnum"] = "base_enum"
@@ -81,7 +84,8 @@ def load_xml(self, xml_file):
 
         for child in root:
             self.replace_tokens(child)
-            self.apply_conventions(child)
+            if child.tag not in ('version', 'module'):
+                self.apply_conventions(child)
             try:
                 if child.tag in self.struct_types:
                     Compound(self, child)
@@ -91,8 +95,8 @@ def load_xml(self, xml_file):
                 #     self.write_basic(child)
                 elif child.tag == "enum":
                     Enum(self, child)
-                # elif child.tag == "module":
-                #     self.read_module(child)
+                elif child.tag == "module":
+                    Module(self, child)
                 elif child.tag == "version":
                     versions.read(child)
                 elif child.tag == "token":
@@ -110,19 +114,6 @@ def read_token(self, token):
                             for sub_token in token],
                             token.attrib["attrs"].split(" ")))
 
-    @staticmethod
-    def get_names(struct):
-        # struct types can be organized in a hierarchy
-        # if inherit attribute is defined, look for corresponding base block
-        class_name = convention.name_class(struct.attrib.get("name"))
-        class_basename = struct.attrib.get("inherit")
-        class_debug_str = clean_comment_str(struct.text, indent="\t")
-        if class_basename:
-            # avoid turning None into 'None' if class doesn't inherit
-            class_basename = convention.name_class(class_basename)
-            # logging.debug(f"Struct {class_name} is based on {class_basename}")
-        return class_name, class_basename, class_debug_str
-
     @staticmethod
     def apply_convention(struct, func, params):
         for k in params:
@@ -137,22 +128,14 @@ def apply_conventions(self, struct):
             if field.tag in FIELD_TYPES:
                 self.apply_convention(field, convention.name_attribute, ("name",))
                 self.apply_convention(field, convention.name_class, ("type",))
+                self.apply_convention(field, convention.name_class, ("onlyT",))
+                self.apply_convention(field, convention.name_class, ("excludeT",))
+                for default in field:
+                    self.apply_convention(field, convention.name_class, ("onlyT",))
 
         # filter comment str
         struct.text = clean_comment_str(struct.text, indent="\t", class_comment='"""')
 
-    def collect_types(self, imports, struct):
-        """Iterate over all fields in struct and collect type references"""
-        # import classes used in the fields
-        for field in struct:
-            if field.tag in ("add", "field", "member"):
-                field_type = convention.name_class(field.attrib["type"])
-                if field_type not in imports:
-                    if field_type == "self.template":
-                        imports.append("typing")
-                    else:
-                        imports.append(field_type)
-
     def method_for_type(self, dtype: str, mode="read", attr="self.dummy", arg=None, template=None):
         if self.tag_dict[dtype.lower()] == "enum":
             storage = self.storage_dict[dtype]
@@ -207,11 +190,6 @@ def replace_tokens(self, xml_struct):
             for op_token, op_str in fixed_tokens:
                 expr_str = expr_str.replace(op_token, op_str)
             xml_struct.attrib[attrib] = expr_str
-        # onlyT & excludeT act as aliases for deprecated cond
-        for t, pref in (("onlyT", ""), ("excludeT", "!")):
-            if t in xml_struct.attrib:
-                xml_struct.attrib["cond"] = pref+xml_struct.attrib[t]
-                break
         for xml_child in xml_struct:
             self.replace_tokens(xml_child)
 
@@ -224,6 +202,17 @@ def copy_src_to_generated():
     copy_tree(src_dir, trg_dir)
 
 
+def create_inits():
+    """Create a __init__.py file in all subdirectories that don't have one, to prevent error on second import"""
+    base_dir = os.path.join(os.getcwd(), 'generated')
+    init_file = "__init__.py"
+    for root, dirs, files in os.walk(base_dir):
+        if init_file not in files:
+            # __init__.py does not exist, create it
+            with open(os.path.join(root, init_file), 'x'): pass
+        # don't go into subdirectories that start with a double underscore
+        dirs[:] = [dirname for dirname in dirs if dirname[:2] != '__']
+
 def generate_classes():
     logging.info("Starting class generation")
     cwd = os.getcwd()
@@ -237,6 +226,7 @@ def generate_classes():
                 logging.info(f"Reading {format_name} format")
                 xmlp = XmlParser(format_name)
                 xmlp.load_xml(xml_path)
+    create_inits()
 
 
 generate_classes()

codegen/BaseClass.py

@@ -40,7 +40,7 @@ def get_code_from_src(self,):
                 if self.parser.format_name in root and py_name == name.lower():
                     src_path = os.path.join(root, name)
                     print("found source", src_path)
-                    with open(src_path, "r") as f:
+                    with open(src_path, "r", encoding=self.parser.encoding) as f:
                         return f.read()
         return ""
 

codegen/Bitfield.py

@@ -22,8 +22,12 @@ def get_mask(self):
 					num_bits = int(field.attrib["numbits"])
 				elif "width" in field.attrib:
 					num_bits = int(field.attrib["width"])
+				elif "bit" in field.attrib:
+					num_bits = 1
+					field.attrib["pos"] = field.attrib["bit"]
+					field.attrib["type"] = "bool"
 				else:
-					raise AttributeError(f"Neither width or mask or numbits are defined for {field.name}")
+					raise AttributeError(f"Neither width, mask, bit or numbits are defined for {field.attrib['name']}")
 				pos = int(field.attrib["pos"])
 
 				mask = ~((~0) << (pos + num_bits)) & ((~0) << pos)
@@ -45,7 +49,7 @@ def read(self):
 		self.class_basename = "BasicBitfield"
 
 		# write to python file
-		with open(self.out_file, "w") as f:
+		with open(self.out_file, "w", encoding=self.parser.encoding) as f:
 			# write the header stuff
 			super().write(f)
 			self.map_pos()

codegen/Compound.py

@@ -3,7 +3,7 @@
 from .Union import Union, get_params
 
 FIELD_TYPES = ("add", "field")
-VER = "stream.version"
+VER = "self.context.version"
 
 
 class Compound(BaseClass):
@@ -27,20 +27,25 @@ def read(self):
 				self.imports.add("numpy")
 
 		# write to python file
-		with open(self.out_file, "w") as f:
+		with open(self.out_file, "w", encoding=self.parser.encoding) as f:
 			# write the header stuff
 			super().write(f)
 
+			if not self.class_basename:
+				f.write(f"\n\n\tcontext = ContextReference()")
+
 			# check all fields/members in this class and write them as fields
 			# for union in self.field_unions.values():
 			# 	union.write_declaration(f)
 
 			if "def __init__" not in self.src_code:
-				f.write(f"\n\n\tdef __init__(self, arg=None, template=None):")
+				f.write(f"\n\n\tdef __init__(self, context, arg=None, template=None):")
 				f.write(f"\n\t\tself.name = ''")
 				# classes that this class inherits from have to be read first
 				if self.class_basename:
-					f.write(f"\n\t\tsuper().__init__(arg, template)")
+					f.write(f"\n\t\tsuper().__init__(context, arg, template)")
+				else:
+					f.write(f"\n\t\tself._context = context")
 				f.write(f"\n\t\tself.arg = arg")
 				f.write(f"\n\t\tself.template = template")
 				f.write(f"\n\t\tself.io_size = 0")
@@ -60,7 +65,6 @@ def read(self):
 				# classes that this class inherits from have to be read first
 				if self.class_basename:
 					f.write(f"\n\t\tsuper().{method_type}(stream)")
-
 				for union in self.field_unions:
 					last_condition = union.write_io(f, method_type, last_condition)
 

codegen/Enum.py

@@ -1,7 +1,7 @@
 from .BaseClass import BaseClass
 
 FIELD_TYPES = ("add", "field")
-VER = "stream.version"
+VER = "self.context.version"
 
 
 class Enum(BaseClass):
@@ -18,7 +18,7 @@ def read(self):
 		self.class_basename = enum_base
 		self.imports.add(enum_base)
 		# write to python file
-		with open(self.out_file, "w") as f:
+		with open(self.out_file, "w", encoding=self.parser.encoding) as f:
 			# write the header stuff
 			super().write(f)
 			for option in self.struct:

codegen/Imports.py

@@ -1,3 +1,6 @@
+from os.path import sep
+
+
 NO_CLASSES = ("Padding",)
 
 
@@ -11,6 +14,9 @@ def __init__(self, parser, xml_struct):
         self.imports = []
         # import parent class
         self.add(xml_struct.attrib.get("inherit"))
+        # import ContextReference class
+        if xml_struct.tag in parser.struct_types and not xml_struct.attrib.get("inherit"):
+            self.add("ContextReference")
 
         # import classes used in the fields
         for field in xml_struct:
@@ -25,9 +31,23 @@ def __init__(self, parser, xml_struct):
                 self.add(field_type)
                 # arr1 needs typing.List
                 arr1 = field.attrib.get("arr1")
+                if arr1 is None:
+                    arr1 = field.attrib.get("length")
                 if arr1:
                     self.add("typing")
                     self.add("Array")
+                type_attribs = ("onlyT", "excludeT")
+                for attrib in type_attribs:
+                    attrib_type = field.attrib.get(attrib)
+                    if attrib_type:
+                        self.add(attrib_type)
+
+                for default in field:
+                    if default.tag in ("default",):
+                        for attrib in type_attribs:
+                            attrib_type = default.attrib.get(attrib)
+                            if attrib_type:
+                                self.add(attrib_type)
 
     def add(self, cls_to_import, import_from=None):
         if cls_to_import:
@@ -43,7 +63,7 @@ def write(self, stream):
             if class_import in NO_CLASSES:
                 continue
             if class_import in self.path_dict:
-                import_path = "generated." + self.path_dict[class_import].replace("\\", ".")
+                import_path = "generated." + self.path_dict[class_import].replace(sep, ".")
                 local_imports.append(f"from {import_path} import {class_import}\n")
             else:
                 module_imports.append(f"import {class_import}\n")

codegen/Module.py

@@ -0,0 +1,24 @@
+import os
+from codegen.naming_conventions import clean_comment_str, name_module
+
+class Module:
+
+    def __init__(self, parser, element):
+        self.parser = parser
+        self.element = element
+        self.read(element)
+        self.write(parser.path_dict[name_module(element.attrib["name"])])
+
+    def read(self, element):
+        self.comment_str = clean_comment_str(element.text, indent="", class_comment='"""')[2:]
+        self.priority = int(element.attrib.get("priority",""))
+        self.depends = [name_module(module) for module in element.attrib.get("depends","").split(" ")]
+        self.custom = bool(eval(element.attrib.get("custom","true").replace("true","True").replace("false","False"),{}))
+
+    def write(self, rel_path):
+        with open(os.path.join(os.getcwd(), "generated", rel_path, "__init__.py"), "w", encoding=self.parser.encoding) as file:
+            file.write(self.comment_str)
+            file.write(f'\n\n__priority__ = {repr(self.priority)}')
+            file.write(f'\n__depends__ = {repr(self.depends)}')
+            file.write(f'\n__custom__ = {repr(self.custom)}')
+            file.write(f'\n')