Merge pull request #37 from HotNoob/v1.1.2

V1.1.2

.gitignore

@@ -11,8 +11,10 @@ config.cfg
 invertermodbustomqtt.service
 log.txt
 variable_mask.txt
+variable_screen.txt
 .~lock.*
 protocols/*custom*
+classes/transports/*custom*
 
 input_registry.json
 holding_registry.json

README.md

@@ -41,7 +41,8 @@ protocol_version = {{version}}
 v0.14 = growatt inverters 2020+
 sigineer_v0.11 = sigineer inverters
 growatt_2020_v1.24 = alt protocol for large growatt inverters - currently untested
-eg4_v58 = eg4 inverters ( EG4-6000XP ) - implemented but untested
+eg4_v58 = eg4 inverters ( EG4-6000XP ) - confirmed working
+srne_v3.9 = SRNE inverters - Untested
 hdhk_16ch_ac_module = some chinese current monitoring device :P
 ```
 
@@ -105,12 +106,19 @@ you can also find the original documented variable names there; to use the origi
 the csvs are using ";" as the delimeter, because that is what open office uses. 
 
 ### variable_mask.txt
-if you want to only send/get specific variables, put them in this file. one variable per line. if list is empty all variables will be sent
+if you want to only send/get specific variables, put them in variable_mask.txt file. one variable per line. if list is empty all variables will be sent
 ```
 variable1
 variable2
 ```
 
+### variable_screen.txt
+if you want to exclude specific variables, put them in the variable_screen.txt file. one variable per line.
+```
+variable_to_exclude
+variable_to_exclude2
+```
+
 ### Any ModBus RTU Device
 As i dive deeper into solar monitoring and general automation, i've come to the realization that ModBus RTU is the "standard" and basically... everything uses it. With how this is setup, it can be used with basically anything running ModBus RTU so long as you have the documentation. 
 

classes/protocol_settings.py

@@ -42,6 +42,40 @@ class Data_Type(Enum):
     _14BIT = 214
     _15BIT = 215
     _16BIT = 216
+    #signed bits
+    _2SBIT = 302
+    _3SBIT = 303
+    _4SBIT = 304
+    _5SBIT = 305
+    _6SBIT = 306
+    _7SBIT = 307
+    _8SBIT = 308
+    _9SBIT = 309
+    _10SBIT = 310
+    _11SBIT = 311
+    _12SBIT = 312
+    _13SBIT = 313
+    _14SBIT = 314
+    _15SBIT = 315
+    _16SBIT = 316
+
+    #signed magnitude  bits
+    _2SMBIT = 402
+    _3SMBIT = 403
+    _4SMBIT = 404
+    _5SMBIT = 405
+    _6SMBIT = 406
+    _7SMBIT = 407
+    _8SMBIT = 408
+    _9SMBIT = 409
+    _10SMBIT = 410
+    _11SMBIT = 411
+    _12SMBIT = 412
+    _13SMBIT = 413
+    _14SMBIT = 414
+    _15SMBIT = 415
+    _16SMBIT = 416
+
     @classmethod
     def fromString(cls, name : str):
         name = name.strip().upper()
@@ -74,7 +108,13 @@ def getSize(cls, data_type : 'Data_Type'):
         if data_type in sizes:
             return sizes[data_type]
 
-        if data_type.value > 200: 
+        if data_type.value > 400:  #signed magnitude bits
+            return data_type.value-400
+
+        if data_type.value > 300:  #signed bits
+            return data_type.value-300
+
+        if data_type.value > 200: #unsigned bits
             return data_type.value-200
 
         return -1 #should never happen
@@ -172,6 +212,9 @@ class protocol_settings:
     transport : str
     settings_dir : str
     variable_mask : list[str]
+    ''' list of variables to allow and exclude all others '''
+    variable_screen : list[str]
+    ''' list of variables to exclude '''
     registry_map : dict[Registry_Type, list[registry_map_entry]] = {}
     registry_map_size : dict[Registry_Type, int] = {}
     registry_map_ranges : dict[Registry_Type, list[tuple]] = {}
@@ -184,6 +227,7 @@ def __init__(self, protocol : str, settings_dir : str = 'protocols'):
         self.protocol = protocol
         self.settings_dir = settings_dir
 
+        #load variable mask
         self.variable_mask = []
         if os.path.isfile('variable_mask.txt'):
             with open('variable_mask.txt') as f:
@@ -193,6 +237,16 @@ def __init__(self, protocol : str, settings_dir : str = 'protocols'):
 
                     self.variable_mask.append(line.strip().lower())
 
+        #load variable screen
+        self.variable_screen = []
+        if os.path.isfile('variable_screen.txt'):
+            with open('variable_screen.txt') as f:
+                for line in f:
+                    if line[0] == '#': #skip comment
+                        continue
+
+                    self.variable_screen.append(line.strip().lower())
+
         self.load__json() #load first, so priority to json codes
 
         if "transport" in self.settings:
@@ -252,13 +306,13 @@ def load__json(self, file : str = '', settings_dir : str = ''):
 
     def load__registry(self, path, registry_type : Registry_Type = Registry_Type.INPUT) -> list[registry_map_entry]: 
         registry_map : list[registry_map_entry] = []
-        register_regex = re.compile(r'(?P<register>x?\d+)\.(b(?P<bit>x?\d{1,2})|(?P<byte>x?\d{1,2}))')
+        register_regex = re.compile(r'(?P<register>(?:0?x[\dA-Z]+|[\d]+))\.(b(?P<bit>x?\d{1,2})|(?P<byte>x?\d{1,2}))')
 
         data_type_regex = re.compile(r'(?P<datatype>\w+)\.(?P<length>\d+)')
 
-        range_regex = re.compile(r'(?P<reverse>r|)(?P<start>x?\d+)[\-~](?P<end>x?\d+)')
+        range_regex = re.compile(r'(?P<reverse>r|)(?P<start>(?:0?x[\dA-Z]+|[\d]+))[\-~](?P<end>(?:0?x[\dA-Z]+|[\d]+))')
         ascii_value_regex = re.compile(r'(?P<regex>^\[.+\]$)')
-        list_regex = re.compile(r'\s*(?:(?P<range_start>x?\d+)-(?P<range_end>x?\d+)|(?P<element>[^,\s][^,]*?))\s*(?:,|$)')
+        list_regex = re.compile(r'\s*(?:(?P<range_start>(?:0?x[\dA-Z]+|[\d]+))-(?P<range_end>(?:0?x[\dA-Z]+|[\d]+))|(?P<element>[^,\s][^,]*?))\s*(?:,|$)')
 
 
         if not os.path.exists(path): #return empty is file doesnt exist.
@@ -513,7 +567,17 @@ def determine_delimiter(first_row) -> str:
                         item.documented_name.strip().lower() not in self.variable_mask 
                         and item.variable_name.strip().lower() not in self.variable_mask
                         ):
-                        del registry_map[index]                  
+                        del registry_map[index]
+
+            #apply variable screen     
+            if self.variable_screen:
+                for index in reversed(range(len(registry_map))):
+                    item = registry_map[index]
+                    if (
+                        item.documented_name.strip().lower() in self.variable_mask 
+                        and item.variable_name.strip().lower() in self.variable_mask
+                        ):
+                        del registry_map[index]      
 
             return registry_map
 
@@ -616,6 +680,35 @@ def process_register_bytes(self, registry : dict[int,bytes], entry : registry_ma
                     else:
                         flags.append("0")
                 value = ''.join(flags)
+
+
+        elif entry.data_type.value > 400: #signed-magnitude bit types ( sign bit is the last bit instead of front )
+            bit_size = Data_Type.getSize(entry.data_type)
+            bit_mask = (1 << bit_size) - 1  # Create a mask for extracting X bits
+            bit_index = entry.register_bit
+
+            # Check if the value is negative
+            if (register >> bit_index) & 1:
+                # If negative, extend the sign bit to fill out the value
+                sign_extension = 0xFFFFFFFFFFFFFFFF << bit_size
+                value = (register >> (bit_index + 1)) | sign_extension
+            else:
+                # If positive, simply extract the value using the bit mask
+                value = (register >> bit_index) & bit_mask
+        elif entry.data_type.value > 300: #signed bit types
+            bit_size = Data_Type.getSize(entry.data_type)
+            bit_mask = (1 << bit_size) - 1  # Create a mask for extracting X bits
+            bit_index = entry.register_bit
+
+            # Check if the value is negative
+            if (register >> (bit_index + bit_size - 1)) & 1:
+                # If negative, extend the sign bit to fill out the value
+                sign_extension = 0xFFFFFFFFFFFFFFFF << bit_size
+                value = (register >> bit_index) | sign_extension
+            else:
+                # If positive, simply extract the value using the bit mask
+                value = (register >> bit_index) & bit_mask
+
         elif entry.data_type.value > 200 or entry.data_type == Data_Type.BYTE: #bit types
             bit_size = Data_Type.getSize(entry.data_type)
             bit_mask = (1 << bit_size) - 1  # Create a mask for extracting X bits

classes/transports/modbus_rtu.py

@@ -3,6 +3,7 @@
 from pymodbus.client.sync import ModbusSerialClient
 from .modbus_base import modbus_base
 from configparser import SectionProxy
+from defs.common import find_usb_serial_port, get_usb_serial_port_info
 
 class modbus_rtu(modbus_base):
     port : str = "/dev/ttyUSB0"
@@ -19,6 +20,9 @@ def __init__(self, settings : SectionProxy, protocolSettings : protocol_settings
         self.port = settings.get("port", "")
         if not self.port:
             raise ValueError("Port is not set")
+
+        self.port = find_usb_serial_port(self.port) 
+        print("Serial Port : " + self.port + " = "+get_usb_serial_port_info(self.port)) #print for config convience
 
         self.baudrate = settings.getint("baudrate", 9600)
 

classes/transports/mqtt.py

@@ -31,7 +31,7 @@ class mqtt(transport_base):
 
     reconnect_attempts : int = 21
 
-    max_precision : int = - 1
+    #max_precision : int = - 1
 
 
     holding_register_prefix : str = ""
@@ -56,7 +56,7 @@ def __init__(self, settings : SectionProxy):
         self.discovery_enabled = strtobool(settings.get('discovery_enabled', self.discovery_enabled))
         self.json = strtobool(settings.get('json', self.json))
         self.reconnect_delay = settings.getint('reconnect_delay', fallback=7)
-        self.max_precision = settings.getint('max_precision', fallback=self.max_precision)
+        #self.max_precision = settings.getint('max_precision', fallback=self.max_precision)
 
         if not isinstance( self.reconnect_delay , int) or self.reconnect_delay < 1: #minumum 1 second
             self.reconnect_delay = 1
@@ -170,9 +170,12 @@ def write_data(self, data : dict[str, str]):
         if(self.json):
             # Serializing json
             json_object = json.dumps(data, indent=4)
-            self.client.publish(self.base_topic, json_object, 0, properties=self.__properties)
+            self.client.publish(self.base_topic, json_object, 0, properties=self.mqtt_properties)
         else:
             for entry, val in data.items():
+                if isinstance(val, float) and self.max_precision >= 0: #apply max_precision on mqtt transport 
+                    val = round(val, self.max_precision)
+
                 self.client.publish(str(self.base_topic+'/'+entry).lower(), str(val))
 
     def client_on_message(self, client, userdata, msg):

classes/transports/serial_pylon.py

@@ -7,6 +7,7 @@
 
 from .serial_frame_client import serial_frame_client
 from .transport_base import transport_base
+from defs.common import find_usb_serial_port, get_usb_serial_port_info
 
 
 
@@ -63,6 +64,9 @@ def __init__(self, settings : 'SectionProxy', protocolSettings : 'protocol_setti
         self.port = settings.get("port", "")
         if not self.port:
             raise ValueError("Port is not set")
+
+        self.port = find_usb_serial_port(self.port) 
+        print("Serial Port : " + self.port + " = "+get_usb_serial_port_info(self.port)) #print for config convience
 
         self.baudrate = settings.getint("baudrate", 9600)
 

classes/transports/transport_base.py

@@ -19,6 +19,7 @@ class transport_base:
     device_model : str = 'hotnoob'
     bridge : str = ''
     write_enabled : bool = False
+    max_precision : int = 2
 
     read_interval : float = 0
     last_read_time : float = 0
@@ -56,10 +57,13 @@ def __init__(self, settings : 'SectionProxy', protocolSettings : 'protocol_setti
             self.device_name = settings.get(['device_name', 'name'], fallback=self.device_manufacturer+"_"+self.device_serial_number)
             self.bridge = settings.get("bridge", self.bridge)
             self.read_interval = settings.getfloat("read_interval", self.read_interval)
+            self.max_precision = settings.getint(["max_precision", "precision"], self.max_precision)
             if "write_enabled" in settings:
                 self.write_enabled = settings.getboolean("write_enabled", self.write_enabled)
             else:
                 self.write_enabled = settings.getboolean("write", self.write_enabled)
+
+
 
     def init_bridge(self, from_transport : 'transport_base'):
         pass

defs/common.py

@@ -1,3 +1,6 @@
+import serial.tools.list_ports
+import re
+
 def strtobool (val):
     """Convert a string representation of truth to true (1) or false (0).
     True values are 'y', 'yes', 't', 'true', 'on', and '1'
@@ -13,4 +16,34 @@ def strtoint(val : str) -> int:
     if val and val[0] == 'x':
         return int.from_bytes(bytes.fromhex(val[1:]), byteorder='big')
 
-    return int(val)
+    if val and val.startswith("0x"):
+        return int.from_bytes(bytes.fromhex(val[2:]), byteorder='big')
+
+    if not val: #empty
+        return 0
+
+    return int(val)
+
+def get_usb_serial_port_info(port : str = '') -> str:
+    for p in serial.tools.list_ports.comports():
+        if str(p.device).upper() == port.upper():
+            return "["+hex(p.vid)+":"+hex(p.pid)+":"+str(p.serial_number)+":"+str(p.location)+"]"
+
+def find_usb_serial_port(port : str =  '', vendor_id : str = '', product_id : str = '', serial_number : str = '', location : str = '') -> str:
+    if not port.startswith('['):
+        return port
+
+    match  = re.match(r"\[(?P<vendor>[x\d]+|):?(?P<product>[x\d]+|):?(?P<serial>\d+|):?(?P<location>[\d\-]+|)\]", port)
+    if match:
+        vendor_id = int(match.group("vendor"), 16) if match.group("vendor") else ''
+        product_id = int(match.group("product"), 16) if match.group("product") else ''
+        serial_number = match.group("serial") if match.group("serial") else ''
+        location = match.group("location") if match.group("location") else ''
+
+    for port in serial.tools.list_ports.comports():
+        if ((not vendor_id or port.vid == vendor_id) and
+            ( not product_id or port.pid == product_id) and
+            ( not serial_number or port.serial_number == serial_number) and
+            ( not location or port.location == location)):
+            return port.device
+    return None

protocol_gateway.py

@@ -71,6 +71,10 @@ def get(self, section, option, *args, **kwargs):
                 raise NoOptionError(option[0], section)
         else:
             value = super().get(section, option, *args, **kwargs)
+
+        if isinstance(value, int):
+            return value
+
         return value.strip() if value is not None else value
 
 class Protocol_Gateway: