diff --git a/.gitignore b/.gitignore index 510cc81..f43faf7 100644 --- a/.gitignore +++ b/.gitignore @@ -13,10 +13,10 @@ pymagnum.code-workspace *.bak credentials.txt source/_build -docs/_build testdata/** testing/** .pymodhis !testdata/allpackets.txt !testdata/allpackets.json +!testdata/allpackets.sql examples/mqttlogger_2.py diff --git a/CHANGES.rst b/CHANGES.rst index 62f8fe8..efc3350 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,18 +1,26 @@ ======================= pyMagnum Release Notes -======================= +======================== +Version 2.0.5 2023/11 +------------------------ +- Enhancement added ``allinone`` method to Magnum class. This creates a single record instead of one per device. +- ``magdump`` has been enhanced to support ``--allinone`` option +- A new example program ``magsql.py`` is available to log data to MariaDB/MySQL. Read the program comments for more information. +- A new tool ``mag2sql`` is available to convert JSON from ``magdump`` output to a MySQL schema definition. + Version 2.0.4 2023/11/05 ------------------------ - Fixed bug in emitted JSON when using multiple devices - Cleaned up code when using dummy data files instead of RS485 serial device - Provided a complete dummy test file testdata/allpackets.txt - Provided a JSON file testdata/allpackets.JSON -- Enhancement added --pretty option to magdump for formatted JSON output +- Fixed All revision values are defined as string. It used to be mixed +- Enhancement --pretty option to magdump for formatted JSON output Version 2.0.3 2023/10/20 ------------------------ -- Fixed problem with installing tzlocal on some system +- Fixed problem with installing tzlocal on some systems - Changed minimum Python version to 3.7 - Added warning about use of ``--break-system-packages`` with Python 3.11 and higher. - Improved how to build information. diff --git a/allpackets.json b/allpackets.json new file mode 100644 index 0000000..06ffc91 --- /dev/null +++ b/allpackets.json @@ -0,0 +1,141 @@ +{ + "datetime":"2023-11-04T09:51:53-04:00", + "device":"MAGNUM", + "comm_device":"!testdata/allpackets.txt", + "data":[ + { + "device":"INVERTER", + "data":{ + "revision":"6.1", + "mode":64, + "mode_text":"INVERT", + "fault":0, + "fault_text":"None", + "vdc":24.6, + "adc":22.0, + "VACout":119.0, + "VACin":0.0, + "invled":1, + "invled_text":"On", + "chgled":0, + "chgled_text":"Off", + "bat":17.0, + "tfmr":51.0, + "fet":36.0, + "model":107, + "model_text":"MS4024PAE", + "stackmode":1, + "stackmode_text":"Parallel stack - master", + "AACin":0.0, + "AACout":5.0, + "Hz":60.0 + } + }, + { + "device":"REMOTE", + "data":{ + "revision":4.0, + "searchwatts":0, + "batterysize":400, + "battype":8, + "absorb":0, + "chargeramps":100, + "ainput":10, + "parallel":0, + "lbco":20.0, + "vaccutout":155.0, + "vsfloat":26.4, + "vEQ":1.2, + "absorbtime":2.0, + "runtime":3.6, + "starttemp":-17.8, + "startvdc":23.0, + "quiettime":0, + "begintime":2400, + "stoptime":0, + "vdcstop":28.8, + "voltstartdelay":120, + "voltstopdelay":120, + "maxrun":12.0, + "socstart":70, + "socstop":85, + "ampstart":0.0, + "ampsstartdelay":120, + "ampstop":20, + "ampsstopdelay":120, + "quietbegintime":2330, + "quietendtime":2345, + "exercisedays":0, + "exercisestart":0, + "exerciseruntime":0, + "topoff":2, + "warmup":30, + "cool":30, + "batteryefficiency":0 + } + }, + { + "device":"BMK", + "data":{ + "revision":"1.0", + "soc":76, + "vdc":25.45, + "adc":11.6, + "vmin":20.16, + "vmax":30.8, + "amph":-104, + "amphtrip":2046.4, + "amphout":2000, + "Fault":1, + "Fault_Text":"Normal" + } + }, + { + "device":"AGS", + "data":{ + "revision":"5.2", + "status":2, + "status_text":"Ready", + "running":false, + "temp":14.4, + "runtime":0.0, + "gen_last_run":0, + "last_full_soc":0, + "gen_total_run":0, + "vdc":25.4 + } + }, + { + "device":"RTR", + "data":{ + "revision":"3" + } + }, + { + "device":"PT100", + "data":{ + "revision":"1.x", + "address":0, + "mode":3, + "mode_text":"Absorb", + "mode_hex":"0X13", + "regulation":1, + "regulation_text":"Voltage", + "fault":0, + "fault_text":"No Fault", + "battery":58.9, + "battery_amps":13.3, + "pv_voltage":146.8, + "charge_time":1.7, + "target_battery_voltage":29.6, + "relay_state":0, + "alarm_state":0, + "fan_on":0, + "day":1, + "battery_temperature":27.0, + "inductor_temperature":49.0, + "fet_temperature":62.0 + } + } + ] +} diff --git a/docs/conf.py b/docs/conf.py index d3c15df..0b7c49b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -24,7 +24,7 @@ # The full version, including alpha/beta/rc tags release = '2.0' -version = '2.0.4' +version = '2.0.5' # -- General configuration --------------------------------------------------- diff --git a/docs/examples.rst b/docs/examples.rst index 1efaa58..001426b 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -9,9 +9,6 @@ Example programs are available in `this repository `_. - These programs support use of the `configuration file `_. -These programs support defining multiple RS485 devices. +These programs support defining multiple RS485 devices, except for `magsql.py`. diff --git a/docs/tools.rst b/docs/tools.rst index b55a20f..400e725 100644 --- a/docs/tools.rst +++ b/docs/tools.rst @@ -46,6 +46,15 @@ You can define more than one device. Just provide multiple ``--device /dev/ttyUS **NOTE:** If the device name is prefixed with a `!` the rest of the name is treated as a filename and is read for data. The format of the text must be the same as the output generated by the ``magtest`` program. this is useful for debugging. +mag2sql +======= + +This tool converts the JSON output from magdump into a draft MySQL definition. Users are urged to edit the output to match their needs. +the example prigram ``examples/magsql.py`` will load MySQL data once the database is defined. + +``mag2sql --help`` + +``magdump | mag2sql > myschema.sql`` Configuration (options) File ============================ diff --git a/examples/magsample.py b/examples/magsample.py index 64eba65..95756a0 100644 --- a/examples/magsample.py +++ b/examples/magsample.py @@ -1,4 +1,5 @@ -# +#!/usr/bin/env python3 +# # Copyright (c) 2018-2022 Charles Godwin # # SPDX-License-Identifier: BSD-3-Clause @@ -47,7 +48,7 @@ def sigint_handler(signal, frame): print(f"Magnum Sample Version:{magnum.__version__}") print(f"Options:{str(args)[10:-1]}") -magnumReaders = dict() +magnumReaders = {} for device in args.device: try: magnumReader = Magnum(device=device, packets=args.packets, trace=args.trace, @@ -65,7 +66,7 @@ def sigint_handler(signal, frame): try: devices = magnumReader.getDevices() if len(magnumReaders) > 1: - data = dict() + data = {} data['comm_device'] = comm_device data["data"] = devices print(json.dumps(data, indent=2)) diff --git a/examples/magserver.py b/examples/magserver.py index 3b001c5..bdc5b1f 100644 --- a/examples/magserver.py +++ b/examples/magserver.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # # Copyright (c) 2018-2022 Charles Godwin # @@ -9,7 +10,6 @@ import signal import sys -from collections import OrderedDict from datetime import datetime, timezone from http import HTTPStatus from http.server import BaseHTTPRequestHandler, HTTPServer @@ -27,7 +27,7 @@ def sigint_handler(signal, frame): signal.signal(signal.SIGINT, sigint_handler) -magnumReaders = dict() +magnumReaders = {} class magServer(BaseHTTPRequestHandler): @@ -59,10 +59,10 @@ def do_GET(self): devices = magnumReader.getDevices() if len(devices) != 0: self._set_headers(contenttype="application/json") - data = OrderedDict() + data = {} data["datetime"] = timestamp data["devices"] = devices - device = dict() + device = {} device['comm_device'] = comm_device device['data'] = data response.append(device) diff --git a/examples/magsql.py b/examples/magsql.py new file mode 100644 index 0000000..6fdea78 --- /dev/null +++ b/examples/magsql.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2023 Charles Godwin +# +# SPDX-License-Identifier: BSD-3-Clause +# +# This code is provided as an example of loading data into MySql/MariaDB database +# run the program with --help for details of options. +# python3 magsql.py --help +# +# You must install mariadb package and dependencies +# `sudo apt install libmariadb3 libmariadb-dev -y` +# `sudo pip3 install python3-dev mariadb` +# +# This has been tested ONLY on a Raspberry Pi +# This is example code and is not supported by the author although he will try to respond to questions. +# +# The --allinone option will log a single flat table row with table name of `log_data` +# +# The program will only load columns that match between the variable in the defined data and columns in the database. +# To see what data is available run magdump.py with same parameters +# +# The user is responsible for defining the MySQL table(s) and database but can exploit the mag2sql module +# +import signal +import sys +import time + +from datetime import datetime, timezone +from xmlrpc.client import Boolean +import mariadb + +import magnum +from magnum.magnum import Magnum +from magnum.magparser import MagnumArgumentParser + +global args, cursor, first_time, db_columns + +first_time = True +db_columns = {} + + +def main(): + global first_time, cursor, args + signal.signal(signal.SIGINT, sigint_handler) + parser = MagnumArgumentParser(description="Magnum SQL Load", prog="Magnum SQL", fromfile_prefix_chars='@', + epilog="Refer to https://github.com/CharlesGodwin/pymagnum for details") + parser.add_argument("-d", "--device", nargs='*', action='append', default=[], + help="Serial device name (default: /dev/ttyUSB0). You can specify ONLY one.") + parser.add_argument("-i", "--interval", default=60, type=int, dest='interval', + help="Interval, in seconds, between dump records, in seconds. 0 means once and exit. (default: %(default)s)") + parser.add_argument('-v', "--verbose", action="store_true", default=False, + help="Display options at runtime (default: %(default)s)") + parser.add_argument("--db_username", default=None, required=True, + help="MySQL User name(default: %(default)s)") + parser.add_argument("--db_password", default=None, required=True, + help="MySQL User password(default: %(default)s)") + parser.add_argument("--db_database", default='magnum', + help="MySQL database name(default: %(default)s)") + parser.add_argument("--db_host", default='localhost', + help="MySQL Server host name(default: %(default)s)") + seldom = parser.add_argument_group("Seldom used") + seldom.add_argument('--version', action='version', + version="%(prog)s Version:{}".format(magnum.__version__)) + seldom.add_argument("--packets", default=50, type=int, + help="Number of packets to generate in reader (default: %(default)s)") + seldom.add_argument("--timeout", default=0.005, type=float, + help="Timeout for serial read (default: %(default)s)") + seldom.add_argument("--trace", action="store_true", default=False, + help="Add most recent raw packet(s) info to data (default: %(default)s)") + seldom.add_argument("--nocleanup", action="store_true", default=False, dest='cleanpackets', + help="Suppress clean up of unknown packets (default: False)") + seldom.add_argument("--db_port", default=3306, type=int, + help="MySQL port(default: %(default)s)") + seldom.add_argument("--allinone", action="store_true", default=False, + help="Process data as a flat single row (default: %(default)s)") + + args = parser.magnum_parse_args() + # Only supports one device + if len(args.device) > 1: + parser.error("magsql only supports 1 device at a time.") + if hasattr(args, 'v1'): # a relic but not harmful + args.allinone = True + if args.verbose: + savepw = args.db_password + args.db_password = "******" + print('Magnum SQL Load Version:{0}'.format(magnum.__version__)) + print("Options:{0}".format(str(args) + .replace('Namespace(', '') + .replace(')', '') + .replace('[', '') + .replace('\'', '') + .replace(']', ''))) + args.db_password = savepw + try: + magnumReader = Magnum(device=args.device[0], packets=args.packets, trace=args.trace, + timeout=args.timeout, cleanpackets=args.cleanpackets) + except Exception as e: + print("{0} {1}".format(args.device, str(e))) + exit(2) + if args.interval != 0 and args.verbose == True: + print(f"Logging every:{args.interval} seconds.") + while True: + start = time.time() + commdevices = [] + timestamp = datetime.now(timezone.utc).replace( + microsecond=0).astimezone().isoformat() + try: + devices = magnumReader.getDevices() + if len(devices) != 0: + alldata = {} + alldata["datetime"] = timestamp + alldata["device"] = 'MAGNUM' + alldata['comm_device'] = magnumReader.getComm_Device() + magnumdata = [] + for device in devices: + data = {} + data["device"] = device["device"] + data["data"] = device["data"] + magnumdata.append(data) + alldata["data"] = magnumdata + commdevices.append(alldata) + if args.allinone: + alldata = magnumReader.allinone(alldata) + # Now the heavy lifting + try: + db_connection = mariadb.connect( + user=args.db_username, + password=args.db_password, + host=args.db_host, + port=args.db_port, + database=args.db_database + ) + db_connection.autocommit = True + cursor = db_connection.cursor() + if first_time: + initialize_tables_def(alldata) + first_time = False + post_data(alldata) + db_connection.commit() + db_connection.close() + except mariadb.Error as e: + print(f"Error connecting to Database Platform: {e}") + sys.exit(1) + + except Exception as e: + print(str(e)) + if args.interval == 0: + break + interval = time.time() - start + sleep = args.interval - interval + if sleep > 0: + time.sleep(sleep) + + +def sigint_handler(signal, frame): + print('Interrupted. Shutting down.') + sys.exit(0) + + +def initialize_tables_def(allstuff): + global args, cursor, db_columns + if type(allstuff) != list: + allstuff = [allstuff] + for alldata in allstuff: + for data in alldata['data']: + db_table = data['device'] + query = f"SELECT GROUP_CONCAT(COLUMN_NAME) AS COLUMNS FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '{args.db_database}' AND TABLE_NAME = '{db_table}';" + try: + cursor.execute(query) + fields = cursor.fetchone() + field_names = fields[0].split(",") + db_columns[db_table] = field_names + except Exception as e: + print( + f"Unable to find table {db_table}: logging of this data will be ignored.") + if len(db_columns) == 0: + print('No tables to load. Shutting down.') + sys.exit(1) + return + + +def post_data(alldata): + global args, cursor, db_columns + if type(alldata) == list: + alldata = alldata[0] + inserts = {} + for data in alldata['data']: + db_table = data['device'] + if db_table not in db_columns: + return + timestamp = datetime.fromisoformat( + alldata['datetime']).strftime('%Y-%m-%d %H:%M:%S') + if 'timestamp' in db_columns[db_table]: + inserts['timestamp'] = f"\"{timestamp}\"" + elif 'datetime' in db_columns[db_table]: + inserts['datetime'] = f"\"{timestamp}\"" + for field in db_columns[db_table]: + if field in data: + if data[field] != None: + value = data[field] + if type(value) == bool: + if value: + value = str(1) + else: + value = str(0) + elif type(value) == float: + value = str(value) + elif type(value) == int: + value = str(value) + else: + value = f"\"{value}\"" + inserts[field] = value + columns = ', '.join(inserts.keys()) + values = list(inserts.values()) + values = ', '.join(values) + query = f"INSERT INTO {db_table} ({columns}) VALUES ({values});" + try: + cursor.execute(query) + except Exception as e: + print(f"Error updating {db_table}:{e}") + return + + +if __name__ == '__main__': + main() diff --git a/examples/mqttlogger-2.py b/examples/mqttlogger-2.py index 6d8b55e..5698920 100644 --- a/examples/mqttlogger-2.py +++ b/examples/mqttlogger-2.py @@ -21,7 +21,6 @@ import sys import time import uuid -from collections import OrderedDict from datetime import datetime, timezone import paho.mqtt.client as mqtt @@ -65,7 +64,7 @@ def publish_data(): try: devices = magnumReader.getDevices() if len(devices) != 0: - data = OrderedDict() + data = {} now = int(time.time()) data["datetime"] = datetime.now(timezone.utc).replace( microsecond=0).astimezone().isoformat() @@ -114,7 +113,7 @@ def publish_data(): if args.topic[-1] != "/": args.topic += "/" print(f"Options:{str(args)[10:-1]}") -magnumReaders = dict() +magnumReaders = {} for device in args.device: try: magnumReader = Magnum(device=device, packets=args.packets, trace=args.trace, diff --git a/examples/mqttlogger.py b/examples/mqttlogger.py index a6a4b9d..c422a79 100644 --- a/examples/mqttlogger.py +++ b/examples/mqttlogger.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # # Copyright (c) 2018-2022 Charles Godwin # @@ -19,7 +20,6 @@ import sys import time import uuid -from collections import OrderedDict from datetime import datetime, timezone import paho.mqtt.client as mqtt @@ -81,7 +81,7 @@ def sigint_handler(signal, frame): if args.topic[-1] != "/": args.topic += "/" print("Options:{}".format(str(args).replace("Namespace(", "").replace(")", ""))) -magnumReaders = dict() +magnumReaders = {} for device in args.device: try: magnumReader = Magnum(device=device, packets=args.packets, trace=args.trace, @@ -117,7 +117,7 @@ def sigint_handler(signal, frame): try: devices = magnumReader.getDevices() if len(devices) != 0: - data = OrderedDict() + data = {} now = int(time.time()) data["datetime"] = datetime.now(timezone.utc).replace( microsecond=0).astimezone().isoformat() diff --git a/magnum/__init__.py b/magnum/__init__.py index 67e8fa0..f55b9ae 100644 --- a/magnum/__init__.py +++ b/magnum/__init__.py @@ -1,10 +1,10 @@ # -# Copyright (c) 2018-2022 Charles Godwin +# Copyright (c) 2018-2023 Charles Godwin # # SPDX-License-Identifier: BSD-3-Clause # # BUILDINFO -__version__="2.0.4" +__version__="2.0.5" # # names of devices # diff --git a/magnum/aclddevice.py b/magnum/aclddevice.py index e89b5fa..9f57987 100644 --- a/magnum/aclddevice.py +++ b/magnum/aclddevice.py @@ -1,4 +1,3 @@ -from collections import OrderedDict from copy import deepcopy from magnum import * @@ -7,8 +6,8 @@ class ACLDDevice: def __init__(self, trace=False): # self.trace = trace self.trace = True # force packet dump - self.data = OrderedDict() - self.deviceData = OrderedDict() + self.data = {} + self.deviceData = {} self.deviceData["device"] = ACLD self.deviceData["data"] = self.data if self.trace: diff --git a/magnum/agsdevice.py b/magnum/agsdevice.py index 09cab9f..14c0d02 100644 --- a/magnum/agsdevice.py +++ b/magnum/agsdevice.py @@ -1,5 +1,4 @@ -from collections import OrderedDict from copy import deepcopy from magnum import * @@ -41,8 +40,8 @@ class AGSDevice: def __init__(self, trace=False): self.trace = trace - self.data = OrderedDict() - self.deviceData = OrderedDict() + self.data = {} + self.deviceData = {} self.deviceData["device"] = AGS self.deviceData["data"] = self.data self.data["revision"] = str('0.0') diff --git a/magnum/bmkdevice.py b/magnum/bmkdevice.py index 85bf806..f2f2bee 100644 --- a/magnum/bmkdevice.py +++ b/magnum/bmkdevice.py @@ -1,5 +1,4 @@ -from collections import OrderedDict from copy import deepcopy from magnum import * @@ -7,8 +6,8 @@ class BMKDevice: def __init__(self, trace=False): self.trace = trace - self.data = OrderedDict() - self.deviceData = OrderedDict() + self.data = {} + self.deviceData = {} self.deviceData["device"] = BMK self.deviceData["data"] = self.data self.data["revision"] = str(0.0) diff --git a/magnum/inverterdevice.py b/magnum/inverterdevice.py index 83cf8fc..cdb2fe3 100644 --- a/magnum/inverterdevice.py +++ b/magnum/inverterdevice.py @@ -1,5 +1,4 @@ -from collections import OrderedDict from copy import deepcopy from magnum import * @@ -101,8 +100,8 @@ class InverterDevice: def __init__(self, trace=False): self.trace = trace - self.data = OrderedDict() - self.deviceData = OrderedDict() + self.data = {} + self.deviceData = {} self.deviceData["device"] = INVERTER self.deviceData["data"] = self.data self.data["revision"] = str(0.0) diff --git a/magnum/mag2sql.py b/magnum/mag2sql.py new file mode 100644 index 0000000..0a55e55 --- /dev/null +++ b/magnum/mag2sql.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2023 Charles Godwin +# +# SPDX-License-Identifier: BSD-3-Clause +# +# This code is provided as an example of generating MySQL schema definition from a JSON file generate by magdump +# run the program with --help for details of options. +# python3 mag2sql.py --help +# +# magdump|python3 mag2sql +# +import sys +import json +import argparse + + +def main(): + parser = argparse.ArgumentParser(description="Magnum JSON to MySQL Generator", prog="Magnum JSON 2 SQL", fromfile_prefix_chars='@', + epilog="Expects input to be JSON generated by magdump. Refer to https://github.com/CharlesGodwin/pymagnum for details") + parser.add_argument("-i", "--input", default=None, + help="Input file name `None` implies stdin (default: %(default)s)") + args = parser.parse_args() + if args.input == None: + inputjson = sys.stdin.read() + alldata = json.loads(inputjson) + else: + f = open(args.input, "r") + alldata = json.load(f) + db_database = alldata['device'] + print(f"create schema if not exists {db_database};") + print(f"use {db_database};") + for item in alldata['data']: + table = item['device'] + print(f"create table if not exists {table} (") + keyfield = "datetime" + print(f"\t`{keyfield}` datetime DEFAULT now(),") + for variable, value in item['data'].items(): + if type(value) == float: + dbtype = "float" + elif type(value) == int: + dbtype = "integer" + elif type(value) == bool: + dbtype = "integer(1)" + else: + dbtype = "varchar(25)" + print(f"\t`{variable}` {dbtype} default NULL,") + print( + f"\tPRIMARY KEY (`{keyfield}`)\n\t) ENGINE=InnoDB;\n") + +if __name__ == '__main__': + main() diff --git a/magnum/magdump.py b/magnum/magdump.py index d7fc88f..b11c6ee 100644 --- a/magnum/magdump.py +++ b/magnum/magdump.py @@ -11,7 +11,6 @@ import sys import time -from collections import OrderedDict from datetime import datetime, timezone # from tzlocal import get_localzone @@ -49,7 +48,11 @@ def main(): help="Add most recent raw packet(s) info to data (default: %(default)s)") seldom.add_argument("--nocleanup", action="store_true", default=False, dest='cleanpackets', help="Suppress clean up of unknown packets (default: False)") + seldom.add_argument("--allinone", action="store_true", default=False, + help="Process data as a flat single row (default: %(default)s)") args = parser.magnum_parse_args() + if hasattr(args, 'v1'): # a relic but not harmful + args.allinone = True if args.verbose: print('Magnum Dump Version:{0}'.format(magnum.__version__)) print("Options:{0}".format(str(args) @@ -58,7 +61,7 @@ def main(): .replace('[', '') .replace('\'', '') .replace(']', ''))) - magnumReaders = dict() + magnumReaders = {} for device in args.device: try: magnumReader = Magnum(device=device, packets=args.packets, trace=args.trace, @@ -85,17 +88,19 @@ def main(): try: devices = magnumReader.getDevices() if len(devices) != 0: - alldata = OrderedDict() + alldata = {} alldata["datetime"] = timestamp alldata["device"] = 'MAGNUM' alldata['comm_device'] = comm_device magnumdata = [] for device in devices: - data = OrderedDict() + data = {} data["device"] = device["device"] data["data"] = device["data"] magnumdata.append(data) alldata["data"] = magnumdata + if args.allinone: + alldata = magnumReader.allinone(alldata) commdevices.append(alldata) except Exception as e: print("{0} {1}".format(comm_device, str(e))) diff --git a/magnum/magnum.py b/magnum/magnum.py index d451275..cb426ba 100644 --- a/magnum/magnum.py +++ b/magnum/magnum.py @@ -79,7 +79,7 @@ class Magnum: REMOTE_C2: default_remote + '7B', # PT100 not used REMOTE_C3: default_remote + '7B', # PT100 not used REMOTE_D0: default_remote + '7B', # ACLD Needs work - ACLD_D1: '7B', # ACLD Needs work + ACLD_D1: '7B', # ACLD Needs work RTR_91: 'BB', UNKNOWN: '' } @@ -442,3 +442,48 @@ def getDevices(self): if deviceinfo: devices.append(deviceinfo) return devices + +# 2023-11-08 15:22:24 Added + +# This merges all device data into one long dictionary. Each variable is prefixed with device name +# consistent with the values used in the old Java based magnum software (v1) +# + def allinone(self, devices): + returndata = [] + + deviceprefixes = {"INVERTER": "INV", + "AGS": "AGS", + "BMK": "BMK", + "RTR": "RTR", + "PT100": "PT", + "REMOTE": "ARC", + "RTR": "RTR" + } + + if (type(devices) != list): + devices = [devices] + for device in devices: + newdata = {} + newdata['datetime'] = device['datetime'] + newdata['comm_device'] = device['comm_device'] + newdata['device'] = device["device"] + data = [] + itemdata={} + for item in device['data']: + if item['device'] in deviceprefixes: + deviceprefix = deviceprefixes[item['device']] + else: + deviceprefix = item['device'] + itemstuff = item['data'] + for itemkey, itemvalue in itemstuff.items(): + key = f"{deviceprefix}_{itemkey}" + itemdata[key] = itemvalue + newblock = {} + newblock['device'] = "log_data" + newblock['data'] = dict(sorted(itemdata.items())) + newdata['data'] = [newblock] + returndata.append(newdata) + if len(returndata) == 1: + return returndata[0] + else: + return returndata diff --git a/magnum/magtest.py b/magnum/magtest.py index c229b2b..ab2ff6c 100644 --- a/magnum/magtest.py +++ b/magnum/magtest.py @@ -66,6 +66,8 @@ def main(): help="Write copy of program output to log file in current directory (default: %(default)s)") args = parser.magnum_parse_args() # Only supports one device + if len(args.device) > 1: + parser.error("magdump only supports 1 device at a time.") args.device = args.device[0] if args.log: logfile = os.path.join(os.getcwd(), "magtest_" + time.strftime("%Y-%m-%dT%H-%M-%S") + ".txt") @@ -85,7 +87,7 @@ def main(): duration = time.time() - start unknown = 0 formatstring = "Length:{0:2} {1:10}=>{2}" - device_list = dict() + device_list = {} for item in [magnum.INVERTER, magnum.REMOTE, magnum.RTR, magnum.BMK, magnum.AGS, magnum.PT100, magnum.ACLD]: device_list[item] = 'NA' device_list[magnum.RTR] = False diff --git a/magnum/pt100device.py b/magnum/pt100device.py index 183157e..06b14fc 100644 --- a/magnum/pt100device.py +++ b/magnum/pt100device.py @@ -1,6 +1,5 @@ import math -from collections import OrderedDict from copy import deepcopy from magnum import * @@ -10,8 +9,8 @@ class PT100Device: def __init__(self, trace=False): # self.trace = trace self.trace = True # force packet dump for now - self.data = OrderedDict() - self.deviceData = OrderedDict() + self.data = {} + self.deviceData = {} self.deviceData["device"] = PT100 self.deviceData["data"] = self.data if self.trace: diff --git a/magnum/remotedevice.py b/magnum/remotedevice.py index 5e99f71..c44659a 100644 --- a/magnum/remotedevice.py +++ b/magnum/remotedevice.py @@ -1,5 +1,4 @@ -from collections import OrderedDict from copy import deepcopy from magnum import * @@ -20,13 +19,13 @@ class RemoteDevice: def __init__(self, trace=False): self.trace = trace - self.data = OrderedDict() - self.deviceData = OrderedDict() + self.data = {} + self.deviceData = {} self.deviceData["device"] = REMOTE self.deviceData["data"] = self.data if self.trace: self.deviceData["trace"] = [] - self.data["revision"] = "0.0" + self.data["revision"] = str("0.0") # self.data["action"] = 0 self.data["searchwatts"] = 0 self.data["batterysize"] = 0 @@ -123,7 +122,7 @@ def setBaseValues(self, unpacked): self.data["battype"] = value self.data["chargeramps"] = unpacked[4] self.data["ainput"] = unpacked[5] - self.data["revision"] = unpacked[6] / 10 + self.data["revision"] = str(unpacked[6] / 10) value = unpacked[7] self.data["parallel"] = (value & 0x0f) * 10 # self.data["force_charge"] = value & 0xf0 diff --git a/magnum/rtrdevice.py b/magnum/rtrdevice.py index 62c0a77..3f9ff91 100644 --- a/magnum/rtrdevice.py +++ b/magnum/rtrdevice.py @@ -1,4 +1,3 @@ -from collections import OrderedDict from copy import deepcopy from magnum import * @@ -6,13 +5,13 @@ class RTRDevice: def __init__(self, trace=False): self.trace = trace - self.data = OrderedDict() - self.deviceData = OrderedDict() + self.data = {} + self.deviceData = {} self.deviceData["device"] = RTR self.deviceData["data"] = self.data if self.trace: self.deviceData["trace"] = [] - self.data["revision"] = "0.0" + self.data["revision"] = "0.0" def parse(self, packet): packetType = packet[0] diff --git a/setup.py b/setup.py index 6f53194..adcb595 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,8 @@ author='Charles Godwin', author_email='magnum@godwin.ca', py_modules=['magnum.magtest', - 'magnum.magdump'], + 'magnum.magdump', + 'magnum.mag2sql'], long_description=long_description, long_description_content_type="text/x-rst", license="BSD", @@ -34,7 +35,8 @@ entry_points={ 'console_scripts': [ 'magdump = magnum.magdump:main', - 'magtest = magnum.magtest:main' + 'magtest = magnum.magtest:main', + 'mag2sql = magnum.mag2sql:main' ], }, keywords='Magnum Energy Renewable Solar Network RS485 IoT' diff --git a/testdata/allpackets.sql b/testdata/allpackets.sql new file mode 100644 index 0000000..3f2a064 --- /dev/null +++ b/testdata/allpackets.sql @@ -0,0 +1,136 @@ +create schema if not exists MAGNUM; +use MAGNUM; +create table if not exists INVERTER ( + `datetime` datetime DEFAULT now(), + `revision` varchar(25) default NULL, + `mode` integer default NULL, + `mode_text` varchar(25) default NULL, + `fault` integer default NULL, + `fault_text` varchar(25) default NULL, + `vdc` float default NULL, + `adc` float default NULL, + `VACout` float default NULL, + `VACin` float default NULL, + `invled` integer default NULL, + `invled_text` varchar(25) default NULL, + `chgled` integer default NULL, + `chgled_text` varchar(25) default NULL, + `bat` float default NULL, + `tfmr` float default NULL, + `fet` float default NULL, + `model` integer default NULL, + `model_text` varchar(25) default NULL, + `stackmode` integer default NULL, + `stackmode_text` varchar(25) default NULL, + `AACin` float default NULL, + `AACout` float default NULL, + `Hz` float default NULL, + PRIMARY KEY (`datetime`) + ) ENGINE=InnoDB; + +create table if not exists REMOTE ( + `datetime` datetime DEFAULT now(), + `revision` varchar(25) default NULL, + `searchwatts` integer default NULL, + `batterysize` integer default NULL, + `battype` integer default NULL, + `absorb` integer default NULL, + `chargeramps` integer default NULL, + `ainput` integer default NULL, + `parallel` integer default NULL, + `lbco` float default NULL, + `vaccutout` float default NULL, + `vsfloat` float default NULL, + `vEQ` float default NULL, + `absorbtime` float default NULL, + `runtime` float default NULL, + `starttemp` float default NULL, + `startvdc` float default NULL, + `quiettime` integer default NULL, + `begintime` integer default NULL, + `stoptime` integer default NULL, + `vdcstop` float default NULL, + `voltstartdelay` integer default NULL, + `voltstopdelay` integer default NULL, + `maxrun` float default NULL, + `socstart` integer default NULL, + `socstop` integer default NULL, + `ampstart` float default NULL, + `ampsstartdelay` integer default NULL, + `ampstop` integer default NULL, + `ampsstopdelay` integer default NULL, + `quietbegintime` integer default NULL, + `quietendtime` integer default NULL, + `exercisedays` integer default NULL, + `exercisestart` integer default NULL, + `exerciseruntime` integer default NULL, + `topoff` integer default NULL, + `warmup` integer default NULL, + `cool` integer default NULL, + `batteryefficiency` integer default NULL, + PRIMARY KEY (`datetime`) + ) ENGINE=InnoDB; + +create table if not exists BMK ( + `datetime` datetime DEFAULT now(), + `revision` varchar(25) default NULL, + `soc` integer default NULL, + `vdc` float default NULL, + `adc` float default NULL, + `vmin` float default NULL, + `vmax` float default NULL, + `amph` integer default NULL, + `amphtrip` float default NULL, + `amphout` integer default NULL, + `Fault` integer default NULL, + `Fault_Text` varchar(25) default NULL, + PRIMARY KEY (`datetime`) + ) ENGINE=InnoDB; + +create table if not exists AGS ( + `datetime` datetime DEFAULT now(), + `revision` varchar(25) default NULL, + `status` integer default NULL, + `status_text` varchar(25) default NULL, + `running` integer(1) default NULL, + `temp` float default NULL, + `runtime` float default NULL, + `gen_last_run` integer default NULL, + `last_full_soc` integer default NULL, + `gen_total_run` integer default NULL, + `vdc` float default NULL, + PRIMARY KEY (`datetime`) + ) ENGINE=InnoDB; + +create table if not exists RTR ( + `datetime` datetime DEFAULT now(), + `revision` varchar(25) default NULL, + PRIMARY KEY (`datetime`) + ) ENGINE=InnoDB; + +create table if not exists PT100 ( + `datetime` datetime DEFAULT now(), + `revision` varchar(25) default NULL, + `address` integer default NULL, + `mode` integer default NULL, + `mode_text` varchar(25) default NULL, + `mode_hex` varchar(25) default NULL, + `regulation` integer default NULL, + `regulation_text` varchar(25) default NULL, + `fault` integer default NULL, + `fault_text` varchar(25) default NULL, + `battery` float default NULL, + `battery_amps` float default NULL, + `pv_voltage` float default NULL, + `charge_time` float default NULL, + `target_battery_voltage` float default NULL, + `relay_state` integer default NULL, + `alarm_state` integer default NULL, + `fan_on` integer default NULL, + `day` integer default NULL, + `battery_temperature` float default NULL, + `inductor_temperature` float default NULL, + `fet_temperature` float default NULL, + PRIMARY KEY (`datetime`) + ) ENGINE=InnoDB; +