From 9a75e585dd767223a44a382a92bcd830e35237d7 Mon Sep 17 00:00:00 2001 From: Joseph McKinsey Date: Fri, 24 May 2024 13:44:24 -0600 Subject: [PATCH 01/19] Split web server into separate package - Eliminated npm part in setup.py commands - Created separate package TODO - rename to helics_cli_extras - move observer stuff to helics_cli_extras - move testing if necessary - setup ci/cd and PyPI package --- .gitignore | 1 + helics/cli.py | 100 +++++++++++---- {client => helics_web/client}/.eslintrc.cjs | 0 {client => helics_web/client}/.gitignore | 0 {client => helics_web/client}/.npmrc | 0 {client => helics_web/client}/.prettierrc | 0 {client => helics_web/client}/README.md | 0 .../client}/package-lock.json | 0 {client => helics_web/client}/package.json | 0 .../client}/playwright.config.js | 0 .../client}/postcss.config.cjs | 0 {client => helics_web/client}/src/app.css | 0 {client => helics_web/client}/src/app.d.ts | 0 {client => helics_web/client}/src/app.html | 0 .../client}/src/lib/BrokerLayout.svelte | 0 .../client}/src/lib/DataFlowGraph.svelte | 0 .../client}/src/lib/Navbar.svelte | 0 .../client}/src/lib/ProfilePlot.svelte | 0 .../client}/src/lib/Switch.svelte | 0 .../client}/src/lib/Table.svelte | 0 .../client}/src/lib/Topology.svelte | 0 .../client}/src/lib/stores.ts | 0 .../client}/src/routes/__layout.svelte | 0 .../client}/src/routes/broker.svelte | 0 .../client}/src/routes/index.svelte | 0 .../client}/src/routes/observe.svelte | 0 .../client}/src/routes/profile.svelte | 0 .../client}/src/routes/run.svelte | 0 .../client}/static/favicon.png | Bin .../client}/svelte.config.js | 0 .../client}/tailwind.config.cjs | 0 {client => helics_web/client}/tests/test.js | 0 {client => helics_web/client}/tsconfig.json | 0 .../helics_web}/__init__.py | 88 +++++++++---- helics_web/pyproject.toml | 74 +++++++++++ setup.py | 119 ++---------------- 36 files changed, 227 insertions(+), 155 deletions(-) rename {client => helics_web/client}/.eslintrc.cjs (100%) rename {client => helics_web/client}/.gitignore (100%) rename {client => helics_web/client}/.npmrc (100%) rename {client => helics_web/client}/.prettierrc (100%) rename {client => helics_web/client}/README.md (100%) rename {client => helics_web/client}/package-lock.json (100%) rename {client => helics_web/client}/package.json (100%) rename {client => helics_web/client}/playwright.config.js (100%) rename {client => helics_web/client}/postcss.config.cjs (100%) rename {client => helics_web/client}/src/app.css (100%) rename {client => helics_web/client}/src/app.d.ts (100%) rename {client => helics_web/client}/src/app.html (100%) rename {client => helics_web/client}/src/lib/BrokerLayout.svelte (100%) rename {client => helics_web/client}/src/lib/DataFlowGraph.svelte (100%) rename {client => helics_web/client}/src/lib/Navbar.svelte (100%) rename {client => helics_web/client}/src/lib/ProfilePlot.svelte (100%) rename {client => helics_web/client}/src/lib/Switch.svelte (100%) rename {client => helics_web/client}/src/lib/Table.svelte (100%) rename {client => helics_web/client}/src/lib/Topology.svelte (100%) rename {client => helics_web/client}/src/lib/stores.ts (100%) rename {client => helics_web/client}/src/routes/__layout.svelte (100%) rename {client => helics_web/client}/src/routes/broker.svelte (100%) rename {client => helics_web/client}/src/routes/index.svelte (100%) rename {client => helics_web/client}/src/routes/observe.svelte (100%) rename {client => helics_web/client}/src/routes/profile.svelte (100%) rename {client => helics_web/client}/src/routes/run.svelte (100%) rename {client => helics_web/client}/static/favicon.png (100%) rename {client => helics_web/client}/svelte.config.js (100%) rename {client => helics_web/client}/tailwind.config.cjs (100%) rename {client => helics_web/client}/tests/test.js (100%) rename {client => helics_web/client}/tsconfig.json (100%) rename {helics/flaskr => helics_web/helics_web}/__init__.py (87%) create mode 100644 helics_web/pyproject.toml diff --git a/.gitignore b/.gitignore index 4a5baff7..5f54e70e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ __helics-server helics/static +helics_web/helics_web/static helics/install docs/api _source diff --git a/helics/cli.py b/helics/cli.py index ab6bf127..b65d8d38 100644 --- a/helics/cli.py +++ b/helics/cli.py @@ -31,7 +31,9 @@ def _get_version(): try: import helics as h - helics_version = "Python HELICS version {}\n\nHELICS Library version {}".format(h.__version__, h.helicsGetVersion()) + helics_version = "Python HELICS version {}\n\nHELICS Library version {}".format( + h.__version__, h.helicsGetVersion() + ) except ImportError: helics_version = "Python `helics` package not installed. Install using `pip install helics --upgrade`." try: @@ -47,28 +49,32 @@ def _get_version(): import helics_apps as ha if not h.helicsGetVersion().startswith(ha.__version__.strip("v")): - echo("`helics` and `helics-apps` versions don't match. You may want to run `pip install helics helics-apps --upgrade`.") + echo( + "`helics` and `helics-apps` versions don't match. You may want to run `pip install helics helics-apps --upgrade`." + ) except ImportError: pass try: import flask except ImportError: - echo('helics-cli\'s web interface is not installed. You may want to run `pip install "helics[cli]"`.') + echo( + 'helics-cli\'s web interface is not installed. You may want to run `pip install "helics[cli]"`.' + ) try: import sqlalchemy except ImportError: - echo('helics-cli\'s observer functionality is not installed. You may want to run `pip install "helics[cli]"`.') + echo( + 'helics-cli\'s observer functionality is not installed. You may want to run `pip install "helics[cli]"`.' + ) return """{} {} {} -""".format( - __version__, helics_version, helics_apps_version - ).strip() +""".format(__version__, helics_version, helics_apps_version).strip() VERSION = _get_version() @@ -99,15 +105,21 @@ def server(open: bool): Run helics web server to access web interface """ import webbrowser - from . import flaskr + import helics_web if open: webbrowser.open("http://127.0.0.1:5000", 1) - flaskr.run() + helics_web.run() @cli.command() -@click.option("--db-folder", prompt="path to database folder", type=click.Path(exists=True, file_okay=False, writable=True, path_type=pathlib.Path)) +@click.option( + "--db-folder", + prompt="path to database folder", + type=click.Path( + exists=True, file_okay=False, writable=True, path_type=pathlib.Path + ), +) def observer(db_folder: pathlib.Path): """ Run helics observer and write data to sqlite file @@ -131,7 +143,14 @@ def observer(db_folder: pathlib.Path): default=True, help="Invert plot", ) -@click.option("--save", prompt=True, prompt_required=False, type=click.Path(), default=None, help="Path to save the plot") +@click.option( + "--save", + prompt=True, + prompt_required=False, + type=click.Path(), + default=None, + help="Path to save the plot", +) def profile_plot(path, save, invert): """ Plot profiler output using matplotlib @@ -164,7 +183,9 @@ def fetch(url, data={}, method="POST"): r.add_header("Content-Length", str(len(bytes))) try: with urllib.request.urlopen(r, bytes) as response: - return json.loads(response.read().decode(response.info().get_param("charset") or "utf-8")) + return json.loads( + response.read().decode(response.info().get_param("charset") or "utf-8") + ) except Exception as e: logger.exception("Unable to post to helics-cli server: {}".format(e)) @@ -179,7 +200,12 @@ def fetch(url, data={}, method="POST"): @click.option("--silent", is_flag=True) @click.option("--connect-server", is_flag=True) @click.option("--no-log-files", is_flag=True, default=False) -@click.option("--no-kill-on-error", is_flag=True, default=False, help="Do not kill all federates on error") +@click.option( + "--no-kill-on-error", + is_flag=True, + default=False, + help="Do not kill all federates on error", +) def run(path, silent, connect_server, no_log_files, no_kill_on_error): """ Run HELICS federation @@ -192,7 +218,12 @@ def run(path, silent, connect_server, no_log_files, no_kill_on_error): if connect_server: with urllib.request.urlopen(r) as response: helics_server_available = ( - json.loads(response.read().decode(response.info().get_param("charset") or "utf-8")).get("status", None) == 200 + json.loads( + response.read().decode( + response.info().get_param("charset") or "utf-8" + ) + ).get("status", None) + == 200 ) except Exception: warn("Unable to connect to helics-cli web server") @@ -205,7 +236,9 @@ def run(path, silent, connect_server, no_log_files, no_kill_on_error): if not os.path.exists(path_to_config): info( - "Unable to find file `config.json` in path: {path_to_config}".format(path_to_config=path_to_config), + "Unable to find file `config.json` in path: {path_to_config}".format( + path_to_config=path_to_config + ), ) return None @@ -218,17 +251,28 @@ def run(path, silent, connect_server, no_log_files, no_kill_on_error): if "broker" in config.keys() and config["broker"] is not False: if not silent: info( - "Adding auto broker (i.e. `helics_broker -f{f}`) to helics-cli subprocesses.".format(f=len(config["federates"])), + "Adding auto broker (i.e. `helics_broker -f{f}`) to helics-cli subprocesses.".format( + f=len(config["federates"]) + ), blink=True, ) config["federates"].append( - {"directory": ".", "exec": "helics_broker -f{}".format(len(config["federates"])), "host": "localhost", "name": "broker"} + { + "directory": ".", + "exec": "helics_broker -f{}".format(len(config["federates"])), + "host": "localhost", + "name": "broker", + } ) names = [c["name"] for c in config["federates"]] if len(set(n for n in names)) != len(config["federates"]): error("Repeated names found in runner.json federates.", blink=True) - for n, c in [(item, count) for item, count in collections.Counter(names).items() if count > 1]: + for n, c in [ + (item, count) + for item, count in collections.Counter(names).items() + if count > 1 + ]: info('Found name "{}" {} times'.format(n, c)) return -1 @@ -241,7 +285,9 @@ def run(path, silent, connect_server, no_log_files, no_kill_on_error): for f in config["federates"]: if not silent: info( - "Running federate {name} as a background process".format(name=f["name"]), + "Running federate {name} as a background process".format( + name=f["name"] + ), ) fname = os.path.abspath(os.path.join(path, "{}.log".format(f["name"]))) @@ -300,8 +346,16 @@ def run(path, silent, connect_server, no_log_files, no_kill_on_error): finally: for p in process_list: t.status(p) - if p.process.returncode != 0 and p.process.returncode != -9 and p.process.returncode is not None: - error("Process {} exited with return code {}".format(p.name, p.process.returncode)) + if ( + p.process.returncode != 0 + and p.process.returncode != -9 + and p.process.returncode is not None + ): + error( + "Process {} exited with return code {}".format( + p.name, p.process.returncode + ) + ) if os.path.exists(p.file): with open(p.file) as f: warn("Last 10 lines of {}.log:".format(p.name), blink=False) @@ -351,7 +405,9 @@ def list_brokers(): else: cmd = "ps aux" - p = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + p = subprocess.Popen( + shlex.split(cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True + ) p.wait() out, _ = p.communicate() for line in out.decode("utf-8").splitlines(): diff --git a/client/.eslintrc.cjs b/helics_web/client/.eslintrc.cjs similarity index 100% rename from client/.eslintrc.cjs rename to helics_web/client/.eslintrc.cjs diff --git a/client/.gitignore b/helics_web/client/.gitignore similarity index 100% rename from client/.gitignore rename to helics_web/client/.gitignore diff --git a/client/.npmrc b/helics_web/client/.npmrc similarity index 100% rename from client/.npmrc rename to helics_web/client/.npmrc diff --git a/client/.prettierrc b/helics_web/client/.prettierrc similarity index 100% rename from client/.prettierrc rename to helics_web/client/.prettierrc diff --git a/client/README.md b/helics_web/client/README.md similarity index 100% rename from client/README.md rename to helics_web/client/README.md diff --git a/client/package-lock.json b/helics_web/client/package-lock.json similarity index 100% rename from client/package-lock.json rename to helics_web/client/package-lock.json diff --git a/client/package.json b/helics_web/client/package.json similarity index 100% rename from client/package.json rename to helics_web/client/package.json diff --git a/client/playwright.config.js b/helics_web/client/playwright.config.js similarity index 100% rename from client/playwright.config.js rename to helics_web/client/playwright.config.js diff --git a/client/postcss.config.cjs b/helics_web/client/postcss.config.cjs similarity index 100% rename from client/postcss.config.cjs rename to helics_web/client/postcss.config.cjs diff --git a/client/src/app.css b/helics_web/client/src/app.css similarity index 100% rename from client/src/app.css rename to helics_web/client/src/app.css diff --git a/client/src/app.d.ts b/helics_web/client/src/app.d.ts similarity index 100% rename from client/src/app.d.ts rename to helics_web/client/src/app.d.ts diff --git a/client/src/app.html b/helics_web/client/src/app.html similarity index 100% rename from client/src/app.html rename to helics_web/client/src/app.html diff --git a/client/src/lib/BrokerLayout.svelte b/helics_web/client/src/lib/BrokerLayout.svelte similarity index 100% rename from client/src/lib/BrokerLayout.svelte rename to helics_web/client/src/lib/BrokerLayout.svelte diff --git a/client/src/lib/DataFlowGraph.svelte b/helics_web/client/src/lib/DataFlowGraph.svelte similarity index 100% rename from client/src/lib/DataFlowGraph.svelte rename to helics_web/client/src/lib/DataFlowGraph.svelte diff --git a/client/src/lib/Navbar.svelte b/helics_web/client/src/lib/Navbar.svelte similarity index 100% rename from client/src/lib/Navbar.svelte rename to helics_web/client/src/lib/Navbar.svelte diff --git a/client/src/lib/ProfilePlot.svelte b/helics_web/client/src/lib/ProfilePlot.svelte similarity index 100% rename from client/src/lib/ProfilePlot.svelte rename to helics_web/client/src/lib/ProfilePlot.svelte diff --git a/client/src/lib/Switch.svelte b/helics_web/client/src/lib/Switch.svelte similarity index 100% rename from client/src/lib/Switch.svelte rename to helics_web/client/src/lib/Switch.svelte diff --git a/client/src/lib/Table.svelte b/helics_web/client/src/lib/Table.svelte similarity index 100% rename from client/src/lib/Table.svelte rename to helics_web/client/src/lib/Table.svelte diff --git a/client/src/lib/Topology.svelte b/helics_web/client/src/lib/Topology.svelte similarity index 100% rename from client/src/lib/Topology.svelte rename to helics_web/client/src/lib/Topology.svelte diff --git a/client/src/lib/stores.ts b/helics_web/client/src/lib/stores.ts similarity index 100% rename from client/src/lib/stores.ts rename to helics_web/client/src/lib/stores.ts diff --git a/client/src/routes/__layout.svelte b/helics_web/client/src/routes/__layout.svelte similarity index 100% rename from client/src/routes/__layout.svelte rename to helics_web/client/src/routes/__layout.svelte diff --git a/client/src/routes/broker.svelte b/helics_web/client/src/routes/broker.svelte similarity index 100% rename from client/src/routes/broker.svelte rename to helics_web/client/src/routes/broker.svelte diff --git a/client/src/routes/index.svelte b/helics_web/client/src/routes/index.svelte similarity index 100% rename from client/src/routes/index.svelte rename to helics_web/client/src/routes/index.svelte diff --git a/client/src/routes/observe.svelte b/helics_web/client/src/routes/observe.svelte similarity index 100% rename from client/src/routes/observe.svelte rename to helics_web/client/src/routes/observe.svelte diff --git a/client/src/routes/profile.svelte b/helics_web/client/src/routes/profile.svelte similarity index 100% rename from client/src/routes/profile.svelte rename to helics_web/client/src/routes/profile.svelte diff --git a/client/src/routes/run.svelte b/helics_web/client/src/routes/run.svelte similarity index 100% rename from client/src/routes/run.svelte rename to helics_web/client/src/routes/run.svelte diff --git a/client/static/favicon.png b/helics_web/client/static/favicon.png similarity index 100% rename from client/static/favicon.png rename to helics_web/client/static/favicon.png diff --git a/client/svelte.config.js b/helics_web/client/svelte.config.js similarity index 100% rename from client/svelte.config.js rename to helics_web/client/svelte.config.js diff --git a/client/tailwind.config.cjs b/helics_web/client/tailwind.config.cjs similarity index 100% rename from client/tailwind.config.cjs rename to helics_web/client/tailwind.config.cjs diff --git a/client/tests/test.js b/helics_web/client/tests/test.js similarity index 100% rename from client/tests/test.js rename to helics_web/client/tests/test.js diff --git a/client/tsconfig.json b/helics_web/client/tsconfig.json similarity index 100% rename from client/tsconfig.json rename to helics_web/client/tsconfig.json diff --git a/helics/flaskr/__init__.py b/helics_web/helics_web/__init__.py similarity index 87% rename from helics/flaskr/__init__.py rename to helics_web/helics_web/__init__.py index 2a066f07..c877b436 100644 --- a/helics/flaskr/__init__.py +++ b/helics_web/helics_web/__init__.py @@ -18,15 +18,21 @@ import re -from .. import database as db +from helics import database as db current_directory = os.path.realpath(os.path.dirname(__file__)) -app = Flask(__name__.split(".")[0], static_url_path="", static_folder=os.path.join(current_directory, "../static")) +app = Flask( + __name__.split(".")[0], + static_url_path="", + static_folder=os.path.join(current_directory, "static"), +) api = Api(app) CORS(app, resources={r"/api/*": {"origins": "*"}}) -app.config["UPLOAD_FOLDER"] = os.path.abspath(os.path.join(os.getcwd(), "__helics-server")) +app.config["UPLOAD_FOLDER"] = os.path.abspath( + os.path.join(os.getcwd(), "__helics-server") +) cache = { "path": os.path.join(app.config["UPLOAD_FOLDER"], "helics-cli.sqlite.db"), @@ -54,12 +60,16 @@ def get(self): def post(self): parser = reqparse.RequestParser() - parser.add_argument("file", type=werkzeug.datastructures.FileStorage, location="files") + parser.add_argument( + "file", type=werkzeug.datastructures.FileStorage, location="files" + ) args = parser.parse_args() file = args["file"] os.makedirs(app.config["UPLOAD_FOLDER"], exist_ok=True) file.save(os.path.join(app.config["UPLOAD_FOLDER"], "helics-cli.sqlite.db")) - cache["path"] = os.path.join(app.config["UPLOAD_FOLDER"], "helics-cli.sqlite.db") + cache["path"] = os.path.join( + app.config["UPLOAD_FOLDER"], "helics-cli.sqlite.db" + ) dm = DatabaseManager() return {"filename": dm.path_to_helics_db} @@ -80,7 +90,11 @@ class Cores(Resource): def get(self): dm = DatabaseManager() cores = dm.session.query(db.Cores).all() - return [{"id": e.id, "name": e.name, "address": e.address} for e in cores if not e.name.startswith("__observer__")] + return [ + {"id": e.id, "name": e.name, "address": e.address} + for e in cores + if not e.name.startswith("__observer__") + ] api.add_resource(Cores, "/api/observer/cores") @@ -90,7 +104,11 @@ class Federates(Resource): def get(self): dm = DatabaseManager() federates = dm.session.query(db.Federates).all() - return [{"id": f.id, "name": f.name, "parent": f.parent} for f in federates if f.name != "__observer__"] + return [ + {"id": f.id, "name": f.name, "parent": f.parent} + for f in federates + if f.name != "__observer__" + ] api.add_resource(Federates, "/api/observer/federates") @@ -161,13 +179,22 @@ def get(self): data["filename"] = cache["runner-file-name"] if data.get("broker", False) is True: data["federates"].append( - {"directory": ".", "exec": "helics_broker -f{}".format(len(data["federates"])), "host": "localhost", "name": "broker"} + { + "directory": ".", + "exec": "helics_broker -f{}".format(len(data["federates"])), + "host": "localhost", + "name": "broker", + } ) for federate in data["federates"]: if federate["directory"].startswith("."): - federate["directory"] = os.path.abspath(os.path.join(data["folder"], federate["directory"])) + federate["directory"] = os.path.abspath( + os.path.join(data["folder"], federate["directory"]) + ) federate["old_name"] = federate["name"] - if os.path.exists(os.path.join(data["folder"], "{}.log".format(federate["name"]))): + if os.path.exists( + os.path.join(data["folder"], "{}.log".format(federate["name"])) + ): federate["log_available"] = True else: federate["log_available"] = False @@ -180,7 +207,9 @@ def get(self): def post(self): parser = reqparse.RequestParser() - parser.add_argument("file", type=werkzeug.datastructures.FileStorage, location="files") + parser.add_argument( + "file", type=werkzeug.datastructures.FileStorage, location="files" + ) args = parser.parse_args() file = args["file"] path = cache["runner-folder"] @@ -200,7 +229,9 @@ def post(self): args = parser.parse_args() name = args["name"] cache["runner-file-name"] = name - cache["runner-path"] = os.path.join(cache["runner-folder"], cache["runner-file-name"]) + cache["runner-path"] = os.path.join( + cache["runner-folder"], cache["runner-file-name"] + ) api.add_resource(RunnerFileName, "/api/runner/file/name") @@ -213,7 +244,9 @@ def post(self): args = parser.parse_args() folder = args["folder"] cache["runner-folder"] = os.path.abspath(os.path.expanduser(folder)) - cache["runner-path"] = os.path.join(cache["runner-folder"], cache["runner-file-name"]) + cache["runner-path"] = os.path.join( + cache["runner-folder"], cache["runner-file-name"] + ) api.add_resource(RunnerFileFolder, "/api/runner/file/folder") @@ -320,7 +353,9 @@ class RunnerLog(Resource): def get(self, name): with open(cache["runner-path"]) as f: data = json.loads(f.read()) - with open(os.path.join(os.path.dirname(cache["runner-path"]), "{}.log".format(name))) as f: + with open( + os.path.join(os.path.dirname(cache["runner-path"]), "{}.log".format(name)) + ) as f: data = f.read() return {"log": data} @@ -340,7 +375,11 @@ def get(self): def post(self): if self.get()["status"]: self.delete() - p = subprocess.Popen(shlex.split("helics run --path {} --connect-server".format(cache["runner-path"]))) + p = subprocess.Popen( + shlex.split( + "helics run --path {} --connect-server".format(cache["runner-path"]) + ) + ) self.runner_server["process"] = p return {"status": True} @@ -396,7 +435,9 @@ def get(self): def post(self): parser = reqparse.RequestParser() parser.add_argument("name", type=str, required=True, help="Name of federate") - parser.add_argument("status", type=str, required=True, help="Status of federate") + parser.add_argument( + "status", type=str, required=True, help="Status of federate" + ) args = parser.parse_args() status_tracker[args["name"]] = args["status"] return {"status": status_tracker[args["name"]]} @@ -414,7 +455,6 @@ def get(self): class Profile(Resource): - # SenderFederate2[131074](initializing)HELICS CODE ENTRY<4570827706580384>[t=-1000000] PATTERN = re.compile( r""" @@ -466,7 +506,9 @@ def get(self): for name in set(names): profile[name].append({}) - for (name, state, message, simtime, realtime) in zip(names, states, messages, simtimes, realtimes): + for name, state, message, simtime, realtime in zip( + names, states, messages, simtimes, realtimes + ): if state == "created": continue if "ENTRY" in message and not invert: @@ -499,7 +541,9 @@ def get(self): def post(self): parser = reqparse.RequestParser() - parser.add_argument("file", type=werkzeug.datastructures.FileStorage, location="files") + parser.add_argument( + "file", type=werkzeug.datastructures.FileStorage, location="files" + ) args = parser.parse_args() file = args["file"] os.makedirs(app.config["UPLOAD_FOLDER"], exist_ok=True) @@ -539,7 +583,9 @@ def get(self): def post(self): parser = reqparse.RequestParser() - parser.add_argument("status", type=bool, required=True, help="requested status of broker server") + parser.add_argument( + "status", type=bool, required=True, help="requested status of broker server" + ) args = parser.parse_args() status = args["status"] if status is True and self.broker_server.get("process", None) is not None: @@ -570,7 +616,7 @@ def post(self): @app.route("/", defaults={"path": "index.html"}) @app.route("/") def index(path): - return send_from_directory(os.path.join(current_directory, "..", "static"), path) + return send_from_directory(os.path.join(current_directory, "static"), path) def run(): diff --git a/helics_web/pyproject.toml b/helics_web/pyproject.toml new file mode 100644 index 00000000..7999e09d --- /dev/null +++ b/helics_web/pyproject.toml @@ -0,0 +1,74 @@ +[build-system] +requires = ["setuptools>=61.0.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "helics-web" +version = "0.0.1" +authors = [{ name = "Dheepak Krishnamurthy", email = "me@kdheepak.com" }] +maintainers = [ + { name = "Dheepak Krishnamurthy", email = "me@kdheepak.com" }, + { name = "Ryan Mast", email = "mast9@llnl.gov" }, +] +description = "Python HELICS bindings" +readme = "README.md" +requires-python = ">=3.6" +keywords = ["helics", "co-simulation", "webserver"] +license = { text = "MIT License" } +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: Unix", + "Operating System :: POSIX", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS", + "Operating System :: Microsoft :: Windows", + "Environment :: Console", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Topic :: Utilities", + "Topic :: Software Development", +] +dependencies = [ + "flask>=2", + "requests", + "flask-restful", + "flask-cors", + "pandas", + "SQLAlchemy", +] + +[project.urls] +Homepage = "hhttps://github.com/GMLC-TDC/pyhelics" +Discussions = "https://github.com/GMLC-TDC/HELICS/discussions" +Documentation = "https://python.helics.org/" +"Issue Tracker" = "https://github.com/GMLC-TDC/pyhelics/issues" +"Source Code" = "https://github.com/GMLC-TDC/pyhelics" + +[tool.setuptools.packages.find] +include = ["helics_web", "helics_web.*"] + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.package-data] +"*" = ["static/*"] + + +#[tool.setuptools_scm] +#version_file = "helics_web/_version.py" + +#[tool.pytest.ini_options] +#addopts = ["--import-mode=importlib"] +#pythonpath = "." diff --git a/setup.py b/setup.py index 3d2e4e74..ee218dc2 100755 --- a/setup.py +++ b/setup.py @@ -64,110 +64,6 @@ def update_package_data(distribution): build_py.finalize_options() -def js_prerelease(command, strict=False): - """decorator for building minified js/css prior to another command""" - - class DecoratedCommand(command): - def run(self): - jsdeps = self.distribution.get_command_obj("jsdeps") - if not IS_REPO and all(os.path.exists(t) for t in jsdeps.targets): - # sdist, nothing to do - command.run(self) - return - - try: - self.distribution.run_command("jsdeps") - except Exception as e: - missing = [t for t in jsdeps.targets if not os.path.exists(t)] - if strict or missing: - log.warn("rebuilding js and css failed") - if missing: - log.error("missing files: %s" % missing) - raise e - else: - log.warn("rebuilding js and css failed (not a problem)") - log.warn(str(e)) - command.run(self) - update_package_data(self.distribution) - - return DecoratedCommand - - -class NPM(Command): - description = "install package.json dependencies using npm" - - user_options = [] - - node_modules = os.path.join(NODE_ROOT, "node_modules") - - targets = ["index.html"] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def get_npm_name(self): - npm_name = "npm" - if platform.system() == "Windows": - npm_name = "npm.cmd" - return npm_name - - def has_npm(self): - npm_name = self.get_npm_name() - try: - subprocess.check_call([npm_name, "--version"]) - return True - except: - return False - - def should_run_npm_install(self): - node_modules_exists = os.path.exists(self.node_modules) - return self.has_npm() and not node_modules_exists - - def run(self): - has_npm = self.has_npm() - if not has_npm: - log.error("`npm` unavailable, skipping npm build. If you're running this command using " "sudo, make sure `npm` is available to sudo") - return - - env = os.environ.copy() - env["PATH"] = NPM_PATH - - npm_name = self.get_npm_name() - - if self.should_run_npm_install(): - log.info("Installing build dependencies with npm. " "This may take a while...") - subprocess.check_call( - [npm_name, "install"], - cwd=NODE_ROOT, - stdout=sys.stdout, - stderr=sys.stderr, - ) - os.utime(self.node_modules, None) - - subprocess.check_call( - [npm_name, "run", "build"], - cwd=NODE_ROOT, - stdout=sys.stdout, - stderr=sys.stderr, - ) - - copy_tree(os.path.join(NODE_ROOT, "build"), os.path.join(STATIC_DIR)) - - for t in self.targets: - if not os.path.exists(os.path.join(STATIC_DIR, t)): - msg = "Missing file: %s" % t - if not has_npm: - msg += "\nnpm is required to build a development version " - "of a widget extension" - raise ValueError(msg) - - # update package data in case this created new files - update_package_data(self.distribution) - - def read(*names, **kwargs): with io.open(join(dirname(__file__), *names), encoding=kwargs.get("encoding", "utf8")) as fh: return fh.read() @@ -567,13 +463,12 @@ def get_tag(self): cmdclass = { "download": HELICSDownloadCommand, - "build_ext": js_prerelease(HELICSCMakeBuild), - "bdist_wheel": js_prerelease(HelicsBdistWheel), - "develop": js_prerelease(develop), - "build_py": js_prerelease(build_py), - "egg_info": js_prerelease(egg_info), - "sdist": js_prerelease(sdist, strict=True), - "jsdeps": NPM, + "build_ext": HELICSCMakeBuild, + "bdist_wheel": HelicsBdistWheel, + "develop": develop, + "build_py": build_py, + "egg_info": egg_info, + "sdist": sdist, } @@ -582,7 +477,7 @@ def is_pure(self): return False -helics_cli_install_requires = ["flask>=2", "requests", "flask-restful", "flask-cors", "pandas", "SQLAlchemy", "matplotlib"] +helics_cli_install_requires = ["requests", "helics_web", "pandas", "SQLAlchemy", "matplotlib"] setup( name="helics", From cbe3b238cb9ef4ba99467fa58f7a824058e83cde Mon Sep 17 00:00:00 2001 From: Joseph McKinsey Date: Fri, 7 Jun 2024 10:02:09 -0600 Subject: [PATCH 02/19] Run formatter --- helics/cli.py | 4 +- helics_web/helics_web/__init__.py | 6 +-- helics_web/pyproject.toml | 9 +--- setup.py | 86 +++++++++++++++++++++---------- 4 files changed, 64 insertions(+), 41 deletions(-) diff --git a/helics/cli.py b/helics/cli.py index b65d8d38..5121e53a 100644 --- a/helics/cli.py +++ b/helics/cli.py @@ -3,6 +3,7 @@ HELICS command line interface """ +from dataclasses import dataclass import json import os import io @@ -160,9 +161,6 @@ def profile_plot(path, save, invert): p.plot(p.profile(path, invert), save=save, kind="realtime") -from dataclasses import dataclass - - @dataclass class Job: name: str diff --git a/helics_web/helics_web/__init__.py b/helics_web/helics_web/__init__.py index c877b436..0e67fd44 100644 --- a/helics_web/helics_web/__init__.py +++ b/helics_web/helics_web/__init__.py @@ -1,15 +1,13 @@ # -*- coding: utf-8 -*- -import logging import json import time import os import shlex import subprocess import sys -from dataclasses import dataclass from typing import cast -from flask import Flask, render_template, send_from_directory, request, jsonify +from flask import Flask, send_from_directory from flask_restful import Resource, Api, reqparse, abort from flask_cors import CORS import sqlalchemy as sa @@ -18,6 +16,7 @@ import re +# HELICS must be installed but cannot be a build depencency from helics import database as db current_directory = os.path.realpath(os.path.dirname(__file__)) @@ -627,5 +626,4 @@ def run(): host = "0.0.0.0" cli = sys.modules["flask.cli"] cli.show_server_banner = lambda *x: None - # os.environ["WERKZEUG_RUN_MAIN"] = "true" app.run(host=host, debug=debug) diff --git a/helics_web/pyproject.toml b/helics_web/pyproject.toml index 7999e09d..6d4d3a7e 100644 --- a/helics_web/pyproject.toml +++ b/helics_web/pyproject.toml @@ -41,6 +41,7 @@ classifiers = [ "Topic :: Software Development", ] dependencies = [ + "helics>=3.0.0", "flask>=2", "requests", "flask-restful", @@ -64,11 +65,3 @@ include-package-data = true [tool.setuptools.package-data] "*" = ["static/*"] - - -#[tool.setuptools_scm] -#version_file = "helics_web/_version.py" - -#[tool.pytest.ini_options] -#addopts = ["--import-mode=importlib"] -#pythonpath = "." diff --git a/setup.py b/setup.py index ee218dc2..8ea14b4e 100755 --- a/setup.py +++ b/setup.py @@ -65,12 +65,18 @@ def update_package_data(distribution): def read(*names, **kwargs): - with io.open(join(dirname(__file__), *names), encoding=kwargs.get("encoding", "utf8")) as fh: + with io.open( + join(dirname(__file__), *names), encoding=kwargs.get("encoding", "utf8") + ) as fh: return fh.read() -PYHELICS_VERSION = read(os.path.join(os.path.dirname(__file__), "helics", "_version.py"), encoding="utf-8") -PYHELICS_VERSION = PYHELICS_VERSION.splitlines()[1].split()[2].strip('"').strip("'").lstrip("v") +PYHELICS_VERSION = read( + os.path.join(os.path.dirname(__file__), "helics", "_version.py"), encoding="utf-8" +) +PYHELICS_VERSION = ( + PYHELICS_VERSION.splitlines()[1].split()[2].strip('"').strip("'").lstrip("v") +) HELICS_VERSION = re.findall(r"(?:(\d+\.(?:\d+\.)*\d+))", PYHELICS_VERSION)[0] # HELICS_VERSION = "{}-beta".format(HELICS_VERSION) @@ -78,18 +84,22 @@ def read(*names, **kwargs): CURRENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) HELICS_SOURCE = os.path.join(CURRENT_DIRECTORY, "./_source") -PYHELICS_INSTALL = os.environ.get("PYHELICS_INSTALL", os.path.join(CURRENT_DIRECTORY, "./helics/install")) +PYHELICS_INSTALL = os.environ.get( + "PYHELICS_INSTALL", os.path.join(CURRENT_DIRECTORY, "./helics/install") +) -DOWNLOAD_URL = "https://github.com/GMLC-TDC/HELICS/releases/download/v{version}/Helics-v{version}-source.tar.gz".format(version=HELICS_VERSION) +DOWNLOAD_URL = "https://github.com/GMLC-TDC/HELICS/releases/download/v{version}/Helics-v{version}-source.tar.gz".format( + version=HELICS_VERSION +) def create_default_url(helics_version, plat_name=""): if "macos" in plat_name.lower(): - if helics_version.startswith("3") and int(helics_version.split(".")[1]) >= 1: # >= 3.1.x - default_url = ( - "https://github.com/GMLC-TDC/HELICS/releases/download/v{helics_version}/Helics-{helics_version}-macOS-universal2.zip".format( - helics_version=helics_version - ) + if ( + helics_version.startswith("3") and int(helics_version.split(".")[1]) >= 1 + ): # >= 3.1.x + default_url = "https://github.com/GMLC-TDC/HELICS/releases/download/v{helics_version}/Helics-{helics_version}-macOS-universal2.zip".format( + helics_version=helics_version ) else: default_url = "https://github.com/GMLC-TDC/HELICS/releases/download/v{helics_version}/Helics-{helics_version}-macOS-x86_64.zip".format( @@ -110,11 +120,11 @@ def create_default_url(helics_version, plat_name=""): helics_version=helics_version ) elif platform.system() == "Darwin": - if helics_version.startswith("3") and int(helics_version.split(".")[1]) >= 1: # >= 3.1.x - default_url = ( - "https://github.com/GMLC-TDC/HELICS/releases/download/v{helics_version}/Helics-{helics_version}-macOS-universal2.zip".format( - helics_version=helics_version - ) + if ( + helics_version.startswith("3") and int(helics_version.split(".")[1]) >= 1 + ): # >= 3.1.x + default_url = "https://github.com/GMLC-TDC/HELICS/releases/download/v{helics_version}/Helics-{helics_version}-macOS-universal2.zip".format( + helics_version=helics_version ) else: default_url = "https://github.com/GMLC-TDC/HELICS/releases/download/v{helics_version}/Helics-{helics_version}-macOS-x86_64.zip".format( @@ -243,7 +253,6 @@ def unzip(zip_file_path, output_dir, permission=None): """ extracted_path = None with zipfile.ZipFile(zip_file_path, "r") as zip_ref: - # For each item in the zip file, extract the file and set permissions if available for file_info in zip_ref.infolist(): extracted_path = _extract(file_info, output_dir, zip_ref) @@ -285,10 +294,14 @@ def run(self): unzip("./tmp.zip", self.pyhelics_install) os.remove("./tmp.zip") - if len(os.listdir(self.pyhelics_install)) == 1 and os.listdir(self.pyhelics_install)[0].startswith("Helics-"): + if len(os.listdir(self.pyhelics_install)) == 1 and os.listdir( + self.pyhelics_install + )[0].startswith("Helics-"): tmp = os.listdir(self.pyhelics_install)[0] for folder in os.listdir(os.path.join(self.pyhelics_install, tmp)): - p = Path(os.path.join(self.pyhelics_install, tmp, folder)).absolute() + p = Path( + os.path.join(self.pyhelics_install, tmp, folder) + ).absolute() parent_dir = p.parents[1] p.rename(parent_dir / p.name) else: @@ -320,9 +333,13 @@ def run(self): IGNOREBLOCK = False print("Writing to {}".format(os.path.abspath(self.pyhelics_install))) for file in files: - if not os.path.isfile(os.path.join(self.pyhelics_install, "include", "helics", file)): + if not os.path.isfile( + os.path.join(self.pyhelics_install, "include", "helics", file) + ): continue - with open(os.path.join(self.pyhelics_install, "include", "helics", file)) as f: + with open( + os.path.join(self.pyhelics_install, "include", "helics", file) + ) as f: lines = [] for line in f: if line.startswith("#ifdef __cplusplus"): @@ -339,7 +356,9 @@ def run(self): data = "\n".join(lines) data = data.replace("HELICS_EXPORT", "") data = data.replace("HELICS_DEPRECATED_EXPORT", "") - with open(os.path.join(self.pyhelics_install, "include", "helics", file), "w") as f: + with open( + os.path.join(self.pyhelics_install, "include", "helics", file), "w" + ) as f: f.write(data) @@ -353,14 +372,19 @@ class HELICSCMakeBuild(build_ext): def run(self): try: out = subprocess.check_output(["cmake", "--version"]) - cmake_version = re.search(r"version\s*([\d.]+)", out.decode().lower()).group(1) + cmake_version = re.search( + r"version\s*([\d.]+)", out.decode().lower() + ).group(1) cmake_version = [int(i) for i in cmake_version.split(".")] if cmake_version < [3, 5, 1]: raise RuntimeError("CMake >= 3.5.1 is required to build helics") except OSError: if not os.path.exists(PYHELICS_INSTALL): - raise RuntimeError("CMake must be installed to build the following extensions: " + ", ".join(e.name for e in self.extensions)) + raise RuntimeError( + "CMake must be installed to build the following extensions: " + + ", ".join(e.name for e in self.extensions) + ) for ext in self.extensions: self.build_extension(ext) @@ -397,7 +421,9 @@ def build_extension(self, ext): build_args = ["--config", cfg] if platform.system() == "Windows": - cmake_args += ["-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}".format(cfg.upper(), extdir)] + cmake_args += [ + "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}".format(cfg.upper(), extdir) + ] if sys.maxsize > 2**32: cmake_args += ["-A", "x64"] build_args += ["--", "/m"] @@ -408,7 +434,9 @@ def build_extension(self, ext): env = os.environ.copy() if not os.path.exists(self.build_temp): os.makedirs(self.build_temp) - subprocess.check_call(["cmake", ext.sourcedir] + cmake_args, cwd=self.build_temp, env=env) + subprocess.check_call( + ["cmake", ext.sourcedir] + cmake_args, cwd=self.build_temp, env=env + ) cmd = " ".join(["cmake", "--build", ".", "--target", "install"] + build_args) print(cmd) subprocess.check_call(shlex.split(cmd), cwd=self.build_temp) @@ -477,7 +505,13 @@ def is_pure(self): return False -helics_cli_install_requires = ["requests", "helics_web", "pandas", "SQLAlchemy", "matplotlib"] +helics_cli_install_requires = [ + "requests", + "helics_web==0.0.1", + "pandas", + "SQLAlchemy", + "matplotlib", +] setup( name="helics", From 5869ede8bfbb3ec1975845480c905096c8785db7 Mon Sep 17 00:00:00 2001 From: Joseph McKinsey Date: Fri, 7 Jun 2024 10:14:55 -0600 Subject: [PATCH 03/19] Add CI for web --- .github/workflows/ci-web.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/ci-web.yml diff --git a/.github/workflows/ci-web.yml b/.github/workflows/ci-web.yml new file mode 100644 index 00000000..aeacd5e1 --- /dev/null +++ b/.github/workflows/ci-web.yml @@ -0,0 +1,36 @@ +name: CI - Web + +on: [push, pull_request] + +jobs: + test: + runs-on: ${{matrix.os}} + strategy: + fail-fast: false + matrix: + os: [macOS-latest, ubuntu-latest, windows-latest] + python-version: ["3.9"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install -U pip install wheel setuptools cffi + - name: Build the web interface first + run: | + cd helics_web + pip install . + - name: Download helics library and run pip install + run: | + python setup.py download + python setup.py build_ext + pip install -e ".[cli]" + - name: Run CLI + run: | + helics server & + sleep 5 + kill %+ From c03e497a41455b17c9f1724305d54b8635ffe663 Mon Sep 17 00:00:00 2001 From: Joseph McKinsey Date: Fri, 7 Jun 2024 10:20:30 -0600 Subject: [PATCH 04/19] Move folder --- .gitignore | 1 - helics_cli_extras/.gitignore | 1 + .../client/.eslintrc.cjs | 0 {helics_web => helics_cli_extras}/client/.gitignore | 0 {helics_web => helics_cli_extras}/client/.npmrc | 0 .../client/.prettierrc | 0 {helics_web => helics_cli_extras}/client/README.md | 0 .../client/package-lock.json | 0 .../client/package.json | 0 .../client/playwright.config.js | 0 .../client/postcss.config.cjs | 0 .../client/src/app.css | 0 .../client/src/app.d.ts | 0 .../client/src/app.html | 0 .../client/src/lib/BrokerLayout.svelte | 0 .../client/src/lib/DataFlowGraph.svelte | 0 .../client/src/lib/Navbar.svelte | 0 .../client/src/lib/ProfilePlot.svelte | 0 .../client/src/lib/Switch.svelte | 0 .../client/src/lib/Table.svelte | 0 .../client/src/lib/Topology.svelte | 0 .../client/src/lib/stores.ts | 0 .../client/src/routes/__layout.svelte | 0 .../client/src/routes/broker.svelte | 0 .../client/src/routes/index.svelte | 0 .../client/src/routes/observe.svelte | 0 .../client/src/routes/profile.svelte | 0 .../client/src/routes/run.svelte | 0 .../client/static/favicon.png | Bin .../client/svelte.config.js | 0 .../client/tailwind.config.cjs | 0 .../client/tests/test.js | 0 .../client/tsconfig.json | 0 .../helics_cli_extras}/__init__.py | 0 {helics_web => helics_cli_extras}/pyproject.toml | 5 ++--- 35 files changed, 3 insertions(+), 4 deletions(-) create mode 100644 helics_cli_extras/.gitignore rename {helics_web => helics_cli_extras}/client/.eslintrc.cjs (100%) rename {helics_web => helics_cli_extras}/client/.gitignore (100%) rename {helics_web => helics_cli_extras}/client/.npmrc (100%) rename {helics_web => helics_cli_extras}/client/.prettierrc (100%) rename {helics_web => helics_cli_extras}/client/README.md (100%) rename {helics_web => helics_cli_extras}/client/package-lock.json (100%) rename {helics_web => helics_cli_extras}/client/package.json (100%) rename {helics_web => helics_cli_extras}/client/playwright.config.js (100%) rename {helics_web => helics_cli_extras}/client/postcss.config.cjs (100%) rename {helics_web => helics_cli_extras}/client/src/app.css (100%) rename {helics_web => helics_cli_extras}/client/src/app.d.ts (100%) rename {helics_web => helics_cli_extras}/client/src/app.html (100%) rename {helics_web => helics_cli_extras}/client/src/lib/BrokerLayout.svelte (100%) rename {helics_web => helics_cli_extras}/client/src/lib/DataFlowGraph.svelte (100%) rename {helics_web => helics_cli_extras}/client/src/lib/Navbar.svelte (100%) rename {helics_web => helics_cli_extras}/client/src/lib/ProfilePlot.svelte (100%) rename {helics_web => helics_cli_extras}/client/src/lib/Switch.svelte (100%) rename {helics_web => helics_cli_extras}/client/src/lib/Table.svelte (100%) rename {helics_web => helics_cli_extras}/client/src/lib/Topology.svelte (100%) rename {helics_web => helics_cli_extras}/client/src/lib/stores.ts (100%) rename {helics_web => helics_cli_extras}/client/src/routes/__layout.svelte (100%) rename {helics_web => helics_cli_extras}/client/src/routes/broker.svelte (100%) rename {helics_web => helics_cli_extras}/client/src/routes/index.svelte (100%) rename {helics_web => helics_cli_extras}/client/src/routes/observe.svelte (100%) rename {helics_web => helics_cli_extras}/client/src/routes/profile.svelte (100%) rename {helics_web => helics_cli_extras}/client/src/routes/run.svelte (100%) rename {helics_web => helics_cli_extras}/client/static/favicon.png (100%) rename {helics_web => helics_cli_extras}/client/svelte.config.js (100%) rename {helics_web => helics_cli_extras}/client/tailwind.config.cjs (100%) rename {helics_web => helics_cli_extras}/client/tests/test.js (100%) rename {helics_web => helics_cli_extras}/client/tsconfig.json (100%) rename {helics_web/helics_web => helics_cli_extras/helics_cli_extras}/__init__.py (100%) rename {helics_web => helics_cli_extras}/pyproject.toml (96%) diff --git a/.gitignore b/.gitignore index 5f54e70e..4a5baff7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ __helics-server helics/static -helics_web/helics_web/static helics/install docs/api _source diff --git a/helics_cli_extras/.gitignore b/helics_cli_extras/.gitignore new file mode 100644 index 00000000..5781fe7b --- /dev/null +++ b/helics_cli_extras/.gitignore @@ -0,0 +1 @@ +helics_cli_extras/static diff --git a/helics_web/client/.eslintrc.cjs b/helics_cli_extras/client/.eslintrc.cjs similarity index 100% rename from helics_web/client/.eslintrc.cjs rename to helics_cli_extras/client/.eslintrc.cjs diff --git a/helics_web/client/.gitignore b/helics_cli_extras/client/.gitignore similarity index 100% rename from helics_web/client/.gitignore rename to helics_cli_extras/client/.gitignore diff --git a/helics_web/client/.npmrc b/helics_cli_extras/client/.npmrc similarity index 100% rename from helics_web/client/.npmrc rename to helics_cli_extras/client/.npmrc diff --git a/helics_web/client/.prettierrc b/helics_cli_extras/client/.prettierrc similarity index 100% rename from helics_web/client/.prettierrc rename to helics_cli_extras/client/.prettierrc diff --git a/helics_web/client/README.md b/helics_cli_extras/client/README.md similarity index 100% rename from helics_web/client/README.md rename to helics_cli_extras/client/README.md diff --git a/helics_web/client/package-lock.json b/helics_cli_extras/client/package-lock.json similarity index 100% rename from helics_web/client/package-lock.json rename to helics_cli_extras/client/package-lock.json diff --git a/helics_web/client/package.json b/helics_cli_extras/client/package.json similarity index 100% rename from helics_web/client/package.json rename to helics_cli_extras/client/package.json diff --git a/helics_web/client/playwright.config.js b/helics_cli_extras/client/playwright.config.js similarity index 100% rename from helics_web/client/playwright.config.js rename to helics_cli_extras/client/playwright.config.js diff --git a/helics_web/client/postcss.config.cjs b/helics_cli_extras/client/postcss.config.cjs similarity index 100% rename from helics_web/client/postcss.config.cjs rename to helics_cli_extras/client/postcss.config.cjs diff --git a/helics_web/client/src/app.css b/helics_cli_extras/client/src/app.css similarity index 100% rename from helics_web/client/src/app.css rename to helics_cli_extras/client/src/app.css diff --git a/helics_web/client/src/app.d.ts b/helics_cli_extras/client/src/app.d.ts similarity index 100% rename from helics_web/client/src/app.d.ts rename to helics_cli_extras/client/src/app.d.ts diff --git a/helics_web/client/src/app.html b/helics_cli_extras/client/src/app.html similarity index 100% rename from helics_web/client/src/app.html rename to helics_cli_extras/client/src/app.html diff --git a/helics_web/client/src/lib/BrokerLayout.svelte b/helics_cli_extras/client/src/lib/BrokerLayout.svelte similarity index 100% rename from helics_web/client/src/lib/BrokerLayout.svelte rename to helics_cli_extras/client/src/lib/BrokerLayout.svelte diff --git a/helics_web/client/src/lib/DataFlowGraph.svelte b/helics_cli_extras/client/src/lib/DataFlowGraph.svelte similarity index 100% rename from helics_web/client/src/lib/DataFlowGraph.svelte rename to helics_cli_extras/client/src/lib/DataFlowGraph.svelte diff --git a/helics_web/client/src/lib/Navbar.svelte b/helics_cli_extras/client/src/lib/Navbar.svelte similarity index 100% rename from helics_web/client/src/lib/Navbar.svelte rename to helics_cli_extras/client/src/lib/Navbar.svelte diff --git a/helics_web/client/src/lib/ProfilePlot.svelte b/helics_cli_extras/client/src/lib/ProfilePlot.svelte similarity index 100% rename from helics_web/client/src/lib/ProfilePlot.svelte rename to helics_cli_extras/client/src/lib/ProfilePlot.svelte diff --git a/helics_web/client/src/lib/Switch.svelte b/helics_cli_extras/client/src/lib/Switch.svelte similarity index 100% rename from helics_web/client/src/lib/Switch.svelte rename to helics_cli_extras/client/src/lib/Switch.svelte diff --git a/helics_web/client/src/lib/Table.svelte b/helics_cli_extras/client/src/lib/Table.svelte similarity index 100% rename from helics_web/client/src/lib/Table.svelte rename to helics_cli_extras/client/src/lib/Table.svelte diff --git a/helics_web/client/src/lib/Topology.svelte b/helics_cli_extras/client/src/lib/Topology.svelte similarity index 100% rename from helics_web/client/src/lib/Topology.svelte rename to helics_cli_extras/client/src/lib/Topology.svelte diff --git a/helics_web/client/src/lib/stores.ts b/helics_cli_extras/client/src/lib/stores.ts similarity index 100% rename from helics_web/client/src/lib/stores.ts rename to helics_cli_extras/client/src/lib/stores.ts diff --git a/helics_web/client/src/routes/__layout.svelte b/helics_cli_extras/client/src/routes/__layout.svelte similarity index 100% rename from helics_web/client/src/routes/__layout.svelte rename to helics_cli_extras/client/src/routes/__layout.svelte diff --git a/helics_web/client/src/routes/broker.svelte b/helics_cli_extras/client/src/routes/broker.svelte similarity index 100% rename from helics_web/client/src/routes/broker.svelte rename to helics_cli_extras/client/src/routes/broker.svelte diff --git a/helics_web/client/src/routes/index.svelte b/helics_cli_extras/client/src/routes/index.svelte similarity index 100% rename from helics_web/client/src/routes/index.svelte rename to helics_cli_extras/client/src/routes/index.svelte diff --git a/helics_web/client/src/routes/observe.svelte b/helics_cli_extras/client/src/routes/observe.svelte similarity index 100% rename from helics_web/client/src/routes/observe.svelte rename to helics_cli_extras/client/src/routes/observe.svelte diff --git a/helics_web/client/src/routes/profile.svelte b/helics_cli_extras/client/src/routes/profile.svelte similarity index 100% rename from helics_web/client/src/routes/profile.svelte rename to helics_cli_extras/client/src/routes/profile.svelte diff --git a/helics_web/client/src/routes/run.svelte b/helics_cli_extras/client/src/routes/run.svelte similarity index 100% rename from helics_web/client/src/routes/run.svelte rename to helics_cli_extras/client/src/routes/run.svelte diff --git a/helics_web/client/static/favicon.png b/helics_cli_extras/client/static/favicon.png similarity index 100% rename from helics_web/client/static/favicon.png rename to helics_cli_extras/client/static/favicon.png diff --git a/helics_web/client/svelte.config.js b/helics_cli_extras/client/svelte.config.js similarity index 100% rename from helics_web/client/svelte.config.js rename to helics_cli_extras/client/svelte.config.js diff --git a/helics_web/client/tailwind.config.cjs b/helics_cli_extras/client/tailwind.config.cjs similarity index 100% rename from helics_web/client/tailwind.config.cjs rename to helics_cli_extras/client/tailwind.config.cjs diff --git a/helics_web/client/tests/test.js b/helics_cli_extras/client/tests/test.js similarity index 100% rename from helics_web/client/tests/test.js rename to helics_cli_extras/client/tests/test.js diff --git a/helics_web/client/tsconfig.json b/helics_cli_extras/client/tsconfig.json similarity index 100% rename from helics_web/client/tsconfig.json rename to helics_cli_extras/client/tsconfig.json diff --git a/helics_web/helics_web/__init__.py b/helics_cli_extras/helics_cli_extras/__init__.py similarity index 100% rename from helics_web/helics_web/__init__.py rename to helics_cli_extras/helics_cli_extras/__init__.py diff --git a/helics_web/pyproject.toml b/helics_cli_extras/pyproject.toml similarity index 96% rename from helics_web/pyproject.toml rename to helics_cli_extras/pyproject.toml index 6d4d3a7e..ae46eaa7 100644 --- a/helics_web/pyproject.toml +++ b/helics_cli_extras/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.0.0", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "helics-web" +name = "helics-cli-extras" version = "0.0.1" authors = [{ name = "Dheepak Krishnamurthy", email = "me@kdheepak.com" }] maintainers = [ @@ -41,7 +41,6 @@ classifiers = [ "Topic :: Software Development", ] dependencies = [ - "helics>=3.0.0", "flask>=2", "requests", "flask-restful", @@ -58,7 +57,7 @@ Documentation = "https://python.helics.org/" "Source Code" = "https://github.com/GMLC-TDC/pyhelics" [tool.setuptools.packages.find] -include = ["helics_web", "helics_web.*"] +include = ["helics_cli_extras", "helics_cli_extras.*"] [tool.setuptools] include-package-data = true From e870042e265f18464e2327b85e27763f22a5cb77 Mon Sep 17 00:00:00 2001 From: Joseph McKinsey Date: Fri, 7 Jun 2024 10:20:59 -0600 Subject: [PATCH 05/19] Rename in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8ea14b4e..496da0c8 100755 --- a/setup.py +++ b/setup.py @@ -507,7 +507,7 @@ def is_pure(self): helics_cli_install_requires = [ "requests", - "helics_web==0.0.1", + "helics_cli_extras==0.0.1", "pandas", "SQLAlchemy", "matplotlib", From 320077df2dc164d571a9dff2697d3400f7cf0604 Mon Sep 17 00:00:00 2001 From: Joseph McKinsey Date: Fri, 7 Jun 2024 10:43:57 -0600 Subject: [PATCH 06/19] Rename in cli.py and improve test --- .github/workflows/ci-web.yml | 9 ++++++++- helics/cli.py | 6 +++--- helics_cli_extras/pyproject.toml | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-web.yml b/.github/workflows/ci-web.yml index aeacd5e1..7219ad3a 100644 --- a/.github/workflows/ci-web.yml +++ b/.github/workflows/ci-web.yml @@ -20,9 +20,15 @@ jobs: - name: Install dependencies run: | python -m pip install -U pip install wheel setuptools cffi + - name: Build NPM + run: | + cd helics_cli_extras/client + npm install + npm run build + cp -r build ../helics_cli_extras/static - name: Build the web interface first run: | - cd helics_web + cd helics_cli_extras pip install . - name: Download helics library and run pip install run: | @@ -33,4 +39,5 @@ jobs: run: | helics server & sleep 5 + curl http://localhost:5000 kill %+ diff --git a/helics/cli.py b/helics/cli.py index 5121e53a..126e2c9a 100644 --- a/helics/cli.py +++ b/helics/cli.py @@ -14,7 +14,7 @@ import urllib.request import logging from ._version import __version__ -from .status_checker import CheckStatusThread, HELICSRuntimeError +from .status_checker import CheckStatusThread import click import pathlib @@ -106,11 +106,11 @@ def server(open: bool): Run helics web server to access web interface """ import webbrowser - import helics_web + import helics_cli_extras if open: webbrowser.open("http://127.0.0.1:5000", 1) - helics_web.run() + helics_cli_extras.run() @cli.command() diff --git a/helics_cli_extras/pyproject.toml b/helics_cli_extras/pyproject.toml index ae46eaa7..6517e5db 100644 --- a/helics_cli_extras/pyproject.toml +++ b/helics_cli_extras/pyproject.toml @@ -63,4 +63,4 @@ include = ["helics_cli_extras", "helics_cli_extras.*"] include-package-data = true [tool.setuptools.package-data] -"*" = ["static/*"] +"*" = ["static/**"] From 7ee0db7a39c8ac18d2abc521d53b4e1e99eb8d0b Mon Sep 17 00:00:00 2001 From: Joseph McKinsey Date: Fri, 7 Jun 2024 10:51:27 -0600 Subject: [PATCH 07/19] Stop duplication of push and pull request --- .github/workflows/ci-web.yml | 10 ++++++++-- .github/workflows/ci.yml | 8 +++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-web.yml b/.github/workflows/ci-web.yml index 7219ad3a..4751cac4 100644 --- a/.github/workflows/ci-web.yml +++ b/.github/workflows/ci-web.yml @@ -1,6 +1,12 @@ name: CI - Web -on: [push, pull_request] +on: + push: + branches: + - main + pull_request: + branches: + - main jobs: test: @@ -8,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - os: [macOS-latest, ubuntu-latest, windows-latest] + os: [ubuntu-latest] python-version: ["3.9"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 35d56545..4e1ebc52 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,12 @@ name: CI -on: [push, pull_request] +on: + push: + branches: + - main + pull_request: + branches: + - main jobs: test: From 52ec2ea0c29b36fb40fa26168b6b0faa88b43590 Mon Sep 17 00:00:00 2001 From: Joseph McKinsey Date: Fri, 7 Jun 2024 10:57:39 -0600 Subject: [PATCH 08/19] Move observer and database into cli_extras --- helics/cli.py | 2 +- .../helics_cli_extras/__init__.py | 631 +----------------- .../helics_cli_extras}/database.py | 11 +- .../helics_cli_extras/flask_app.py | 627 +++++++++++++++++ .../helics_cli_extras}/observer.py | 52 +- 5 files changed, 682 insertions(+), 641 deletions(-) rename {helics => helics_cli_extras/helics_cli_extras}/database.py (95%) create mode 100644 helics_cli_extras/helics_cli_extras/flask_app.py rename {helics => helics_cli_extras/helics_cli_extras}/observer.py (79%) diff --git a/helics/cli.py b/helics/cli.py index 126e2c9a..7276dda0 100644 --- a/helics/cli.py +++ b/helics/cli.py @@ -125,7 +125,7 @@ def observer(db_folder: pathlib.Path): """ Run helics observer and write data to sqlite file """ - from .observer import HelicsObserverFederate + from helics_cli_extras import HelicsObserverFederate o = HelicsObserverFederate(folder=db_folder) o.run() diff --git a/helics_cli_extras/helics_cli_extras/__init__.py b/helics_cli_extras/helics_cli_extras/__init__.py index 0e67fd44..62e038e0 100644 --- a/helics_cli_extras/helics_cli_extras/__init__.py +++ b/helics_cli_extras/helics_cli_extras/__init__.py @@ -1,629 +1,2 @@ -# -*- coding: utf-8 -*- -import json -import time -import os -import shlex -import subprocess -import sys -from typing import cast - -from flask import Flask, send_from_directory -from flask_restful import Resource, Api, reqparse, abort -from flask_cors import CORS -import sqlalchemy as sa -from sqlalchemy.ext.automap import automap_base -import werkzeug - -import re - -# HELICS must be installed but cannot be a build depencency -from helics import database as db - -current_directory = os.path.realpath(os.path.dirname(__file__)) - -app = Flask( - __name__.split(".")[0], - static_url_path="", - static_folder=os.path.join(current_directory, "static"), -) -api = Api(app) -CORS(app, resources={r"/api/*": {"origins": "*"}}) - -app.config["UPLOAD_FOLDER"] = os.path.abspath( - os.path.join(os.getcwd(), "__helics-server") -) - -cache = { - "path": os.path.join(app.config["UPLOAD_FOLDER"], "helics-cli.sqlite.db"), - "profile-path": os.path.join(app.config["UPLOAD_FOLDER"], "profile.txt"), - "runner-path": os.path.join(app.config["UPLOAD_FOLDER"], "runner.json"), - "runner-folder": app.config["UPLOAD_FOLDER"], - "runner-file-name": "runner.json", -} - - -class DatabaseManager: - def __init__(self, path_to_helics_db=cache["path"]): - self.path_to_helics_db = path_to_helics_db - self.instaniate() - - def instaniate(self): - self.engine = db.create_engine(f"sqlite:///{self.path_to_helics_db}") - self.connection = self.engine.connect() - self.session = db.Session(self.engine) - - -class Database(Resource): - def get(self): - return cache["path"] - - def post(self): - parser = reqparse.RequestParser() - parser.add_argument( - "file", type=werkzeug.datastructures.FileStorage, location="files" - ) - args = parser.parse_args() - file = args["file"] - os.makedirs(app.config["UPLOAD_FOLDER"], exist_ok=True) - file.save(os.path.join(app.config["UPLOAD_FOLDER"], "helics-cli.sqlite.db")) - cache["path"] = os.path.join( - app.config["UPLOAD_FOLDER"], "helics-cli.sqlite.db" - ) - dm = DatabaseManager() - return {"filename": dm.path_to_helics_db} - - -api.add_resource(Database, "/api/observer/database") - - -class SystemInfo(Resource): - def get(self): - dm = DatabaseManager() - return dm.session.query(db.SystemInfo).one().data - - -api.add_resource(SystemInfo, "/api/observer/systeminfo") - - -class Cores(Resource): - def get(self): - dm = DatabaseManager() - cores = dm.session.query(db.Cores).all() - return [ - {"id": e.id, "name": e.name, "address": e.address} - for e in cores - if not e.name.startswith("__observer__") - ] - - -api.add_resource(Cores, "/api/observer/cores") - - -class Federates(Resource): - def get(self): - dm = DatabaseManager() - federates = dm.session.query(db.Federates).all() - return [ - {"id": f.id, "name": f.name, "parent": f.parent} - for f in federates - if f.name != "__observer__" - ] - - -api.add_resource(Federates, "/api/observer/federates") - - -class Graph(Resource): - def get(self): - dm = DatabaseManager() - federate = dm.session.query(db.FederateGraph).one().data - data = dm.session.query(db.DataGraph).one().data - return {"federate": federate, "data": data} - - -api.add_resource(Graph, "/api/observer/graphs") - - -class Subscriptions(Resource): - def get(self): - dm = DatabaseManager() - subscriptions = dm.session.query(db.Subscriptions).all() - return [db.as_dict(i) for i in subscriptions] - - -api.add_resource(Subscriptions, "/api/observer/subscriptions") - - -class Inputs(Resource): - def get(self): - dm = DatabaseManager() - inputs = dm.session.query(db.Inputs).all() - return [db.as_dict(i) for i in inputs] - - -api.add_resource(Inputs, "/api/observer/inputs") - - -class Publications(Resource): - def get(self): - dm = DatabaseManager() - publications = dm.session.query(db.Publications).all() - return [db.as_dict(i) for i in publications] - - -api.add_resource(Publications, "/api/observer/publications") - - -class DataTable(Resource): - def get(self): - dm = DatabaseManager() - Base = automap_base() - Base.prepare(dm.engine, reflect=True) - return [db.as_dict(i) for i in dm.session.query(Base.classes.datatable).all()] - - -api.add_resource(DataTable, "/api/observer/data") - -status_tracker = {} - - -class RunnerFile(Resource): - def get(self): - if not os.path.exists(cache["runner-path"]): - return {} - with open(cache["runner-path"]) as f: - data = json.loads(f.read()) - data["folder"] = cache["runner-folder"] - data["path"] = cache["runner-path"] - data["filename"] = cache["runner-file-name"] - if data.get("broker", False) is True: - data["federates"].append( - { - "directory": ".", - "exec": "helics_broker -f{}".format(len(data["federates"])), - "host": "localhost", - "name": "broker", - } - ) - for federate in data["federates"]: - if federate["directory"].startswith("."): - federate["directory"] = os.path.abspath( - os.path.join(data["folder"], federate["directory"]) - ) - federate["old_name"] = federate["name"] - if os.path.exists( - os.path.join(data["folder"], "{}.log".format(federate["name"])) - ): - federate["log_available"] = True - else: - federate["log_available"] = False - - if federate["name"] in status_tracker: - federate["status"] = status_tracker[federate["name"]] - else: - federate["status"] = None - return data - - def post(self): - parser = reqparse.RequestParser() - parser.add_argument( - "file", type=werkzeug.datastructures.FileStorage, location="files" - ) - args = parser.parse_args() - file = args["file"] - path = cache["runner-folder"] - name = cache["runner-file-name"] - os.makedirs(path, exist_ok=True) - file.save(os.path.join(path, name)) - cache["runner-path"] = os.path.join(path, name) - - -api.add_resource(RunnerFile, "/api/runner/file") - - -class RunnerFileName(Resource): - def post(self): - parser = reqparse.RequestParser() - parser.add_argument("name", type=str) - args = parser.parse_args() - name = args["name"] - cache["runner-file-name"] = name - cache["runner-path"] = os.path.join( - cache["runner-folder"], cache["runner-file-name"] - ) - - -api.add_resource(RunnerFileName, "/api/runner/file/name") - - -class RunnerFileFolder(Resource): - def post(self): - parser = reqparse.RequestParser() - parser.add_argument("folder", type=str) - args = parser.parse_args() - folder = args["folder"] - cache["runner-folder"] = os.path.abspath(os.path.expanduser(folder)) - cache["runner-path"] = os.path.join( - cache["runner-folder"], cache["runner-file-name"] - ) - - -api.add_resource(RunnerFileFolder, "/api/runner/file/folder") - - -class RunnerFilePath(Resource): - def post(self): - parser = reqparse.RequestParser() - parser.add_argument("path", type=str) - args = parser.parse_args() - path = os.path.abspath(os.path.expanduser(args["path"])) - cache["runner-path"] = path - cache["runner-folder"] = os.path.dirname(path) - cache["runner-file-name"] = os.path.basename(path) - - -api.add_resource(RunnerFilePath, "/api/runner/file/path") - - -class RunnerFileEdit(Resource): - def delete(self): - parser = reqparse.RequestParser() - parser.add_argument("old_name", type=str) - args = parser.parse_args() - old_name = args["old_name"] - with open(cache["runner-path"]) as f: - data = json.loads(f.read()) - - delete_index = None - for i, f in enumerate(data["federates"]): - if f["name"] == old_name: - delete_index = i - break - data["federates"].pop(delete_index) - with open(cache["runner-path"], "w") as f: - f.write(json.dumps(data)) - return {"status": 200} - - def put(self): - parser = reqparse.RequestParser() - parser.add_argument("old_name", type=str) - parser.add_argument("name", type=str) - parser.add_argument("exec", type=str) - parser.add_argument("directory", type=str) - args = parser.parse_args() - old_name = args["old_name"] - name = args["name"] - exec = args["exec"] - directory = args["directory"] - - with open(cache["runner-path"]) as f: - data = json.loads(f.read()) - - for f in data["federates"]: - if f["name"] == old_name: - f["name"] = name - f["exec"] = exec - f["directory"] = directory - break - else: - abort(417, description="Unknown name='{}'".format(old_name)) - - with open(cache["runner-path"], "w") as f: - f.write(json.dumps(data)) - - return {"status": 200} - - def post(self): - parser = reqparse.RequestParser() - parser.add_argument("name", type=str) - parser.add_argument("exec", type=str) - parser.add_argument("directory", type=str) - args = parser.parse_args() - name = args["name"] - exec = args["exec"] - directory = args["directory"] - - with open(cache["runner-path"]) as f: - data = json.loads(f.read()) - - for f in data["federates"]: - if f["name"] == name: - return abort(417, description="Name already exists {}".format(name)) - - data["federates"].append( - { - "directory": directory, - "exec": exec, - "host": "localhost", - "name": name, - } - ) - - with open(cache["runner-path"], "w") as f: - f.write(json.dumps(data)) - - return {"status": 200} - - -api.add_resource(RunnerFileEdit, "/api/runner/file/edit") - - -class RunnerLog(Resource): - def get(self, name): - with open(cache["runner-path"]) as f: - data = json.loads(f.read()) - with open( - os.path.join(os.path.dirname(cache["runner-path"]), "{}.log".format(name)) - ) as f: - data = f.read() - return {"log": data} - - -api.add_resource(RunnerLog, "/api/runner/log/") - - -class RunnerRun(Resource): - runner_server = {} - - def get(self): - if self.runner_server.get("process", None) is not None: - return {"status": True} - else: - return {"status": False} - - def post(self): - if self.get()["status"]: - self.delete() - p = subprocess.Popen( - shlex.split( - "helics run --path {} --connect-server".format(cache["runner-path"]) - ) - ) - self.runner_server["process"] = p - return {"status": True} - - def delete(self): - self.runner_server["process"].terminate() - self.runner_server["process"].kill() - counter = 0 - while self.runner_server["process"].poll() is None or counter > 5: - time.sleep(1) - self.runner_server["process"].terminate() - self.runner_server["process"].kill() - counter += 1 - del self.runner_server["process"] - global status_tracker - status_tracker = {} - return {"status": False} - - -api.add_resource(RunnerRun, "/api/runner/run") - - -class RunnerKillBroker(Resource): - def post(self): - name = "helics_broker" - import platform - - if platform.system().lower().startswith("windows"): - # windows OS - if not name.endswith(".exe"): - name = name + ".exe" - os.system("taskkill /f /im {}".format(name)) - # os.system(r"start /b pcs.exe 2>&1") - # with os.popen("tasklist | findstr \"pcs.exe\"") as f: - # temp_content = f.read() - # if len(temp_content) == 0: - # pass - else: - if name.endswith(".exe"): - name = name.replace(".exe", "") - os.system("killall -9 {} 2>&1".format(name)) - - -api.add_resource(RunnerKillBroker, "/api/runner/kill/broker") - - -class RunnerStatus(Resource): - def get(self): - parser = reqparse.RequestParser() - parser.add_argument("name", type=str, required=True, help="Name of federate") - args = parser.parse_args() - return {"status": status_tracker[args["name"]]} - - def post(self): - parser = reqparse.RequestParser() - parser.add_argument("name", type=str, required=True, help="Name of federate") - parser.add_argument( - "status", type=str, required=True, help="Status of federate" - ) - args = parser.parse_args() - status_tracker[args["name"]] = args["status"] - return {"status": status_tracker[args["name"]]} - - -api.add_resource(RunnerStatus, "/api/runner/status") - - -class Health(Resource): - def get(self): - return {"status": 200} - - -api.add_resource(Health, "/api/health") - - -class Profile(Resource): - # SenderFederate2[131074](initializing)HELICS CODE ENTRY<4570827706580384>[t=-1000000] - PATTERN = re.compile( - r""" - (?P\w+) # SenderFederate2 - \[(\d+)\] # [131074] - \((?P\w+)\) # (initializing) - (?P(?:\w|\s)+) # HELICS CODE ENTRY - \<(?P\d+(?:\|\d+)?)\> # <4570827706580384|534534523453> - \[t=(?P-?\d*\.{0,1}\d+)\] # [t=-1000000] - """, - re.X, - ) - - def get(self): - with open(cache["profile-path"]) as f: - data = f.read() - data = data.replace(r"", "").replace(r"", "") - names = [] - states = [] - messages = [] - simtimes = [] - realtimes = [] - time_marker = {} - for line in data.splitlines(): - m = self.PATTERN.match(line) - m = cast(re.Match[str], m) - name = m.group("name") - state = m.group("state") - message = m.group("message") - simtime = float(m.group("simtime")) - try: - realtime = float(m.group("realtime")) - except ValueError: - realtime, markertime = m.group("realtime").split("|") - time_marker[name] = float(markertime) - realtime = float(realtime) - names.append(name) - states.append(state) - messages.append(message) - simtimes.append(simtime) - realtimes.append(realtime) - - profile = {} - for name in set(names): - profile[name] = [] - - invert = True - if invert: - for name in set(names): - profile[name].append({}) - - for name, state, message, simtime, realtime in zip( - names, states, messages, simtimes, realtimes - ): - if state == "created": - continue - if "ENTRY" in message and not invert: - profile[name].append({"s_enter": simtime, "r_enter": realtime}) - elif "EXIT" in message and not invert: - profile[name][-1]["s_end"] = simtime - profile[name][-1]["r_end"] = realtime - elif "EXIT" in message and invert: - profile[name].append({"s_enter": simtime, "r_enter": realtime}) - elif "ENTRY" in message and invert: - profile[name][-1]["s_end"] = simtime - profile[name][-1]["r_end"] = realtime - profiles = {} - names = {k: i for i, k in enumerate(sorted(profile.keys()))} - - end = "r_end" - enter = "r_enter" - scaling = 1e9 - - for k in profile.keys(): - profiles[k] = [] - for i in profile[k]: - if end in i.keys() and enter in i.keys(): - i["name"] = k - i[enter] = i[enter] / scaling - i[end] = i[end] / scaling - profiles[k].append(i) - - return profiles - - def post(self): - parser = reqparse.RequestParser() - parser.add_argument( - "file", type=werkzeug.datastructures.FileStorage, location="files" - ) - args = parser.parse_args() - file = args["file"] - os.makedirs(app.config["UPLOAD_FOLDER"], exist_ok=True) - file.save(os.path.join(app.config["UPLOAD_FOLDER"], "profile.txt")) - cache["profile-path"] = os.path.join(app.config["UPLOAD_FOLDER"], "profile.txt") - return {"status": "success"} - - -api.add_resource(Profile, "/api/profiler/") - - -class APIException(Exception): - def __init__(self, code, message): - self._code = code - self._message = message - - @property - def code(self): - return self._code - - @property - def message(self): - return self._message - - def __str__(self): - return self.__class__.__name__ + ": " + self.message - - -class BrokerServer(Resource): - broker_server = {} - - def get(self): - if self.broker_server.get("process", None) is not None: - return {"status": True} - else: - return {"status": False} - - def post(self): - parser = reqparse.RequestParser() - parser.add_argument( - "status", type=bool, required=True, help="requested status of broker server" - ) - args = parser.parse_args() - status = args["status"] - if status is True and self.broker_server.get("process", None) is not None: - return abort(417, description="Unable to start server", status=True) - elif status is False and self.broker_server.get("process", None) is None: - return abort(417, description="Unable to stop server", status=False) - elif status is True: - p = subprocess.Popen(shlex.split("helics_broker_server --http")) - self.broker_server["process"] = p - return {"status": status} - elif status is False: - self.broker_server["process"].terminate() - self.broker_server["process"].kill() - counter = 0 - # TODO: Find out why broker server is not being terminated - while self.broker_server["process"].poll() is None or counter > 5: - time.sleep(1) - self.broker_server["process"].terminate() - self.broker_server["process"].kill() - counter += 1 - del self.broker_server["process"] - return {"status": status} - - -api.add_resource(BrokerServer, "/api/broker-server") - - -@app.route("/", defaults={"path": "index.html"}) -@app.route("/") -def index(path): - return send_from_directory(os.path.join(current_directory, "static"), path) - - -def run(): - debug = bool(os.environ.get("PYHELICS_FLASK_DEBUG", False)) - if debug: - host = None - else: - host = "0.0.0.0" - cli = sys.modules["flask.cli"] - cli.show_server_banner = lambda *x: None - app.run(host=host, debug=debug) +from .flask_app import run +from .observer import HelicsObserverFederate diff --git a/helics/database.py b/helics_cli_extras/helics_cli_extras/database.py similarity index 95% rename from helics/database.py rename to helics_cli_extras/helics_cli_extras/database.py index 1e701784..3915c211 100644 --- a/helics/database.py +++ b/helics_cli_extras/helics_cli_extras/database.py @@ -1,7 +1,14 @@ # -*- coding: utf-8 -*- -from sqlalchemy import Column, Integer, Float, String, JSON, Sequence, Table, Boolean, create_engine +from sqlalchemy import ( + Column, + Integer, + Float, + String, + JSON, + Sequence, + Boolean, +) from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import Session Base = declarative_base() diff --git a/helics_cli_extras/helics_cli_extras/flask_app.py b/helics_cli_extras/helics_cli_extras/flask_app.py new file mode 100644 index 00000000..1800b684 --- /dev/null +++ b/helics_cli_extras/helics_cli_extras/flask_app.py @@ -0,0 +1,627 @@ +# -*- coding: utf-8 -*- +import json +import time +import os +import shlex +import subprocess +import sys +from typing import cast + +from flask import Flask, send_from_directory +from flask_restful import Resource, Api, reqparse, abort +from flask_cors import CORS +import sqlalchemy as sa +from sqlalchemy.ext.automap import automap_base +import werkzeug + +import re +from . import database as db + +current_directory = os.path.realpath(os.path.dirname(__file__)) + +app = Flask( + __name__.split(".")[0], + static_url_path="", + static_folder=os.path.join(current_directory, "static"), +) +api = Api(app) +CORS(app, resources={r"/api/*": {"origins": "*"}}) + +app.config["UPLOAD_FOLDER"] = os.path.abspath( + os.path.join(os.getcwd(), "__helics-server") +) + +cache = { + "path": os.path.join(app.config["UPLOAD_FOLDER"], "helics-cli.sqlite.db"), + "profile-path": os.path.join(app.config["UPLOAD_FOLDER"], "profile.txt"), + "runner-path": os.path.join(app.config["UPLOAD_FOLDER"], "runner.json"), + "runner-folder": app.config["UPLOAD_FOLDER"], + "runner-file-name": "runner.json", +} + + +class DatabaseManager: + def __init__(self, path_to_helics_db=cache["path"]): + self.path_to_helics_db = path_to_helics_db + self.instaniate() + + def instaniate(self): + self.engine = db.create_engine(f"sqlite:///{self.path_to_helics_db}") + self.connection = self.engine.connect() + self.session = db.Session(self.engine) + + +class Database(Resource): + def get(self): + return cache["path"] + + def post(self): + parser = reqparse.RequestParser() + parser.add_argument( + "file", type=werkzeug.datastructures.FileStorage, location="files" + ) + args = parser.parse_args() + file = args["file"] + os.makedirs(app.config["UPLOAD_FOLDER"], exist_ok=True) + file.save(os.path.join(app.config["UPLOAD_FOLDER"], "helics-cli.sqlite.db")) + cache["path"] = os.path.join( + app.config["UPLOAD_FOLDER"], "helics-cli.sqlite.db" + ) + dm = DatabaseManager() + return {"filename": dm.path_to_helics_db} + + +api.add_resource(Database, "/api/observer/database") + + +class SystemInfo(Resource): + def get(self): + dm = DatabaseManager() + return dm.session.query(db.SystemInfo).one().data + + +api.add_resource(SystemInfo, "/api/observer/systeminfo") + + +class Cores(Resource): + def get(self): + dm = DatabaseManager() + cores = dm.session.query(db.Cores).all() + return [ + {"id": e.id, "name": e.name, "address": e.address} + for e in cores + if not e.name.startswith("__observer__") + ] + + +api.add_resource(Cores, "/api/observer/cores") + + +class Federates(Resource): + def get(self): + dm = DatabaseManager() + federates = dm.session.query(db.Federates).all() + return [ + {"id": f.id, "name": f.name, "parent": f.parent} + for f in federates + if f.name != "__observer__" + ] + + +api.add_resource(Federates, "/api/observer/federates") + + +class Graph(Resource): + def get(self): + dm = DatabaseManager() + federate = dm.session.query(db.FederateGraph).one().data + data = dm.session.query(db.DataGraph).one().data + return {"federate": federate, "data": data} + + +api.add_resource(Graph, "/api/observer/graphs") + + +class Subscriptions(Resource): + def get(self): + dm = DatabaseManager() + subscriptions = dm.session.query(db.Subscriptions).all() + return [db.as_dict(i) for i in subscriptions] + + +api.add_resource(Subscriptions, "/api/observer/subscriptions") + + +class Inputs(Resource): + def get(self): + dm = DatabaseManager() + inputs = dm.session.query(db.Inputs).all() + return [db.as_dict(i) for i in inputs] + + +api.add_resource(Inputs, "/api/observer/inputs") + + +class Publications(Resource): + def get(self): + dm = DatabaseManager() + publications = dm.session.query(db.Publications).all() + return [db.as_dict(i) for i in publications] + + +api.add_resource(Publications, "/api/observer/publications") + + +class DataTable(Resource): + def get(self): + dm = DatabaseManager() + Base = automap_base() + Base.prepare(dm.engine, reflect=True) + return [db.as_dict(i) for i in dm.session.query(Base.classes.datatable).all()] + + +api.add_resource(DataTable, "/api/observer/data") + +status_tracker = {} + + +class RunnerFile(Resource): + def get(self): + if not os.path.exists(cache["runner-path"]): + return {} + with open(cache["runner-path"]) as f: + data = json.loads(f.read()) + data["folder"] = cache["runner-folder"] + data["path"] = cache["runner-path"] + data["filename"] = cache["runner-file-name"] + if data.get("broker", False) is True: + data["federates"].append( + { + "directory": ".", + "exec": "helics_broker -f{}".format(len(data["federates"])), + "host": "localhost", + "name": "broker", + } + ) + for federate in data["federates"]: + if federate["directory"].startswith("."): + federate["directory"] = os.path.abspath( + os.path.join(data["folder"], federate["directory"]) + ) + federate["old_name"] = federate["name"] + if os.path.exists( + os.path.join(data["folder"], "{}.log".format(federate["name"])) + ): + federate["log_available"] = True + else: + federate["log_available"] = False + + if federate["name"] in status_tracker: + federate["status"] = status_tracker[federate["name"]] + else: + federate["status"] = None + return data + + def post(self): + parser = reqparse.RequestParser() + parser.add_argument( + "file", type=werkzeug.datastructures.FileStorage, location="files" + ) + args = parser.parse_args() + file = args["file"] + path = cache["runner-folder"] + name = cache["runner-file-name"] + os.makedirs(path, exist_ok=True) + file.save(os.path.join(path, name)) + cache["runner-path"] = os.path.join(path, name) + + +api.add_resource(RunnerFile, "/api/runner/file") + + +class RunnerFileName(Resource): + def post(self): + parser = reqparse.RequestParser() + parser.add_argument("name", type=str) + args = parser.parse_args() + name = args["name"] + cache["runner-file-name"] = name + cache["runner-path"] = os.path.join( + cache["runner-folder"], cache["runner-file-name"] + ) + + +api.add_resource(RunnerFileName, "/api/runner/file/name") + + +class RunnerFileFolder(Resource): + def post(self): + parser = reqparse.RequestParser() + parser.add_argument("folder", type=str) + args = parser.parse_args() + folder = args["folder"] + cache["runner-folder"] = os.path.abspath(os.path.expanduser(folder)) + cache["runner-path"] = os.path.join( + cache["runner-folder"], cache["runner-file-name"] + ) + + +api.add_resource(RunnerFileFolder, "/api/runner/file/folder") + + +class RunnerFilePath(Resource): + def post(self): + parser = reqparse.RequestParser() + parser.add_argument("path", type=str) + args = parser.parse_args() + path = os.path.abspath(os.path.expanduser(args["path"])) + cache["runner-path"] = path + cache["runner-folder"] = os.path.dirname(path) + cache["runner-file-name"] = os.path.basename(path) + + +api.add_resource(RunnerFilePath, "/api/runner/file/path") + + +class RunnerFileEdit(Resource): + def delete(self): + parser = reqparse.RequestParser() + parser.add_argument("old_name", type=str) + args = parser.parse_args() + old_name = args["old_name"] + with open(cache["runner-path"]) as f: + data = json.loads(f.read()) + + delete_index = None + for i, f in enumerate(data["federates"]): + if f["name"] == old_name: + delete_index = i + break + data["federates"].pop(delete_index) + with open(cache["runner-path"], "w") as f: + f.write(json.dumps(data)) + return {"status": 200} + + def put(self): + parser = reqparse.RequestParser() + parser.add_argument("old_name", type=str) + parser.add_argument("name", type=str) + parser.add_argument("exec", type=str) + parser.add_argument("directory", type=str) + args = parser.parse_args() + old_name = args["old_name"] + name = args["name"] + exec = args["exec"] + directory = args["directory"] + + with open(cache["runner-path"]) as f: + data = json.loads(f.read()) + + for f in data["federates"]: + if f["name"] == old_name: + f["name"] = name + f["exec"] = exec + f["directory"] = directory + break + else: + abort(417, description="Unknown name='{}'".format(old_name)) + + with open(cache["runner-path"], "w") as f: + f.write(json.dumps(data)) + + return {"status": 200} + + def post(self): + parser = reqparse.RequestParser() + parser.add_argument("name", type=str) + parser.add_argument("exec", type=str) + parser.add_argument("directory", type=str) + args = parser.parse_args() + name = args["name"] + exec = args["exec"] + directory = args["directory"] + + with open(cache["runner-path"]) as f: + data = json.loads(f.read()) + + for f in data["federates"]: + if f["name"] == name: + return abort(417, description="Name already exists {}".format(name)) + + data["federates"].append( + { + "directory": directory, + "exec": exec, + "host": "localhost", + "name": name, + } + ) + + with open(cache["runner-path"], "w") as f: + f.write(json.dumps(data)) + + return {"status": 200} + + +api.add_resource(RunnerFileEdit, "/api/runner/file/edit") + + +class RunnerLog(Resource): + def get(self, name): + with open(cache["runner-path"]) as f: + data = json.loads(f.read()) + with open( + os.path.join(os.path.dirname(cache["runner-path"]), "{}.log".format(name)) + ) as f: + data = f.read() + return {"log": data} + + +api.add_resource(RunnerLog, "/api/runner/log/") + + +class RunnerRun(Resource): + runner_server = {} + + def get(self): + if self.runner_server.get("process", None) is not None: + return {"status": True} + else: + return {"status": False} + + def post(self): + if self.get()["status"]: + self.delete() + p = subprocess.Popen( + shlex.split( + "helics run --path {} --connect-server".format(cache["runner-path"]) + ) + ) + self.runner_server["process"] = p + return {"status": True} + + def delete(self): + self.runner_server["process"].terminate() + self.runner_server["process"].kill() + counter = 0 + while self.runner_server["process"].poll() is None or counter > 5: + time.sleep(1) + self.runner_server["process"].terminate() + self.runner_server["process"].kill() + counter += 1 + del self.runner_server["process"] + global status_tracker + status_tracker = {} + return {"status": False} + + +api.add_resource(RunnerRun, "/api/runner/run") + + +class RunnerKillBroker(Resource): + def post(self): + name = "helics_broker" + import platform + + if platform.system().lower().startswith("windows"): + # windows OS + if not name.endswith(".exe"): + name = name + ".exe" + os.system("taskkill /f /im {}".format(name)) + # os.system(r"start /b pcs.exe 2>&1") + # with os.popen("tasklist | findstr \"pcs.exe\"") as f: + # temp_content = f.read() + # if len(temp_content) == 0: + # pass + else: + if name.endswith(".exe"): + name = name.replace(".exe", "") + os.system("killall -9 {} 2>&1".format(name)) + + +api.add_resource(RunnerKillBroker, "/api/runner/kill/broker") + + +class RunnerStatus(Resource): + def get(self): + parser = reqparse.RequestParser() + parser.add_argument("name", type=str, required=True, help="Name of federate") + args = parser.parse_args() + return {"status": status_tracker[args["name"]]} + + def post(self): + parser = reqparse.RequestParser() + parser.add_argument("name", type=str, required=True, help="Name of federate") + parser.add_argument( + "status", type=str, required=True, help="Status of federate" + ) + args = parser.parse_args() + status_tracker[args["name"]] = args["status"] + return {"status": status_tracker[args["name"]]} + + +api.add_resource(RunnerStatus, "/api/runner/status") + + +class Health(Resource): + def get(self): + return {"status": 200} + + +api.add_resource(Health, "/api/health") + + +class Profile(Resource): + # SenderFederate2[131074](initializing)HELICS CODE ENTRY<4570827706580384>[t=-1000000] + PATTERN = re.compile( + r""" + (?P\w+) # SenderFederate2 + \[(\d+)\] # [131074] + \((?P\w+)\) # (initializing) + (?P(?:\w|\s)+) # HELICS CODE ENTRY + \<(?P\d+(?:\|\d+)?)\> # <4570827706580384|534534523453> + \[t=(?P-?\d*\.{0,1}\d+)\] # [t=-1000000] + """, + re.X, + ) + + def get(self): + with open(cache["profile-path"]) as f: + data = f.read() + data = data.replace(r"", "").replace(r"", "") + names = [] + states = [] + messages = [] + simtimes = [] + realtimes = [] + time_marker = {} + for line in data.splitlines(): + m = self.PATTERN.match(line) + m = cast(re.Match[str], m) + name = m.group("name") + state = m.group("state") + message = m.group("message") + simtime = float(m.group("simtime")) + try: + realtime = float(m.group("realtime")) + except ValueError: + realtime, markertime = m.group("realtime").split("|") + time_marker[name] = float(markertime) + realtime = float(realtime) + names.append(name) + states.append(state) + messages.append(message) + simtimes.append(simtime) + realtimes.append(realtime) + + profile = {} + for name in set(names): + profile[name] = [] + + invert = True + if invert: + for name in set(names): + profile[name].append({}) + + for name, state, message, simtime, realtime in zip( + names, states, messages, simtimes, realtimes + ): + if state == "created": + continue + if "ENTRY" in message and not invert: + profile[name].append({"s_enter": simtime, "r_enter": realtime}) + elif "EXIT" in message and not invert: + profile[name][-1]["s_end"] = simtime + profile[name][-1]["r_end"] = realtime + elif "EXIT" in message and invert: + profile[name].append({"s_enter": simtime, "r_enter": realtime}) + elif "ENTRY" in message and invert: + profile[name][-1]["s_end"] = simtime + profile[name][-1]["r_end"] = realtime + profiles = {} + names = {k: i for i, k in enumerate(sorted(profile.keys()))} + + end = "r_end" + enter = "r_enter" + scaling = 1e9 + + for k in profile.keys(): + profiles[k] = [] + for i in profile[k]: + if end in i.keys() and enter in i.keys(): + i["name"] = k + i[enter] = i[enter] / scaling + i[end] = i[end] / scaling + profiles[k].append(i) + + return profiles + + def post(self): + parser = reqparse.RequestParser() + parser.add_argument( + "file", type=werkzeug.datastructures.FileStorage, location="files" + ) + args = parser.parse_args() + file = args["file"] + os.makedirs(app.config["UPLOAD_FOLDER"], exist_ok=True) + file.save(os.path.join(app.config["UPLOAD_FOLDER"], "profile.txt")) + cache["profile-path"] = os.path.join(app.config["UPLOAD_FOLDER"], "profile.txt") + return {"status": "success"} + + +api.add_resource(Profile, "/api/profiler/") + + +class APIException(Exception): + def __init__(self, code, message): + self._code = code + self._message = message + + @property + def code(self): + return self._code + + @property + def message(self): + return self._message + + def __str__(self): + return self.__class__.__name__ + ": " + self.message + + +class BrokerServer(Resource): + broker_server = {} + + def get(self): + if self.broker_server.get("process", None) is not None: + return {"status": True} + else: + return {"status": False} + + def post(self): + parser = reqparse.RequestParser() + parser.add_argument( + "status", type=bool, required=True, help="requested status of broker server" + ) + args = parser.parse_args() + status = args["status"] + if status is True and self.broker_server.get("process", None) is not None: + return abort(417, description="Unable to start server", status=True) + elif status is False and self.broker_server.get("process", None) is None: + return abort(417, description="Unable to stop server", status=False) + elif status is True: + p = subprocess.Popen(shlex.split("helics_broker_server --http")) + self.broker_server["process"] = p + return {"status": status} + elif status is False: + self.broker_server["process"].terminate() + self.broker_server["process"].kill() + counter = 0 + # TODO: Find out why broker server is not being terminated + while self.broker_server["process"].poll() is None or counter > 5: + time.sleep(1) + self.broker_server["process"].terminate() + self.broker_server["process"].kill() + counter += 1 + del self.broker_server["process"] + return {"status": status} + + +api.add_resource(BrokerServer, "/api/broker-server") + + +@app.route("/", defaults={"path": "index.html"}) +@app.route("/") +def index(path): + return send_from_directory(os.path.join(current_directory, "static"), path) + + +def run(): + debug = bool(os.environ.get("PYHELICS_FLASK_DEBUG", False)) + if debug: + host = None + else: + host = "0.0.0.0" + cli = sys.modules["flask.cli"] + cli.show_server_banner = lambda *x: None + app.run(host=host, debug=debug) diff --git a/helics/observer.py b/helics_cli_extras/helics_cli_extras/observer.py similarity index 79% rename from helics/observer.py rename to helics_cli_extras/helics_cli_extras/observer.py index 6635f46a..d2ee880f 100644 --- a/helics/observer.py +++ b/helics_cli_extras/helics_cli_extras/observer.py @@ -7,6 +7,7 @@ from typing import Dict, List, cast from datetime import datetime +# HELICS is a runtime dependency here. import helics as h from . import database as db @@ -37,7 +38,9 @@ def _setup_engine(self): db_file = os.path.abspath(os.path.join(self._folder, "helics-cli.sqlite.db")) if os.path.exists(db_file): os.remove(db_file) - self.engine = db.create_engine("sqlite+pysqlite:///{}".format(db_file), echo=False, future=True) + self.engine = db.create_engine( + "sqlite+pysqlite:///{}".format(db_file), echo=False, future=True + ) db.Base.metadata.create_all(self.engine) self.session = db.Session(bind=self.engine) self.session.add(db.MetaData(name="helics_version", value=h.helicsGetVersion())) @@ -76,7 +79,9 @@ def publications(self) -> List[str]: @property def subscriptions(self) -> List[str]: - return [cast(str, name) for name in self.federate.query("root", "subscriptions")] + return [ + cast(str, name) for name in self.federate.query("root", "subscriptions") + ] @property def inputs(self) -> List[str]: @@ -106,7 +111,13 @@ def get_data(self): for federate in core["federates"]: assert federate["attributes"]["parent"] == core["attributes"]["id"] logger.info(f"Adding federate {federate}") - self.session.add(db.Federates(id=federate["attributes"]["id"], name=federate["attributes"]["name"], parent=core["attributes"]["id"])) + self.session.add( + db.Federates( + id=federate["attributes"]["id"], + name=federate["attributes"]["name"], + parent=core["attributes"]["id"], + ) + ) if "inputs" in federate.keys(): for input in federate["inputs"]: @@ -115,16 +126,28 @@ def get_data(self): self.session.add(db.Inputs(source=input["federate"])) else: for s in input["sources"]: - self.session.add(db.Inputs(source=input["federate"], target=s["federate"])) + self.session.add( + db.Inputs( + source=input["federate"], target=s["federate"] + ) + ) if "publications" in federate.keys(): for publication in federate["publications"]: logger.info(f"Adding publication {publication}") if "targets" not in publication.keys(): - self.session.add(db.Publications(source=publication["federate"])) + self.session.add( + db.Publications(source=publication["federate"]) + ) else: for t in publication["targets"]: - self.session.add(db.Publications(name=publication["key"], target=publication["federate"], source=t["federate"])) + self.session.add( + db.Publications( + name=publication["key"], + target=publication["federate"], + source=t["federate"], + ) + ) self.session.commit() @@ -138,7 +161,12 @@ def get_data(self): ] columns.insert(0, db.Column("simulation_time", db.Float)) columns.insert(0, db.Column("updated_at", db.Float)) - columns.insert(0, db.Column("id", db.Integer, db.Sequence("id"), primary_key=True, nullable=False)) + columns.insert( + 0, + db.Column( + "id", db.Integer, db.Sequence("id"), primary_key=True, nullable=False + ), + ) datatable = db.Table("datatable", db.Base.metadata, *columns) class DataTable(db.Base): @@ -154,7 +182,9 @@ def hook(self): def wait(self): federates = self.federates - while not all(self.federate.query(name, "isinit") is True for name in federates): + while not all( + self.federate.query(name, "isinit") is True for name in federates + ): for name in federates: logger.debug(f"{name} isinit = {self.federate.query(name, 'isinit')}") time.sleep(1) @@ -191,7 +221,11 @@ def run(self): self.session.commit() if ( - all(self.federate.query(name, "state") == "disconnected" for name in self.federates if name != "__observer__") + all( + self.federate.query(name, "state") == "disconnected" + for name in self.federates + if name != "__observer__" + ) or simulation_time >= 9223372036.3 ): break From e88c4a7c5e48d3643f4c9bf22b51a9bea757d6d9 Mon Sep 17 00:00:00 2001 From: Joseph McKinsey Date: Fri, 7 Jun 2024 11:10:33 -0600 Subject: [PATCH 09/19] Fix imports on observer and change ci-web to ci-cli.yml --- .github/workflows/{ci-web.yml => ci-cli.yml} | 11 +++++++++-- helics_cli_extras/helics_cli_extras/observer.py | 6 ++++-- 2 files changed, 13 insertions(+), 4 deletions(-) rename .github/workflows/{ci-web.yml => ci-cli.yml} (85%) diff --git a/.github/workflows/ci-web.yml b/.github/workflows/ci-cli.yml similarity index 85% rename from .github/workflows/ci-web.yml rename to .github/workflows/ci-cli.yml index 4751cac4..c827c1d8 100644 --- a/.github/workflows/ci-web.yml +++ b/.github/workflows/ci-cli.yml @@ -1,4 +1,4 @@ -name: CI - Web +name: CI - CLI on: push: @@ -41,9 +41,16 @@ jobs: python setup.py download python setup.py build_ext pip install -e ".[cli]" - - name: Run CLI + - name: Run Server run: | helics server & sleep 5 curl http://localhost:5000 kill %+ + - name: Run Observer + run: | + mkdir db + helics observer & + sleep 5 + curl http://localhost:5000 + kill %+ \ No newline at end of file diff --git a/helics_cli_extras/helics_cli_extras/observer.py b/helics_cli_extras/helics_cli_extras/observer.py index d2ee880f..1eb5e30c 100644 --- a/helics_cli_extras/helics_cli_extras/observer.py +++ b/helics_cli_extras/helics_cli_extras/observer.py @@ -11,6 +11,8 @@ import helics as h from . import database as db +from sqlalchemy import create_engine +from sqlalchemy.orm import Session logger = logging.getLogger(__name__) hdlr = logging.StreamHandler() @@ -38,11 +40,11 @@ def _setup_engine(self): db_file = os.path.abspath(os.path.join(self._folder, "helics-cli.sqlite.db")) if os.path.exists(db_file): os.remove(db_file) - self.engine = db.create_engine( + self.engine = create_engine( "sqlite+pysqlite:///{}".format(db_file), echo=False, future=True ) db.Base.metadata.create_all(self.engine) - self.session = db.Session(bind=self.engine) + self.session = Session(bind=self.engine) self.session.add(db.MetaData(name="helics_version", value=h.helicsGetVersion())) self.session.add(db.MetaData(name="created", value=datetime.now().isoformat())) self.session.add(db.SystemInfo(data=h.helicsGetSystemInfo())) From f58c2f2b0774720f4fa0f7d2cb48608cd148394a Mon Sep 17 00:00:00 2001 From: Joseph McKinsey Date: Fri, 7 Jun 2024 11:12:42 -0600 Subject: [PATCH 10/19] .github/workflows/ci-cli.yml --- .github/workflows/ci-cli.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci-cli.yml b/.github/workflows/ci-cli.yml index c827c1d8..40164ee7 100644 --- a/.github/workflows/ci-cli.yml +++ b/.github/workflows/ci-cli.yml @@ -52,5 +52,4 @@ jobs: mkdir db helics observer & sleep 5 - curl http://localhost:5000 kill %+ \ No newline at end of file From def0ac4cdff7e714a78579377ce15bfabcf1684b Mon Sep 17 00:00:00 2001 From: Joseph McKinsey Date: Fri, 7 Jun 2024 11:35:39 -0600 Subject: [PATCH 11/19] Rename and create cd --- .github/workflows/cd-cli-extras.yml | 70 +++++++++++++++++++ .../{ci-cli.yml => ci-cli-extras.yml} | 0 2 files changed, 70 insertions(+) create mode 100644 .github/workflows/cd-cli-extras.yml rename .github/workflows/{ci-cli.yml => ci-cli-extras.yml} (100%) diff --git a/.github/workflows/cd-cli-extras.yml b/.github/workflows/cd-cli-extras.yml new file mode 100644 index 00000000..db643c03 --- /dev/null +++ b/.github/workflows/cd-cli-extras.yml @@ -0,0 +1,70 @@ +name: CD - CLI Extras + +on: + push: + branches: + - main + tags: + - v* + +jobs: + build-wheels: + runs-on: ubuntu-latest + strategy: + fail-fast: false + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + + - name: Set up Python3 + uses: actions/setup-python@v5 + with: + python-version: 3.8 + + - name: Install python3 dependencies + run: | + python -m pip install -U pip install wheel setuptools cffi build + - name: Build cli_extras wheel + run: | + cd helics_cli_extras + python -m build + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: python-dist + path: helics_cli_extras/dist/* + + publish-helics: + needs: [build-wheels] + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/helics-cli-extras + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + steps: + - name: Get the built packages + uses: actions/download-artifact@v4 + with: + merge-multiple: true + path: dist + + - name: Publish package to TestPyPI + if: startsWith(github.ref, 'refs/tags/') + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.TEST_PYPI_PASSWORD }} + repository-url: https://test.pypi.org/legacy/ + + - name: Publish package to PyPI + if: startsWith(github.ref, 'refs/tags/') + uses: pypa/gh-action-pypi-publish@release/v1 + + - name: GitHub Release + if: startsWith(github.ref, 'refs/tags/') + uses: softprops/action-gh-release@v2 + with: + files: "dist/*" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci-cli.yml b/.github/workflows/ci-cli-extras.yml similarity index 100% rename from .github/workflows/ci-cli.yml rename to .github/workflows/ci-cli-extras.yml From 9e06de45ff51ed0702ba35399aca729118bcd9d0 Mon Sep 17 00:00:00 2001 From: Joseph McKinsey Date: Fri, 7 Jun 2024 11:40:55 -0600 Subject: [PATCH 12/19] Add NPM and environment name --- .github/workflows/cd-cli-extras.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cd-cli-extras.yml b/.github/workflows/cd-cli-extras.yml index db643c03..fe73e086 100644 --- a/.github/workflows/cd-cli-extras.yml +++ b/.github/workflows/cd-cli-extras.yml @@ -27,6 +27,9 @@ jobs: - name: Build cli_extras wheel run: | cd helics_cli_extras + npm install + npm run build + cp -r build helics_cli_extras/static python -m build - name: Upload artifacts uses: actions/upload-artifact@v4 @@ -38,7 +41,7 @@ jobs: needs: [build-wheels] runs-on: ubuntu-latest environment: - name: pypi + name: pypi-cli-extras url: https://pypi.org/p/helics-cli-extras permissions: id-token: write # IMPORTANT: this permission is mandatory for trusted publishing From 448025e52c5a1f9c4d78f73d73187cfe0049bbe7 Mon Sep 17 00:00:00 2001 From: Joseph McKinsey Date: Fri, 7 Jun 2024 11:51:14 -0600 Subject: [PATCH 13/19] Fix cd-cli-extras.yml --- .github/workflows/cd-cli-extras.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/cd-cli-extras.yml b/.github/workflows/cd-cli-extras.yml index fe73e086..42813303 100644 --- a/.github/workflows/cd-cli-extras.yml +++ b/.github/workflows/cd-cli-extras.yml @@ -34,7 +34,7 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: python-dist + name: python-cli-extras-dist path: helics_cli_extras/dist/* publish-helics: @@ -50,7 +50,7 @@ jobs: uses: actions/download-artifact@v4 with: merge-multiple: true - path: dist + path: helics_cli_extras/dist - name: Publish package to TestPyPI if: startsWith(github.ref, 'refs/tags/') @@ -59,15 +59,10 @@ jobs: user: __token__ password: ${{ secrets.TEST_PYPI_PASSWORD }} repository-url: https://test.pypi.org/legacy/ + packages-dir: helics_cli_extras/dist - name: Publish package to PyPI if: startsWith(github.ref, 'refs/tags/') uses: pypa/gh-action-pypi-publish@release/v1 - - - name: GitHub Release - if: startsWith(github.ref, 'refs/tags/') - uses: softprops/action-gh-release@v2 with: - files: "dist/*" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + packages-dir: helics_cli_extras/dist \ No newline at end of file From b4bc70bab4e26d3c57018899a33da59ebac1f44b Mon Sep 17 00:00:00 2001 From: Joseph McKinsey Date: Fri, 7 Jun 2024 12:01:41 -0600 Subject: [PATCH 14/19] Run on workflow dispatch --- .github/workflows/cd-cli-extras.yml | 6 ++++-- .github/workflows/cd.yml | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cd-cli-extras.yml b/.github/workflows/cd-cli-extras.yml index 42813303..86c2b85c 100644 --- a/.github/workflows/cd-cli-extras.yml +++ b/.github/workflows/cd-cli-extras.yml @@ -6,6 +6,7 @@ on: - main tags: - v* + workflow_dispatch: jobs: build-wheels: @@ -26,10 +27,11 @@ jobs: python -m pip install -U pip install wheel setuptools cffi build - name: Build cli_extras wheel run: | - cd helics_cli_extras + cd helics_cli_extras/client npm install npm run build - cp -r build helics_cli_extras/static + cp -r build ../helics_cli_extras/static + cd .. python -m build - name: Upload artifacts uses: actions/upload-artifact@v4 diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index e998fa51..926d865a 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -6,6 +6,7 @@ on: - main tags: - v* + workflow_dispatch: jobs: build-wheels: @@ -97,7 +98,7 @@ jobs: with: user: __token__ password: ${{ secrets.TEST_PYPI_PASSWORD }} - repository_url: https://test.pypi.org/legacy/ + repository-url: https://test.pypi.org/legacy/ - name: Publish package to PyPI if: startsWith(github.ref, 'refs/tags/') From 8b221961c927fafe61f5399616fef33fb25d7a67 Mon Sep 17 00:00:00 2001 From: Joseph McKinsey <31461013+josephmckinsey@users.noreply.github.com> Date: Wed, 24 Jul 2024 09:24:44 -0600 Subject: [PATCH 15/19] Fix typo in helics_cli_extras/pyproject.toml Co-authored-by: Ryan Mast <3969255+nightlark@users.noreply.github.com> --- helics_cli_extras/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helics_cli_extras/pyproject.toml b/helics_cli_extras/pyproject.toml index 6517e5db..283689c3 100644 --- a/helics_cli_extras/pyproject.toml +++ b/helics_cli_extras/pyproject.toml @@ -50,7 +50,7 @@ dependencies = [ ] [project.urls] -Homepage = "hhttps://github.com/GMLC-TDC/pyhelics" +Homepage = "https://github.com/GMLC-TDC/pyhelics" Discussions = "https://github.com/GMLC-TDC/HELICS/discussions" Documentation = "https://python.helics.org/" "Issue Tracker" = "https://github.com/GMLC-TDC/pyhelics/issues" From aa025526f41eb933fec42472edaf3530a98b8e67 Mon Sep 17 00:00:00 2001 From: Joseph McKinsey Date: Wed, 24 Jul 2024 09:32:26 -0600 Subject: [PATCH 16/19] Remove pypi package 'install' --- .github/workflows/cd-cli-extras.yml | 2 +- .github/workflows/cd.yml | 2 +- .github/workflows/ci-cli-extras.yml | 2 +- .github/workflows/ci.yml | 2 +- .github/workflows/docs.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cd-cli-extras.yml b/.github/workflows/cd-cli-extras.yml index 86c2b85c..176d10c7 100644 --- a/.github/workflows/cd-cli-extras.yml +++ b/.github/workflows/cd-cli-extras.yml @@ -24,7 +24,7 @@ jobs: - name: Install python3 dependencies run: | - python -m pip install -U pip install wheel setuptools cffi build + python -m pip install -U pip wheel setuptools cffi build - name: Build cli_extras wheel run: | cd helics_cli_extras/client diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 926d865a..673fd783 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -38,7 +38,7 @@ jobs: - name: Install python3 dependencies run: | - python -m pip install -U pip install wheel setuptools cffi + python -m pip install -U pip wheel setuptools cffi - name: Download helics library run: python setup.py download --plat-name=${{ matrix.target }} diff --git a/.github/workflows/ci-cli-extras.yml b/.github/workflows/ci-cli-extras.yml index 40164ee7..562f829f 100644 --- a/.github/workflows/ci-cli-extras.yml +++ b/.github/workflows/ci-cli-extras.yml @@ -25,7 +25,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install -U pip install wheel setuptools cffi + python -m pip install -U pip wheel setuptools cffi - name: Build NPM run: | cd helics_cli_extras/client diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e1ebc52..c88eb0b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install -U pip install wheel setuptools cffi + python -m pip install -U pip wheel setuptools cffi - name: Download helics library and run pip install run: | python setup.py download diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index ef426d2c..6769293f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -16,7 +16,7 @@ jobs: python-version: "3.12" - name: Install dependencies run: | - python -m pip install -U pip install wheel setuptools + python -m pip install -U pip wheel setuptools python setup.py download pip install -e ".[cli,docs]" - name: Copy README.md From 5cecf0d9a7a88f06e6dd16234dadc9fc0961a4ca Mon Sep 17 00:00:00 2001 From: Joseph McKinsey Date: Thu, 10 Oct 2024 13:16:18 -0600 Subject: [PATCH 17/19] Add import check for helics_cli_extras. Direct to helics[cli] --- helics/cli.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/helics/cli.py b/helics/cli.py index 7276dda0..c85a62c6 100644 --- a/helics/cli.py +++ b/helics/cli.py @@ -106,7 +106,13 @@ def server(open: bool): Run helics web server to access web interface """ import webbrowser - import helics_cli_extras + + try: + import helics_cli_extras + except ImportError: + error( + 'helics-cli\'s web interface is not installed. You may want to run `pip install "helics[cli]"`.' + ) if open: webbrowser.open("http://127.0.0.1:5000", 1) @@ -125,7 +131,12 @@ def observer(db_folder: pathlib.Path): """ Run helics observer and write data to sqlite file """ - from helics_cli_extras import HelicsObserverFederate + try: + from helics_cli_extras import HelicsObserverFederate + except ImportError: + error( + 'helics-cli\'s observer functionality is not installed. You may want to run `pip install "helics[cli]"`.' + ) o = HelicsObserverFederate(folder=db_folder) o.run() From bc92c0cc9dbc888a9ef161f18327d3e6b9da9bd7 Mon Sep 17 00:00:00 2001 From: Joseph McKinsey Date: Fri, 11 Oct 2024 10:22:01 -0600 Subject: [PATCH 18/19] Remove sqlalchemy, pandas, and requests --- helics/cli.py | 13 +++---------- setup.py | 3 --- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/helics/cli.py b/helics/cli.py index c85a62c6..9ef72f84 100644 --- a/helics/cli.py +++ b/helics/cli.py @@ -57,17 +57,10 @@ def _get_version(): pass try: - import flask - except ImportError: - echo( - 'helics-cli\'s web interface is not installed. You may want to run `pip install "helics[cli]"`.' - ) - - try: - import sqlalchemy + import helics_cli_extras except ImportError: echo( - 'helics-cli\'s observer functionality is not installed. You may want to run `pip install "helics[cli]"`.' + 'helics_cli_extras is not installed. You may want to run `pip install "helics[cli]"`.' ) return """{} @@ -111,7 +104,7 @@ def server(open: bool): import helics_cli_extras except ImportError: error( - 'helics-cli\'s web interface is not installed. You may want to run `pip install "helics[cli]"`.' + 'helics_cli_extras is not installed. You may want to run `pip install "helics[cli]"`.' ) if open: diff --git a/setup.py b/setup.py index 496da0c8..1b8b9725 100755 --- a/setup.py +++ b/setup.py @@ -506,10 +506,7 @@ def is_pure(self): helics_cli_install_requires = [ - "requests", "helics_cli_extras==0.0.1", - "pandas", - "SQLAlchemy", "matplotlib", ] From 134accadcd8d253cad812c9f8fd1c3db886d0be0 Mon Sep 17 00:00:00 2001 From: Joseph McKinsey Date: Fri, 11 Oct 2024 15:28:39 -0600 Subject: [PATCH 19/19] Make cd-cli-extras happen only on workflow dispatch (will require manual version updates) --- .github/workflows/cd-cli-extras.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cd-cli-extras.yml b/.github/workflows/cd-cli-extras.yml index 176d10c7..f7070dd4 100644 --- a/.github/workflows/cd-cli-extras.yml +++ b/.github/workflows/cd-cli-extras.yml @@ -4,9 +4,13 @@ on: push: branches: - main - tags: - - v* workflow_dispatch: + inputs: + publish_to_pypi: + description: 'Publish to TestPyPI and PyPI' + required: true + type: boolean + default: false jobs: build-wheels: @@ -55,7 +59,7 @@ jobs: path: helics_cli_extras/dist - name: Publish package to TestPyPI - if: startsWith(github.ref, 'refs/tags/') + if: ${{ github.event.inputs.publish_to_pypi == true }} uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ @@ -64,7 +68,7 @@ jobs: packages-dir: helics_cli_extras/dist - name: Publish package to PyPI - if: startsWith(github.ref, 'refs/tags/') + if: ${{ github.event.inputs.publish_to_pypi == true }} uses: pypa/gh-action-pypi-publish@release/v1 with: packages-dir: helics_cli_extras/dist \ No newline at end of file