Merge pull request #110 from tum-esm/main

v4.0.5

.github/workflows/test-on-push-to-main.yml

@@ -0,0 +1,76 @@
+name: 'test-on-push-to-main'
+on:
+    push:
+        branches:
+            - main
+    pull_request:
+        branches:
+            - main
+
+jobs:
+    test-typescript-codebase:
+        runs-on: ubuntu-latest
+        steps:
+            # check-out repo and set-up python
+            - name: Check out repository
+              uses: actions/checkout@v3
+            - name: Set up NodeJS with Yarn
+              uses: actions/setup-node@v3
+              with:
+                  node-version: '14'
+                  cache: 'yarn'
+                  cache-dependency-path: packages/ui/yarn.lock
+            - name: Install dependencies
+              run: yarn install
+              working-directory: packages/ui
+            - name: Build frontend
+              run: yarn build
+              working-directory: packages/ui
+
+    test-python-codebase:
+        runs-on: ubuntu-latest
+        steps:
+            # check-out repo and set-up python
+            - name: Check out repository
+              uses: actions/checkout@v3
+            - name: Set up Python 3.10.6
+              id: setup-python
+              uses: actions/setup-python@v3
+              with:
+                  python-version: 3.10.6
+
+            # install & configure poetry
+            - name: Install Poetry
+              uses: snok/install-poetry@v1
+              with:
+                  virtualenvs-create: true
+                  virtualenvs-in-project: true
+                  installer-parallel: true
+
+            # load cached venv if cache exists
+            - name: Load cached venv
+              id: cached-poetry-dependencies
+              uses: actions/cache@v3
+              with:
+                  path: .venv
+                  key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('poetry.lock') }}
+
+            # install dependencies if cache does not exist
+            - name: Install dependencies
+              if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+              run: poetry install --no-interaction --no-root
+
+            # install your root project, if required
+            - name: Install library
+              run: poetry install --no-interaction
+
+            # run test suite
+            - name: Run mypy static type analysis
+              run: |
+                  source .venv/bin/activate
+                  bash scripts/run_type_analysis.sh
+            - name: Run pytest tests
+              run: |
+                  source .venv/bin/activate
+                  pytest -m "ci" --cov=packages tests
+                  coverage report

.gitignore

@@ -3,10 +3,12 @@
 *.pyc
 .pytest_cache
 __pycache__
+.coverage
+test-tmp/
 
 # config
 config/config.json
-config/config.original.json
+config/config.tmp.json
 config/*.lock
 .env
 packages/cli/alias/pyra-cli.bat
@@ -31,8 +33,9 @@ runtime-data/
 pyra-core-process-state.json
 logs/persistent-state.json
 logs/activity/*.json
-logs/vbdsd/*.jpg
-logs/vbdsd-autoexposure/*.jpg
+logs/**/*.jpg
+!logs/helios/.gitkeep
+logs/helios/*
 
 # pyra ui
 packages/electron-ui/node_modules/
@@ -43,4 +46,6 @@ packages/ui/node_modules/
 # development
 .vscode/
 .idea
-hidden
+hidden
+website/.docusaurus
+website/node_modules

README.md

@@ -1,58 +1,33 @@
-**Work in progress! Do not use it yet.**
+# PYRA
 
-<br/>
-
-# Pyra Version 4
-
-## Set up with
-
-Dependency management using https://python-poetry.org/.
-
-```bash
-# create a virtual environment (copy of the python interpreter)
-python3.10 -m venv .venv
-
-# activate virtual environment
-source .venv/bin/activate   # unix
-.venv\Scripts\activate.bat  # windows
-
-# when your venv is activated your command line has a (.venv) prefix
-# install dependencies using poetry
-poetry install
-```
+**For installation, see https://github.com/tum-esm/pyra-setup-tool**
 
 <br/>
 
-## Configuration Files
-
-Two types of config files:
+## Repository Management & CI
 
-1. **`setup.json`** contains all information about the static setup: Which parts does the enclosure consist of? This should be written once and only changes when the hardware changes.
-2. **`parameters.json`** contains all dynamic parameters that can be set when operating pyra. This should be manipulated either via the CLI (coming soon) or the graphical user interface (coming soon, similar to Pyra version <= 3).
+**Branches:** `development-...`, `integration-x.y.z`, `main`, `release`, `prerelease`
 
-For each file, there is a `*.default.json` file present in the repository. A full reference can be found here soon.
+**Hierarchy:** `development-...` contains stuff in active development and will be merged into `integration-x.y.z`. `integration-x.y.z`: Is used during active integration on the stations and will be merged into `main`. `main` contains the latest running version that passed the integration and will be merged into `release` once enough changes have accumulated. Any branch can be released into `prerelease` to run the CI-Pipeline on demand. `prerelease` will not be merged into anything else and is just used for development purposes.
 
-<br/>
+**Continuous Integration:** The CI-Pipeline runs every time a commit/a series of commits is added to the `release` branch. The CI compiles and bundles the frontend code into an installable windows-application. Then it creates a new release draft and attaches the `.msi` file to the draft. We can then manually add the release description and submit the release.
 
-## CLI
+**Testing (not in an active CI):** We could add automated tests to the main- and integration branches. However, most things we could test make use of OPUS, Camtracker, Helios, or the enclosure, hence we can only do a subset of our tests in an isolated CI environment without the system present.
 
-_documentation coming soon_
+**Issues:** Things we work on are managed via issues - which are bundled into milestones (each milestone represents a release). The issues should be closed once they are on the `main` branch via commit messages ("closes #87", "fixes #70", etc. see [this list of keywords](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword)). Issues that have been finished but are not on the `main` branch yet, can be labeled using the white label "implemented". This way, we can oversee incompleted issues, but don't forget to merge them.
 
-Make `pyra-cli` command available:
+<br/>
 
-```bash
-alias pyra-cli=".../.venv/bin/python .../packages/cli/main.py"
-```
+## Elements
 
-TODO: Find a way to set up autocompletion on the `pyra-cli` command.
+### FileLocks
 
-<br/>
+Since we have parallel processes interacting with state, config, and logs, we need to control the access to these resources to avoid race conditions. We use the python module [filelock](https://pypi.org/project/filelock/) for this. Before working with one of these resources, a process has to acquire a file lock for the respective `.state.lock`/`.config.lock`/`.logs.lock` file. When it cannot acquire a lock for 10 seconds, it throws a `TimeoutError`.
 
-## Graphical User Interface
+When running into a deadlock, with timeout errors (never happened to us yet), the CLI command `pyra-cli remove-filelocks` removes all present lock files.
 
-_documentation coming soon_
+### Version numbers
 
-Less Secure Apps have been deactivated.
-https://support.google.com/accounts/answer/6010255?hl=de&visit_id=637914296292859831-802637670&p=less-secure-apps&rd=1
+Versions up to `4.0.4` are alpha and beta versions that should not be used regularly. PYRA can be generally used starting from version `4.0.5`.
 
-Solution: Use "App passwords", requires 2FA
+Inside the codebase, the version number is included 3 times: `pyproject.toml`, `packages/ui/package.json`, `packages/ui/src-tauri/tauri.conf.json`. The script `scripts/sync_version_numbers.py` takes the version number from the `.toml` file and pastes it into the other locations. This script can be run in a [git-hook](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks).

config/config.default.json

@@ -1,5 +1,6 @@
 {
     "general": {
+        "version": "4.0.5",
         "seconds_per_core_interval": 30,
         "test_mode": false,
         "station_id": "...",
@@ -34,7 +35,7 @@
     "measurement_triggers": {
         "consider_time": true,
         "consider_sun_elevation": true,
-        "consider_vbdsd": false,
+        "consider_helios": false,
         "start_time": {
             "hour": 7,
             "minute": 0,
@@ -48,5 +49,6 @@
         "min_sun_elevation": 0
     },
     "tum_plc": null,
-    "vbdsd": null
+    "helios": null,
+    "upload": null
 }

config/vbdsd.config.default.json → config/helios.config.default.json

@@ -3,5 +3,6 @@
     "evaluation_size": 15,
     "seconds_per_interval": 6,
     "measurement_threshold": 0.6,
+    "edge_detection_threshold": 0.02,
     "save_images": false
 }

config/upload.config.default.json

@@ -0,0 +1,12 @@
+{
+    "host": "1.2.3.4",
+    "user": "...",
+    "password": "...",
+    "upload_ifgs": false,
+    "src_directory_ifgs": "...",
+    "dst_directory_ifgs": "...",
+    "remove_src_ifgs_after_upload": false,
+    "upload_helios": false,
+    "dst_directory_helios": "...",
+    "remove_src_helios_after_upload": true
+}

packages/cli/commands/__init__.py

@@ -0,0 +1,6 @@
+from .config import config_command_group
+from .core import core_command_group
+from .logs import logs_command_group
+from .plc import plc_command_group
+from .remove_filelocks import remove_filelocks
+from .state import state_command_group

packages/cli/commands/config.py

@@ -3,6 +3,7 @@
 import click
 import os
 import sys
+from packages.core import types
 from packages.core.utils import with_filelock, update_dict_recursively
 
 dir = os.path.dirname
@@ -13,16 +14,19 @@
 
 
 sys.path.append(PROJECT_DIR)
-from packages.core.utils import ConfigValidation
 
-error_handler = lambda text: click.echo(click.style(text, fg="red"))
-success_handler = lambda text: click.echo(click.style(text, fg="green"))
-ConfigValidation.logging_handler = error_handler
+
+def print_green(text: str) -> None:
+    click.echo(click.style(text, fg="green"))
+
+
+def print_red(text: str) -> None:
+    click.echo(click.style(text, fg="red"))
 
 
 @click.command(help="Read the current config.json file.")
 @with_filelock(CONFIG_LOCK_PATH)
-def _get_config():
+def _get_config() -> None:
     if not os.path.isfile(CONFIG_FILE_PATH):
         shutil.copyfile(DEFAULT_CONFIG_FILE_PATH, CONFIG_FILE_PATH)
     with open(CONFIG_FILE_PATH, "r") as f:
@@ -31,7 +35,7 @@ def _get_config():
         except:
             raise AssertionError("file not in a valid json format")
 
-    ConfigValidation.check_structure(content)
+    types.validate_config_dict(content, partial=False, skip_filepaths=True)
     click.echo(json.dumps(content))
 
 
@@ -41,35 +45,62 @@ def _get_config():
 )
 @click.argument("content", default="{}")
 @with_filelock(CONFIG_LOCK_PATH)
-def _update_config(content: str):
-    # The validation itself might print stuff using the error_handler
-    if not ConfigValidation.check_partial_config_string(content):
+def _update_config(content: str) -> None:
+    # try to load the dict
+    try:
+        new_partial_json = json.loads(content)
+    except:
+        print_red("content argument is not a valid JSON string")
         return
-    new_partial_json = json.loads(content)
 
-    with open(CONFIG_FILE_PATH, "r") as f:
-        current_json: dict = json.load(f)
+    # validate the dict's integrity
+    try:
+        types.validate_config_dict(new_partial_json, partial=True)
+    except Exception as e:
+        print_red(str(e))
+        return
 
+    # load the current json file
+    try:
+        with open(CONFIG_FILE_PATH, "r") as f:
+            current_json = json.load(f)
+    except:
+        print_red("Could not load the current config.json file")
+        return
+
+    # merge current config and new partial config
     merged_json = update_dict_recursively(current_json, new_partial_json)
     with open(CONFIG_FILE_PATH, "w") as f:
         json.dump(merged_json, f, indent=4)
 
-    success_handler("Updated config file")
+    print_green("Updated config file")
 
 
 @click.command(
     help=f"Validate the current config.json file.\n\nThe required schema can be found in the documentation."
 )
 @with_filelock(CONFIG_LOCK_PATH)
-def _validate_current_config():
-    # The validation itself might print stuff using the error_handler
-    file_is_valid, _ = ConfigValidation.check_current_config_file()
-    if file_is_valid:
-        success_handler(f"Current config file is valid")
+def _validate_current_config() -> None:
+    # load the current json file
+    try:
+        with open(CONFIG_FILE_PATH, "r") as f:
+            current_json = json.load(f)
+    except:
+        print_red("Could not load the current config.json file")
+        return
+
+    # validate its integrity
+    try:
+        types.validate_config_dict(current_json, partial=False)
+    except Exception as e:
+        print_red(str(e))
+        return
+
+    print_green(f"Current config file is valid")
 
 
 @click.group()
-def config_command_group():
+def config_command_group() -> None:
     pass