refactor: redo entire app build

- Pull in built assets from other repos
- Added unit tests

Dockerfile

@@ -1,114 +1,89 @@
 # Packet Forwarder Docker File
-# (C) Nebra Ltd 2019
+# (C) Nebra Ltd 2021
 # Licensed under the MIT License.
 
 ####################################################################################################
-################################## Stage: builder ##################################################
+########################### Stage: PktFwd Python App Builder #######################################
+FROM balenalib/raspberry-pi-debian:buster-build as pktfwd-builder
 
-FROM balenalib/raspberry-pi-debian:buster-build as builder
+# Variables used internally to this stage
+ENV INPUT_DIR=/opt/input
 
-# Move to correct working directory
-WORKDIR /opt/iotloragateway/dev
+# Variables to be referenced pktfwd-runner stage
+ENV OUTPUT_DIR=/opt/output/pktfwd-dependencies
 
-# Copy python dependencies for `pip install` later
-COPY requirements.txt requirements.txt
-
-# This will be the path that venv uses for installation below
-ENV PATH="/opt/iotloragateway/dev/venv/bin:$PATH"
+WORKDIR "$INPUT_DIR"
 
-# Install build tools
+# Install python3 and pip3
 # hadolint ignore=DL3008
 RUN apt-get update && \
     apt-get -y install --no-install-recommends \
-        automake \
-        libtool \
-        autoconf \
-        git \
-        ca-certificates \
-        pkg-config \
-        build-essential \
-        python3 \
-        python3-pip \
-        python3-venv && \
-    # Because the PATH is already updated above, this command creates a new venv AND activates it
-    python3 -m venv /opt/iotloragateway/dev/venv && \
-    # Given venv is active, this `pip` refers to the python3 variant
-    pip install --no-cache-dir -r requirements.txt
-
-# Copy the buildfiles and sx1302 concentrator fixes
-COPY buildfiles buildfiles
-COPY sx1302fixes sx1302fixes
-
-# Clone the lora gateway and packet forwarder repos
-RUN git clone https://github.com/NebraLtd/lora_gateway.git
-RUN git clone https://github.com/NebraLtd/packet_forwarder.git
-
-# Create folder needed by packetforwarder compiler
-RUN mkdir -p /opt/iotloragateway/packetforwarder
-
-# Compile for sx1301 concentrator on all the necessary SPI buses
-RUN ./buildfiles/compileSX1301.sh spidev0.0
-RUN ./buildfiles/compileSX1301.sh spidev0.1
-RUN ./buildfiles/compileSX1301.sh spidev1.0
-RUN ./buildfiles/compileSX1301.sh spidev1.1
-RUN ./buildfiles/compileSX1301.sh spidev1.2
-RUN ./buildfiles/compileSX1301.sh spidev2.0
-RUN ./buildfiles/compileSX1301.sh spidev2.1
-RUN ./buildfiles/compileSX1301.sh spidev32766.0
-
-# Compile for sx1302 concentrator
-RUN ./buildfiles/compileSX1302.sh
-
-# No need to cleanup the builder
+        python3=3.7.3-1 \
+        python3-pip=18.1-5+rpt1
 
-####################################################################################################
-################################### Stage: runner ##################################################
+# Copy python dependencies for `pip3 install` later
+COPY requirements.txt requirements.txt
 
-FROM balenalib/raspberry-pi-debian:buster-run as runner
+RUN pip3 install --target="$OUTPUT_DIR" --no-cache-dir -r requirements.txt
+# TODO remove once published
+RUN pip3 install setuptools wheel
+RUN pip3 install --target="$OUTPUT_DIR" git+https://github.com/NebraLtd/hm-pyhelper@marvinmarnold/fix-logger-packaging --upgrade --no-cache-dir
 
-# Start in sx1301 directory
-WORKDIR /opt/iotloragateway/packet_forwarder/sx1301
+###################################################################################################
+################################## Stage: runner ##################################################
+FROM balenalib/raspberry-pi-debian:buster-run as pktfwd-runner
 
-# Install python3-venv and python3-rpi.gpio
-# hadolint ignore=DL3008
-RUN apt-get update && \
-    apt-get -y install \
-        python3-venv \
-        python3-rpi.gpio && \
-    apt-get autoremove -y && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
+ENV ROOT_DIR=/opt
+
+# Copy from: Locations of build assets within images of earlier stages
+ENV SX1301_LORA_GATEWAY_OUTPUT_RESET_LGW_FILEPATH=/opt/output/reset_lgw.sh
+ENV SX1301_PACKET_FORWARDER_OUTPUT_DIR=/opt/output
+ENV PKTFWD_BUILDER_OUTPUT_DIR=/opt/output/pktfwd-dependencies
+ENV SX1302_HAL_CHIP_ID_OUTPUT_FILEPATH=/opt/util_chip_id/chip_id
+ENV SX1302_HAL_RESET_LGW_OUTPUT_FILEPATH=/opt/tools/reset_lgw.sh
 
-# Copy sx1301 packetforwader from builder
-COPY --from=builder /opt/iotloragateway/packetforwarder .
+# Copy to: Locations build assets from earlier stages/source are copied into
+ENV PYTHON_APP_DIR="$ROOT_DIR/pktfwd"
+ENV PYTHON_DEPENDENCIES_DIR="$ROOT_DIR/pktfwd-dependencies"
 
-# Copy sx1301 regional config templates
-COPY lora_templates_sx1301 lora_templates_sx1301/
+# Variables required for pktfwd python app
+ENV SX1301_REGION_CONFIGS_DIR="$PYTHON_APP_DIR/config/lora_templates_sx1301"
+ENV SX1302_REGION_CONFIGS_DIR="$PYTHON_APP_DIR/config/lora_templates_sx1302"
+ENV SX1301_DIR="$ROOT_DIR/sx1301"
+ENV SX1302_DIR="$ROOT_DIR/sx1302"
+ENV SX1302_LORA_PKT_FWD_FILEPATH=TODO
+ENV SX1301_RESET_LGW_FILEPATH="$SX1301_DIR/reset_lgw.sh"
+ENV SX1302_RESET_LGW_FILEPATH="$SX1302_DIR/reset_lgw.sh"
+ENV UTIL_CHIP_ID_FILEPATH="$SX1302_DIR/chip_id"
 
-# Use EU config as initial default
-COPY lora_templates_sx1301/local_conf.json local_conf.json
-COPY lora_templates_sx1301/EU-global_conf.json global_conf.json
+WORKDIR "$ROOT_DIR"
 
-# Move to sx1302 directory
-WORKDIR /opt/iotloragateway/packet_forwarder/sx1302
+# Copy python app
+COPY pktfwd/ "$PYTHON_APP_DIR"
 
-# Copy sx1302 hal from builder
-COPY --from=builder /opt/iotloragateway/dev/sx1302_hal-2.1.0 .
+# Copy sx1301 lora_pkt_fwd_SPI_BUS
+COPY --from=nebraltd/packet_forwarder:83fd72d2db4c69faffd387d757452cbe9d594a08 "$SX1301_PACKET_FORWARDER_OUTPUT_DIR" "$SX1301_DIR"
 
-# Copy sx1302 regional config templates
-COPY lora_templates_sx1302 lora_templates_sx1302/
+# Copy sx1301 reset_lgw.sh
+COPY --from=nebraltd/lora_gateway:9c4b1d0c79645c3065aa4c2f3019c14da6cb2675 "$SX1301_LORA_GATEWAY_OUTPUT_RESET_LGW_FILEPATH" "$SX1301_RESET_LGW_FILEPATH"
 
-# Use EU config as initial default
-COPY lora_templates_sx1302/local_conf.json packet_forwarder/local_conf.json
-COPY lora_templates_sx1302/EU-global_conf.json packet_forwarder/global_conf.json
+# Copy sx1302 chip_id utility and reset script
+COPY --from=nebraltd/sx1302_hal:acc0770a1880d77faae6ad4682f94c567be759b4 "$SX1302_HAL_CHIP_ID_OUTPUT_FILEPATH" "$UTIL_CHIP_ID_FILEPATH"
+COPY --from=nebraltd/sx1302_hal:acc0770a1880d77faae6ad4682f94c567be759b4 "$SX1302_HAL_RESET_LGW_OUTPUT_FILEPATH" "$SX1302_RESET_LGW_FILEPATH"
 
-# Move to main packet forwarder directory and copy source code
-WORKDIR /opt/iotloragateway/packet_forwarder
-COPY files/* .
+# Copy pktfwd python app dependencies
+COPY --from=pktfwd-builder "$PKTFWD_BUILDER_OUTPUT_DIR" "$PYTHON_DEPENDENCIES_DIR"
+
+# hadolint ignore=DL3008
+# Install python3 then cleanup
+RUN apt-get update && \
+    apt-get -y install --no-install-recommends python3=3.7.3-1 && \
+    apt-get autoremove -y && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
-# Copy venv from builder and update PATH to activate it
-COPY --from=builder /opt/iotloragateway/dev/venv /opt/iotloragateway/dev/venv
-ENV PATH="/opt/iotloragateway/dev/venv/bin:$PATH"
+# Add python dependencies to PYTHONPATH
+ENV PYTHONPATH="${PYTHONPATH}:${PYTHON_DEPENDENCIES_DIR}"
 
-# Run run_pkt script
-ENTRYPOINT ["sh", "/opt/iotloragateway/packet_forwarder/run_pkt.sh"]
+# Run pktfwd/__main__.py
+ENTRYPOINT ["python3", "pktfwd"]

README.md

@@ -1,9 +1,15 @@
 # hm-pktfwd
 Helium Miner Packet Forwarder
 
-This compiles the packet forwarder container on the Nebra Miners.
+This is a Python app that uses prebuilt utilities to detect the correct concentrator chip and region, then start the concentrator accordingly.
 
-# Supported Region Plans
+hm-pktfwd builds off three other repos which each built a portion of the code required to run the packet forwarder.
+
+- [lora_gateway](https://github.com/NebraLtd/lora_gateway)
+- [packet_forwarder](https://github.com/NebraLtd/packet_forwarder)
+- [sx1302_hal](https://github.com/NebraLtd/sx1302_hal)
+
+## Supported Region Plans
 
 You can typically find the exact region plan you need to use at [What Helium Region](https://whatheliumregion.xyz/) or on the [Helium Miner GitHub repo](https://github.com/helium/miner/blob/master/priv/countries_reg_domains.csv) however the table below provides a rough guide...
 
@@ -27,10 +33,28 @@ Please note:
 | --- | --- |
 | CN779 | NOT YET SUPPORTED |
 
-## Pre built containers
+## Building
+
+### Pre built containers
 
 This repo automatically builds docker containers and uploads them to two repositories for easy access:
 - [hm-pktfwd on DockerHub](https://hub.docker.com/r/nebraltd/hm-pktfwd)
 - [hm-pktfwd on GitHub Packages](https://github.com/NebraLtd/hm-pktfwd/pkgs/container/hm-pktfwd)
 
 The images are tagged using the docker long and short commit SHAs for that release. The current version deployed to miners can be found in the [helium-miner-software repo](https://github.com/NebraLtd/helium-miner-software/blob/production/docker-compose.yml).
+
+### Manual build
+
+When developing, it is faster to build locally instead of relying on the pre-built container to generate.
+
+```bash
+# Cross-compile
+docker buildx build --platform linux/arm64/v8 --progress=plain -t DOCKERHUB_USER/hm-pktfwd .
+
+# To stop at an intermediary stage
+docker buildx build --platform linux/arm64/v8 --progress=plain --target pktfwd-builder -t pktfwd-builder .
+
+# Tag and push image
+docker image tag docker.io/DOCKERHUB_USER/hm-pktfwd DOCKERHUB_USER/hm-pktfwd:0.0.X
+docker push DOCKERHUB_USER/hm-pktfwd:0.0.X
+```

pktfwd/__main__.py

@@ -0,0 +1,108 @@
+import os
+from hm_pyhelper.logger import get_logger
+from pktfwd.pktfwd_app import PktfwdApp
+
+
+LOGGER = get_logger(__name__)
+
+
+#
+# Mandatory
+#
+VARIANT = os.environ['VARIANT']
+SX1301_REGION_CONFIGS_DIR = os.environ['SX1301_REGION_CONFIGS_DIR']
+SX1302_REGION_CONFIGS_DIR = os.environ['SX1302_REGION_CONFIGS_DIR']
+UTIL_CHIP_ID_FILEPATH = os.environ['UTIL_CHIP_ID_FILEPATH']
+SX1301_RESET_LGW_FILEPATH = os.environ['SX1301_RESET_LGW_FILEPATH']
+SX1302_RESET_LGW_FILEPATH = os.environ['SX1302_RESET_LGW_FILEPATH']
+# Directory the python script will be run from.
+# global_conf.json will also be copied here
+ROOT_DIR = os.environ['ROOT_DIR']
+SX1302_LORA_PKT_FWD_FILEPATH = os.environ['SX1302_LORA_PKT_FWD_FILEPATH']
+SX1301_LORA_PKT_FWD_DIR = os.environ['SX1301_DIR']
+
+#
+# Defaults
+#
+
+# File where hm-miner outputs region info
+# https://github.com/NebraLtd/hm-miner/blob/98107f50257de420a42dec50cca6e2f667f899ad/gen-region.sh#L9
+REGION_FILEPATH = os.getenv('REGION_FILEPATH', '/var/pktfwd/region')
+
+# File that pktfwd outputs diagnostic info to.
+# This is different than diagnostics managed by hm-diag.
+# Currently this file only contains "true" or "false",
+# corresponding to whether pktfwd is actively running.
+DIAGNOSTICS_FILEPATH = os.getenv('DIAGNOSTICS_FILEPATH', '/var/pktfwd/diagnostics')  # noqa
+
+# Sleep time before attempting to start concentrator.
+AWAIT_SYSTEM_SLEEP_SECONDS = int(os.getenv('AWAIT_SYSTEM_SLEEP_SECONDS', '5'))
+
+# If False, Sentry will not be enabled
+SENTRY_KEY = os.getenv('SENTRY_PKTFWD', False)
+
+#
+# Optional
+#
+
+# Overrides the region detected in $REGION_FILEPATH, if defined
+REGION_OVERRIDE = os.getenv('REGION_OVERRIDE', False)
+
+# Balena vars used with Sentry
+BALENA_ID = os.getenv('BALENA_DEVICE_UUID')
+BALENA_APP = os.getenv('BALENA_APP_NAME')
+
+
+def main():
+    validate_env()
+    start()
+
+
+def validate_env():
+    LOGGER.debug("Starting with the following ENV:\n\
+        VARIANT=%s\n\
+        REGION_OVERRIDE=%s\n\
+        REGION_FILEPATH=%s\n\
+        SX1301_REGION_CONFIGS_DIR=%s\n\
+        SX1302_REGION_CONFIGS_DIR=%s\n\
+        SENTRY_KEY=%s\n\
+        BALENA_ID=%s\n\
+        BALENA_APP=%s\n\
+        DIAGNOSTICS_FILEPATH=%s\n\
+        AWAIT_SYSTEM_SLEEP_SECONDS=%s\n\
+        SX1301_RESET_LGW_FILEPATH=%s\n\
+        SX1302_RESET_LGW_FILEPATH=%s\n\
+        UTIL_CHIP_ID_FILEPATH=%s\n\
+        ROOT_DIR=%s\n\
+        SX1302_LORA_PKT_FWD_FILEPATH=%s\n\
+        SX1301_LORA_PKT_FWD_DIR=%s\n" %
+            (VARIANT, REGION_OVERRIDE, REGION_FILEPATH,  # noqa E128
+             SX1301_REGION_CONFIGS_DIR, SX1302_REGION_CONFIGS_DIR, SENTRY_KEY,
+             BALENA_ID, BALENA_APP, DIAGNOSTICS_FILEPATH,
+             AWAIT_SYSTEM_SLEEP_SECONDS, SX1301_RESET_LGW_FILEPATH,
+             SX1302_RESET_LGW_FILEPATH, UTIL_CHIP_ID_FILEPATH, ROOT_DIR,
+             SX1302_LORA_PKT_FWD_FILEPATH, SX1301_LORA_PKT_FWD_DIR))
+
+
+def start():
+    pktfwd_app = PktfwdApp(VARIANT, REGION_OVERRIDE, REGION_FILEPATH,
+                           SX1301_REGION_CONFIGS_DIR,
+                           SX1302_REGION_CONFIGS_DIR, SENTRY_KEY,
+                           BALENA_ID, BALENA_APP,
+                           DIAGNOSTICS_FILEPATH,
+                           AWAIT_SYSTEM_SLEEP_SECONDS,
+                           SX1301_RESET_LGW_FILEPATH,
+                           SX1302_RESET_LGW_FILEPATH,
+                           UTIL_CHIP_ID_FILEPATH, ROOT_DIR,
+                           SX1302_LORA_PKT_FWD_FILEPATH,
+                           SX1301_LORA_PKT_FWD_DIR)
+
+    try:
+        pktfwd_app.start()
+    except Exception:
+        LOGGER.exception('__main__ failed for unknown reason')
+        pktfwd_app.stop()
+
+
+if __name__ == "__main__":
+    main()

pktfwd/config/region_config_filenames.py

@@ -0,0 +1,14 @@
+REGION_CONFIG_FILENAMES = {
+    "AS923_1": "AS923-1-global_conf.json",
+    "AS923_2": "AS923-2-global_conf.json",
+    "AS923_3": "AS923-3-global_conf.json",
+    "AS923_4": "AS923-4-global_conf.json",
+    "AU915": "AU-global_conf.json",
+    "CN470": "CN-global_conf.json",
+    "EU433": "EU433-global_conf.json",
+    "EU868": "EU-global_conf.json",
+    "IN865": "IN-global_conf.json",
+    "KR920": "KR-global_conf.json",
+    "RU864": "RU-global_conf.json",
+    "US915": "US-global_conf.json"
+}