Merge pull request #452 from MyElectricalData/0.9.4

0.9.4

.env.example

@@ -0,0 +1,4 @@
+APPLICATION_PATH=src/
+APPLICATION_PATH_DATA=data/cache/
+APPLICATION_PATH_LOG=data/log
+DEBUG=false

.github/workflows/semantic_release.yaml

@@ -0,0 +1,41 @@
+name: "semantic_release"
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+
+  #################################################################################################################
+  # SEMANTIC RELEASE
+  Semantic-Release:
+    runs-on: [self-hosted, decathlon]
+    environment: production
+    concurrency: release
+    permissions:
+      contents: write
+      id-token: write
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      ################################################################################################################
+      # PRE CLEAN
+      - name: Pre clean
+        uses: dktunited/.github/actions/cleaner@main
+
+      #################################################################################################################
+      # GENERATE RELEASE
+      - name: Release
+        uses: python-semantic-release/python-semantic-release@v8.7.0
+        with:
+         directory: ./
+         github_token: ${{ secrets.GITHUB_TOKEN }}
+         push: true
+
+      #################################################################################################################
+      # Post CLEAN
+      - name: Post clean
+        uses: dktunited/.github/actions/cleaner@main

.gitignore

@@ -1,16 +1,20 @@
-/.idea
-/.env
-/data/*
-/tmp
-/data/config.yaml
-/mosquitto
-/data
+.idea
+.env
+data/*
+tmp
+data/config.yaml
+mosquitto
+data
+.venv
+.pytest_cache
+.coverage
 
 /__pycache__/
-/app/alembic/__pycache__/
-/app/alembic/versions/__pycache__/
-/app/__pycache__/
-/app/templates/__pycache__/
-/app/templates/models/__pycache__/
-/app/models/__pycache__/
-/app/routers/__pycache__/
+/src/alembic/__pycache__/
+/src/alembic/versions/__pycache__/
+/src/__pycache__/
+/src/templates/__pycache__/
+/src/templates/models/__pycache__/
+/src/models/__pycache__/
+/src/routers/__pycache__/
+tests/__pycache__/test_jobs.cpython-310-pytest-7.4.0.pyc

.pre-commit-config.yaml

@@ -0,0 +1,26 @@
+repos:
+  - repo: local
+    hooks:
+      - id: black
+        name: black
+        always_run: true
+        entry: bash -c 'make black'
+        language: system
+        types: [python]
+        pass_filenames: false
+
+#      - id: flake8
+#        name: flake8
+#        always_run: true
+#        entry: bash -c 'make flake8'
+#        language: system
+#        types: [python]
+#        pass_filenames: false
+
+#      - id: pylint
+#        name: pylint
+#        always_run: true
+#        entry: bash -c 'make pylint'
+#        language: system
+#        types: [python]
+#        pass_filenames: false

.tool-versions

@@ -1 +1,2 @@
-vault 1.12.2+ent
+poetry 1.7.1
+python 3.12.1

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11-slim
+FROM python:3.12.1-slim
 
 RUN apt-get update && \
     apt-get install -y \
@@ -17,20 +17,14 @@ ENV TZ=Europe/Paris
 
 RUN pip install --upgrade pip pip-tools setuptools
 
-COPY ./app /app
+COPY ./src /app
 
 RUN pip-compile -o /app/requirements.txt /app/pyproject.toml
 RUN pip install -r /app/requirements.txt
-RUN #pip install git+https://github.com/influxdata/influxdb-client-python.git@master
 
 RUN mkdir /data
 RUN mkdir /log
 
-#RUN sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
-#RUN apt-get install wget
-#RUN /bin/wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
-#RUN apt-get update
-#RUN apt-get -y install postgresql
 RUN apt-get clean
 
 CMD ["python", "-u", "/app/main.py"]

Makefile

@@ -1,87 +1,138 @@
-COMPOSE=docker compose -f dev/docker-compose.dev.yaml
-COMPOSE_TEST=docker compose -f dev/docker-compose.test.yaml
-
-.DEFAULT_GOAL := wizard
-## Run wizard
-wizard:
-	python3 main.py action=wizard
-
-## Connect to EnedisGateway container : DEV
-debug:
-	$(COMPOSE) stop myelectricaldata_import
-	$(COMPOSE) rm -f myelectricaldata_import
-	python3 main.py action=debug
-
-## Connect to EnedisGateway container : DEV
-run:
-	$(COMPOSE) stop myelectricaldata_import
-	$(COMPOSE) rm -f myelectricaldata_import
-	python3 main.py action=run
-
-## Connect to EnedisGateway container : DEV
-run-test:
-	$(COMPOSE_TEST) stop myelectricaldata_import
-	$(COMPOSE_TEST) rm -f myelectricaldata_import
-	python3 main.py action=run env=test
-
-## Connect to EnedisGateway container : DEV
-run-production:
-	$(COMPOSE) stop myelectricaldata_import
-	$(COMPOSE) rm -f myelectricaldata_import
-	python3 main.py action=run env=production
-
-## Start docker conatiners for dev
-up:
-	@echo "Start docker container for dev"
-	$(COMPOSE) up -d
-	@echo ""
-	@echo "\033[0;33mMQTT Explorer:\033[0m    \033[0;32mhttp://127.0.0.1:4000\033[0m    Auth info: (host: mosquitto)"
-	@echo "\033[0;33mInflux DB:    \033[0m    \033[0;32mhttp://127.0.0.1:8086\033[0m    Auth info: (user: enedisgateway2mqtt, pawword: enedisgateway2mqtt)"
-
-## Stop docker conatiners for dev
-down:
-	@echo "Start docker container for dev"
-	$(COMPOSE) down
-
-## Start in app
-start:
-	$(COMPOSE) exec myelectricaldata_import python -u /app/main.py
-
-## Connect to enedisgateway2mqtt container
-bash:
-	$(COMPOSE) exec myelectricaldata_import bash
-
-## Create git branch
-version=
-git_branch:
-	git branch $(version) || true
-	git checkout $(version) || true
-	echo -n $(version) > app/VERSION
-
-## Create add/commit/push
-current_version := $(shell cat app/VERSION)
-comment=
-.PHONY: git_push
-git_push:
-	set -x
-	@(echo "git add --all")
-	git add --all
-	@if [ "$(comment)" = "" ]; then comment="maj"; fi; \
-    echo "git commit -m '$${comment}'"
-	git commit -m "$${comment}"
-	@(echo "git push origin $(current_version)")
-	git push origin $(current_version)
+SHELL := /bin/bash
+
+VERSION :=  $(shell cat src/VERSION)
+
+PY = python3
+VENV = .venv
+BIN=$(VENV)/bin
 
+define title
+@/bin/echo -e "\n------------------------------------------------\n${1}\n------------------------------------------------\n"
+endef
+define poetry
+$(call title, ${2})
+touch .env
+poetry run -vvv ${1}
+endef
+
+check: check_asdf
+
+check_asdf:
+	@if ! command -v asdf &> /dev/null; then \
+		echo "Error: asdf is not installed. Please install asdf first."; \
+		echo " => https://asdf-vm.com/guide/getting-started.html"; \
+		exit 1; \
+	fi
+
+.ONESHELL:
+.DEFAULT_GOAL := run
 
 ## Generate requirements.txt
 generate-dependencies:
-	cd app; pip-compile -o requirements.txt pyproject.toml; cd -
+	poetry export --without-hashes --output src/requirements.txt
+
+## Install dev environment
+install: install_libdev install_asdf upgrade_pip configure_poetry
+
+install_libdev:
+	@$(call title, Install libdev requirement for python\n=> https://github.com/pyenv/pyenv/wiki#suggested-build-environment)
+	if [ "$(shell uname)" == "Darwin" ]; then \
+		brew install openssl readline sqlite3 xz zlib tcl-tk; \
+	else\
+		sudo apt update; \
+		sudo apt install -y build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev curl libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev; \
+	fi
+
+install_asdf:
+	@$(call title,"ASDF Install in .tool-versions")
+	asdf plugin-add poetry https://github.com/asdf-community/asdf-poetry.git
+	asdf plugin-add python
+	asdf install
+
+upgrade_pip:
+	@$(call poetry, pip install --upgrade pip, "Upgrade pip")
+
+configure_poetry:
+	@$(call title,"Poetry set python version")
+	sed -i 's/python = .*/python = "$(shell cat .tool-versions|grep 'python' | cut -d " " -f 2)"/g' pyproject.toml
+	sed -i 's/FROM python:.*/FROM python:'$(shell cat .tool-versions|grep 'python' | cut -d " " -f 2)'-slim/g' Dockerfile
+	poetry env use ~/.asdf/installs/python/$(shell cat .tool-versions|grep 'python' | cut -d " " -f 2)/bin/python
+	@$(call title,"Poetry self plugin")
+	poetry self add poetry-plugin-export poetry-dotenv-plugin
+	@$(call title,"Poetry install")
+	poetry install
+	poetry lock
+
+
+clean: python-clean
+## Clean venv
+python-clean:
+	rm -Rf $(VENV)
+
+## Init local environment
+init:
+	if [ ! -f ".env" ]; then \
+  		cp env.example .env; \
+	fi
+
+## Git Init
+git-init:
+	@$(call poetry, pre-commit install, "Install Pre-Commit in Git Hooks")
+
+## Bootstap application
+bootstrap:
+	@$(call poetry, --ansi python src/main.py, "Run main.py")
+
+## Enable debug mode
+enable_debug:
+	sed -i "s/DEBUG=.*/DEBUG=true/g" .env
+
+## Disable debug mode
+disable_debug:
+	sed -i "s/DEBUG=.*/DEBUG=false/g" .env
+
+## Run in local
+run: init disable_debug bootstrap
+
+## Run in local in debug
+debug: init enable_debug up bootstrap down
+
+## Start all external ressource necessary to debug (MQTT, InfluxDB,...)
+up:
+	docker compose -f dev/docker-compose.dev.yaml start
+
+## Stop all external ressource necessary to debug (MQTT, InfluxDB,...)
+down:
+	docker compose -f dev/docker-compose.dev.yaml stop
+
+## Run PyTest only
+test: pytest
+
+tnr: tox
+## Run regression testing
+tox: init
+	@$(call poetry, tox, "Run TOX")
+
+## Run PyTest
+pytest: init
+	@$(call poetry, tox -e pytest, "Run PyTest")
+
+## Run black
+black: init
+	@$(call poetry, tox -e black, "Run Black")
+
+## Run Black formater
+black-format: init
+	@$(call poetry, black --line-length 119 --color ., "Run Black formater")
+
+##Run flake8
+flake8: init
+	@$(call poetry, tox -e flake8, "Run Flake8")
 
-clean: generate-dependencies
-	docker image rm -f dev-myelectricaldata_import
-	docker image rm -f dev_myelectricaldata_import
-#	docker build ./
+##Run pylint
+pylint: init
+	@$(call poetry, tox -e pylint, "Run pylint")
 
-## Create github release (prod)
-create-release:
-	python3 main.py action=create_release
+##Run pre-commit
+pre-commit: init
+	@$(call poetry, pre-commit run --all-files, "Run pre-commit test")