Merge branch '0.8.13' into master

Makefile

@@ -1,4 +1,5 @@
 COMPOSE=docker compose -f dev/docker-compose.dev.yaml
+COMPOSE_TEST=docker compose -f dev/docker-compose.test.yaml
 
 .DEFAULT_GOAL := wizard
 ## Run wizard
@@ -17,6 +18,12 @@ run:
 	$(COMPOSE) rm -f myelectricaldata_import
 	python3 main.py action=run
 
+## Connect to EnedisGateway container : DEV
+run-test:
+	$(COMPOSE_TEST) stop myelectricaldata_import
+	$(COMPOSE_TEST) rm -f myelectricaldata_import
+	python3 main.py action=run env=test
+
 ## Connect to EnedisGateway container : DEV
 run-production:
 	$(COMPOSE) stop myelectricaldata_import

app/VERSION

@@ -1 +1 @@
-0.8.12-beta2
+0.8.13-beta3

app/main.py

@@ -165,6 +165,11 @@ def lock():
     def gateway_status():
         return Ajax().gateway_status()
 
+    @APP.route("/datatable/<usage_point_id>/<measurement_direction>", methods=['GET'])
+    @APP.route("/datatable/<usage_point_id>/<measurement_direction>/", methods=['GET'])
+    def datatable(usage_point_id, measurement_direction):
+        return Ajax(usage_point_id).datatable(measurement_direction, request.args)
+
 
     @APP.route("/configuration/<usage_point_id>", methods=['POST'])
     @APP.route("/configuration/<usage_point_id>/", methods=['POST'])
@@ -201,6 +206,11 @@ def import_data(usage_point_id, target):
     def reset_all_data(usage_point_id):
         return Ajax(usage_point_id).reset_all_data()
 
+    @APP.route("/delete/<usage_point_id>", methods=['GET'])
+    @APP.route("/delete/<usage_point_id>/", methods=['GET'])
+    def delete_all_data(usage_point_id):
+        return Ajax(usage_point_id).delete_all_data()
+
 
     @APP.route("/reset_gateway/<usage_point_id>", methods=['GET'])
     @APP.route("/reset_gateway/<usage_point_id>/", methods=['GET'])

app/models/database.py

@@ -4,10 +4,6 @@
 from datetime import datetime, timedelta
 from os.path import exists
 
-from sqlalchemy import (create_engine, delete, inspect, select)
-from sqlalchemy.orm import sessionmaker
-
-from config import MAX_IMPORT_TRY
 from db_schema import (
     Config,
     Contracts,
@@ -22,6 +18,10 @@
 from dependencies import str2bool
 from models.config import get_version
 from models.log import Log
+from sqlalchemy import (create_engine, delete, inspect, select, func, desc, asc)
+from sqlalchemy.orm import sessionmaker
+
+from config import MAX_IMPORT_TRY
 
 LOG = Log()
 
@@ -39,14 +39,16 @@ def __init__(self, path="/data"):
 
         self.lock_file = f"{self.path}/.lock"
 
+        self.yesterday = datetime.combine(datetime.now() - timedelta(days=1), datetime.max.time())
+
         # MIGRATE v7 to v8
         if os.path.isfile(f"{self.path}/enedisgateway.db"):
             LOG.title_warning("Migration de l'ancienne base de données vers la nouvelle structure.")
             self.migratev7tov8()
 
     def migratev7tov8(self):
         uri = f'sqlite:///{self.path}/enedisgateway.db'
-        engine = create_engine(uri, echo=False, query_cache_size=0)
+        engine = create_engine(uri, echo=True, query_cache_size=0)
         session = sessionmaker(engine)(autocommit=True, autoflush=True)
 
         for measurement_direction in ["consumption", "production"]:
@@ -250,25 +252,25 @@ def set_usage_point(self, usage_point_id, data):
         else:
             production_price = 0
         if (
-            "consumption_price_base" in data
-            and data["consumption_price_base"] is not None
-            and data["consumption_price_base"] != ""
+                "consumption_price_base" in data
+                and data["consumption_price_base"] is not None
+                and data["consumption_price_base"] != ""
         ):
             consumption_price_base = data["consumption_price_base"]
         else:
             consumption_price_base = 0
         if (
-            "consumption_price_hc" in data
-            and data["consumption_price_hc"] is not None
-            and data["consumption_price_hc"] != ""
+                "consumption_price_hc" in data
+                and data["consumption_price_hc"] is not None
+                and data["consumption_price_hc"] != ""
         ):
             consumption_price_hc = data["consumption_price_hc"]
         else:
             consumption_price_hc = 0
         if (
-            "consumption_price_hp" in data
-            and data["consumption_price_hp"] is not None
-            and data["consumption_price_hp"] != ""
+                "consumption_price_hp" in data
+                and data["consumption_price_hp"] is not None
+                and data["consumption_price_hp"] != ""
         ):
             consumption_price_hp = data["consumption_price_hp"]
         else:
@@ -621,6 +623,46 @@ def get_daily_all(self, usage_point_id, measurement_direction="consumption"):
             .order_by(table.date.desc())
         ).all()
 
+    def get_daily_datatable(self, usage_point_id, order_column="date", order_dir="asc", search=None,
+                            measurement_direction="consumption"):
+        if measurement_direction == "consumption":
+            table = ConsumptionDaily
+            relation = UsagePoints.relation_consumption_daily
+        else:
+            table = ProductionDaily
+            relation = UsagePoints.relation_production_daily
+
+        sort = asc(order_column) if order_dir == "desc" else desc(order_column)
+
+        if search is not None and search != "":
+            result = self.session.scalars(select(table)
+                                          .join(relation)
+                                          .where(UsagePoints.usage_point_id == usage_point_id)
+                                          .where((table.date.like(f"%{search}%")) | (table.value.like(f"%{search}%")))
+                                          .where(table.date <= self.yesterday)
+                                          .order_by(sort)
+                                          )
+        else:
+            result = self.session.scalars(select(table)
+                                          .join(relation)
+                                          .where(UsagePoints.usage_point_id == usage_point_id)
+                                          .where(table.date <= self.yesterday)
+                                          .order_by(sort)
+                                          )
+        return result.all()
+
+    def get_daily_count(self, usage_point_id, measurement_direction="consumption"):
+        if measurement_direction == "consumption":
+            table = ConsumptionDaily
+            relation = UsagePoints.relation_consumption_daily
+        else:
+            table = ProductionDaily
+            relation = UsagePoints.relation_production_daily
+        return self.session.scalars(
+            select([func.count()]).select_from(table).join(relation)
+            .where(UsagePoints.usage_point_id == usage_point_id)
+        ).one_or_none()
+
     def get_daily_date(self, usage_point_id, date, measurement_direction="consumption"):
         unique_id = hashlib.md5(f"{usage_point_id}/{date}".encode('utf-8')).hexdigest()
         if measurement_direction == "consumption":
@@ -840,6 +882,16 @@ def insert_daily(self, usage_point_id, date, value, blacklist=0, fail_count=0,
                 )
             )
 
+    def reset_daily(self, usage_point_id, date=None, mesure_type="consumption"):
+        daily = self.get_daily_date(usage_point_id, date, mesure_type)
+        if daily is not None:
+            daily.value = 0
+            daily.blacklist = 0
+            daily.fail_count = 0
+            return True
+        else:
+            return False
+
     def delete_daily(self, usage_point_id, date=None, measurement_direction="consumption"):
         if measurement_direction == "consumption":
             table = ConsumptionDaily
@@ -916,6 +968,45 @@ def get_detail_all(self, usage_point_id, begin=None, end=None, measurement_direc
                 .order_by(table.date)
             ).all()
 
+    def get_detail_datatable(self, usage_point_id, order_column="date", order_dir="asc", search=None,
+                             measurement_direction="consumption"):
+        if measurement_direction == "consumption":
+            table = ConsumptionDetail
+            relation = UsagePoints.relation_consumption_detail
+        else:
+            table = ProductionDetail
+            relation = UsagePoints.relation_production_detail
+
+        sort = asc(order_column) if order_dir == "desc" else desc(order_column)
+        if search is not None and search != "":
+            result = self.session.scalars(select(table)
+                                          .join(relation)
+                                          .where(UsagePoints.usage_point_id == usage_point_id)
+                                          .where((table.date.like(f"%{search}%")) | (table.value.like(f"%{search}%")))
+                                          .where(table.date <= self.yesterday)
+                                          .order_by(sort)
+                                          )
+        else:
+            result = self.session.scalars(select(table)
+                                          .join(relation)
+                                          .where(UsagePoints.usage_point_id == usage_point_id)
+                                          .where(table.date <= self.yesterday)
+                                          .order_by(sort)
+                                          )
+        return result.all()
+
+    def get_detail_count(self, usage_point_id, measurement_direction="consumption"):
+        if measurement_direction == "consumption":
+            table = ConsumptionDetail
+            relation = UsagePoints.relation_consumption_detail
+        else:
+            table = ProductionDetail
+            relation = UsagePoints.relation_production_detail
+        return self.session.scalars(
+            select([func.count()]).select_from(table).join(relation)
+            .where(UsagePoints.usage_point_id == usage_point_id)
+        ).one_or_none()
+
     def get_detail_date(self, usage_point_id, date, measurement_direction="consumption"):
         unique_id = hashlib.md5(f"{usage_point_id}/{date}".encode('utf-8')).hexdigest()
         if measurement_direction == "consumption":
@@ -1050,6 +1141,31 @@ def insert_detail(self, usage_point_id, date, value, interval, measure_type, bla
                 )
             )
 
+    def reset_detail(self, usage_point_id, date=None, mesure_type="consumption"):
+        detail = self.get_detail_date(usage_point_id, date, mesure_type)
+        if detail is not None:
+            detail.value = 0
+            detail.interval = 0
+            detail.blacklist = 0
+            detail.fail_count = 0
+            self.session.flush()
+            return True
+        else:
+            return False
+
+    def reset_detail_range(self, usage_point_id, begin, end, mesure_type="consumption"):
+        detail = self.get_detail_range(usage_point_id, begin, end, mesure_type)
+        if detail is not None:
+            for row in detail:
+                row.value = 0
+                row.interval = 0
+                row.blacklist = 0
+                row.fail_count = 0
+            self.session.flush()
+            return True
+        else:
+            return False
+
     def delete_detail(self, usage_point_id, date=None, mesure_type="consumption"):
         if mesure_type == "consumption":
             table = ConsumptionDetail
@@ -1065,6 +1181,21 @@ def delete_detail(self, usage_point_id, date=None, mesure_type="consumption"):
             self.session.execute(delete(table).where(table.usage_point_id == usage_point_id))
         return True
 
+    def delete_detail_range(self, usage_point_id, begin, end, mesure_type="consumption"):
+        if mesure_type == "consumption":
+            table = ConsumptionDetail
+        else:
+            table = ProductionDetail
+        if date is not None:
+            unique_id = hashlib.md5(f"{usage_point_id}/{date}".encode('utf-8')).hexdigest()
+            self.session.execute(
+                delete(table)
+                .where(table.id == unique_id)
+            )
+        else:
+            self.session.execute(delete(table).where(table.usage_point_id == usage_point_id))
+        return True
+
     def get_ratio_hc_hp(self, usage_point_id, begin, end, mesure_type="consumption"):
         result = {
             "HC": 0,
@@ -1279,6 +1410,31 @@ def insert_daily_max_power(self, usage_point_id, date, event_date, value, blackl
                 )
             )
 
+    def get_daily_max_power_count(self, usage_point_id):
+        return self.session.scalars(
+            select([func.count()]).select_from(ConsumptionDailyMaxPower).join(UsagePoints.relation_consumption_daily_max_power)
+            .where(UsagePoints.usage_point_id == usage_point_id)
+        ).one_or_none()
+
+    def get_daily_max_power_datatable(self, usage_point_id, order_column="date", order_dir="asc", search=None):
+        sort = asc(order_column) if order_dir == "desc" else desc(order_column)
+        if search is not None and search != "":
+            result = self.session.scalars(select(ConsumptionDailyMaxPower)
+                                          .join(UsagePoints.relation_consumption_daily_max_power)
+                                          .where(UsagePoints.usage_point_id == usage_point_id)
+                                          .where((ConsumptionDailyMaxPower.date.like(f"%{search}%")) | (ConsumptionDailyMaxPower.value.like(f"%{search}%")))
+                                          .where(ConsumptionDailyMaxPower.date <= self.yesterday)
+                                          .order_by(sort)
+                                          )
+        else:
+            result = self.session.scalars(select(ConsumptionDailyMaxPower)
+                                          .join(UsagePoints.relation_consumption_daily_max_power)
+                                          .where(UsagePoints.usage_point_id == usage_point_id)
+                                          .where(ConsumptionDailyMaxPower.date <= self.yesterday)
+                                          .order_by(sort)
+                                          )
+        return result.all()
+
     def daily_max_power_fail_increment(self, usage_point_id, date):
         unique_id = hashlib.md5(f"{usage_point_id}/{date}".encode('utf-8')).hexdigest()
         daily = self.get_daily_max_power_date(usage_point_id, date)
@@ -1311,6 +1467,17 @@ def daily_max_power_fail_increment(self, usage_point_id, date):
             )
         return fail_count
 
+    def reset_daily_max_power(self, usage_point_id, date=None):
+        daily = self.get_daily_max_power_date(usage_point_id, date)
+        if daily is not None:
+            daily.event_date = None
+            daily.value = 0
+            daily.blacklist = 0
+            daily.fail_count = 0
+            return True
+        else:
+            return False
+
     def delete_daily_max_power(self, usage_point_id, date=None):
         if date is not None:
             unique_id = hashlib.md5(f"{usage_point_id}/{date}".encode('utf-8')).hexdigest()
@@ -1319,7 +1486,8 @@ def delete_daily_max_power(self, usage_point_id, date=None):
                 .where(ConsumptionDailyMaxPower.id == unique_id)
             )
         else:
-            self.session.execute(delete(ConsumptionDailyMaxPower).where(ConsumptionDailyMaxPower.usage_point_id == usage_point_id))
+            self.session.execute(
+                delete(ConsumptionDailyMaxPower).where(ConsumptionDailyMaxPower.usage_point_id == usage_point_id))
         return True
 
     def blacklist_daily_max_power(self, usage_point_id, date, action=True):
@@ -1347,5 +1515,6 @@ def get_daily_max_power_fail_count(self, usage_point_id, date):
         else:
             return 0
 
+
 os.system("cd /app; alembic upgrade head")
 Database().init_database()

app/models/jobs.py

@@ -106,8 +106,8 @@ def job_import_data(self, wait=True, target=None):
                         traceback.print_exc()
                         LOG.error([f"Erreur lors de la récupération de votre production détaillée", e])
                     try:
-                        # if target == "consumption_max_power" or target is None:
-                        self.get_consumption_max_power()
+                        if target == "consumption_max_power" or target is None:
+                            self.get_consumption_max_power()
                     except Exception as e:
                         traceback.print_exc()
                         LOG.error([f"Erreur lors de la récupération de votre puissance maximum journalière", e])
@@ -226,6 +226,7 @@ def job_import_data(self, wait=True, target=None):
                     LOG.log(
                         f" => Point de livraison Désactivé dans la configuration (Exemple: https://tinyurl.com/2kbd62s9).")
             LOG.finish()
+            self.usage_point_id = None
             DB.unlock()
             return {
                 "status": True,

app/models/log.py

@@ -32,7 +32,7 @@ def show(self, message):
         self.separator_warning()
         if type(message) is list:
             for msg in message:
-                pprint(f" {msg}")
+                pprint(f"{msg}")
         else:
             pprint(f" {message}")
         self.separator_warning()

app/models/query_cache.py

@@ -22,6 +22,7 @@ def reset(self):
 
         target = f"{self.url}/cache/{self.usage_point_id}"
         response = Query(endpoint=target, headers=self.headers).delete()
+        print(response.text)
         if response.status_code == 200:
             try:
                 status = json.loads(response.text)