From a9c7270167a1c0af045d2b76f13b8017ce205e7a Mon Sep 17 00:00:00 2001 From: Dennis Siemensma Date: Sun, 29 Jan 2023 17:31:29 +0100 Subject: [PATCH] Disabled an automatic data migration in v5.10 until further notice #1770 --- docs/reference/changelog.rst | 6 + ...s_fix_total_gas_consumption_retroactive.py | 196 +++++++++--------- dsmrreader/__init__.py | 2 +- dsmrreader/provisioning/downgrade/v5.10.2.sh | 17 ++ 4 files changed, 124 insertions(+), 97 deletions(-) create mode 100644 dsmrreader/provisioning/downgrade/v5.10.2.sh diff --git a/docs/reference/changelog.rst b/docs/reference/changelog.rst index 7be6aa934..e21cebc23 100644 --- a/docs/reference/changelog.rst +++ b/docs/reference/changelog.rst @@ -25,6 +25,12 @@ Current release series :depth: 1 +v5.10.1 - January 2023 +---------------------- + +- ``Fixed`` [`#1770 `_] Disabled an automatic data migration in ``v5.10`` until further notice. + + v5.10.1 - January 2023 ---------------------- diff --git a/dsmr_stats/migrations/0020_day_statistics_fix_total_gas_consumption_retroactive.py b/dsmr_stats/migrations/0020_day_statistics_fix_total_gas_consumption_retroactive.py index 57373e298..74f8c5a1a 100644 --- a/dsmr_stats/migrations/0020_day_statistics_fix_total_gas_consumption_retroactive.py +++ b/dsmr_stats/migrations/0020_day_statistics_fix_total_gas_consumption_retroactive.py @@ -10,103 +10,107 @@ def regenerate_data(apps, schema_editor): """Try to fix any invalid gas data due to #1770.""" - DayStatistics = apps.get_model("dsmr_stats", "DayStatistics") - HourStatistics = apps.get_model("dsmr_stats", "HourStatistics") - Notification = apps.get_model("dsmr_frontend", "Notification") - import dsmr_backend.services.backend - - if dsmr_backend.services.backend.is_recent_installation(): - # Skip for new installations. - return - - # No DB index, but should be fine. - days = DayStatistics.objects.all() - day_count = days.count() - x = 0 - total_difference_occurrences = 0 - total_gas_difference = Decimal(0) - difference_occurrences_per_year = collections.defaultdict(int) - total_difference_per_year = collections.defaultdict(Decimal) - per_day_messages = [] - - for current in days: - x += 1 - print( - "Data migration: Retroactively checking/recalculating gas consumption for day statistics: {} ({}/{})".format( - current.day, x, day_count - ) - ) - - # Straight copy from create_daily_statistics() - hours_in_day = dsmr_backend.services.backend.hours_in_day(day=current.day) - start_of_day = timezone.make_aware( - timezone.datetime( - year=current.day.year, - month=current.day.month, - day=current.day.day, - hour=0, - minute=0, - ) - ) - end_of_day = start_of_day + timezone.timedelta(hours=hours_in_day) - hours_gas_sum = HourStatistics.objects.filter( - hour_start__gte=start_of_day, - hour_start__lt=end_of_day, - ).aggregate(gas_sum=Sum("gas"),)["gas_sum"] - - if not hours_gas_sum or not current.gas: - continue - - current_day_gas_diff = hours_gas_sum - current.gas - - # Only override when consumption is MORE - if current_day_gas_diff > 0: - output_message = f"{current.day} | Before: {current.gas} m³ - Recalculated: {hours_gas_sum} m³ | Difference: {current_day_gas_diff} m³" - print(output_message) - per_day_messages.append(output_message) - - total_gas_difference += current_day_gas_diff - total_difference_occurrences += 1 - difference_occurrences_per_year[current.day.year] += 1 - total_difference_per_year[current.day.year] += current_day_gas_diff - - # Update day, prices will be recalculated below. - current.gas = hours_gas_sum - current.save() - - if total_difference_occurrences > 0: - # Reflect changes for prices. - import dsmr_stats.services - - dsmr_stats.services.recalculate_prices() - - total_text = "DSMR-reader found and fixed historic gas differences in favor of issue #1770. Note that the number format is English and not localized.\n" - - total_text += f"\nDifferences by year ({total_gas_difference} m³ in total):\n" - for year, total in total_difference_per_year.items(): - total_text += f"{year} | {total} m³\n" - - total_text += ( - f"\nOccurrences by year ({total_difference_occurrences} x in total):\n" - ) - for year, count in difference_occurrences_per_year.items(): - total_text += f"{year} | {count} x\n" - - print(total_text) - - # Total - Notification.objects.create(message=total_text) - # Per-day - per_day_messages.insert( - 0, - "Any per-day differences in favor of issue #1770 are displayed below. Note that the number format is English and not localized.\n", - ) - Notification.objects.create(message="\n".join(per_day_messages)) - - print( - "The information above should also be logged as notifications, accessible on the Dashboard page of DSMR-reader." - ) + # @TODO: Temporarily no-op until further notice. + return + + # DayStatistics = apps.get_model("dsmr_stats", "DayStatistics") + # HourStatistics = apps.get_model("dsmr_stats", "HourStatistics") + # Notification = apps.get_model("dsmr_frontend", "Notification") + # + # import dsmr_backend.services.backend + # + # if dsmr_backend.services.backend.is_recent_installation(): + # # Skip for new installations. + # return + # + # # No DB index, but should be fine. + # days = DayStatistics.objects.all() + # day_count = days.count() + # x = 0 + # total_difference_occurrences = 0 + # total_gas_difference = Decimal(0) + # difference_occurrences_per_year = collections.defaultdict(int) + # total_difference_per_year = collections.defaultdict(Decimal) + # per_day_messages = [] + # + # for current in days: + # x += 1 + # print( + # "Data migration: Retroactively checking/recalculating gas consumption for day statistics: {} ({}/{})".format( + # current.day, x, day_count + # ) + # ) + # + # # Straight copy from create_daily_statistics() + # hours_in_day = dsmr_backend.services.backend.hours_in_day(day=current.day) + # start_of_day = timezone.make_aware( + # timezone.datetime( + # year=current.day.year, + # month=current.day.month, + # day=current.day.day, + # hour=0, + # minute=0, + # ) + # ) + # end_of_day = start_of_day + timezone.timedelta(hours=hours_in_day) + # hours_gas_sum = HourStatistics.objects.filter( + # hour_start__gte=start_of_day, + # hour_start__lt=end_of_day, + # ).aggregate(gas_sum=Sum("gas"),)["gas_sum"] + # + # if not hours_gas_sum or not current.gas: + # continue + # + # current_day_gas_diff = hours_gas_sum - current.gas + # + # # Only override when consumption is MORE + # if current_day_gas_diff > 0: + # output_message = f"{current.day} | Before: {current.gas} m³ - Recalculated: {hours_gas_sum} m³ | Difference: {current_day_gas_diff} m³" + # print(output_message) + # per_day_messages.append(output_message) + # + # total_gas_difference += current_day_gas_diff + # total_difference_occurrences += 1 + # difference_occurrences_per_year[current.day.year] += 1 + # total_difference_per_year[current.day.year] += current_day_gas_diff + # + # # Update day, prices will be recalculated below. + # current.gas = hours_gas_sum + # current.save() + # + # if total_difference_occurrences > 0: + # # Reflect changes for prices. + # import dsmr_stats.services + # + # dsmr_stats.services.recalculate_prices() + # + # total_text = "DSMR-reader found and fixed historic gas differences in favor of issue #1770. Note that the number format is English and not localized.\n" + # + # total_text += f"\nDifferences by year ({total_gas_difference} m³ in total):\n" + # for year, total in total_difference_per_year.items(): + # total_text += f"{year} | {total} m³\n" + # + # total_text += ( + # f"\nOccurrences by year ({total_difference_occurrences} x in total):\n" + # ) + # for year, count in difference_occurrences_per_year.items(): + # total_text += f"{year} | {count} x\n" + # + # print(total_text) + # + # # Total + # Notification.objects.create(message=total_text) + # # Per-day + # per_day_messages.insert( + # 0, + # "Any per-day differences in favor of issue #1770 are displayed below. Note that the number format is English and not localized.\n", + # ) + # Notification.objects.create(message="\n".join(per_day_messages)) + # + # print( + # "The information above should also be logged as notifications, accessible on the Dashboard page of DSMR-reader." + # ) def noop(apps, schema_editor): diff --git a/dsmrreader/__init__.py b/dsmrreader/__init__.py index 34b07d821..cf338e86f 100644 --- a/dsmrreader/__init__.py +++ b/dsmrreader/__init__.py @@ -1,5 +1,5 @@ from django.utils.version import get_version -VERSION = (5, 10, 1, "final", 0) +VERSION = (5, 10, 2, "final", 0) __version__ = get_version(VERSION) diff --git a/dsmrreader/provisioning/downgrade/v5.10.2.sh b/dsmrreader/provisioning/downgrade/v5.10.2.sh new file mode 100644 index 000000000..b6874f1df --- /dev/null +++ b/dsmrreader/provisioning/downgrade/v5.10.2.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# Dump for DSMR-reader v5.10.2 +./manage.py migrate dsmr_api 0003_create_api_user +./manage.py migrate dsmr_backend 0015_backend_restart_required +./manage.py migrate dsmr_backup 0017_backup_interval_in_days +./manage.py migrate dsmr_consumption 0021_schedule_quarter_hour_peaks_calculation +./manage.py migrate dsmr_datalogger 0032_dsmr_extra_device_channel +./manage.py migrate dsmr_dropbox 0001_schedule_dropbox +./manage.py migrate dsmr_frontend 0049_alter_notification_options +./manage.py migrate dsmr_influxdb 0006_influxdb_settings_field_size +./manage.py migrate dsmr_mindergas 0005_schedule_mindergas_export +./manage.py migrate dsmr_mqtt 0021_quarter_hour_peak_mqtt +./manage.py migrate dsmr_notification 0008_dummy_notification_provider +./manage.py migrate dsmr_pvoutput 0004_pvoutput_setting_refactoring +./manage.py migrate dsmr_stats 0020_day_statistics_fix_total_gas_consumption_retroactive +./manage.py migrate dsmr_weather 0006_schedule_weather_update