v1.15.0

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,11 @@
+*Let op: Het DSMR-reader project is gefocused op het uitlezen van de slimme meter en om die data met jou op zoveel mogelijk manieren te delen. Het inlezen van andere (externe) data in DSMR-reader valt buiten de scope van dit project.*
+
+### Jouw omgeving
+*(Indien van toepassing)*
+* DSMR-reader versie *(rechtsbovenin applicatie zichtbaar)*: `v1.X.Y`
+* Hardware *(RaspberryPi 2, 3 of iets anders)*: `RaspberryPi X...`
+* DSMR-protocol versie van slimme meter *(v4 of v5)*: `v...`
+
+### Bug, wens of iets anders
+* Bijvoorbeeld stappen hoe een probleem te reproduceren is...
+* Of simpelweg een beschijving van een feature die je graag wenst te zien...

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,3 @@
+### Check
+
+- Make sure you are making a pull request against the `development` branch. Also you should start your branch off the `development` branch.

docs/changelog.rst

@@ -18,13 +18,25 @@ Please make sure you have a fresh **database backup** before upgrading! Upgradin
 
 
 
+v1.15.0 - 2018-03-21
+^^^^^^^^^^^^^^^^^^^^
+
+**Tickets resolved in this release:**
+
+- [`#449 <https://github.com/dennissiemensma/dsmr-reader/issues/449>`_] Meterstatistieken via MQTT beschikbaar
+- [`#208 <https://github.com/dennissiemensma/dsmr-reader/issues/208>`_] Notificatie bij uitblijven gegevens uit slimme meter
+- [`#342 <https://github.com/dennissiemensma/dsmr-reader/issues/342>`_] Backup to dropbox never finish (free plan no more space)
+
+
+
 v1.14.0 - 2018-03-11
 ^^^^^^^^^^^^^^^^^^^^
 
 **Tickets resolved in this release:**
 
 - [`#441 <https://github.com/dennissiemensma/dsmr-reader/issues/441>`_] PVOutput exports schedulen naar ingestelde upload interval - by pyrocumulus
 - [`#436 <https://github.com/dennissiemensma/dsmr-reader/issues/436>`_] Update docs: authentication method for public webinterface
+- [`#449 <https://github.com/dennissiemensma/dsmr-reader/issues/449>`_] Meterstatistieken via MQTT beschikbaar
 - [`#445 <https://github.com/dennissiemensma/dsmr-reader/issues/445>`_] Upload/export to PVoutput doesn't work
 - [`#432 <https://github.com/dennissiemensma/dsmr-reader/issues/432>`_] [API] Gas cost missing at start of day 
 - [`#367 <https://github.com/dennissiemensma/dsmr-reader/issues/367>`_] Dagverbruik en teruglevering via MQTT

dsmr_backup/services/dropbox.py

@@ -1,10 +1,12 @@
 import os
 
+from django.utils.translation import ugettext_lazy as gettext
 from django.utils import timezone
 from django.conf import settings
 import dropbox
 
 from dsmr_backup.models.settings import DropboxSettings
+from dsmr_frontend.models.message import Notification
 import dsmr_backup.services.backup
 
 
@@ -32,22 +34,44 @@ def sync():
     for (_, _, filenames) in os.walk(backup_directory):
         for current_file in filenames:
             current_file_path = os.path.join(backup_directory, current_file)
-            file_stats = os.stat(current_file_path)
+            check_synced_file(file_path=current_file_path, dropbox_settings=dropbox_settings)
 
-            # Ignore empty files.
-            if file_stats.st_size == 0:
-                continue
+    DropboxSettings.objects.update(latest_sync=timezone.now())
 
-            last_modified = timezone.datetime.fromtimestamp(file_stats.st_mtime)
-            last_modified = timezone.make_aware(last_modified)
 
-            # Ignore when file was not altered since last sync.
-            if dropbox_settings.latest_sync and last_modified < dropbox_settings.latest_sync:
-                continue
+def check_synced_file(file_path, dropbox_settings):
 
-            upload_chunked(file_path=current_file_path)
+    file_stats = os.stat(file_path)
 
-    DropboxSettings.objects.update(latest_sync=timezone.now())
+    # Ignore empty files.
+    if file_stats.st_size == 0:
+        return
+
+    last_modified = timezone.datetime.fromtimestamp(file_stats.st_mtime)
+    last_modified = timezone.make_aware(last_modified)
+
+    # Ignore when file was not altered since last sync.
+    if dropbox_settings.latest_sync and last_modified < dropbox_settings.latest_sync:
+        return
+
+    try:
+        upload_chunked(file_path=file_path)
+    except dropbox.exceptions.ApiError as exception:
+        print(' - Dropbox error: {}'.format(exception.error))
+
+        if 'insufficient_space' in str(exception.error):
+            # Notify user.
+            Notification.objects.create(message=gettext(
+                "[{}] Unable to upload files to Dropbox due to insufficient space. "
+                "Ignoring new files for the next {} hours...".format(
+                    timezone.now(), settings.DSMRREADER_DROPBOX_ERROR_INTERVAL
+                )
+            ))
+            DropboxSettings.objects.update(
+                latest_sync=timezone.now() + timezone.timedelta(hours=settings.DSMRREADER_DROPBOX_ERROR_INTERVAL)
+            )
+
+        raise
 
 
 def upload_chunked(file_path):

dsmr_backup/tests/test_services.py

@@ -6,9 +6,11 @@
 from django.test import TestCase
 from django.utils import timezone
 from django.conf import settings
+import dropbox
 
 from dsmr_backend.tests.mixins import InterceptStdoutMixin
 from dsmr_backup.models.settings import BackupSettings, DropboxSettings
+from dsmr_frontend.models.message import Notification
 import dsmr_backup.services.backup
 import dsmr_backup.services.dropbox
 
@@ -246,6 +248,47 @@ def test_sync_last_modified(self, now_mock, upload_chunked_mock, get_backup_dire
             dsmr_backup.services.dropbox.sync()
             self.assertFalse(upload_chunked_mock.called)
 
+    @mock.patch('dsmr_backup.services.dropbox.upload_chunked')
+    @mock.patch('django.utils.timezone.now')
+    def test_sync_insufficient_space(self, now_mock, upload_chunked_mock):
+        now_mock.return_value = timezone.make_aware(timezone.datetime(2000, 1, 1))
+
+        # Crash the party, no more space available!
+        upload_chunked_mock.side_effect = dropbox.exceptions.ApiError(
+            12345,
+            "UploadError('path', UploadWriteFailed(reason=WriteError('insufficient_space', None), ...)",
+            'x',
+            'y'
+        )
+
+        Notification.objects.all().delete()
+        self.assertEqual(Notification.objects.count(), 0)
+
+        with self.assertRaises(dropbox.exceptions.ApiError):
+            dsmr_backup.services.dropbox.sync()
+
+        # Warning message should be created and next sync should be skipped ahead.
+        self.assertEqual(Notification.objects.count(), 1)
+        self.assertGreater(
+            DropboxSettings.get_solo().latest_sync, timezone.now() + timezone.timedelta(
+                hours=settings.DSMRREADER_DROPBOX_ERROR_INTERVAL - 1
+                )
+        )
+
+        # Test alternate path.
+        DropboxSettings.objects.update(latest_sync=timezone.now() - timezone.timedelta(hours=24))
+        upload_chunked_mock.side_effect = dropbox.exceptions.ApiError(
+            12345,
+            "UploadError('path', UploadWriteFailed(reason=WriteError('other_error', None), ...)",
+            'x',
+            'y'
+        )
+
+        with self.assertRaises(dropbox.exceptions.ApiError):
+            dsmr_backup.services.dropbox.sync()
+
+        self.assertEqual(Notification.objects.count(), 1)
+
     @mock.patch('dropbox.Dropbox.files_upload')
     @mock.patch('dropbox.Dropbox.files_upload_session_start')
     @mock.patch('dropbox.Dropbox.files_upload_session_append')

dsmr_datalogger/management/commands/dsmr_datalogger.py

@@ -11,7 +11,6 @@
 class Command(InfiniteManagementCommandMixin, BaseCommand):
     help = _('Performs an DSMR P1 telegram reading on the COM port.')
     name = __name__  # Required for PID file.
-    sleep_time = 0.25
     sleep_time = settings.DSMRREADER_DATALOGGER_SLEEP
 
     def run(self, **options):

dsmr_frontend/migrations/0006_notifications_initial.py

@@ -26,13 +26,6 @@ def insert_notifications(apps, schema_editor):
         redirect_to='admin:index'
     )
 
-    Notification.objects.create(
-        message=dsmr_frontend.services.get_translated_string(
-            text=_('You may check the status of your readings and data in the Status page.')
-        ),
-        redirect_to='frontend:status'
-    )
-
 
 class Migration(migrations.Migration):
     dependencies = [

dsmr_frontend/migrations/0008_merge_electricity_tariffs_notification.py

@@ -2,23 +2,11 @@
 from __future__ import unicode_literals
 
 from django.db import migrations
-from django.utils.translation import ugettext_lazy as _
 
 
 def insert_notifications(apps, schema_editor):
-    import dsmr_frontend.services
-    Notification = apps.get_model('dsmr_frontend', 'Notification')
-
-    Notification.objects.create(
-        message=dsmr_frontend.services.get_translated_string(
-            text=_(
-                "It's now possible to have the electricity tariffs merged to a single one. This might come handy when "
-                "you pay your energy supplier fo a single tariff instead of high/low. You can enable this feature in "
-                " frontend configuration: 'Merge electricity tariffs'"
-            )
-        ),
-        redirect_to='admin:index'
-    )
+    # Removed for new installations.
+    pass
 
 
 class Migration(migrations.Migration):

dsmr_mindergas/migrations/0002_mindergas_notification.py

@@ -2,22 +2,11 @@
 from __future__ import unicode_literals
 
 from django.db import migrations
-from django.utils.translation import ugettext_lazy as _
 
 
 def insert_notifications(apps, schema_editor):
-    import dsmr_frontend.services
-    Notification = apps.get_model('dsmr_frontend', 'Notification')
-
-    Notification.objects.create(
-        message=dsmr_frontend.services.get_translated_string(
-            text=_(
-                "It's now possible to automatically export your gas meter positions to your own Mindergas.nl account! "
-                "See the FAQ in the documentation for a guide on how to create and/or link your account."
-            )
-        ),
-        redirect_to='frontend:docs-redirect'
-    )
+    # Removed for new installations.
+    pass
 
 
 class Migration(migrations.Migration):

dsmr_mindergas/services.py

@@ -11,10 +11,10 @@
 
 def should_export():
     """ Checks whether we should export data yet. Once every day. """
-    settings = MinderGasSettings.get_solo()
+    mindergas_settings = MinderGasSettings.get_solo()
 
     # Only when enabled and token set.
-    if not settings.export or not settings.auth_token:
+    if not mindergas_settings.export or not mindergas_settings.auth_token:
         return False
 
     # Nonsense when having no data.
@@ -23,7 +23,7 @@ def should_export():
     if not capabilities['gas']:
         return False
 
-    return dsmr_backend.services.is_timestamp_passed(timestamp=settings.next_export)
+    return dsmr_backend.services.is_timestamp_passed(timestamp=mindergas_settings.next_export)
 
 
 def export():
@@ -55,13 +55,13 @@ def export():
         last_gas_reading = None
         print(' - MinderGas | No gas readings found for uploading')
     else:
-        settings = MinderGasSettings.get_solo()
+        mindergas_settings = MinderGasSettings.get_solo()
         print(' - MinderGas | Uploading gas meter position: {}'.format(last_gas_reading.delivered))
 
         # Register telegram by simply sending it to the application with a POST request.
         response = requests.post(
             MinderGasSettings.API_URL,
-            headers={'Content-Type': 'application/json', 'AUTH-TOKEN': settings.auth_token},
+            headers={'Content-Type': 'application/json', 'AUTH-TOKEN': mindergas_settings.auth_token},
             data=json.dumps({
                 'date': last_gas_reading.read_at.date().isoformat(),
                 'reading': str(last_gas_reading.delivered)
@@ -74,6 +74,6 @@ def export():
             print(' [!] MinderGas upload failed (HTTP {}): {}'.format(response.status_code, response.text))
 
     print(' - MinderGas | Delaying the next upload until: {}'.format(next_export))
-    settings = MinderGasSettings.get_solo()
-    settings.next_export = next_export
-    settings.save()
+    mindergas_settings = MinderGasSettings.get_solo()
+    mindergas_settings.next_export = next_export
+    mindergas_settings.save()

dsmr_mqtt/admin.py

@@ -2,7 +2,7 @@
 from django.utils.translation import ugettext_lazy as _
 from solo.admin import SingletonModelAdmin
 
-from dsmr_mqtt.models.settings import broker, day_totals, telegram
+from dsmr_mqtt.models.settings import broker, day_totals, telegram, meter_statistics
 
 
 @admin.register(broker.MQTTBrokerSettings)
@@ -189,3 +189,37 @@ class SplitTopicDayTotalsMQTTSettingsAdmin(SingletonModelAdmin):
             }
         ),
     )
+
+
+@admin.register(meter_statistics.SplitTopicMeterStatisticsMQTTSettings)
+class SplitTopicMeterStatisticsMQTTSettingsAdmin(SingletonModelAdmin):
+    fieldsets = (
+        (
+            None, {
+                'fields': ['enabled', 'formatting'],
+                'description': _(
+                    'Triggered by any method of reading insertion (datalogger or API). '
+                    'Allows you to send meter statistics to the MQTT broker, splitted per field. You can '
+                    'designate each field name to a different topic. Removing lines will prevent those fields from '
+                    'being broadcast as well. '
+                    '''Default value:
+<pre>
+[mapping]
+# DATA = TOPIC PATH
+dsmr_version = dsmr/meter-stats/dsmr_version
+electricity_tariff = dsmr/meter-stats/electricity_tariff
+power_failure_count = dsmr/meter-stats/power_failure_count
+long_power_failure_count = dsmr/meter-stats/long_power_failure_count
+voltage_sag_count_l1 = dsmr/meter-stats/voltage_sag_count_l1
+voltage_sag_count_l2 = dsmr/meter-stats/voltage_sag_count_l2
+voltage_sag_count_l3 = dsmr/meter-stats/voltage_sag_count_l3
+voltage_swell_count_l1 = dsmr/meter-stats/voltage_swell_count_l1
+voltage_swell_count_l2 = dsmr/meter-stats/voltage_swell_count_l2
+voltage_swell_count_l3 = dsmr/meter-stats/voltage_swell_count_l3
+rejected_telegrams = dsmr/meter-stats/rejected_telegrams
+</pre>
+'''
+                )
+            }
+        ),
+    )

dsmr_mqtt/apps.py

@@ -52,3 +52,8 @@ def _on_dsmrreading_created_signal(self, sender, instance, created, **kwargs):
             dsmr_mqtt.services.publish_day_totals()
         except Exception as error:
             logger.error('publish_day_totals() failed: {}'.format(error))
+
+        try:
+            dsmr_mqtt.services.publish_split_topic_meter_statistics()
+        except Exception as error:
+            logger.error('publish_split_topic_meter_statistics() failed: {}'.format(error))

dsmr_mqtt/migrations/0002_daytotalsmqttsettings.py

@@ -28,10 +28,10 @@ class Migration(migrations.Migration):
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                 ('enabled', models.BooleanField(default=False, help_text='Whether day totals are sent to the broker, having each field sent to a different topic.', verbose_name='Enabled')),
-                ('formatting', models.TextField(default='\n[mapping]\n# DATA = JSON FIELD\nelectricity1 = dsmr/day-totals/electricity1\nelectricity2 = dsmr/day-totals/electricity2\nelectricity1_returned = dsmr/day-totals/electricity1_returned\nelectricity2_returned = dsmr/day-totals/electricity2_returned\nelectricity_merged = dsmr/day-totals/electricity_merged\nelectricity_returned_merged = dsmr/day-totals/electricity_returned_merged\nelectricity1_cost = dsmr/day-totals/electricity1_cost\nelectricity2_cost = dsmr/day-totals/electricity2_cost\nelectricity_cost_merged = dsmr/day-totals/electricity_cost_merged\n\n# Gas (if any)\ngas = dsmr/day-totals/gas\ngas_cost = dsmr/day-totals/gas_cost\ntotal_cost = dsmr/day-totals/total_cost\n\n# Your energy supplier prices (if set)\nenergy_supplier_price_electricity_delivered_1 = dsmr/day-totals/energy_supplier_price_electricity_delivered_1\nenergy_supplier_price_electricity_delivered_2 = dsmr/day-totals/energy_supplier_price_electricity_delivered_2\nenergy_supplier_price_electricity_returned_1 = dsmr/day-totals/energy_supplier_price_electricity_returned_1\nenergy_supplier_price_electricity_returned_2 = dsmr/day-totals/energy_supplier_price_electricity_returned_2\nenergy_supplier_price_gas = dsmr/day-totals/energy_supplier_price_gas\n', help_text='Maps the field names to separate topics sent to the broker.', verbose_name='Formatting')),
+                ('formatting', models.TextField(default='\n[mapping]\n# DATA = TOPIC PATH\nelectricity1 = dsmr/day-totals/electricity1\nelectricity2 = dsmr/day-totals/electricity2\nelectricity1_returned = dsmr/day-totals/electricity1_returned\nelectricity2_returned = dsmr/day-totals/electricity2_returned\nelectricity_merged = dsmr/day-totals/electricity_merged\nelectricity_returned_merged = dsmr/day-totals/electricity_returned_merged\nelectricity1_cost = dsmr/day-totals/electricity1_cost\nelectricity2_cost = dsmr/day-totals/electricity2_cost\nelectricity_cost_merged = dsmr/day-totals/electricity_cost_merged\n\n# Gas (if any)\ngas = dsmr/day-totals/gas\ngas_cost = dsmr/day-totals/gas_cost\ntotal_cost = dsmr/day-totals/total_cost\n\n# Your energy supplier prices (if set)\nenergy_supplier_price_electricity_delivered_1 = dsmr/day-totals/energy_supplier_price_electricity_delivered_1\nenergy_supplier_price_electricity_delivered_2 = dsmr/day-totals/energy_supplier_price_electricity_delivered_2\nenergy_supplier_price_electricity_returned_1 = dsmr/day-totals/energy_supplier_price_electricity_returned_1\nenergy_supplier_price_electricity_returned_2 = dsmr/day-totals/energy_supplier_price_electricity_returned_2\nenergy_supplier_price_gas = dsmr/day-totals/energy_supplier_price_gas\n', help_text='Maps the field names to separate topics sent to the broker.', verbose_name='Formatting')),
             ],
             options={
-                'verbose_name': 'MQTT:  Day totals (per split topic) configuration',
+                'verbose_name': 'MQTT: Day totals (per split topic) configuration',
                 'default_permissions': (),
             },
         ),

dsmr_mqtt/migrations/0003_splittopicmeterstatisticsmqttsettings.py

@@ -0,0 +1,25 @@
+# Generated by Django 2.0.2 on 2018-03-13 19:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dsmr_mqtt', '0002_daytotalsmqttsettings'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='SplitTopicMeterStatisticsMQTTSettings',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('enabled', models.BooleanField(default=False, help_text='Whether meter statistics are sent to the broker, having each field sent to a different topic.', verbose_name='Enabled')),
+                ('formatting', models.TextField(default='\n[mapping]\n# DATA = TOPIC PATH\ndsmr_version = dsmr/meter-stats/dsmr_version\nelectricity_tariff = dsmr/meter-stats/electricity_tariff\npower_failure_count = dsmr/meter-stats/power_failure_count\nlong_power_failure_count = dsmr/meter-stats/long_power_failure_count\nvoltage_sag_count_l1 = dsmr/meter-stats/voltage_sag_count_l1\nvoltage_sag_count_l2 = dsmr/meter-stats/voltage_sag_count_l2\nvoltage_sag_count_l3 = dsmr/meter-stats/voltage_sag_count_l3\nvoltage_swell_count_l1 = dsmr/meter-stats/voltage_swell_count_l1\nvoltage_swell_count_l2 = dsmr/meter-stats/voltage_swell_count_l2\nvoltage_swell_count_l3 = dsmr/meter-stats/voltage_swell_count_l3\nrejected_telegrams = dsmr/meter-stats/rejected_telegrams\n', help_text='Maps the field names to separate topics sent to the broker.', verbose_name='Formatting')),
+            ],
+            options={
+                'verbose_name': 'MQTT: Meter Statistics (per split topic) configuration',
+                'default_permissions': (),
+            },
+        ),
+    ]