From 2b4439ac777d013064a410b97b47cf8b6c6547fa Mon Sep 17 00:00:00 2001 From: Oli Parker Date: Tue, 18 Apr 2023 18:11:11 +0100 Subject: [PATCH 1/4] feat: add weather is forecase, updarte cron job, add logging for weather --- apps/weather/admin.py | 3 ++- .../management/commands/populate_weather.py | 3 +++ .../migrations/0003_weather_is_forecast.py | 18 +++++++++++++ .../migrations/0004_set_default_weather.py | 19 ++++++++++++++ .../0005_alter_weather_is_forecast.py | 18 +++++++++++++ .../0006_weather_created_weather_updated.py | 25 +++++++++++++++++++ apps/weather/models.py | 5 ++++ project/settings/local.py | 3 +++ 8 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 apps/weather/migrations/0003_weather_is_forecast.py create mode 100644 apps/weather/migrations/0004_set_default_weather.py create mode 100644 apps/weather/migrations/0005_alter_weather_is_forecast.py create mode 100644 apps/weather/migrations/0006_weather_created_weather_updated.py diff --git a/apps/weather/admin.py b/apps/weather/admin.py index 4026fb9..ab2e0c9 100644 --- a/apps/weather/admin.py +++ b/apps/weather/admin.py @@ -5,4 +5,5 @@ @admin.register(Weather) class WeatherAdmin(admin.ModelAdmin): - list_filter = ["date_time", "location"] + list_filter = ["date_time", "location", "is_forecast"] + read_only_fields = ["is_forecast", "created", "updated"] diff --git a/apps/weather/management/commands/populate_weather.py b/apps/weather/management/commands/populate_weather.py index a06e584..5071d51 100644 --- a/apps/weather/management/commands/populate_weather.py +++ b/apps/weather/management/commands/populate_weather.py @@ -2,6 +2,7 @@ from django.contrib.gis.geos import Point from django.core.management.base import BaseCommand from weather.models import Weather +import datetime as dt class Command(BaseCommand): @@ -19,6 +20,7 @@ def handle(self, *args, **options): # TODO get lat/lon from database. for lat, lon in [(52.52, 13.41), (52.40, -2.01)]: resp = requests.get(self.url.format(lat, lon)).json() + now = dt.datetime.now() for time, temp, apparent_temp, rain, weather_code, cloud_cover in zip( resp["hourly"]["time"], @@ -38,5 +40,6 @@ def handle(self, *args, **options): "rain": rain, "cloud_cover": cloud_cover, "weather_code": weather_code, + "is_forecast": time > now }, ) diff --git a/apps/weather/migrations/0003_weather_is_forecast.py b/apps/weather/migrations/0003_weather_is_forecast.py new file mode 100644 index 0000000..4fdc867 --- /dev/null +++ b/apps/weather/migrations/0003_weather_is_forecast.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2 on 2023-04-18 17:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('weather', '0002_rename_apparent_temperate_weather_apparent_temperature'), + ] + + operations = [ + migrations.AddField( + model_name='weather', + name='is_forecast', + field=models.BooleanField(default=False), + ), + ] diff --git a/apps/weather/migrations/0004_set_default_weather.py b/apps/weather/migrations/0004_set_default_weather.py new file mode 100644 index 0000000..7f5e706 --- /dev/null +++ b/apps/weather/migrations/0004_set_default_weather.py @@ -0,0 +1,19 @@ +from django.db import migrations, models +import datetime as dt + + +def update_weather(apps, schema_editor): + Weather = apps.get_model("weather", "Weather") + now = dt.datetime.now() + for weather in Weather.objects.all(): + weather.is_forecast = weather.date_time > now + weather.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ("weather", "0003_weather_is_forecast"), + ] + + operations = [migrations.RunPython(update_weather)] diff --git a/apps/weather/migrations/0005_alter_weather_is_forecast.py b/apps/weather/migrations/0005_alter_weather_is_forecast.py new file mode 100644 index 0000000..d18e8f9 --- /dev/null +++ b/apps/weather/migrations/0005_alter_weather_is_forecast.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2 on 2023-04-18 17:06 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('weather', '0004_set_default_weather'), + ] + + operations = [ + migrations.AlterField( + model_name='weather', + name='is_forecast', + field=models.BooleanField(), + ), + ] diff --git a/apps/weather/migrations/0006_weather_created_weather_updated.py b/apps/weather/migrations/0006_weather_created_weather_updated.py new file mode 100644 index 0000000..e64c53f --- /dev/null +++ b/apps/weather/migrations/0006_weather_created_weather_updated.py @@ -0,0 +1,25 @@ +# Generated by Django 4.2 on 2023-04-18 17:09 + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('weather', '0005_alter_weather_is_forecast'), + ] + + operations = [ + migrations.AddField( + model_name='weather', + name='created', + field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now), + preserve_default=False, + ), + migrations.AddField( + model_name='weather', + name='updated', + field=models.DateTimeField(auto_now=True), + ), + ] diff --git a/apps/weather/models.py b/apps/weather/models.py index d07e6ad..a77fc55 100644 --- a/apps/weather/models.py +++ b/apps/weather/models.py @@ -22,6 +22,11 @@ class Weather(models.Model): cloud_cover = models.FloatField() weather_code = models.FloatField() # TODO there are choices. + is_forecast = models.BooleanField() + + created = models.DateTimeField(auto_now_add=True) + updated = models.DateTimeField(auto_now=True) + class Meta: constraints = [ models.UniqueConstraint(name="unique_weather_for_location_time", fields=["date_time", "location"]) diff --git a/project/settings/local.py b/project/settings/local.py index cbf14bb..432e856 100644 --- a/project/settings/local.py +++ b/project/settings/local.py @@ -36,3 +36,6 @@ "bootstrap/", ), } + +GDAL_LIBRARY_PATH = '/opt/homebrew/opt/gdal/lib/libgdal.dylib' +GEOS_LIBRARY_PATH = '/opt/homebrew/opt/geos/lib/libgeos_c.dylib' From 7cb62868dbb712028774a6a5317c03515ddd7fa8 Mon Sep 17 00:00:00 2001 From: Oli Date: Tue, 18 Apr 2023 18:52:41 +0100 Subject: [PATCH 2/4] fix: migration fixes etc --- .../management/commands/populate_weather.py | 17 +++++++++++++---- .../migrations/0004_set_default_weather.py | 8 +++----- poetry.lock | 4 ++-- project/settings/local.py | 3 --- pyproject.toml | 1 + 5 files changed, 19 insertions(+), 14 deletions(-) diff --git a/apps/weather/management/commands/populate_weather.py b/apps/weather/management/commands/populate_weather.py index 5071d51..a0137ac 100644 --- a/apps/weather/management/commands/populate_weather.py +++ b/apps/weather/management/commands/populate_weather.py @@ -1,8 +1,14 @@ +import logging + import requests from django.contrib.gis.geos import Point from django.core.management.base import BaseCommand +from django.utils.timezone import datetime, now +from pytz import utc from weather.models import Weather -import datetime as dt + +# Get an instance of a logger +logger = logging.getLogger(__name__) class Command(BaseCommand): @@ -19,8 +25,9 @@ class Command(BaseCommand): def handle(self, *args, **options): # TODO get lat/lon from database. for lat, lon in [(52.52, 13.41), (52.40, -2.01)]: + logger.debug(f"Fetching data for {lat}, {lon}") resp = requests.get(self.url.format(lat, lon)).json() - now = dt.datetime.now() + time_now = now() for time, temp, apparent_temp, rain, weather_code, cloud_cover in zip( resp["hourly"]["time"], @@ -30,9 +37,11 @@ def handle(self, *args, **options): resp["hourly"]["weathercode"], resp["hourly"]["cloudcover"], ): + tz_aware_time = datetime.fromisoformat(time) + tz_aware_time = utc.localize(tz_aware_time) # this is bad! optimise later! Weather.objects.update_or_create( - date_time=time, + date_time=tz_aware_time, location=Point(lat, lon), defaults={ "temperature": temp, @@ -40,6 +49,6 @@ def handle(self, *args, **options): "rain": rain, "cloud_cover": cloud_cover, "weather_code": weather_code, - "is_forecast": time > now + "is_forecast": tz_aware_time > time_now, }, ) diff --git a/apps/weather/migrations/0004_set_default_weather.py b/apps/weather/migrations/0004_set_default_weather.py index 7f5e706..599797a 100644 --- a/apps/weather/migrations/0004_set_default_weather.py +++ b/apps/weather/migrations/0004_set_default_weather.py @@ -1,17 +1,15 @@ -from django.db import migrations, models -import datetime as dt +from django.db import migrations +from django.utils.timezone import now def update_weather(apps, schema_editor): Weather = apps.get_model("weather", "Weather") - now = dt.datetime.now() for weather in Weather.objects.all(): - weather.is_forecast = weather.date_time > now + weather.is_forecast = weather.date_time > now() weather.save() class Migration(migrations.Migration): - dependencies = [ ("weather", "0003_weather_is_forecast"), ] diff --git a/poetry.lock b/poetry.lock index 2fcda1c..ab4495d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.1 and should not be changed by hand. [[package]] name = "asgiref" @@ -1598,4 +1598,4 @@ brotli = ["Brotli"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "1a38d538a4560345fc8893477d11e61686c06af8162836c24824bb33d08085e2" +content-hash = "0963ecc8769c7a6c1557f2893ecf3cf37843ae3f314533b16ef7c25e146db177" diff --git a/project/settings/local.py b/project/settings/local.py index 432e856..cbf14bb 100644 --- a/project/settings/local.py +++ b/project/settings/local.py @@ -36,6 +36,3 @@ "bootstrap/", ), } - -GDAL_LIBRARY_PATH = '/opt/homebrew/opt/gdal/lib/libgdal.dylib' -GEOS_LIBRARY_PATH = '/opt/homebrew/opt/geos/lib/libgeos_c.dylib' diff --git a/pyproject.toml b/pyproject.toml index 9fd358d..8cd8ea6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ parameterized = "^0.9.0" psycopg = "^3.1.8" whitenoise = "^6.4.0" crispy-bootstrap5 = "^0.7" +pytz = "^2023.3" [tool.poetry.group.local.dependencies] From 2350de563c18a60f5b466841f092186d59332b5f Mon Sep 17 00:00:00 2001 From: Oli Date: Tue, 18 Apr 2023 18:56:05 +0100 Subject: [PATCH 3/4] fix: migration fixes etc --- .pre-commit-config.yaml | 6 ------ apps/weather/migrations/0003_weather_is_forecast.py | 7 +++---- .../migrations/0005_alter_weather_is_forecast.py | 7 +++---- .../0006_weather_created_weather_updated.py | 13 ++++++------- 4 files changed, 12 insertions(+), 21 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0db3d59..092a75d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,12 +12,6 @@ repos: - id: black - - repo: https://github.com/pycqa/isort - rev: 5.12.0 - hooks: - - id: isort - name: isort (python) - - repo: https://github.com/compilerla/conventional-pre-commit rev: 'v2.1.1' hooks: diff --git a/apps/weather/migrations/0003_weather_is_forecast.py b/apps/weather/migrations/0003_weather_is_forecast.py index 4fdc867..3b30dbf 100644 --- a/apps/weather/migrations/0003_weather_is_forecast.py +++ b/apps/weather/migrations/0003_weather_is_forecast.py @@ -4,15 +4,14 @@ class Migration(migrations.Migration): - dependencies = [ - ('weather', '0002_rename_apparent_temperate_weather_apparent_temperature'), + ("weather", "0002_rename_apparent_temperate_weather_apparent_temperature"), ] operations = [ migrations.AddField( - model_name='weather', - name='is_forecast', + model_name="weather", + name="is_forecast", field=models.BooleanField(default=False), ), ] diff --git a/apps/weather/migrations/0005_alter_weather_is_forecast.py b/apps/weather/migrations/0005_alter_weather_is_forecast.py index d18e8f9..1730447 100644 --- a/apps/weather/migrations/0005_alter_weather_is_forecast.py +++ b/apps/weather/migrations/0005_alter_weather_is_forecast.py @@ -4,15 +4,14 @@ class Migration(migrations.Migration): - dependencies = [ - ('weather', '0004_set_default_weather'), + ("weather", "0004_set_default_weather"), ] operations = [ migrations.AlterField( - model_name='weather', - name='is_forecast', + model_name="weather", + name="is_forecast", field=models.BooleanField(), ), ] diff --git a/apps/weather/migrations/0006_weather_created_weather_updated.py b/apps/weather/migrations/0006_weather_created_weather_updated.py index e64c53f..9a4826b 100644 --- a/apps/weather/migrations/0006_weather_created_weather_updated.py +++ b/apps/weather/migrations/0006_weather_created_weather_updated.py @@ -1,25 +1,24 @@ # Generated by Django 4.2 on 2023-04-18 17:09 -from django.db import migrations, models import django.utils.timezone +from django.db import migrations, models class Migration(migrations.Migration): - dependencies = [ - ('weather', '0005_alter_weather_is_forecast'), + ("weather", "0005_alter_weather_is_forecast"), ] operations = [ migrations.AddField( - model_name='weather', - name='created', + model_name="weather", + name="created", field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now), preserve_default=False, ), migrations.AddField( - model_name='weather', - name='updated', + model_name="weather", + name="updated", field=models.DateTimeField(auto_now=True), ), ] From c2c97b920e43b68d605d7fd8e9700a7c333bd114 Mon Sep 17 00:00:00 2001 From: Oli Date: Tue, 18 Apr 2023 18:56:17 +0100 Subject: [PATCH 4/4] fix: migration fixes etc --- .github/workflows/ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98bfe18..b23e82c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,9 +34,6 @@ jobs: - name: Black Lint run: poetry run black --line-length 120 --exclude '/migrations/' --check apps project - - name: Isort Lint - run: poetry run isort --check-only --diff apps project -l 120 - - name: Ruff Lint run: poetry run ruff check .