perf(weather): bulk upsert weather for quicker query (#245)

*

apps/alerts/admin.py

@@ -1,7 +1,5 @@
-from django.contrib import admin
-
 from alerts.models import Alert, AlertLog
-
+from django.contrib import admin
 
 # Register your models here.
 

apps/alerts/management/commands/scan_for_alerts.py

@@ -1,8 +1,7 @@
 import logging
 
-from django.core.management.base import BaseCommand
-
 from alerts.models import Alert, AlertLog
+from django.core.management.base import BaseCommand
 
 # Get an instance of a logger
 logger = logging.getLogger(__name__)

apps/alerts/migrations/0001_initial.py

@@ -1,8 +1,8 @@
 # Generated by Django 4.2 on 2023-04-30 09:19
 
+import django.db.models.deletion
 from django.conf import settings
 from django.db import migrations, models
-import django.db.models.deletion
 
 
 class Migration(migrations.Migration):

apps/weather/management/commands/populate_weather.py

@@ -22,12 +22,16 @@ class Command(BaseCommand):
         "rain,weathercode,cloudcover&past_days=7&forecast_days=7"
     )
 
+    def get_weather_data(self, lat, lon):
+        return requests.get(self.url.format(lat, lon)).json()
+
     def handle(self, *args, **options):
         # TODO get lat/lon from database.
         for lat, lon in [(52.52, 13.41), (52.40, -2.01)]:
-            logger.debug(f"Fetching data for {lat}, {lon}")
-            resp = requests.get(self.url.format(lat, lon)).json()
+            self.stdout.write(f"Fetching data for {lat}, {lon}")
+            resp = self.get_weather_data(lat, lon)
             time_now = now()
+            weather_objs = []
 
             for time, temp, apparent_temp, rain, weather_code, cloud_cover in zip(
                 resp["hourly"]["time"],
@@ -39,16 +43,30 @@ def handle(self, *args, **options):
             ):
                 tz_aware_time = datetime.fromisoformat(time)
                 tz_aware_time = utc.localize(tz_aware_time)
-                # this is bad! optimise later!
-                Weather.objects.update_or_create(
-                    date_time=tz_aware_time,
-                    location=Point(lat, lon),
-                    defaults={
-                        "temperature": temp,
-                        "apparent_temperature": apparent_temp,
-                        "rain": rain,
-                        "cloud_cover": cloud_cover,
-                        "weather_code": weather_code,
-                        "is_forecast": tz_aware_time > time_now,
-                    },
+                weather_objs.append(
+                    Weather(
+                        date_time=tz_aware_time,
+                        location=Point(lat, lon),
+                        temperature=temp,
+                        apparent_temperature=apparent_temp,
+                        rain=rain,
+                        cloud_cover=cloud_cover,
+                        weather_code=weather_code,
+                        is_forecast=tz_aware_time > time_now,
+                    )
                 )
+
+            # Bulk insert them, updating where needed
+            Weather.objects.bulk_create(
+                weather_objs,
+                update_conflicts=True,
+                update_fields=[
+                    "temperature",
+                    "apparent_temperature",
+                    "rain",
+                    "cloud_cover",
+                    "weather_code",
+                    "is_forecast",
+                ],
+                unique_fields=["date_time", "location"],
+            )

apps/weather/models.py

@@ -1,6 +1,5 @@
 from django.contrib.gis.db import models as gis_models
 from django.db import models
-
 from weather import choices
 
 # Create your models here.

apps/weather/tests/test_populate_weather_command.py

@@ -0,0 +1,15 @@
+from unittest.mock import patch
+
+from django.core.management import call_command
+from django.test import TestCase
+
+
+class TestPopulateWeatherCommand(TestCase):
+    @patch("weather.management.commands.populate_weather.Command.get_weather_data")
+    def do_command(self, func, return_value):
+        """Helper function for mocking the response to an api call."""
+        func.return_value = return_value
+
+        call_command("populate_weather")
+
+    # TODO.