perf: follow-up historicalanswer migration using SQL

caluma/caluma_form/migrations/0035_historicalanswer_history_question_type.py

@@ -1,27 +1,29 @@
 # Generated by Django 2.2.12 on 2020-06-19 14:21
 
-from django.core.exceptions import ObjectDoesNotExist
 from django.db import migrations, models
 
+INITIAL_UPDATE = """
+    UPDATE caluma_form_historicalanswer histans
+    SET history_question_type = hq_latest.type
 
-def add_initial_historical_question_type(apps, schema_editor):
-    HistoricalAnswer = apps.get_model("caluma_form", "HistoricalAnswer")
-    HistoricalQuestion = apps.get_model("caluma_form", "HistoricalQuestion")
-
-    for answer in HistoricalAnswer.objects.all():
-        try:
-            question = (
-                HistoricalQuestion.objects.filter(
-                    slug=answer.question_id, history_date__lte=answer.history_date
-                )
-                .order_by("-history_date")
-                .latest()
+    FROM (
+        SELECT ha.history_id, hq.type, hq.slug, ha.history_date ha_date, hq.history_date hq_date
+        FROM caluma_form_historicalanswer ha
+        JOIN caluma_form_historicalquestion hq ON hq.slug = ha.question_id AND hq.history_date = (
+            SELECT MAX(hq.history_date) FROM caluma_form_historicalquestion hq WHERE (
+                hq.slug = ha.question_id AND hq.history_date <= ha.history_date
             )
-        except ObjectDoesNotExist:
-            # historical question not found, falling back to current question
-            question = answer.question
-        answer.history_question_type = question.type
-        answer.save()
+        )
+    ) hq_latest
+    WHERE histans.history_id = hq_latest.history_id;
+"""
+
+FALLBACK_UPDATE = """
+    UPDATE caluma_form_historicalanswer ha
+    SET history_question_type = q.type
+    FROM caluma_form_question q
+    WHERE ha.history_question_type IS NULL AND q.slug = ha.question_id;
+"""
 
 
 class Migration(migrations.Migration):
@@ -32,10 +34,15 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name="historicalanswer",
             name="history_question_type",
-            field=models.CharField(default="text", max_length=23),
+            field=models.CharField(default=None, max_length=23, null=True),
             preserve_default=False,
         ),
-        migrations.RunPython(
-            add_initial_historical_question_type, migrations.RunPython.noop
+        migrations.RunSQL(INITIAL_UPDATE, reverse_sql=migrations.RunSQL.noop),
+        migrations.RunSQL(FALLBACK_UPDATE, reverse_sql=migrations.RunSQL.noop),
+        migrations.AlterField(
+            model_name="historicalanswer",
+            name="history_question_type",
+            field=models.CharField(default="", max_length=23),
+            preserve_default=False,
         ),
     ]

caluma/caluma_form/tests/test_migration.py

@@ -378,14 +378,14 @@ def test_migrate_answer_history_question_type(transactional_db):
     now = timezone.now()
 
     # create historical records
-    hist_quest = OldHistQuest.objects.create(
+    old_hist_quest = OldHistQuest.objects.create(
         slug=question.slug,
         type="text",
         created_at=now,
         modified_at=now,
         history_date=now,
     )
-    OldHistAns.objects.create(
+    old_hist_ans = OldHistAns.objects.create(
         id=answer.id,
         value=answer.value,
         document_id=document.pk,
@@ -394,14 +394,36 @@ def test_migrate_answer_history_question_type(transactional_db):
         modified_at=now,
         history_date=now,
     )
+    # create another set of question / answer
+    now = timezone.now()
 
-    # Migrate forwards.
-    executor.loader.build_graph()  # reload.
+    new_hist_quest = OldHistQuest.objects.create(
+        slug=question.slug,
+        type="integer",
+        created_at=now,
+        modified_at=now,
+        history_date=now,
+    )
+    new_hist_ans = OldHistAns.objects.create(
+        id=answer.id,
+        value=answer.value,
+        document_id=document.pk,
+        question_id=question.slug,
+        created_at=now,
+        modified_at=now,
+        history_date=now,
+    )
+    # Migrate forward
+    executor.loader.build_graph()
     executor.migrate(migrate_to)
 
     new_apps = executor.loader.project_state(migrate_to).apps
 
-    # Test the new data.
-    NewHistAns = new_apps.get_model(app, "HistoricalAnswer")
-    new_hist_ans = NewHistAns.objects.get()
-    assert new_hist_ans.history_question_type == hist_quest.type
+    MigratedHistAns = new_apps.get_model(app, "HistoricalAnswer")
+
+    # Test the new data
+    old_hist_ans = MigratedHistAns.objects.get(history_id=old_hist_ans.history_id)
+    assert old_hist_ans.history_question_type == old_hist_quest.type
+
+    new_hist_ans = MigratedHistAns.objects.get(history_id=new_hist_ans.history_id)
+    assert new_hist_ans.history_question_type == new_hist_quest.type