Celery Agent Merging (#3838)

* celery task for processing agent merging in the background

* remove circular model import

* fix Agent import error

* prevent other Agent import errors

* revised agent merging unit tests

* add status url

* fix progress in unit tests

* Spmerging django migration

* fix django migration issues by moving spmerging

* fix serialization error

* clean up status of failed merges

* move starting notification to be inside the celery task

* fix progress closure

* confirmed successful celery task execution

* catch unhandled errors and send notification

* add additional information fields for spmerging

* tmp github unit test fix

* github test database restart to fix unit testing

* fix the test_SpecifyDB for unit testing

* remove previous test_SpecifyDB

* try a different test database

* remove deleting test database action

* revert temp test db

* add github action step for debugging

* debug db connection

* fix debug db connection test

* add verbosity to testing

* move spmerging model definition outside of the specify app and into notifications to prevent errors

* revert GitHub actions to previous state

* add additional info to notifications

.github/workflows/test.yml

@@ -124,8 +124,16 @@ jobs:
       - name: Run Mypy type checker
         run: VIRTUAL_ENV=./ve make typecheck
 
+      # - name: Delete existing test database (if any). Use if database is corrupted
+      #   env:
+      #     PORT: ${{ job.services.mariadb.ports[3306] }}
+      #   run: |
+      #     mysql -h 127.0.0.1 -P $PORT -u MasterUser -p'MasterPassword' -e 'DROP DATABASE IF EXISTS test_SpecifyDB;';
+      #     mysql -h 127.0.0.1 -P $PORT -u MasterUser -p'MasterPassword' -e 'SHOW DATABASES;'
+
       - name: Run test suite
-        run: ./ve/bin/python manage.py test --keepdb
+        run: ./ve/bin/python manage.py test --verbosity=3 --keepdb
+        # run: ./ve/bin/python manage.py test --verbosity=3 --noinput
 
   test-front-end:
     name: Run front-end tests

specifyweb/notifications/migrations/0003_spmerging.py

@@ -0,0 +1,37 @@
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('specify', '__first__'),
+        ('notifications', '0002_message_read'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Spmerging',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=256)),
+                ('taskid', models.CharField(max_length=256)),
+                ('mergingstatus', models.CharField(max_length=256)),
+                ('table', models.CharField(max_length=256)),
+                ('newrecordid', models.SmallIntegerField(null=True)),
+                ('newrecordata', models.JSONField(null=True)),
+                ('oldrecordids', models.JSONField(null=True)),
+                ('collection', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='specify.Collection')),
+                ('specifyuser', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+                ('timestampcreated', models.DateTimeField(default=django.utils.timezone.now)),
+                ('timestampmodified', models.DateTimeField(auto_now=True)),
+                ('createdbyagent', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='specify.agent')),
+                ('modifiedbyagent', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='specify.agent')),
+            ],
+            options={
+                'db_table': 'spmerging'
+            },
+        ),
+    ] 

specifyweb/notifications/models.py

@@ -1,6 +1,6 @@
-
-
 from django.db import models
+from django.utils import timezone
+from specifyweb.specify import models as spmodels
 from ..specify.models import Specifyuser
 
 class Message(models.Model):
@@ -9,3 +9,25 @@ class Message(models.Model):
     content = models.TextField()
     read = models.BooleanField(default=False)
 
+Collection = getattr(spmodels, 'Collection')
+Specifyuser = getattr(spmodels, 'Specifyuser')
+Agent = getattr(spmodels, 'Agent')
+
+class Spmerging(models.Model):
+    name = models.CharField(max_length=256) 
+    taskid = models.CharField(max_length=256) 
+    mergingstatus = models.CharField(max_length=256)
+    table = models.CharField(max_length=256)
+    newrecordid = models.SmallIntegerField(null=True)
+    newrecordata = models.JSONField(null=True)
+    oldrecordids = models.JSONField(null=True)
+    collection = models.ForeignKey(Collection, on_delete=models.CASCADE)
+    specifyuser = models.ForeignKey(Specifyuser, on_delete=models.CASCADE)
+    timestampcreated = models.DateTimeField(default=timezone.now) 
+    timestampmodified = models.DateTimeField(auto_now=True) 
+    createdbyagent = models.ForeignKey(Agent, null=True, on_delete=models.SET_NULL, related_name="+") 
+    modifiedbyagent = models.ForeignKey(Agent, null=True, on_delete=models.SET_NULL, related_name="+")
+
+    class Meta:
+        db_table = 'spmerging'
+        # managed = False

specifyweb/specify/api_tests.py

@@ -579,7 +579,8 @@ def test_replace_agents(self):
         response = c.post(
             f'/api/specify/agent/replace/{agent_2.id}/',
             data=json.dumps({
-                'old_record_ids': [agent_1.id]
+                'old_record_ids': [agent_1.id],
+                'background': False
             }),
             content_type='application/json'
         )
@@ -597,7 +598,8 @@ def test_replace_agents(self):
         response = c.post(
             f'/api/specify/agent/replace/{agent_2.id}/',
             data=json.dumps({
-                'old_record_ids': [agent_1.id]
+                'old_record_ids': [agent_1.id],
+                'background': False
             }),
             content_type='application/json'
         )
@@ -651,7 +653,8 @@ def test_record_recursive_merge(self):
             f'/api/specify/agent/replace/{agent_2.id}/',
             data=json.dumps({
                 'old_record_ids': [agent_1.id],
-                'new_record_data': None
+                'new_record_data': None,
+                'background': False
             }),
             content_type='application/json'
         )
@@ -702,7 +705,8 @@ def test_agent_address_replacement(self):
             f'/api/specify/agent/replace/{agent_1.id}/',
             data=json.dumps({
                 'old_record_ids': [agent_2.id],
-                'new_record_data': None
+                'new_record_data': None,
+                'background': False
             }),
             content_type='application/json'
         )
@@ -788,7 +792,8 @@ def test_agent_address_multiple_replacement(self):
                         }
                     ],
                     'jobtitle': 'shardbearer'
-                }
+                },
+                'background': False
             }),
             content_type='application/json')
         self.assertEqual(response.status_code, 204)

specifyweb/specify/urls.py

@@ -10,6 +10,7 @@
 urlpatterns = [
     # replace record
     url(r'^specify/(?P<model_name>\w+)/replace/(?P<new_model_id>\d+)/$', views.record_merge),
+    url(r'^specify/merging_status/(?P<merge_id>\d+)/', views.merging_status),
 
     # the main business data API
     url(r'^specify_schema/openapi.json$', schema.openapi),