Migrate stack to Postgresql

Dockerfile

@@ -8,7 +8,7 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK=1
 ENV PATH="/venv/bin:$PATH"
 
 COPY docker/bin/apt-install /usr/local/bin/
-RUN apt-install build-essential default-libmysqlclient-dev libxslt1.1 libxml2 libxml2-dev libxslt1-dev
+RUN apt-install build-essential libxslt1.1 libxml2 libxml2-dev libxslt1-dev
 
 RUN python -m venv /venv
 
@@ -42,7 +42,7 @@ WORKDIR /app
 ENV DJANGO_SETTINGS_MODULE=basket.settings
 
 COPY docker/bin/apt-install /usr/local/bin/
-RUN apt-install default-libmysqlclient-dev libxslt1.1 libxml2
+RUN apt-install libxslt1.1 libxml2 postgresql-client
 
 ARG GIT_SHA=latest
 ENV GIT_SHA=${GIT_SHA}

basket/petition/tests/test_views.py

@@ -74,7 +74,7 @@ def test_petition_post_invalid(client, mocker):
         },
     }
     assert Petition.objects.count() == 0
-    assert mock_send_mail.call_count == 0
+    assert mock_send_mail.delay.call_count == 0
 
 
 @pytest.mark.django_db
@@ -96,7 +96,7 @@ def test_petition_email_invalid(client, mocker):
         },
     }
     assert Petition.objects.count() == 0
-    assert mock_send_mail.call_count == 0
+    assert mock_send_mail.delay.call_count == 0
 
 
 @pytest.mark.django_db
@@ -118,11 +118,11 @@ def test_petition_name_invalid(client, mocker):
         },
     }
     assert Petition.objects.count() == 0
-    assert mock_send_mail.call_count == 0
+    assert mock_send_mail.delay.call_count == 0
 
 
 @pytest.mark.django_db
-def test_petition_db_error(client, mocker):
+def test_petition_db_emoji(client, mocker):
     mock_send_mail = mocker.patch("basket.petition.models.send_email_confirmation")
     url = reverse("sign-petition")
     data = {
@@ -134,12 +134,9 @@ def test_petition_db_error(client, mocker):
     response = client.post(url, data=data)
     assert response.status_code == 200
     assert response.json() == {
-        "status": "error",
-        "errors": {
-            "__all__": "Database error",
-        },
+        "status": "success",
     }
-    assert mock_send_mail.call_count == 0
+    assert mock_send_mail.delay.call_count == 1
 
 
 def test_petition_cors(client):

basket/settings.py

@@ -45,19 +45,15 @@ def path(*args):
 # avoids a warning from django
 TEST_RUNNER = "django.test.runner.DiscoverRunner"
 
-# Production uses MySQL, but Sqlite should be sufficient for local development.
-# Our CI server tests against MySQL.
+# Production uses Postgres, but Sqlite should be sufficient for local development.
+# Our CI server tests against Postgres.
 DATABASES = {
     "default": config(
         "DATABASE_URL",
         default="sqlite:///basket.db",
         cast=dj_database_url.parse,
     ),
 }
-if DATABASES["default"]["ENGINE"] == "django.db.backends.mysql":
-    DATABASES["default"]["OPTIONS"] = {
-        "init_command": "SET sql_mode='STRICT_TRANS_TABLES'",
-    }
 DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
 
 # CACHE_URL and RQ_URL are derived from REDIS_URL.

bin/run-dev.sh

@@ -1,5 +1,4 @@
 #!/bin/bash -ex
 
-urlwait
 python manage.py migrate --noinput
 python manage.py runserver 0.0.0.0:8000

bin/run-tests.sh

@@ -4,7 +4,6 @@ set -exo pipefail
 
 ruff check basket/
 ruff format --check basket/
-urlwait
 python manage.py makemigrations | grep "No changes detected"
 python manage.py migrate --noinput
 py.test basket \

docker-compose.yml

@@ -1,10 +1,16 @@
 version: "3.4"
 services:
-  db:
-    image: mariadb
+  postgres:
+    image: postgres:15-alpine
     environment:
-      - MYSQL_ALLOW_EMPTY_PASSWORD=yes
-      - MYSQL_DATABASE=basket
+      POSTGRES_DB: basket
+      POSTGRES_USER: basket
+      POSTGRES_PASSWORD: basket
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready"]
+      interval: 2s
+      timeout: 2s
+      retries: 10
 
   redis:
     image: redis
@@ -25,7 +31,7 @@ services:
     ports:
       - "8000:8000"
     depends_on:
-      - db
+      - postgres
       - redis
     command:
       ./bin/run-dev.sh
@@ -40,7 +46,7 @@ services:
       - docker/envfiles/local.env
       - .env
     depends_on:
-      - db
+      - postgres
       - redis
     command:
       ./bin/run-worker.sh
@@ -54,7 +60,7 @@ services:
       - docker/envfiles/local.env
       - docker/envfiles/test.env
     depends_on:
-      - db
+      - postgres
       - redis
     command:
       ./bin/run-tests.sh
@@ -66,7 +72,7 @@ services:
       - docker/envfiles/local.env
       - docker/envfiles/test.env
     depends_on:
-      - db
+      - postgres
       - redis
     command:
       ./bin/run-tests.sh

docker/envfiles/local.env

@@ -1,9 +1,7 @@
 ALLOWED_HOSTS=localhost,127.0.0.1,testserver
-DATABASE_URL=mysql://root@db/basket
-DEBUG=True
-DJANGO_LOG_LEVEL=DEBUG
+DATABASE_URL=postgres://basket:basket@postgres/basket
+DDJANGO_LOG_LEVEL=DEBUG
 LOCAL_DEV=True
 REDIS_URL=redis://redis:6379
-URLWAIT_TIMEOUT=30
 # Setting max_jobs low to test worker completion and restarts via Docker compose.
 RQ_MAX_JOBS=5

docker/envfiles/test.env

@@ -1,6 +1,6 @@
 ADMINS=["thedude@example.com"]
 ALLOWED_HOSTS=*
-DATABASE_URL=mysql://root@db/basket
+DATABASE_URL=postgres://basket:basket@postgres/basket
 
 DEBUG=False
 DEV=False

docs/install.rst

@@ -42,7 +42,7 @@ code according to our style and check for errors for every commit.
 Use Docker
 ----------
 
-Basket requires a database (either MySQL or SQLite locally, depending on the ``DATABASE_URL`` setting) and Redis. We use Docker to run these services.
+Basket requires a database (either Postgres or SQLite locally, depending on the ``DATABASE_URL`` setting) and Redis. We use Docker to run these services.
 
 The steps to get up and running are these:
 

requirements/dev.in

@@ -5,5 +5,4 @@ pytest-cov==4.1.0
 pytest-datadir==1.5.0
 pytest-django==4.8.0
 pytest-mock==3.12.0
-urlwait==1.0
 ruff==0.2.0

requirements/dev.txt

@@ -616,15 +616,6 @@ mozilla-django-oidc==4.0.0 \
 msgpack-python==0.5.6 \
     --hash=sha256:378cc8a6d3545b532dfd149da715abae4fda2a3adb6d74e525d0d5e51f46909b
     # via -r requirements/prod.txt
-mysqlclient==2.1.1 \
-    --hash=sha256:0d1cd3a5a4d28c222fa199002810e8146cffd821410b67851af4cc80aeccd97c \
-    --hash=sha256:828757e419fb11dd6c5ed2576ec92c3efaa93a0f7c39e263586d1ee779c3d782 \
-    --hash=sha256:996924f3483fd36a34a5812210c69e71dea5a3d5978d01199b78b7f6d485c855 \
-    --hash=sha256:b355c8b5a7d58f2e909acdbb050858390ee1b0e13672ae759e5e784110022994 \
-    --hash=sha256:c1ed71bd6244993b526113cca3df66428609f90e4652f37eb51c33496d478b37 \
-    --hash=sha256:c812b67e90082a840efb82a8978369e6e69fc62ce1bda4ca8f3084a9d862308b \
-    --hash=sha256:dea88c8d3f5a5d9293dfe7f087c16dd350ceb175f2f6631c9cf4caf3e19b7a96
-    # via -r requirements/prod.txt
 newrelic==9.6.0 \
     --hash=sha256:01c0eb630bb18261241a37aa0a70cb6f706079a1f58f59f2bb64f26fda54ffc5 \
     --hash=sha256:09dad0db993402e166e37d99302c2ad5588b4ff1e5b814819540ca5ec2bd3cea \
@@ -673,6 +664,80 @@ pluggy==1.4.0 \
     --hash=sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981 \
     --hash=sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be
     # via pytest
+psycopg2-binary==2.9.9 \
+    --hash=sha256:03ef7df18daf2c4c07e2695e8cfd5ee7f748a1d54d802330985a78d2a5a6dca9 \
+    --hash=sha256:0a602ea5aff39bb9fac6308e9c9d82b9a35c2bf288e184a816002c9fae930b77 \
+    --hash=sha256:0c009475ee389757e6e34611d75f6e4f05f0cf5ebb76c6037508318e1a1e0d7e \
+    --hash=sha256:0ef4854e82c09e84cc63084a9e4ccd6d9b154f1dbdd283efb92ecd0b5e2b8c84 \
+    --hash=sha256:1236ed0952fbd919c100bc839eaa4a39ebc397ed1c08a97fc45fee2a595aa1b3 \
+    --hash=sha256:143072318f793f53819048fdfe30c321890af0c3ec7cb1dfc9cc87aa88241de2 \
+    --hash=sha256:15208be1c50b99203fe88d15695f22a5bed95ab3f84354c494bcb1d08557df67 \
+    --hash=sha256:1873aade94b74715be2246321c8650cabf5a0d098a95bab81145ffffa4c13876 \
+    --hash=sha256:18d0ef97766055fec15b5de2c06dd8e7654705ce3e5e5eed3b6651a1d2a9a152 \
+    --hash=sha256:1ea665f8ce695bcc37a90ee52de7a7980be5161375d42a0b6c6abedbf0d81f0f \
+    --hash=sha256:2293b001e319ab0d869d660a704942c9e2cce19745262a8aba2115ef41a0a42a \
+    --hash=sha256:246b123cc54bb5361588acc54218c8c9fb73068bf227a4a531d8ed56fa3ca7d6 \
+    --hash=sha256:275ff571376626195ab95a746e6a04c7df8ea34638b99fc11160de91f2fef503 \
+    --hash=sha256:281309265596e388ef483250db3640e5f414168c5a67e9c665cafce9492eda2f \
+    --hash=sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493 \
+    --hash=sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996 \
+    --hash=sha256:30dcc86377618a4c8f3b72418df92e77be4254d8f89f14b8e8f57d6d43603c0f \
+    --hash=sha256:31a34c508c003a4347d389a9e6fcc2307cc2150eb516462a7a17512130de109e \
+    --hash=sha256:323ba25b92454adb36fa425dc5cf6f8f19f78948cbad2e7bc6cdf7b0d7982e59 \
+    --hash=sha256:34eccd14566f8fe14b2b95bb13b11572f7c7d5c36da61caf414d23b91fcc5d94 \
+    --hash=sha256:3a58c98a7e9c021f357348867f537017057c2ed7f77337fd914d0bedb35dace7 \
+    --hash=sha256:3f78fd71c4f43a13d342be74ebbc0666fe1f555b8837eb113cb7416856c79682 \
+    --hash=sha256:4154ad09dac630a0f13f37b583eae260c6aa885d67dfbccb5b02c33f31a6d420 \
+    --hash=sha256:420f9bbf47a02616e8554e825208cb947969451978dceb77f95ad09c37791dae \
+    --hash=sha256:4686818798f9194d03c9129a4d9a702d9e113a89cb03bffe08c6cf799e053291 \
+    --hash=sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe \
+    --hash=sha256:60989127da422b74a04345096c10d416c2b41bd7bf2a380eb541059e4e999980 \
+    --hash=sha256:64cf30263844fa208851ebb13b0732ce674d8ec6a0c86a4e160495d299ba3c93 \
+    --hash=sha256:68fc1f1ba168724771e38bee37d940d2865cb0f562380a1fb1ffb428b75cb692 \
+    --hash=sha256:6e6f98446430fdf41bd36d4faa6cb409f5140c1c2cf58ce0bbdaf16af7d3f119 \
+    --hash=sha256:729177eaf0aefca0994ce4cffe96ad3c75e377c7b6f4efa59ebf003b6d398716 \
+    --hash=sha256:72dffbd8b4194858d0941062a9766f8297e8868e1dd07a7b36212aaa90f49472 \
+    --hash=sha256:75723c3c0fbbf34350b46a3199eb50638ab22a0228f93fb472ef4d9becc2382b \
+    --hash=sha256:77853062a2c45be16fd6b8d6de2a99278ee1d985a7bd8b103e97e41c034006d2 \
+    --hash=sha256:78151aa3ec21dccd5cdef6c74c3e73386dcdfaf19bced944169697d7ac7482fc \
+    --hash=sha256:7f01846810177d829c7692f1f5ada8096762d9172af1b1a28d4ab5b77c923c1c \
+    --hash=sha256:804d99b24ad523a1fe18cc707bf741670332f7c7412e9d49cb5eab67e886b9b5 \
+    --hash=sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab \
+    --hash=sha256:8359bf4791968c5a78c56103702000105501adb557f3cf772b2c207284273984 \
+    --hash=sha256:83791a65b51ad6ee6cf0845634859d69a038ea9b03d7b26e703f94c7e93dbcf9 \
+    --hash=sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf \
+    --hash=sha256:876801744b0dee379e4e3c38b76fc89f88834bb15bf92ee07d94acd06ec890a0 \
+    --hash=sha256:8dbf6d1bc73f1d04ec1734bae3b4fb0ee3cb2a493d35ede9badbeb901fb40f6f \
+    --hash=sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212 \
+    --hash=sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb \
+    --hash=sha256:977646e05232579d2e7b9c59e21dbe5261f403a88417f6a6512e70d3f8a046be \
+    --hash=sha256:9dba73be7305b399924709b91682299794887cbbd88e38226ed9f6712eabee90 \
+    --hash=sha256:a148c5d507bb9b4f2030a2025c545fccb0e1ef317393eaba42e7eabd28eb6041 \
+    --hash=sha256:a6cdcc3ede532f4a4b96000b6362099591ab4a3e913d70bcbac2b56c872446f7 \
+    --hash=sha256:ac05fb791acf5e1a3e39402641827780fe44d27e72567a000412c648a85ba860 \
+    --hash=sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d \
+    --hash=sha256:b58b4710c7f4161b5e9dcbe73bb7c62d65670a87df7bcce9e1faaad43e715245 \
+    --hash=sha256:b6356793b84728d9d50ead16ab43c187673831e9d4019013f1402c41b1db9b27 \
+    --hash=sha256:b76bedd166805480ab069612119ea636f5ab8f8771e640ae103e05a4aae3e417 \
+    --hash=sha256:bc7bb56d04601d443f24094e9e31ae6deec9ccb23581f75343feebaf30423359 \
+    --hash=sha256:c2470da5418b76232f02a2fcd2229537bb2d5a7096674ce61859c3229f2eb202 \
+    --hash=sha256:c332c8d69fb64979ebf76613c66b985414927a40f8defa16cf1bc028b7b0a7b0 \
+    --hash=sha256:c6af2a6d4b7ee9615cbb162b0738f6e1fd1f5c3eda7e5da17861eacf4c717ea7 \
+    --hash=sha256:c77e3d1862452565875eb31bdb45ac62502feabbd53429fdc39a1cc341d681ba \
+    --hash=sha256:ca08decd2697fdea0aea364b370b1249d47336aec935f87b8bbfd7da5b2ee9c1 \
+    --hash=sha256:ca49a8119c6cbd77375ae303b0cfd8c11f011abbbd64601167ecca18a87e7cdd \
+    --hash=sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07 \
+    --hash=sha256:d2997c458c690ec2bc6b0b7ecbafd02b029b7b4283078d3b32a852a7ce3ddd98 \
+    --hash=sha256:d3f82c171b4ccd83bbaf35aa05e44e690113bd4f3b7b6cc54d2219b132f3ae55 \
+    --hash=sha256:dc4926288b2a3e9fd7b50dc6a1909a13bbdadfc67d93f3374d984e56f885579d \
+    --hash=sha256:ead20f7913a9c1e894aebe47cccf9dc834e1618b7aa96155d2091a626e59c972 \
+    --hash=sha256:ebdc36bea43063116f0486869652cb2ed7032dbc59fbcb4445c4862b5c1ecf7f \
+    --hash=sha256:ed1184ab8f113e8d660ce49a56390ca181f2981066acc27cf637d5c1e10ce46e \
+    --hash=sha256:ee825e70b1a209475622f7f7b776785bd68f34af6e7a46e2e42f27b659b5bc26 \
+    --hash=sha256:f7ae5d65ccfbebdfa761585228eb4d0df3a8b15cfb53bd953e713e09fbb12957 \
+    --hash=sha256:f7fc5a5acafb7d6ccca13bfa8c90f8c51f13d8fb87d95656d3950f0158d3ce53 \
+    --hash=sha256:f9b5571d33660d5009a8b3c25dc1db560206e2d2f89d3df1cb32d72c0d117d52
+    # via -r requirements/prod.txt
 pybrowserid==0.14.0 \
     --hash=sha256:6c227669e87cc25796ae76f6a0ef65025528c8ad82d352679fa9a3e5663a71e3 \
     --hash=sha256:8e237d6a2bc9ead849a4472a84d3e6a9309bec99cf8e10d36213710dda8df8ca
@@ -913,10 +978,6 @@ urllib3==1.26.18 \
     #   botocore
     #   requests
     #   sentry-sdk
-urlwait==1.0 \
-    --hash=sha256:a9bf2da792fa6983fa93f6360108e16615066ab0f9cfb7f53e5faee5f5dffaac \
-    --hash=sha256:eae2c20001efc915166cac79c04bac0088ad5787ec64b36f27afd2f359953b2b
-    # via -r requirements/dev.in
 user-agents==2.2.0 \
     --hash=sha256:a98c4dc72ecbc64812c4534108806fb0a0b3a11ec3fd1eafe807cee5b0a942e7 \
     --hash=sha256:d36d25178db65308d1458c5fa4ab39c9b2619377010130329f3955e7626ead26

requirements/prod.in

@@ -20,8 +20,8 @@ lxml==5.1.0
 markus[datadog]==4.2.0
 mozilla-django-oidc==4.0.0
 msgpack-python==0.5.6
-mysqlclient==2.1.1
 newrelic==9.6.0
+psycopg2-binary==2.9.9
 PyFxA==0.7.7
 pyOpenSSL==24.0.0
 pysilverpop==0.2.6