chore(conf): migrate production static storage to GCS buckets

Dockerfile

@@ -6,7 +6,7 @@ ARG APP_HOME=/app
 WORKDIR ${APP_HOME}
 
 COPY . ${APP_HOME}
-RUN npm install && npm cache clean --force
+RUN npm ci --no-audit && npm cache clean --force
 RUN npm run build
 
 # Define an alias for the specfic python version used in this file.
@@ -62,12 +62,15 @@ COPY --from=python-build-stage /usr/src/app/wheels  /wheels/
 RUN pip install --no-cache-dir --no-index --find-links=/wheels/ /wheels/* \
   && rm -rf /wheels/
 
-
-COPY ./entrypoint /entrypoint
-RUN sed -i 's/\r$//g' /entrypoint
-RUN chmod +x /entrypoint
-
 # Copy application code to WORKDIR
 COPY --from=client-builder ${APP_HOME} ${APP_HOME}
 
-ENTRYPOINT ["/entrypoint"]
+
+# Run the web service on container startup.
+# Timeout is set to 0 to disable the timeouts of the workers to allow Cloud
+# Run to handle instance scaling.
+CMD gunicorn config.asgi \
+    --bind 0.0.0.0:$PORT \
+    --timeout 0 \
+    --chdir=/app \
+    -k uvicorn.workers.UvicornWorker

cloudbuild.yaml

@@ -1,6 +1,7 @@
 steps:
-  # build the container image
-  - name: "gcr.io/cloud-builders/docker"
+  # Build the container image
+  - id: "build image"
+    name: "gcr.io/cloud-builders/docker"
     args:
       [
         "build",
@@ -10,12 +11,69 @@ steps:
       ]
 
   # Push the container image to Container Registry
-  - name: "gcr.io/cloud-builders/docker"
+  - id: "push image"
+    name: "gcr.io/cloud-builders/docker"
     args:
-      ["push", "europe-west1-docker.pkg.dev/$PROJECT_ID/fyj/idr-server-${_DEPLOYMENT_TYPE}:$COMMIT_SHA"]
+      [
+        "push",
+        "europe-west1-docker.pkg.dev/$PROJECT_ID/fyj/idr-server-${_DEPLOYMENT_TYPE}:$COMMIT_SHA"
+      ]
+
+  # Apply the latest migrations
+  - id: "apply migrations"
+    name: "gcr.io/google-appengine/exec-wrapper"
+    args:
+      [
+        "-i", "europe-west1-docker.pkg.dev/$PROJECT_ID/fyj/idr-server-${_DEPLOYMENT_TYPE}:$COMMIT_SHA",
+        "-s", "${_CLOUDSQL_INSTANCE_CONNECTION_NAME}",
+        "-e", "DJANGO_SETTINGS_MODULE=config.settings.production",
+        "-e", "GOOGLE_CLOUD_PROJECT=$PROJECT_ID",
+        "-e", "SETTINGS_NAME=${_SETTINGS_NAME}",
+        "--", "python", "/app/manage.py", "migrate",
+      ]
+
+  # Create cache table
+  - id: "create cache table"
+    name: "gcr.io/google-appengine/exec-wrapper"
+    args:
+      [
+        "-i", "europe-west1-docker.pkg.dev/$PROJECT_ID/fyj/idr-server-${_DEPLOYMENT_TYPE}:$COMMIT_SHA",
+        "-s", "${_CLOUDSQL_INSTANCE_CONNECTION_NAME}",
+        "-e", "DJANGO_SETTINGS_MODULE=config.settings.production",
+        "-e", "GOOGLE_CLOUD_PROJECT=$PROJECT_ID",
+        "-e", "SETTINGS_NAME=${_SETTINGS_NAME}",
+        "--", "python", "/app/manage.py", "createcachetable",
+      ]
+
+  # Collect static files
+  - id: "collect static files"
+    name: "gcr.io/google-appengine/exec-wrapper"
+    args:
+      [
+        "-i", "europe-west1-docker.pkg.dev/$PROJECT_ID/fyj/idr-server-${_DEPLOYMENT_TYPE}:$COMMIT_SHA",
+        "-s", "${_CLOUDSQL_INSTANCE_CONNECTION_NAME}",
+        "-e", "DJANGO_SETTINGS_MODULE=config.settings.production",
+        "-e", "GOOGLE_CLOUD_PROJECT=$PROJECT_ID",
+        "-e", "SETTINGS_NAME=${_SETTINGS_NAME}",
+        "--", "python", "/app/manage.py", "collectstatic", "--noinput"
+      ]
+
+  # Collect static files
+  - id: "compress static assets"
+    name: "gcr.io/google-appengine/exec-wrapper"
+    args:
+      [
+        "-i", "europe-west1-docker.pkg.dev/$PROJECT_ID/fyj/idr-server-${_DEPLOYMENT_TYPE}:$COMMIT_SHA",
+        "-s", "${_CLOUDSQL_INSTANCE_CONNECTION_NAME}",
+        "-e", "DJANGO_SETTINGS_MODULE=config.settings.production",
+        "-e", "GOOGLE_CLOUD_PROJECT=$PROJECT_ID",
+        "-e", "SETTINGS_NAME=${_SETTINGS_NAME}",
+        "--", "python", "/app/manage.py", "compress"
+      ]
 
   # Deploy an image from Container Registry to Cloud Run
-  - name: "gcr.io/cloud-builders/gcloud"
+  - id: "deploy to cloud run"
+    name: "gcr.io/cloud-builders/gcloud"
     args: [
       "beta",
       "run",
@@ -26,7 +84,7 @@ steps:
       "--platform", "managed",
       "--allow-unauthenticated",
       "--add-cloudsql-instances", "${_CLOUDSQL_INSTANCE_CONNECTION_NAME}",
-      "--set-env-vars", "GOOGLE_CLOUD_PROJECT=$PROJECT_ID,SETTINGS_NAME=${_SETTINGS_NAME},DJANGO_SETTINGS_MODULE=config.settings.production",
+      "--set-env-vars", "GOOGLE_CLOUD_PROJECT=$PROJECT_ID,SETTINGS_NAME=${_SETTINGS_NAME},DJANGO_SETTINGS_MODULE=config.settings.production,ENV_PATH=/tmp/secrets/.env",
       "--min-instances", "1",
       "--max-instances", "8",
       "--memory", "512M",

config/settings/production.py

@@ -1,7 +1,13 @@
+import io
 import json
 import logging
 
+import google.auth
+import google.auth.exceptions
 import sentry_sdk
+from django.conf import ImproperlyConfigured
+from dotenv import load_dotenv
+from google.cloud import secretmanager
 from google.oauth2 import service_account
 from sentry_sdk.integrations.django import DjangoIntegration
 from sentry_sdk.integrations.logging import LoggingIntegration
@@ -13,8 +19,31 @@
 # READ ENVIRONMENT
 ###############################################################################
 
-ENV_PATH = env.str("ENV_PATH", default="/tmp/secrets/.env")
-env.read_env(path=ENV_PATH, override=True)
+ENV_PATH = env.str("ENV_PATH", default=None)
+# First, try and load the environment variables from an .env file if a path to
+# the file is provided.
+if ENV_PATH:
+    env.read_env(path=ENV_PATH, override=True)
+# Else, load the variables from Google Secrets Manager
+else:
+    SETTINGS_NAME = env.str("SETTINGS_NAME")
+    try:
+        GCP_PROJECT_ID = env.str(
+            "GOOGLE_CLOUD_PROJECT", default=google.auth.default()
+        )
+    except google.auth.exceptions.DefaultCredentialsError:
+        raise ImproperlyConfigured(
+            "No local .env or GOOGLE_CLOUD_PROJECT detected. No secrets found."
+        )
+
+    secret_manager_client = secretmanager.SecretManagerServiceClient()
+    secrets_name = "projects/{}/secrets/{}/versions/latest".format(
+        GCP_PROJECT_ID, SETTINGS_NAME
+    )
+    payload = secret_manager_client.access_secret_version(
+        name=secrets_name
+    ).payload.data.decode("UTF-8")
+    load_dotenv(stream=io.StringIO(payload), override=True)
 
 
 ###############################################################################
@@ -119,7 +148,7 @@
 
 INSTALLED_APPS += ["storages"]  # noqa: F405
 GS_BUCKET_NAME = env.str("DJANGO_GCP_STORAGE_BUCKET_NAME")
-GS_DEFAULT_ACL = "project-private"
+GS_DEFAULT_ACL = "projectPrivate"
 
 
 ###############################################################################
@@ -128,20 +157,15 @@
 
 DEFAULT_FILE_STORAGE = "utils.storages.MediaRootGoogleCloudStorage"
 MEDIA_URL = "https://storage.googleapis.com/%s/media/" % GS_BUCKET_NAME
-STATICFILES_STORAGE = "whitenoise.storage.CompressedStaticFilesStorage"
+STATIC_URL = "https://storage.googleapis.com/%s/static/" % GS_BUCKET_NAME
+STATICFILES_STORAGE = "utils.storages.StaticRootGoogleCloudStorage"
 
 
 ###############################################################################
 # DJANGO COMPRESSOR
 ###############################################################################
 
 COMPRESS_ENABLED = True
-# https://django-compressor.readthedocs.io/en/latest/settings/#django.conf.settings.COMPRESS_URL
-COMPRESS_URL = STATIC_URL  # noqa F405
-# https://django-compressor.readthedocs.io/en/latest/settings/#django.conf.settings.COMPRESS_OFFLINE
-COMPRESS_OFFLINE = (
-    True  # Offline compression is required when using Whitenoise
-)
 # https://django-compressor.readthedocs.io/en/latest/settings/#django.conf.settings.COMPRESS_FILTERS
 COMPRESS_FILTERS = {
     "css": [
@@ -150,6 +174,14 @@
     ],
     "js": ["compressor.filters.jsmin.JSMinFilter"],
 }
+# https://django-compressor.readthedocs.io/en/latest/settings/#django.conf.settings.COMPRESS_OFFLINE
+COMPRESS_OFFLINE = True
+# https://django-compressor.readthedocs.io/en/stable/settings.html#django.conf.settings.COMPRESS_OFFLINE_MANIFEST_STORAGE
+COMPRESS_OFFLINE_MANIFEST_STORAGE = STATICFILES_STORAGE
+# https://django-compressor.readthedocs.io/en/stable/settings.html#django.conf.settings.COMPRESS_STORAGE
+COMPRESS_STORAGE = STATICFILES_STORAGE
+# https://django-compressor.readthedocs.io/en/latest/settings/#django.conf.settings.COMPRESS_URL
+COMPRESS_URL = STATIC_URL
 
 
 ###############################################################################

config/urls.py

@@ -41,30 +41,35 @@
             url=settings.STATIC_URL + "favicon.ico", permanent=True
         ),
     ),
-] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
+]
 
 if settings.DEBUG:
     # Static file serving when using Gunicorn + Uvicorn for local web socket
     # development.
     urlpatterns += staticfiles_urlpatterns()
-    if settings.DEBUG:
-        # This allows the error pages to be debugged during development, just
-        # visit these url in browser to see how these error pages look like.
-        urlpatterns += [
-            path(
-                "400/",
-                default_views.bad_request,
-                kwargs={"exception": Exception("Bad Request!")},
-            ),
-            path(
-                "403/",
-                default_views.permission_denied,
-                kwargs={"exception": Exception("Permission Denied")},
-            ),
-            path(
-                "404/",
-                default_views.page_not_found,
-                kwargs={"exception": Exception("Page not Found")},
-            ),
-            path("500/", default_views.server_error),
-        ]
+
+    # Server media files during local development.
+    urlpatterns += static(
+        settings.MEDIA_URL, document_root=settings.MEDIA_ROOT
+    )
+
+    # This allows the error pages to be debugged during development, just
+    # visit these url in browser to see how these error pages look like.
+    urlpatterns += [
+        path(
+            "400/",
+            default_views.bad_request,
+            kwargs={"exception": Exception("Bad Request!")},
+        ),
+        path(
+            "403/",
+            default_views.permission_denied,
+            kwargs={"exception": Exception("Permission Denied")},
+        ),
+        path(
+            "404/",
+            default_views.page_not_found,
+            kwargs={"exception": Exception("Page not Found")},
+        ),
+        path("500/", default_views.server_error),
+    ]

requirements/production.txt

@@ -19,5 +19,6 @@ grpcio~=1.53.0
 # Others
 # -----------------------------------------------------------------------------
 gunicorn~=20.1.0
-sentry-sdk~=1.18.0
+python-dotenv~=1.0.0
+sentry-sdk~=1.19.0
 uvicorn~=0.21.1

utils/storages.py

@@ -2,10 +2,12 @@
 
 
 class StaticRootGoogleCloudStorage(GoogleCloudStorage):
+    default_acl = "publicRead"
+    file_overwrite = True
     location = "static"
-    default_acl = "project-private"
 
 
 class MediaRootGoogleCloudStorage(GoogleCloudStorage):
-    location = "media"
+    default_acl = "projectPrivate"
     file_overwrite = False
+    location = "media"