fix: handle database permissions errors and workflow exceptions (#609)

* fix: fix zarf templating typo * fix: change user running the migrations * fix: delay removing the supabase deployment wihtin our e2e tests * fix: fix duplicate id name in e2e.yaml workflow * fix: test for the proper http return codes when testing row_level_security * chore: bump version of uds-core for the e2e tests * fix: created test utility for getting api key to test against --------- Co-authored-by: Gregory Horvath <gphorvath@defenseunicorns.com>
defenseunicorns · Jun 12, 2024 · e910f06 · e910f06
1 parent e8bf312
commit e910f06
Show file tree

Hide file tree

Showing 12 changed files with 119 additions and 82 deletions.
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -69,7 +69,7 @@ jobs:
 
         - name: Create UDS Cluster
           run: |
-            uds deploy k3d-core-slim-dev:0.18.0 --confirm
+            uds deploy k3d-core-slim-dev:0.22.1 --confirm
 
           ##########
           # Supabase
@@ -85,18 +85,13 @@ jobs:
         - name: Set environment variable
           id: set-env-var
           run: |
-            echo "API_KEY=$(uds zarf tools kubectl get secret supabase-bootstrap-jwt -n leapfrogai -o jsonpath='{.data.anon-key}' | base64 -d)" >> "$GITHUB_ENV"
+            echo "ANON_KEY=$(uds zarf tools kubectl get secret supabase-bootstrap-jwt -n leapfrogai -o jsonpath='{.data.anon-key}' | base64 -d)" >> "$GITHUB_ENV"
 
         - name: Test Supabase
           run: |
             python -m pip install requests
             python -m pytest ./tests/e2e/test_supabase.py -v
 
-        # This cleanup may need to be moved/removed when other packages depend on Supabase
-        - name: Cleanup Supabase
-          run: |
-            uds zarf package remove supabase -l=trace --confirm
-
         ##########
         # API
         ##########
@@ -106,11 +101,6 @@ jobs:
             docker image prune -af
             uds zarf package deploy packages/api/zarf-package-leapfrogai-api-amd64-e2e-test.tar.zst --set=EXPOSE_OPENAPI_SCHEMA=true --confirm
             rm packages/api/zarf-package-leapfrogai-api-amd64-e2e-test.tar.zst
-            
-        - name: Set environment variable
-          id: set-env-var
-          run: |
-            echo "ANON_KEY=$(uds zarf tools kubectl get secret supabase-bootstrap-jwt -n leapfrogai -o jsonpath='{.data.anon-key}' | base64 -d)" >> "$GITHUB_ENV"
 
         - name: Test API
           run: |
@@ -171,6 +161,10 @@ jobs:
           run: |
             uds zarf package remove whisper -l=trace --confirm
 
+        # This cleanup may need to be moved/removed when other packages depend on Supabase
+        - name: Cleanup Supabase
+          run: |
+            uds zarf package remove supabase -l=trace --confirm
 
         ##########
         # vLLM

diff --git a/packages/api/chart/templates/migration-job.yaml b/packages/api/chart/templates/migration-job.yaml
@@ -26,7 +26,11 @@ spec:
         args:
           - -c
           - >-
-            supabase migration fetch --db-url="postgresql://postgres:$POSTGRES_PASSWORD@$MIGRATION_SERVICE_NAME.$MIGRATION_NAMESPACE.svc.cluster.local:$MIGRATION_SERVICE_PORT" --debug || true &&
-            supabase db push --db-url="postgresql://postgres:$POSTGRES_PASSWORD@$MIGRATION_SERVICE_NAME.$MIGRATION_NAMESPACE.svc.cluster.local:$MIGRATION_SERVICE_PORT" --debug
+            supabase migration fetch --db-url="postgresql://supabase_admin:$POSTGRES_PASSWORD@$MIGRATION_SERVICE_NAME.$MIGRATION_NAMESPACE.svc.cluster.local:$MIGRATION_SERVICE_PORT/postgres" --debug || true &&
+            supabase db push --db-url="postgresql://supabase_admin:$POSTGRES_PASSWORD@$MIGRATION_SERVICE_NAME.$MIGRATION_NAMESPACE.svc.cluster.local:$MIGRATION_SERVICE_PORT/postgres" --include-all --debug
+        securityContext:
+          runAsUser: {{ .Values.image.securityContext.runAsUser }}
+          runAsGroup: {{ .Values.image.securityContext.runAsGroup }}
+          fsGroup: {{ .Values.image.securityContext.fsGroup }}
       restartPolicy: Never
   backoffLimit: 4
diff --git a/packages/api/chart/values.yaml b/packages/api/chart/values.yaml
@@ -4,6 +4,11 @@ image:
   # x-release-please-end
   kiwigridTag: 1.23.3
 
+  securityContext:
+    runAsUser: 65532
+    runAsGroup: 65532
+    fsGroup: 65532
+
 supabase:
   url: "https://supabase-kong.###ZARF_VAR_HOSTED_DOMAIN###"
 

diff --git a/packages/api/zarf.yaml b/packages/api/zarf.yaml
@@ -45,6 +45,6 @@ components:
           - wait:
               cluster:
                 kind: Job
-                name: api-migrations-###ZARF_PKG_TMPL_IMAGE_VERSION###
+                name: api-migrations-###ZARF_PKG_TMPL_LEAPFROGAI_IMAGE_VERSION###
                 namespace: leapfrogai
                 condition: complete
diff --git a/packages/ui/chart/templates/ui/migration-job.yaml b/packages/ui/chart/templates/ui/migration-job.yaml
@@ -26,7 +26,11 @@ spec:
         args:
           - -c
           - >-
-            supabase migration fetch --db-url="postgresql://postgres:$POSTGRES_PASSWORD@$MIGRATION_SERVICE_NAME.$MIGRATION_NAMESPACE.svc.cluster.local:$MIGRATION_SERVICE_PORT" --debug || true &&
-            supabase db push --db-url="postgresql://postgres:$POSTGRES_PASSWORD@$MIGRATION_SERVICE_NAME.$MIGRATION_NAMESPACE.svc.cluster.local:$MIGRATION_SERVICE_PORT" --debug
+            supabase migration fetch --db-url="postgresql://supabase_admin:$POSTGRES_PASSWORD@$MIGRATION_SERVICE_NAME.$MIGRATION_NAMESPACE.svc.cluster.local:$MIGRATION_SERVICE_PORT/postgres" --debug || true &&
+            supabase db push --db-url="postgresql://supabase_admin:$POSTGRES_PASSWORD@$MIGRATION_SERVICE_NAME.$MIGRATION_NAMESPACE.svc.cluster.local:$MIGRATION_SERVICE_PORT/postgres" --include-all --debug
+        securityContext:
+          runAsUser: {{ .Values.image.securityContext.runAsUser }}
+          runAsGroup: {{ .Values.image.securityContext.runAsGroup }}
+          fsGroup: {{ .Values.image.securityContext.fsGroup }}
       restartPolicy: Never
   backoffLimit: 4
diff --git a/packages/ui/chart/ui-values.yaml b/packages/ui/chart/ui-values.yaml
@@ -4,6 +4,11 @@ image:
   # Overrides the image tag whose default is the chart appVersion.
   tag: '###ZARF_CONST_IMAGE_VERSION###'
 
+  securityContext:
+    runAsUser: 65532
+    runAsGroup: 65532
+    fsGroup: 65532
+
 package:
   name: leapfrogai-ui
   subdomain: '###ZARF_VAR_SUBDOMAIN###'

diff --git a/tests/e2e/test_api.py b/tests/e2e/test_api.py
@@ -4,18 +4,18 @@
 
 import pytest as pytest
 import requests
-import os
-import json
+
+from .utils import create_test_user
 
 logger = logging.getLogger(__name__)
 test_id = str(uuid.uuid4())
 
 get_urls = {
-    "assistants_url": "http://leapfrogai-api.uds.dev/openai/v1/assistants",
-    "assistants_id_url": f"http://leapfrogai-api.uds.dev/openai/v1/assistants/{test_id}",
-    "files_url": "http://leapfrogai-api.uds.dev/openai/v1/files",
-    "files_specific_url": f"http://leapfrogai-api.uds.dev/openai/v1/files/{test_id}",
-    "files_specific_content_url": f"http://leapfrogai-api.uds.dev/openai/v1/files/{test_id}/content",
+    "assistants_url": "https://leapfrogai-api.uds.dev/openai/v1/assistants",
+    "assistants_id_url": f"https://leapfrogai-api.uds.dev/openai/v1/assistants/{test_id}",
+    "files_url": "https://leapfrogai-api.uds.dev/openai/v1/files",
+    "files_specific_url": f"https://leapfrogai-api.uds.dev/openai/v1/files/{test_id}",
+    "files_specific_content_url": f"https://leapfrogai-api.uds.dev/openai/v1/files/{test_id}/content",
 }
 
 post_urls = {
@@ -25,15 +25,13 @@
 }
 
 delete_urls = {
-    "assistants_id_url": f"http://leapfrogai-api.uds.dev/openai/v1/assistants/{test_id}",
-    "files_specific_url": f"http://leapfrogai-api.uds.dev/openai/v1/files/{test_id}",
+    "assistants_id_url": f"https://leapfrogai-api.uds.dev/openai/v1/assistants/{test_id}",
+    "files_specific_url": f"https://leapfrogai-api.uds.dev/openai/v1/files/{test_id}",
 }
 
-# This is the anon_key for supabase, it provides access to the endpoints that would otherwise be inaccessible
-anon_key = os.environ["ANON_KEY"]
 
-test_email: str = "fakeuser@test.com"
-test_password: str = "password"
+# We need a jwt token that is properly decodeable but invalid in regards to not being a token for a valid user
+invalid_jwt_token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c"
 
 mock_assistant_body = {
     "name": "Test Assistant",
@@ -49,46 +47,13 @@
 }
 
 
-def create_test_user():
-    headers = {
-        "apikey": f"{anon_key}",
-        "Authorization": f"Bearer {anon_key}",
-        "Content-Type": "application/json",
-    }
-    requests.post(
-        url="https://supabase-kong.uds.dev/auth/v1/signup",
-        headers=headers,
-        json={
-            "email": test_email,
-            "password": test_password,
-            "confirmPassword": test_password,
-        },
-    )
-
-
-def get_jwt_token(api_key: str):
-    url = "https://supabase-kong.uds.dev/auth/v1/token?grant_type=password"
-    headers = {"apikey": f"{api_key}", "Content-Type": "application/json"}
-    data = {"email": test_email, "password": test_password}
-
-    response = requests.post(url, headers=headers, json=data)
-
-    if response.status_code != 200:
-        pytest.fail(
-            f"Request for the JWT token failed with status code {response.status_code} expected 200",
-            False,
-        )
-
-    return json.loads(response.content)["access_token"]
-
-
 def verify_request(
-    urls: dict[str, str], request_type: str, jwt_token: str, legitimate: True
+    urls: dict[str, str], request_type: str, jwt_token: str, legitimate: bool
 ):
     headers = (
         {"Authorization": f"Bearer {jwt_token}"}
         if legitimate
-        else {"Authorization": "Bearer faketoken"}
+        else {"Authorization": f"Bearer {invalid_jwt_token}"}
     )
 
     # Verify that legitimate requests are not forbidden
@@ -116,19 +81,26 @@ def verify_request(
             elif request_type == "delete":
                 response = requests.delete(urls[url], headers=headers)
 
-            if legitimate and response.status_code == 403:
+            # 'legitimate' requests should never return a 401 or a 403
+            if legitimate and (
+                response.status_code == 401 or response.status_code == 403
+            ):
                 response.raise_for_status()
 
-            if not legitimate and response.status_code != 403:
+            # 'illegitimate' requests should always return a 401 or a 403
+            if (
+                not legitimate
+                and response.status_code != 401
+                and response.status_code != 403
+            ):
                 raise Exception("An illegitimate request has been allowed")
 
         except requests.exceptions.RequestException:
             pytest.fail(f"Request failed with status code {response.status_code}", True)
 
 
 def test_api_row_level_security():
-    create_test_user()
-    jwt_token = get_jwt_token(anon_key)
+    jwt_token = create_test_user()
 
     # Confirm that legitimate requests are allowed
     verify_request(get_urls, "get", jwt_token, True)

diff --git a/tests/e2e/test_llama.py b/tests/e2e/test_llama.py
@@ -3,9 +3,10 @@
 import pytest
 from openai import InternalServerError, OpenAI
 
+from .utils import create_test_user
+
 client = OpenAI(
-    base_url="https://leapfrogai-api.uds.dev/openai/v1",
-    api_key="Free the models",
+    base_url="https://leapfrogai-api.uds.dev/openai/v1", api_key=create_test_user()
 )
 
 model_name = "llama-cpp-python"

diff --git a/tests/e2e/test_supabase.py b/tests/e2e/test_supabase.py
@@ -1,22 +1,18 @@
 import requests
-import os
+
+from .utils import ANON_KEY
 
 health_urls = {
     "auth_health_url": "http://supabase-kong.uds.dev/auth/v1/health",
     "rest_health_url": "http://supabase-kong.uds.dev/rest/v1/",
     "storage_health_url": "http://supabase-kong.uds.dev/storage/v1/status",
 }
 
-# This is the Supabase anon key, it provides access to the health endpoints that would otherwise be inaccessible
-anon_api_key = os.environ["API_KEY"]
-
 
 def test_studio():
     try:
         for url_name in health_urls:
-            response = requests.get(
-                health_urls[url_name], headers={"apikey": anon_api_key}
-            )
+            response = requests.get(health_urls[url_name], headers={"apikey": ANON_KEY})
             response.raise_for_status()
     except requests.exceptions.RequestException as e:
         print(f"Error: Request failed with status code {response.status_code}")

diff --git a/tests/e2e/test_text_embeddings.py b/tests/e2e/test_text_embeddings.py
@@ -3,9 +3,10 @@
 import pytest
 from openai import InternalServerError, OpenAI
 
+from .utils import create_test_user
+
 client = OpenAI(
-    base_url="https://leapfrogai-api.uds.dev/openai/v1",
-    api_key="Free the models",
+    base_url="https://leapfrogai-api.uds.dev/openai/v1", api_key=create_test_user()
 )
 
 model_name = "text-embeddings"

diff --git a/tests/e2e/test_whisper.py b/tests/e2e/test_whisper.py
@@ -3,9 +3,10 @@
 import pytest
 from openai import InternalServerError, OpenAI
 
+from .utils import create_test_user
+
 client = OpenAI(
-    base_url="https://leapfrogai-api.uds.dev/openai/v1",
-    api_key="Free the models",
+    base_url="https://leapfrogai-api.uds.dev/openai/v1", api_key=create_test_user()
 )
 
 

diff --git a/tests/e2e/utils.py b/tests/e2e/utils.py
@@ -0,0 +1,54 @@
+import json
+import os
+
+import pytest
+import requests
+
+# This is the anon_key for supabase, it provides access to the endpoints that would otherwise be inaccessible
+ANON_KEY = os.environ["ANON_KEY"]
+
+DEFAULT_TEST_EMAIL = "fakeuser1@test.com"
+DEFAULT_TEST_PASSWORD = "password"
+
+
+def create_test_user(
+    anon_key: str = ANON_KEY,
+    email: str = DEFAULT_TEST_EMAIL,
+    password: str = DEFAULT_TEST_PASSWORD,
+) -> str:
+    headers = {
+        "apikey": f"{anon_key}",
+        "Authorization": f"Bearer {anon_key}",
+        "Content-Type": "application/json",
+    }
+
+    requests.post(
+        url="https://supabase-kong.uds.dev/auth/v1/signup",
+        headers=headers,
+        json={
+            "email": email,
+            "password": password,
+            "confirmPassword": password,
+        },
+    )
+
+    return get_jwt_token(anon_key, email, password)
+
+
+def get_jwt_token(
+    api_key: str,
+    test_email: str = DEFAULT_TEST_EMAIL,
+    test_password: str = DEFAULT_TEST_PASSWORD,
+) -> str:
+    url = "https://supabase-kong.uds.dev/auth/v1/token?grant_type=password"
+    headers = {"apikey": f"{api_key}", "Content-Type": "application/json"}
+    data = {"email": test_email, "password": test_password}
+
+    response = requests.post(url, headers=headers, json=data)
+    if response.status_code != 200:
+        pytest.fail(
+            f"Request for the JWT token failed with status code {response.status_code} expected 200",
+            False,
+        )
+
+    return json.loads(response.content)["access_token"]