Adding self test for the get_memory function

.github/workflows/get_memory_test.yml

@@ -0,0 +1,96 @@
+name: Recipes server tests
+
+on: [push]
+
+jobs:
+  test:
+      runs-on: ubuntu-latest
+      environment: "GitHub Actions 1"
+      env: 
+        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        ASSISTANTS_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        ASSISTANTS_API_TYPE: ${{ secrets.ASSISTANTS_API_TYPE }}
+        ASSISTANTS_ID: ${{ secrets.ASSISTANTS_ID }} 
+        ASSISTANTS_BASE_URL: ${{ secrets.ASSISTANTS_BASE_URL }} 
+        ASSISTANTS_MODEL: ${{ secrets.ASSISTANTS_MODEL }} 
+        ASSISTANTS_BOT_NAME: ${{ secrets.ASSISTANTS_BOT_NAME }} 
+
+        POSTGRES_DATA_HOST: ${{ secrets.POSTGRES_DATA_HOST }} 
+        POSTGRES_DATA_PORT: ${{ secrets.POSTGRES_DATA_PORT }} 
+        POSTGRES_DATA_DB: ${{ secrets.POSTGRES_DATA_DB }} 
+        POSTGRES_DATA_USER: ${{ secrets.POSTGRES_DATA_USER }} 
+        POSTGRES_DATA_PASSWORD: ${{ secrets.POSTGRES_DATA_PASSWORD }} 
+
+        DATA_DB_CONN_STRING: ${{ secrets.DATA_DB_CONN_STRING }}
+
+        POSTGRES_RECIPE_HOST: ${{ secrets.POSTGRES_RECIPE_HOST }} 
+        POSTGRES_RECIPE_PORT: ${{ secrets.POSTGRES_RECIPE_PORT }} 
+        POSTGRES_RECIPE_DB: ${{ secrets.POSTGRES_RECIPE_DB }} 
+        POSTGRES_RECIPE_USER: ${{ secrets.POSTGRES_RECIPE_USER }} 
+        POSTGRES_RECIPE_PASSWORD: ${{ secrets.POSTGRES_RECIPE_PASSWORD }} 
+
+        RECIPE_DB_CONN_STRING: "postgresql://${{ secrets.POSTGRES_RECIPE_USER }}:${{ secrets.POSTGRES_RECIPE_PASSWORD }}@${{ secrets.POSTGRES_RECIPE_HOST }}:${{ secrets.POSTGRES_RECIPE_PORT }}/${{ secrets.POSTGRES_RECIPE_DB }}"
+
+        RECIPES_OPENAI_API_TYPE: ${{ secrets.RECIPES_OPENAI_API_TYPE }} 
+        RECIPES_OPENAI_API_KEY: ${{ secrets.RECIPES_OPENAI_API_KEY }} 
+        RECIPES_MODEL: ${{ secrets.RECIPES_MODEL }} 
+        RECIPES_OPENAI_TEXT_COMPLETION_DEPLOYMENT_NAME: ${{ secrets.RECIPES_OPENAI_TEXT_COMPLETION_DEPLOYMENT_NAME }} 
+        RECIPES_MEMORY_SIMILARITY_CUTOFF: ${{ secrets.RECIPES_MEMORY_SIMILARITY_CUTOFF }} 
+        RECIPES_RECIPE_SIMILARITY_CUTOFF: ${{ secrets.RECIPES_RECIPE_SIMILARITY_CUTOFF }} 
+        RECIPES_HELPER_FUNCTION_SIMILARITY_CUTOFF: ${{ secrets.RECIPES_HELPER_FUNCTION_SIMILARITY_CUTOFF }} 
+        RECIPES_MODEL_TEMP: ${{ secrets.RECIPES_MODEL_TEMP }} 
+        RECIPES_MODEL_MAX_TOKENS: ${{ secrets.RECIPES_MODEL_MAX_TOKENS }} 
+
+        IMAGE_HOST: ${{ secrets.IMAGE_HOST }} 
+        RECIPE_SERVER_API: ${{ secrets.RECIPE_SERVER_API }} 
+
+        CHAINLIT_AUTH_SECRET: ${{ secrets.CHAINLIT_AUTH_SECRET }} 
+        USER_LOGIN: ${{ secrets.USER_LOGIN }} 
+        USER_PASSWORD: ${{ secrets.USER_PASSWORD }} 
+
+        COLUMNS: 150
+
+      steps:
+
+        - name: Checkout
+          uses: actions/checkout@v3
+
+        #- name: Checkout integration tests data
+        #  uses: actions/checkout@master
+        #  with:
+        #    repository: datakind/recipes-ai-test-data
+        #    ssh-key: ${{ secrets.GITHUB_SSH_PRIVATE_KEY}} 
+        #    path: recipes-ai-test-data
+
+        - name: Expose GitHub Runtime
+          uses: crazy-max/ghaction-github-runtime@v2
+
+        - name: Spin up DB and recipes server
+          run: |
+            env > .env 
+
+            echo "Installing demo data ..."
+            pip3 install gdown==5.2.0
+            cd data && python3 download_demo_data.py && cd ..
+            ls data/datadb
+
+            echo "Starting docker containers for dbs and server ..."
+            docker-compose -f ./docker-compose-github.yml pull
+            docker-compose -f ./docker-compose-github.yml up -d --build 
+            echo "logs datadb ..."
+            docker-compose -f docker-compose-github.yml  logs datadb
+            docker ps
+
+        - name: Run tests
+          uses: actions/setup-python@v4
+          with:
+            python-version: "3.11.4"
+        - run: |
+            pip3 install pytest==8.2.2
+            pip3 install requests==2.32.3
+            pip3 install python-dotenv==1.0.1
+
+        - run: |
+            echo "exec into container ..."
+            docker exec recipes-ai-server bash -c "cd tests/ && pytest"
+

docker-compose-github.yml

@@ -66,8 +66,8 @@ services:
        - ./templates:/app/templates
        - ./utils:/app/utils
        - ./management/skills.py:/app/recipes/skills.py
-
-
+       - ./tests:/app/tests
+       - ./utils:/app/tests/utils
 
 volumes:
   pgdata2:

requirements.txt

@@ -28,3 +28,4 @@ robocorp-actions
 robocorp-truststore
 seaborn==0.13.2
 sqlalchemy==2.0.30
+pytest==8.2.2

server/fastapi/Dockerfile

@@ -14,6 +14,7 @@ COPY ./templates /app/templates
 COPY ./utils /app/utils
 COPY ./management/skills.py /app/recipes/skills.py
 COPY requirements.txt /app
+COPY ../../utils /app/tests/utils
 
 # Install any needed packages specified in requirements.txt.
 RUN pip install --upgrade pip

tests/test_cases_get_memory.json

@@ -0,0 +1,66 @@
+{
+  "tests": [
+    {
+      "test_case": "Test Case 1",
+      "test_case_description": "Test to evaluate a query that returns a text response",
+      "user_input": "Get all recipes",
+      "chat_history": "[{'role': 'user', 'content': 'Get all recipes'}]",
+      "generate_intent": "true",
+      "expected_output": {
+        "result": {
+          "type": "text",
+          "file": "",
+          "value": "|    | type   | intent                                                                                                                                               |\n|---:|:-------|:-----------------------------------------------------------------------------------------------------------------------------------------------------|\n|  0 | recipe | plot a line chart of conflict events by month for a country using HDX data as an image                                                               |\n|  1 | recipe | plot a scatterplot of food price movements and number of fatalities by country using HDXData data, including regression line as an image             |\n|  2 | recipe | plot a line chart of fatalities by month for a country using HDX data as an image                                                                    |\n|  3 | recipe | plot a line chart of commodity prices monthly relative change for a country from 2008-01-01 using HDX data as an image                               |\n|  4 | recipe | plot a bar chart of humanitarian organizations by sector for a given region using Humanitarian Data Exchange data as an image                        |\n|  5 | recipe | provide a list of organizations providing food security for a region in a country                                                                    |\n|  6 | recipe | provide the total population of a provided country using HDX data as text                                                                            |\n|  7 | recipe | plot population pyramids by age for a country using HDX data as an image                                                                             |\n|  8 | recipe | plot a scatterplot of food price movements and number of fatalities by country using HDXData data, including regression line as an image             |\n|  9 | recipe | plot a map of population by admin1 or admin2 for a country using HAPI data as an image                                                               |\n| 10 | recipe | plot a line chart of commodity prices monthly relative change for a country from 2008-01-01 using HDX data as an image                               |\n| 11 | recipe | List organizations in top 3 states by population in provided IPC Phase in a country, using HAPI data                                                 |\n| 12 | recipe | plot a bar chart of humanitarian organizations by sector for a given region using Humanitarian Data Exchange data as an image                        |\n| 13 | recipe | provide a text summary of metadata by subnational region using HAPI data as text                                                                     |\n| 14 | recipe | get all recipes                                                                                                                                      |\n| 15 | recipe | plot a map of IPC phase data by admin_1 using HDX data as an image                                                                                   |\n|  0 | memory | plot a line chart of conflict events by month for Chad using HDX data as an image                                                                    |\n|  1 | memory | plot a scatterplot of food price movements and number of fatalities in TCD from 2008-01-01 using HDXData data, including regression line as an image |\n|  2 | memory | plot a line chart of fatalities by month for Chad using HDX data as an image                                                                         |\n|  3 | memory | plot a line chart of commodity prices monthly relative change for Chad from 2008-01-01 using HDX data as an image                                    |\n|  4 | memory | plot a bar chart of humanitarian organizations in Wadi Fira by sector using Humanitarian Data Exchange data as an image                              |\n|  5 | memory | provide a list of organizations providing food security in Wadi Fira, Chad                                                                           |\n|  6 | memory | provide the total population of Mali using HDX data as text                                                                                          |\n|  7 | memory | plot population pyramids by age for Chad using HDX data as an image                                                                                  |\n|  8 | memory | plot a scatterplot of food price movements and number of fatalities in TCD from 2008-01-01 using HDXData data, including regression line as an image |\n|  9 | memory | plot a map of population by admin1 for Haiti using HAPI data as an image                                                                             |\n| 10 | memory | plot a line chart of commodity prices monthly relative change for Chad from 2008-01-01 using HDX data as an image                                    |\n| 11 | memory | List organizations in the top 3 states by population in IPC Phase 3+ in Chad, using HAPI data                                                        |\n| 12 | memory | plot a bar chart of humanitarian organizations in Wadi Fira by sector using Humanitarian Data Exchange data as an image                              |\n| 13 | memory | provide a text summary of metadata for Wadi Fira using HAPI data as text                                                                             |\n| 14 | memory | retrieve all recipes                                                                                                                                 |\n| 15 | memory | plot a map of IPC phase 3 data by admin_1 in Chad using HDX data as an image                                                                         |"
+        },
+        "metadata": "{\"params\": {}, \"attribution\": \"\", \"data_url\": \"\", \"time_period\": {\"start\": \"\", \"end\": \"\"}}",
+        "memory_type": "memory",
+        "memory": "retrieve all recipes",
+        "memory_found": "true"
+      }
+    },
+    {
+      "test_case": "Test Case 2",
+      "test_case_description": "Test to evaluate a query that returns a text response",
+      "user_input": "provide the total population of Mali using HDX data as text",
+      "chat_history": "[{'role': 'user', 'content': 'Get all recipes'}, {'role': 'user', 'content': 'provide the total population of Mali using HDX data as text'}]",
+      "generate_intent": "true",
+      "expected_output": {
+        "result": {
+          "type": "number",
+          "file": "",
+          "value": "17907114"
+        },
+        "metadata": "{\"params\": {\"country_code\": \"MLI\"}, \"attribution\": \"https://data.humdata.org/dataset/ce21c7db-d8f0-40f8-adc2-452d2d2d105c\", \"data_url\": \"https://data.humdata.org/dataset/ce21c7db-d8f0-40f8-adc2-452d2d2d105c/resource/6f243ba2-4d4a-4663-a7c4-e917dbbde73a/download/mli_pop_adm0_v2.csv\", \"time_period\": {\"start\": \"2018-01-01\", \"end\": \"2018-12-31T23:59:59\"}}",
+        "memory_type": "memory",
+        "memory": "provide the total population of Mali using HDX data as text",
+        "memory_found": "true"
+      }
+    },
+    {
+      "test_case": "Test Case 3",
+      "test_case_description": "Test to evaluate a query that returns an image response",
+      "user_input": "plot a bar chart of humanitarian organizations in Wadi Fira by sector using Humanitarian Data Exchange data as an image",
+      "chat_history": "[{'role': 'user', 'content': 'Get all recipes'}, {'role': 'user', 'content': 'provide the total population of Mali using HDX data as text'}, {'role': 'user', 'content': 'plot a bar chart of humanitarian organizations in Wadi Fira by sector using Humanitarian Data Exchange data as an image'}]",
+      "generate_intent": "true",
+      "expected_output": {
+        "result": {
+          "type": "image",
+          "file": "http://localhost:8000/public/images/memory_image_e9d3e3ee-977f-4291-a51a-7e4e3e4cd5f3.png",
+          "value": ""
+        },
+        "metadata": "{\"params\": {\"region\": \"Wadi Fira\"}, \"attribution\": \"https://data.humdata.org/dataset/682c3db6-e253-430f-a3f3-305ef079e2de\", \"data_url\": \"https://data.humdata.org/dataset/682c3db6-e253-430f-a3f3-305ef079e2de/resource/1e3ba1f4-2dbc-4f46-9ce7-2274e157188e/download/3w-tcd-20240508.xlsx\", \"time_period\": {\"start\": \"2023-12-01\", \"end\": \"2024-05-31T23:59:59.999999\"}}",
+        "memory_type": "memory",
+        "memory": "plot a bar chart of humanitarian organizations in Wadi Fira by sector using Humanitarian Data Exchange data as an image",
+        "memory_found": "true"
+      }
+    },
+    {
+      "test_case": "Test Case 4",
+      "test_case_description": "Test to evaluate a query that does not return a memory",
+      "user_input": "Plot the distribution of internet access in Chad",
+      "chat_history": "[{'role': 'user', 'content': 'Get all recipes'}, {'role': 'user', 'content': 'provide the total population of Mali using HDX data as text'}, {'role': 'user', 'content': 'plot a bar chart of humanitarian organizations in Wadi Fira by sector using Humanitarian Data Exchange data as an image'}, {'role': 'user', 'content': 'Plot the distribution of internet access in Chad'}]",
+      "generate_intent": "true",
+      "expected_output": {"result":"Sorry, no recipe or memory found","memory_found":"false"}
+    }
+  ]
+}

tests/test_get_memory.py

@@ -0,0 +1,30 @@
+import json
+
+import pytest
+
+from utils.general import call_execute_query_api, call_get_memory_recipe_api
+
+
+# load json file into variable and print it
+@pytest.fixture
+def get_test_cases():
+    """
+    Loads test cases from test_cases_get_memory.json.
+    """
+    with open("test_cases_get_memory.json") as f:
+        test_data = json.load(f)
+        return test_data
+
+
+def test_get_memory_recipe(get_test_cases):
+    """
+    Tests the get memory recipe API endpoint.
+    """
+
+    for test in get_test_cases.get("tests", []):
+        user_input = test["user_input"]
+        chat_history = test["chat_history"]
+        generate_intent = test["generate_intent"]
+        expected_output = test["expected_output"]
+        response = call_get_memory_recipe_api(user_input, chat_history, generate_intent)
+        assert response == expected_output

utils/general.py

@@ -5,7 +5,6 @@
 import sys
 import warnings
 
-import pandas as pd
 import requests
 from dotenv import load_dotenv