ci: preliminary ci support (#368)

* ci: preliminary ci support

* style: fix whitespace

* ci: run some horde_sdk tests too

* ci: fix use local testing API for SDK tests

* ci: show all output for `pytest` invocations

* ci: skip model list checking for now

* style: fix

.github/workflows/maintests.yml

@@ -0,0 +1,84 @@
+name: AI-Horde main tests
+
+on:
+    push:
+      branches:
+        - main
+      paths:
+        - '**.py'
+        - '**.json'
+        - 'tox.ini'
+        - '.github/workflows/maintests.yml'
+        - '.github/workflows/prtests.yml'
+        - '.github/workflows/release.yml'
+
+jobs:
+  runner-job:
+    runs-on: ubuntu-latest
+    # runs-on: self-hosted
+    env:
+      POSTGRES_URL: "localhost:5432/horde_test"
+      POSTGRES_PASS: "postgres"
+      PGPASSWORD: "postgres"
+      REDIS_IP: "localhost"
+      REDIS_SERVERS: '["localhost"]'
+      USE_SQLITE: 0
+      ADMINS: '["test_user#1"]'
+      R2_TRANSIENT_ACCOUNT: ${{ secrets.R2_TRANSIENT_ACCOUNT }}
+      R2_PERMANENT_ACCOUNT: ${{ secrets.R2_PERMANENT_ACCOUNT }}
+      SHARED_AWS_ACCESS_ID: ${{ secrets.SHARED_AWS_ACCESS_ID }}
+      SHARED_AWS_ACCESS_KEY: ${{ secrets.SHARED_AWS_ACCESS_KEY }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      KUDOS_TRUST_THRESHOLD: 100
+      AI_HORDE_DEV_URL: "http://localhost:7001/api/" # For horde_sdk tests
+
+    services:
+      postgres:
+        image: postgres:15.6-bullseye
+        env:
+          POSTGRES_PASSWORD: postgres
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+      redis:
+        image: redis
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+          # cache: 'pip'
+      - run: python -m pip install --upgrade pip wheel setuptools
+      - name: Install and run lint/format checkers
+        run: |
+          python -m pip install -r requirements.dev.txt 
+          black --check .
+          ruff .
+      - name: Install and run tests
+        run: |
+          python -m pip install -r requirements.txt
+          psql -h localhost -U postgres -c "CREATE DATABASE horde_test;"
+          python server.py -vvvvi --horde stable &
+          sleep 5
+          curl -X POST --data-raw 'username=test_user' http://localhost:7001/register | grep -Po '<p style="background-color:darkorange;">\K.*(?=<\/p>)' > tests/apikey.txt
+          export AI_HORDE_DEV_APIKEY=$(cat tests/apikey.txt)
+          pytest tests/ -s
+          python -m pip download --no-deps --no-binary :all: horde_sdk
+          tar -xvf horde_sdk-*.tar.gz
+          cd horde_sdk**/
+          pytest tests/ --ignore-glob=*api_calls.py --ignore-glob=*test_model_meta.py -s

.gitignore

@@ -136,4 +136,10 @@ horde.log
 horde*.bz2
 horde.db
 /.idea
-/boto3oeo.py
+/boto3oeo.py
+
+
+apikey.txt
+.vscode/
+
+horde_sdk**

horde/apis/models/v2.py

@@ -1,6 +1,8 @@
 from flask_restx import fields, reqparse
+
 from horde.exceptions import KNOWN_RC
 
+
 class Parsers:
     def __init__(self):
         self.generate_parser = reqparse.RequestParser()

horde/classes/base/processing_generation.py

@@ -25,7 +25,7 @@ class ProcessingGeneration(db.Model):
     id = db.Column(uuid_column_type(), primary_key=True, default=get_db_uuid)
     procgen_type = db.Column(db.String(30), nullable=False, index=True)
     generation = db.Column(db.Text)
-    gen_metadata = db.Column(json_column_type, nullable=False)
+    gen_metadata = db.Column(json_column_type, nullable=True)
 
     model = db.Column(db.String(255), default="", nullable=False)
     seed = db.Column(db.BigInteger, default=0, nullable=False)

horde/database/threads.py

@@ -295,6 +295,9 @@ def prune_stats():
 def store_patreon_members():
     api_client = patreon.API(os.getenv("PATREON_CREATOR_ACCESS_TOKEN"))
     # campaign_id = api_client.get_campaigns(10).data()[0].id()
+    if api_client is None:
+        logger.error("Failed to get patreon API client")
+        return
     cursor = None
     members = []
     while True:

horde/exceptions.py

@@ -111,6 +111,7 @@
     "Locked",
 ]
 
+
 class BadRequest(wze.BadRequest):
     def __init__(self, message, log=None, rc="BadRequest"):
         self.specific = message

horde/flask.py

@@ -22,6 +22,7 @@
     HORDE.config["SQLALCHEMY_ENGINE_OPTIONS"] = {
         "pool_size": 50,
         "max_overflow": -1,
+        # "pool_pre_ping": True,
     }
 HORDE.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
 db = SQLAlchemy(HORDE)

horde/model_reference.py

@@ -130,3 +130,4 @@ def has_nsfw_models(self, model_names):
 
 
 model_reference = ModelReference(3600, None)
+model_reference.call_function()

horde/patreon.py

@@ -17,6 +17,8 @@ def call_function(self):
             # logger.debug(self.patrons)
         except (TypeError, AttributeError):
             logger.warning("Patreon cache could not be retrieved from redis. Leaving existing cache.")
+        except Exception as e:
+            logger.error(f"Error retrieving patreon cache from redis: {e}")
 
     def is_patron(self, user_id):
         return user_id in self.patrons

requirements.dev.txt

@@ -1,4 +1,5 @@
 black==23.12.1
 ruff==0.1.13
+pytest==8.0.0
 tox~=4.12.1
 horde_sdk>=0.7.29

requirements.txt

@@ -2,7 +2,7 @@ werkzeug~=2.2.2
 Flask~=2.2.2
 flask-restx
 flask_limiter~=2.8.1
-Flask-Caching   
+Flask-Caching
 waitress~=2.1.2
 requests >= 2.27
 Markdown~=3.4.1
@@ -28,4 +28,4 @@ git+https://github.com/Patreon/patreon-python.git
 torch
 emoji
 semver >= 3.0.2
-numpy ~= 1.24.1 # better_profanity fails on later versions of numpy
+numpy ~= 1.24.1 # better_profanity fails on later versions of numpy

tests/conftest.py

@@ -0,0 +1,45 @@
+import pathlib
+
+import pytest
+import requests
+
+
+@pytest.fixture
+def CIVERSION() -> str:
+    return "0.1.1"
+
+
+@pytest.fixture
+def HORDE_URL() -> str:
+    return "localhost:7001"
+
+
+@pytest.fixture
+def api_key() -> str:
+    key_file = pathlib.Path(__file__).parent / "apikey.txt"
+    if key_file.exists():
+        return key_file.read_text().strip()
+
+    raise ValueError("No api key file found")
+
+
+@pytest.fixture(autouse=True)
+def increase_kudos(api_key: str, HORDE_URL: str, CIVERSION: str) -> None:
+    headers = {"apikey": api_key, "Client-Agent": f"aihorde_ci_client:{CIVERSION}:(discord)db0#1625", "user_id": "1"}
+
+    payload_set_to_mod = {
+        "trusted": True,
+        "moderator": True,
+    }
+
+    response_set_to_mod = requests.put(f"http://{HORDE_URL}/api/v2/users/1", json=payload_set_to_mod, headers=headers)
+
+    assert response_set_to_mod.ok, response_set_to_mod.text
+
+    payload_set_kudos = {
+        "kudos": 10000,
+    }
+
+    response_kudos = requests.put(f"http://{HORDE_URL}/api/v2/users/1", json=payload_set_kudos, headers=headers)
+
+    assert response_kudos.ok, response_kudos.text

tests/test_alchemy.py

@@ -1,20 +1,16 @@
 import requests
 
-CIVERSION = "0.1.1"
-HORDE_URL = "dev.stablehorde.net"
-TEST_MODELS = ["elinas/chronos-70b-v2"]
 
-
-def test_simple_alchemy() -> None:
-    headers = {"apikey": "2bc5XkMeLAWiN9O5s7bhfg", "Client-Agent": f"aihorde_ci_client:{CIVERSION}:(discord)db0#1625"}  # ci/cd user
+def test_simple_alchemy(api_key: str, HORDE_URL: str, CIVERSION: str) -> None:
+    headers = {"apikey": api_key, "Client-Agent": f"aihorde_ci_client:{CIVERSION}:(discord)db0#1625"}  # ci/cd user
     async_dict = {
         "forms": [
             {"name": "caption"},
         ],
         "source_image": "https://github.com/Haidra-Org/AI-Horde/blob/main/icon.png?raw=true",
     }
-    async_req = requests.post(f"https://{HORDE_URL}/api/v2/interrogate/async", json=async_dict, headers=headers)
-    assert async_req.ok
+    async_req = requests.post(f"http://{HORDE_URL}/api/v2/interrogate/async", json=async_dict, headers=headers)
+    assert async_req.ok, async_req.text
     async_results = async_req.json()
     req_id = async_results["id"]
     # print(async_results)
@@ -25,27 +21,27 @@ def test_simple_alchemy() -> None:
         "max_tiles": 96,
     }
     try:
-        pop_req = requests.post(f"https://{HORDE_URL}/api/v2/interrogate/pop", json=pop_dict, headers=headers)
+        pop_req = requests.post(f"http://{HORDE_URL}/api/v2/interrogate/pop", json=pop_dict, headers=headers)
     except Exception:
-        requests.delete(f"https://{HORDE_URL}/api/v2/interrogate/status/{req_id}", headers=headers)
+        requests.delete(f"http://{HORDE_URL}/api/v2/interrogate/status/{req_id}", headers=headers)
         raise
-    assert pop_req.ok
+    assert pop_req.ok, pop_req.text
     pop_results = pop_req.json()
     # print(json.dumps(pop_results, indent=4))
 
     job_id = pop_results["forms"][0]["id"]
-    assert job_id is not None
+    assert job_id is not None, pop_results
     submit_dict = {
         "id": job_id,
         "result": {"caption": "Test"},
         "state": "ok",
     }
-    submit_req = requests.post(f"https://{HORDE_URL}/api/v2/interrogate/submit", json=submit_dict, headers=headers)
-    assert submit_req.ok
+    submit_req = requests.post(f"http://{HORDE_URL}/api/v2/interrogate/submit", json=submit_dict, headers=headers)
+    assert submit_req.ok, submit_req.text
     submit_results = submit_req.json()
     assert submit_results["reward"] > 0
-    retrieve_req = requests.get(f"https://{HORDE_URL}/api/v2/interrogate/status/{req_id}", headers=headers)
-    assert retrieve_req.ok
+    retrieve_req = requests.get(f"http://{HORDE_URL}/api/v2/interrogate/status/{req_id}", headers=headers)
+    assert retrieve_req.ok, retrieve_req.text
     retrieve_results = retrieve_req.json()
     # print(json.dumps(retrieve_results,indent=4))
     assert len(retrieve_results["forms"]) == 1

tests/test_image.py

@@ -1,12 +1,10 @@
 import requests
 
-CIVERSION = "0.1.1"
-HORDE_URL = "dev.stablehorde.net"
 TEST_MODELS = ["Fustercluck", "AlbedoBase XL (SDXL)"]
 
 
-def test_simple_image_gen() -> None:
-    headers = {"apikey": "2bc5XkMeLAWiN9O5s7bhfg", "Client-Agent": f"aihorde_ci_client:{CIVERSION}:(discord)db0#1625"}  # ci/cd user
+def test_simple_image_gen(api_key: str, HORDE_URL: str, CIVERSION: str) -> None:
+    headers = {"apikey": api_key, "Client-Agent": f"aihorde_ci_client:{CIVERSION}:(discord)db0#1625"}  # ci/cd user
     async_dict = {
         "prompt": "a horde of cute stable robots in a sprawling server room repairing a massive mainframe",
         "nsfw": True,
@@ -22,8 +20,8 @@ def test_simple_image_gen() -> None:
         "models": TEST_MODELS,
         "loras": [{"name": "247778", "is_version": True}],
     }
-    async_req = requests.post(f"https://{HORDE_URL}/api/v2/generate/async", json=async_dict, headers=headers)
-    assert async_req.ok
+    async_req = requests.post(f"http://{HORDE_URL}/api/v2/generate/async", json=async_dict, headers=headers)
+    assert async_req.ok, async_req.text
     async_results = async_req.json()
     req_id = async_results["id"]
     # print(async_results)
@@ -40,25 +38,25 @@ def test_simple_image_gen() -> None:
         "allow_controlnet": True,
         "allow_lora": True,
     }
-    pop_req = requests.post(f"https://{HORDE_URL}/api/v2/generate/pop", json=pop_dict, headers=headers)
-    assert pop_req.ok
+    pop_req = requests.post(f"http://{HORDE_URL}/api/v2/generate/pop", json=pop_dict, headers=headers)
+    assert pop_req.ok, pop_req.text
     pop_results = pop_req.json()
     # print(json.dumps(pop_results, indent=4))
 
     job_id = pop_results["id"]
-    assert job_id is not None
+    assert job_id is not None, pop_results
     submit_dict = {
         "id": job_id,
         "generation": "R2",
         "state": "ok",
         "seed": 0,
     }
-    submit_req = requests.post(f"https://{HORDE_URL}/api/v2/generate/submit", json=submit_dict, headers=headers)
-    assert submit_req.ok
+    submit_req = requests.post(f"http://{HORDE_URL}/api/v2/generate/submit", json=submit_dict, headers=headers)
+    assert submit_req.ok, submit_req.text
     submit_results = submit_req.json()
     assert submit_results["reward"] > 0
-    retrieve_req = requests.get(f"https://{HORDE_URL}/api/v2/generate/status/{req_id}", headers=headers)
-    assert retrieve_req.ok
+    retrieve_req = requests.get(f"http://{HORDE_URL}/api/v2/generate/status/{req_id}", headers=headers)
+    assert retrieve_req.ok, retrieve_req.text
     retrieve_results = retrieve_req.json()
     # print(json.dumps(retrieve_results,indent=4))
     assert len(retrieve_results["generations"]) == 1