Merge pull request #1 from AIPowerGrid/urek

Upgrading Hordelib
AIPowerGrid · Dec 6, 2024 · 5307ac9 · 5307ac9
2 parents b741b4d + 7dffec8
commit 5307ac9
Show file tree

Hide file tree

Showing 213 changed files with 33,308 additions and 5,943 deletions.
diff --git a/.github/workflows/maintests.yml b/.github/workflows/maintests.yml
@@ -1,4 +1,4 @@
-name: Tests
+name: Main Tests
 
 on:
   push:
@@ -18,6 +18,8 @@ jobs:
       - uses: actions/checkout@v3
       - name: Run pre-commit
         uses: pre-commit/action@v3.0.0
+        with:
+          extra_args: --all-files
 
   build:
     runs-on: self-hosted
@@ -38,8 +40,8 @@ jobs:
       - name: Install tox and any other packages
         run: |
           python -m pip install --upgrade pip
-          pip install --upgrade -r requirements.dev.txt
-      - name: Check build_helper.py hordelib imports have no breaking dependency changes
+          pip install --upgrade tox
+      - name: Check build_helper.py horde-engine imports have no breaking dependency changes
         run: tox -e test-build-helper
       - name: Build unit test environment, confirm CUDA is available on host
         run: tox -e tests -- -k test_cuda

diff --git a/.github/workflows/prtests.yml b/.github/workflows/prtests.yml
@@ -1,4 +1,4 @@
-name: Unstable Tests
+name: Pull Request Tests
 
 on:
   pull_request_target:
@@ -19,8 +19,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
       - name: Run pre-commit
         uses: pre-commit/action@v3.0.0
+        with:
+          extra_args: --all-files
 
   build:
     runs-on: self-hosted
@@ -43,8 +47,8 @@ jobs:
       - name: Install tox and any other packages
         run: |
           python -m pip install --upgrade pip
-          pip install --upgrade -r requirements.dev.txt
-      - name: Check build_helper.py hordelib imports have no breaking dependency changes
+          pip install --upgrade tox
+      - name: Check build_helper.py horde-engine imports have no breaking dependency changes
         run: tox -e test-build-helper
       - name: Build unit test environment, confirm CUDA is available on host
         run: tox -e tests -- -k test_cuda

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -1,18 +1,24 @@
-name: hordelib build and publish
+name: horde-engine build and publish
 
 on:
   push:
     branches:
       - releases
 
 permissions:
+  id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
   contents: write
   pull-requests: read
 
 jobs:
   build-n-publish:
-    name: hordelib build and publish
+    name: horde-engine build and publish
     runs-on: ubuntu-latest
+
+    environment: # see https://github.com/pypa/gh-action-pypi-publish/tree/release/v1/
+      name: pypi
+      url: https://pypi.org/p/horde-engine/
+
     steps:
 
     # Version bumps the project and creates a tag by creating
@@ -46,8 +52,8 @@ jobs:
         python-version: "3.10"
 
     # Install build deps
-    # NOTE If any hordelib imports used by build_helper.py are changed, or the specific modules
-    # imported from hordelib depend on a package not included here, running build_helper.py later on will fail.
+    # NOTE If any horde-engine imports used by build_helper.py are changed, or the specific modules
+    # imported from horde-engine depend on a package not included here, running build_helper.py later on will fail.
     # See `build_helper.py` for more information.
     - name: "🛠 Install pypa/build"
       if: ${{ steps.release.outputs.version != '' }}
@@ -70,8 +76,6 @@ jobs:
       with:
         add: 'CHANGELOG.md'
         message: 'ci: update changelog'
-        committer_name: GitHub Actions
-        committer_email: actions@github.com
 
     # Patches our requirements.txt and pyproject.toml
     # Build a changelog
@@ -90,8 +94,6 @@ jobs:
     - name: "📦 Publish distribution to PyPI"
       if: ${{ steps.release.outputs.version != '' }}
       uses: pypa/gh-action-pypi-publish@release/v1
-      with:
-        password: ${{ secrets.PYPI_API_TOKEN }}
 
     # Update the badges (maybe, if we're lucky)
     - name: "📦 Refresh pypi badge cache"
@@ -104,9 +106,9 @@ jobs:
         curl -X PURGE
         https://camo.githubusercontent.com/769edfb1778d4cbc3f93bc5ad0be9597bbd2d9c162cc1e9fb44172a5b660af01/68747470733a2f2f706570792e746563682f62616467652f686f7264656c6962
 
-    # - name: "Inform with Discord Webhook"
-    #   if: ${{ steps.release.outputs.version != '' }}
-    #   uses: tsickert/discord-webhook@v5.3.0
-    #   with:
-    #     webhook-url: ${{ secrets.DISCORD_WEBHOOK_URL }}
-    #     content: "New version of hordelib has been published to pypi: ${{ steps.release.outputs.version }}. You can update your worker by running `./update-runtime.sh --hordelib` (Linux) or `update-runtime.cmd --hordelib` (Windows). Changelog: https://t.ly/z2vQ"
+    - name: "Inform with Discord Webhook"
+      if: ${{ steps.release.outputs.version != '' }}
+      uses: tsickert/discord-webhook@v5.3.0
+      with:
+        webhook-url: ${{ secrets.DISCORD_WEBHOOK_URL }}
+        content: "New version of horde-engine has been published to pypi: ${{ steps.release.outputs.version }}. Changelog: https://github.com/Haidra-Org/hordelib/blob/releases/CHANGELOG.md"
diff --git a/.gitignore b/.gitignore
@@ -20,7 +20,7 @@ parts/
 sdist/
 var/
 wheels/
-models/
+hordelib/models/
 pip-wheel-metadata/
 share/python-wheels/
 *.egg-info/
@@ -151,6 +151,7 @@ hordelib/model_database/stable_diffusion.json
 hordelib/model_database/lora.json
 ComfyUI
 model.ckpt
+models/
 coverage.lcov
 profiles/
 longprompts.zip

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,22 +1,39 @@
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
     -   id: check-yaml
     -   id: end-of-file-fixer
     -   id: trailing-whitespace
 -   repo: https://github.com/psf/black
-    rev: 24.3.0
+    rev: 24.4.2
     hooks:
     -   id: black
         exclude: ^hordelib/nodes/.*\..*$
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.3.3
+    rev: v0.4.3
     hooks:
     -    id: ruff
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v1.9.0'
+    rev: 'v1.10.0'
     hooks:
     -   id: mypy
         exclude: ^examples/.*$ # FIXME
-        additional_dependencies: [pydantic, strenum, types-colorama, types-docutils, types-Pillow, types-psutil, types-Pygments, types-pywin32, types-PyYAML, types-regex, types-requests, types-setuptools, types-tabulate, types-tqdm, types-urllib3, horde_sdk==0.9.1]
+        additional_dependencies: [
+            pydantic,
+            strenum,
+            types-colorama,
+            types-docutils,
+            types-Pillow, types-psutil,
+            types-Pygments,
+            types-pywin32,
+            types-PyYAML,
+            types-regex,
+            types-requests,
+            types-setuptools,
+            types-tabulate,
+            types-tqdm,
+            types-urllib3,
+            horde_sdk==0.14.3,
+            horde_model_reference==0.9.0,
+        ]
diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-# hordelib
+# horde-engine
 
 [![PyPI Version][pypi-image]][pypi-url]
 [![Downloads][downloads-image]][downloads-url]
@@ -10,11 +10,13 @@
 [![All Models][all-model-images]][all-model-url]
 [![Release Changelog][changelog-image]][changelog-url]
 
-`hordelib` is a wrapper around [ComfyUI](https://github.com/comfyanonymous/ComfyUI) primarily to enable the [AI Horde](https://aihorde.net/) to run inference pipelines designed visually in the ComfyUI GUI.
+> Note: This project was formerly known as `hordelib`. The project namespace will be changed in the near future to reflect this change.
 
-The developers of `hordelib` can be found in the AI Horde Discord server: [https://discord.gg/3DxrhksKzn](https://discord.gg/3DxrhksKzn)
+`horde-engine` is a wrapper around [ComfyUI](https://github.com/comfyanonymous/ComfyUI) primarily to enable the [AI Horde](https://aihorde.net/) to run inference pipelines designed visually in the ComfyUI GUI.
 
-`hordelib` has been the default inference backend library of the [AI Horde](https://aihorde.net/) since `hordelib` v1.0.0.
+The developers of `horde-engine` can be found in the AI Horde Discord server: [https://discord.gg/3DxrhksKzn](https://discord.gg/3DxrhksKzn)
+
+Note that `horde-engine` (previously known as `hordelib`) has been the default inference backend library of the [AI Horde](https://aihorde.net/) since `hordelib` v1.0.0.
 
 ## Purpose
 
@@ -45,19 +47,25 @@ If you only have 16GB of RAM you will also need swap space. So if you typically
 Horde payloads can be processed simply with (for example):
 
 ```python
-import os
+# import os
+# Wherever your models are
+# os.environ["AIWORKER_CACHE_HOME"] = "f:/ai/models" # Defaults to `models/` in the current working directory
+
 import hordelib
-hordelib.initialise()
+
+hordelib.initialise()  # This must be called before any other hordelib functions
 
 from hordelib.horde import HordeLib
 from hordelib.shared_model_manager import SharedModelManager
 
-# Wherever your models are
-os.environ["AIWORKER_CACHE_HOME"] = "f:/ai/models"
-
 generate = HordeLib()
-SharedModelManager.loadModelManagers(compvis=True)
-SharedModelManager.manager.load("Deliberate")
+
+if SharedModelManager.manager.compvis is None:
+    raise Exception("Failed to load compvis model manager")
+
+SharedModelManager.manager.compvis.download_model("Deliberate")
+SharedModelManager.manager.compvis.validate_model("Deliberate")
+
 
 data = {
     "sampler_name": "k_dpmpp_2m",
@@ -79,7 +87,12 @@ data = {
     "model": "Deliberate",
 }
 pil_image = generate.basic_inference_single_image(data).image
+
+if pil_image is None:
+    raise Exception("Failed to generate image")
+
 pil_image.save("test.png")
+
 ```
 
 Note that `hordelib.initialise()` will erase all command line arguments from argv. So make sure you parse them before you call that.
@@ -88,7 +101,7 @@ See `tests/run_*.py` for more standalone examples.
 
 ### Logging
 
-If you don't want `hordelib` to setup and control the logging configuration initialise with:
+If you don't want `hordelib` to setup and control the logging configuration (we use [loguru](https://loguru.readthedocs.io/en/stable/)) initialise with:
 
 ```python
 import hordelib
@@ -111,13 +124,9 @@ Custom nodes for ComfyUI providing Controlnet preprocessing capability. Licened
 
 Custom nodes for ComfyUI providing face restoration.
 
-## DMCA Abuse
-
-On 26th May 2023 an [individual](https://github.com/hlky) issued a [DMCA takedown notice](https://github.com/github/dmca/blob/master/2023/05/2023-05-26-nataili.md) to Github against `hordelib` which claimed their name had been removed from the copyright header in the AGPL license in the 7 files listed in the takedown notice. This claim was true, and this attribution had been removed by a `hordelib` contributor prior to being committed into the `hordelib` repository.
-
-Unfortunately, it appears the individual making the DMCA claim was acting in bad faith, and even when their name was restored to the copyright attribution in the files, they persisted to press the DMCA takedown claim, which, due to the nature of the Github process, resulted in hordelib being subject to a DMCA takedown on Github.
+### [ComfyQR](https://gitlab.com/sofuego-comfy-nodes/ComfyQR)
 
-This version of `hordelib` has the 7 files mentioned in the DMCA takedown removed and replaced with alternatives and the functionality required to run the AI Horde Worker was restored on 7th June 2023.
+Nodes for generating QR codes
 
 ## Development
 
@@ -177,9 +186,9 @@ In this example we install the dependencies in the OS default environment. When
 
 `pip install -r requirements.txt --extra-index-url https://download.pytorch.org/whl/cu118 --upgrade`
 
-Ensure ComfyUI is installed and patched, one way is running the tests:
+Ensure ComfyUI is installed, one way is running the tests:
 
-`tox`
+`tox -- -k test_comfy_install`
 
 From then on to run ComfyUI:
 
@@ -243,43 +252,37 @@ python run_controlnet.py
 The `images/` directory should have our test images.
 ```
 
-### Updating the embedded version of ComfyUI
-
-We use a ComfyUI version pinned to a specific commit, see `hordelib/consts.py:COMFYUI_VERSION`
-
-To test if the latest version works and upgrade to it, from the project root simply:
+### Creating a local build of hordelib
 
-1. `cd ComfyUI` _Change CWD to the embedded comfy_
-1. `git checkout master` _Switch to master branch_
-1. `git pull` _Get the latest comfyui code_
-1. `git rev-parse HEAD` _Update the hash in `hordelib.consts:COMFYUI_VERSION`_
-1. `cd ..` _Get back to the hordelib project root_
-1. `tox` _See if everything still works_
+This is useful when testing new nodes  via the horde-reGen-worker etc
 
-Now ComfyUI is pinned to a new version.
+```bash
+python build_helper.py
+python -m build --sdist --wheel --outdir dist/ .
+python build_helper.py --fix
+```
 
-### ComfyUI Patching
+On the venv where you want to install th new version
 
-We patch the ComfyUI source code to:
+```bash
+python -m pip install /path/to/hordelib/dist/horde_engine-*.whl
+```
 
-1. Modify the model manager to allow us to dynamically move models between VRAM, RAM and disk cache.
-2. Allow make ComfyUI output some handy JSON we need for development purposes.
+### Updating the embedded version of ComfyUI
 
-To create a patch file:
-- Make the required changes to a clean install of ComfyUI and then run `git diff > yourfile.patch` then move the patch file to wherever you want to save it.
+- Change the value in `consts.py` to the desired ComfyUI version.
+- Run the test suite via `tox`
 
-Note that the patch file _really_ needs to be in UTF-8 format and some common terminals, e.g. Powershell, won't do this by default. In Powershell to create a patch file use: `git diff | Set-Content -Encoding utf8 -Path yourfile.patch`
 
-Patches can be applied with the `hordelib.install_comfyui.Installer` classes `apply_patch()` method.
 
 <!-- Badges: -->
 
 [pypi-image]: https://badge.fury.io/py/hordelib.svg?branch=main&kill_cache=1
-[pypi-url]: https://badge.fury.io/py/hordelib
-[downloads-image]: https://pepy.tech/badge/hordelib
-[downloads-url]: https://pepy.tech/project/hordelib
-[license-url]: https://img.shields.io/github/license/jug-dev/hordelib
-[build-image]: https://github.com/jug-dev/hordelib/actions/workflows/maintests.yml/badge.svg?branch=main
+[pypi-url]: https://badge.fury.io/py/horde-engine
+[downloads-image]: https://pepy.tech/badge/horde-engine
+[downloads-url]: https://pepy.tech/project/horde-engine
+[license-url]: https://img.shields.io/github/license/Haidra-Org/hordelib
+[build-image]: https://github.com/Haidra-Org/hordelib/actions/workflows/maintests.yml/badge.svg?branch=main
 [all-model-images]: https://badgen.net/badge/all-models/images/blue?icon=awesome
 [build-url]: https://tests.hordelib.org/
 [main-test-image]: https://badgen.net/badge/main/latest-images/blue?icon=awesome
@@ -288,4 +291,4 @@ Patches can be applied with the `hordelib.install_comfyui.Installer` classes `ap
 [pr-test-url]: https://tests.hordelib.org/unstable/index.html
 [all-model-url]: https://tests.hordelib.org/all_models/
 [changelog-image]: https://img.shields.io/badge/Release-Changelog-yellow
-[changelog-url]: https://github.com/jug-dev/hordelib/blob/releases/CHANGELOG.md
+[changelog-url]: https://github.com/Haidra-Org/hordelib/blob/releases/CHANGELOG.md