From a46fefb9d2eea7f319d685838503ba2f8dc96d4d Mon Sep 17 00:00:00 2001 From: Alexander Piskun Date: Tue, 24 Sep 2024 11:57:11 +0300 Subject: [PATCH] fix for more precise benchmarking + docs for it Signed-off-by: Alexander Piskun --- docs/how_to_benchmark.md | 109 +++++++++++++ docs/index.md | 4 + ...ary-2024-09-13-i9_10900-7900XTX-OTHER.json | 152 ------------------ ...ary-2024-09-24-i9_10900-7900XTX-FLUX.json} | 11 +- ...ry-2024-09-24-i9_10900-7900XTX-HEAVY.json} | 8 +- ...ary-2024-09-24-i9_10900-7900XTX-OTHER.json | 67 ++++++++ ...2024-09-24-i9_10900-7900XTX-PORTRAITS.json | 75 +++++++++ ...ary-2024-09-24-i9_10900-7900XTX-SDXL.json} | 47 ++---- mkdocs.yml | 1 + requirements.txt | 1 + scripts/benchmarks/benchmark.py | 55 +++++-- 11 files changed, 318 insertions(+), 212 deletions(-) create mode 100644 docs/how_to_benchmark.md delete mode 100644 hardware_results/summary-2024-09-13-i9_10900-7900XTX-OTHER.json rename hardware_results/{summary-2024-09-13-i9_10900-7900XTX-FLUX.json => summary-2024-09-24-i9_10900-7900XTX-FLUX.json} (61%) rename hardware_results/{summary-2024-09-13-i9_10900-7900XTX-HEAVY.json => summary-2024-09-24-i9_10900-7900XTX-HEAVY.json} (55%) create mode 100644 hardware_results/summary-2024-09-24-i9_10900-7900XTX-OTHER.json create mode 100644 hardware_results/summary-2024-09-24-i9_10900-7900XTX-PORTRAITS.json rename hardware_results/{summary-2024-09-13-i9_10900-7900XTX-SDXL.json => summary-2024-09-24-i9_10900-7900XTX-SDXL.json} (58%) diff --git a/docs/how_to_benchmark.md b/docs/how_to_benchmark.md new file mode 100644 index 0000000..b610633 --- /dev/null +++ b/docs/how_to_benchmark.md @@ -0,0 +1,109 @@ +--- +title: How to Benchmark +--- + +# Introduction + +!!! note + + For benchmarking, it's preferable to use the `SERVER` + `WORKER` modes. + + You can find the guide on how to install in this configuration [here](AdminManual/WorkingModes/worker_to_database.md). + +Assuming you have Visionatrix installed, you can proceed with benchmarking. + +You don't need to run the benchmark script on the same machine where Visionatrix is installed. Therefore, clone the Visionatrix documentation repository wherever it's convenient for you: + +```bash +git clone https://github.com/Visionatrix/VixFlowsDocs.git +``` + +## Setting Up a Virtual Environment (Optional) + +It's recommended to use a virtual environment to avoid conflicts with other Python packages on your system. Here's how you can set up a virtual environment that works on macOS and Linux (assuming all OS packages are installed): + +```bash +cd VixFlowsDocs && python3 -m venv venv && source venv/bin/activate +``` + +This creates a new virtual environment named `venv` and activates it. You can now proceed to install the dependencies: + +```bash +pip install -r requirements.txt +``` + +If you prefer not to use a virtual environment, you can install the dependencies directly: + +```bash +cd VixFlowsDocs && pip install -r requirements.txt +``` + +## Running the Benchmark Script + +You can then run the script with the default parameters. The working directory doesn't matter because the script uses relative paths from its own location. + +If your Visionatrix instance does not have authentication enabled (for example, it's running in `DEFAULT` mode), simply run the script: + +```bash +python3 scripts/benchmarks/benchmark.py +``` + +The script will offer you options for what you want to test; just enter the number of your choice. + +!!! note + + The script will automatically install the flows from the test set. It will also start testing as soon as the first flow is installed and will install the next one in parallel. + + This will not significantly affect performance (unless your Visionatrix is running in `DEFAULT` mode). + +Upon completion of the tests, a folder with results will appear in the `results` directory. The most important file is the JSON file located at the root of this folder. + +You should set the environment variable `HARDWARE` in the format "CPU-GraphicCard" or, after testing is complete, rename the results file (replace "YOUR_CPU" and "YOUR_GPU" with your own hardware specifications). + +If you want to add your results to the documentation, simply open a pull request with this file in the `hardware_results` folder. + +# Supported Environment Variables + +As mentioned earlier, the `HARDWARE` variable is supported. Example usage: + +```bash +HARDWARE="EPYC_75F3-4090" python3 scripts/benchmarks/benchmark.py +``` + +Another supported variable is `SERVER_URL`, for example: + +```bash +SERVER_URL="http://192.168.1.10:8288" python3 scripts/benchmarks/benchmark.py +``` + +Use this when Visionatrix is located on another machine. + +If you need to provide authentication credentials for the Visionatrix server, you can do so using the variables `USER_NAME` and `USER_PASSWORD`. + +Example usage along with `SERVER_URL`: + +```bash +SERVER_URL="http://192.168.1.10:8288" USER_NAME="admin" USER_PASSWORD="admin" python3 scripts/benchmarks/benchmark.py +``` + +The `COUNT` variable controls the number of tasks created for each flow. By default, it is set to `2`. You can set it to `1`, `3`, `4`, etc. The higher the number, the more accurate the hardware test will be. + +Also supported is the `REMOVE_RESULTS` variable. By default, it is `1`, but you can set it to `0` if you don't want the tasks created during testing to be deleted. + +If you have a slow internet connection and not all flows from your selected test set are installed, you might want to set a custom value for the `FLOW_INSTALL_TIMEOUT` variable. + +By default, it is set to 1800 seconds (30 minutes). If the flow does not download within this time, the script will produce an error (but this does not cancel the flow installation, and eventually it will be installed on the server). + +The last variables are `PAUSE_INTERVAL` (default value `0`) and `PAUSE_INTERVAL_AFTER_WARMUP` (default value `0`). + +- The `PAUSE_INTERVAL` variable defines the pause time between test sets (this might be useful if, during tests, your device heats up and you want to prevent overheating). + +- The `PAUSE_INTERVAL_AFTER_WARMUP` variable is slightly more useful. It defines the interval of time to wait after the first test run, during which the models from the flow are loaded into memory. This is useful on laptops or devices with insufficient cooling. It is only beneficial if you do not set the `WARMUP` variable to `0`. + +Theoretically, to measure speed closer to production, measurements with `WARMUP` set to `0` will be more accurate. However, we conduct testing without changing the `WARMUP` value, which is `1` by default. + +# Conclusion + +By following the steps outlined above, you can successfully benchmark your Visionatrix setup. Adjust the environment variables as needed to suit your specific hardware and network conditions. If you encounter any issues or have results you'd like to share, feel free to contribute to the documentation by opening a pull request. + +Happy benchmarking! diff --git a/docs/index.md b/docs/index.md index 5e60635..9818549 100644 --- a/docs/index.md +++ b/docs/index.md @@ -22,4 +22,8 @@ Common information: - [FAQ](faq.md) - [Hardware FAQ](hardware_faq.md) - [Hardware Results](hardware_results.md) + +Specialized stuff: + +- [How To Perform Benchmarks](how_to_benchmark.md) - [Swagger API](swagger.html) diff --git a/hardware_results/summary-2024-09-13-i9_10900-7900XTX-OTHER.json b/hardware_results/summary-2024-09-13-i9_10900-7900XTX-OTHER.json deleted file mode 100644 index 0f87a4e..0000000 --- a/hardware_results/summary-2024-09-13-i9_10900-7900XTX-OTHER.json +++ /dev/null @@ -1,152 +0,0 @@ -{ - "test_time": "2024/09/13", - "flows": [ - { - "flow_name": "stable_cascade", - "flow_display_name": "Stable Cascade", - "test_cases": [ - { - "test_case": "one_pass", - "avg_exec_time": 7.68705482949963, - "hardware_desc": "i9_10900-7900XTX" - }, - { - "test_case": "two_pass", - "avg_exec_time": 27.010453875000167, - "hardware_desc": "i9_10900-7900XTX" - }, - { - "test_case": "three_pass", - "avg_exec_time": 65.37532646525096, - "hardware_desc": "i9_10900-7900XTX" - } - ] - }, - { - "flow_name": "hunyuan_dit", - "flow_display_name": "HunyuanDiT", - "test_cases": [ - { - "test_case": "20steps", - "avg_exec_time": 20.942034714749752, - "hardware_desc": "i9_10900-7900XTX" - }, - { - "test_case": "40steps", - "avg_exec_time": 41.45832211549987, - "hardware_desc": "i9_10900-7900XTX" - } - ] - }, - { - "flow_name": "vintage_portrait", - "flow_display_name": "Vintage Portrait", - "test_cases": [ - { - "test_case": "default", - "avg_exec_time": 25.496262035999735, - "hardware_desc": "i9_10900-7900XTX" - } - ] - }, - { - "flow_name": "sketch_portrait", - "flow_display_name": "Sketch Portrait", - "test_cases": [ - { - "test_case": "default", - "avg_exec_time": 4.2926650005006195, - "hardware_desc": "i9_10900-7900XTX" - } - ] - }, - { - "flow_name": "comicu_portrait", - "flow_display_name": "ComicU Portrait", - "test_cases": [ - { - "test_case": "default", - "avg_exec_time": 4.442787754249821, - "hardware_desc": "i9_10900-7900XTX" - } - ] - }, - { - "flow_name": "ghibli_portrait", - "flow_display_name": "Ghibli Portrait", - "test_cases": [ - { - "test_case": "default", - "avg_exec_time": 4.06410050049999, - "hardware_desc": "i9_10900-7900XTX" - } - ] - }, - { - "flow_name": "memoji_portrait", - "flow_display_name": "Memoji Portrait", - "test_cases": [ - { - "test_case": "default", - "avg_exec_time": 3.76674200575053, - "hardware_desc": "i9_10900-7900XTX" - } - ] - }, - { - "flow_name": "photomaker_1", - "flow_display_name": "Photo from 1 image", - "test_cases": [ - { - "test_case": "default", - "avg_exec_time": 13.05911071850096, - "hardware_desc": "i9_10900-7900XTX" - } - ] - }, - { - "flow_name": "remove_background_birefnet", - "flow_display_name": "Remove background (BiRefNet)", - "test_cases": [ - { - "test_case": "1024x1024", - "avg_exec_time": 0.38839321525028936, - "hardware_desc": "i9_10900-7900XTX" - } - ] - }, - { - "flow_name": "remove_background_bria", - "flow_display_name": "Remove background (Bria)", - "test_cases": [ - { - "test_case": "1024x1024", - "avg_exec_time": 0.2585432169998967, - "hardware_desc": "i9_10900-7900XTX" - } - ] - }, - { - "flow_name": "supir_upscaler", - "flow_display_name": "SUPIR Upscaler", - "test_cases": [ - { - "test_case": "1MPx1.5", - "avg_exec_time": 124.54452258699985, - "hardware_desc": "i9_10900-7900XTX" - } - ] - }, - { - "flow_name": "photo_stickers", - "flow_display_name": "Photo Stickers", - "test_cases": [ - { - "test_case": "default", - "avg_exec_time": 33.268059159749555, - "hardware_desc": "i9_10900-7900XTX" - } - ] - } - ] -} diff --git a/hardware_results/summary-2024-09-13-i9_10900-7900XTX-FLUX.json b/hardware_results/summary-2024-09-24-i9_10900-7900XTX-FLUX.json similarity index 61% rename from hardware_results/summary-2024-09-13-i9_10900-7900XTX-FLUX.json rename to hardware_results/summary-2024-09-24-i9_10900-7900XTX-FLUX.json index 417da49..e347c0a 100644 --- a/hardware_results/summary-2024-09-13-i9_10900-7900XTX-FLUX.json +++ b/hardware_results/summary-2024-09-24-i9_10900-7900XTX-FLUX.json @@ -1,5 +1,5 @@ { - "test_time": "2024/09/13", + "test_time": "2024/09/24", "flows": [ { "flow_name": "flux1_dev_8bit", @@ -7,13 +7,11 @@ "test_cases": [ { "test_case": "20steps", - "avg_exec_time": 37.42710090300034, - "hardware_desc": "i9_10900-7900XTX" + "avg_exec_time": 38.754069298000104 }, { "test_case": "40steps", - "avg_exec_time": 74.38036600125042, - "hardware_desc": "i9_10900-7900XTX" + "avg_exec_time": 77.04926321249877 } ] }, @@ -23,8 +21,7 @@ "test_cases": [ { "test_case": "default", - "avg_exec_time": 7.357450785000765, - "hardware_desc": "i9_10900-7900XTX" + "avg_exec_time": 8.171019977999094 } ] } diff --git a/hardware_results/summary-2024-09-13-i9_10900-7900XTX-HEAVY.json b/hardware_results/summary-2024-09-24-i9_10900-7900XTX-HEAVY.json similarity index 55% rename from hardware_results/summary-2024-09-13-i9_10900-7900XTX-HEAVY.json rename to hardware_results/summary-2024-09-24-i9_10900-7900XTX-HEAVY.json index 6dd25f8..a586be9 100644 --- a/hardware_results/summary-2024-09-13-i9_10900-7900XTX-HEAVY.json +++ b/hardware_results/summary-2024-09-24-i9_10900-7900XTX-HEAVY.json @@ -1,5 +1,5 @@ { - "test_time": "2024/09/13", + "test_time": "2024/09/24", "flows": [ { "flow_name": "flux1_dev", @@ -7,13 +7,11 @@ "test_cases": [ { "test_case": "20steps", - "avg_exec_time": 52.623215481249645, - "hardware_desc": "i9_10900-7900XTX" + "avg_exec_time": 56.20099137950092 }, { "test_case": "40steps", - "avg_exec_time": 103.50971343000037, - "hardware_desc": "i9_10900-7900XTX" + "avg_exec_time": 110.1237817695 } ] } diff --git a/hardware_results/summary-2024-09-24-i9_10900-7900XTX-OTHER.json b/hardware_results/summary-2024-09-24-i9_10900-7900XTX-OTHER.json new file mode 100644 index 0000000..d06e50c --- /dev/null +++ b/hardware_results/summary-2024-09-24-i9_10900-7900XTX-OTHER.json @@ -0,0 +1,67 @@ +{ + "test_time": "2024/09/24", + "flows": [ + { + "flow_name": "stable_cascade", + "flow_display_name": "Stable Cascade", + "test_cases": [ + { + "test_case": "one_pass", + "avg_exec_time": 8.559083316002216 + }, + { + "test_case": "two_pass", + "avg_exec_time": 28.99868852649888 + }, + { + "test_case": "three_pass", + "avg_exec_time": 68.67816362149824 + } + ] + }, + { + "flow_name": "hunyuan_dit", + "flow_display_name": "HunyuanDiT", + "test_cases": [ + { + "test_case": "20steps", + "avg_exec_time": 23.851970283500123 + }, + { + "test_case": "40steps", + "avg_exec_time": 47.45772618000046 + } + ] + }, + { + "flow_name": "remove_background_birefnet", + "flow_display_name": "Remove background (BiRefNet)", + "test_cases": [ + { + "test_case": "1024x1024", + "avg_exec_time": 0.37308207750174915 + } + ] + }, + { + "flow_name": "remove_background_bria", + "flow_display_name": "Remove background (Bria)", + "test_cases": [ + { + "test_case": "1024x1024", + "avg_exec_time": 0.25507376949462923 + } + ] + }, + { + "flow_name": "supir_upscaler", + "flow_display_name": "SUPIR Upscaler", + "test_cases": [ + { + "test_case": "1MPx1.5", + "avg_exec_time": 127.17051447850099 + } + ] + } + ] +} diff --git a/hardware_results/summary-2024-09-24-i9_10900-7900XTX-PORTRAITS.json b/hardware_results/summary-2024-09-24-i9_10900-7900XTX-PORTRAITS.json new file mode 100644 index 0000000..ac13895 --- /dev/null +++ b/hardware_results/summary-2024-09-24-i9_10900-7900XTX-PORTRAITS.json @@ -0,0 +1,75 @@ +{ + "test_time": "2024/09/24", + "flows": [ + { + "flow_name": "vintage_portrait", + "flow_display_name": "Vintage Portrait", + "test_cases": [ + { + "test_case": "default", + "avg_exec_time": 27.309107471995958 + } + ] + }, + { + "flow_name": "sketch_portrait", + "flow_display_name": "Sketch Portrait", + "test_cases": [ + { + "test_case": "default", + "avg_exec_time": 5.244705069999327 + } + ] + }, + { + "flow_name": "comicu_portrait", + "flow_display_name": "ComicU Portrait", + "test_cases": [ + { + "test_case": "default", + "avg_exec_time": 5.440893882998353 + } + ] + }, + { + "flow_name": "ghibli_portrait", + "flow_display_name": "Ghibli Portrait", + "test_cases": [ + { + "test_case": "default", + "avg_exec_time": 17.521222452502116 + } + ] + }, + { + "flow_name": "memoji_portrait", + "flow_display_name": "Memoji Portrait", + "test_cases": [ + { + "test_case": "default", + "avg_exec_time": 4.734728722 + } + ] + }, + { + "flow_name": "photomaker_1", + "flow_display_name": "Photo from 1 image", + "test_cases": [ + { + "test_case": "default", + "avg_exec_time": 14.028759131004335 + } + ] + }, + { + "flow_name": "photo_stickers", + "flow_display_name": "Photo Stickers", + "test_cases": [ + { + "test_case": "default", + "avg_exec_time": 35.01521601149943 + } + ] + } + ] +} diff --git a/hardware_results/summary-2024-09-13-i9_10900-7900XTX-SDXL.json b/hardware_results/summary-2024-09-24-i9_10900-7900XTX-SDXL.json similarity index 58% rename from hardware_results/summary-2024-09-13-i9_10900-7900XTX-SDXL.json rename to hardware_results/summary-2024-09-24-i9_10900-7900XTX-SDXL.json index 4eae022..eca70c4 100644 --- a/hardware_results/summary-2024-09-13-i9_10900-7900XTX-SDXL.json +++ b/hardware_results/summary-2024-09-24-i9_10900-7900XTX-SDXL.json @@ -1,5 +1,5 @@ { - "test_time": "2024/09/13", + "test_time": "2024/09/24", "flows": [ { "flow_name": "sdxl_lighting", @@ -7,13 +7,11 @@ "test_cases": [ { "test_case": "4_steps", - "avg_exec_time": 1.0666292267496829, - "hardware_desc": "i9_10900-7900XTX" + "avg_exec_time": 1.0567749305046163 }, { "test_case": "8_steps", - "avg_exec_time": 1.6902707809997537, - "hardware_desc": "i9_10900-7900XTX" + "avg_exec_time": 1.6828510370032745 } ] }, @@ -23,8 +21,7 @@ "test_cases": [ { "test_case": "default", - "avg_exec_time": 9.208366818750164, - "hardware_desc": "i9_10900-7900XTX" + "avg_exec_time": 9.214472276002198 } ] }, @@ -34,13 +31,11 @@ "test_cases": [ { "test_case": "default", - "avg_exec_time": 15.148308121500122, - "hardware_desc": "i9_10900-7900XTX" + "avg_exec_time": 15.11067810700115 }, { "test_case": "fast_run", - "avg_exec_time": 6.2556433127501805, - "hardware_desc": "i9_10900-7900XTX" + "avg_exec_time": 6.240951770996617 } ] }, @@ -50,23 +45,19 @@ "test_cases": [ { "test_case": "fast_run_30steps", - "avg_exec_time": 1.95916982225026, - "hardware_desc": "i9_10900-7900XTX" + "avg_exec_time": 1.9713126410024415 }, { "test_case": "fast_run_60steps", - "avg_exec_time": 3.4866955709999274, - "hardware_desc": "i9_10900-7900XTX" + "avg_exec_time": 3.484776608496759 }, { "test_case": "usual_run_30steps", - "avg_exec_time": 5.01547002250004, - "hardware_desc": "i9_10900-7900XTX" + "avg_exec_time": 5.018865888996515 }, { "test_case": "usual_run_60steps", - "avg_exec_time": 9.575216640000008, - "hardware_desc": "i9_10900-7900XTX" + "avg_exec_time": 9.5855497679986 } ] }, @@ -76,13 +67,11 @@ "test_cases": [ { "test_case": "fast_run_30steps", - "avg_exec_time": 7.173761420250003, - "hardware_desc": "i9_10900-7900XTX" + "avg_exec_time": 7.292398761997902 }, { "test_case": "usual_run_30steps", - "avg_exec_time": 17.319431959749863, - "hardware_desc": "i9_10900-7900XTX" + "avg_exec_time": 17.363493708002352 } ] }, @@ -92,13 +81,11 @@ "test_cases": [ { "test_case": "default", - "avg_exec_time": 15.068115058749981, - "hardware_desc": "i9_10900-7900XTX" + "avg_exec_time": 15.02141429350013 }, { "test_case": "fast_run", - "avg_exec_time": 6.229756580750063, - "hardware_desc": "i9_10900-7900XTX" + "avg_exec_time": 6.232963144000678 } ] }, @@ -108,13 +95,11 @@ "test_cases": [ { "test_case": "25steps", - "avg_exec_time": 8.885974457999964, - "hardware_desc": "i9_10900-7900XTX" + "avg_exec_time": 8.869329255001503 }, { "test_case": "50steps", - "avg_exec_time": 17.365900846500153, - "hardware_desc": "i9_10900-7900XTX" + "avg_exec_time": 17.392088781001803 } ] } diff --git a/mkdocs.yml b/mkdocs.yml index 71053b6..3b01ab3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -46,6 +46,7 @@ nav: - FAQ: faq.md - Hardware FAQ: hardware_faq.md - Hardware Results: hardware_results.md + - How To Benchmark: how_to_benchmark.md - Swagger API: swagger.html markdown_extensions: diff --git a/requirements.txt b/requirements.txt index 7d56c44..f4dde7a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ mkdocs-git-revision-date-localized-plugin mkdocs-material[imaging] requests huggingface_hub +httpx diff --git a/scripts/benchmarks/benchmark.py b/scripts/benchmarks/benchmark.py index c6bded2..e8518db 100644 --- a/scripts/benchmarks/benchmark.py +++ b/scripts/benchmarks/benchmark.py @@ -16,23 +16,26 @@ SERVER_URL = os.environ.get("SERVER_URL", "http://127.0.0.1:8288") REMOVE_RESULTS_FROM_VISIONATRIX = int(os.environ.get("REMOVE_RESULTS", "1")) -DEFAULT_NUMBER_OF_TEST_CASE_RUNS = int(os.environ.get("COUNT", "3")) -HARDWARE = os.environ.get("HARDWARE", "HW_REPLACE44").strip("\"'") +DEFAULT_NUMBER_OF_TEST_CASE_RUNS = int(os.environ.get("COUNT", "2")) +HARDWARE = os.environ.get("HARDWARE", "YOUR_CPU-YOUR_GPU").strip("\"'") +WARMUP = bool(int(os.environ.get("WARMUP", "1"))) FLOW_INSTALL_TIMEOUT = int(os.environ.get("FLOW_INSTALL_TIMEOUT", "1800")) TEST_START_TIME = datetime.now() RESULTS_DIR = Path("results").joinpath( f"{TEST_START_TIME.strftime('%Y-%m-%d')}-{HARDWARE}" ) -USER_NAME, USER_PASS = os.getenv("USERNAME", ""), os.getenv("USERPASS", "") -BASIC_AUTH = httpx.BasicAuth(USER_NAME, USER_PASS) if USER_NAME and USER_PASS else None +USER_NAME, USER_PASSWORD = os.getenv("USER_NAME", ""), os.getenv("USER_PASSWORD", "") +BASIC_AUTH = ( + httpx.BasicAuth(USER_NAME, USER_PASSWORD) if USER_NAME and USER_PASSWORD else None +) if BASIC_AUTH: print("Using authentication for connect") PAUSE_INTERVAL = int(os.environ.get("PAUSE_INTERVAL", "0")) -FIRST_TEST_FLAG = True +PAUSE_INTERVAL_AFTER_WARMUP = int(os.environ.get("PAUSE_INTERVAL_AFTER_WARMUP", "0")) +FIRST_TEST_FLAG = True SELECTED_TEST_FLOW_SUITE = [] - INSTALLED_FLOWS_CACHE = [] @@ -49,7 +52,7 @@ class TestCase(BaseModel): description="List of file paths to pass to the flow during the task creation", ) warm_up: bool = Field( - True, + WARMUP, description="If set to True, the warm-up run will be executed before test start.", ) count: int = Field( @@ -627,11 +630,33 @@ async def run_test_case( input_params = test_case.input_params input_files = test_case.input_files - warm_up = test_case.warm_up - count = test_case.count + 1 if warm_up else test_case.count + if test_case.warm_up: + print("Warming up...") + warmup_task_id = await create_task(flow_name, input_params, 1, input_files) + if not warmup_task_id: + print( + f"Failed to create warmup task for flow '{flow_name}', test case: {test_case.name}" + ) + return + r = await get_task_progress(warmup_task_id[0]) + if r.get("error", ""): + print( + f"Failed to finish warmup task for flow '{flow_name}', test case: {test_case.name}" + f"\nError: {r['error']}" + ) + return + if REMOVE_RESULTS_FROM_VISIONATRIX: + await delete_task(warmup_task_id[0]) + if PAUSE_INTERVAL_AFTER_WARMUP: + print( + f"Pausing for {PAUSE_INTERVAL_AFTER_WARMUP} seconds after warmup." + ) + await asyncio.sleep(PAUSE_INTERVAL_AFTER_WARMUP) # Create tasks, passing input files if present - task_ids = await create_task(flow_name, input_params, count, input_files) + task_ids = await create_task( + flow_name, input_params, test_case.count, input_files + ) if not task_ids: print( f"Failed to create tasks for flow '{flow_name}', test case: {test_case.name}" @@ -648,11 +673,10 @@ async def run_test_case( await save_flow_comfy(flow_name, test_case.name, result["flow_comfy"]) break # No need to check further once saved - if warm_up: - task_results = task_results[1:] - # Save results and capture summary - summary = await save_results(flow_name, test_case.name, test_case, task_results) + summary = await save_results( + flow_name, test_case.name, test_case, task_results + ) # noqa if summary: results_summary[flow_name].append( { @@ -758,9 +782,6 @@ async def benchmarker(): await asyncio.gather(*flow_tasks) - # Generate the JSON results summary file - await generate_results_summary_json(results_summary) - async def save_results( flow_name: str, test_case_name: str, test_case: TestCase, task_results: list