Add Python version benchmarks (#421)

emmett-framework · Oct 28, 2024 · 2cf7818 · 2cf7818
1 parent c8147f2
commit 2cf7818
Show file tree

Hide file tree

Showing 8 changed files with 339 additions and 297 deletions.
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
@@ -83,21 +83,70 @@ jobs:
         pip install -r benchmarks/envs/asgi.txt
         pip install -r benchmarks/envs/wsgi.txt
     - name: benchmark
-      env:
-        BENCHMARK_BASE: false
-        BENCHMARK_VS: true
       working-directory: ./benchmarks
       run: |
-        python benchmarks.py
+        python benchmarks.py vs
     - name: upload results
       uses: actions/upload-artifact@v4
       with:
         name: results-vs
         path: benchmarks/results/*
 
+  benchmark-pyver:
+    runs-on: ubuntu-latest
+    needs: [toolchain]
+
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: |
+          3.10
+          3.11
+          3.12
+          3.13
+    - uses: actions/download-artifact@v4
+      with:
+        name: rewrk
+    - run: |
+        sudo mv rewrk /usr/local/bin && chmod +x /usr/local/bin/rewrk
+    - uses: pyo3/maturin-action@v1
+      with:
+        command: build
+        args: --release --interpreter python3.10 python3.11 python3.12 python3.13
+        target: x64
+        manylinux: auto
+        container: off
+    - name: setup venvs
+      run: |
+        python3.10 -m venv .venv310
+        python3.11 -m venv .venv311
+        python3.12 -m venv .venv312
+        python3.13 -m venv .venv313
+        .venv310/bin/pip install $(ls target/wheels/granian-*-cp310-*.whl)
+        .venv311/bin/pip install $(ls target/wheels/granian-*-cp311-*.whl)
+        .venv312/bin/pip install $(ls target/wheels/granian-*-cp312-*.whl)
+        .venv313/bin/pip install $(ls target/wheels/granian-*-cp313-*.whl)
+    - name: benchmark
+      working-directory: ./benchmarks
+      run: |
+        BENCHMARK_EXC_PREFIX=${{ github.workspace }}/.venv310/bin ${{ github.workspace }}/.venv310/bin/python benchmarks.py interfaces
+        mv results/data.json results/py310.json
+        BENCHMARK_EXC_PREFIX=${{ github.workspace }}/.venv311/bin ${{ github.workspace }}/.venv311/bin/python benchmarks.py interfaces
+        mv results/data.json results/py311.json
+        BENCHMARK_EXC_PREFIX=${{ github.workspace }}/.venv312/bin ${{ github.workspace }}/.venv312/bin/python benchmarks.py interfaces
+        mv results/data.json results/py312.json
+        BENCHMARK_EXC_PREFIX=${{ github.workspace }}/.venv313/bin ${{ github.workspace }}/.venv313/bin/python benchmarks.py interfaces
+        mv results/data.json results/py313.json
+    - name: upload results
+      uses: actions/upload-artifact@v4
+      with:
+        name: results-pyver
+        path: benchmarks/results/*
+
   results:
     runs-on: ubuntu-latest
-    needs: [benchmark-base, benchmark-vs]
+    needs: [benchmark-base, benchmark-vs, benchmark-pyver]
 
     steps:
     - uses: actions/checkout@v4
@@ -114,11 +163,22 @@ jobs:
         path: benchmarks/results
     - run: |
         mv benchmarks/results/data.json benchmarks/results/vs.json
+    - uses: actions/download-artifact@v4
+      with:
+        name: results-pyver
+        path: benchmarks/results
     - name: render
       working-directory: ./benchmarks
       run: |
         noir -c data:results/base.json -v 'benv=GHA Linux x86_64' templates/main.md > README.md
         noir -c data:results/vs.json -v 'benv=GHA Linux x86_64' templates/vs.md > vs.md
+        noir \
+          -c data310:results/py310.json \
+          -c data311:results/py311.json \
+          -c data312:results/py312.json \
+          -c data313:results/py313.json \
+          -v pyvb=310 -v 'benv=GHA Linux x86_64' \
+          templates/pyver.md > pyver.md
     - name: open PR
       uses: peter-evans/create-pull-request@v6
       with:
@@ -131,3 +191,4 @@ jobs:
         add-paths: |
           benchmarks/README.md
           benchmarks/vs.md
+          benchmarks/pyver.md
diff --git a/benchmarks/app/asgi.py b/benchmarks/app/asgi.py
@@ -2,82 +2,59 @@
 import pathlib
 import sys
 
+
 PLAINTEXT_RESPONSE = {
     'type': 'http.response.start',
     'status': 200,
     'headers': [
         (b'content-type', b'text/plain; charset=utf-8'),
-    ]
+    ],
 }
 MEDIA_RESPONSE = {
     'type': 'http.response.start',
     'status': 200,
     'headers': [[b'content-type', b'image/png'], [b'content-length', b'95']],
 }
 
-BODY_BYTES_SHORT = b"Test"
-BODY_BYTES_LONG = b"Test" * 20_000
-BODY_STR_SHORT = "Test"
-BODY_STR_LONG = "Test" * 20_000
+BODY_BYTES_SHORT = b'Test'
+BODY_BYTES_LONG = b'Test' * 20_000
+BODY_STR_SHORT = 'Test'
+BODY_STR_LONG = 'Test' * 20_000
 
 MEDIA_PATH = pathlib.Path(__file__).parent.parent / 'files' / 'media.png'
 
 
 async def b_short(scope, receive, send):
     await send(PLAINTEXT_RESPONSE)
-    await send({
-        'type': 'http.response.body',
-        'body': BODY_BYTES_SHORT,
-        'more_body': False
-    })
+    await send({'type': 'http.response.body', 'body': BODY_BYTES_SHORT, 'more_body': False})
 
 
 async def b_long(scope, receive, send):
     await send(PLAINTEXT_RESPONSE)
-    await send({
-        'type': 'http.response.body',
-        'body': BODY_BYTES_LONG,
-        'more_body': False
-    })
+    await send({'type': 'http.response.body', 'body': BODY_BYTES_LONG, 'more_body': False})
 
 
 async def s_short(scope, receive, send):
     await send(PLAINTEXT_RESPONSE)
-    await send({
-        'type': 'http.response.body',
-        'body': BODY_STR_SHORT.encode("utf8"),
-        'more_body': False
-    })
+    await send({'type': 'http.response.body', 'body': BODY_STR_SHORT.encode('utf8'), 'more_body': False})
 
 
 async def s_long(scope, receive, send):
     await send(PLAINTEXT_RESPONSE)
-    await send({
-        'type': 'http.response.body',
-        'body': BODY_STR_LONG.encode("utf8"),
-        'more_body': False
-    })
+    await send({'type': 'http.response.body', 'body': BODY_STR_LONG.encode('utf8'), 'more_body': False})
 
 
 async def echo(scope, receive, send):
     await send(PLAINTEXT_RESPONSE)
     msg = await receive()
-    await send({
-        'type': 'http.response.body',
-        'body': msg['body'],
-        'more_body': False
-    })
+    await send({'type': 'http.response.body', 'body': msg['body'], 'more_body': False})
 
 
 async def file_body(scope, receive, send):
     await send(MEDIA_RESPONSE)
     with MEDIA_PATH.open('rb') as f:
         data = f.read()
-    await send({
-        'type': 'http.response.body',
-        'body': data,
-        'more_body': False
-    })
+    await send({'type': 'http.response.body', 'body': data, 'more_body': False})
 
 
 async def file_pathsend(scope, receive, send):
@@ -87,25 +64,19 @@ async def file_pathsend(scope, receive, send):
 
 def io_builder(wait):
     wait = wait / 1000
+
     async def io(scope, receive, send):
         await send(PLAINTEXT_RESPONSE)
         await asyncio.sleep(wait)
-        await send({
-            'type': 'http.response.body',
-            'body': BODY_BYTES_SHORT,
-            'more_body': False
-        })
+        await send({'type': 'http.response.body', 'body': BODY_BYTES_SHORT, 'more_body': False})
+
     return io
 
 
 async def handle_404(scope, receive, send):
     content = b'Not found'
     await send(PLAINTEXT_RESPONSE)
-    await send({
-        'type': 'http.response.body',
-        'body': content,
-        'more_body': False
-    })
+    await send({'type': 'http.response.body', 'body': content, 'more_body': False})
 
 
 routes = {
@@ -133,8 +104,9 @@ async def async_app(scope, receive, send):
 
 def granian(wrk, thr):
     from granian import Granian
-    Granian("asgi:app", workers=int(wrk), threads=int(thr), interface="asgi").serve()
+
+    Granian('asgi:app', workers=int(wrk), threads=int(thr), interface='asgi').serve()
 
 
-if __name__ == "__main__":
+if __name__ == '__main__':
     granian(sys.argv[1], sys.argv[2])
diff --git a/benchmarks/app/rsgi.py b/benchmarks/app/rsgi.py
@@ -2,83 +2,54 @@
 import pathlib
 import sys
 
+
 HEADERS = [('content-type', 'text/plain; charset=utf-8')]
 HEADERS_MEDIA = [('content-type', 'image/png'), ('content-length', '95')]
 
-BODY_BYTES_SHORT = b"Test"
-BODY_BYTES_LONG = b"Test" * 20_000
-BODY_STR_SHORT = "Test"
-BODY_STR_LONG = "Test" * 20_000
+BODY_BYTES_SHORT = b'Test'
+BODY_BYTES_LONG = b'Test' * 20_000
+BODY_STR_SHORT = 'Test'
+BODY_STR_LONG = 'Test' * 20_000
 
 MEDIA_PATH = str(pathlib.Path(__file__).parent.parent / 'files' / 'media.png')
 
 
 async def b_short(scope, proto):
-    proto.response_bytes(
-        200,
-        HEADERS,
-        BODY_BYTES_SHORT
-    )
+    proto.response_bytes(200, HEADERS, BODY_BYTES_SHORT)
 
 
 async def b_long(scope, proto):
-    proto.response_bytes(
-        200,
-        HEADERS,
-        BODY_BYTES_LONG
-    )
+    proto.response_bytes(200, HEADERS, BODY_BYTES_LONG)
 
 
 async def s_short(scope, proto):
-    proto.response_str(
-        200,
-        HEADERS,
-        BODY_STR_SHORT
-    )
+    proto.response_str(200, HEADERS, BODY_STR_SHORT)
 
 
 async def s_long(scope, proto):
-    proto.response_str(
-        200,
-        HEADERS,
-        BODY_STR_LONG
-    )
+    proto.response_str(200, HEADERS, BODY_STR_LONG)
 
 
 async def echo(scope, proto):
-    proto.response_bytes(
-        200,
-        HEADERS,
-        await proto()
-    )
+    proto.response_bytes(200, HEADERS, await proto())
 
 
 async def file(scope, proto):
-    proto.response_file(
-        200,
-        HEADERS_MEDIA,
-        MEDIA_PATH
-    )
+    proto.response_file(200, HEADERS_MEDIA, MEDIA_PATH)
 
 
 def io_builder(wait):
     wait = wait / 1000
+
     async def io(scope, proto):
         await asyncio.sleep(wait)
-        proto.response_bytes(
-            200,
-            HEADERS,
-            BODY_BYTES_SHORT
-        )
+        proto.response_bytes(200, HEADERS, BODY_BYTES_SHORT)
+
     return io
 
 
 async def handle_404(scope, proto):
-    proto.response_str(
-        404,
-        HEADERS,
-        "not found"
-    )
+    proto.response_str(404, HEADERS, 'not found')
 
 
 routes = {
@@ -100,8 +71,9 @@ def app(scope, proto):
 
 def granian(wrk, thr):
     from granian import Granian
-    Granian("rsgi:app", workers=int(wrk), threads=int(thr), interface="rsgi").serve()
+
+    Granian('rsgi:app', workers=int(wrk), threads=int(thr), interface='rsgi').serve()
 
 
-if __name__ == "__main__":
+if __name__ == '__main__':
     granian(sys.argv[1], sys.argv[2])