Use HTTP Livestreaming for audio/video streaming out

.config/playwright.config.js

@@ -19,7 +19,22 @@ const base = defineConfig({
 	testMatch: /.*\.spec\.ts/,
 	testDir: "..",
 	workers: process.env.CI ? 1 : undefined,
-	retries: 3
+	retries: 3,
+	projects: [
+		{
+			name: "firefox",
+			use: { ...devices["Desktop Firefox"] },
+			testMatch: /.stream_(audio|video)_out\.spec\.ts/
+		},
+		{
+			name: "chrome",
+			use: {
+				...devices["Desktop Chrome"],
+				permissions: ["clipboard-read", "clipboard-write", "microphone"]
+			},
+			testIgnore: /.stream_(audio|video)_out\.spec\.ts/
+		}
+	]
 });
 
 const normal = defineConfig(base, {

.github/workflows/test-functional.yml

@@ -51,11 +51,12 @@ jobs:
         with:
           always_install_pnpm: true
           build_lite: true
-      - name: install outbreak_forecast dependencies
+      - name: install demo dependencies
         run: |
           . venv/bin/activate
           python -m pip install -r demo/outbreak_forecast/requirements.txt
           python -m pip install -r demo/gradio_pdf_demo/requirements.txt
+          python -m pip install -r demo/stream_video_out/requirements.txt
       - run: pnpm exec playwright install chromium firefox
       - name: run browser tests
         run: |

client/python/test/conftest.py

@@ -238,6 +238,30 @@ def show(n):
     return demo
 
 
+@pytest.fixture
+def count_generator_demo_exception():
+    def count(n):
+        for i in range(int(n)):
+            time.sleep(0.01)
+            if i == 5:
+                raise ValueError("Oh no!")
+            yield i
+
+    def show(n):
+        return str(list(range(int(n))))
+
+    with gr.Blocks() as demo:
+        with gr.Column():
+            num = gr.Number(value=10)
+            with gr.Row():
+                count_btn = gr.Button("Count")
+        with gr.Column():
+            out = gr.Textbox()
+
+        count_btn.click(count, num, out, api_name="count")
+    return demo
+
+
 @pytest.fixture
 def file_io_demo():
     demo = gr.Interface(

demo/outbreak_forecast/requirements.txt

@@ -2,4 +2,5 @@ numpy
 matplotlib
 bokeh
 plotly
-altair
+altair
+opencv-python

demo/outbreak_forecast/run.ipynb

@@ -1 +1 @@
-{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: outbreak_forecast\n", "### Generate a plot based on 5 inputs.\n", "        "]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio numpy matplotlib bokeh plotly altair"]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import altair\n", "\n", "import gradio as gr\n", "from math import sqrt\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import plotly.express as px\n", "import pandas as pd\n", "\n", "\n", "def outbreak(plot_type, r, month, countries, social_distancing):\n", "    months = [\"January\", \"February\", \"March\", \"April\", \"May\"]\n", "    m = months.index(month)\n", "    start_day = 30 * m\n", "    final_day = 30 * (m + 1)\n", "    x = np.arange(start_day, final_day + 1)\n", "    pop_count = {\"USA\": 350, \"Canada\": 40, \"Mexico\": 300, \"UK\": 120}\n", "    if social_distancing:\n", "        r = sqrt(r)\n", "    df = pd.DataFrame({\"day\": x})\n", "    for country in countries:\n", "        df[country] = x ** (r) * (pop_count[country] + 1)\n", "\n", "    if plot_type == \"Matplotlib\":\n", "        fig = plt.figure()\n", "        plt.plot(df[\"day\"], df[countries].to_numpy())\n", "        plt.title(\"Outbreak in \" + month)\n", "        plt.ylabel(\"Cases\")\n", "        plt.xlabel(\"Days since Day 0\")\n", "        plt.legend(countries)\n", "        return fig\n", "    elif plot_type == \"Plotly\":\n", "        fig = px.line(df, x=\"day\", y=countries)\n", "        fig.update_layout(\n", "            title=\"Outbreak in \" + month,\n", "            xaxis_title=\"Cases\",\n", "            yaxis_title=\"Days Since Day 0\",\n", "        )\n", "        return fig\n", "    elif plot_type == \"Altair\":\n", "        df = df.melt(id_vars=\"day\").rename(columns={\"variable\": \"country\"})\n", "        fig = altair.Chart(df).mark_line().encode(x=\"day\", y=\"value\", color=\"country\")\n", "        return fig\n", "    else:\n", "        raise ValueError(\"A plot type must be selected\")\n", "\n", "\n", "inputs = [\n", "    gr.Dropdown([\"Matplotlib\", \"Plotly\", \"Altair\"], label=\"Plot Type\"),\n", "    gr.Slider(1, 4, 3.2, label=\"R\"),\n", "    gr.Dropdown([\"January\", \"February\", \"March\", \"April\", \"May\"], label=\"Month\"),\n", "    gr.CheckboxGroup(\n", "        [\"USA\", \"Canada\", \"Mexico\", \"UK\"], label=\"Countries\", value=[\"USA\", \"Canada\"]\n", "    ),\n", "    gr.Checkbox(label=\"Social Distancing?\"),\n", "]\n", "outputs = gr.Plot()\n", "\n", "demo = gr.Interface(\n", "    fn=outbreak,\n", "    inputs=inputs,\n", "    outputs=outputs,\n", "    examples=[\n", "        [\"Matplotlib\", 2, \"March\", [\"Mexico\", \"UK\"], True],\n", "        [\"Altair\", 2, \"March\", [\"Mexico\", \"Canada\"], True],\n", "        [\"Plotly\", 3.6, \"February\", [\"Canada\", \"Mexico\", \"UK\"], False],\n", "    ],\n", "    cache_examples=True,\n", ")\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}
+{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: outbreak_forecast\n", "### Generate a plot based on 5 inputs.\n", "        "]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio numpy matplotlib bokeh plotly altair opencv-python"]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import altair\n", "\n", "import gradio as gr\n", "from math import sqrt\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import plotly.express as px\n", "import pandas as pd\n", "\n", "\n", "def outbreak(plot_type, r, month, countries, social_distancing):\n", "    months = [\"January\", \"February\", \"March\", \"April\", \"May\"]\n", "    m = months.index(month)\n", "    start_day = 30 * m\n", "    final_day = 30 * (m + 1)\n", "    x = np.arange(start_day, final_day + 1)\n", "    pop_count = {\"USA\": 350, \"Canada\": 40, \"Mexico\": 300, \"UK\": 120}\n", "    if social_distancing:\n", "        r = sqrt(r)\n", "    df = pd.DataFrame({\"day\": x})\n", "    for country in countries:\n", "        df[country] = x ** (r) * (pop_count[country] + 1)\n", "\n", "    if plot_type == \"Matplotlib\":\n", "        fig = plt.figure()\n", "        plt.plot(df[\"day\"], df[countries].to_numpy())\n", "        plt.title(\"Outbreak in \" + month)\n", "        plt.ylabel(\"Cases\")\n", "        plt.xlabel(\"Days since Day 0\")\n", "        plt.legend(countries)\n", "        return fig\n", "    elif plot_type == \"Plotly\":\n", "        fig = px.line(df, x=\"day\", y=countries)\n", "        fig.update_layout(\n", "            title=\"Outbreak in \" + month,\n", "            xaxis_title=\"Cases\",\n", "            yaxis_title=\"Days Since Day 0\",\n", "        )\n", "        return fig\n", "    elif plot_type == \"Altair\":\n", "        df = df.melt(id_vars=\"day\").rename(columns={\"variable\": \"country\"})\n", "        fig = altair.Chart(df).mark_line().encode(x=\"day\", y=\"value\", color=\"country\")\n", "        return fig\n", "    else:\n", "        raise ValueError(\"A plot type must be selected\")\n", "\n", "\n", "inputs = [\n", "    gr.Dropdown([\"Matplotlib\", \"Plotly\", \"Altair\"], label=\"Plot Type\"),\n", "    gr.Slider(1, 4, 3.2, label=\"R\"),\n", "    gr.Dropdown([\"January\", \"February\", \"March\", \"April\", \"May\"], label=\"Month\"),\n", "    gr.CheckboxGroup(\n", "        [\"USA\", \"Canada\", \"Mexico\", \"UK\"], label=\"Countries\", value=[\"USA\", \"Canada\"]\n", "    ),\n", "    gr.Checkbox(label=\"Social Distancing?\"),\n", "]\n", "outputs = gr.Plot()\n", "\n", "demo = gr.Interface(\n", "    fn=outbreak,\n", "    inputs=inputs,\n", "    outputs=outputs,\n", "    examples=[\n", "        [\"Matplotlib\", 2, \"March\", [\"Mexico\", \"UK\"], True],\n", "        [\"Altair\", 2, \"March\", [\"Mexico\", \"Canada\"], True],\n", "        [\"Plotly\", 3.6, \"February\", [\"Canada\", \"Mexico\", \"UK\"], False],\n", "    ],\n", "    cache_examples=True,\n", ")\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}

demo/stream_audio_out/run.ipynb

@@ -1 +1 @@
-{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: stream_audio_out"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["# Downloading files from the demo repo\n", "import os\n", "os.mkdir('audio')\n", "!wget -q -O audio/cantina.wav https://github.com/gradio-app/gradio/raw/main/demo/stream_audio_out/audio/cantina.wav"]}, {"cell_type": "code", "execution_count": null, "id": "44380577570523278879349135829904343037", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "from pydub import AudioSegment\n", "from time import sleep\n", "\n", "with gr.Blocks() as demo:\n", "    input_audio = gr.Audio(label=\"Input Audio\", type=\"filepath\", format=\"mp3\")\n", "    with gr.Row():\n", "        with gr.Column():\n", "            stream_as_file_btn = gr.Button(\"Stream as File\")\n", "            format = gr.Radio([\"wav\", \"mp3\"], value=\"wav\", label=\"Format\")\n", "            stream_as_file_output = gr.Audio(streaming=True)\n", "\n", "            def stream_file(audio_file, format):\n", "                audio = AudioSegment.from_file(audio_file)\n", "                i = 0\n", "                chunk_size = 1000\n", "                while chunk_size * i < len(audio):\n", "                    chunk = audio[chunk_size * i : chunk_size * (i + 1)]\n", "                    i += 1\n", "                    if chunk:\n", "                        file = f\"/tmp/{i}.{format}\"\n", "                        chunk.export(file, format=format)\n", "                        yield file\n", "                        sleep(0.5)\n", "\n", "            stream_as_file_btn.click(\n", "                stream_file, [input_audio, format], stream_as_file_output\n", "            )\n", "\n", "            gr.Examples(\n", "                [[\"audio/cantina.wav\", \"wav\"], [\"audio/cantina.wav\", \"mp3\"]],\n", "                [input_audio, format],\n", "                fn=stream_file,\n", "                outputs=stream_as_file_output,\n", "            )\n", "\n", "        with gr.Column():\n", "            stream_as_bytes_btn = gr.Button(\"Stream as Bytes\")\n", "            stream_as_bytes_output = gr.Audio(streaming=True)\n", "\n", "            def stream_bytes(audio_file):\n", "                chunk_size = 20_000\n", "                with open(audio_file, \"rb\") as f:\n", "                    while True:\n", "                        chunk = f.read(chunk_size)\n", "                        if chunk:\n", "                            yield chunk\n", "                            sleep(1)\n", "                        else:\n", "                            break\n", "            stream_as_bytes_btn.click(stream_bytes, input_audio, stream_as_bytes_output)\n", "\n", "if __name__ == \"__main__\":\n", "    demo.queue().launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}
+{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: stream_audio_out"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["# Downloading files from the demo repo\n", "import os\n", "os.mkdir('audio')\n", "!wget -q -O audio/cantina.wav https://github.com/gradio-app/gradio/raw/main/demo/stream_audio_out/audio/cantina.wav"]}, {"cell_type": "code", "execution_count": null, "id": "44380577570523278879349135829904343037", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "from pydub import AudioSegment\n", "from time import sleep\n", "import os\n", "\n", "with gr.Blocks() as demo:\n", "    input_audio = gr.Audio(label=\"Input Audio\", type=\"filepath\", format=\"mp3\")\n", "    with gr.Row():\n", "        with gr.Column():\n", "            stream_as_file_btn = gr.Button(\"Stream as File\")\n", "            format = gr.Radio([\"wav\", \"mp3\"], value=\"wav\", label=\"Format\")\n", "            stream_as_file_output = gr.Audio(streaming=True, elem_id=\"stream_as_file_output\", autoplay=True)\n", "\n", "            def stream_file(audio_file, format):\n", "                audio = AudioSegment.from_file(audio_file)\n", "                i = 0\n", "                chunk_size = 1000\n", "                while chunk_size * i < len(audio):\n", "                    chunk = audio[chunk_size * i : chunk_size * (i + 1)]\n", "                    i += 1\n", "                    if chunk:\n", "                        file = f\"/tmp/{i}.{format}\"\n", "                        chunk.export(file, format=format)\n", "                        yield file\n", "                        sleep(0.5)\n", "\n", "            stream_as_file_btn.click(\n", "                stream_file, [input_audio, format], stream_as_file_output\n", "            )\n", "\n", "            gr.Examples(\n", "                [[os.path.join(os.path.abspath(''), \"audio/cantina.wav\"), \"wav\"],\n", "                 [os.path.join(os.path.abspath(''), \"audio/cantina.wav\"), \"mp3\"]],\n", "                [input_audio, format],\n", "                fn=stream_file,\n", "                outputs=stream_as_file_output,\n", "                cache_examples=False,\n", "            )\n", "\n", "        with gr.Column():\n", "            stream_as_bytes_btn = gr.Button(\"Stream as Bytes\")\n", "            stream_as_bytes_output = gr.Audio(streaming=True, elem_id=\"stream_as_bytes_output\", autoplay=True)\n", "\n", "            def stream_bytes(audio_file):\n", "                chunk_size = 20_000\n", "                with open(audio_file, \"rb\") as f:\n", "                    while True:\n", "                        chunk = f.read(chunk_size)\n", "                        if chunk:\n", "                            yield chunk\n", "                            sleep(1)\n", "                        else:\n", "                            break\n", "            stream_as_bytes_btn.click(stream_bytes, input_audio, stream_as_bytes_output)\n", "\n", "if __name__ == \"__main__\":\n", "    demo.queue().launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}

demo/stream_audio_out/run.py

@@ -1,14 +1,15 @@
 import gradio as gr
 from pydub import AudioSegment
 from time import sleep
+import os
 
 with gr.Blocks() as demo:
     input_audio = gr.Audio(label="Input Audio", type="filepath", format="mp3")
     with gr.Row():
         with gr.Column():
             stream_as_file_btn = gr.Button("Stream as File")
             format = gr.Radio(["wav", "mp3"], value="wav", label="Format")
-            stream_as_file_output = gr.Audio(streaming=True)
+            stream_as_file_output = gr.Audio(streaming=True, elem_id="stream_as_file_output", autoplay=True)
 
             def stream_file(audio_file, format):
                 audio = AudioSegment.from_file(audio_file)
@@ -28,15 +29,17 @@ def stream_file(audio_file, format):
             )
 
             gr.Examples(
-                [["audio/cantina.wav", "wav"], ["audio/cantina.wav", "mp3"]],
+                [[os.path.join(os.path.dirname(__file__), "audio/cantina.wav"), "wav"],
+                 [os.path.join(os.path.dirname(__file__), "audio/cantina.wav"), "mp3"]],
                 [input_audio, format],
                 fn=stream_file,
                 outputs=stream_as_file_output,
+                cache_examples=False,
             )
 
         with gr.Column():
             stream_as_bytes_btn = gr.Button("Stream as Bytes")
-            stream_as_bytes_output = gr.Audio(streaming=True)
+            stream_as_bytes_output = gr.Audio(streaming=True, elem_id="stream_as_bytes_output", autoplay=True)
 
             def stream_bytes(audio_file):
                 chunk_size = 20_000

demo/stream_video_out/run.ipynb

@@ -0,0 +1 @@
+{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: stream_video_out"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio opencv-python"]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["# Downloading files from the demo repo\n", "import os\n", "os.mkdir('video')\n", "!wget -q -O video/compliment_bot_screen_recording_3x.mp4 https://github.com/gradio-app/gradio/raw/main/demo/stream_video_out/video/compliment_bot_screen_recording_3x.mp4"]}, {"cell_type": "code", "execution_count": null, "id": "44380577570523278879349135829904343037", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import cv2\n", "import os\n", "\n", "\n", "def process_video(input_video, stream_as_mp4):\n", "    cap = cv2.VideoCapture(input_video)\n", "\n", "    video_codec = cv2.VideoWriter_fourcc(*\"mp4v\") if stream_as_mp4 else cv2.VideoWriter_fourcc(*\"x264\") # type: ignore\n", "    fps = int(cap.get(cv2.CAP_PROP_FPS))\n", "    width  = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))\n", "    height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))\n", "\n", "    iterating, frame = cap.read()\n", "\n", "    n_frames = 0\n", "    n_chunks = 0\n", "    name = f\"output_{n_chunks}\" + \".mp4\" if stream_as_mp4 else \".ts\"\n", "    segment_file = cv2.VideoWriter(name, video_codec, fps, (width, height)) # type: ignore\n", "\n", "    while iterating:\n", "\n", "        # flip frame vertically\n", "        frame = cv2.flip(frame, 0)\n", "        display_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n", "        segment_file.write(display_frame)\n", "        n_frames += 1\n", "        if n_frames == 3 * fps:\n", "            n_chunks += 1\n", "            segment_file.release()\n", "            n_frames = 0\n", "            yield name\n", "            name = f\"output_{n_chunks}\" + \".mp4\" if stream_as_mp4 else \".ts\"\n", "            segment_file = cv2.VideoWriter(name, video_codec, fps, (width, height)) # type: ignore\n", "\n", "        iterating, frame = cap.read()\n", "\n", "    segment_file.release()\n", "    yield name\n", "\n", "with gr.Blocks() as demo:\n", "    gr.Markdown(\"# Video Streaming Out \ud83d\udcf9\")\n", "    with gr.Row():\n", "        with gr.Column():\n", "            input_video = gr.Video(label=\"input\")\n", "            checkbox = gr.Checkbox(label=\"Stream as MP4 file?\", value=False)\n", "        with gr.Column():\n", "            processed_frames = gr.Video(label=\"stream\", streaming=True, autoplay=True, elem_id=\"stream_video_output\")\n", "    with gr.Row():\n", "        process_video_btn = gr.Button(\"process video\")\n", "\n", "    process_video_btn.click(process_video, [input_video, checkbox], [processed_frames])\n", "\n", "    gr.Examples(\n", "        [[os.path.join(os.path.abspath(''), \"video/compliment_bot_screen_recording_3x.mp4\"), False],\n", "            [os.path.join(os.path.abspath(''), \"video/compliment_bot_screen_recording_3x.mp4\"), True]],\n", "        [input_video, checkbox],\n", "        fn=process_video,\n", "        outputs=processed_frames,\n", "        cache_examples=False,\n", "    )\n", "\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}