Use HTTP Livestreaming for audio/video streaming out (#8906)

* HTTP live streaming

* type check

* fix code

* Fix code

* add code

* Video demo

* Fix tests

* Update notebook

* Add guide

* Fix demo

* Allow downloading

* revert

* Fix download filename

* lint

* notebooks

* fix video demo

* Fix config

* Fix audio repeated play bug

* Improve guide

* fix audio?

* Use cantina

* Code

* type check

* add code

* Use runtimeerror

* Add code

.github/workflows/test-functional.yml

@@ -51,11 +51,12 @@ jobs:
         with:
           always_install_pnpm: true
           build_lite: true
-      - name: install outbreak_forecast dependencies
+      - name: install demo dependencies
         run: |
           . venv/bin/activate
           python -m pip install -r demo/outbreak_forecast/requirements.txt
           python -m pip install -r demo/gradio_pdf_demo/requirements.txt
+          python -m pip install -r demo/stream_video_out/requirements.txt
       - run: pnpm exec playwright install chromium firefox
       - name: run browser tests
         run: |

client/python/test/conftest.py

@@ -238,6 +238,30 @@ def show(n):
     return demo
 
 
+@pytest.fixture
+def count_generator_demo_exception():
+    def count(n):
+        for i in range(int(n)):
+            time.sleep(0.01)
+            if i == 5:
+                raise ValueError("Oh no!")
+            yield i
+
+    def show(n):
+        return str(list(range(int(n))))
+
+    with gr.Blocks() as demo:
+        with gr.Column():
+            num = gr.Number(value=10)
+            with gr.Row():
+                count_btn = gr.Button("Count")
+        with gr.Column():
+            out = gr.Textbox()
+
+        count_btn.click(count, num, out, api_name="count")
+    return demo
+
+
 @pytest.fixture
 def file_io_demo():
     demo = gr.Interface(

demo/outbreak_forecast/requirements.txt

@@ -2,4 +2,5 @@ numpy
 matplotlib
 bokeh
 plotly
-altair
+altair
+opencv-python

demo/outbreak_forecast/run.ipynb

@@ -1 +1 @@
-{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: outbreak_forecast\n", "### Generate a plot based on 5 inputs.\n", "        "]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio numpy matplotlib bokeh plotly altair"]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import altair\n", "\n", "import gradio as gr\n", "from math import sqrt\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import plotly.express as px\n", "import pandas as pd\n", "\n", "def outbreak(plot_type, r, month, countries, social_distancing):\n", "    months = [\"January\", \"February\", \"March\", \"April\", \"May\"]\n", "    m = months.index(month)\n", "    start_day = 30 * m\n", "    final_day = 30 * (m + 1)\n", "    x = np.arange(start_day, final_day + 1)\n", "    pop_count = {\"USA\": 350, \"Canada\": 40, \"Mexico\": 300, \"UK\": 120}\n", "    if social_distancing:\n", "        r = sqrt(r)\n", "    df = pd.DataFrame({\"day\": x})\n", "    for country in countries:\n", "        df[country] = x ** (r) * (pop_count[country] + 1)\n", "\n", "    if plot_type == \"Matplotlib\":\n", "        fig = plt.figure()\n", "        plt.plot(df[\"day\"], df[countries].to_numpy())\n", "        plt.title(\"Outbreak in \" + month)\n", "        plt.ylabel(\"Cases\")\n", "        plt.xlabel(\"Days since Day 0\")\n", "        plt.legend(countries)\n", "        return fig\n", "    elif plot_type == \"Plotly\":\n", "        fig = px.line(df, x=\"day\", y=countries)\n", "        fig.update_layout(\n", "            title=\"Outbreak in \" + month,\n", "            xaxis_title=\"Cases\",\n", "            yaxis_title=\"Days Since Day 0\",\n", "        )\n", "        return fig\n", "    elif plot_type == \"Altair\":\n", "        df = df.melt(id_vars=\"day\").rename(columns={\"variable\": \"country\"})\n", "        fig = altair.Chart(df).mark_line().encode(x=\"day\", y=\"value\", color=\"country\")\n", "        return fig\n", "    else:\n", "        raise ValueError(\"A plot type must be selected\")\n", "\n", "inputs = [\n", "    gr.Dropdown([\"Matplotlib\", \"Plotly\", \"Altair\"], label=\"Plot Type\"),\n", "    gr.Slider(1, 4, 3.2, label=\"R\"),\n", "    gr.Dropdown([\"January\", \"February\", \"March\", \"April\", \"May\"], label=\"Month\"),\n", "    gr.CheckboxGroup(\n", "        [\"USA\", \"Canada\", \"Mexico\", \"UK\"], label=\"Countries\", value=[\"USA\", \"Canada\"]\n", "    ),\n", "    gr.Checkbox(label=\"Social Distancing?\"),\n", "]\n", "outputs = gr.Plot()\n", "\n", "demo = gr.Interface(\n", "    fn=outbreak,\n", "    inputs=inputs,\n", "    outputs=outputs,\n", "    examples=[\n", "        [\"Matplotlib\", 2, \"March\", [\"Mexico\", \"UK\"], True],\n", "        [\"Altair\", 2, \"March\", [\"Mexico\", \"Canada\"], True],\n", "        [\"Plotly\", 3.6, \"February\", [\"Canada\", \"Mexico\", \"UK\"], False],\n", "    ],\n", "    cache_examples=True,\n", ")\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}
+{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: outbreak_forecast\n", "### Generate a plot based on 5 inputs.\n", "        "]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio numpy matplotlib bokeh plotly altair opencv-python"]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import altair\n", "\n", "import gradio as gr\n", "from math import sqrt\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import plotly.express as px\n", "import pandas as pd\n", "\n", "def outbreak(plot_type, r, month, countries, social_distancing):\n", "    months = [\"January\", \"February\", \"March\", \"April\", \"May\"]\n", "    m = months.index(month)\n", "    start_day = 30 * m\n", "    final_day = 30 * (m + 1)\n", "    x = np.arange(start_day, final_day + 1)\n", "    pop_count = {\"USA\": 350, \"Canada\": 40, \"Mexico\": 300, \"UK\": 120}\n", "    if social_distancing:\n", "        r = sqrt(r)\n", "    df = pd.DataFrame({\"day\": x})\n", "    for country in countries:\n", "        df[country] = x ** (r) * (pop_count[country] + 1)\n", "\n", "    if plot_type == \"Matplotlib\":\n", "        fig = plt.figure()\n", "        plt.plot(df[\"day\"], df[countries].to_numpy())\n", "        plt.title(\"Outbreak in \" + month)\n", "        plt.ylabel(\"Cases\")\n", "        plt.xlabel(\"Days since Day 0\")\n", "        plt.legend(countries)\n", "        return fig\n", "    elif plot_type == \"Plotly\":\n", "        fig = px.line(df, x=\"day\", y=countries)\n", "        fig.update_layout(\n", "            title=\"Outbreak in \" + month,\n", "            xaxis_title=\"Cases\",\n", "            yaxis_title=\"Days Since Day 0\",\n", "        )\n", "        return fig\n", "    elif plot_type == \"Altair\":\n", "        df = df.melt(id_vars=\"day\").rename(columns={\"variable\": \"country\"})\n", "        fig = altair.Chart(df).mark_line().encode(x=\"day\", y=\"value\", color=\"country\")\n", "        return fig\n", "    else:\n", "        raise ValueError(\"A plot type must be selected\")\n", "\n", "inputs = [\n", "    gr.Dropdown([\"Matplotlib\", \"Plotly\", \"Altair\"], label=\"Plot Type\"),\n", "    gr.Slider(1, 4, 3.2, label=\"R\"),\n", "    gr.Dropdown([\"January\", \"February\", \"March\", \"April\", \"May\"], label=\"Month\"),\n", "    gr.CheckboxGroup(\n", "        [\"USA\", \"Canada\", \"Mexico\", \"UK\"], label=\"Countries\", value=[\"USA\", \"Canada\"]\n", "    ),\n", "    gr.Checkbox(label=\"Social Distancing?\"),\n", "]\n", "outputs = gr.Plot()\n", "\n", "demo = gr.Interface(\n", "    fn=outbreak,\n", "    inputs=inputs,\n", "    outputs=outputs,\n", "    examples=[\n", "        [\"Matplotlib\", 2, \"March\", [\"Mexico\", \"UK\"], True],\n", "        [\"Altair\", 2, \"March\", [\"Mexico\", \"Canada\"], True],\n", "        [\"Plotly\", 3.6, \"February\", [\"Canada\", \"Mexico\", \"UK\"], False],\n", "    ],\n", "    cache_examples=True,\n", ")\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}

demo/stream_audio_out/run.ipynb

@@ -1 +1 @@
-{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: stream_audio_out"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["# Downloading files from the demo repo\n", "import os\n", "os.mkdir('audio')\n", "!wget -q -O audio/cantina.wav https://github.com/gradio-app/gradio/raw/main/demo/stream_audio_out/audio/cantina.wav"]}, {"cell_type": "code", "execution_count": null, "id": "44380577570523278879349135829904343037", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "from pydub import AudioSegment\n", "from time import sleep\n", "\n", "with gr.Blocks() as demo:\n", "    input_audio = gr.Audio(label=\"Input Audio\", type=\"filepath\", format=\"mp3\")\n", "    with gr.Row():\n", "        with gr.Column():\n", "            stream_as_file_btn = gr.Button(\"Stream as File\")\n", "            format = gr.Radio([\"wav\", \"mp3\"], value=\"wav\", label=\"Format\")\n", "            stream_as_file_output = gr.Audio(streaming=True)\n", "\n", "            def stream_file(audio_file, format):\n", "                audio = AudioSegment.from_file(audio_file)\n", "                i = 0\n", "                chunk_size = 1000\n", "                while chunk_size * i < len(audio):\n", "                    chunk = audio[chunk_size * i : chunk_size * (i + 1)]\n", "                    i += 1\n", "                    if chunk:\n", "                        file = f\"/tmp/{i}.{format}\"\n", "                        chunk.export(file, format=format)\n", "                        yield file\n", "                        sleep(0.5)\n", "\n", "            stream_as_file_btn.click(\n", "                stream_file, [input_audio, format], stream_as_file_output\n", "            )\n", "\n", "            gr.Examples(\n", "                [[\"audio/cantina.wav\", \"wav\"], [\"audio/cantina.wav\", \"mp3\"]],\n", "                [input_audio, format],\n", "                fn=stream_file,\n", "                outputs=stream_as_file_output,\n", "            )\n", "\n", "        with gr.Column():\n", "            stream_as_bytes_btn = gr.Button(\"Stream as Bytes\")\n", "            stream_as_bytes_output = gr.Audio(streaming=True)\n", "\n", "            def stream_bytes(audio_file):\n", "                chunk_size = 20_000\n", "                with open(audio_file, \"rb\") as f:\n", "                    while True:\n", "                        chunk = f.read(chunk_size)\n", "                        if chunk:\n", "                            yield chunk\n", "                            sleep(1)\n", "                        else:\n", "                            break\n", "            stream_as_bytes_btn.click(stream_bytes, input_audio, stream_as_bytes_output)\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}
+{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: stream_audio_out"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["# Downloading files from the demo repo\n", "import os\n", "os.mkdir('audio')\n", "!wget -q -O audio/cantina.wav https://github.com/gradio-app/gradio/raw/main/demo/stream_audio_out/audio/cantina.wav"]}, {"cell_type": "code", "execution_count": null, "id": "44380577570523278879349135829904343037", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "from pydub import AudioSegment\n", "from time import sleep\n", "import os\n", "\n", "with gr.Blocks() as demo:\n", "    input_audio = gr.Audio(label=\"Input Audio\", type=\"filepath\", format=\"mp3\")\n", "    with gr.Row():\n", "        with gr.Column():\n", "            stream_as_file_btn = gr.Button(\"Stream as File\")\n", "            format = gr.Radio([\"wav\", \"mp3\"], value=\"wav\", label=\"Format\")\n", "            stream_as_file_output = gr.Audio(streaming=True, elem_id=\"stream_as_file_output\", autoplay=True)\n", "\n", "            def stream_file(audio_file, format):\n", "                audio = AudioSegment.from_file(audio_file)\n", "                i = 0\n", "                chunk_size = 1000\n", "                while chunk_size * i < len(audio):\n", "                    chunk = audio[chunk_size * i : chunk_size * (i + 1)]\n", "                    i += 1\n", "                    if chunk:\n", "                        file = f\"/tmp/{i}.{format}\"\n", "                        chunk.export(file, format=format)\n", "                        yield file\n", "                        sleep(0.5)\n", "\n", "            stream_as_file_btn.click(\n", "                stream_file, [input_audio, format], stream_as_file_output\n", "            )\n", "\n", "            gr.Examples(\n", "                [[os.path.join(os.path.abspath(''), \"audio/cantina.wav\"), \"wav\"],\n", "                 [os.path.join(os.path.abspath(''), \"audio/cantina.wav\"), \"mp3\"]],\n", "                [input_audio, format],\n", "                fn=stream_file,\n", "                outputs=stream_as_file_output,\n", "                cache_examples=False,\n", "            )\n", "\n", "        with gr.Column():\n", "            stream_as_bytes_btn = gr.Button(\"Stream as Bytes\")\n", "            stream_as_bytes_output = gr.Audio(streaming=True, elem_id=\"stream_as_bytes_output\", autoplay=True)\n", "\n", "            def stream_bytes(audio_file):\n", "                chunk_size = 20_000\n", "                with open(audio_file, \"rb\") as f:\n", "                    while True:\n", "                        chunk = f.read(chunk_size)\n", "                        if chunk:\n", "                            yield chunk\n", "                            sleep(1)\n", "                        else:\n", "                            break\n", "            stream_as_bytes_btn.click(stream_bytes, input_audio, stream_as_bytes_output)\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}

demo/stream_audio_out/run.py

@@ -1,14 +1,15 @@
 import gradio as gr
 from pydub import AudioSegment
 from time import sleep
+import os
 
 with gr.Blocks() as demo:
     input_audio = gr.Audio(label="Input Audio", type="filepath", format="mp3")
     with gr.Row():
         with gr.Column():
             stream_as_file_btn = gr.Button("Stream as File")
             format = gr.Radio(["wav", "mp3"], value="wav", label="Format")
-            stream_as_file_output = gr.Audio(streaming=True)
+            stream_as_file_output = gr.Audio(streaming=True, elem_id="stream_as_file_output", autoplay=True)
 
             def stream_file(audio_file, format):
                 audio = AudioSegment.from_file(audio_file)
@@ -28,15 +29,17 @@ def stream_file(audio_file, format):
             )
 
             gr.Examples(
-                [["audio/cantina.wav", "wav"], ["audio/cantina.wav", "mp3"]],
+                [[os.path.join(os.path.dirname(__file__), "audio/cantina.wav"), "wav"],
+                 [os.path.join(os.path.dirname(__file__), "audio/cantina.wav"), "mp3"]],
                 [input_audio, format],
                 fn=stream_file,
                 outputs=stream_as_file_output,
+                cache_examples=False,
             )
 
         with gr.Column():
             stream_as_bytes_btn = gr.Button("Stream as Bytes")
-            stream_as_bytes_output = gr.Audio(streaming=True)
+            stream_as_bytes_output = gr.Audio(streaming=True, elem_id="stream_as_bytes_output", autoplay=True)
 
             def stream_bytes(audio_file):
                 chunk_size = 20_000

demo/stream_video_out/run.ipynb

@@ -0,0 +1 @@
+{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: stream_video_out"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio opencv-python"]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["# Downloading files from the demo repo\n", "import os\n", "os.mkdir('video')\n", "!wget -q -O video/compliment_bot_screen_recording_3x.mp4 https://github.com/gradio-app/gradio/raw/main/demo/stream_video_out/video/compliment_bot_screen_recording_3x.mp4"]}, {"cell_type": "code", "execution_count": null, "id": "44380577570523278879349135829904343037", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import cv2\n", "import os\n", "from pathlib import Path\n", "import atexit\n", "\n", "current_dir = Path(__file__).resolve().parent\n", "\n", "\n", "def delete_files():\n", "    for p in Path(current_dir).glob(\"*.ts\"):\n", "        p.unlink()\n", "    for p in Path(current_dir).glob(\"*.mp4\"):\n", "        p.unlink()\n", "\n", "atexit.register(delete_files)\n", "\n", "\n", "def process_video(input_video, stream_as_mp4):\n", "    cap = cv2.VideoCapture(input_video)\n", "\n", "    video_codec = cv2.VideoWriter_fourcc(*\"mp4v\") if stream_as_mp4 else cv2.VideoWriter_fourcc(*\"x264\") # type: ignore\n", "    fps = int(cap.get(cv2.CAP_PROP_FPS))\n", "    width  = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))\n", "    height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))\n", "\n", "    iterating, frame = cap.read()\n", "\n", "    n_frames = 0\n", "    n_chunks = 0\n", "    name = str(current_dir / f\"output_{n_chunks}{'.mp4' if stream_as_mp4 else '.ts'}\")\n", "    segment_file = cv2.VideoWriter(name, video_codec, fps, (width, height)) # type: ignore\n", "\n", "    while iterating:\n", "\n", "        # flip frame vertically\n", "        frame = cv2.flip(frame, 0)\n", "        display_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n", "        segment_file.write(display_frame)\n", "        n_frames += 1\n", "        if n_frames == 3 * fps:\n", "            n_chunks += 1\n", "            segment_file.release()\n", "            n_frames = 0\n", "            yield name\n", "            name = str(current_dir / f\"output_{n_chunks}{'.mp4' if stream_as_mp4 else '.ts'}\")\n", "            segment_file = cv2.VideoWriter(name, video_codec, fps, (width, height)) # type: ignore\n", "\n", "        iterating, frame = cap.read()\n", "\n", "    segment_file.release()\n", "    yield name\n", "\n", "with gr.Blocks() as demo:\n", "    gr.Markdown(\"# Video Streaming Out \ud83d\udcf9\")\n", "    with gr.Row():\n", "        with gr.Column():\n", "            input_video = gr.Video(label=\"input\")\n", "            checkbox = gr.Checkbox(label=\"Stream as MP4 file?\", value=False)\n", "        with gr.Column():\n", "            processed_frames = gr.Video(label=\"stream\", streaming=True, autoplay=True, elem_id=\"stream_video_output\")\n", "    with gr.Row():\n", "        process_video_btn = gr.Button(\"process video\")\n", "\n", "    process_video_btn.click(process_video, [input_video, checkbox], [processed_frames])\n", "\n", "    gr.Examples(\n", "        [[os.path.join(os.path.abspath(''), \"video/compliment_bot_screen_recording_3x.mp4\"), False],\n", "            [os.path.join(os.path.abspath(''), \"video/compliment_bot_screen_recording_3x.mp4\"), True]],\n", "        [input_video, checkbox],\n", "        fn=process_video,\n", "        outputs=processed_frames,\n", "        cache_examples=False,\n", "    )\n", "\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}