Skip to content

Commit

Permalink
Video gallery (#9052)
Browse files Browse the repository at this point in the history 
* video support

* tests and backend changes

* undo main merge

* upload fix

* Revert "undo main merge"

This reverts commit e2a26e6.

* type fixes

* format

* pr fixes

* Update gradio/components/gallery.py

Co-authored-by: Abubakar Abid <abubakar@huggingface.co>

* Update gradio/components/gallery.py

Co-authored-by: Abubakar Abid <abubakar@huggingface.co>

* type fix

* thumbnails

* thumbnail type

* remove thumbnail generation

* add changeset

* test fixes

* test fixes

* python test fix

* python test fixc

* fix

* fix

* story fix

---------

Co-authored-by: Abubakar Abid <abubakar@huggingface.co>
Co-authored-by: gradio-pr-bot <gradio-pr-bot@users.noreply.github.com>
  • Loading branch information
@dawoodkhan82 @abidlabs @gradio-pr-bot
3 people authored Aug 20, 2024
1 parent 864cd0f commit f3652eb
Show file tree
Hide file tree
Showing 19 changed files with 276 additions and 103 deletions.
7 changes: 7 additions & 0 deletions .changeset/light-bats-arrive.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
"@gradio/core": minor
"@gradio/gallery": minor
"gradio": minor
---

feat:Video gallery
Binary file added demo/gallery_component/files/cheetah.jpg
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added demo/gallery_component/files/world.mp4
View file
Binary file not shown.
2 changes: 1 addition & 1 deletion demo/gallery_component/run.ipynb
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: gallery_component"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "\n", "with gr.Blocks() as demo:\n", " cheetahs = [\n", " \"https://upload.wikimedia.org/wikipedia/commons/0/09/TheCheethcat.jpg\",\n", " \"https://nationalzoo.si.edu/sites/default/files/animals/cheetah-003.jpg\",\n", " \"https://img.etimg.com/thumb/msid-50159822,width-650,imgsize-129520,,resizemode-4,quality-100/.jpg\",\n", " \"https://nationalzoo.si.edu/sites/default/files/animals/cheetah-002.jpg\",\n", " \"https://images.theconversation.com/files/375893/original/file-20201218-13-a8h8uq.jpg?ixlib=rb-1.1.0&rect=16%2C407%2C5515%2C2924&q=45&auto=format&w=496&fit=clip\",\n", " ]\n", " gr.Gallery(value=cheetahs, columns=4)\n", "\n", "demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}
{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: gallery_component"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["# Downloading files from the demo repo\n", "import os\n", "os.mkdir('files')\n", "!wget -q -O files/cheetah.jpg https://github.com/gradio-app/gradio/raw/main/demo/gallery_component/files/cheetah.jpg\n", "!wget -q -O files/world.mp4 https://github.com/gradio-app/gradio/raw/main/demo/gallery_component/files/world.mp4"]}, {"cell_type": "code", "execution_count": null, "id": "44380577570523278879349135829904343037", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "\n", "with gr.Blocks() as demo:\n", " gallery_items = [\n", " (\"https://upload.wikimedia.org/wikipedia/commons/0/09/TheCheethcat.jpg\", \"cheetah1\"),\n", " (\"https://nationalzoo.si.edu/sites/default/files/animals/cheetah-003.jpg\", \"cheetah2\"),\n", " (\"https://videos.pexels.com/video-files/3209828/3209828-uhd_2560_1440_25fps.mp4\", \"world\"),\n", " (\"files/cheetah.jpg\", \"cheetah3\"),\n", " (\"files/world.mp4\", \"world2\")\n", " ]\n", " gr.Gallery(value=gallery_items, columns=4)\n", "\n", "demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}
14 changes: 7 additions & 7 deletions demo/gallery_component/run.py
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
import gradio as gr

with gr.Blocks() as demo:
cheetahs = [
"https://upload.wikimedia.org/wikipedia/commons/0/09/TheCheethcat.jpg",
"https://nationalzoo.si.edu/sites/default/files/animals/cheetah-003.jpg",
"https://img.etimg.com/thumb/msid-50159822,width-650,imgsize-129520,,resizemode-4,quality-100/.jpg",
"https://nationalzoo.si.edu/sites/default/files/animals/cheetah-002.jpg",
"https://images.theconversation.com/files/375893/original/file-20201218-13-a8h8uq.jpg?ixlib=rb-1.1.0&rect=16%2C407%2C5515%2C2924&q=45&auto=format&w=496&fit=clip",
gallery_items = [
("https://upload.wikimedia.org/wikipedia/commons/0/09/TheCheethcat.jpg", "cheetah1"),
("https://nationalzoo.si.edu/sites/default/files/animals/cheetah-003.jpg", "cheetah2"),
("https://videos.pexels.com/video-files/3209828/3209828-uhd_2560_1440_25fps.mp4", "world"),
("files/cheetah.jpg", "cheetah3"),
("files/world.mp4", "world2")
]
gr.Gallery(value=cheetahs, columns=4)
gr.Gallery(value=gallery_items, columns=4)

demo.launch()
2 changes: 1 addition & 1 deletion demo/gallery_component_events/run.ipynb
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: gallery_component_events"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "\n", "with gr.Blocks() as demo:\n", " cheetahs = [\n", " \"https://gradio-builds.s3.amazonaws.com/assets/cheetah-003.jpg\",\n", " \"https://gradio-builds.s3.amazonaws.com/assets/lite-logo.png\",\n", " \"https://gradio-builds.s3.amazonaws.com/assets/TheCheethcat.jpg\",\n", " ]\n", " with gr.Row():\n", " with gr.Column():\n", " gal = gr.Gallery(columns=4, interactive=True, label=\"Input Gallery\")\n", " btn = gr.Button()\n", " with gr.Column():\n", " output_gal = gr.Gallery(columns=4, interactive=True, label=\"Output Gallery\")\n", " with gr.Row():\n", " textbox = gr.Json(label=\"uploaded files\")\n", " num_upload = gr.Number(value=0, label=\"Num Upload\")\n", " num_change = gr.Number(value=0, label=\"Num Change\")\n", " select_output = gr.Textbox(label=\"Select Data\")\n", " gal.upload(lambda v,n: (v, v, n+1), [gal, num_upload], [textbox, output_gal, num_upload])\n", " gal.change(lambda v,n: (v, v, n+1), [gal, num_change], [textbox, output_gal, num_change])\n", "\n", " btn.click(lambda: cheetahs, None, [output_gal])\n", "\n", " def select(select_data: gr.SelectData):\n", " return select_data.value['image']['url']\n", "\n", " output_gal.select(select, None, select_output)\n", "\n", "if __name__ == \"__main__\":\n", " demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}
{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: gallery_component_events"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "\n", "with gr.Blocks() as demo:\n", " files = [\n", " \"https://gradio-builds.s3.amazonaws.com/assets/cheetah-003.jpg\",\n", " \"https://gradio-static-files.s3.amazonaws.com/world.mp4\",\n", " \"https://gradio-builds.s3.amazonaws.com/assets/TheCheethcat.jpg\",\n", " ]\n", " with gr.Row():\n", " with gr.Column():\n", " gal = gr.Gallery(columns=4, interactive=True, label=\"Input Gallery\")\n", " btn = gr.Button()\n", " with gr.Column():\n", " output_gal = gr.Gallery(columns=4, interactive=True, label=\"Output Gallery\")\n", " with gr.Row():\n", " textbox = gr.Json(label=\"uploaded files\")\n", " num_upload = gr.Number(value=0, label=\"Num Upload\")\n", " num_change = gr.Number(value=0, label=\"Num Change\")\n", " select_output = gr.Textbox(label=\"Select Data\")\n", " gal.upload(lambda v,n: (v, v, n+1), [gal, num_upload], [textbox, output_gal, num_upload])\n", " gal.change(lambda v,n: (v, v, n+1), [gal, num_change], [textbox, output_gal, num_change])\n", "\n", " btn.click(lambda: files, None, [output_gal])\n", "\n", " def select(select_data: gr.SelectData):\n", " return select_data.value['image']['url'] if 'image' in select_data.value else select_data.value['video']['url']\n", "\n", " output_gal.select(select, None, select_output)\n", "\n", "if __name__ == \"__main__\":\n", " demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}
8 changes: 4 additions & 4 deletions demo/gallery_component_events/run.py
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import gradio as gr

with gr.Blocks() as demo:
cheetahs = [
files = [
"https://gradio-builds.s3.amazonaws.com/assets/cheetah-003.jpg",
"https://gradio-builds.s3.amazonaws.com/assets/lite-logo.png",
"https://gradio-static-files.s3.amazonaws.com/world.mp4",
"https://gradio-builds.s3.amazonaws.com/assets/TheCheethcat.jpg",
]
with gr.Row():
Expand All @@ -20,10 +20,10 @@
gal.upload(lambda v,n: (v, v, n+1), [gal, num_upload], [textbox, output_gal, num_upload])
gal.change(lambda v,n: (v, v, n+1), [gal, num_change], [textbox, output_gal, num_change])

btn.click(lambda: cheetahs, None, [output_gal])
btn.click(lambda: files, None, [output_gal])

def select(select_data: gr.SelectData):
return select_data.value['image']['url']
return select_data.value['image']['url'] if 'image' in select_data.value else select_data.value['video']['url']

output_gal.select(select, None, select_output)

Expand Down
66 changes: 49 additions & 17 deletions gradio/components/gallery.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import numpy as np
import PIL.Image
from gradio_client import handle_file
from gradio_client import utils as client_utils
from gradio_client.documentation import document
from gradio_client.utils import is_http_url_like

Expand All @@ -28,24 +29,29 @@
if TYPE_CHECKING:
from gradio.components import Timer

GalleryImageType = Union[np.ndarray, PIL.Image.Image, Path, str]
CaptionedGalleryImageType = tuple[GalleryImageType, str]
GalleryMediaType = Union[np.ndarray, PIL.Image.Image, Path, str]
CaptionedGalleryMediaType = tuple[GalleryMediaType, str]


class GalleryImage(GradioModel):
image: FileData
caption: Optional[str] = None


class GalleryVideo(GradioModel):
video: FileData
caption: Optional[str] = None


class GalleryData(GradioRootModel):
root: list[GalleryImage]
root: list[Union[GalleryImage, GalleryVideo]]


@document()
class Gallery(Component):
"""
Creates a gallery component that allows displaying a grid of images, and optionally captions. If used as an input, the user can upload images to the gallery.
If used as an output, the user can click on individual images to view them at a higher resolution.
Creates a gallery component that allows displaying a grid of images or videos, and optionally captions. If used as an input, the user can upload images or videos to the gallery.
If used as an output, the user can click on individual images or videos to view them at a higher resolution.
Demos: fake_gan
"""
Expand All @@ -63,6 +69,7 @@ def __init__(
) = None,
*,
format: str = "webp",
file_types: list[str] | None = None,
label: str | None = None,
every: Timer | float | None = None,
inputs: Component | Sequence[Component] | set[Component] | None = None,
Expand Down Expand Up @@ -92,8 +99,9 @@ def __init__(
):
"""
Parameters:
value: List of images to display in the gallery by default. If callable, the function will be called whenever the app loads to set the initial value of the component.
value: List of images or videos to display in the gallery by default. If callable, the function will be called whenever the app loads to set the initial value of the component.
format: Format to save images before they are returned to the frontend, such as 'jpeg' or 'png'. This parameter only applies to images that are returned from the prediction function as numpy arrays or PIL Images. The format should be supported by the PIL library.
file_types: List of file extensions or types of files to be uploaded (e.g. ['image', 'video']), when this is used as an input component. "image" allows only image files to be uploaded, "video" allows only video files to be uploaded, ".mp4" allows only mp4 files to be uploaded, etc. If None, any image and video files types are allowed.
label: The label for this component. Appears above the component and is also used as the header if there are a table of examples for this component. If None and used in a `gr.Interface`, the label will be the name of the parameter this component is assigned to.
every: Continously calls `value` to recalculate it if `value` is a function (has no effect otherwise). Can provide a Timer whose tick resets `value`, or a float that provides the regular interval for the reset Timer.
inputs: Components that are used as inputs to calculate `value` if `value` is a function (has no effect otherwise). `value` is recalculated any time the inputs change.
Expand Down Expand Up @@ -134,6 +142,7 @@ def __init__(
self.selected_index = selected_index
self.type = type
self.show_fullscreen_button = show_fullscreen_button
self.file_types = file_types

self.show_share_button = (
(utils.get_space() is not None)
Expand Down Expand Up @@ -167,27 +176,31 @@ def preprocess(
):
"""
Parameters:
payload: a list of images, or list of (image, caption) tuples
payload: a list of images or videos, or list of (media, caption) tuples
Returns:
Passes the list of images as a list of (image, caption) tuples, or a list of (image, None) tuples if no captions are provided (which is usually the case). The image can be a `str` file path, a `numpy` array, or a `PIL.Image` object depending on `type`.
Passes the list of images or videos as a list of (media, caption) tuples, or a list of (media, None) tuples if no captions are provided (which is usually the case). Images can be a `str` file path, a `numpy` array, or a `PIL.Image` object depending on `type`. Videos are always `str` file path.
"""
if payload is None or not payload.root:
return None
data = []
for gallery_element in payload.root:
image = self.convert_to_type(gallery_element.image.path, self.type) # type: ignore
data.append((image, gallery_element.caption))
media = (
gallery_element.video.path
if (type(gallery_element) is GalleryVideo)
else self.convert_to_type(gallery_element.image.path, self.type) # type: ignore
)
data.append((media, gallery_element.caption))
return data

def postprocess(
self,
value: list[GalleryImageType | CaptionedGalleryImageType] | None,
value: list[GalleryMediaType | CaptionedGalleryMediaType] | None,
) -> GalleryData:
"""
Parameters:
value: Expects the function to return a `list` of images, or `list` of (image, `str` caption) tuples. Each image can be a `str` file path, a `numpy` array, or a `PIL.Image` object.
value: Expects the function to return a `list` of images or videos, or `list` of (media, `str` caption) tuples. Each image can be a `str` file path, a `numpy` array, or a `PIL.Image` object. Each video can be a `str` file path.
Returns:
a list of images, or list of (image, caption) tuples
a list of images or videos, or list of (media, caption) tuples
"""
if value is None:
return GalleryData(root=[])
Expand All @@ -197,6 +210,7 @@ def _save(img):
url = None
caption = None
orig_name = None
mime_type = None
if isinstance(img, (tuple, list)):
img, caption = img
if isinstance(img, np.ndarray):
Expand All @@ -211,6 +225,7 @@ def _save(img):
file_path = str(utils.abspath(file))
elif isinstance(img, str):
file_path = img
mime_type = client_utils.get_mimetype(file_path)
if is_http_url_like(img):
url = img
orig_name = Path(urlparse(img).path).name
Expand All @@ -220,12 +235,29 @@ def _save(img):
elif isinstance(img, Path):
file_path = str(img)
orig_name = img.name
mime_type = client_utils.get_mimetype(file_path)
else:
raise ValueError(f"Cannot process type as image: {type(img)}")
return GalleryImage(
image=FileData(path=file_path, url=url, orig_name=orig_name),
caption=caption,
)
if mime_type is not None and "video" in mime_type:
return GalleryVideo(
video=FileData(
path=file_path,
url=url,
orig_name=orig_name,
mime_type=mime_type,
),
caption=caption,
)
else:
return GalleryImage(
image=FileData(
path=file_path,
url=url,
orig_name=orig_name,
mime_type=mime_type,
),
caption=caption,
)

if wasm_utils.IS_WASM:
for img in value:
Expand Down
2 changes: 1 addition & 1 deletion js/core/src/lang/en.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@
"drop_file": "Drop File Here",
"drop_image": "Drop Image Here",
"drop_video": "Drop Video Here",
"drop_gallery": "Drop Image(s) Here",
"drop_gallery": "Drop Media Here",
"paste_clipboard": "Paste from Clipboard"
}
}
10 changes: 9 additions & 1 deletion js/gallery/Gallery.stories.svelte
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,14 @@
},
caption: "A fast cheetah"
},
{
video: {
path: "https://gradio-builds.s3.amazonaws.com/demo-files/world.mp4",
url: "https://gradio-builds.s3.amazonaws.com/demo-files/world.mp4",
orig_name: "world.mp4"
},
caption: "The world"
},
{
image: {
path: "https://gradio-builds.s3.amazonaws.com/demo-files/cheetah-002.jpg",
Expand Down Expand Up @@ -139,7 +147,7 @@
play={async ({ canvasElement }) => {
const canvas = within(canvasElement);

const image = canvas.getByLabelText("Thumbnail 1 of 7");
const image = canvas.getByLabelText("Thumbnail 1 of 8");
await userEvent.click(image);
const expand_btn = canvas.getByRole("button", {
name: "View in full screen"
Expand Down
Loading

0 comments on commit f3652eb

Please sign in to comment.