From a2e44f2f1e29e04a01874ef7fbfe8e27c97e4cc9 Mon Sep 17 00:00:00 2001
From: freddyaboulton <alfonsoboulton@gmail.com>
Date: Fri, 16 Aug 2024 19:58:38 -0400
Subject: [PATCH 1/9] fix

---
 client/js/src/helpers/api_info.ts            |  4 +-
 client/js/src/types.ts                       |  1 +
 client/js/src/utils/submit.ts                |  3 +-
 js/audio/Index.svelte                        |  2 +
 js/audio/interactive/InteractiveAudio.svelte | 11 +++++-
 js/audio/streaming/StreamAudio.svelte        | 36 +++++++++++++++++-
 js/core/src/Blocks.svelte                    | 20 +++++++++-
 js/core/src/init.ts                          | 10 +++--
 js/core/src/lang/en.json                     |  3 +-
 js/icons/src/Spinner.svelte                  | 40 ++++++++++++++++++++
 js/icons/src/index.ts                        |  1 +
 js/image/Index.svelte                        |  2 +
 js/image/shared/ImageUploader.svelte         |  2 +
 js/image/shared/Webcam.svelte                | 40 ++++++++++++++++----
 14 files changed, 157 insertions(+), 18 deletions(-)
 create mode 100644 js/icons/src/Spinner.svelte

diff --git a/client/js/src/helpers/api_info.ts b/client/js/src/helpers/api_info.ts
index 6d9d5bb37b0a1..b500c539fc018 100644
--- a/client/js/src/helpers/api_info.ts
+++ b/client/js/src/helpers/api_info.ts
@@ -247,6 +247,7 @@ export function handle_message(
 		| "unexpected_error";
 	data?: any;
 	status?: Status;
+	original_msg?: string;
 } {
 	const queue = true;
 	switch (data.msg) {
@@ -373,7 +374,8 @@ export function handle_message(
 					position: 0,
 					success: data.success,
 					eta: data.eta
-				}
+				},
+				original_msg: "process_starts"
 			};
 	}
 
diff --git a/client/js/src/types.ts b/client/js/src/types.ts
index 01afb0d39b783..bb3d38703e00f 100644
--- a/client/js/src/types.ts
+++ b/client/js/src/types.ts
@@ -363,6 +363,7 @@ export interface StatusMessage extends Status {
 	type: "status";
 	endpoint: string;
 	fn_index: number;
+	original_msg?: string;
 }
 
 export interface PayloadMessage extends Payload {
diff --git a/client/js/src/utils/submit.ts b/client/js/src/utils/submit.ts
index a109784fa01fa..afcf5e588e41a 100644
--- a/client/js/src/utils/submit.ts
+++ b/client/js/src/utils/submit.ts
@@ -598,7 +598,7 @@ export function submit(
 							event_id_final = event_id;
 							let callback = async function (_data: object): Promise<void> {
 								try {
-									const { type, status, data } = handle_message(
+									const { type, status, data, original_msg } = handle_message(
 										_data,
 										last_status[fn_index]
 									);
@@ -614,6 +614,7 @@ export function submit(
 											endpoint: _endpoint,
 											fn_index,
 											time: new Date(),
+											original_msg: original_msg,
 											...status
 										});
 									} else if (type === "complete") {
diff --git a/js/audio/Index.svelte b/js/audio/Index.svelte
index 6bced2a49843e..9e9ed23e231ff 100644
--- a/js/audio/Index.svelte
+++ b/js/audio/Index.svelte
@@ -42,6 +42,7 @@
 	export let stream_every: number;
 
 	export let close_stream: () => void;
+	export let open_stream: () => void;
 	export let set_time_limit: (time: number) => void;
 	export let gradio: Gradio<{
 		input: never;
@@ -246,6 +247,7 @@
 			{trim_region_settings}
 			{stream_every}
 			bind:close_stream
+			bind:open_stream
 			bind:set_time_limit
 			upload={gradio.client.upload}
 			stream_handler={gradio.client.stream}
diff --git a/js/audio/interactive/InteractiveAudio.svelte b/js/audio/interactive/InteractiveAudio.svelte
index 17325c4798896..31a8f77c2bb78 100644
--- a/js/audio/interactive/InteractiveAudio.svelte
+++ b/js/audio/interactive/InteractiveAudio.svelte
@@ -41,13 +41,19 @@
 	export let stream_every: number;
 
 	let time_limit: number | null = null;
+	let queue_joined = false;
 
 	export const close_stream: () => void = () => {
 		time_limit = null;
+		stream_open = false;
+		queue_joined = false;
 	};
 	export const set_time_limit = (time: number): void => {
 		if (recording) time_limit = time;
 	};
+	export const open_stream: () => void = () => {
+		if (recording) stream_open = true;
+	};
 
 	$: dispatch("drag", dragging);
 
@@ -60,6 +66,7 @@
 	let pending_stream: Uint8Array[] = [];
 	let submit_pending_stream_on_pending_end = false;
 	let inited = false;
+	let stream_open = false;
 
 	const NUM_HEADER_BYTES = 44;
 	let audio_chunks: Blob[] = [];
@@ -167,7 +174,8 @@
 			pending_stream.push(payload);
 		} else {
 			let blobParts = [header].concat(pending_stream, [payload]);
-			dispatch_blob(blobParts, "stream");
+			if (stream_open || !queue_joined) dispatch_blob(blobParts, "stream");
+			queue_joined = true;
 			pending_stream = [];
 		}
 	}
@@ -240,6 +248,7 @@
 					{i18n}
 					{waveform_settings}
 					{waveform_options}
+					waiting={queue_joined && !stream_open}
 				/>
 			{:else}
 				<AudioRecorder
diff --git a/js/audio/streaming/StreamAudio.svelte b/js/audio/streaming/StreamAudio.svelte
index fe3ef5c6d59c2..b775794cc13ed 100644
--- a/js/audio/streaming/StreamAudio.svelte
+++ b/js/audio/streaming/StreamAudio.svelte
@@ -1,6 +1,7 @@
 <script lang="ts">
 	import { onMount } from "svelte";
 	import type { I18nFormatter } from "@gradio/utils";
+	import { Spinner } from "@gradio/icons";
 	import WaveSurfer from "wavesurfer.js";
 	import RecordPlugin from "wavesurfer.js/dist/plugins/record.js";
 	import type { WaveformOptions } from "../shared/types";
@@ -15,6 +16,8 @@
 	export let waveform_options: WaveformOptions = {
 		show_recording_waveform: true
 	};
+	export let waiting = false;
+	$: console.log("waiting", waiting);
 
 	let micWaveform: WaveSurfer;
 	let waveformRecord: RecordPlugin;
@@ -48,7 +51,7 @@
 		/>
 	{/if}
 	<div class="controls">
-		{#if recording}
+		{#if recording && !waiting}
 			<button
 				class={paused_recording ? "stop-button-paused" : "stop-button"}
 				on:click={() => {
@@ -62,6 +65,18 @@
 				</span>
 				{paused_recording ? i18n("audio.pause") : i18n("audio.stop")}
 			</button>
+		{:else if recording && waiting}
+			<button
+				class="spinner-button"
+				on:click={() => {
+					stop();
+				}}
+			>
+				<div class="icon">
+					<Spinner />
+				</div>
+				{i18n("audio.waiting")}
+			</button>
 		{:else}
 			<button
 				class="record-button"
@@ -95,6 +110,13 @@
 		margin: var(--spacing-xl);
 	}
 
+	.icon {
+		width: var(--size-4);
+		height: var(--size-4);
+		fill: var(--primary-600);
+		stroke: var(--primary-600);
+	}
+
 	.stop-button-paused {
 		display: none;
 		height: var(--size-8);
@@ -136,6 +158,18 @@
 		display: flex;
 	}
 
+	.spinner-button {
+		height: var(--size-8);
+		width: var(--size-24);
+		background-color: var(--block-background-fill);
+		border-radius: var(--radius-3xl);
+		align-items: center;
+		border: 1px solid var(--primary-600);
+		margin: 0 var(--spacing-xl);
+		display: flex;
+		justify-content: space-evenly;
+	}
+
 	.record-button::before {
 		content: "";
 		height: var(--size-4);
diff --git a/js/core/src/Blocks.svelte b/js/core/src/Blocks.svelte
index 63de8eabcf511..156f3e3ab4aec 100644
--- a/js/core/src/Blocks.svelte
+++ b/js/core/src/Blocks.svelte
@@ -51,7 +51,7 @@
 		targets,
 		update_value,
 		get_data,
-		close_stream,
+		modify_stream,
 		set_time_limit,
 		loading_status,
 		scheduled_updates,
@@ -371,6 +371,19 @@
 				];
 			}
 
+			function open_stream_events(
+				status: StatusMessage,
+				id: number,
+				dep: Dependency
+			): void {
+				if (
+					status.original_msg === "process_starts" &&
+					dep.connection === "stream"
+				) {
+					modify_stream(id, "open");
+				}
+			}
+
 			function handle_status_update(message: StatusMessage): void {
 				const { fn_index, ...status } = message;
 				if (status.stage === "streaming" && status.time_limit) {
@@ -378,6 +391,9 @@
 						set_time_limit(id, status.time_limit);
 					});
 				}
+				dep.inputs.forEach((id) => {
+					open_stream_events(message, id, dep);
+				});
 				//@ts-ignore
 				loading_status.update({
 					...status,
@@ -428,7 +444,7 @@
 						}
 					});
 					dep.inputs.forEach((id) => {
-						close_stream(id);
+						modify_stream(id, "close");
 					});
 					submit_map.delete(dep_index);
 				}
diff --git a/js/core/src/init.ts b/js/core/src/init.ts
index 7fe927dadff08..614afb87255b5 100644
--- a/js/core/src/init.ts
+++ b/js/core/src/init.ts
@@ -28,7 +28,7 @@ export function create_components(): {
 	targets: Writable<TargetMap>;
 	update_value: (updates: UpdateTransaction[]) => void;
 	get_data: (id: number) => any | Promise<any>;
-	close_stream: (id: number) => void;
+	modify_stream: (id: number, action: "open" | "close") => void;
 	set_time_limit: (id: number, time_limit: number | undefined) => void;
 	loading_status: ReturnType<typeof create_loading_status_store>;
 	scheduled_updates: Writable<boolean>;
@@ -347,10 +347,12 @@ export function create_components(): {
 		return comp.props.value;
 	}
 
-	function close_stream(id: number): void {
+	function modify_stream(id: number, action: "open" | "close"): void {
 		const comp = _component_map.get(id);
-		if (comp && comp.instance.close_stream) {
+		if (comp && comp.instance.close_stream && action == "close") {
 			comp.instance.close_stream();
+		} else if (comp && comp.instance.open_stream && action == "open") {
+			comp.instance.open_stream();
 		}
 	}
 
@@ -366,7 +368,7 @@ export function create_components(): {
 		targets: target_map,
 		update_value,
 		get_data,
-		close_stream,
+		modify_stream,
 		set_time_limit,
 		loading_status,
 		scheduled_updates: update_scheduled_store,
diff --git a/js/core/src/lang/en.json b/js/core/src/lang/en.json
index 35d662c993602..5b60e7cd5d1af 100644
--- a/js/core/src/lang/en.json
+++ b/js/core/src/lang/en.json
@@ -16,7 +16,8 @@
 		"record": "Record",
 		"no_microphone": "No microphone found",
 		"pause": "Pause",
-		"play": "Play"
+		"play": "Play",
+		"waiting": "Waiting"
 	},
 	"blocks": {
 		"connection_can_break": "On mobile, the connection can break if this tab is unfocused or the device sleeps, losing your position in queue.",
diff --git a/js/icons/src/Spinner.svelte b/js/icons/src/Spinner.svelte
new file mode 100644
index 0000000000000..1f9c0c8c56951
--- /dev/null
+++ b/js/icons/src/Spinner.svelte
@@ -0,0 +1,40 @@
+<svg
+	xmlns="http://www.w3.org/2000/svg"
+	width="100%"
+	height="100%"
+	viewBox="0 0 50 50"
+>
+	<circle
+		cx="25"
+		cy="25"
+		r="20"
+		fill="none"
+		stroke-width="3.0"
+		stroke-linecap="round"
+		stroke-dasharray="94.2477796076938 94.2477796076938"
+		stroke-dashoffset="0"
+	>
+		<animateTransform
+			attributeName="transform"
+			type="rotate"
+			from="0 25 25"
+			to="360 25 25"
+			repeatCount="indefinite"
+		/>
+	</circle>
+</svg>
+
+<style>
+	svg {
+		animation: spin 1.5s linear infinite;
+	}
+
+	@keyframes spin {
+		0% {
+			transform: rotate(0deg);
+		}
+		100% {
+			transform: rotate(360deg);
+		}
+	}
+</style>
diff --git a/js/icons/src/index.ts b/js/icons/src/index.ts
index d2f1fedac84ee..360494856ae5c 100644
--- a/js/icons/src/index.ts
+++ b/js/icons/src/index.ts
@@ -58,3 +58,4 @@ export { default as VolumeHigh } from "./VolumeHigh.svelte";
 export { default as VolumeMuted } from "./VolumeMuted.svelte";
 export { default as Warning } from "./Warning.svelte";
 export { default as Webcam } from "./Webcam.svelte";
+export { default as Spinner } from "./Spinner.svelte";
diff --git a/js/image/Index.svelte b/js/image/Index.svelte
index d95b0cb99ede5..c488bf88541bf 100644
--- a/js/image/Index.svelte
+++ b/js/image/Index.svelte
@@ -23,6 +23,7 @@
 	type sources = "upload" | "webcam" | "clipboard" | null;
 
 	export let close_stream: () => void;
+	export let open_stream: () => void;
 	export let set_time_limit: (arg0: number) => void;
 	export let value_is_output = false;
 	export let elem_id = "";
@@ -174,6 +175,7 @@
 			{mirror_webcam}
 			{stream_every}
 			bind:close_stream
+			bind:open_stream
 			bind:set_time_limit
 			max_file_size={gradio.max_file_size}
 			i18n={gradio.i18n}
diff --git a/js/image/shared/ImageUploader.svelte b/js/image/shared/ImageUploader.svelte
index 32e76010158f2..57fce22ced992 100644
--- a/js/image/shared/ImageUploader.svelte
+++ b/js/image/shared/ImageUploader.svelte
@@ -31,6 +31,7 @@
 	export let stream_every: number;
 
 	export let close_stream: () => void;
+	export let open_stream: () => void;
 	export let set_time_limit: (arg0: number) => void;
 
 	let upload_input: Upload;
@@ -150,6 +151,7 @@
 				{i18n}
 				{upload}
 				bind:close_stream
+				bind:open_stream
 				bind:set_time_limit
 			/>
 		{:else if value !== null && !streaming}
diff --git a/js/image/shared/Webcam.svelte b/js/image/shared/Webcam.svelte
index 2349267197ea8..0ef6227382bef 100644
--- a/js/image/shared/Webcam.svelte
+++ b/js/image/shared/Webcam.svelte
@@ -1,6 +1,12 @@
 <script lang="ts">
 	import { createEventDispatcher, onMount } from "svelte";
-	import { Camera, Circle, Square, DropdownArrow } from "@gradio/icons";
+	import {
+		Camera,
+		Circle,
+		Square,
+		DropdownArrow,
+		Spinner
+	} from "@gradio/icons";
 	import type { I18nFormatter } from "@gradio/utils";
 	import { StreamingBar } from "@gradio/statustracker";
 	import { type FileData, type Client, prepare_files } from "@gradio/client";
@@ -15,12 +21,20 @@
 	let video_source: HTMLVideoElement;
 	let available_video_devices: MediaDeviceInfo[] = [];
 	let selected_device: MediaDeviceInfo | null = null;
-	let interval_id = 0;
-	let stop_button: HTMLButtonElement;
 	let time_limit: number | null = null;
+	let queue_joined = false;
+	let stream_open = false;
 
 	export const close_stream: () => void = () => {
 		time_limit = null;
+		queue_joined = false;
+		stream_open = false;
+	};
+
+	export const open_stream: () => void = () => {
+		if (!stream_open) {
+			stream_open = true;
+		}
 	};
 
 	export const set_time_limit = (time: number): void => {
@@ -122,7 +136,7 @@
 				context.drawImage(video_source, -video_source.videoWidth, 0);
 			}
 
-			if (streaming && !recording) {
+			if (streaming && (!recording || (queue_joined && !stream_open))) {
 				return;
 			}
 
@@ -133,9 +147,17 @@
 				`image/${streaming ? "jpeg" : "png"}`,
 				0.8
 			);
+			if (!queue_joined) queue_joined = true;
 		}
 	}
 
+	$: console.log(
+		"queue_joined stream_open recording",
+		queue_joined,
+		stream_open,
+		recording
+	);
+
 	let recording = false;
 	let recorded_blobs: BlobPart[] = [];
 	let stream: MediaStream;
@@ -203,11 +225,12 @@
 			stream.getTracks().forEach((track) => track.stop());
 			video_source.srcObject = null;
 			webcam_accessed = false;
+			queue_joined = false;
 		}
 	}
 
 	if (streaming && mode === "image") {
-		interval_id = window.setInterval(() => {
+		window.setInterval(() => {
 			if (video_source && !pending) {
 				take_picture();
 			}
@@ -263,12 +286,15 @@
 	{:else}
 		<div class="button-wrap">
 			<button
-				bind:this={stop_button}
 				on:click={record_video_or_photo}
 				aria-label={mode === "image" ? "capture photo" : "start recording"}
 			>
 				{#if mode === "video" || streaming}
-					{#if recording}
+					{#if streaming && queue_joined && !stream_open}
+						<div class="icon color-primary" title="spinner">
+							<Spinner />
+						</div>
+					{:else if (streaming && stream_open) || (!streaming && recording)}
 						<div class="icon color-primary" title="stop recording">
 							<Square />
 						</div>

From 288516c239a320b3aab034751dad12fd32d7c91b Mon Sep 17 00:00:00 2001
From: freddyaboulton <alfonsoboulton@gmail.com>
Date: Mon, 19 Aug 2024 16:31:57 -0400
Subject: [PATCH 2/9] submit logic happens in Blocks

---
 js/audio/Index.svelte                        | 14 ++--
 js/audio/interactive/InteractiveAudio.svelte | 26 ++++----
 js/audio/streaming/StreamAudio.svelte        |  1 -
 js/core/src/Blocks.svelte                    | 24 ++++---
 js/core/src/init.ts                          | 23 +++++--
 js/image/Index.svelte                        | 14 ++--
 js/image/shared/ImageUploader.svelte         |  6 +-
 js/image/shared/Webcam.svelte                | 68 +++++++++++---------
 8 files changed, 108 insertions(+), 68 deletions(-)

diff --git a/js/audio/Index.svelte b/js/audio/Index.svelte
index 9e9ed23e231ff..f417f6df90602 100644
--- a/js/audio/Index.svelte
+++ b/js/audio/Index.svelte
@@ -41,8 +41,15 @@
 	export let streaming: boolean;
 	export let stream_every: number;
 
-	export let close_stream: () => void;
-	export let open_stream: () => void;
+	let stream_state = "closed";
+	let _modify_stream: (state: "open" | "closed" | "waiting") => void;
+	export function modify_stream_state(
+		state: "open" | "closed" | "waiting"
+	): void {
+		stream_state = state;
+		_modify_stream(state);
+	}
+	export const get_stream_state: () => void = () => stream_state;
 	export let set_time_limit: (time: number) => void;
 	export let gradio: Gradio<{
 		input: never;
@@ -246,8 +253,7 @@
 			{waveform_options}
 			{trim_region_settings}
 			{stream_every}
-			bind:close_stream
-			bind:open_stream
+			bind:modify_stream={_modify_stream}
 			bind:set_time_limit
 			upload={gradio.client.upload}
 			stream_handler={gradio.client.stream}
diff --git a/js/audio/interactive/InteractiveAudio.svelte b/js/audio/interactive/InteractiveAudio.svelte
index 31a8f77c2bb78..aac2cf351647c 100644
--- a/js/audio/interactive/InteractiveAudio.svelte
+++ b/js/audio/interactive/InteractiveAudio.svelte
@@ -41,19 +41,24 @@
 	export let stream_every: number;
 
 	let time_limit: number | null = null;
-	let queue_joined = false;
+	let stream_state: "open" | "waiting" | "closed" = "closed";
 
-	export const close_stream: () => void = () => {
-		time_limit = null;
-		stream_open = false;
-		queue_joined = false;
+	export const modify_stream: (state: "open" | "closed" | "waiting") => void = (
+		state: "open" | "closed" | "waiting"
+	) => {
+		if (state === "closed") {
+			time_limit = null;
+			stream_state = "closed";
+		} else if (state === "waiting") {
+			stream_state = "waiting";
+		} else {
+			stream_state = "open";
+		}
 	};
+
 	export const set_time_limit = (time: number): void => {
 		if (recording) time_limit = time;
 	};
-	export const open_stream: () => void = () => {
-		if (recording) stream_open = true;
-	};
 
 	$: dispatch("drag", dragging);
 
@@ -174,8 +179,7 @@
 			pending_stream.push(payload);
 		} else {
 			let blobParts = [header].concat(pending_stream, [payload]);
-			if (stream_open || !queue_joined) dispatch_blob(blobParts, "stream");
-			queue_joined = true;
+			dispatch_blob(blobParts, "stream");
 			pending_stream = [];
 		}
 	}
@@ -248,7 +252,7 @@
 					{i18n}
 					{waveform_settings}
 					{waveform_options}
-					waiting={queue_joined && !stream_open}
+					waiting={stream_state === "waiting"}
 				/>
 			{:else}
 				<AudioRecorder
diff --git a/js/audio/streaming/StreamAudio.svelte b/js/audio/streaming/StreamAudio.svelte
index b775794cc13ed..1a806b7341397 100644
--- a/js/audio/streaming/StreamAudio.svelte
+++ b/js/audio/streaming/StreamAudio.svelte
@@ -17,7 +17,6 @@
 		show_recording_waveform: true
 	};
 	export let waiting = false;
-	$: console.log("waiting", waiting);
 
 	let micWaveform: WaveSurfer;
 	let waveformRecord: RecordPlugin;
diff --git a/js/core/src/Blocks.svelte b/js/core/src/Blocks.svelte
index 156f3e3ab4aec..c9cf8e616cf72 100644
--- a/js/core/src/Blocks.svelte
+++ b/js/core/src/Blocks.svelte
@@ -52,6 +52,7 @@
 		update_value,
 		get_data,
 		modify_stream,
+		get_stream_state,
 		set_time_limit,
 		loading_status,
 		scheduled_updates,
@@ -280,13 +281,20 @@
 
 			let submission: ReturnType<typeof app.submit>;
 			app.set_current_payload(payload);
-			if (streaming && submit_map.has(dep_index)) {
-				await app.post_data(
-					// @ts-ignore
-					`${app.config.root}/stream/${submit_map.get(dep_index).event_id()}`,
-					{ ...payload, session_hash: app.session_hash }
-				);
-				return;
+			if (streaming) {
+				if (!submit_map.has(dep_index)) {
+					dep.inputs.forEach((id) => modify_stream(id, "waiting"));
+				} else if (
+					submit_map.has(dep_index) &&
+					dep.inputs.some((id) => get_stream_state(id) === "open")
+				) {
+					await app.post_data(
+						// @ts-ignore
+						`${app.config.root}/stream/${submit_map.get(dep_index).event_id()}`,
+						{ ...payload, session_hash: app.session_hash }
+					);
+					return;
+				}
 			}
 			try {
 				submission = app.submit(
@@ -444,7 +452,7 @@
 						}
 					});
 					dep.inputs.forEach((id) => {
-						modify_stream(id, "close");
+						modify_stream(id, "closed");
 					});
 					submit_map.delete(dep_index);
 				}
diff --git a/js/core/src/init.ts b/js/core/src/init.ts
index 614afb87255b5..e52e2a5e6bd38 100644
--- a/js/core/src/init.ts
+++ b/js/core/src/init.ts
@@ -28,7 +28,8 @@ export function create_components(): {
 	targets: Writable<TargetMap>;
 	update_value: (updates: UpdateTransaction[]) => void;
 	get_data: (id: number) => any | Promise<any>;
-	modify_stream: (id: number, action: "open" | "close") => void;
+	modify_stream: (id: number, state: "open" | "waiting" | "closed") => void;
+	get_stream_state: (id: number) => "open" | "waiting" | "closed" | "not_set";
 	set_time_limit: (id: number, time_limit: number | undefined) => void;
 	loading_status: ReturnType<typeof create_loading_status_store>;
 	scheduled_updates: Writable<boolean>;
@@ -347,15 +348,24 @@ export function create_components(): {
 		return comp.props.value;
 	}
 
-	function modify_stream(id: number, action: "open" | "close"): void {
+	function modify_stream(
+		id: number,
+		state: "open" | "closed" | "waiting"
+	): void {
 		const comp = _component_map.get(id);
-		if (comp && comp.instance.close_stream && action == "close") {
-			comp.instance.close_stream();
-		} else if (comp && comp.instance.open_stream && action == "open") {
-			comp.instance.open_stream();
+		if (comp && comp.instance.modify_stream_state) {
+			comp.instance.modify_stream_state(state);
 		}
 	}
 
+	function get_stream_state(
+		id: number
+	): "open" | "closed" | "waiting" | "not_set" {
+		const comp = _component_map.get(id);
+		if (comp) return comp.instance.get_stream_state();
+		return "not_set";
+	}
+
 	function set_time_limit(id: number, time_limit: number | undefined): void {
 		const comp = _component_map.get(id);
 		if (comp && comp.instance.set_time_limit) {
@@ -369,6 +379,7 @@ export function create_components(): {
 		update_value,
 		get_data,
 		modify_stream,
+		get_stream_state,
 		set_time_limit,
 		loading_status,
 		scheduled_updates: update_scheduled_store,
diff --git a/js/image/Index.svelte b/js/image/Index.svelte
index c488bf88541bf..bc7db5814f57b 100644
--- a/js/image/Index.svelte
+++ b/js/image/Index.svelte
@@ -22,8 +22,15 @@
 
 	type sources = "upload" | "webcam" | "clipboard" | null;
 
-	export let close_stream: () => void;
-	export let open_stream: () => void;
+	let stream_state = "closed";
+	let _modify_stream: (state: "open" | "closed" | "waiting") => void;
+	export function modify_stream_state(
+		state: "open" | "closed" | "waiting"
+	): void {
+		stream_state = state;
+		_modify_stream(state);
+	}
+	export const get_stream_state: () => void = () => stream_state;
 	export let set_time_limit: (arg0: number) => void;
 	export let value_is_output = false;
 	export let elem_id = "";
@@ -174,8 +181,7 @@
 			{streaming}
 			{mirror_webcam}
 			{stream_every}
-			bind:close_stream
-			bind:open_stream
+			bind:modify_stream={_modify_stream}
 			bind:set_time_limit
 			max_file_size={gradio.max_file_size}
 			i18n={gradio.i18n}
diff --git a/js/image/shared/ImageUploader.svelte b/js/image/shared/ImageUploader.svelte
index 57fce22ced992..6c613b77b569f 100644
--- a/js/image/shared/ImageUploader.svelte
+++ b/js/image/shared/ImageUploader.svelte
@@ -30,8 +30,7 @@
 	export let stream_handler: Client["stream"];
 	export let stream_every: number;
 
-	export let close_stream: () => void;
-	export let open_stream: () => void;
+	export let modify_stream: (state: "open" | "closed" | "waiting") => void;
 	export let set_time_limit: (arg0: number) => void;
 
 	let upload_input: Upload;
@@ -150,8 +149,7 @@
 				include_audio={false}
 				{i18n}
 				{upload}
-				bind:close_stream
-				bind:open_stream
+				bind:modify_stream
 				bind:set_time_limit
 			/>
 		{:else if value !== null && !streaming}
diff --git a/js/image/shared/Webcam.svelte b/js/image/shared/Webcam.svelte
index 0ef6227382bef..b25b7e9221443 100644
--- a/js/image/shared/Webcam.svelte
+++ b/js/image/shared/Webcam.svelte
@@ -22,18 +22,18 @@
 	let available_video_devices: MediaDeviceInfo[] = [];
 	let selected_device: MediaDeviceInfo | null = null;
 	let time_limit: number | null = null;
-	let queue_joined = false;
-	let stream_open = false;
-
-	export const close_stream: () => void = () => {
-		time_limit = null;
-		queue_joined = false;
-		stream_open = false;
-	};
-
-	export const open_stream: () => void = () => {
-		if (!stream_open) {
-			stream_open = true;
+	let stream_state: "open" | "waiting" | "closed" = "closed";
+
+	export const modify_stream: (state: "open" | "closed" | "waiting") => void = (
+		state: "open" | "closed" | "waiting"
+	) => {
+		if (state === "closed") {
+			time_limit = null;
+			stream_state = "closed";
+		} else if (state === "waiting") {
+			stream_state = "waiting";
+		} else {
+			stream_state = "open";
 		}
 	};
 
@@ -136,7 +136,7 @@
 				context.drawImage(video_source, -video_source.videoWidth, 0);
 			}
 
-			if (streaming && (!recording || (queue_joined && !stream_open))) {
+			if (streaming && !recording) {
 				return;
 			}
 
@@ -147,17 +147,9 @@
 				`image/${streaming ? "jpeg" : "png"}`,
 				0.8
 			);
-			if (!queue_joined) queue_joined = true;
 		}
 	}
 
-	$: console.log(
-		"queue_joined stream_open recording",
-		queue_joined,
-		stream_open,
-		recording
-	);
-
 	let recording = false;
 	let recorded_blobs: BlobPart[] = [];
 	let stream: MediaStream;
@@ -225,7 +217,6 @@
 			stream.getTracks().forEach((track) => track.stop());
 			video_source.srcObject = null;
 			webcam_accessed = false;
-			queue_joined = false;
 		}
 	}
 
@@ -290,17 +281,26 @@
 				aria-label={mode === "image" ? "capture photo" : "start recording"}
 			>
 				{#if mode === "video" || streaming}
-					{#if streaming && queue_joined && !stream_open}
-						<div class="icon color-primary" title="spinner">
-							<Spinner />
+					{#if streaming && stream_state === "waiting"}
+						<div class="icon-with-text" style="width:var(--size-24);">
+							<div class="icon color-primary" title="spinner">
+								<Spinner />
+							</div>
+							{i18n("audio.waiting")}
 						</div>
-					{:else if (streaming && stream_open) || (!streaming && recording)}
-						<div class="icon color-primary" title="stop recording">
-							<Square />
+					{:else if (streaming && stream_state === "open") || (!streaming && recording)}
+						<div class="icon-with-text">
+							<div class="icon color-primary" title="stop recording">
+								<Square />
+							</div>
+							{i18n("audio.stop")}
 						</div>
 					{:else}
-						<div class="icon color-primary" title="start recording">
-							<Circle />
+						<div class="icon-with-text">
+							<div class="icon color-primary" title="start recording">
+								<Circle />
+							</div>
+							{i18n("audio.record")}
 						</div>
 					{/if}
 				{:else}
@@ -382,6 +382,14 @@
 		color: var(--button-secondary-text-color);
 	}
 
+	.icon-with-text {
+		width: var(--size-20);
+		align-items: center;
+		margin: 0 var(--spacing-xl);
+		display: flex;
+		justify-content: space-evenly;
+	}
+
 	@media (--screen-md) {
 		button {
 			bottom: var(--size-4);

From d4a7edabb431e19f7425ee9dac1ca8b1ec702b55 Mon Sep 17 00:00:00 2001
From: gradio-pr-bot <gradio-pr-bot@users.noreply.github.com>
Date: Mon, 19 Aug 2024 20:48:16 +0000
Subject: [PATCH 3/9] add changeset

---
 .changeset/easy-files-serve.md | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 .changeset/easy-files-serve.md

diff --git a/.changeset/easy-files-serve.md b/.changeset/easy-files-serve.md
new file mode 100644
index 0000000000000..f8a389d5c3606
--- /dev/null
+++ b/.changeset/easy-files-serve.md
@@ -0,0 +1,10 @@
+---
+"@gradio/audio": minor
+"@gradio/client": minor
+"@gradio/core": minor
+"@gradio/icons": minor
+"@gradio/image": minor
+"gradio": minor
+---
+
+feat:Open audio/image input stream only when queue is ready

From cc660b89c0e8fa467a624f0ad8a58ab4e8c9d57d Mon Sep 17 00:00:00 2001
From: freddyaboulton <alfonsoboulton@gmail.com>
Date: Mon, 19 Aug 2024 17:09:54 -0400
Subject: [PATCH 4/9] trigger ci


From 66c19d66744ed54517a7f3339910d0556a1718a4 Mon Sep 17 00:00:00 2001
From: freddyaboulton <alfonsoboulton@gmail.com>
Date: Mon, 19 Aug 2024 17:27:44 -0400
Subject: [PATCH 5/9] trigger ci


From 6188ba369c9ca42148daddfc872651ea1b38f824 Mon Sep 17 00:00:00 2001
From: freddyaboulton <alfonsoboulton@gmail.com>
Date: Mon, 19 Aug 2024 17:49:25 -0400
Subject: [PATCH 6/9] Add code

---
 client/js/src/test/api_info.test.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/client/js/src/test/api_info.test.ts b/client/js/src/test/api_info.test.ts
index 40e9d1f57fd5f..a717069723168 100644
--- a/client/js/src/test/api_info.test.ts
+++ b/client/js/src/test/api_info.test.ts
@@ -238,6 +238,7 @@ describe("handle_message", () => {
 		const result = handle_message(data, last_status);
 		expect(result).toEqual({
 			type: "update",
+			original_msg: "process_starts",
 			status: {
 				queue: true,
 				stage: "pending",

From 4e1d0014d413cbec981720ad0f0d4b5b5af77604 Mon Sep 17 00:00:00 2001
From: freddyaboulton <alfonsoboulton@gmail.com>
Date: Mon, 19 Aug 2024 18:17:27 -0400
Subject: [PATCH 7/9] Add code

---
 js/core/src/init.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/js/core/src/init.ts b/js/core/src/init.ts
index e52e2a5e6bd38..80829c12c6c21 100644
--- a/js/core/src/init.ts
+++ b/js/core/src/init.ts
@@ -362,7 +362,8 @@ export function create_components(): {
 		id: number
 	): "open" | "closed" | "waiting" | "not_set" {
 		const comp = _component_map.get(id);
-		if (comp) return comp.instance.get_stream_state();
+		if (comp && comp.instance.get_stream_state)
+			return comp.instance.get_stream_state();
 		return "not_set";
 	}
 

From 9b67e99aa47a2aa188e66c4e148aa34c577d3abc Mon Sep 17 00:00:00 2001
From: freddyaboulton <alfonsoboulton@gmail.com>
Date: Mon, 19 Aug 2024 19:16:43 -0400
Subject: [PATCH 8/9] Fix retrigger refactor

---
 js/audio/interactive/InteractiveAudio.svelte | 1 +
 js/core/src/Blocks.svelte                    | 5 +++++
 js/image/shared/Webcam.svelte                | 2 +-
 3 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/js/audio/interactive/InteractiveAudio.svelte b/js/audio/interactive/InteractiveAudio.svelte
index aac2cf351647c..d5a3b3adf6c74 100644
--- a/js/audio/interactive/InteractiveAudio.svelte
+++ b/js/audio/interactive/InteractiveAudio.svelte
@@ -179,6 +179,7 @@
 			pending_stream.push(payload);
 		} else {
 			let blobParts = [header].concat(pending_stream, [payload]);
+			if (!recording || stream_state === "waiting") return;
 			dispatch_blob(blobParts, "stream");
 			pending_stream = [];
 		}
diff --git a/js/core/src/Blocks.svelte b/js/core/src/Blocks.svelte
index c9cf8e616cf72..979abecd1189b 100644
--- a/js/core/src/Blocks.svelte
+++ b/js/core/src/Blocks.svelte
@@ -284,6 +284,11 @@
 			if (streaming) {
 				if (!submit_map.has(dep_index)) {
 					dep.inputs.forEach((id) => modify_stream(id, "waiting"));
+				} else if (
+					submit_map.has(dep_index) &&
+					dep.inputs.some((id) => get_stream_state(id) === "waiting")
+				) {
+					return;
 				} else if (
 					submit_map.has(dep_index) &&
 					dep.inputs.some((id) => get_stream_state(id) === "open")
diff --git a/js/image/shared/Webcam.svelte b/js/image/shared/Webcam.svelte
index b25b7e9221443..8e2f2b997f76d 100644
--- a/js/image/shared/Webcam.svelte
+++ b/js/image/shared/Webcam.svelte
@@ -136,7 +136,7 @@
 				context.drawImage(video_source, -video_source.videoWidth, 0);
 			}
 
-			if (streaming && !recording) {
+			if (streaming && (!recording || stream_state === "waiting")) {
 				return;
 			}
 

From 4fccd03cc11593ed4997d87f1fc426c73884fb06 Mon Sep 17 00:00:00 2001
From: freddyaboulton <alfonsoboulton@gmail.com>
Date: Tue, 20 Aug 2024 12:26:06 -0400
Subject: [PATCH 9/9] Add code

---
 js/core/src/lang/zh-CN.json | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/js/core/src/lang/zh-CN.json b/js/core/src/lang/zh-CN.json
index 4f1e080370ddb..94bef996d6417 100644
--- a/js/core/src/lang/zh-CN.json
+++ b/js/core/src/lang/zh-CN.json
@@ -16,7 +16,8 @@
 		"record": "录制",
 		"no_microphone": "找不到麦克风",
 		"pause": "暂停",
-		"play": "播放"
+		"play": "播放",
+		"waiting": "等待"
 	},
 	"blocks": {
 		"connection_can_break": "在移动设备上，如果此标签页失去焦点或设备休眠，连接可能会中断，导致您在队列中失去位置。",