From df9b5456a1c24eb283d76f2f236f3f3210cf338a Mon Sep 17 00:00:00 2001 From: nleve Date: Fri, 16 Jun 2023 02:38:37 -0400 Subject: [PATCH] initial screensharing stream piping and interaction --- src/engine/resource/schema.ts | 1 + .../screenshares/screenshares.common.ts | 3 + src/plugins/screenshares/screenshares.game.ts | 19 ++++++ src/ui/hooks/useWorldInteraction.ts | 10 +++ src/ui/views/gltf-viewer/GLTFViewer.tsx | 6 +- .../session/entity-tooltip/EntityTooltip.tsx | 45 +++++++++---- src/ui/views/session/reticle/Reticle.tsx | 5 +- .../views/session/world/WorldInteraction.tsx | 66 ++++++++++++++++--- 8 files changed, 131 insertions(+), 24 deletions(-) create mode 100644 src/plugins/screenshares/screenshares.common.ts create mode 100644 src/plugins/screenshares/screenshares.game.ts diff --git a/src/engine/resource/schema.ts b/src/engine/resource/schema.ts index 5fc6aaa41..4ebae4a76 100644 --- a/src/engine/resource/schema.ts +++ b/src/engine/resource/schema.ts @@ -454,6 +454,7 @@ export enum InteractableType { Player = 3, Portal = 4, UI = 5, + Screenshare = 6, } export const InteractableResource = defineResource("interactable", ResourceType.Interactable, { diff --git a/src/plugins/screenshares/screenshares.common.ts b/src/plugins/screenshares/screenshares.common.ts new file mode 100644 index 000000000..092139b27 --- /dev/null +++ b/src/plugins/screenshares/screenshares.common.ts @@ -0,0 +1,3 @@ +export interface ScreenshareProps { + peerId: string; +} diff --git a/src/plugins/screenshares/screenshares.game.ts b/src/plugins/screenshares/screenshares.game.ts new file mode 100644 index 000000000..0ab651767 --- /dev/null +++ b/src/plugins/screenshares/screenshares.game.ts @@ -0,0 +1,19 @@ +import { addComponent } from "bitecs"; + +import { GameState } from "../../engine/GameTypes"; +import { getModule } from "../../engine/module/module.common"; +import { PhysicsModule } from "../../engine/physics/physics.game"; +import { RemoteNode } from "../../engine/resource/RemoteResources"; +import { InteractableType } from "../../engine/resource/schema"; +import { addInteractableComponent } from "../interaction/interaction.game"; +import { ScreenshareProps } from "./screenshares.common"; + +export const ScreenshareComponent = new Map(); + +export const addScreenshareComponent = (ctx: GameState, node: RemoteNode, data: ScreenshareProps) => { + const physics = getModule(ctx, PhysicsModule); + addInteractableComponent(ctx, physics, node, InteractableType.Screenshare); + addComponent(ctx.world, ScreenshareComponent, node.eid); + ScreenshareComponent.set(node.eid, data); + return node; +}; diff --git a/src/ui/hooks/useWorldInteraction.ts b/src/ui/hooks/useWorldInteraction.ts index 297d53729..56611af05 100644 --- a/src/ui/hooks/useWorldInteraction.ts +++ b/src/ui/hooks/useWorldInteraction.ts @@ -68,6 +68,16 @@ export function useWorldInteraction( }); return; } + + if (interactableType === InteractableType.Screenshare) { + interactionCallback({ + interactableType, + action, + name: "Screenshare", + peerId: message.peerId, + held: false, + }); + } }; const onExitedWorld = (ctx: IMainThreadContext, message: ExitedWorldMessage) => { diff --git a/src/ui/views/gltf-viewer/GLTFViewer.tsx b/src/ui/views/gltf-viewer/GLTFViewer.tsx index 6cad575cd..8b5405c2b 100644 --- a/src/ui/views/gltf-viewer/GLTFViewer.tsx +++ b/src/ui/views/gltf-viewer/GLTFViewer.tsx @@ -209,6 +209,10 @@ function GLTFViewerUI() { console.log("Interacted with portal", interaction); return; } + if (interactableType === InteractableType.Screenshare) { + console.log("Interacted with screenshare", interaction); + return; + } } if (interactableType === InteractableType.Player) { @@ -230,7 +234,7 @@ function GLTFViewerUI() { <> {editorEnabled && } - {activeEntity && } + {activeEntity && } ); diff --git a/src/ui/views/session/entity-tooltip/EntityTooltip.tsx b/src/ui/views/session/entity-tooltip/EntityTooltip.tsx index 24249f93b..951501603 100644 --- a/src/ui/views/session/entity-tooltip/EntityTooltip.tsx +++ b/src/ui/views/session/entity-tooltip/EntityTooltip.tsx @@ -4,19 +4,15 @@ import { Icon } from "../../../atoms/icon/Icon"; import { Dots } from "../../../atoms/loading/Dots"; import { Text } from "../../../atoms/text/Text"; import { InteractionState } from "../../../hooks/useWorldInteraction"; +import { IInteractionProcess } from "../world/WorldInteraction"; import "./EntityTooltip.css"; -export interface IPortalProcess { - joining?: boolean; - error?: Error; -} - interface EntityTooltipProps { activeEntity: InteractionState; - portalProcess: IPortalProcess; + interactionProcess: IInteractionProcess; } -export function EntityTooltip({ activeEntity, portalProcess }: EntityTooltipProps) { +export function EntityTooltip({ activeEntity, interactionProcess }: EntityTooltipProps) { return (
{activeEntity.interactableType === InteractableType.Player && ( @@ -101,20 +97,20 @@ export function EntityTooltip({ activeEntity, portalProcess }: EntityTooltipProp )} {activeEntity.interactableType === InteractableType.Portal && ( <> - {portalProcess.joining && } + {interactionProcess.loading && } - {portalProcess.joining ? "Joining portal" : "Portal"} + {interactionProcess.loading ? "Joining portal" : "Portal"}
{activeEntity.name} - {portalProcess.error && ( + {interactionProcess.error && ( - {portalProcess.error.message ?? "Unknown error joining portal."} + {interactionProcess.error.message ?? "Unknown error joining portal."} )} - {!portalProcess.joining && ( + {!interactionProcess.loading && ( E / @@ -124,6 +120,31 @@ export function EntityTooltip({ activeEntity, portalProcess }: EntityTooltipProp
)} + {activeEntity.interactableType === InteractableType.Screenshare && ( + <> + {interactionProcess.loading && } + + {interactionProcess.loading ? "Sharing screen" : "Share screen"} + +
+ + {activeEntity.name} + + {interactionProcess.error && ( + + {`Failed to share screen: ${interactionProcess.error.message}`} + + )} + {!interactionProcess.loading && ( + + E / + + Share your screen + + )} +
+ + )}
); } diff --git a/src/ui/views/session/reticle/Reticle.tsx b/src/ui/views/session/reticle/Reticle.tsx index 00f917610..c3f5469ff 100644 --- a/src/ui/views/session/reticle/Reticle.tsx +++ b/src/ui/views/session/reticle/Reticle.tsx @@ -26,7 +26,10 @@ export function Reticle() { (activeEntity.interactableType === InteractableType.Grabbable || activeEntity.interactableType === InteractableType.Interactable), Reticle__player: activeEntity && activeEntity.interactableType === InteractableType.Player, - Reticle__portal: activeEntity && activeEntity.interactableType === InteractableType.Portal, + Reticle__portal: + activeEntity && + (activeEntity.interactableType === InteractableType.Portal || + activeEntity.interactableType === InteractableType.Screenshare), })} /> ); diff --git a/src/ui/views/session/world/WorldInteraction.tsx b/src/ui/views/session/world/WorldInteraction.tsx index f628e9306..79e58a328 100644 --- a/src/ui/views/session/world/WorldInteraction.tsx +++ b/src/ui/views/session/world/WorldInteraction.tsx @@ -18,9 +18,10 @@ import { CameraRigModule } from "../../../../plugins/camera/CameraRig.main"; import { Reticle } from "../reticle/Reticle"; import { useWorldNavigator } from "../../../hooks/useWorldNavigator"; import { useWorldLoader } from "../../../hooks/useWorldLoader"; +import { useHydrogen } from "../../../hooks/useHydrogen"; -export interface IPortalProcess { - joining?: boolean; +export interface IInteractionProcess { + loading?: boolean; error?: Error; } @@ -35,20 +36,62 @@ export function WorldInteraction({ session, world, activeCall }: WorldInteractio const camRigModule = getModule(mainThread, CameraRigModule); const [activeEntity, setActiveEntity] = useMemoizedState(); - const [portalProcess, setPortalProcess] = useMemoizedState({}); + const [interactionProcess, setInteractionProcess] = useMemoizedState({}); const [members, setMembers] = useState(false); + const { platform } = useHydrogen(true); const { navigateEnterWorld } = useWorldNavigator(session); const { exitWorld } = useWorldLoader(); const selectWorld = useSetAtom(overlayWorldAtom); const isMounted = useIsMounted(); + const handleScreenshareGrab = useCallback( + async (interaction) => { + setInteractionProcess({}); + + if (!activeCall) { + setInteractionProcess({ error: new Error("no active call") }); + return; + } + const { peerId } = interaction; + if (peerId) { + // Someone else is already using it. + if (peerId === session.userId) { + // "someone else" is you. Stop using it. (bonus: don't stop screen sharing if you have multiple going) + console.log("Stopping screensharing."); + activeCall.localMedia?.screenShare?.getTracks().forEach((track) => track.stop()); + } else { + // Tell the user to try again later. + const currentUser = peerId + ? activeCall?.members.get(peerId)?.member.displayName || getMxIdUsername(peerId) + : "Player"; + setInteractionProcess({ error: new Error(`${currentUser} is currently using this.`) }); + } + } else { + // Nobody is using it, go ahead and share your screen. + if (!activeCall.localMedia) { + setInteractionProcess({ error: new Error("activeCall.localMedia not initialized") }); + } else { + try { + // TODO: use platform.mediaDevices.getDisplayMedia({audio: true, video: true}) to get screenshare with PC audio? + // See hydrogen-web/src/platform/web/dom/MediaDevices.ts + const screenStream = await platform.mediaDevices.getScreenShareTrack(); + if (screenStream) await activeCall.setMedia(activeCall.localMedia.withScreenShare(screenStream)); + } catch (err) { + setInteractionProcess({ error: err as Error }); + } + } + } + }, + [activeCall, platform.mediaDevices, session.userId, setInteractionProcess] + ); + const handlePortalGrab = useCallback( async (interaction) => { let unSubStatusObserver: () => void | undefined; try { - setPortalProcess({}); + setInteractionProcess({}); const { uri } = interaction; if (!uri) throw Error("Portal does not have valid matrix id/alias"); @@ -62,11 +105,11 @@ export function WorldInteraction({ session, world, activeCall }: WorldInteractio const roomId = roomIdOrAlias.startsWith("#") ? aliasToRoomId(session.rooms, parsedUri.mxid1) : parsedUri.mxid1; if (!roomId) { - setPortalProcess({ joining: true }); + setInteractionProcess({ loading: true }); const rId = await session.joinRoom(roomIdOrAlias); if (!isMounted()) return; - setPortalProcess({}); + setInteractionProcess({}); const roomStatusObserver = await session.observeRoomStatus(rId); unSubStatusObserver = roomStatusObserver.subscribe(async (roomStatus) => { const newWorld = session.rooms.get(rId); @@ -99,13 +142,13 @@ export function WorldInteraction({ session, world, activeCall }: WorldInteractio } } catch (err) { if (!isMounted()) return; - setPortalProcess({ error: err as Error }); + setInteractionProcess({ error: err as Error }); } return () => { unSubStatusObserver?.(); }; }, - [session, selectWorld, exitWorld, navigateEnterWorld, isMounted, setPortalProcess] + [session, selectWorld, exitWorld, navigateEnterWorld, isMounted, setInteractionProcess] ); const handleInteraction = useCallback( @@ -123,6 +166,9 @@ export function WorldInteraction({ session, world, activeCall }: WorldInteractio handlePortalGrab(interaction); return; } + if (interactableType === InteractableType.Screenshare) { + handleScreenshareGrab(interaction); + } } if (interactableType === InteractableType.Player) { @@ -135,7 +181,7 @@ export function WorldInteraction({ session, world, activeCall }: WorldInteractio setActiveEntity(interaction); }, - [handlePortalGrab, setActiveEntity, activeCall] + [handlePortalGrab, handleScreenshareGrab, setActiveEntity, activeCall] ); useWorldInteraction(mainThread, handleInteraction); @@ -149,7 +195,7 @@ export function WorldInteraction({ session, world, activeCall }: WorldInteractio )} {!camRigModule.orbiting && } {activeEntity && !camRigModule.orbiting && ( - + )} );