matrix-org · nl3v · Jun 16, 2023
diff --git a/src/engine/resource/schema.ts b/src/engine/resource/schema.ts
@@ -454,6 +454,7 @@ export enum InteractableType {
   Player = 3,
   Portal = 4,
   UI = 5,
+  Screenshare = 6,
 }
 
 export const InteractableResource = defineResource("interactable", ResourceType.Interactable, {

diff --git a/src/plugins/screenshares/screenshares.common.ts b/src/plugins/screenshares/screenshares.common.ts
@@ -0,0 +1,3 @@
+export interface ScreenshareProps {
+  peerId: string;
+}
diff --git a/src/plugins/screenshares/screenshares.game.ts b/src/plugins/screenshares/screenshares.game.ts
@@ -0,0 +1,19 @@
+import { addComponent } from "bitecs";
+
+import { GameState } from "../../engine/GameTypes";
+import { getModule } from "../../engine/module/module.common";
+import { PhysicsModule } from "../../engine/physics/physics.game";
+import { RemoteNode } from "../../engine/resource/RemoteResources";
+import { InteractableType } from "../../engine/resource/schema";
+import { addInteractableComponent } from "../interaction/interaction.game";
+import { ScreenshareProps } from "./screenshares.common";
+
+export const ScreenshareComponent = new Map<number, ScreenshareProps>();
+
+export const addScreenshareComponent = (ctx: GameState, node: RemoteNode, data: ScreenshareProps) => {
+  const physics = getModule(ctx, PhysicsModule);
+  addInteractableComponent(ctx, physics, node, InteractableType.Screenshare);
+  addComponent(ctx.world, ScreenshareComponent, node.eid);
+  ScreenshareComponent.set(node.eid, data);
+  return node;
+};
diff --git a/src/ui/hooks/useWorldInteraction.ts b/src/ui/hooks/useWorldInteraction.ts
@@ -68,6 +68,16 @@ export function useWorldInteraction(
         });
         return;
       }
+
+      if (interactableType === InteractableType.Screenshare) {
+        interactionCallback({
+          interactableType,
+          action,
+          name: "Screenshare",
+          peerId: message.peerId,
+          held: false,
+        });
+      }
     };
 
     const onExitedWorld = (ctx: IMainThreadContext, message: ExitedWorldMessage) => {

diff --git a/src/ui/views/gltf-viewer/GLTFViewer.tsx b/src/ui/views/gltf-viewer/GLTFViewer.tsx
@@ -209,6 +209,10 @@ function GLTFViewerUI() {
           console.log("Interacted with portal", interaction);
           return;
         }
+        if (interactableType === InteractableType.Screenshare) {
+          console.log("Interacted with screenshare", interaction);
+          return;
+        }
       }
 
       if (interactableType === InteractableType.Player) {
@@ -230,7 +234,7 @@ function GLTFViewerUI() {
     <>
       <Stats statsEnabled={statsEnabled} />
       {editorEnabled && <EditorView />}
-      {activeEntity && <EntityTooltip activeEntity={activeEntity} portalProcess={{ joining: false }} />}
+      {activeEntity && <EntityTooltip activeEntity={activeEntity} interactionProcess={{ loading: false }} />}
       <Reticle />
     </>
   );

diff --git a/src/ui/views/session/entity-tooltip/EntityTooltip.tsx b/src/ui/views/session/entity-tooltip/EntityTooltip.tsx
@@ -4,19 +4,15 @@ import { Icon } from "../../../atoms/icon/Icon";
 import { Dots } from "../../../atoms/loading/Dots";
 import { Text } from "../../../atoms/text/Text";
 import { InteractionState } from "../../../hooks/useWorldInteraction";
+import { IInteractionProcess } from "../world/WorldInteraction";
 import "./EntityTooltip.css";
 
-export interface IPortalProcess {
-  joining?: boolean;
-  error?: Error;
-}
-
 interface EntityTooltipProps {
   activeEntity: InteractionState;
-  portalProcess: IPortalProcess;
+  interactionProcess: IInteractionProcess;
 }
 
-export function EntityTooltip({ activeEntity, portalProcess }: EntityTooltipProps) {
+export function EntityTooltip({ activeEntity, interactionProcess }: EntityTooltipProps) {
   return (
     <div className="EntityTooltip">
       {activeEntity.interactableType === InteractableType.Player && (
@@ -101,20 +97,20 @@ export function EntityTooltip({ activeEntity, portalProcess }: EntityTooltipProp
       )}
       {activeEntity.interactableType === InteractableType.Portal && (
         <>
-          {portalProcess.joining && <Dots color="world" size="sm" />}
+          {interactionProcess.loading && <Dots color="world" size="sm" />}
           <Text weight="bold" color="world">
-            {portalProcess.joining ? "Joining portal" : "Portal"}
+            {interactionProcess.loading ? "Joining portal" : "Portal"}
           </Text>
           <div className="flex flex-column gap-xxs">
             <Text variant="b3" color="world">
               {activeEntity.name}
             </Text>
-            {portalProcess.error && (
+            {interactionProcess.error && (
               <Text variant="b3" color="world">
-                {portalProcess.error.message ?? "Unknown error joining portal."}
+                {interactionProcess.error.message ?? "Unknown error joining portal."}
               </Text>
             )}
-            {!portalProcess.joining && (
+            {!interactionProcess.loading && (
               <Text variant="b3" color="world">
                 <span className="EntityTooltip__boxedKey">E</span> /
                 <Icon src={MouseIC} size="sm" className="EntityTooltip__mouseIcon" color="world" />
@@ -124,6 +120,31 @@ export function EntityTooltip({ activeEntity, portalProcess }: EntityTooltipProp
           </div>
         </>
       )}
+      {activeEntity.interactableType === InteractableType.Screenshare && (
+        <>
+          {interactionProcess.loading && <Dots color="world" size="sm" />}
+          <Text weight="bold" color="world">
+            {interactionProcess.loading ? "Sharing screen" : "Share screen"}
+          </Text>
+          <div className="flex flex-column gap-xxs">
+            <Text variant="b3" color="world">
+              {activeEntity.name}
+            </Text>
+            {interactionProcess.error && (
+              <Text variant="b3" color="world">
+                {`Failed to share screen: ${interactionProcess.error.message}`}
+              </Text>
+            )}
+            {!interactionProcess.loading && (
+              <Text variant="b3" color="world">
+                <span className="EntityTooltip__boxedKey">E</span> /
+                <Icon src={MouseIC} size="sm" className="EntityTooltip__mouseIcon" color="world" />
+                <span> Share your screen</span>
+              </Text>
+            )}
+          </div>
+        </>
+      )}
     </div>
   );
 }
diff --git a/src/ui/views/session/reticle/Reticle.tsx b/src/ui/views/session/reticle/Reticle.tsx
@@ -26,7 +26,10 @@ export function Reticle() {
           (activeEntity.interactableType === InteractableType.Grabbable ||
             activeEntity.interactableType === InteractableType.Interactable),
         Reticle__player: activeEntity && activeEntity.interactableType === InteractableType.Player,
-        Reticle__portal: activeEntity && activeEntity.interactableType === InteractableType.Portal,
+        Reticle__portal:
+          activeEntity &&
+          (activeEntity.interactableType === InteractableType.Portal ||
+            activeEntity.interactableType === InteractableType.Screenshare),
       })}
     />
   );

diff --git a/src/ui/views/session/world/WorldInteraction.tsx b/src/ui/views/session/world/WorldInteraction.tsx
@@ -18,9 +18,10 @@ import { CameraRigModule } from "../../../../plugins/camera/CameraRig.main";
 import { Reticle } from "../reticle/Reticle";
 import { useWorldNavigator } from "../../../hooks/useWorldNavigator";
 import { useWorldLoader } from "../../../hooks/useWorldLoader";
+import { useHydrogen } from "../../../hooks/useHydrogen";
 
-export interface IPortalProcess {
-  joining?: boolean;
+export interface IInteractionProcess {
+  loading?: boolean;
   error?: Error;
 }
 
@@ -35,20 +36,62 @@ export function WorldInteraction({ session, world, activeCall }: WorldInteractio
   const camRigModule = getModule(mainThread, CameraRigModule);
 
   const [activeEntity, setActiveEntity] = useMemoizedState<InteractionState | undefined>();
-  const [portalProcess, setPortalProcess] = useMemoizedState<IPortalProcess>({});
+  const [interactionProcess, setInteractionProcess] = useMemoizedState<IInteractionProcess>({});
   const [members, setMembers] = useState(false);
+  const { platform } = useHydrogen(true);
 
   const { navigateEnterWorld } = useWorldNavigator(session);
   const { exitWorld } = useWorldLoader();
   const selectWorld = useSetAtom(overlayWorldAtom);
   const isMounted = useIsMounted();
 
+  const handleScreenshareGrab = useCallback(
+    async (interaction) => {
+      setInteractionProcess({});
+
+      if (!activeCall) {
+        setInteractionProcess({ error: new Error("no active call") });
+        return;
+      }
+      const { peerId } = interaction;
+      if (peerId) {
+        // Someone else is already using it.
+        if (peerId === session.userId) {
+          // "someone else" is you. Stop using it. (bonus: don't stop screen sharing if you have multiple going)
+          console.log("Stopping screensharing.");
+          activeCall.localMedia?.screenShare?.getTracks().forEach((track) => track.stop());
+        } else {
+          // Tell the user to try again later.
+          const currentUser = peerId
+            ? activeCall?.members.get(peerId)?.member.displayName || getMxIdUsername(peerId)
+            : "Player";
+          setInteractionProcess({ error: new Error(`${currentUser} is currently using this.`) });
+        }
+      } else {
+        // Nobody is using it, go ahead and share your screen.
+        if (!activeCall.localMedia) {
+          setInteractionProcess({ error: new Error("activeCall.localMedia not initialized") });
+        } else {
+          try {
+            // TODO: use platform.mediaDevices.getDisplayMedia({audio: true, video: true}) to get screenshare with PC audio?
+            // See hydrogen-web/src/platform/web/dom/MediaDevices.ts
+            const screenStream = await platform.mediaDevices.getScreenShareTrack();
+            if (screenStream) await activeCall.setMedia(activeCall.localMedia.withScreenShare(screenStream));
+          } catch (err) {
+            setInteractionProcess({ error: err as Error });
+          }
+        }
+      }
+    },
+    [activeCall, platform.mediaDevices, session.userId, setInteractionProcess]
+  );
+
   const handlePortalGrab = useCallback(
     async (interaction) => {
       let unSubStatusObserver: () => void | undefined;
 
       try {
-        setPortalProcess({});
+        setInteractionProcess({});
         const { uri } = interaction;
         if (!uri) throw Error("Portal does not have valid matrix id/alias");
 
@@ -62,11 +105,11 @@ export function WorldInteraction({ session, world, activeCall }: WorldInteractio
         const roomId = roomIdOrAlias.startsWith("#") ? aliasToRoomId(session.rooms, parsedUri.mxid1) : parsedUri.mxid1;
 
         if (!roomId) {
-          setPortalProcess({ joining: true });
+          setInteractionProcess({ loading: true });
           const rId = await session.joinRoom(roomIdOrAlias);
           if (!isMounted()) return;
 
-          setPortalProcess({});
+          setInteractionProcess({});
           const roomStatusObserver = await session.observeRoomStatus(rId);
           unSubStatusObserver = roomStatusObserver.subscribe(async (roomStatus) => {
             const newWorld = session.rooms.get(rId);
@@ -99,13 +142,13 @@ export function WorldInteraction({ session, world, activeCall }: WorldInteractio
         }
       } catch (err) {
         if (!isMounted()) return;
-        setPortalProcess({ error: err as Error });
+        setInteractionProcess({ error: err as Error });
       }
       return () => {
         unSubStatusObserver?.();
       };
     },
-    [session, selectWorld, exitWorld, navigateEnterWorld, isMounted, setPortalProcess]
+    [session, selectWorld, exitWorld, navigateEnterWorld, isMounted, setInteractionProcess]
   );
 
   const handleInteraction = useCallback(
@@ -123,6 +166,9 @@ export function WorldInteraction({ session, world, activeCall }: WorldInteractio
           handlePortalGrab(interaction);
           return;
         }
+        if (interactableType === InteractableType.Screenshare) {
+          handleScreenshareGrab(interaction);
+        }
       }
 
       if (interactableType === InteractableType.Player) {
@@ -135,7 +181,7 @@ export function WorldInteraction({ session, world, activeCall }: WorldInteractio
 
       setActiveEntity(interaction);
     },
-    [handlePortalGrab, setActiveEntity, activeCall]
+    [handlePortalGrab, handleScreenshareGrab, setActiveEntity, activeCall]
   );
 
   useWorldInteraction(mainThread, handleInteraction);
@@ -149,7 +195,7 @@ export function WorldInteraction({ session, world, activeCall }: WorldInteractio
       )}
       {!camRigModule.orbiting && <Reticle />}
       {activeEntity && !camRigModule.orbiting && (
-        <EntityTooltip activeEntity={activeEntity} portalProcess={portalProcess} />
+        <EntityTooltip activeEntity={activeEntity} interactionProcess={interactionProcess} />
       )}
     </div>
   );