Remote screenshare support

package.json

@@ -1,6 +1,6 @@
 {
     "name": "@whereby.com/browser-sdk",
-    "version": "2.0.0-alpha15",
+    "version": "2.0.0-alpha16",
     "description": "Modules for integration Whereby video in web apps",
     "author": "Whereby AS",
     "license": "MIT",
@@ -70,7 +70,7 @@
     },
     "dependencies": {
         "@swc/helpers": "^0.3.13",
-        "@whereby/jslib-media": "whereby/jslib-media.git#0.2.0",
+        "@whereby/jslib-media": "whereby/jslib-media.git#0.2.1",
         "assert": "^2.0.0",
         "axios": "^1.2.3",
         "btoa": "^1.2.1",

src/lib/RoomConnection.ts

@@ -15,7 +15,7 @@ import {
     RoomService,
 } from "./api";
 
-import { LocalParticipant, RemoteParticipant, StreamState, WaitingParticipant } from "./RoomParticipant";
+import { LocalParticipant, RemoteParticipant, Screenshare, StreamState, WaitingParticipant } from "./RoomParticipant";
 
 import ServerSocket, {
     ChatMessage as SignalChatMessage,
@@ -29,6 +29,8 @@ import ServerSocket, {
     RoomKnockedEvent as SignalRoomKnockedEvent,
     SignalClient,
     SocketManager,
+    ScreenshareStartedEvent as SignalScreenshareStartedEvent,
+    ScreenshareStoppedEvent as SignalScreenshareStoppedEvent,
 } from "@whereby/jslib-media/src/utils/ServerSocket";
 import { sdkVersion } from "./index";
 import LocalMedia from "./LocalMedia";
@@ -103,6 +105,18 @@ type ParticipantMetadataChangedEvent = {
     displayName: string;
 };
 
+type ScreenshareStartedEvent = {
+    participantId: string;
+    id: string;
+    hasAudioTrack: boolean;
+    stream: MediaStream;
+};
+
+type ScreenshareStoppedEvent = {
+    participantId: string;
+    id: string;
+};
+
 type WaitingParticipantJoinedEvent = {
     participantId: string;
     displayName: string | null;
@@ -123,6 +137,8 @@ interface RoomEventsMap {
     participant_video_enabled: CustomEvent<ParticipantVideoEnabledEvent>;
     room_connection_status_changed: CustomEvent<RoomConnectionStatusChangedEvent>;
     room_joined: CustomEvent<RoomJoinedEvent>;
+    screenshare_started: CustomEvent<ScreenshareStartedEvent>;
+    screenshare_stopped: CustomEvent<ScreenshareStoppedEvent>;
     streaming_started: CustomEvent<StreamingState>;
     waiting_participant_joined: CustomEvent<WaitingParticipantJoinedEvent>;
     waiting_participant_left: CustomEvent<WaitingParticipantLeftEvent>;
@@ -178,6 +194,7 @@ export default class RoomConnection extends TypedEventTarget {
     public localParticipant: LocalParticipant | null = null;
     public roomUrl: URL;
     public remoteParticipants: RemoteParticipant[] = [];
+    public screenshares: Screenshare[] = [];
     public readonly localMediaConstraints?: MediaStreamConstraints;
     public readonly roomName: string;
     private organizationId: string;
@@ -266,6 +283,8 @@ export default class RoomConnection extends TypedEventTarget {
         this.signalSocket.on("room_joined", this._handleRoomJoined.bind(this));
         this.signalSocket.on("room_knocked", this._handleRoomKnocked.bind(this));
         this.signalSocket.on("cloud_recording_stopped", this._handleCloudRecordingStopped.bind(this));
+        this.signalSocket.on("screenshare_started", this._handleScreenshareStarted.bind(this));
+        this.signalSocket.on("screenshare_stopped", this._handleScreenshareStopped.bind(this));
         this.signalSocket.on("streaming_stopped", this._handleStreamingStopped.bind(this));
         this.signalSocket.on("disconnect", this._handleDisconnect.bind(this));
         this.signalSocket.on("connect_error", this._handleDisconnect.bind(this));
@@ -539,6 +558,41 @@ export default class RoomConnection extends TypedEventTarget {
         this.dispatchEvent(new CustomEvent("streaming_stopped"));
     }
 
+    private _handleScreenshareStarted(screenshare: SignalScreenshareStartedEvent) {
+        const { clientId: participantId, streamId: id, hasAudioTrack } = screenshare;
+        const remoteParticipant = this.remoteParticipants.find((p) => p.id === participantId);
+
+        if (!remoteParticipant) {
+            this.logger.log("WARN: Could not find participant for screenshare");
+            return;
+        }
+
+        const foundScreenshare = this.screenshares.find((s) => s.id === id);
+        if (foundScreenshare) {
+            this.logger.log("WARN: Screenshare already exists");
+            return;
+        }
+
+        remoteParticipant.addStream(id, "to_accept");
+        this._handleAcceptStreams([remoteParticipant]);
+
+        this.screenshares = [...this.screenshares, { participantId, id, hasAudioTrack, stream: undefined }];
+    }
+
+    private _handleScreenshareStopped(screenshare: SignalScreenshareStoppedEvent) {
+        const { clientId: participantId, streamId: id } = screenshare;
+        const remoteParticipant = this.remoteParticipants.find((p) => p.id === participantId);
+
+        if (!remoteParticipant) {
+            this.logger.log("WARN: Could not find participant for screenshare");
+            return;
+        }
+
+        remoteParticipant.removeStream(id);
+        this.screenshares = this.screenshares.filter((s) => !(s.participantId === participantId && s.id === id));
+        this.dispatchEvent(new CustomEvent("screenshare_stopped", { detail: { participantId, id } }));
+    }
+
     private _handleRtcEvent<K extends keyof RtcEvents>(eventName: K, data: RtcEvents[K]) {
         if (eventName === "rtc_manager_created") {
             return this._handleRtcManagerCreated(data as RtcManagerCreatedPayload);
@@ -644,15 +698,32 @@ export default class RoomConnection extends TypedEventTarget {
         });
     }
 
-    private _handleStreamAdded({ clientId, stream, streamId }: RtcStreamAddedPayload) {
+    private _handleStreamAdded({ clientId, stream, streamId, streamType }: RtcStreamAddedPayload) {
         const remoteParticipant = this.remoteParticipants.find((p) => p.id === clientId);
         if (!remoteParticipant) {
             this.logger.log("WARN: Could not find participant for incoming stream");
             return;
         }
 
+        const remoteParticipantStream = remoteParticipant.streams.find((s) => s.id === streamId);
+
+        if (
+            (remoteParticipant.stream && remoteParticipant.stream.id === streamId) ||
+            (!remoteParticipant.stream && streamType === "webcam") ||
+            (!remoteParticipant.stream &&
+                !streamType &&
+                remoteParticipantStream &&
+                remoteParticipant.streams.indexOf(remoteParticipantStream) < 1)
+        ) {
+            this.dispatchEvent(
+                new CustomEvent("participant_stream_added", { detail: { participantId: clientId, stream, streamId } })
+            );
+            return;
+        }
+
+        // screenshare
         this.dispatchEvent(
-            new CustomEvent("participant_stream_added", { detail: { participantId: clientId, stream, streamId } })
+            new CustomEvent("screenshare_started", { detail: { participantId: clientId, stream, id: streamId } })
         );
     }
 

src/lib/RoomParticipant.ts

@@ -60,6 +60,17 @@ export class RemoteParticipant extends RoomParticipant {
         this.streams = streams.map((streamId) => ({ id: streamId, state: newJoiner ? "new_accept" : "to_accept" }));
     }
 
+    addStream(streamId: string, state: StreamState) {
+        this.streams.push({ id: streamId, state });
+    }
+
+    removeStream(streamId: string) {
+        const index = this.streams.findIndex((s) => s.id === streamId);
+        if (index !== -1) {
+            this.streams.splice(index, 1);
+        }
+    }
+
     updateStreamState(streamId: string, state: StreamState) {
         const stream = this.streams.find((s) => s.id === streamId);
         if (stream) {
@@ -80,3 +91,17 @@ export interface WaitingParticipant {
     id: string;
     displayName: string | null;
 }
+
+export class Screenshare {
+    public readonly participantId: string;
+    public readonly id: string;
+    public readonly hasAudioTrack: boolean;
+    public readonly stream?: MediaStream;
+
+    constructor({ participantId, id, hasAudioTrack, stream }: Screenshare) {
+        this.participantId = participantId;
+        this.id = id;
+        this.hasAudioTrack = hasAudioTrack;
+        this.stream = stream;
+    }
+}

src/lib/react/useRoomConnection.ts

@@ -8,7 +8,7 @@ import RoomConnection, {
     RoomConnectionStatus,
     StreamingState,
 } from "../RoomConnection";
-import { LocalParticipant, RemoteParticipant, WaitingParticipant } from "../RoomParticipant";
+import { LocalParticipant, RemoteParticipant, Screenshare, WaitingParticipant } from "../RoomParticipant";
 
 type RemoteParticipantState = Omit<RemoteParticipant, "updateStreamState">;
 
@@ -20,6 +20,7 @@ export interface RoomConnectionState {
     localParticipant?: LocalParticipant;
     mostRecentChatMessage: ChatMessage | null;
     remoteParticipants: RemoteParticipantState[];
+    screenshares: Screenshare[];
     roomConnectionStatus: RoomConnectionStatus;
     streaming: StreamingState;
     waitingParticipants: WaitingParticipant[];
@@ -36,6 +37,7 @@ const initialState: RoomConnectionState = {
     mostRecentChatMessage: null,
     remoteParticipants: [],
     roomConnectionStatus: "",
+    screenshares: [],
     streaming: {
         status: "",
         startedAt: null,
@@ -115,6 +117,22 @@ type RoomConnectionEvents =
               displayName: string;
           };
       }
+    | {
+          type: "SCREENSHARE_STARTED";
+          payload: {
+              participantId: string;
+              stream: MediaStream;
+              id: string;
+              hasAudioTrack: boolean;
+          };
+      }
+    | {
+          type: "SCREENSHARE_STOPPED";
+          payload: {
+              participantId: string;
+              id: string;
+          };
+      }
     | {
           type: "STREAMING_STARTED";
           payload: StreamingState;
@@ -154,6 +172,14 @@ function updateParticipant(
     ];
 }
 
+function addScreenshare(screenshares: Screenshare[], screenshare: Screenshare): Screenshare[] {
+    const existingScreenshare = screenshares.find((ss) => ss.id === screenshare.id);
+    if (existingScreenshare) {
+        return screenshares;
+    }
+    return [...screenshares, screenshare];
+}
+
 function reducer(state: RoomConnectionState, action: RoomConnectionEvents): RoomConnectionState {
     switch (action.type) {
         case "CHAT_MESSAGE":
@@ -237,6 +263,23 @@ function reducer(state: RoomConnectionState, action: RoomConnectionEvents): Room
                 ...state,
                 localParticipant: { ...state.localParticipant, displayName: action.payload.displayName },
             };
+        case "SCREENSHARE_STARTED":
+            return {
+                ...state,
+                screenshares: addScreenshare(state.screenshares, {
+                    participantId: action.payload.participantId,
+                    id: action.payload.id,
+                    hasAudioTrack: action.payload.hasAudioTrack,
+                    stream: action.payload.stream,
+                }),
+            };
+        case "SCREENSHARE_STOPPED":
+            return {
+                ...state,
+                screenshares: state.screenshares.filter(
+                    (ss) => ss.participantId !== action.payload.participantId || ss.id !== action.payload.id
+                ),
+            };
         case "STREAMING_STARTED":
             return {
                 ...state,
@@ -364,6 +407,17 @@ export default function useRoomConnection(
             dispatch({ type: "PARTICIPANT_METADATA_CHANGED", payload: { participantId, displayName } });
         });
 
+        roomConnection.addEventListener("screenshare_started", (e) => {
+            const { participantId, id, hasAudioTrack, stream } = e.detail;
+
+            dispatch({ type: "SCREENSHARE_STARTED", payload: { participantId, id, hasAudioTrack, stream } });
+        });
+
+        roomConnection.addEventListener("screenshare_stopped", (e) => {
+            const { participantId, id } = e.detail;
+            dispatch({ type: "SCREENSHARE_STOPPED", payload: { participantId, id } });
+        });
+
         roomConnection.addEventListener("streaming_started", (e) => {
             const { status, startedAt } = e.detail;
             dispatch({ type: "STREAMING_STARTED", payload: { status, startedAt } });

src/types.d.ts

@@ -33,6 +33,7 @@ declare module "@whereby/jslib-media/src/webrtc/RtcManagerDispatcher" {
         clientId: string;
         stream: MediaStream;
         streamId: string;
+        streamType: "webcam" | "screenshare";
     }
 
     type RtcEvents = {
@@ -177,6 +178,17 @@ declare module "@whereby/jslib-media/src/utils/ServerSocket" {
         liveVideo: boolean;
     }
 
+    interface ScreenshareStartedEvent {
+        clientId: string;
+        streamId: string;
+        hasAudioTrack: boolean;
+    }
+
+    interface ScreenshareStoppedEvent {
+        clientId: string;
+        streamId: string;
+    }
+
     interface VideoEnabledEvent {
         clientId: string;
         isVideoEnabled: boolean;
@@ -204,6 +216,8 @@ declare module "@whereby/jslib-media/src/utils/ServerSocket" {
         room_joined: RoomJoinedEvent;
         room_knocked: RoomKnockedEvent;
         room_left: void;
+        screenshare_started: ScreenshareStartedEvent;
+        screenshare_stopped: ScreenshareStoppedEvent;
         streaming_stopped: void;
         video_enabled: VideoEnabledEvent;
     }

yarn.lock

@@ -3686,9 +3686,9 @@
     "@webassemblyjs/wast-parser" "1.9.0"
     "@xtuc/long" "4.2.2"
 
-"@whereby/jslib-media@whereby/jslib-media.git#0.2.0":
-  version "0.2.0"
-  resolved "https://codeload.github.com/whereby/jslib-media/tar.gz/2e0354ca8c76810e710282f8051e08bde482090f"
+"@whereby/jslib-media@whereby/jslib-media.git#0.2.1":
+  version "0.2.1"
+  resolved "https://codeload.github.com/whereby/jslib-media/tar.gz/c728ca326eaeab8dc1b2aa4b67c0130f0a17bdb6"
   dependencies:
     assert "^2.0.0"
     mediasoup-client "^3.6.82"