Proactively fix stuck devices in video rooms

src/Lifecycle.ts

@@ -60,6 +60,8 @@ import StorageEvictedDialog from "./components/views/dialogs/StorageEvictedDialo
 import { setSentryUser } from "./sentry";
 import SdkConfig from "./SdkConfig";
 import { DialogOpener } from "./utils/DialogOpener";
+import VideoChannelStore from "./stores/VideoChannelStore";
+import { fixStuckDevices } from "./utils/VideoChannelUtils";
 import { Action } from "./dispatcher/actions";
 import AbstractLocalStorageSettingsHandler from "./settings/handlers/AbstractLocalStorageSettingsHandler";
 
@@ -835,6 +837,11 @@ async function startMatrixClient(startSyncing = true): Promise<void> {
     // Now that we have a MatrixClientPeg, update the Jitsi info
     Jitsi.getInstance().start();
 
+    // In case we disconnected uncleanly from a video room, clean up the stuck device
+    if (VideoChannelStore.instance.roomId) {
+        fixStuckDevices(MatrixClientPeg.get().getRoom(VideoChannelStore.instance.roomId), false);
+    }
+
     // dispatch that we finished starting up to wire up any other bits
     // of the matrix client that cannot be set prior to starting up.
     dis.dispatch({ action: 'client_started' });

src/components/structures/VideoRoomView.tsx

@@ -14,14 +14,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import React, { FC, useContext, useState, useMemo } from "react";
+import React, { FC, useContext, useState, useMemo, useEffect } from "react";
 import { logger } from "matrix-js-sdk/src/logger";
 import { Room } from "matrix-js-sdk/src/models/room";
 
 import MatrixClientContext from "../../contexts/MatrixClientContext";
 import { useEventEmitter } from "../../hooks/useEventEmitter";
 import WidgetUtils from "../../utils/WidgetUtils";
-import { addVideoChannel, getVideoChannel } from "../../utils/VideoChannelUtils";
+import { addVideoChannel, getVideoChannel, fixStuckDevices } from "../../utils/VideoChannelUtils";
 import WidgetStore, { IApp } from "../../stores/WidgetStore";
 import { UPDATE_EVENT } from "../../stores/AsyncStore";
 import VideoChannelStore, { VideoChannelEvent } from "../../stores/VideoChannelStore";
@@ -62,6 +62,12 @@ const VideoRoomView: FC<IProps> = ({ room, resizing }) => {
         }
     }, [room, widgetStoreReady, widgetLoaded]); // eslint-disable-line react-hooks/exhaustive-deps
 
+    // We'll also take this opportunity to fix any stuck devices.
+    // The linter thinks that store.connected should be a dependency, but we explicitly
+    // *only* want this to happen at mount to avoid racing with normal device updates.
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+    useEffect(() => { fixStuckDevices(room, store.connected); }, [room]);
+
     const [connected, setConnected] = useState(store.connected && store.roomId === room.roomId);
     useEventEmitter(store, VideoChannelEvent.Connect, () => setConnected(store.roomId === room.roomId));
     useEventEmitter(store, VideoChannelEvent.Disconnect, () => setConnected(false));

src/settings/Settings.tsx

@@ -929,6 +929,18 @@ export const SETTINGS: {[setting: string]: ISetting} = {
         supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS,
         default: false,
     },
+    "audioInputMuted": {
+        supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS,
+        default: false,
+    },
+    "videoInputMuted": {
+        supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS,
+        default: false,
+    },
+    "videoChannelRoomId": {
+        supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS,
+        default: null,
+    },
     [UIFeature.RoomHistorySettings]: {
         supportedLevels: LEVELS_UI_FEATURE,
         default: true,

src/stores/VideoChannelStore.ts

@@ -18,15 +18,13 @@ import EventEmitter from "events";
 import { Room, RoomEvent } from "matrix-js-sdk/src/models/room";
 import { ClientWidgetApi, IWidgetApiRequest } from "matrix-widget-api";
 
+import SettingsStore from "../settings/SettingsStore";
+import { SettingLevel } from "../settings/SettingLevel";
 import defaultDispatcher from "../dispatcher/dispatcher";
 import { ActionPayload } from "../dispatcher/payloads";
 import { ElementWidgetActions } from "./widgets/ElementWidgetActions";
 import { WidgetMessagingStore, WidgetMessagingStoreEvent } from "./widgets/WidgetMessagingStore";
-import {
-    VIDEO_CHANNEL_MEMBER,
-    IVideoChannelMemberContent,
-    getVideoChannel,
-} from "../utils/VideoChannelUtils";
+import { getVideoChannel, addOurDevice, removeOurDevice } from "../utils/VideoChannelUtils";
 import { timeout } from "../utils/promise";
 import WidgetUtils from "../utils/WidgetUtils";
 import { AsyncStoreWithClient } from "./AsyncStoreWithClient";
@@ -83,9 +81,13 @@ export default class VideoChannelStore extends AsyncStoreWithClient<null> {
 
     private activeChannel: ClientWidgetApi;
 
-    private _roomId: string;
-    public get roomId(): string { return this._roomId; }
-    private set roomId(value: string) { this._roomId = value; }
+    // This is persisted to settings so we can detect unclean disconnects
+    public get roomId(): string | null { return SettingsStore.getValue("videoChannelRoomId"); }
+    private set roomId(value: string | null) {
+        SettingsStore.setValue("videoChannelRoomId", null, SettingLevel.DEVICE, value);
+    }
+
+    private get room(): Room { return this.matrixClient.getRoom(this.roomId); }
 
     private _connected = false;
     public get connected(): boolean { return this._connected; }
@@ -95,18 +97,14 @@ export default class VideoChannelStore extends AsyncStoreWithClient<null> {
     public get participants(): IJitsiParticipant[] { return this._participants; }
     private set participants(value: IJitsiParticipant[]) { this._participants = value; }
 
-    private _audioMuted = localStorage.getItem("mx_audioMuted") === "true";
-    public get audioMuted(): boolean { return this._audioMuted; }
+    public get audioMuted(): boolean { return SettingsStore.getValue("audioInputMuted"); }
     public set audioMuted(value: boolean) {
-        this._audioMuted = value;
-        localStorage.setItem("mx_audioMuted", value.toString());
+        SettingsStore.setValue("audioInputMuted", null, SettingLevel.DEVICE, value);
     }
 
-    private _videoMuted = localStorage.getItem("mx_videoMuted") === "true";
-    public get videoMuted(): boolean { return this._videoMuted; }
+    public get videoMuted(): boolean { return SettingsStore.getValue("videoInputMuted"); }
     public set videoMuted(value: boolean) {
-        this._videoMuted = value;
-        localStorage.setItem("mx_videoMuted", value.toString());
+        SettingsStore.setValue("videoInputMuted", null, SettingLevel.DEVICE, value);
     }
 
     public connect = async (roomId: string, audioDevice: MediaDeviceInfo, videoDevice: MediaDeviceInfo) => {
@@ -198,13 +196,13 @@ export default class VideoChannelStore extends AsyncStoreWithClient<null> {
         }
 
         this.connected = true;
-        this.matrixClient.getRoom(roomId).on(RoomEvent.MyMembership, this.onMyMembership);
+        this.room.on(RoomEvent.MyMembership, this.onMyMembership);
         window.addEventListener("beforeunload", this.setDisconnected);
 
         this.emit(VideoChannelEvent.Connect, roomId);
 
         // Tell others that we're connected, by adding our device to room state
-        this.updateDevices(roomId, devices => Array.from(new Set(devices).add(this.matrixClient.getDeviceId())));
+        await addOurDevice(this.room);
     };
 
     public disconnect = async () => {
@@ -221,10 +219,11 @@ export default class VideoChannelStore extends AsyncStoreWithClient<null> {
 
     public setDisconnected = async () => {
         const roomId = this.roomId;
+        const room = this.room;
 
         this.activeChannel.off(`action:${ElementWidgetActions.HangupCall}`, this.onHangup);
         this.activeChannel.off(`action:${ElementWidgetActions.CallParticipants}`, this.onParticipants);
-        this.matrixClient.getRoom(roomId).off(RoomEvent.MyMembership, this.onMyMembership);
+        room.off(RoomEvent.MyMembership, this.onMyMembership);
         window.removeEventListener("beforeunload", this.setDisconnected);
 
         this.activeChannel = null;
@@ -235,11 +234,7 @@ export default class VideoChannelStore extends AsyncStoreWithClient<null> {
         this.emit(VideoChannelEvent.Disconnect, roomId);
 
         // Tell others that we're disconnected, by removing our device from room state
-        await this.updateDevices(roomId, devices => {
-            const devicesSet = new Set(devices);
-            devicesSet.delete(this.matrixClient.getDeviceId());
-            return Array.from(devicesSet);
-        });
+        await removeOurDevice(room);
     };
 
     private ack = (ev: CustomEvent<IWidgetApiRequest>) => {
@@ -248,18 +243,6 @@ export default class VideoChannelStore extends AsyncStoreWithClient<null> {
         this.activeChannel.transport.reply(ev.detail, {});
     };
 
-    private updateDevices = async (roomId: string, fn: (devices: string[]) => string[]) => {
-        const room = this.matrixClient.getRoom(roomId);
-        if (room.getMyMembership() !== "join") return;
-
-        const devicesState = room.currentState.getStateEvents(VIDEO_CHANNEL_MEMBER, this.matrixClient.getUserId());
-        const devices = devicesState?.getContent<IVideoChannelMemberContent>()?.devices ?? [];
-
-        await this.matrixClient.sendStateEvent(
-            roomId, VIDEO_CHANNEL_MEMBER, { devices: fn(devices) }, this.matrixClient.getUserId(),
-        );
-    };
-
     private onHangup = async (ev: CustomEvent<IWidgetApiRequest>) => {
         this.ack(ev);
         // In case this hangup is caused by Jitsi Meet crashing at startup,

src/utils/VideoChannelUtils.ts

@@ -16,6 +16,8 @@ limitations under the License.
 
 import { useState } from "react";
 import { throttle } from "lodash";
+import { Optional } from "matrix-events-sdk";
+import { IMyDevice } from "matrix-js-sdk/src/client";
 import { CallType } from "matrix-js-sdk/src/webrtc/call";
 import { Room } from "matrix-js-sdk/src/models/room";
 import { RoomStateEvent } from "matrix-js-sdk/src/models/room-state";
@@ -26,14 +28,16 @@ import WidgetStore, { IApp } from "../stores/WidgetStore";
 import { WidgetType } from "../widgets/WidgetType";
 import WidgetUtils from "./WidgetUtils";
 
-export const VIDEO_CHANNEL = "io.element.video";
-export const VIDEO_CHANNEL_MEMBER = "io.element.video.member";
+const STUCK_DEVICE_TIMEOUT_MS = 1000 * 60 * 60;
 
-export interface IVideoChannelMemberContent {
+interface IVideoChannelMemberContent {
     // Connected device IDs
     devices: string[];
 }
 
+export const VIDEO_CHANNEL = "io.element.video";
+export const VIDEO_CHANNEL_MEMBER = "io.element.video.member";
+
 export const getVideoChannel = (roomId: string): IApp => {
     const apps = WidgetStore.instance.getApps(roomId);
     return apps.find(app => WidgetType.JITSI.matches(app.type) && app.id === VIDEO_CHANNEL);
@@ -72,3 +76,54 @@ export const useConnectedMembers = (
     }, throttleMs, { leading: true, trailing: true }));
     return members;
 };
+
+const updateDevices = async (room: Optional<Room>, fn: (devices: string[] | null) => string[]) => {
+    if (room?.getMyMembership() !== "join") return;
+
+    const devicesState = room.currentState.getStateEvents(VIDEO_CHANNEL_MEMBER, room.client.getUserId());
+    const devices = devicesState?.getContent<IVideoChannelMemberContent>()?.devices ?? [];
+    const newDevices = fn(devices);
+
+    if (newDevices) {
+        await room.client.sendStateEvent(
+            room.roomId, VIDEO_CHANNEL_MEMBER, { devices: newDevices }, room.client.getUserId(),
+        );
+    }
+};
+
+export const addOurDevice = async (room: Room) => {
+    await updateDevices(room, devices => Array.from(new Set(devices).add(room.client.getDeviceId())));
+};
+
+export const removeOurDevice = async (room: Room) => {
+    await updateDevices(room, devices => {
+        const devicesSet = new Set(devices);
+        devicesSet.delete(room.client.getDeviceId());
+        return Array.from(devicesSet);
+    });
+};
+
+/**
+ * Fixes devices that may have gotten stuck in video channel member state after
+ * an unclean disconnection, by filtering out logged out devices, inactive
+ * devices, and our own device (if we're disconnected).
+ * @param {Room} room The room to fix
+ * @param {boolean} connectedLocalEcho Local echo of whether this device is connected
+ */
+export const fixStuckDevices = async (room: Room, connectedLocalEcho: boolean) => {
+    const now = new Date().valueOf();
+    const { devices: myDevices } = await room.client.getDevices();
+    const deviceMap = new Map<string, IMyDevice>(myDevices.map(d => [d.device_id, d]));
+
+    await updateDevices(room, devices => {
+        const newDevices = devices.filter(d => {
+            const device = deviceMap.get(d);
+            return device?.last_seen_ts
+                && !(d === room.client.getDeviceId() && !connectedLocalEcho)
+                && (now - device.last_seen_ts) < STUCK_DEVICE_TIMEOUT_MS;
+        });
+
+        // Skip the update if the devices are unchanged
+        return newDevices.length === devices.length ? null : newDevices;
+    });
+};

test/components/structures/VideoRoomView-test.tsx

@@ -17,7 +17,8 @@ limitations under the License.
 import React from "react";
 import { mount } from "enzyme";
 import { act } from "react-dom/test-utils";
-import { MatrixClient } from "matrix-js-sdk/src/client";
+import { mocked } from "jest-mock";
+import { MatrixClient, IMyDevice } from "matrix-js-sdk/src/client";
 import { Room } from "matrix-js-sdk/src/models/room";
 import { MatrixWidgetType } from "matrix-widget-api";
 
@@ -27,9 +28,11 @@ import {
     StubVideoChannelStore,
     mkRoom,
     wrapInMatrixClientContext,
+    mockStateEventImplementation,
+    mkVideoChannelMember,
 } from "../../test-utils";
 import { MatrixClientPeg } from "../../../src/MatrixClientPeg";
-import { VIDEO_CHANNEL } from "../../../src/utils/VideoChannelUtils";
+import { VIDEO_CHANNEL, VIDEO_CHANNEL_MEMBER } from "../../../src/utils/VideoChannelUtils";
 import WidgetStore from "../../../src/stores/WidgetStore";
 import _VideoRoomView from "../../../src/components/structures/VideoRoomView";
 import VideoLobby from "../../../src/components/views/voip/VideoLobby";
@@ -64,6 +67,37 @@ describe("VideoRoomView", () => {
         room = mkRoom(cli, "!1:example.org");
     });
 
+    it("removes stuck devices on mount", async () => {
+        // Simulate an unclean disconnect
+        store.roomId = "!1:example.org";
+
+        const devices: IMyDevice[] = [
+            {
+                device_id: cli.getDeviceId(),
+                last_seen_ts: new Date().valueOf(),
+            },
+            {
+                device_id: "went offline 2 hours ago",
+                last_seen_ts: new Date().valueOf() - 1000 * 60 * 60 * 2,
+            },
+        ];
+        mocked(cli).getDevices.mockResolvedValue({ devices });
+
+        // Make both devices be stuck
+        mocked(room.currentState).getStateEvents.mockImplementation(mockStateEventImplementation([
+            mkVideoChannelMember(cli.getUserId(), devices.map(d => d.device_id)),
+        ]));
+
+        mount(<VideoRoomView room={room} resizing={false} />);
+        // Wait for state to settle
+        await act(() => Promise.resolve());
+
+        // All devices should have been removed
+        expect(cli.sendStateEvent).toHaveBeenLastCalledWith(
+            "!1:example.org", VIDEO_CHANNEL_MEMBER, { devices: [] }, cli.getUserId(),
+        );
+    });
+
     it("shows lobby and keeps widget loaded when disconnected", async () => {
         const view = mount(<VideoRoomView room={room} resizing={false} />);
         // Wait for state to settle

test/stores/VideoChannelStore-test.ts

@@ -17,7 +17,7 @@ limitations under the License.
 import { mocked } from "jest-mock";
 import { Widget, ClientWidgetApi, MatrixWidgetType, IWidgetApiRequest } from "matrix-widget-api";
 
-import { stubClient, setupAsyncStoreWithClient } from "../test-utils";
+import { stubClient, setupAsyncStoreWithClient, mkRoom } from "../test-utils";
 import { MatrixClientPeg } from "../../src/MatrixClientPeg";
 import WidgetStore, { IApp } from "../../src/stores/WidgetStore";
 import { WidgetMessagingStore } from "../../src/stores/widgets/WidgetMessagingStore";
@@ -51,6 +51,7 @@ describe("VideoChannelStore", () => {
         const cli = MatrixClientPeg.get();
         setupAsyncStoreWithClient(WidgetMessagingStore.instance, cli);
         setupAsyncStoreWithClient(store, cli);
+        mocked(cli).getRoom.mockReturnValue(mkRoom(cli, "!1:example.org"));
 
         let resolveMessageSent: () => void;
         messageSent = new Promise(resolve => resolveMessageSent = resolve);

test/test-utils/test-utils.ts

@@ -79,6 +79,7 @@ export function createTestClient(): MatrixClient {
         getUserId: jest.fn().mockReturnValue("@userId:matrix.rog"),
         getUser: jest.fn().mockReturnValue({ on: jest.fn() }),
         getDeviceId: jest.fn().mockReturnValue("ABCDEFGHI"),
+        getDevices: jest.fn().mockResolvedValue({ devices: [{ device_id: "ABCDEFGHI" }] }),
         credentials: { userId: "@userId:matrix.rog" },
 
         getPushActionsForEvent: jest.fn(),

test/test-utils/video.ts

@@ -22,24 +22,25 @@ import { VIDEO_CHANNEL_MEMBER } from "../../src/utils/VideoChannelUtils";
 import VideoChannelStore, { VideoChannelEvent, IJitsiParticipant } from "../../src/stores/VideoChannelStore";
 
 export class StubVideoChannelStore extends EventEmitter {
-    private _roomId: string;
-    public get roomId(): string { return this._roomId; }
+    private _roomId: string | null;
+    public get roomId(): string | null { return this._roomId; }
+    public set roomId(value: string | null) { this._roomId = value; }
     private _connected: boolean;
     public get connected(): boolean { return this._connected; }
     public get participants(): IJitsiParticipant[] { return []; }
 
     public startConnect = (roomId: string) => {
-        this._roomId = roomId;
+        this.roomId = roomId;
         this.emit(VideoChannelEvent.StartConnect, roomId);
     };
     public connect = jest.fn((roomId: string) => {
-        this._roomId = roomId;
+        this.roomId = roomId;
         this._connected = true;
         this.emit(VideoChannelEvent.Connect, roomId);
     });
     public disconnect = jest.fn(() => {
         const roomId = this._roomId;
-        this._roomId = null;
+        this.roomId = null;
         this._connected = false;
         this.emit(VideoChannelEvent.Disconnect, roomId);
     });