diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index dd0bfc9ebfa..dd7aeb18c10 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,6 +1 @@ -* @matrix-org/element-web -/.github/workflows/** @matrix-org/element-web-app-team -/package.json @matrix-org/element-web-app-team -/yarn.lock @matrix-org/element-web-app-team -/src/webrtc @matrix-org/element-call-reviewers -/spec/*/webrtc @matrix-org/element-call-reviewers +* @matrix-org/element-call-reviewers diff --git a/package.json b/package.json index 1d60216b7ef..e7058354fac 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,7 @@ "jwt-decode": "^3.1.2", "loglevel": "^1.7.1", "matrix-events-sdk": "0.0.1", - "matrix-widget-api": "^1.5.0", + "matrix-widget-api": "^1.6.0", "oidc-client-ts": "^2.2.4", "p-retry": "4", "sdp-transform": "^2.14.1", diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index d4d50b458d7..41ab56c84dd 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -485,6 +485,7 @@ export class MockCallMatrixClient extends TypedEventEmitter().mockReturnValue([]); public getRoom = jest.fn(); + public getFoci = jest.fn(); public supportsThreads(): boolean { return true; diff --git a/spec/unit/embedded.spec.ts b/spec/unit/embedded.spec.ts index caab40ac056..115e0a419ca 100644 --- a/spec/unit/embedded.spec.ts +++ b/spec/unit/embedded.spec.ts @@ -23,7 +23,14 @@ limitations under the License. // eslint-disable-next-line no-restricted-imports import { EventEmitter } from "events"; import { MockedObject } from "jest-mock"; -import { WidgetApi, WidgetApiToWidgetAction, MatrixCapabilities, ITurnServer, IRoomEvent } from "matrix-widget-api"; +import { + WidgetApi, + WidgetApiToWidgetAction, + MatrixCapabilities, + ITurnServer, + IRoomEvent, + IOpenIDCredentials, +} from "matrix-widget-api"; import { createRoomWidgetClient, MsgType } from "../../src/matrix"; import { MatrixClient, ClientEvent, ITurnServer as IClientTurnServer } from "../../src/client"; @@ -33,6 +40,12 @@ import { MatrixEvent } from "../../src/models/event"; import { ToDeviceBatch } from "../../src/models/ToDeviceMessage"; import { DeviceInfo } from "../../src/crypto/deviceinfo"; +const testOIDCToken = { + access_token: "12345678", + expires_in: "10", + matrix_server_name: "homeserver.oabc", + token_type: "Bearer", +}; class MockWidgetApi extends EventEmitter { public start = jest.fn(); public requestCapability = jest.fn(); @@ -49,8 +62,15 @@ class MockWidgetApi extends EventEmitter { public sendRoomEvent = jest.fn(() => ({ event_id: `$${Math.random()}` })); public sendStateEvent = jest.fn(); public sendToDevice = jest.fn(); + public requestOpenIDConnectToken = jest.fn(() => { + return testOIDCToken; + return new Promise(() => { + return testOIDCToken; + }); + }); public readStateEvents = jest.fn(() => []); public getTurnServers = jest.fn(() => []); + public sendContentLoaded = jest.fn(); public transport = { reply: jest.fn() }; } @@ -285,7 +305,12 @@ describe("RoomWidgetClient", () => { expect(await emittedSync).toEqual(SyncState.Syncing); }); }); - + describe("oidc token", () => { + it("requests an oidc token", async () => { + await makeClient({}); + expect(await client.getOpenIdToken()).toStrictEqual(testOIDCToken); + }); + }); it("gets TURN servers", async () => { const server1: ITurnServer = { uris: [ diff --git a/spec/unit/matrixrtc/CallMembership.spec.ts b/spec/unit/matrixrtc/CallMembership.spec.ts new file mode 100644 index 00000000000..a8ae0a8224a --- /dev/null +++ b/spec/unit/matrixrtc/CallMembership.spec.ts @@ -0,0 +1,139 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { MatrixEvent } from "../../../src"; +import { CallMembership, CallMembershipData } from "../../../src/matrixrtc/CallMembership"; + +const membershipTemplate: CallMembershipData = { + call_id: "", + scope: "m.room", + application: "m.call", + device_id: "AAAAAAA", + expires: 5000, +}; + +function makeMockEvent(originTs = 0): MatrixEvent { + return { + getTs: jest.fn().mockReturnValue(originTs), + sender: { + userId: "@alice:example.org", + }, + } as unknown as MatrixEvent; +} + +describe("CallMembership", () => { + it("rejects membership with no expiry", () => { + expect(() => { + new CallMembership(makeMockEvent(), Object.assign({}, membershipTemplate, { expires: undefined })); + }).toThrow(); + }); + + it("rejects membership with no device_id", () => { + expect(() => { + new CallMembership(makeMockEvent(), Object.assign({}, membershipTemplate, { device_id: undefined })); + }).toThrow(); + }); + + it("rejects membership with no call_id", () => { + expect(() => { + new CallMembership(makeMockEvent(), Object.assign({}, membershipTemplate, { call_id: undefined })); + }).toThrow(); + }); + + it("rejects membership with no scope", () => { + expect(() => { + new CallMembership(makeMockEvent(), Object.assign({}, membershipTemplate, { scope: undefined })); + }).toThrow(); + }); + + it("uses event timestamp if no created_ts", () => { + const membership = new CallMembership(makeMockEvent(12345), membershipTemplate); + expect(membership.createdTs()).toEqual(12345); + }); + + it("uses created_ts if present", () => { + const membership = new CallMembership( + makeMockEvent(12345), + Object.assign({}, membershipTemplate, { created_ts: 67890 }), + ); + expect(membership.createdTs()).toEqual(67890); + }); + + it("computes absolute expiry time", () => { + const membership = new CallMembership(makeMockEvent(1000), membershipTemplate); + expect(membership.getAbsoluteExpiry()).toEqual(5000 + 1000); + }); + + it("considers memberships unexpired if local age low enough", () => { + const fakeEvent = makeMockEvent(1000); + fakeEvent.getLocalAge = jest.fn().mockReturnValue(3000); + const membership = new CallMembership(fakeEvent, membershipTemplate); + expect(membership.isExpired()).toEqual(false); + }); + + it("considers memberships expired when local age large", () => { + const fakeEvent = makeMockEvent(1000); + fakeEvent.getLocalAge = jest.fn().mockReturnValue(6000); + const membership = new CallMembership(fakeEvent, membershipTemplate); + expect(membership.isExpired()).toEqual(true); + }); + + it("returns active foci", () => { + const fakeEvent = makeMockEvent(); + const mockFocus = { type: "this_is_a_mock_focus" }; + const membership = new CallMembership( + fakeEvent, + Object.assign({}, membershipTemplate, { foci_active: [mockFocus] }), + ); + expect(membership.getActiveFoci()).toEqual([mockFocus]); + }); + + describe("expiry calculation", () => { + let fakeEvent: MatrixEvent; + let membership: CallMembership; + + beforeEach(() => { + // server origin timestamp for this event is 1000 + fakeEvent = makeMockEvent(1000); + // our clock would have been at 2000 at the creation time (our clock at event receive time - age) + // (ie. the local clock is 1 second ahead of the servers' clocks) + fakeEvent.localTimestamp = 2000; + + // for simplicity's sake, we say that the event's age is zero + fakeEvent.getLocalAge = jest.fn().mockReturnValue(0); + + membership = new CallMembership(fakeEvent!, membershipTemplate); + + jest.useFakeTimers(); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it("converts expiry time into local clock", () => { + // for sanity's sake, make sure the server-relative expiry time is what we expect + expect(membership.getAbsoluteExpiry()).toEqual(6000); + // therefore the expiry time converted to our clock should be 1 second later + expect(membership.getLocalExpiry()).toEqual(7000); + }); + + it("calculates time until expiry", () => { + jest.setSystemTime(2000); + expect(membership.getMsUntilExpiry()).toEqual(5000); + }); + }); +}); diff --git a/spec/unit/matrixrtc/MatrixRTCSession.spec.ts b/spec/unit/matrixrtc/MatrixRTCSession.spec.ts new file mode 100644 index 00000000000..9572e1f8bf2 --- /dev/null +++ b/spec/unit/matrixrtc/MatrixRTCSession.spec.ts @@ -0,0 +1,405 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { EventTimeline, EventType, MatrixClient, Room } from "../../../src"; +import { CallMembershipData } from "../../../src/matrixrtc/CallMembership"; +import { MatrixRTCSession, MatrixRTCSessionEvent } from "../../../src/matrixrtc/MatrixRTCSession"; +import { randomString } from "../../../src/randomstring"; +import { makeMockRoom, mockRTCEvent } from "./mocks"; + +const membershipTemplate: CallMembershipData = { + call_id: "", + scope: "m.room", + application: "m.call", + device_id: "AAAAAAA", + expires: 60 * 60 * 1000, +}; + +const mockFocus = { type: "mock" }; + +describe("MatrixRTCSession", () => { + let client: MatrixClient; + let sess: MatrixRTCSession | undefined; + + beforeEach(() => { + client = new MatrixClient({ baseUrl: "base_url" }); + client.getUserId = jest.fn().mockReturnValue("@alice:example.org"); + client.getDeviceId = jest.fn().mockReturnValue("AAAAAAA"); + }); + + afterEach(() => { + client.stopClient(); + client.matrixRTC.stop(); + if (sess) sess.stop(); + sess = undefined; + }); + + it("Creates a room-scoped session from room state", () => { + const mockRoom = makeMockRoom([membershipTemplate]); + + sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom); + expect(sess?.memberships.length).toEqual(1); + expect(sess?.memberships[0].callId).toEqual(""); + expect(sess?.memberships[0].scope).toEqual("m.room"); + expect(sess?.memberships[0].application).toEqual("m.call"); + expect(sess?.memberships[0].deviceId).toEqual("AAAAAAA"); + expect(sess?.memberships[0].isExpired()).toEqual(false); + }); + + it("ignores expired memberships events", () => { + const expiredMembership = Object.assign({}, membershipTemplate); + expiredMembership.expires = 1000; + expiredMembership.device_id = "EXPIRED"; + const mockRoom = makeMockRoom([membershipTemplate, expiredMembership], () => 10000); + + sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom); + expect(sess?.memberships.length).toEqual(1); + expect(sess?.memberships[0].deviceId).toEqual("AAAAAAA"); + }); + + it("honours created_ts", () => { + const expiredMembership = Object.assign({}, membershipTemplate); + expiredMembership.created_ts = 500; + expiredMembership.expires = 1000; + const mockRoom = makeMockRoom([expiredMembership]); + sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom); + expect(sess?.memberships[0].getAbsoluteExpiry()).toEqual(1500); + }); + + it("returns empty session if no membership events are present", () => { + const mockRoom = makeMockRoom([]); + sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom); + expect(sess?.memberships).toHaveLength(0); + }); + + it("safely ignores events with no memberships section", () => { + const mockRoom = { + roomId: randomString(8), + getLiveTimeline: jest.fn().mockReturnValue({ + getState: jest.fn().mockReturnValue({ + getStateEvents: (_type: string, _stateKey: string) => [ + { + getType: jest.fn().mockReturnValue(EventType.GroupCallMemberPrefix), + getContent: jest.fn().mockReturnValue({}), + getSender: jest.fn().mockReturnValue("@mock:user.example"), + getTs: jest.fn().mockReturnValue(1000), + getLocalAge: jest.fn().mockReturnValue(0), + }, + ], + }), + }), + }; + sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom as unknown as Room); + expect(sess.memberships).toHaveLength(0); + }); + + it("safely ignores events with junk memberships section", () => { + const mockRoom = { + roomId: randomString(8), + getLiveTimeline: jest.fn().mockReturnValue({ + getState: jest.fn().mockReturnValue({ + getStateEvents: (_type: string, _stateKey: string) => [ + { + getType: jest.fn().mockReturnValue(EventType.GroupCallMemberPrefix), + getContent: jest.fn().mockReturnValue({ memberships: "i am a fish" }), + getSender: jest.fn().mockReturnValue("@mock:user.example"), + getTs: jest.fn().mockReturnValue(1000), + getLocalAge: jest.fn().mockReturnValue(0), + }, + ], + }), + }), + }; + sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom as unknown as Room); + expect(sess.memberships).toHaveLength(0); + }); + + it("ignores memberships with no expires_ts", () => { + const expiredMembership = Object.assign({}, membershipTemplate); + (expiredMembership.expires as number | undefined) = undefined; + const mockRoom = makeMockRoom([expiredMembership]); + sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom); + expect(sess.memberships).toHaveLength(0); + }); + + it("ignores memberships with no device_id", () => { + const testMembership = Object.assign({}, membershipTemplate); + (testMembership.device_id as string | undefined) = undefined; + const mockRoom = makeMockRoom([testMembership]); + const sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom); + expect(sess.memberships).toHaveLength(0); + }); + + it("ignores memberships with no call_id", () => { + const testMembership = Object.assign({}, membershipTemplate); + (testMembership.call_id as string | undefined) = undefined; + const mockRoom = makeMockRoom([testMembership]); + sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom); + expect(sess.memberships).toHaveLength(0); + }); + + it("ignores memberships with no scope", () => { + const testMembership = Object.assign({}, membershipTemplate); + (testMembership.scope as string | undefined) = undefined; + const mockRoom = makeMockRoom([testMembership]); + sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom); + expect(sess.memberships).toHaveLength(0); + }); + + it("ignores anything that's not a room-scoped call (for now)", () => { + const testMembership = Object.assign({}, membershipTemplate); + testMembership.scope = "m.user"; + const mockRoom = makeMockRoom([testMembership]); + sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom); + expect(sess.memberships).toHaveLength(0); + }); + + describe("getOldestMembership", () => { + it("returns the oldest membership event", () => { + const mockRoom = makeMockRoom([ + Object.assign({}, membershipTemplate, { device_id: "foo", created_ts: 3000 }), + Object.assign({}, membershipTemplate, { device_id: "old", created_ts: 1000 }), + Object.assign({}, membershipTemplate, { device_id: "bar", created_ts: 2000 }), + ]); + + sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom); + expect(sess.getOldestMembership()!.deviceId).toEqual("old"); + }); + }); + + describe("joining", () => { + let mockRoom: Room; + + beforeEach(() => { + mockRoom = makeMockRoom([]); + sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom); + }); + + afterEach(() => { + // stop the timers + sess!.leaveRoomSession(); + }); + + it("starts un-joined", () => { + expect(sess!.isJoined()).toEqual(false); + }); + + it("shows joined once join is called", () => { + sess!.joinRoomSession([mockFocus]); + expect(sess!.isJoined()).toEqual(true); + }); + + it("sends a membership event when joining a call", () => { + client.sendStateEvent = jest.fn(); + + sess!.joinRoomSession([mockFocus]); + + expect(client.sendStateEvent).toHaveBeenCalledWith( + mockRoom!.roomId, + EventType.GroupCallMemberPrefix, + { + memberships: [ + { + application: "m.call", + scope: "m.room", + call_id: "", + device_id: "AAAAAAA", + expires: 3600000, + foci_active: [{ type: "mock" }], + }, + ], + }, + "@alice:example.org", + ); + }); + + it("does nothing if join called when already joined", () => { + const sendStateEventMock = jest.fn(); + client.sendStateEvent = sendStateEventMock; + + sess!.joinRoomSession([mockFocus]); + + expect(client.sendStateEvent).toHaveBeenCalledTimes(1); + + sess!.joinRoomSession([mockFocus]); + expect(client.sendStateEvent).toHaveBeenCalledTimes(1); + }); + + it("renews membership event before expiry time", async () => { + jest.useFakeTimers(); + let resolveFn: ((_roomId: string, _type: string, val: Record) => void) | undefined; + + const eventSentPromise = new Promise>((r) => { + resolveFn = (_roomId: string, _type: string, val: Record) => { + r(val); + }; + }); + try { + const sendStateEventMock = jest.fn().mockImplementation(resolveFn); + client.sendStateEvent = sendStateEventMock; + + sess!.joinRoomSession([mockFocus]); + + const eventContent = await eventSentPromise; + + // definitely should have renewed by 1 second before the expiry! + const timeElapsed = 60 * 60 * 1000 - 1000; + mockRoom.getLiveTimeline().getState(EventTimeline.FORWARDS)!.getStateEvents = jest + .fn() + .mockReturnValue(mockRTCEvent(eventContent.memberships, mockRoom.roomId, () => timeElapsed)); + + const eventReSentPromise = new Promise>((r) => { + resolveFn = (_roomId: string, _type: string, val: Record) => { + r(val); + }; + }); + + sendStateEventMock.mockReset().mockImplementation(resolveFn); + + jest.setSystemTime(Date.now() + timeElapsed); + jest.advanceTimersByTime(timeElapsed); + await eventReSentPromise; + + expect(sendStateEventMock).toHaveBeenCalledWith( + mockRoom.roomId, + EventType.GroupCallMemberPrefix, + { + memberships: [ + { + application: "m.call", + scope: "m.room", + call_id: "", + device_id: "AAAAAAA", + expires: 3600000 * 2, + foci_active: [{ type: "mock" }], + created_ts: 1000, + }, + ], + }, + "@alice:example.org", + ); + } finally { + jest.useRealTimers(); + } + }); + }); + + it("emits an event at the time a membership event expires", () => { + jest.useFakeTimers(); + try { + let eventAge = 0; + + const membership = Object.assign({}, membershipTemplate); + const mockRoom = makeMockRoom([membership], () => eventAge); + + sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom); + const membershipObject = sess.memberships[0]; + + const onMembershipsChanged = jest.fn(); + sess.on(MatrixRTCSessionEvent.MembershipsChanged, onMembershipsChanged); + + eventAge = 61 * 1000 * 1000; + jest.advanceTimersByTime(61 * 1000 * 1000); + + expect(onMembershipsChanged).toHaveBeenCalledWith([membershipObject], []); + expect(sess?.memberships.length).toEqual(0); + } finally { + jest.useRealTimers(); + } + }); + + it("prunes expired memberships on update", () => { + client.sendStateEvent = jest.fn(); + + let eventAge = 0; + + const mockRoom = makeMockRoom( + [ + Object.assign({}, membershipTemplate, { + device_id: "OTHERDEVICE", + expires: 1000, + }), + ], + () => eventAge, + ); + sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom); + + // sanity check + expect(sess.memberships).toHaveLength(1); + expect(sess.memberships[0].deviceId).toEqual("OTHERDEVICE"); + + eventAge = 10000; + + sess.joinRoomSession([mockFocus]); + + expect(client.sendStateEvent).toHaveBeenCalledWith( + mockRoom!.roomId, + EventType.GroupCallMemberPrefix, + { + memberships: [ + { + application: "m.call", + scope: "m.room", + call_id: "", + device_id: "AAAAAAA", + expires: 3600000, + foci_active: [mockFocus], + }, + ], + }, + "@alice:example.org", + ); + }); + + it("fills in created_ts for other memberships on update", () => { + client.sendStateEvent = jest.fn(); + + const mockRoom = makeMockRoom([ + Object.assign({}, membershipTemplate, { + device_id: "OTHERDEVICE", + }), + ]); + sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom); + + sess.joinRoomSession([mockFocus]); + + expect(client.sendStateEvent).toHaveBeenCalledWith( + mockRoom!.roomId, + EventType.GroupCallMemberPrefix, + { + memberships: [ + { + application: "m.call", + scope: "m.room", + call_id: "", + device_id: "OTHERDEVICE", + expires: 3600000, + created_ts: 1000, + }, + { + application: "m.call", + scope: "m.room", + call_id: "", + device_id: "AAAAAAA", + expires: 3600000, + foci_active: [mockFocus], + }, + ], + }, + "@alice:example.org", + ); + }); +}); diff --git a/spec/unit/matrixrtc/MatrixRTCSessionManager.spec.ts b/spec/unit/matrixrtc/MatrixRTCSessionManager.spec.ts new file mode 100644 index 00000000000..fb96c3e53ae --- /dev/null +++ b/spec/unit/matrixrtc/MatrixRTCSessionManager.spec.ts @@ -0,0 +1,80 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { ClientEvent, EventTimeline, MatrixClient } from "../../../src"; +import { RoomStateEvent } from "../../../src/models/room-state"; +import { CallMembershipData } from "../../../src/matrixrtc/CallMembership"; +import { MatrixRTCSessionManagerEvents } from "../../../src/matrixrtc/MatrixRTCSessionManager"; +import { makeMockRoom } from "./mocks"; + +const membershipTemplate: CallMembershipData = { + call_id: "", + scope: "m.room", + application: "m.call", + device_id: "AAAAAAA", + expires: 60 * 60 * 1000, +}; + +describe("MatrixRTCSessionManager", () => { + let client: MatrixClient; + + beforeEach(async () => { + client = new MatrixClient({ baseUrl: "base_url" }); + client.matrixRTC.start(); + }); + + afterEach(() => { + client.stopClient(); + client.matrixRTC.stop(); + }); + + it("Fires event when session starts", () => { + const onStarted = jest.fn(); + client.matrixRTC.on(MatrixRTCSessionManagerEvents.SessionStarted, onStarted); + + try { + const room1 = makeMockRoom([membershipTemplate]); + jest.spyOn(client, "getRooms").mockReturnValue([room1]); + + client.emit(ClientEvent.Room, room1); + expect(onStarted).toHaveBeenCalledWith(room1.roomId, client.matrixRTC.getActiveRoomSession(room1)); + } finally { + client.matrixRTC.off(MatrixRTCSessionManagerEvents.SessionStarted, onStarted); + } + }); + + it("Fires event when session ends", () => { + const onEnded = jest.fn(); + client.matrixRTC.on(MatrixRTCSessionManagerEvents.SessionEnded, onEnded); + + const memberships = [membershipTemplate]; + + const room1 = makeMockRoom(memberships); + jest.spyOn(client, "getRooms").mockReturnValue([room1]); + jest.spyOn(client, "getRoom").mockReturnValue(room1); + + client.emit(ClientEvent.Room, room1); + + memberships.splice(0, 1); + + const roomState = room1.getLiveTimeline().getState(EventTimeline.FORWARDS)!; + const membEvent = roomState.getStateEvents("")[0]; + + client.emit(RoomStateEvent.Events, membEvent, roomState, null); + + expect(onEnded).toHaveBeenCalledWith(room1.roomId, client.matrixRTC.getActiveRoomSession(room1)); + }); +}); diff --git a/spec/unit/matrixrtc/mocks.ts b/spec/unit/matrixrtc/mocks.ts new file mode 100644 index 00000000000..fa7d948e620 --- /dev/null +++ b/spec/unit/matrixrtc/mocks.ts @@ -0,0 +1,66 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { EventType, MatrixEvent, Room } from "../../../src"; +import { CallMembershipData } from "../../../src/matrixrtc/CallMembership"; +import { randomString } from "../../../src/randomstring"; + +export function makeMockRoom( + memberships: CallMembershipData[], + getLocalAge: (() => number) | undefined = undefined, +): Room { + const roomId = randomString(8); + return { + roomId: roomId, + getLiveTimeline: jest.fn().mockReturnValue({ + getState: jest.fn().mockReturnValue(makeMockRoomState(memberships, roomId, getLocalAge)), + }), + } as unknown as Room; +} + +function makeMockRoomState(memberships: CallMembershipData[], roomId: string, getLocalAge: (() => number) | undefined) { + return { + getStateEvents: (_: string, stateKey: string) => { + const event = mockRTCEvent(memberships, roomId, getLocalAge); + + if (stateKey !== undefined) return event; + return [event]; + }, + }; +} + +export function mockRTCEvent( + memberships: CallMembershipData[], + roomId: string, + getLocalAge: (() => number) | undefined, +): MatrixEvent { + const getLocalAgeFn = getLocalAge ?? (() => 10); + + return { + getType: jest.fn().mockReturnValue(EventType.GroupCallMemberPrefix), + getContent: jest.fn().mockReturnValue({ + memberships: memberships, + }), + getSender: jest.fn().mockReturnValue("@mock:user.example"), + getTs: jest.fn().mockReturnValue(1000), + getLocalAge: getLocalAgeFn, + localTimestamp: Date.now(), + getRoomId: jest.fn().mockReturnValue(roomId), + sender: { + userId: "@mock:user.example", + }, + } as unknown as MatrixEvent; +} diff --git a/spec/unit/webrtc/groupCallEventHandler.spec.ts b/spec/unit/webrtc/groupCallEventHandler.spec.ts index c8d65538c60..7fe5cb6a014 100644 --- a/spec/unit/webrtc/groupCallEventHandler.spec.ts +++ b/spec/unit/webrtc/groupCallEventHandler.spec.ts @@ -71,7 +71,8 @@ describe("Group Call Event Handler", function () { getMember: (userId: string) => (userId === FAKE_USER_ID ? mockMember : null), } as unknown as Room; - (mockClient as any).getRoom = jest.fn().mockReturnValue(mockRoom); + mockClient.getRoom = jest.fn().mockReturnValue(mockRoom); + mockClient.getFoci.mockReturnValue([{}]); }); describe("reacts to state changes", () => { diff --git a/src/client.ts b/src/client.ts index cb6ce2ddae2..a3376ba2ab3 100644 --- a/src/client.ts +++ b/src/client.ts @@ -219,6 +219,7 @@ import { ServerSideSecretStorageImpl, } from "./secret-storage"; import { RegisterRequest, RegisterResponse } from "./@types/registration"; +import { MatrixRTCSessionManager } from "./matrixrtc/MatrixRTCSessionManager"; export type Store = IStore; @@ -382,6 +383,8 @@ export interface ICreateClientOpts { */ useE2eForGroupCall?: boolean; + livekitServiceURL?: string; + /** * Crypto callbacks provided by the application */ @@ -399,6 +402,12 @@ export interface ICreateClientOpts { * Default: false. */ isVoipWithNoMediaAllowed?: boolean; + + /** + * If true, group calls will not establish media connectivity and only create the signaling events, + * so that livekit media can be used in the application layert (js-sdk contains no livekit code). + */ + useLivekitForGroupCalls?: boolean; } export interface IMatrixClientCreateOpts extends ICreateClientOpts { @@ -1211,6 +1220,8 @@ export class MatrixClient extends TypedEventEmitter { @@ -1344,6 +1360,10 @@ export class MatrixClient extends TypedEventEmitter { if (this.isInitialSyncComplete()) { - this.callEventHandler!.start(); - this.groupCallEventHandler!.start(); + if (supportsMatrixCall()) { + this.callEventHandler!.start(); + this.groupCallEventHandler!.start(); + } + this.off(ClientEvent.Sync, this.startCallEventHandler); } }; + private startMatrixRTC = (): void => { + if (this.isInitialSyncComplete()) { + this.matrixRTC.start(); + + this.off(ClientEvent.Sync, this.startMatrixRTC); + } + }; + /** * Once the client has been initialised, we want to clear notifications we * know for a fact should be here. diff --git a/src/embedded.ts b/src/embedded.ts index e20c64a28b4..178d74b2ec7 100644 --- a/src/embedded.ts +++ b/src/embedded.ts @@ -29,7 +29,14 @@ import { MatrixEvent, IEvent, IContent, EventStatus } from "./models/event"; import { ISendEventResponse } from "./@types/requests"; import { EventType } from "./@types/event"; import { logger } from "./logger"; -import { MatrixClient, ClientEvent, IMatrixClientCreateOpts, IStartClientOpts, SendToDeviceContentMap } from "./client"; +import { + MatrixClient, + ClientEvent, + IMatrixClientCreateOpts, + IStartClientOpts, + SendToDeviceContentMap, + IOpenIDToken, +} from "./client"; import { SyncApi, SyncState } from "./sync"; import { SlidingSyncSdk } from "./sliding-sync-sdk"; import { User } from "./models/user"; @@ -153,6 +160,12 @@ export class RoomWidgetClient extends MatrixClient { // Open communication with the host widgetApi.start(); + // Send a content loaded event now we've started the widget API + // Note that element-web currently does not use waitForIFrameLoad=false and so + // does *not* (yes, that is the right way around) wait for this event. Let's + // start sending this, then once this has rolled out, we can change element-web to + // use waitForIFrameLoad=false and have a widget API that's less racy. + widgetApi.sendContentLoaded(); } public async startClient(opts: IStartClientOpts = {}): Promise { @@ -197,6 +210,8 @@ export class RoomWidgetClient extends MatrixClient { this.setSyncState(SyncState.Syncing); logger.info("Finished backfilling events"); + this.matrixRTC.start(); + // Watch for TURN servers, if requested if (this.capabilities.turnServers) this.watchTurnServers(); } @@ -241,6 +256,18 @@ export class RoomWidgetClient extends MatrixClient { return {}; } + public async getOpenIdToken(): Promise { + const token = await this.widgetApi.requestOpenIDConnectToken(); + // the IOpenIDCredentials from the widget-api and IOpenIDToken form the matrix-js-sdk are compatible. + // we still recreate the token to make this transparent and catch'able by the linter in case the types change in the future. + return { + access_token: token.access_token, + expires_in: token.expires_in, + matrix_server_name: token.matrix_server_name, + token_type: token.token_type, + }; + } + public async queueToDevice({ eventType, batch }: ToDeviceBatch): Promise { // map: user Id → device Id → payload const contentMap: MapWithDefault> = new MapWithDefault(() => new Map()); diff --git a/src/matrixrtc/CallMembership.ts b/src/matrixrtc/CallMembership.ts new file mode 100644 index 00000000000..e67909d1e32 --- /dev/null +++ b/src/matrixrtc/CallMembership.ts @@ -0,0 +1,95 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { MatrixEvent, RoomMember } from "../matrix"; +import { deepCompare } from "../utils"; +import { Focus } from "./focus"; + +type CallScope = "m.room" | "m.user"; + +// Represents an entry in the memberships section of an m.call.member event as it is on the wire +export interface CallMembershipData { + application?: string; + call_id: string; + scope: CallScope; + device_id: string; + created_ts?: number; + expires: number; + foci_active?: Focus[]; +} + +export class CallMembership { + public static equal(a: CallMembership, b: CallMembership): boolean { + return deepCompare(a.data, b.data); + } + + public constructor(private parentEvent: MatrixEvent, private data: CallMembershipData) { + if (typeof data.expires !== "number") throw new Error("Malformed membership: expires must be numeric"); + if (typeof data.device_id !== "string") throw new Error("Malformed membership event: device_id must be string"); + if (typeof data.call_id !== "string") throw new Error("Malformed membership event: call_id must be string"); + if (typeof data.scope !== "string") throw new Error("Malformed membership event: scope must be string"); + if (!parentEvent.sender) throw new Error("Invalid parent event: sender is null"); + } + + public get member(): RoomMember { + return this.parentEvent.sender!; + } + + public get callId(): string { + return this.data.call_id; + } + + public get deviceId(): string { + return this.data.device_id; + } + + public get application(): string | undefined { + return this.data.application; + } + + public get scope(): CallScope { + return this.data.scope; + } + + public createdTs(): number { + return this.data.created_ts ?? this.parentEvent.getTs(); + } + + public getAbsoluteExpiry(): number { + return this.createdTs() + this.data.expires; + } + + // gets the expiry time of the event, converted into the device's local time + public getLocalExpiry(): number { + const relativeCreationTime = this.parentEvent.getTs() - this.createdTs(); + + const localCreationTs = this.parentEvent.localTimestamp - relativeCreationTime; + + return localCreationTs + this.data.expires; + } + + public getMsUntilExpiry(): number { + return this.getLocalExpiry() - Date.now(); + } + + public isExpired(): boolean { + return this.getAbsoluteExpiry() < this.parentEvent.getTs() + this.parentEvent.getLocalAge(); + } + + public getActiveFoci(): Focus[] { + return this.data.foci_active ?? []; + } +} diff --git a/src/matrixrtc/MatrixRTCSession.ts b/src/matrixrtc/MatrixRTCSession.ts new file mode 100644 index 00000000000..5f9ce7cc572 --- /dev/null +++ b/src/matrixrtc/MatrixRTCSession.ts @@ -0,0 +1,418 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { logger } from "../logger"; +import { TypedEventEmitter } from "../models/typed-event-emitter"; +import { EventTimeline } from "../models/event-timeline"; +import { Room } from "../models/room"; +import { MatrixClient } from "../client"; +import { EventType } from "../@types/event"; +import { CallMembership, CallMembershipData } from "./CallMembership"; +import { Focus } from "./focus"; +import { MatrixEvent } from "../matrix"; + +const MEMBERSHIP_EXPIRY_TIME = 60 * 60 * 1000; +const MEMBER_EVENT_CHECK_PERIOD = 2 * 60 * 1000; // How often we check to see if we need to re-send our member event +const CALL_MEMBER_EVENT_RETRY_DELAY_MIN = 3000; + +export enum MatrixRTCSessionEvent { + // A member joined, left, or updated a property of their membership. + MembershipsChanged = "memberships_changed", + // We joined or left the session: our own local idea of whether we are joined, + // separate from MembershipsChanged, ie. independent of whether our member event + // has succesfully gone through. + JoinStateChanged = "join_state_changed", +} + +export type MatrixRTCSessionEventHandlerMap = { + [MatrixRTCSessionEvent.MembershipsChanged]: ( + oldMemberships: CallMembership[], + newMemberships: CallMembership[], + ) => void; + [MatrixRTCSessionEvent.JoinStateChanged]: (isJoined: boolean) => void; +}; + +/** + * A MatrixRTCSession manages the membership & properties of a MatrixRTC session. + * This class doesn't deal with media at all, just membership & properties of a session. + */ +export class MatrixRTCSession extends TypedEventEmitter { + // How many ms after we joined the call, that our membership should expire, or undefined + // if we're not yet joined + private relativeExpiry: number | undefined; + + private memberEventTimeout?: ReturnType; + private expiryTimeout?: ReturnType; + + private activeFoci: Focus[] | undefined; + + private updateCallMembershipRunning = false; + private needCallMembershipUpdate = false; + + /** + * Returns all the call memberships for a room, oldest first + */ + public static callMembershipsForRoom(room: Room): CallMembership[] { + const roomState = room.getLiveTimeline().getState(EventTimeline.FORWARDS); + if (!roomState) { + logger.warn("Couldn't get state for room " + room.roomId); + throw new Error("Could't get state for room " + room.roomId); + } + const callMemberEvents = roomState.getStateEvents(EventType.GroupCallMemberPrefix); + + const callMemberships: CallMembership[] = []; + for (const memberEvent of callMemberEvents) { + const eventMemberships: CallMembershipData[] = memberEvent.getContent()["memberships"]; + if (eventMemberships === undefined) { + logger.warn(`Ignoring malformed member event from ${memberEvent.getSender()}: no memberships section`); + continue; + } + if (!Array.isArray(eventMemberships)) { + logger.warn(`Malformed member event from ${memberEvent.getSender()}: memberships is not an array`); + continue; + } + + for (const membershipData of eventMemberships) { + try { + const membership = new CallMembership(memberEvent, membershipData); + + if (membership.callId !== "" || membership.scope !== "m.room") { + // for now, just ignore anything that isn't the a room scope call + logger.info(`Ignoring user-scoped call`); + continue; + } + + if (membership.isExpired()) { + logger.info( + `Ignoring expired device membership ${memberEvent.getSender()}/${membership.deviceId}`, + ); + continue; + } + callMemberships.push(membership); + } catch (e) { + logger.warn("Couldn't construct call membership: ", e); + } + } + } + + callMemberships.sort((a, b) => a.createdTs() - b.createdTs()); + logger.debug( + "Call memberships, in order: ", + callMemberships.map((m) => [m.createdTs(), m.member.userId]), + ); + + return callMemberships; + } + + /** + * Return a the MatrixRTC for the room, whether there are currently active members or not + */ + public static roomSessionForRoom(client: MatrixClient, room: Room): MatrixRTCSession { + const callMemberships = MatrixRTCSession.callMembershipsForRoom(room); + + return new MatrixRTCSession(client, room, callMemberships); + } + + private constructor( + private readonly client: MatrixClient, + public readonly room: Room, + public memberships: CallMembership[], + ) { + super(); + this.setExpiryTimer(); + } + + /* + * Returns true if we intend to be participating in the MatrixRTC session. + */ + public isJoined(): boolean { + return this.relativeExpiry !== undefined; + } + + /** + * Performs cleanup & removes timers for client shutdown + */ + public stop(): void { + this.leaveRoomSession(); + if (this.expiryTimeout) { + clearTimeout(this.expiryTimeout); + this.expiryTimeout = undefined; + } + if (this.memberEventTimeout) { + clearTimeout(this.memberEventTimeout); + this.memberEventTimeout = undefined; + } + } + + /** + * Announces this user and device as joined to the MatrixRTC session, + * and continues to update the membership event to keep it valid until + * leaveRoomSession() is called + * This will not subscribe to updates: remember to call subscribe() separately if + * desired. + * This method will return immediately and the session will be joined in the background. + */ + public joinRoomSession(activeFoci: Focus[]): void { + if (this.isJoined()) { + logger.info(`Already joined to session in room ${this.room.roomId}: ignoring join call`); + return; + } + + logger.info(`Joining call session in room ${this.room.roomId}`); + this.activeFoci = activeFoci; + this.relativeExpiry = MEMBERSHIP_EXPIRY_TIME; + this.emit(MatrixRTCSessionEvent.JoinStateChanged, true); + // We don't wait for this, mostly because it may fail and schedule a retry, so this + // function returning doesn't really mean anything at all. + this.triggerCallMembershipEventUpdate(); + } + + /** + * Announces this user and device as having left the MatrixRTC session + * and stops scheduled updates. + * This will not unsubscribe from updates: remember to call unsubscribe() separately if + * desired. + */ + public leaveRoomSession(): void { + if (!this.isJoined()) { + logger.info(`Not joined to session in room ${this.room.roomId}: ignoring leave call`); + return; + } + + logger.info(`Leaving call session in room ${this.room.roomId}`); + this.relativeExpiry = undefined; + this.activeFoci = undefined; + this.emit(MatrixRTCSessionEvent.JoinStateChanged, false); + this.triggerCallMembershipEventUpdate(); + } + + /** + * Sets a timer for the soonest membership expiry + */ + private setExpiryTimer(): void { + if (this.expiryTimeout) { + clearTimeout(this.expiryTimeout); + this.expiryTimeout = undefined; + } + + let soonestExpiry; + for (const membership of this.memberships) { + const thisExpiry = membership.getMsUntilExpiry(); + if (soonestExpiry === undefined || thisExpiry < soonestExpiry) { + soonestExpiry = thisExpiry; + } + } + + if (soonestExpiry != undefined) { + this.expiryTimeout = setTimeout(this.onMembershipUpdate, soonestExpiry); + } + } + + public getOldestMembership(): CallMembership | undefined { + return this.memberships[0]; + } + + public onMembershipUpdate = (): void => { + const oldMemberships = this.memberships; + this.memberships = MatrixRTCSession.callMembershipsForRoom(this.room); + + const changed = + oldMemberships.length != this.memberships.length || + oldMemberships.some((m, i) => !CallMembership.equal(m, this.memberships[i])); + + if (changed) { + logger.info(`Memberships for call in room ${this.room.roomId} have changed: emitting`); + this.emit(MatrixRTCSessionEvent.MembershipsChanged, oldMemberships, this.memberships); + } + + this.setExpiryTimer(); + }; + + /** + * Constructs our own membership + * @param prevEvent - The previous version of our call membership, if any + */ + private makeMyMembership(prevMembership?: CallMembership): CallMembershipData { + if (this.relativeExpiry === undefined) { + throw new Error("Tried to create our own membership event when we're not joined!"); + } + + const m: CallMembershipData = { + call_id: "", + scope: "m.room", + application: "m.call", + device_id: this.client.getDeviceId()!, + expires: this.relativeExpiry, + foci_active: this.activeFoci, + }; + + if (prevMembership) m.created_ts = prevMembership.createdTs(); + + return m; + } + + /** + * Returns true if our membership event needs to be updated + */ + private membershipEventNeedsUpdate( + myPrevMembershipData?: CallMembershipData, + myPrevMembership?: CallMembership, + ): boolean { + // work out if we need to update our membership event + let needsUpdate = false; + // Need to update if there's a membership for us but we're not joined (valid or otherwise) + if (!this.isJoined() && myPrevMembershipData) needsUpdate = true; + if (this.isJoined()) { + // ...or if we are joined, but there's no valid membership event + if (!myPrevMembership) { + needsUpdate = true; + } else if (myPrevMembership.getMsUntilExpiry() < MEMBERSHIP_EXPIRY_TIME / 2) { + // ...or if the expiry time needs bumping + needsUpdate = true; + this.relativeExpiry! += MEMBERSHIP_EXPIRY_TIME; + } + } + + return needsUpdate; + } + + /** + * Makes a new membership list given the old list alonng with this user's previous membership event + * (if any) and this device's previous membership (if any) + */ + private makeNewMemberships( + oldMemberships: CallMembershipData[], + myCallMemberEvent?: MatrixEvent, + myPrevMembership?: CallMembership, + ): CallMembershipData[] { + const localDeviceId = this.client.getDeviceId(); + if (!localDeviceId) throw new Error("Local device ID is null!"); + + const filterExpired = (m: CallMembershipData): boolean => { + let membershipObj; + try { + membershipObj = new CallMembership(myCallMemberEvent!, m); + } catch (e) { + return false; + } + + return !membershipObj.isExpired(); + }; + + const transformMemberships = (m: CallMembershipData): CallMembershipData => { + if (m.created_ts === undefined) { + // we need to fill this in with the origin_server_ts from its original event + m.created_ts = myCallMemberEvent!.getTs(); + } + + return m; + }; + + // Filter our any invalid or expired memberships, and also our own - we'll add that back in next + let newMemberships = oldMemberships.filter(filterExpired).filter((m) => m.device_id !== localDeviceId); + + // Fix up any memberships that need their created_ts adding + newMemberships = newMemberships.map(transformMemberships); + + // If we're joined, add our own + if (this.isJoined()) { + newMemberships.push(this.makeMyMembership(myPrevMembership)); + } + + return newMemberships; + } + + private triggerCallMembershipEventUpdate = async (): Promise => { + if (this.updateCallMembershipRunning) { + this.needCallMembershipUpdate = true; + return; + } + + this.updateCallMembershipRunning = true; + try { + // if anything triggers an update while the update is running, do another update afterwards + do { + this.needCallMembershipUpdate = false; + await this.updateCallMembershipEvent(); + } while (this.needCallMembershipUpdate); + } finally { + this.updateCallMembershipRunning = false; + } + }; + + private async updateCallMembershipEvent(): Promise { + if (this.memberEventTimeout) { + clearTimeout(this.memberEventTimeout); + this.memberEventTimeout = undefined; + } + + const roomState = this.room.getLiveTimeline().getState(EventTimeline.FORWARDS); + if (!roomState) throw new Error("Couldn't get room state for room " + this.room.roomId); + + const localUserId = this.client.getUserId(); + const localDeviceId = this.client.getDeviceId(); + if (!localUserId || !localDeviceId) throw new Error("User ID or device ID was null!"); + + const myCallMemberEvent = roomState.getStateEvents(EventType.GroupCallMemberPrefix, localUserId) ?? undefined; + const content = myCallMemberEvent?.getContent>() ?? {}; + const memberships: CallMembershipData[] = Array.isArray(content["memberships"]) ? content["memberships"] : []; + + const myPrevMembershipData = memberships.find((m) => m.device_id === localDeviceId); + let myPrevMembership; + try { + if (myCallMemberEvent && myPrevMembershipData) { + myPrevMembership = new CallMembership(myCallMemberEvent, myPrevMembershipData); + } + } catch (e) { + // This would indicate a bug or something weird if our own call membership + // wasn't valid + logger.warn("Our previous call membership was invalid - this shouldn't happen.", e); + } + + if (myPrevMembership) { + logger.debug(`${myPrevMembership.getMsUntilExpiry()} until our membership expires`); + } + + if (!this.membershipEventNeedsUpdate(myPrevMembershipData, myPrevMembership)) { + // nothing to do - reschedule the check again + this.memberEventTimeout = setTimeout(this.triggerCallMembershipEventUpdate, MEMBER_EVENT_CHECK_PERIOD); + return; + } + + const newContent = { + memberships: this.makeNewMemberships(memberships, myCallMemberEvent, myPrevMembership), + }; + + let resendDelay; + try { + await this.client.sendStateEvent( + this.room.roomId, + EventType.GroupCallMemberPrefix, + newContent, + localUserId, + ); + logger.info(`Sent updated call member event.`); + + // check periodically to see if we need to refresh our member event + if (this.isJoined()) resendDelay = MEMBER_EVENT_CHECK_PERIOD; + } catch (e) { + resendDelay = CALL_MEMBER_EVENT_RETRY_DELAY_MIN + Math.random() * 2000; + logger.warn(`Failed to send call member event: retrying in ${resendDelay}`); + } + + if (resendDelay) this.memberEventTimeout = setTimeout(this.triggerCallMembershipEventUpdate, resendDelay); + } +} diff --git a/src/matrixrtc/MatrixRTCSessionManager.ts b/src/matrixrtc/MatrixRTCSessionManager.ts new file mode 100644 index 00000000000..6f643a26416 --- /dev/null +++ b/src/matrixrtc/MatrixRTCSessionManager.ts @@ -0,0 +1,128 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { logger } from "../logger"; +import { MatrixClient, ClientEvent } from "../client"; +import { TypedEventEmitter } from "../models/typed-event-emitter"; +import { Room } from "../models/room"; +import { RoomState, RoomStateEvent } from "../models/room-state"; +import { MatrixEvent } from "../models/event"; +import { MatrixRTCSession } from "./MatrixRTCSession"; + +export enum MatrixRTCSessionManagerEvents { + // A member has joined the MatrixRTC session, creating an active session in a room where there wasn't previously + SessionStarted = "session_started", + // All participants have left a given MatrixRTC session. + SessionEnded = "session_ended", +} + +type EventHandlerMap = { + [MatrixRTCSessionManagerEvents.SessionStarted]: (roomId: string, session: MatrixRTCSession) => void; + [MatrixRTCSessionManagerEvents.SessionEnded]: (roomId: string, session: MatrixRTCSession) => void; +}; + +/** + * Holds all active MatrixRTC session objects and creates new ones as events arrive. + * This interface is UNSTABLE and may change without warning. + */ +export class MatrixRTCSessionManager extends TypedEventEmitter { + // All the room-scoped sessions we know about. This will include any where the app + // has queried for the MatrixRTC sessions in a room, whether it's ever had any members + // or not). We keep a (lazily created) session object for every room to ensure that there + // is only ever one single room session object for any given room for the lifetime of the + // client: that way there can never be any code holding onto a stale object that is no + // longer the correct session object for the room. + private roomSessions = new Map(); + + public constructor(private client: MatrixClient) { + super(); + } + + public start(): void { + // We shouldn't need to null-check here, but matrix-client.spec.ts mocks getRooms + // returing nothing, and breaks tests if you change it to return an empty array :'( + for (const room of this.client.getRooms() ?? []) { + const session = MatrixRTCSession.roomSessionForRoom(this.client, room); + if (session.memberships.length > 0) { + this.roomSessions.set(room.roomId, session); + } + } + + this.client.on(ClientEvent.Room, this.onRoom); + this.client.on(RoomStateEvent.Events, this.onRoomState); + } + + public stop(): void { + for (const sess of this.roomSessions.values()) { + sess.stop(); + } + this.roomSessions.clear(); + + this.client.removeListener(ClientEvent.Room, this.onRoom); + this.client.removeListener(RoomStateEvent.Events, this.onRoomState); + } + + /** + * Gets the main MatrixRTC session for a room, or undefined if there is + * no current session + */ + public getActiveRoomSession(room: Room): MatrixRTCSession | undefined { + return this.roomSessions.get(room.roomId)!; + } + + /** + * Gets the main MatrixRTC session for a room, returning an empty session + * if no members are currently participating + */ + public getRoomSession(room: Room): MatrixRTCSession { + if (!this.roomSessions.has(room.roomId)) { + this.roomSessions.set(room.roomId, MatrixRTCSession.roomSessionForRoom(this.client, room)); + } + + return this.roomSessions.get(room.roomId)!; + } + + private onRoom = (room: Room): void => { + this.refreshRoom(room); + }; + + private onRoomState = (event: MatrixEvent, _state: RoomState): void => { + const room = this.client.getRoom(event.getRoomId()); + if (!room) { + logger.error(`Got room state event for unknown room ${event.getRoomId()}!`); + return; + } + + this.refreshRoom(room); + }; + + private refreshRoom(room: Room): void { + const isNewSession = !this.roomSessions.has(room.roomId); + const sess = this.getRoomSession(room); + + const wasActiveAndKnown = sess.memberships.length > 0 && !isNewSession; + + sess.onMembershipUpdate(); + + const nowActive = sess.memberships.length > 0; + + if (wasActiveAndKnown && !nowActive) { + this.emit(MatrixRTCSessionManagerEvents.SessionEnded, room.roomId, this.roomSessions.get(room.roomId)!); + } else if (!wasActiveAndKnown && nowActive) { + this.emit(MatrixRTCSessionManagerEvents.SessionStarted, room.roomId, this.roomSessions.get(room.roomId)!); + } + } +} diff --git a/src/matrixrtc/focus.ts b/src/matrixrtc/focus.ts new file mode 100644 index 00000000000..6892bbf15c0 --- /dev/null +++ b/src/matrixrtc/focus.ts @@ -0,0 +1,24 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/** + * Information about a MatrixRTC conference focus. The only attribute that + * the js-sdk (currently) knows about is the type: applications can extend + * this class for different types of focus. + */ +export interface Focus { + type: string; +} diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 4c3224435cd..4307a7a25ed 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -170,6 +170,8 @@ export interface IGroupCallRoomState { // TODO: Specify data-channels "dataChannelsEnabled"?: boolean; "dataChannelOptions"?: IGroupCallDataChannelOptions; + + "io.element.livekit_service_url"?: string; } export interface IGroupCallRoomMemberFeed { @@ -250,6 +252,7 @@ export class GroupCall extends TypedEventEmitter< private initWithAudioMuted = false; private initWithVideoMuted = false; private initCallFeedPromise?: Promise; + private _livekitServiceURL?: string; private stats: GroupCallStats | undefined; /** @@ -268,10 +271,16 @@ export class GroupCall extends TypedEventEmitter< private dataChannelsEnabled?: boolean, private dataChannelOptions?: IGroupCallDataChannelOptions, isCallWithoutVideoAndAudio?: boolean, + // this tells the js-sdk not to actually establish any calls to exchange media and just to + // create the group call signaling events, with the intention that the actual media will be + // handled using livekit. The js-sdk doesn't contain any code to do the actual livekit call though. + private useLivekit = false, + livekitServiceURL?: string, ) { super(); this.reEmitter = new ReEmitter(this); this.groupCallId = groupCallId ?? genCallID(); + this._livekitServiceURL = livekitServiceURL; this.creationTs = room.currentState.getStateEvents(EventType.GroupCallPrefix, this.groupCallId)?.getTs() ?? null; this.updateParticipants(); @@ -320,6 +329,12 @@ export class GroupCall extends TypedEventEmitter< this.client.groupCallEventHandler!.groupCalls.set(this.room.roomId, this); this.client.emit(GroupCallEventHandlerEvent.Outgoing, this); + await this.sendCallStateEvent(); + + return this; + } + + private async sendCallStateEvent(): Promise { const groupCallState: IGroupCallRoomState = { "m.intent": this.intent, "m.type": this.type, @@ -328,10 +343,20 @@ export class GroupCall extends TypedEventEmitter< "dataChannelsEnabled": this.dataChannelsEnabled, "dataChannelOptions": this.dataChannelsEnabled ? this.dataChannelOptions : undefined, }; + if (this.livekitServiceURL) { + groupCallState["io.element.livekit_service_url"] = this.livekitServiceURL; + } await this.client.sendStateEvent(this.room.roomId, EventType.GroupCallPrefix, groupCallState, this.groupCallId); + } - return this; + public get livekitServiceURL(): string | undefined { + return this._livekitServiceURL; + } + + public updateLivekitServiceURL(newURL: string): Promise { + this._livekitServiceURL = newURL; + return this.sendCallStateEvent(); } private _state = GroupCallState.LocalCallFeedUninitialized; @@ -442,6 +467,11 @@ export class GroupCall extends TypedEventEmitter< } public async initLocalCallFeed(): Promise { + if (this.useLivekit) { + logger.info("Livekit group call: not starting local call feed."); + return; + } + if (this.state !== GroupCallState.LocalCallFeedUninitialized) { throw new Error(`Cannot initialize local call feed in the "${this.state}" state.`); } @@ -537,11 +567,13 @@ export class GroupCall extends TypedEventEmitter< this.onIncomingCall(call); } - this.retryCallLoopInterval = setInterval(this.onRetryCallLoop, this.retryCallInterval); + if (!this.useLivekit) { + this.retryCallLoopInterval = setInterval(this.onRetryCallLoop, this.retryCallInterval); - this.activeSpeaker = undefined; - this.onActiveSpeakerLoop(); - this.activeSpeakerLoopInterval = setInterval(this.onActiveSpeakerLoop, this.activeSpeakerInterval); + this.activeSpeaker = undefined; + this.onActiveSpeakerLoop(); + this.activeSpeakerLoopInterval = setInterval(this.onActiveSpeakerLoop, this.activeSpeakerInterval); + } } private dispose(): void { @@ -923,6 +955,11 @@ export class GroupCall extends TypedEventEmitter< return; } + if (this.useLivekit) { + logger.info("Received incoming call whilst in signaling-only mode! Ignoring."); + return; + } + const deviceMap = this.calls.get(opponentUserId) ?? new Map(); const prevCall = deviceMap.get(newCall.getOpponentDeviceId()!); @@ -1629,7 +1666,7 @@ export class GroupCall extends TypedEventEmitter< } }); - if (this.state === GroupCallState.Entered) this.placeOutgoingCalls(); + if (this.state === GroupCallState.Entered && !this.useLivekit) this.placeOutgoingCalls(); // Update the participants stored in the stats object }; diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts index 1500e191862..3dae0ce4029 100644 --- a/src/webrtc/groupCallEventHandler.ts +++ b/src/webrtc/groupCallEventHandler.ts @@ -84,6 +84,7 @@ export class GroupCallEventHandler { } public stop(): void { + this.client.removeListener(ClientEvent.Room, this.onRoomsChanged); this.client.removeListener(RoomStateEvent.Events, this.onRoomStateChanged); } @@ -189,6 +190,8 @@ export class GroupCallEventHandler { content?.dataChannelsEnabled || this.client.isVoipWithNoMediaAllowed, dataChannelOptions, this.client.isVoipWithNoMediaAllowed, + this.client.useLivekitForGroupCalls, + content["io.element.livekit_service_url"], ); this.groupCalls.set(room.roomId, groupCall); diff --git a/yarn.lock b/yarn.lock index 48f28607990..89bbc614c48 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5291,7 +5291,7 @@ matrix-mock-request@^2.5.0: dependencies: expect "^28.1.0" -matrix-widget-api@^1.5.0: +matrix-widget-api@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.6.0.tgz#f0075411edffc6de339580ade7e6e6e6edb01af4" integrity sha512-VXIJyAZ/WnBmT4C7ePqevgMYGneKMCP/0JuCOqntSsaNlCRHJvwvTxmqUU+ufOpzIF5gYNyIrAjbgrEbK3iqJQ==