From 21f107d7340e118d4ca6cf5ab51fc5e4fff7d39f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 27 Jul 2021 14:19:54 +0200 Subject: [PATCH 01/15] Add types for MSC3291 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/@types/event.ts | 1 + src/webrtc/callEventTypes.ts | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/src/@types/event.ts b/src/@types/event.ts index 42c7d0529c7..f1f5094f8f8 100644 --- a/src/@types/event.ts +++ b/src/@types/event.ts @@ -53,6 +53,7 @@ export enum EventType { CallReject = "m.call.reject", CallSelectAnswer = "m.call.select_answer", CallNegotiate = "m.call.negotiate", + CallSDPStreamMetadataChanged = "org.matrix.call.sdp_stream_metadata_changed", CallReplaces = "m.call.replaces", CallAssertedIdentity = "m.call.asserted_identity", CallAssertedIdentityPrefix = "org.matrix.call.asserted_identity", diff --git a/src/webrtc/callEventTypes.ts b/src/webrtc/callEventTypes.ts index 5325678438b..c380181f073 100644 --- a/src/webrtc/callEventTypes.ts +++ b/src/webrtc/callEventTypes.ts @@ -11,6 +11,8 @@ export enum SDPStreamMetadataPurpose { export interface SDPStreamMetadataObject { purpose: SDPStreamMetadataPurpose; + audio_muted: boolean; + video_muted: boolean; } export interface SDPStreamMetadata { @@ -40,6 +42,10 @@ export interface MCallOfferNegotiate { [SDPStreamMetadataKey]: SDPStreamMetadata; } +export interface MCallSDPStreamMetadataChanged { + [SDPStreamMetadataKey]: SDPStreamMetadata; +} + export interface MCallReplacesTarget { id: string; display_name: string; From 8e6040ad6f8ec31c9a908cea4d86e5fc4b51dc1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 27 Jul 2021 14:23:58 +0200 Subject: [PATCH 02/15] Give CallFeed a mute state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/callFeed.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 69d6170abeb..085291b8c22 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -21,6 +21,7 @@ import { RoomMember } from "../models/room-member"; export enum CallFeedEvent { NewStream = "new_stream", + MuteStateChanged = "mute_state_changed" } export class CallFeed extends EventEmitter { @@ -30,6 +31,8 @@ export class CallFeed extends EventEmitter { public purpose: SDPStreamMetadataPurpose, private client: MatrixClient, private roomId: string, + private audioMuted?: boolean, + private videoMuted?: boolean, ) { super(); } @@ -51,15 +54,13 @@ export class CallFeed extends EventEmitter { return this.userId === this.client.getUserId(); } - // TODO: The two following methods should be later replaced - // by something that will also check if the remote is muted /** * Returns true if audio is muted or if there are no audio * tracks, otherwise returns false * @returns {boolean} is audio muted? */ public isAudioMuted(): boolean { - return this.stream.getAudioTracks().length === 0; + return this.stream.getAudioTracks().length === 0 || this.audioMuted; } /** @@ -69,7 +70,7 @@ export class CallFeed extends EventEmitter { */ public isVideoMuted(): boolean { // We assume only one video track - return this.stream.getVideoTracks().length === 0; + return this.stream.getVideoTracks().length === 0 || this.videoMuted; } /** @@ -81,4 +82,14 @@ export class CallFeed extends EventEmitter { this.stream = newStream; this.emit(CallFeedEvent.NewStream, this.stream); } + + public setAudioMuted(muted: boolean): void { + this.audioMuted = muted; + this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); + } + + public setVideoMuted(muted: boolean): void { + this.videoMuted = muted; + this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); + } } From 1df90f8fc7ffed34023ebcb3bb0a5b32bc02fd94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 27 Jul 2021 14:31:13 +0200 Subject: [PATCH 03/15] Basic implementation of MSC3291 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 59 +++++++++++++++++++++++----------- src/webrtc/callEventHandler.ts | 9 ++++++ 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 6a9a8be7667..3c25e22acd7 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -36,6 +36,7 @@ import { SDPStreamMetadataPurpose, SDPStreamMetadata, SDPStreamMetadataKey, + MCallSDPStreamMetadataChanged, } from './callEventTypes'; import { CallFeed } from './callFeed'; @@ -353,8 +354,6 @@ export class MatrixCall extends EventEmitter { this.makingOffer = false; this.remoteOnHold = false; - this.micMuted = false; - this.vidMuted = false; this.feeds = []; @@ -398,6 +397,14 @@ export class MatrixCall extends EventEmitter { return this.remoteAssertedIdentity; } + public get localUsermediaFeed(): CallFeed { + return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); + } + + private getFeedByStreamId(streamId: string): CallFeed { + return this.getFeeds().find((feed) => feed.stream.id === streamId); + } + /** * Returns an array of all CallFeeds * @returns {Array} CallFeeds @@ -427,10 +434,12 @@ export class MatrixCall extends EventEmitter { * @returns {SDPStreamMetadata} localSDPStreamMetadata */ private getLocalSDPStreamMetadata(): SDPStreamMetadata { - const metadata = {}; + const metadata: SDPStreamMetadata = {}; for (const localFeed of this.getLocalFeeds()) { metadata[localFeed.stream.id] = { purpose: localFeed.purpose, + audio_muted: localFeed.isAudioMuted(), + video_muted: localFeed.isVideoMuted(), }; } logger.debug("Got local SDPStreamMetadata", metadata); @@ -455,6 +464,8 @@ export class MatrixCall extends EventEmitter { const userId = this.getOpponentMember().userId; const purpose = this.remoteSDPStreamMetadata[stream.id].purpose; + const audioMuted = this.remoteSDPStreamMetadata[stream.id].audio_muted; + const videoMuted = this.remoteSDPStreamMetadata[stream.id].video_muted; if (!purpose) { logger.warn(`Ignoring stream with id ${stream.id} because we didn't get any metadata about it`); @@ -467,7 +478,7 @@ export class MatrixCall extends EventEmitter { if (existingFeed) { existingFeed.setNewStream(stream); } else { - this.feeds.push(new CallFeed(stream, userId, purpose, this.client, this.roomId)); + this.feeds.push(new CallFeed(stream, userId, purpose, this.client, this.roomId, audioMuted, videoMuted)); this.emit(CallEvent.FeedsChanged, this.feeds); } @@ -494,7 +505,7 @@ export class MatrixCall extends EventEmitter { // Try to find a feed with the same stream id as the new stream, // if we find it replace the old stream with the new one - const feed = this.feeds.find((feed) => feed.stream.id === stream.id); + const feed = this.getFeedByStreamId(stream.id); if (feed) { feed.setNewStream(stream); } else { @@ -513,7 +524,7 @@ export class MatrixCall extends EventEmitter { if (existingFeed) { existingFeed.setNewStream(stream); } else { - this.feeds.push(new CallFeed(stream, userId, purpose, this.client, this.roomId)); + this.feeds.push(new CallFeed(stream, userId, purpose, this.client, this.roomId, true, true)); this.emit(CallEvent.FeedsChanged, this.feeds); } @@ -551,7 +562,7 @@ export class MatrixCall extends EventEmitter { private deleteFeedByStream(stream: MediaStream) { logger.debug(`Removing feed with stream id ${stream.id}`); - const feed = this.feeds.find((feed) => feed.stream.id === stream.id); + const feed = this.getFeedByStreamId(stream.id); if (!feed) { logger.warn(`Didn't find the feed with stream id ${stream.id} to delete`); return; @@ -887,7 +898,7 @@ export class MatrixCall extends EventEmitter { * @param {boolean} muted True to mute the outbound video. */ setLocalVideoMuted(muted: boolean) { - this.vidMuted = muted; + this.localUsermediaFeed?.setVideoMuted(muted); this.updateMuteStatus(); } @@ -901,8 +912,7 @@ export class MatrixCall extends EventEmitter { * (including if the call is not set up yet). */ isLocalVideoMuted(): boolean { - if (this.type === CallType.Voice) return true; - return this.vidMuted; + return this.localUsermediaFeed?.isVideoMuted(); } /** @@ -910,7 +920,7 @@ export class MatrixCall extends EventEmitter { * @param {boolean} muted True to mute the mic. */ setMicrophoneMuted(muted: boolean) { - this.micMuted = muted; + this.localUsermediaFeed?.setAudioMuted(muted); this.updateMuteStatus(); } @@ -924,7 +934,7 @@ export class MatrixCall extends EventEmitter { * is not set up yet). */ isMicrophoneMuted(): boolean { - return this.micMuted; + return this.localUsermediaFeed?.isAudioMuted(); } /** @@ -987,14 +997,14 @@ export class MatrixCall extends EventEmitter { } private updateMuteStatus() { - if (!this.localAVStream) { - return; - } + this.sendVoipEvent(EventType.CallSDPStreamMetadataChanged, { + [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(), + }); - const micShouldBeMuted = this.micMuted || this.remoteOnHold; - setTracksEnabled(this.localAVStream.getAudioTracks(), !micShouldBeMuted); + const micShouldBeMuted = this.localUsermediaFeed?.isAudioMuted() || this.remoteOnHold; + const vidShouldBeMuted = this.localUsermediaFeed?.isVideoMuted() || this.remoteOnHold; - const vidShouldBeMuted = this.vidMuted || this.remoteOnHold; + setTracksEnabled(this.localAVStream.getAudioTracks(), !micShouldBeMuted); setTracksEnabled(this.localAVStream.getVideoTracks(), !vidShouldBeMuted); } @@ -1318,6 +1328,19 @@ export class MatrixCall extends EventEmitter { } } + public onSDPStreamMetadataChangedReceived(event: MatrixEvent): void { + // TODO: What if the values is missing + // TODO: Handle purpose changes + const content = event.getContent(); + const metadata = content[SDPStreamMetadataKey]; + this.remoteSDPStreamMetadata = metadata; + for (const feed of this.getRemoteFeeds()) { + const streamId = feed.stream.id; + feed.setAudioMuted(metadata[streamId].audio_muted); + feed.setVideoMuted(metadata[streamId].video_muted); + } + } + async onAssertedIdentityReceived(event: MatrixEvent) { if (!event.getContent().asserted_identity) return; diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index 263ddbf9bcf..fa498e63335 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -297,6 +297,15 @@ export class CallEventHandler { } call.onAssertedIdentityReceived(event); + } else if (event.getType() === EventType.CallSDPStreamMetadataChanged) { + if (!call) return; + + if (event.getContent().party_id === call.ourPartyId) { + // Ignore remote echo + return; + } + + call.onSDPStreamMetadataChangedReceived(event); } } } From 6fda2a0c57d298e58376bbcef378123fda461a24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 28 Jul 2021 14:15:00 +0200 Subject: [PATCH 04/15] Handle purpose changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 3c25e22acd7..82eccf5ca40 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1330,7 +1330,6 @@ export class MatrixCall extends EventEmitter { public onSDPStreamMetadataChangedReceived(event: MatrixEvent): void { // TODO: What if the values is missing - // TODO: Handle purpose changes const content = event.getContent(); const metadata = content[SDPStreamMetadataKey]; this.remoteSDPStreamMetadata = metadata; @@ -1338,6 +1337,7 @@ export class MatrixCall extends EventEmitter { const streamId = feed.stream.id; feed.setAudioMuted(metadata[streamId].audio_muted); feed.setVideoMuted(metadata[streamId].video_muted); + feed.purpose = metadata[streamId].purpose; } } From 8de6c5aad182aff1b36de17b24dd5f901ea2686e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 28 Jul 2021 14:59:57 +0200 Subject: [PATCH 05/15] Fix typo which caused all feeds to be muted by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 82eccf5ca40..89891665821 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -509,7 +509,7 @@ export class MatrixCall extends EventEmitter { if (feed) { feed.setNewStream(stream); } else { - this.feeds.push(new CallFeed(stream, userId, purpose, this.client, this.roomId)); + this.feeds.push(new CallFeed(stream, userId, purpose, this.client, this.roomId, false, false)); this.emit(CallEvent.FeedsChanged, this.feeds); } @@ -524,7 +524,7 @@ export class MatrixCall extends EventEmitter { if (existingFeed) { existingFeed.setNewStream(stream); } else { - this.feeds.push(new CallFeed(stream, userId, purpose, this.client, this.roomId, true, true)); + this.feeds.push(new CallFeed(stream, userId, purpose, this.client, this.roomId, false, false)); this.emit(CallEvent.FeedsChanged, this.feeds); } From 057eb0f2a547ba9769a210457806d6e6a2ecd842 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 28 Jul 2021 15:00:21 +0200 Subject: [PATCH 06/15] Make mute state props mandetory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/callFeed.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 085291b8c22..b82c68535bd 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -31,8 +31,8 @@ export class CallFeed extends EventEmitter { public purpose: SDPStreamMetadataPurpose, private client: MatrixClient, private roomId: string, - private audioMuted?: boolean, - private videoMuted?: boolean, + private audioMuted: boolean, + private videoMuted: boolean, ) { super(); } From b11a8459d8161a26bead13cc19516a99005d65e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 28 Jul 2021 15:16:24 +0200 Subject: [PATCH 07/15] Add a method for setting remoteSDPStreamMetadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 89891665821..1ef551dbcc3 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -612,7 +612,7 @@ export class MatrixCall extends EventEmitter { const sdpStreamMetadata = invite[SDPStreamMetadataKey]; if (sdpStreamMetadata) { - this.remoteSDPStreamMetadata = sdpStreamMetadata; + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); } else { logger.debug("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); } @@ -1221,7 +1221,7 @@ export class MatrixCall extends EventEmitter { const sdpStreamMetadata = event.getContent()[SDPStreamMetadataKey]; if (sdpStreamMetadata) { - this.remoteSDPStreamMetadata = sdpStreamMetadata; + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); } else { logger.warn("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); } @@ -1296,9 +1296,9 @@ export class MatrixCall extends EventEmitter { const prevLocalOnHold = this.isLocalOnHold(); - const metadata = event.getContent()[SDPStreamMetadataKey]; - if (metadata) { - this.remoteSDPStreamMetadata = metadata; + const sdpStreamMetadata = event.getContent()[SDPStreamMetadataKey]; + if (sdpStreamMetadata) { + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); } else { logger.warn("Received negotiation event without SDPStreamMetadata!"); } @@ -1328,19 +1328,22 @@ export class MatrixCall extends EventEmitter { } } - public onSDPStreamMetadataChangedReceived(event: MatrixEvent): void { - // TODO: What if the values is missing - const content = event.getContent(); - const metadata = content[SDPStreamMetadataKey]; + private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { this.remoteSDPStreamMetadata = metadata; for (const feed of this.getRemoteFeeds()) { const streamId = feed.stream.id; - feed.setAudioMuted(metadata[streamId].audio_muted); - feed.setVideoMuted(metadata[streamId].video_muted); - feed.purpose = metadata[streamId].purpose; + feed.setAudioMuted(metadata[streamId]?.audio_muted ?? feed.isAudioMuted()); + feed.setVideoMuted(metadata[streamId]?.video_muted ?? feed.isVideoMuted()); + feed.purpose = metadata[streamId]?.purpose ?? feed.purpose; } } + public onSDPStreamMetadataChangedReceived(event: MatrixEvent): void { + const content = event.getContent(); + const metadata = content[SDPStreamMetadataKey]; + this.updateRemoteSDPStreamMetadata(metadata); + } + async onAssertedIdentityReceived(event: MatrixEvent) { if (!event.getContent().asserted_identity) return; From 23ab3e3ec0a8c6bedba9be80aac5a2b8e9814601 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 28 Jul 2021 15:29:28 +0200 Subject: [PATCH 08/15] Test mute metadata gets mapped onto feeds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- spec/unit/webrtc/call.spec.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 69a3553b01c..400c5ecb360 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -321,16 +321,19 @@ describe('Call', function() { [SDPStreamMetadataKey]: { "stream_id": { purpose: SDPStreamMetadataPurpose.Usermedia, + audio_muted: true, + video_muted: false, }, }, }; }, }); - call.pushRemoteFeed({ id: "stream_id" }); - expect(call.getFeeds().find((feed) => { - return feed.stream.id === "stream_id"; - })?.purpose).toBe(SDPStreamMetadataPurpose.Usermedia); + call.pushRemoteFeed({ id: "stream_id", getAudioTracks: () => ["track1"], getVideoTracks: () => ["track1"] }); + const feed = call.getFeeds().find((feed) => feed.stream.id === "stream_id"); + expect(feed?.purpose).toBe(SDPStreamMetadataPurpose.Usermedia); + expect(feed?.isAudioMuted()).toBeTruthy(); + expect(feed?.isVideoMuted()).not.toBeTruthy(); }); it("should fallback to replaceTrack() if the other side doesn't support SPDStreamMetadata", async () => { From ca7a4570947c2369963d88dddbe43151fd4c386f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 28 Jul 2021 17:56:51 +0200 Subject: [PATCH 09/15] Add recursivelyAssign() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/utils.ts | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/utils.ts b/src/utils.ts index 5abed06af2a..9d646087861 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -693,3 +693,25 @@ const collator = new Intl.Collator(); export function compare(a: string, b: string): number { return collator.compare(a, b); } + +/** + * This function is similar to Object.assign() but it assigns recursively and + * allows you to ignore nullish values from the source + * + * @param {Object} target + * @param {Object} source + * @returns the target object + */ +export function recursivelyAssign(target: T, source: S, ignoreNullish = false): T { + for (const [sourceKey, sourceValue] of Object.entries(source)) { + if (target[sourceKey] instanceof Object && sourceValue) { + recursivelyAssign(target[sourceKey], sourceValue); + continue; + } + if ((sourceValue !== null && sourceValue !== undefined) || !ignoreNullish) { + target[sourceKey] = sourceValue; + continue; + } + } + return target; +} From 2d25150a213ce63ba60f3be43f687f64d64005c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 28 Jul 2021 17:57:16 +0200 Subject: [PATCH 10/15] Write tests for recursivelyAssign() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- spec/unit/utils.spec.ts | 64 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/spec/unit/utils.spec.ts b/spec/unit/utils.spec.ts index 9611511b40d..12a83b235df 100644 --- a/spec/unit/utils.spec.ts +++ b/spec/unit/utils.spec.ts @@ -493,4 +493,68 @@ describe("utils", function() { expect(deepSortedObjectEntries(input)).toMatchObject(output); }); }); + + describe("recursivelyAssign", () => { + it("doesn't override with null/undefined", () => { + const result = utils.recursivelyAssign( + { + string: "Hello world", + object: {}, + float: 0.1, + }, { + string: null, + object: undefined, + }, + true, + ); + + expect(result).toStrictEqual({ + string: "Hello world", + object: {}, + float: 0.1, + }); + }); + + it("assigns recursively", () => { + const result = utils.recursivelyAssign( + { + number: 42, + object: { + message: "Hello world", + day: "Monday", + langs: { + compiled: ["c++"], + }, + }, + thing: "string", + }, { + number: 2, + object: { + message: "How are you", + day: "Friday", + langs: { + compiled: ["c++", "c"], + }, + }, + thing: { + aSubThing: "something", + }, + }, + ); + + expect(result).toStrictEqual({ + number: 2, + object: { + message: "How are you", + day: "Friday", + langs: { + compiled: ["c++", "c"], + }, + }, + thing: { + aSubThing: "something", + }, + }); + }); + }); }); From 606aa29381b7d1e692bbd08f7c74619798859327 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 28 Jul 2021 18:37:48 +0200 Subject: [PATCH 11/15] Use recursivelyAssign MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 1ef551dbcc3..b6687fb6b60 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1329,7 +1329,7 @@ export class MatrixCall extends EventEmitter { } private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { - this.remoteSDPStreamMetadata = metadata; + metadata = utils.recursivelyAssign(this.remoteSDPStreamMetadata, metadata, true); for (const feed of this.getRemoteFeeds()) { const streamId = feed.stream.id; feed.setAudioMuted(metadata[streamId]?.audio_muted ?? feed.isAudioMuted()); From b3c66848e25c0713db813158d46d79521e532d4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 28 Jul 2021 19:06:24 +0200 Subject: [PATCH 12/15] Handle missing params MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/utils.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/utils.ts b/src/utils.ts index 9d646087861..74dbb4c9fff 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -703,6 +703,9 @@ export function compare(a: string, b: string): number { * @returns the target object */ export function recursivelyAssign(target: T, source: S, ignoreNullish = false): T { + if (!target) target = Object({}); + if (!source) source = Object({}); + for (const [sourceKey, sourceValue] of Object.entries(source)) { if (target[sourceKey] instanceof Object && sourceValue) { recursivelyAssign(target[sourceKey], sourceValue); From 87fd3521ea2449b8c6034d04e8862804618815f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 28 Jul 2021 20:11:51 +0200 Subject: [PATCH 13/15] Fix types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/utils.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index 74dbb4c9fff..b5701df67a7 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -702,10 +702,7 @@ export function compare(a: string, b: string): number { * @param {Object} source * @returns the target object */ -export function recursivelyAssign(target: T, source: S, ignoreNullish = false): T { - if (!target) target = Object({}); - if (!source) source = Object({}); - +export function recursivelyAssign(target: Object, source: Object, ignoreNullish = false): any { for (const [sourceKey, sourceValue] of Object.entries(source)) { if (target[sourceKey] instanceof Object && sourceValue) { recursivelyAssign(target[sourceKey], sourceValue); From ca042b36479435a9e9c699ffda5211ba709e7a79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 28 Jul 2021 20:12:21 +0200 Subject: [PATCH 14/15] Fix handling when remoteSDPStreamMetadata is null MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index b6687fb6b60..a74246c160c 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1329,12 +1329,12 @@ export class MatrixCall extends EventEmitter { } private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { - metadata = utils.recursivelyAssign(this.remoteSDPStreamMetadata, metadata, true); + this.remoteSDPStreamMetadata = utils.recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); for (const feed of this.getRemoteFeeds()) { const streamId = feed.stream.id; - feed.setAudioMuted(metadata[streamId]?.audio_muted ?? feed.isAudioMuted()); - feed.setVideoMuted(metadata[streamId]?.video_muted ?? feed.isVideoMuted()); - feed.purpose = metadata[streamId]?.purpose ?? feed.purpose; + feed.setAudioMuted(this.remoteSDPStreamMetadata[streamId]?.audio_muted); + feed.setVideoMuted(this.remoteSDPStreamMetadata[streamId]?.video_muted); + feed.purpose = this.remoteSDPStreamMetadata[streamId]?.purpose; } } From e6696f78f4b3a521a1f643efc841f3a59ec82865 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 29 Jul 2021 13:00:48 +0200 Subject: [PATCH 15/15] Allow recieving unprefixed version of m.call.sdp_stream_metadata_changed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/@types/event.ts | 3 ++- src/webrtc/call.ts | 2 +- src/webrtc/callEventHandler.ts | 5 ++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/@types/event.ts b/src/@types/event.ts index f1f5094f8f8..56ac83d8a38 100644 --- a/src/@types/event.ts +++ b/src/@types/event.ts @@ -53,7 +53,8 @@ export enum EventType { CallReject = "m.call.reject", CallSelectAnswer = "m.call.select_answer", CallNegotiate = "m.call.negotiate", - CallSDPStreamMetadataChanged = "org.matrix.call.sdp_stream_metadata_changed", + CallSDPStreamMetadataChanged = "m.call.sdp_stream_metadata_changed", + CallSDPStreamMetadataChangedPrefix = "org.matrix.call.sdp_stream_metadata_changed", CallReplaces = "m.call.replaces", CallAssertedIdentity = "m.call.asserted_identity", CallAssertedIdentityPrefix = "org.matrix.call.asserted_identity", diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index c42ae2d1cc5..a18ccfae7d2 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1001,7 +1001,7 @@ export class MatrixCall extends EventEmitter { } private updateMuteStatus() { - this.sendVoipEvent(EventType.CallSDPStreamMetadataChanged, { + this.sendVoipEvent(EventType.CallSDPStreamMetadataChangedPrefix, { [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(), }); diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index fa498e63335..96b233e0f1d 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -297,7 +297,10 @@ export class CallEventHandler { } call.onAssertedIdentityReceived(event); - } else if (event.getType() === EventType.CallSDPStreamMetadataChanged) { + } else if ( + event.getType() === EventType.CallSDPStreamMetadataChanged || + event.getType() === EventType.CallSDPStreamMetadataChangedPrefix + ) { if (!call) return; if (event.getContent().party_id === call.ourPartyId) {