From 82303b908d60aac27365fb091d4e13d57a5ba142 Mon Sep 17 00:00:00 2001 From: Timo K Date: Tue, 30 May 2023 17:18:58 +0200 Subject: [PATCH 01/52] send expected peer connections to posthog. (based on roomState event) --- src/webrtc/groupCall.ts | 5 ++++- src/webrtc/stats/groupCallStats.ts | 17 ++++++++++++++-- src/webrtc/stats/statsReport.ts | 4 ++++ .../stats/summaryStatsReportGatherer.ts | 20 ++++++++++++++++++- 4 files changed, 42 insertions(+), 4 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 6d98586c92b..b882d947bca 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1595,6 +1595,9 @@ export class GroupCall extends TypedEventEmitter< }); if (this.state === GroupCallState.Entered) this.placeOutgoingCalls(); + + // Update the participants stored in the stats object + this.stats?.setParticipants(this.participants); }; private onStateChanged = (newState: GroupCallState, oldState: GroupCallState): void => { @@ -1628,7 +1631,7 @@ export class GroupCall extends TypedEventEmitter< public getGroupCallStats(): GroupCallStats { if (this.stats === undefined) { const userID = this.client.getUserId() || "unknown"; - this.stats = new GroupCallStats(this.groupCallId, userID, this.statsCollectIntervalTime); + this.stats = new GroupCallStats(this.groupCallId, userID, this.participants, this.statsCollectIntervalTime); this.stats.reports.on(StatsReport.CONNECTION_STATS, this.onConnectionStats); this.stats.reports.on(StatsReport.BYTE_SENT_STATS, this.onByteSentStats); this.stats.reports.on(StatsReport.SUMMARY_STATS, this.onSummaryStats); diff --git a/src/webrtc/stats/groupCallStats.ts b/src/webrtc/stats/groupCallStats.ts index 054d51f8154..e5a53fcc323 100644 --- a/src/webrtc/stats/groupCallStats.ts +++ b/src/webrtc/stats/groupCallStats.ts @@ -17,6 +17,8 @@ import { CallStatsReportGatherer } from "./callStatsReportGatherer"; import { StatsReportEmitter } from "./statsReportEmitter"; import { CallStatsReportSummary } from "./callStatsReportSummary"; import { SummaryStatsReportGatherer } from "./summaryStatsReportGatherer"; +import { ParticipantState } from "../groupCall"; +import { RoomMember } from "../../matrix"; export class GroupCallStats { private timer: undefined | ReturnType; @@ -24,7 +26,12 @@ export class GroupCallStats { public readonly reports = new StatsReportEmitter(); private readonly summaryStatsReportGatherer = new SummaryStatsReportGatherer(this.reports); - public constructor(private groupCallId: string, private userId: string, private interval: number = 10000) {} + public constructor( + private groupCallId: string, + private userId: string, + private participants: Map>, + private interval: number = 10000, + ) {} public start(): void { if (this.timer === undefined && this.interval > 0) { @@ -67,10 +74,16 @@ export class GroupCallStats { summary.push(c.processStats(this.groupCallId, this.userId)); }); - Promise.all(summary).then((s: Awaited[]) => this.summaryStatsReportGatherer.build(s)); + Promise.all(summary).then((s: Awaited[]) => this.summaryStatsReportGatherer.build(s, this.participants)); } public setInterval(interval: number): void { this.interval = interval; } + public setParticipants(participants: Map>): void { + this.participants = participants; + } + public getParticipants(): Map> { + return this.participants; + } } diff --git a/src/webrtc/stats/statsReport.ts b/src/webrtc/stats/statsReport.ts index cdfa751f464..3293a71fced 100644 --- a/src/webrtc/stats/statsReport.ts +++ b/src/webrtc/stats/statsReport.ts @@ -79,4 +79,8 @@ export interface SummaryStatsReport { maxPacketLoss: number; percentageConcealedAudio: number; peerConnections: number; + roomStateExpectedPeerConnections: number; + missingPeerConnections: number; + percentageEstablishedPeerConnections: number; + // Todo: Decide if we want an index (or a timestamp) of this report in relation to the group call, to help differenciate when issues occur and ignore/track initial connection delays. } diff --git a/src/webrtc/stats/summaryStatsReportGatherer.ts b/src/webrtc/stats/summaryStatsReportGatherer.ts index 87601dcac7a..93f5e042001 100644 --- a/src/webrtc/stats/summaryStatsReportGatherer.ts +++ b/src/webrtc/stats/summaryStatsReportGatherer.ts @@ -13,6 +13,8 @@ limitations under the License. import { StatsReportEmitter } from "./statsReportEmitter"; import { CallStatsReportSummary } from "./callStatsReportSummary"; import { SummaryStatsReport } from "./statsReport"; +import { ParticipantState } from "../groupCall"; +import { RoomMember } from "../../matrix"; interface CallStatsReportSummaryCounter { receivedAudio: number; @@ -25,7 +27,10 @@ interface CallStatsReportSummaryCounter { export class SummaryStatsReportGatherer { public constructor(private emitter: StatsReportEmitter) {} - public build(allSummary: CallStatsReportSummary[]): void { + public build( + allSummary: CallStatsReportSummary[], + callParticipants: Map>, + ): void { // Filter all stats which collect the first time webrtc stats. // Because stats based on time interval and the first collection of a summery stats has no previous // webrtcStats as basement all the calculation are 0. We don't want track the 0 stats. @@ -34,6 +39,16 @@ export class SummaryStatsReportGatherer { if (summaryTotalCount === 0) { return; } + // Calculate the actual number of devices based on the participants state event + // (this is used, to compare the expected participant count from the room state with the acutal peer connections) + // const devices = callParticipants.() + const devices: ({ id: String } & ParticipantState)[] = []; + for (const userEntry of callParticipants) { + for (const device of userEntry[1]) { + devices.push({ id: device[0], ...device[1] }); + } + } + const summaryCounter: CallStatsReportSummaryCounter = { receivedAudio: 0, receivedVideo: 0, @@ -66,6 +81,9 @@ export class SummaryStatsReportGatherer { : 0, ), peerConnections: summaryTotalCount, + roomStateExpectedPeerConnections: devices.length - 1, + missingPeerConnections: summaryTotalCount - (devices.length - 1), + percentageEstablishedPeerConnections: summaryTotalCount / (devices.length - 1), } as SummaryStatsReport; this.emitter.emitSummaryStatsReport(report); } From 91dfdd464fedc695031fe9adc13b95584d104d9c Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 31 May 2023 14:17:21 +0200 Subject: [PATCH 02/52] add tests --- spec/test-utils/webrtc.ts | 75 ++++++++ .../stats/summaryStatsReportGatherer.spec.ts | 163 ++++++++++++++---- .../stats/summaryStatsReportGatherer.ts | 8 +- 3 files changed, 207 insertions(+), 39 deletions(-) diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index 2037767cc94..c0a209424e4 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -745,3 +745,78 @@ export const REMOTE_SFU_DESCRIPTION = "a=sctp-port:5000\n" + "a=ice-ufrag:obZwzAcRtxwuozPZ\n" + "a=ice-pwd:TWXNaPeyKTTvRLyIQhWHfHlZHJjtcoKs"; + +export const groupCallParticipantsFourOtherDevices = new Map([ + [ + new RoomMember("roomId0", "user1"), + new Map([ + [ + "deviceId0", + { + sessionId: "0", + screensharing: false, + }, + ], + [ + "deviceId1", + { + sessionId: "1", + screensharing: false, + }, + ], + [ + "deviceId2", + { + sessionId: "2", + screensharing: false, + }, + ], + ]), + ], + [ + new RoomMember("roomId0", "user2"), + new Map([ + [ + "deviceId3", + { + sessionId: "0", + screensharing: false, + }, + ], + [ + "deviceId4", + { + sessionId: "1", + screensharing: false, + }, + ], + ]), + ], +]); + +export const groupCallParticipantsOneOtherDevice = new Map([ + [ + new RoomMember("roomId1", "thisMember"), + new Map([ + [ + "deviceId0", + { + sessionId: "0", + screensharing: false, + }, + ], + ]), + ], + [ + new RoomMember("roomId1", "opponentMember"), + new Map([ + [ + "deviceId1", + { + sessionId: "1", + screensharing: false, + }, + ], + ]), + ], +]); \ No newline at end of file diff --git a/spec/unit/webrtc/stats/summaryStatsReportGatherer.spec.ts b/spec/unit/webrtc/stats/summaryStatsReportGatherer.spec.ts index eefff0d9b49..a71e6a53875 100644 --- a/spec/unit/webrtc/stats/summaryStatsReportGatherer.spec.ts +++ b/spec/unit/webrtc/stats/summaryStatsReportGatherer.spec.ts @@ -15,7 +15,7 @@ limitations under the License. */ import { SummaryStatsReportGatherer } from "../../../../src/webrtc/stats/summaryStatsReportGatherer"; import { StatsReportEmitter } from "../../../../src/webrtc/stats/statsReportEmitter"; - +import { groupCallParticipantsOneOtherDevice, groupCallParticipantsFourOtherDevices } from "../../../test-utils/webrtc"; describe("SummaryStatsReportGatherer", () => { let reporter: SummaryStatsReportGatherer; let emitter: StatsReportEmitter; @@ -27,34 +27,37 @@ describe("SummaryStatsReportGatherer", () => { describe("build Summary Stats Report", () => { it("should do nothing if summary list empty", async () => { - reporter.build([]); + reporter.build([], new Map()); expect(emitter.emitSummaryStatsReport).not.toHaveBeenCalled(); }); it("should do nothing if a summary stats element collection the is first time", async () => { - reporter.build([ - { - isFirstCollection: true, - receivedMedia: 10, - receivedAudioMedia: 4, - receivedVideoMedia: 6, - audioTrackSummary: { - count: 1, - muted: 0, - maxJitter: 0, - maxPacketLoss: 0, - concealedAudio: 0, - totalAudio: 100, - }, - videoTrackSummary: { - count: 1, - muted: 0, - maxJitter: 0, - maxPacketLoss: 0, - concealedAudio: 0, - totalAudio: 0, - }, - }, - ]); + reporter.build( + [ + { + isFirstCollection: true, + receivedMedia: 10, + receivedAudioMedia: 4, + receivedVideoMedia: 6, + audioTrackSummary: { + count: 1, + muted: 0, + maxJitter: 0, + maxPacketLoss: 0, + concealedAudio: 0, + totalAudio: 100, + }, + videoTrackSummary: { + count: 1, + muted: 0, + maxJitter: 0, + maxPacketLoss: 0, + concealedAudio: 0, + totalAudio: 0, + }, + }, + ], + groupCallParticipantsOneOtherDevice, + ); expect(emitter.emitSummaryStatsReport).not.toHaveBeenCalled(); }); @@ -149,7 +152,7 @@ describe("SummaryStatsReportGatherer", () => { }, }, ]; - reporter.build(summary); + reporter.build(summary, groupCallParticipantsFourOtherDevices); expect(emitter.emitSummaryStatsReport).toHaveBeenCalledWith({ percentageReceivedMedia: 0.5, percentageReceivedAudioMedia: 0.5, @@ -158,6 +161,9 @@ describe("SummaryStatsReportGatherer", () => { maxPacketLoss: 0, peerConnections: 4, percentageConcealedAudio: 0.0375, + missingPeerConnections: 0, + percentageEstablishedPeerConnections: 1, + roomStateExpectedPeerConnections: 4, }); }); @@ -186,7 +192,7 @@ describe("SummaryStatsReportGatherer", () => { }, }, ]; - reporter.build(summary); + reporter.build(summary, groupCallParticipantsOneOtherDevice); expect(emitter.emitSummaryStatsReport).toHaveBeenCalledWith({ percentageReceivedMedia: 1, percentageReceivedAudioMedia: 1, @@ -195,6 +201,9 @@ describe("SummaryStatsReportGatherer", () => { maxPacketLoss: 0, peerConnections: 1, percentageConcealedAudio: 0, + missingPeerConnections: 0, + percentageEstablishedPeerConnections: 1, + roomStateExpectedPeerConnections: 1, }); }); @@ -223,7 +232,7 @@ describe("SummaryStatsReportGatherer", () => { }, }, ]; - reporter.build(summary); + reporter.build(summary, groupCallParticipantsOneOtherDevice); expect(emitter.emitSummaryStatsReport).toHaveBeenCalledWith({ percentageReceivedMedia: 0, percentageReceivedAudioMedia: 1, @@ -232,6 +241,9 @@ describe("SummaryStatsReportGatherer", () => { maxPacketLoss: 0, peerConnections: 1, percentageConcealedAudio: 0, + missingPeerConnections: 0, + percentageEstablishedPeerConnections: 1, + roomStateExpectedPeerConnections: 1, }); }); @@ -260,7 +272,7 @@ describe("SummaryStatsReportGatherer", () => { }, }, ]; - reporter.build(summary); + reporter.build(summary, groupCallParticipantsOneOtherDevice); expect(emitter.emitSummaryStatsReport).toHaveBeenCalledWith({ percentageReceivedMedia: 0, percentageReceivedAudioMedia: 0, @@ -269,6 +281,9 @@ describe("SummaryStatsReportGatherer", () => { maxPacketLoss: 0, peerConnections: 1, percentageConcealedAudio: 0, + missingPeerConnections: 0, + percentageEstablishedPeerConnections: 1, + roomStateExpectedPeerConnections: 1, }); }); @@ -363,7 +378,7 @@ describe("SummaryStatsReportGatherer", () => { }, }, ]; - reporter.build(summary); + reporter.build(summary, groupCallParticipantsFourOtherDevices); expect(emitter.emitSummaryStatsReport).toHaveBeenCalledWith({ percentageReceivedMedia: 1, percentageReceivedAudioMedia: 1, @@ -372,6 +387,9 @@ describe("SummaryStatsReportGatherer", () => { maxPacketLoss: 40, peerConnections: 4, percentageConcealedAudio: 0, + missingPeerConnections: 0, + percentageEstablishedPeerConnections: 1, + roomStateExpectedPeerConnections: 4, }); }); @@ -400,7 +418,7 @@ describe("SummaryStatsReportGatherer", () => { }, }, ]; - reporter.build(summary); + reporter.build(summary, groupCallParticipantsOneOtherDevice); expect(emitter.emitSummaryStatsReport).toHaveBeenCalledWith({ percentageReceivedMedia: 1, percentageReceivedAudioMedia: 1, @@ -409,6 +427,9 @@ describe("SummaryStatsReportGatherer", () => { maxPacketLoss: 0, peerConnections: 1, percentageConcealedAudio: 0, + missingPeerConnections: 0, + percentageEstablishedPeerConnections: 1, + roomStateExpectedPeerConnections: 1, }); }); @@ -437,7 +458,7 @@ describe("SummaryStatsReportGatherer", () => { }, }, ]; - reporter.build(summary); + reporter.build(summary, groupCallParticipantsOneOtherDevice); expect(emitter.emitSummaryStatsReport).toHaveBeenCalledWith({ percentageReceivedMedia: 1, percentageReceivedAudioMedia: 1, @@ -446,6 +467,9 @@ describe("SummaryStatsReportGatherer", () => { maxPacketLoss: 0, peerConnections: 1, percentageConcealedAudio: 0, + missingPeerConnections: 0, + percentageEstablishedPeerConnections: 1, + roomStateExpectedPeerConnections: 1, }); }); @@ -474,7 +498,7 @@ describe("SummaryStatsReportGatherer", () => { }, }, ]; - reporter.build(summary); + reporter.build(summary, groupCallParticipantsOneOtherDevice); expect(emitter.emitSummaryStatsReport).toHaveBeenCalledWith({ percentageReceivedMedia: 1, percentageReceivedAudioMedia: 1, @@ -483,6 +507,9 @@ describe("SummaryStatsReportGatherer", () => { maxPacketLoss: 0, peerConnections: 1, percentageConcealedAudio: 0, + missingPeerConnections: 0, + percentageEstablishedPeerConnections: 1, + roomStateExpectedPeerConnections: 1, }); }); @@ -577,15 +604,79 @@ describe("SummaryStatsReportGatherer", () => { }, }, ]; - reporter.build(summary); + reporter.build(summary, groupCallParticipantsFourOtherDevices); + expect(emitter.emitSummaryStatsReport).toHaveBeenCalledWith({ + percentageReceivedMedia: 1, + percentageReceivedAudioMedia: 1, + percentageReceivedVideoMedia: 1, + maxJitter: 2, + maxPacketLoss: 40, + peerConnections: 4, + percentageConcealedAudio: 0, + missingPeerConnections: 0, + percentageEstablishedPeerConnections: 1, + roomStateExpectedPeerConnections: 4, + }); + }); + it("should report missing peer connections", async () => { + const summary = [ + { + isFirstCollection: true, + receivedMedia: 1, + receivedAudioMedia: 1, + receivedVideoMedia: 1, + audioTrackSummary: { + count: 1, + muted: 0, + maxJitter: 20, + maxPacketLoss: 5, + concealedAudio: 0, + totalAudio: 0, + }, + videoTrackSummary: { + count: 1, + muted: 0, + maxJitter: 0, + maxPacketLoss: 0, + concealedAudio: 0, + totalAudio: 0, + }, + }, + { + isFirstCollection: false, + receivedMedia: 1, + receivedAudioMedia: 1, + receivedVideoMedia: 1, + audioTrackSummary: { + count: 1, + muted: 0, + maxJitter: 2, + maxPacketLoss: 5, + concealedAudio: 0, + totalAudio: 0, + }, + videoTrackSummary: { + count: 1, + muted: 0, + maxJitter: 0, + maxPacketLoss: 40, + concealedAudio: 0, + totalAudio: 0, + }, + }, + ]; + reporter.build(summary, groupCallParticipantsFourOtherDevices); expect(emitter.emitSummaryStatsReport).toHaveBeenCalledWith({ percentageReceivedMedia: 1, percentageReceivedAudioMedia: 1, percentageReceivedVideoMedia: 1, maxJitter: 2, maxPacketLoss: 40, - peerConnections: 3, + peerConnections: 2, percentageConcealedAudio: 0, + missingPeerConnections: 2, + percentageEstablishedPeerConnections: 0.5, + roomStateExpectedPeerConnections: 4, }); }); }); diff --git a/src/webrtc/stats/summaryStatsReportGatherer.ts b/src/webrtc/stats/summaryStatsReportGatherer.ts index 93f5e042001..6045cbb2ae8 100644 --- a/src/webrtc/stats/summaryStatsReportGatherer.ts +++ b/src/webrtc/stats/summaryStatsReportGatherer.ts @@ -36,6 +36,8 @@ export class SummaryStatsReportGatherer { // webrtcStats as basement all the calculation are 0. We don't want track the 0 stats. const summary = allSummary.filter((s) => !s.isFirstCollection); const summaryTotalCount = summary.length; + // For counting the peer connections we also want to consider the ignored summaries + const peerConnectionsCount= allSummary.length; if (summaryTotalCount === 0) { return; } @@ -80,10 +82,10 @@ export class SummaryStatsReportGatherer { ? (summaryCounter.concealedAudio / summaryCounter.totalAudio).toFixed(decimalPlaces) : 0, ), - peerConnections: summaryTotalCount, + peerConnections: peerConnectionsCount, roomStateExpectedPeerConnections: devices.length - 1, - missingPeerConnections: summaryTotalCount - (devices.length - 1), - percentageEstablishedPeerConnections: summaryTotalCount / (devices.length - 1), + missingPeerConnections: (devices.length - 1) - peerConnectionsCount, + percentageEstablishedPeerConnections: peerConnectionsCount / (devices.length - 1), } as SummaryStatsReport; this.emitter.emitSummaryStatsReport(report); } From 538b78765af055b0bcbf3c1403c45083ebbd1874 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 31 May 2023 14:23:31 +0200 Subject: [PATCH 03/52] change GroupCallStats initialized --- src/webrtc/groupCall.ts | 3 ++- src/webrtc/stats/groupCallStats.ts | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index b882d947bca..35c724ff109 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1631,7 +1631,8 @@ export class GroupCall extends TypedEventEmitter< public getGroupCallStats(): GroupCallStats { if (this.stats === undefined) { const userID = this.client.getUserId() || "unknown"; - this.stats = new GroupCallStats(this.groupCallId, userID, this.participants, this.statsCollectIntervalTime); + this.stats = new GroupCallStats(this.groupCallId, userID, this.statsCollectIntervalTime); + this.stats.setParticipants(this.participants); this.stats.reports.on(StatsReport.CONNECTION_STATS, this.onConnectionStats); this.stats.reports.on(StatsReport.BYTE_SENT_STATS, this.onByteSentStats); this.stats.reports.on(StatsReport.SUMMARY_STATS, this.onSummaryStats); diff --git a/src/webrtc/stats/groupCallStats.ts b/src/webrtc/stats/groupCallStats.ts index e5a53fcc323..b410e177530 100644 --- a/src/webrtc/stats/groupCallStats.ts +++ b/src/webrtc/stats/groupCallStats.ts @@ -25,11 +25,11 @@ export class GroupCallStats { private readonly gatherers: Map = new Map(); public readonly reports = new StatsReportEmitter(); private readonly summaryStatsReportGatherer = new SummaryStatsReportGatherer(this.reports); + private participants: Map> = new Map(); public constructor( private groupCallId: string, private userId: string, - private participants: Map>, private interval: number = 10000, ) {} From 0bc2bfd4a8fdb78c91c5141cd5d8465ad03d4a07 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 31 May 2023 15:53:45 +0200 Subject: [PATCH 04/52] prettier --- spec/test-utils/webrtc.ts | 2 +- src/webrtc/stats/groupCallStats.ts | 10 ++++------ src/webrtc/stats/summaryStatsReportGatherer.ts | 4 ++-- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index c0a209424e4..d4d50b458d7 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -819,4 +819,4 @@ export const groupCallParticipantsOneOtherDevice = new Map([ ], ]), ], -]); \ No newline at end of file +]); diff --git a/src/webrtc/stats/groupCallStats.ts b/src/webrtc/stats/groupCallStats.ts index b410e177530..f285581d0bc 100644 --- a/src/webrtc/stats/groupCallStats.ts +++ b/src/webrtc/stats/groupCallStats.ts @@ -27,11 +27,7 @@ export class GroupCallStats { private readonly summaryStatsReportGatherer = new SummaryStatsReportGatherer(this.reports); private participants: Map> = new Map(); - public constructor( - private groupCallId: string, - private userId: string, - private interval: number = 10000, - ) {} + public constructor(private groupCallId: string, private userId: string, private interval: number = 10000) {} public start(): void { if (this.timer === undefined && this.interval > 0) { @@ -74,7 +70,9 @@ export class GroupCallStats { summary.push(c.processStats(this.groupCallId, this.userId)); }); - Promise.all(summary).then((s: Awaited[]) => this.summaryStatsReportGatherer.build(s, this.participants)); + Promise.all(summary).then((s: Awaited[]) => + this.summaryStatsReportGatherer.build(s, this.participants), + ); } public setInterval(interval: number): void { diff --git a/src/webrtc/stats/summaryStatsReportGatherer.ts b/src/webrtc/stats/summaryStatsReportGatherer.ts index 6045cbb2ae8..e450ef6ee15 100644 --- a/src/webrtc/stats/summaryStatsReportGatherer.ts +++ b/src/webrtc/stats/summaryStatsReportGatherer.ts @@ -37,7 +37,7 @@ export class SummaryStatsReportGatherer { const summary = allSummary.filter((s) => !s.isFirstCollection); const summaryTotalCount = summary.length; // For counting the peer connections we also want to consider the ignored summaries - const peerConnectionsCount= allSummary.length; + const peerConnectionsCount = allSummary.length; if (summaryTotalCount === 0) { return; } @@ -84,7 +84,7 @@ export class SummaryStatsReportGatherer { ), peerConnections: peerConnectionsCount, roomStateExpectedPeerConnections: devices.length - 1, - missingPeerConnections: (devices.length - 1) - peerConnectionsCount, + missingPeerConnections: devices.length - peerConnectionsCount - 1, percentageEstablishedPeerConnections: peerConnectionsCount / (devices.length - 1), } as SummaryStatsReport; this.emitter.emitSummaryStatsReport(report); From dd21ee2dd1d4b7aef60cb2cb3f0af2209ff9a028 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 31 May 2023 16:07:55 +0200 Subject: [PATCH 05/52] more test and catch for promise --- spec/unit/webrtc/stats/groupCallStats.spec.ts | 8 ++++++++ src/webrtc/stats/groupCallStats.ts | 6 +++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/spec/unit/webrtc/stats/groupCallStats.spec.ts b/spec/unit/webrtc/stats/groupCallStats.spec.ts index 6aa45f307f9..d5ab5cc7e12 100644 --- a/spec/unit/webrtc/stats/groupCallStats.spec.ts +++ b/spec/unit/webrtc/stats/groupCallStats.spec.ts @@ -15,6 +15,7 @@ limitations under the License. */ import { GroupCallStats } from "../../../../src/webrtc/stats/groupCallStats"; import { CallStatsReportSummary } from "../../../../src/webrtc/stats/callStatsReportSummary"; +import { groupCallParticipantsOneOtherDevice } from "../../../test-utils/webrtc"; const GROUP_CALL_ID = "GROUP_ID"; const LOCAL_USER_ID = "LOCAL_USER_ID"; @@ -160,6 +161,13 @@ describe("GroupCallStats", () => { expect(clearInterval).not.toHaveBeenCalled(); }); }); + + describe("on setting participants", () => { + it("should return the same participants", async () => { + stats.setParticipants(groupCallParticipantsOneOtherDevice); + expect(stats.getParticipants()).toBe(groupCallParticipantsOneOtherDevice); + }); + }); }); const mockRTCPeerConnection = (): RTCPeerConnection => { diff --git a/src/webrtc/stats/groupCallStats.ts b/src/webrtc/stats/groupCallStats.ts index f285581d0bc..ab86df64a85 100644 --- a/src/webrtc/stats/groupCallStats.ts +++ b/src/webrtc/stats/groupCallStats.ts @@ -70,9 +70,9 @@ export class GroupCallStats { summary.push(c.processStats(this.groupCallId, this.userId)); }); - Promise.all(summary).then((s: Awaited[]) => - this.summaryStatsReportGatherer.build(s, this.participants), - ); + Promise.all(summary) + .then((s: Awaited[]) => this.summaryStatsReportGatherer.build(s, this.participants)) + .catch((err) => alert(err)); } public setInterval(interval: number): void { From b23b8fdad3c5b94f41dccbe74d5c3d00e1cf6b9b Mon Sep 17 00:00:00 2001 From: Timo K Date: Thu, 1 Jun 2023 22:06:18 +0200 Subject: [PATCH 06/52] seperate the participant logic in a summary extend function Signed-off-by: Timo K --- spec/unit/webrtc/stats/groupCallStats.spec.ts | 7 - .../stats/summaryStatsReportGatherer.spec.ts | 158 ++++++++++-------- src/webrtc/groupCall.ts | 10 +- src/webrtc/stats/groupCallStats.ts | 8 +- src/webrtc/stats/statsReport.ts | 7 +- .../stats/summaryStatsReportGatherer.ts | 39 +++-- 6 files changed, 125 insertions(+), 104 deletions(-) diff --git a/spec/unit/webrtc/stats/groupCallStats.spec.ts b/spec/unit/webrtc/stats/groupCallStats.spec.ts index d5ab5cc7e12..5ee060ca8fe 100644 --- a/spec/unit/webrtc/stats/groupCallStats.spec.ts +++ b/spec/unit/webrtc/stats/groupCallStats.spec.ts @@ -161,13 +161,6 @@ describe("GroupCallStats", () => { expect(clearInterval).not.toHaveBeenCalled(); }); }); - - describe("on setting participants", () => { - it("should return the same participants", async () => { - stats.setParticipants(groupCallParticipantsOneOtherDevice); - expect(stats.getParticipants()).toBe(groupCallParticipantsOneOtherDevice); - }); - }); }); const mockRTCPeerConnection = (): RTCPeerConnection => { diff --git a/spec/unit/webrtc/stats/summaryStatsReportGatherer.spec.ts b/spec/unit/webrtc/stats/summaryStatsReportGatherer.spec.ts index a71e6a53875..871fb2f883e 100644 --- a/spec/unit/webrtc/stats/summaryStatsReportGatherer.spec.ts +++ b/spec/unit/webrtc/stats/summaryStatsReportGatherer.spec.ts @@ -15,7 +15,8 @@ limitations under the License. */ import { SummaryStatsReportGatherer } from "../../../../src/webrtc/stats/summaryStatsReportGatherer"; import { StatsReportEmitter } from "../../../../src/webrtc/stats/statsReportEmitter"; -import { groupCallParticipantsOneOtherDevice, groupCallParticipantsFourOtherDevices } from "../../../test-utils/webrtc"; +import { groupCallParticipantsFourOtherDevices } from "../../../test-utils/webrtc"; + describe("SummaryStatsReportGatherer", () => { let reporter: SummaryStatsReportGatherer; let emitter: StatsReportEmitter; @@ -27,37 +28,34 @@ describe("SummaryStatsReportGatherer", () => { describe("build Summary Stats Report", () => { it("should do nothing if summary list empty", async () => { - reporter.build([], new Map()); + reporter.build([]); expect(emitter.emitSummaryStatsReport).not.toHaveBeenCalled(); }); it("should do nothing if a summary stats element collection the is first time", async () => { - reporter.build( - [ - { - isFirstCollection: true, - receivedMedia: 10, - receivedAudioMedia: 4, - receivedVideoMedia: 6, - audioTrackSummary: { - count: 1, - muted: 0, - maxJitter: 0, - maxPacketLoss: 0, - concealedAudio: 0, - totalAudio: 100, - }, - videoTrackSummary: { - count: 1, - muted: 0, - maxJitter: 0, - maxPacketLoss: 0, - concealedAudio: 0, - totalAudio: 0, - }, - }, - ], - groupCallParticipantsOneOtherDevice, - ); + reporter.build([ + { + isFirstCollection: true, + receivedMedia: 10, + receivedAudioMedia: 4, + receivedVideoMedia: 6, + audioTrackSummary: { + count: 1, + muted: 0, + maxJitter: 0, + maxPacketLoss: 0, + concealedAudio: 0, + totalAudio: 100, + }, + videoTrackSummary: { + count: 1, + muted: 0, + maxJitter: 0, + maxPacketLoss: 0, + concealedAudio: 0, + totalAudio: 0, + }, + }, + ]); expect(emitter.emitSummaryStatsReport).not.toHaveBeenCalled(); }); @@ -152,7 +150,7 @@ describe("SummaryStatsReportGatherer", () => { }, }, ]; - reporter.build(summary, groupCallParticipantsFourOtherDevices); + reporter.build(summary); expect(emitter.emitSummaryStatsReport).toHaveBeenCalledWith({ percentageReceivedMedia: 0.5, percentageReceivedAudioMedia: 0.5, @@ -161,9 +159,6 @@ describe("SummaryStatsReportGatherer", () => { maxPacketLoss: 0, peerConnections: 4, percentageConcealedAudio: 0.0375, - missingPeerConnections: 0, - percentageEstablishedPeerConnections: 1, - roomStateExpectedPeerConnections: 4, }); }); @@ -192,7 +187,7 @@ describe("SummaryStatsReportGatherer", () => { }, }, ]; - reporter.build(summary, groupCallParticipantsOneOtherDevice); + reporter.build(summary); expect(emitter.emitSummaryStatsReport).toHaveBeenCalledWith({ percentageReceivedMedia: 1, percentageReceivedAudioMedia: 1, @@ -201,9 +196,6 @@ describe("SummaryStatsReportGatherer", () => { maxPacketLoss: 0, peerConnections: 1, percentageConcealedAudio: 0, - missingPeerConnections: 0, - percentageEstablishedPeerConnections: 1, - roomStateExpectedPeerConnections: 1, }); }); @@ -232,7 +224,7 @@ describe("SummaryStatsReportGatherer", () => { }, }, ]; - reporter.build(summary, groupCallParticipantsOneOtherDevice); + reporter.build(summary); expect(emitter.emitSummaryStatsReport).toHaveBeenCalledWith({ percentageReceivedMedia: 0, percentageReceivedAudioMedia: 1, @@ -241,9 +233,6 @@ describe("SummaryStatsReportGatherer", () => { maxPacketLoss: 0, peerConnections: 1, percentageConcealedAudio: 0, - missingPeerConnections: 0, - percentageEstablishedPeerConnections: 1, - roomStateExpectedPeerConnections: 1, }); }); @@ -272,7 +261,7 @@ describe("SummaryStatsReportGatherer", () => { }, }, ]; - reporter.build(summary, groupCallParticipantsOneOtherDevice); + reporter.build(summary); expect(emitter.emitSummaryStatsReport).toHaveBeenCalledWith({ percentageReceivedMedia: 0, percentageReceivedAudioMedia: 0, @@ -281,9 +270,6 @@ describe("SummaryStatsReportGatherer", () => { maxPacketLoss: 0, peerConnections: 1, percentageConcealedAudio: 0, - missingPeerConnections: 0, - percentageEstablishedPeerConnections: 1, - roomStateExpectedPeerConnections: 1, }); }); @@ -378,7 +364,7 @@ describe("SummaryStatsReportGatherer", () => { }, }, ]; - reporter.build(summary, groupCallParticipantsFourOtherDevices); + reporter.build(summary); expect(emitter.emitSummaryStatsReport).toHaveBeenCalledWith({ percentageReceivedMedia: 1, percentageReceivedAudioMedia: 1, @@ -387,9 +373,6 @@ describe("SummaryStatsReportGatherer", () => { maxPacketLoss: 40, peerConnections: 4, percentageConcealedAudio: 0, - missingPeerConnections: 0, - percentageEstablishedPeerConnections: 1, - roomStateExpectedPeerConnections: 4, }); }); @@ -418,7 +401,7 @@ describe("SummaryStatsReportGatherer", () => { }, }, ]; - reporter.build(summary, groupCallParticipantsOneOtherDevice); + reporter.build(summary); expect(emitter.emitSummaryStatsReport).toHaveBeenCalledWith({ percentageReceivedMedia: 1, percentageReceivedAudioMedia: 1, @@ -427,9 +410,6 @@ describe("SummaryStatsReportGatherer", () => { maxPacketLoss: 0, peerConnections: 1, percentageConcealedAudio: 0, - missingPeerConnections: 0, - percentageEstablishedPeerConnections: 1, - roomStateExpectedPeerConnections: 1, }); }); @@ -458,7 +438,7 @@ describe("SummaryStatsReportGatherer", () => { }, }, ]; - reporter.build(summary, groupCallParticipantsOneOtherDevice); + reporter.build(summary); expect(emitter.emitSummaryStatsReport).toHaveBeenCalledWith({ percentageReceivedMedia: 1, percentageReceivedAudioMedia: 1, @@ -467,9 +447,6 @@ describe("SummaryStatsReportGatherer", () => { maxPacketLoss: 0, peerConnections: 1, percentageConcealedAudio: 0, - missingPeerConnections: 0, - percentageEstablishedPeerConnections: 1, - roomStateExpectedPeerConnections: 1, }); }); @@ -498,7 +475,7 @@ describe("SummaryStatsReportGatherer", () => { }, }, ]; - reporter.build(summary, groupCallParticipantsOneOtherDevice); + reporter.build(summary); expect(emitter.emitSummaryStatsReport).toHaveBeenCalledWith({ percentageReceivedMedia: 1, percentageReceivedAudioMedia: 1, @@ -507,9 +484,6 @@ describe("SummaryStatsReportGatherer", () => { maxPacketLoss: 0, peerConnections: 1, percentageConcealedAudio: 0, - missingPeerConnections: 0, - percentageEstablishedPeerConnections: 1, - roomStateExpectedPeerConnections: 1, }); }); @@ -604,7 +578,7 @@ describe("SummaryStatsReportGatherer", () => { }, }, ]; - reporter.build(summary, groupCallParticipantsFourOtherDevices); + reporter.build(summary); expect(emitter.emitSummaryStatsReport).toHaveBeenCalledWith({ percentageReceivedMedia: 1, percentageReceivedAudioMedia: 1, @@ -613,9 +587,6 @@ describe("SummaryStatsReportGatherer", () => { maxPacketLoss: 40, peerConnections: 4, percentageConcealedAudio: 0, - missingPeerConnections: 0, - percentageEstablishedPeerConnections: 1, - roomStateExpectedPeerConnections: 4, }); }); it("should report missing peer connections", async () => { @@ -665,7 +636,7 @@ describe("SummaryStatsReportGatherer", () => { }, }, ]; - reporter.build(summary, groupCallParticipantsFourOtherDevices); + reporter.build(summary); expect(emitter.emitSummaryStatsReport).toHaveBeenCalledWith({ percentageReceivedMedia: 1, percentageReceivedAudioMedia: 1, @@ -674,9 +645,58 @@ describe("SummaryStatsReportGatherer", () => { maxPacketLoss: 40, peerConnections: 2, percentageConcealedAudio: 0, - missingPeerConnections: 2, - percentageEstablishedPeerConnections: 0.5, - roomStateExpectedPeerConnections: 4, + }); + }); + }); + describe("extend Summary Stats Report", () => { + it("should extend the report with the appropriate data based on a user map", async () => { + const summary = { + percentageReceivedMedia: 1, + percentageReceivedAudioMedia: 1, + percentageReceivedVideoMedia: 1, + maxJitter: 2, + maxPacketLoss: 40, + peerConnections: 4, + percentageConcealedAudio: 0, + }; + SummaryStatsReportGatherer.extendSummaryReport(summary, groupCallParticipantsFourOtherDevices); + expect(summary).toStrictEqual({ + percentageReceivedMedia: 1, + percentageReceivedAudioMedia: 1, + percentageReceivedVideoMedia: 1, + maxJitter: 2, + maxPacketLoss: 40, + peerConnections: 4, + percentageConcealedAudio: 0, + oppUsersInCall: 1, + oppDevicesInCall: 4, + diffDevicesToPeerConnections: 0, + ratioPeerConnectionToDevices: 1, + }); + }); + it("should extend the report data based on a user map", async () => { + const summary = { + percentageReceivedMedia: 1, + percentageReceivedAudioMedia: 1, + percentageReceivedVideoMedia: 1, + maxJitter: 2, + maxPacketLoss: 40, + peerConnections: 4, + percentageConcealedAudio: 0, + }; + SummaryStatsReportGatherer.extendSummaryReport(summary, new Map()); + expect(summary).toStrictEqual({ + percentageReceivedMedia: 1, + percentageReceivedAudioMedia: 1, + percentageReceivedVideoMedia: 1, + maxJitter: 2, + maxPacketLoss: 40, + peerConnections: 4, + percentageConcealedAudio: 0, + oppUsersInCall: 0, + oppDevicesInCall: 0, + diffDevicesToPeerConnections: -4, + ratioPeerConnectionToDevices: 0, }); }); }); diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 35c724ff109..d6f7b4f1ecf 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -26,6 +26,7 @@ import { IScreensharingOpts } from "./mediaHandler"; import { mapsEqual } from "../utils"; import { GroupCallStats } from "./stats/groupCallStats"; import { ByteSentStatsReport, ConnectionStatsReport, StatsReport, SummaryStatsReport } from "./stats/statsReport"; +import { SummaryStatsReportGatherer } from "./stats/summaryStatsReportGatherer"; export enum GroupCallIntent { Ring = "m.ring", @@ -98,6 +99,9 @@ export enum GroupCallStatsReportEvent { SummaryStats = "GroupCall.summary_stats", } +/** + * The final report-events that get consumed by client. + */ export type GroupCallStatsReportEventHandlerMap = { [GroupCallStatsReportEvent.ConnectionStats]: (report: GroupCallStatsReport) => void; [GroupCallStatsReportEvent.ByteSentStats]: (report: GroupCallStatsReport) => void; @@ -269,14 +273,18 @@ export class GroupCall extends TypedEventEmitter< } private onConnectionStats = (report: ConnectionStatsReport): void => { + // Final emit of the summary event, to be consumed by the client this.emit(GroupCallStatsReportEvent.ConnectionStats, { report }); }; private onByteSentStats = (report: ByteSentStatsReport): void => { + // Final emit of the summary event, to be consumed by the client this.emit(GroupCallStatsReportEvent.ByteSentStats, { report }); }; private onSummaryStats = (report: SummaryStatsReport): void => { + SummaryStatsReportGatherer.extendSummaryReport(report, this.participants); + // Final emit of the summary event, to be consumed by the client this.emit(GroupCallStatsReportEvent.SummaryStats, { report }); }; @@ -1597,7 +1605,6 @@ export class GroupCall extends TypedEventEmitter< if (this.state === GroupCallState.Entered) this.placeOutgoingCalls(); // Update the participants stored in the stats object - this.stats?.setParticipants(this.participants); }; private onStateChanged = (newState: GroupCallState, oldState: GroupCallState): void => { @@ -1632,7 +1639,6 @@ export class GroupCall extends TypedEventEmitter< if (this.stats === undefined) { const userID = this.client.getUserId() || "unknown"; this.stats = new GroupCallStats(this.groupCallId, userID, this.statsCollectIntervalTime); - this.stats.setParticipants(this.participants); this.stats.reports.on(StatsReport.CONNECTION_STATS, this.onConnectionStats); this.stats.reports.on(StatsReport.BYTE_SENT_STATS, this.onByteSentStats); this.stats.reports.on(StatsReport.SUMMARY_STATS, this.onSummaryStats); diff --git a/src/webrtc/stats/groupCallStats.ts b/src/webrtc/stats/groupCallStats.ts index ab86df64a85..1804e27786c 100644 --- a/src/webrtc/stats/groupCallStats.ts +++ b/src/webrtc/stats/groupCallStats.ts @@ -71,17 +71,11 @@ export class GroupCallStats { }); Promise.all(summary) - .then((s: Awaited[]) => this.summaryStatsReportGatherer.build(s, this.participants)) + .then((s: Awaited[]) => this.summaryStatsReportGatherer.build(s)) .catch((err) => alert(err)); } public setInterval(interval: number): void { this.interval = interval; } - public setParticipants(participants: Map>): void { - this.participants = participants; - } - public getParticipants(): Map> { - return this.participants; - } } diff --git a/src/webrtc/stats/statsReport.ts b/src/webrtc/stats/statsReport.ts index 3293a71fced..b5d83f2a47f 100644 --- a/src/webrtc/stats/statsReport.ts +++ b/src/webrtc/stats/statsReport.ts @@ -79,8 +79,9 @@ export interface SummaryStatsReport { maxPacketLoss: number; percentageConcealedAudio: number; peerConnections: number; - roomStateExpectedPeerConnections: number; - missingPeerConnections: number; - percentageEstablishedPeerConnections: number; + oppUsersInCall?: number; + oppDevicesInCall?: number; + diffDevicesToPeerConnections?: number; + ratioPeerConnectionToDevices?: number; // Todo: Decide if we want an index (or a timestamp) of this report in relation to the group call, to help differenciate when issues occur and ignore/track initial connection delays. } diff --git a/src/webrtc/stats/summaryStatsReportGatherer.ts b/src/webrtc/stats/summaryStatsReportGatherer.ts index e450ef6ee15..4edc14ac7e0 100644 --- a/src/webrtc/stats/summaryStatsReportGatherer.ts +++ b/src/webrtc/stats/summaryStatsReportGatherer.ts @@ -27,10 +27,7 @@ interface CallStatsReportSummaryCounter { export class SummaryStatsReportGatherer { public constructor(private emitter: StatsReportEmitter) {} - public build( - allSummary: CallStatsReportSummary[], - callParticipants: Map>, - ): void { + public build(allSummary: CallStatsReportSummary[]): void { // Filter all stats which collect the first time webrtc stats. // Because stats based on time interval and the first collection of a summery stats has no previous // webrtcStats as basement all the calculation are 0. We don't want track the 0 stats. @@ -41,15 +38,6 @@ export class SummaryStatsReportGatherer { if (summaryTotalCount === 0) { return; } - // Calculate the actual number of devices based on the participants state event - // (this is used, to compare the expected participant count from the room state with the acutal peer connections) - // const devices = callParticipants.() - const devices: ({ id: String } & ParticipantState)[] = []; - for (const userEntry of callParticipants) { - for (const device of userEntry[1]) { - devices.push({ id: device[0], ...device[1] }); - } - } const summaryCounter: CallStatsReportSummaryCounter = { receivedAudio: 0, @@ -83,13 +71,32 @@ export class SummaryStatsReportGatherer { : 0, ), peerConnections: peerConnectionsCount, - roomStateExpectedPeerConnections: devices.length - 1, - missingPeerConnections: devices.length - peerConnectionsCount - 1, - percentageEstablishedPeerConnections: peerConnectionsCount / (devices.length - 1), } as SummaryStatsReport; this.emitter.emitSummaryStatsReport(report); } + public static extendSummaryReport( + report: SummaryStatsReport, + callParticipants: Map>, + ): void { + // Calculate the actual number of devices based on the participants state event + // (this is used, to compare the expected participant count from the room state with the acutal peer connections) + // const devices = callParticipants.() + const devices: [string, ParticipantState][] = []; + const users: [RoomMember, Map][] = []; + for (const userEntry of callParticipants) { + users.push(userEntry); + for (const device of userEntry[1]) { + devices.push(device); + } + } + report.oppDevicesInCall = Math.max(0, devices.length - 1); + report.oppUsersInCall = Math.max(0, users.length - 1); + report.diffDevicesToPeerConnections = Math.max(0, devices.length - 1) - report.peerConnections; + report.ratioPeerConnectionToDevices = + Math.max(0, devices.length - 1) == 0 ? 0 : report.peerConnections / (devices.length - 1); + } + private countTrackListReceivedMedia(counter: CallStatsReportSummaryCounter, stats: CallStatsReportSummary): void { let hasReceivedAudio = false; let hasReceivedVideo = false; From 45e8ae46775e2a9456455730b0e985c3c1504459 Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 2 Jun 2023 09:48:22 +0200 Subject: [PATCH 07/52] remove unused Signed-off-by: Timo K --- spec/unit/webrtc/stats/groupCallStats.spec.ts | 1 - src/webrtc/stats/groupCallStats.ts | 3 --- 2 files changed, 4 deletions(-) diff --git a/spec/unit/webrtc/stats/groupCallStats.spec.ts b/spec/unit/webrtc/stats/groupCallStats.spec.ts index 5ee060ca8fe..6aa45f307f9 100644 --- a/spec/unit/webrtc/stats/groupCallStats.spec.ts +++ b/spec/unit/webrtc/stats/groupCallStats.spec.ts @@ -15,7 +15,6 @@ limitations under the License. */ import { GroupCallStats } from "../../../../src/webrtc/stats/groupCallStats"; import { CallStatsReportSummary } from "../../../../src/webrtc/stats/callStatsReportSummary"; -import { groupCallParticipantsOneOtherDevice } from "../../../test-utils/webrtc"; const GROUP_CALL_ID = "GROUP_ID"; const LOCAL_USER_ID = "LOCAL_USER_ID"; diff --git a/src/webrtc/stats/groupCallStats.ts b/src/webrtc/stats/groupCallStats.ts index 1804e27786c..ee9104d87cf 100644 --- a/src/webrtc/stats/groupCallStats.ts +++ b/src/webrtc/stats/groupCallStats.ts @@ -17,15 +17,12 @@ import { CallStatsReportGatherer } from "./callStatsReportGatherer"; import { StatsReportEmitter } from "./statsReportEmitter"; import { CallStatsReportSummary } from "./callStatsReportSummary"; import { SummaryStatsReportGatherer } from "./summaryStatsReportGatherer"; -import { ParticipantState } from "../groupCall"; -import { RoomMember } from "../../matrix"; export class GroupCallStats { private timer: undefined | ReturnType; private readonly gatherers: Map = new Map(); public readonly reports = new StatsReportEmitter(); private readonly summaryStatsReportGatherer = new SummaryStatsReportGatherer(this.reports); - private participants: Map> = new Map(); public constructor(private groupCallId: string, private userId: string, private interval: number = 10000) {} From 1b423134546a177a39fbf514a43457dc939540cb Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 2 Jun 2023 11:47:50 +0200 Subject: [PATCH 08/52] rename summaryStatsReportGatherer to "Reporter" for the summary stats there is only one instance because there is only one summary. Since we dont have a list of gatherers it this class only reports. Hence we rename it to be a reporter. Signed-off-by: Timo K --- .../webrtc/stats/summaryStatsReportGatherer.spec.ts | 10 +++++----- src/webrtc/groupCall.ts | 4 ++-- src/webrtc/stats/groupCallStats.ts | 4 ++-- ...yStatsReportGatherer.ts => summaryStatsReporter.ts} | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) rename src/webrtc/stats/{summaryStatsReportGatherer.ts => summaryStatsReporter.ts} (99%) diff --git a/spec/unit/webrtc/stats/summaryStatsReportGatherer.spec.ts b/spec/unit/webrtc/stats/summaryStatsReportGatherer.spec.ts index 871fb2f883e..fa368f22c8a 100644 --- a/spec/unit/webrtc/stats/summaryStatsReportGatherer.spec.ts +++ b/spec/unit/webrtc/stats/summaryStatsReportGatherer.spec.ts @@ -13,17 +13,17 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -import { SummaryStatsReportGatherer } from "../../../../src/webrtc/stats/summaryStatsReportGatherer"; +import { SummaryStatsReporter } from "../../../../src/webrtc/stats/summaryStatsReporter"; import { StatsReportEmitter } from "../../../../src/webrtc/stats/statsReportEmitter"; import { groupCallParticipantsFourOtherDevices } from "../../../test-utils/webrtc"; describe("SummaryStatsReportGatherer", () => { - let reporter: SummaryStatsReportGatherer; + let reporter: SummaryStatsReporter; let emitter: StatsReportEmitter; beforeEach(() => { emitter = new StatsReportEmitter(); emitter.emitSummaryStatsReport = jest.fn(); - reporter = new SummaryStatsReportGatherer(emitter); + reporter = new SummaryStatsReporter(emitter); }); describe("build Summary Stats Report", () => { @@ -659,7 +659,7 @@ describe("SummaryStatsReportGatherer", () => { peerConnections: 4, percentageConcealedAudio: 0, }; - SummaryStatsReportGatherer.extendSummaryReport(summary, groupCallParticipantsFourOtherDevices); + SummaryStatsReporter.extendSummaryReport(summary, groupCallParticipantsFourOtherDevices); expect(summary).toStrictEqual({ percentageReceivedMedia: 1, percentageReceivedAudioMedia: 1, @@ -684,7 +684,7 @@ describe("SummaryStatsReportGatherer", () => { peerConnections: 4, percentageConcealedAudio: 0, }; - SummaryStatsReportGatherer.extendSummaryReport(summary, new Map()); + SummaryStatsReporter.extendSummaryReport(summary, new Map()); expect(summary).toStrictEqual({ percentageReceivedMedia: 1, percentageReceivedAudioMedia: 1, diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index d6f7b4f1ecf..20ed9fe200c 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -26,7 +26,7 @@ import { IScreensharingOpts } from "./mediaHandler"; import { mapsEqual } from "../utils"; import { GroupCallStats } from "./stats/groupCallStats"; import { ByteSentStatsReport, ConnectionStatsReport, StatsReport, SummaryStatsReport } from "./stats/statsReport"; -import { SummaryStatsReportGatherer } from "./stats/summaryStatsReportGatherer"; +import { SummaryStatsReporter } from "./stats/summaryStatsReporter"; export enum GroupCallIntent { Ring = "m.ring", @@ -283,7 +283,7 @@ export class GroupCall extends TypedEventEmitter< }; private onSummaryStats = (report: SummaryStatsReport): void => { - SummaryStatsReportGatherer.extendSummaryReport(report, this.participants); + SummaryStatsReporter.extendSummaryReport(report, this.participants); // Final emit of the summary event, to be consumed by the client this.emit(GroupCallStatsReportEvent.SummaryStats, { report }); }; diff --git a/src/webrtc/stats/groupCallStats.ts b/src/webrtc/stats/groupCallStats.ts index 3e146efdf00..17182ca1824 100644 --- a/src/webrtc/stats/groupCallStats.ts +++ b/src/webrtc/stats/groupCallStats.ts @@ -16,13 +16,13 @@ limitations under the License. import { CallStatsReportGatherer } from "./callStatsReportGatherer"; import { StatsReportEmitter } from "./statsReportEmitter"; import { CallStatsReportSummary } from "./callStatsReportSummary"; -import { SummaryStatsReportGatherer } from "./summaryStatsReportGatherer"; +import { SummaryStatsReporter } from "./summaryStatsReporter"; export class GroupCallStats { private timer: undefined | ReturnType; private readonly gatherers: Map = new Map(); public readonly reports = new StatsReportEmitter(); - private readonly summaryStatsReportGatherer = new SummaryStatsReportGatherer(this.reports); + private readonly summaryStatsReportGatherer = new SummaryStatsReporter(this.reports); public constructor(private groupCallId: string, private userId: string, private interval: number = 10000) {} diff --git a/src/webrtc/stats/summaryStatsReportGatherer.ts b/src/webrtc/stats/summaryStatsReporter.ts similarity index 99% rename from src/webrtc/stats/summaryStatsReportGatherer.ts rename to src/webrtc/stats/summaryStatsReporter.ts index 4edc14ac7e0..c44ac6e0f0d 100644 --- a/src/webrtc/stats/summaryStatsReportGatherer.ts +++ b/src/webrtc/stats/summaryStatsReporter.ts @@ -24,7 +24,7 @@ interface CallStatsReportSummaryCounter { totalAudio: number; } -export class SummaryStatsReportGatherer { +export class SummaryStatsReporter { public constructor(private emitter: StatsReportEmitter) {} public build(allSummary: CallStatsReportSummary[]): void { From 1f99cd00c6ed1fbe2197aaddf11c6a484de36da3 Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 5 Jun 2023 18:54:11 +0200 Subject: [PATCH 09/52] review Signed-off-by: Timo K --- spec/unit/webrtc/stats/summaryStatsReportGatherer.spec.ts | 8 ++++---- src/webrtc/stats/groupCallStats.ts | 5 ++++- src/webrtc/stats/statsReport.ts | 4 ++-- src/webrtc/stats/summaryStatsReporter.ts | 4 ++-- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/spec/unit/webrtc/stats/summaryStatsReportGatherer.spec.ts b/spec/unit/webrtc/stats/summaryStatsReportGatherer.spec.ts index fa368f22c8a..d4a98392f77 100644 --- a/spec/unit/webrtc/stats/summaryStatsReportGatherer.spec.ts +++ b/spec/unit/webrtc/stats/summaryStatsReportGatherer.spec.ts @@ -668,8 +668,8 @@ describe("SummaryStatsReportGatherer", () => { maxPacketLoss: 40, peerConnections: 4, percentageConcealedAudio: 0, - oppUsersInCall: 1, - oppDevicesInCall: 4, + opponentUsersInCall: 1, + opponentDevicesInCall: 4, diffDevicesToPeerConnections: 0, ratioPeerConnectionToDevices: 1, }); @@ -693,8 +693,8 @@ describe("SummaryStatsReportGatherer", () => { maxPacketLoss: 40, peerConnections: 4, percentageConcealedAudio: 0, - oppUsersInCall: 0, - oppDevicesInCall: 0, + opponentUsersInCall: 0, + opponentDevicesInCall: 0, diffDevicesToPeerConnections: -4, ratioPeerConnectionToDevices: 0, }); diff --git a/src/webrtc/stats/groupCallStats.ts b/src/webrtc/stats/groupCallStats.ts index 17182ca1824..a6adc1aa9f7 100644 --- a/src/webrtc/stats/groupCallStats.ts +++ b/src/webrtc/stats/groupCallStats.ts @@ -17,6 +17,7 @@ import { CallStatsReportGatherer } from "./callStatsReportGatherer"; import { StatsReportEmitter } from "./statsReportEmitter"; import { CallStatsReportSummary } from "./callStatsReportSummary"; import { SummaryStatsReporter } from "./summaryStatsReporter"; +import { logger } from "../../logger"; export class GroupCallStats { private timer: undefined | ReturnType; @@ -77,7 +78,9 @@ export class GroupCallStats { Promise.all(summary) .then((s: Awaited[]) => this.summaryStatsReportGatherer.build(s)) - .catch((err) => alert(err)); + .catch((err) => { + logger.error("Cloud not build syummary stats report", err); + }); } public setInterval(interval: number): void { diff --git a/src/webrtc/stats/statsReport.ts b/src/webrtc/stats/statsReport.ts index c28f0f421ee..737e8db11f5 100644 --- a/src/webrtc/stats/statsReport.ts +++ b/src/webrtc/stats/statsReport.ts @@ -83,8 +83,8 @@ export interface SummaryStatsReport { maxPacketLoss: number; percentageConcealedAudio: number; peerConnections: number; - oppUsersInCall?: number; - oppDevicesInCall?: number; + opponentUsersInCall?: number; + opponentDevicesInCall?: number; diffDevicesToPeerConnections?: number; ratioPeerConnectionToDevices?: number; // Todo: Decide if we want an index (or a timestamp) of this report in relation to the group call, to help differenciate when issues occur and ignore/track initial connection delays. diff --git a/src/webrtc/stats/summaryStatsReporter.ts b/src/webrtc/stats/summaryStatsReporter.ts index c44ac6e0f0d..dae0abe77df 100644 --- a/src/webrtc/stats/summaryStatsReporter.ts +++ b/src/webrtc/stats/summaryStatsReporter.ts @@ -90,8 +90,8 @@ export class SummaryStatsReporter { devices.push(device); } } - report.oppDevicesInCall = Math.max(0, devices.length - 1); - report.oppUsersInCall = Math.max(0, users.length - 1); + report.opponentDevicesInCall = Math.max(0, devices.length - 1); + report.opponentUsersInCall = Math.max(0, users.length - 1); report.diffDevicesToPeerConnections = Math.max(0, devices.length - 1) - report.peerConnections; report.ratioPeerConnectionToDevices = Math.max(0, devices.length - 1) == 0 ? 0 : report.peerConnections / (devices.length - 1); From c4e3b90582810b79266ac889be75dc62d660a16a Mon Sep 17 00:00:00 2001 From: Timo <16718859+toger5@users.noreply.github.com> Date: Mon, 5 Jun 2023 20:35:31 +0200 Subject: [PATCH 10/52] Update src/webrtc/stats/groupCallStats.ts Co-authored-by: Robin --- src/webrtc/stats/groupCallStats.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/stats/groupCallStats.ts b/src/webrtc/stats/groupCallStats.ts index a6adc1aa9f7..7cdf395441f 100644 --- a/src/webrtc/stats/groupCallStats.ts +++ b/src/webrtc/stats/groupCallStats.ts @@ -79,7 +79,7 @@ export class GroupCallStats { Promise.all(summary) .then((s: Awaited[]) => this.summaryStatsReportGatherer.build(s)) .catch((err) => { - logger.error("Cloud not build syummary stats report", err); + logger.error("Could not build summary stats report", err); }); } From 39465e3fba3d3fa794de0f4adbeebf5024b0fa74 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 7 Jun 2023 11:26:46 +0200 Subject: [PATCH 11/52] revert rename Signed-off-by: Timo K --- .../webrtc/stats/summaryStatsReportGatherer.spec.ts | 10 +++++----- src/webrtc/groupCall.ts | 4 ++-- ...yStatsReporter.ts => SummaryStatsReportGatherer.ts} | 2 +- src/webrtc/stats/groupCallStats.ts | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) rename src/webrtc/stats/{summaryStatsReporter.ts => SummaryStatsReportGatherer.ts} (99%) diff --git a/spec/unit/webrtc/stats/summaryStatsReportGatherer.spec.ts b/spec/unit/webrtc/stats/summaryStatsReportGatherer.spec.ts index d4a98392f77..2cf2460a3b9 100644 --- a/spec/unit/webrtc/stats/summaryStatsReportGatherer.spec.ts +++ b/spec/unit/webrtc/stats/summaryStatsReportGatherer.spec.ts @@ -13,17 +13,17 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -import { SummaryStatsReporter } from "../../../../src/webrtc/stats/summaryStatsReporter"; +import { SummaryStatsReportGatherer } from "../../../../src/webrtc/stats/SummaryStatsReportGatherer"; import { StatsReportEmitter } from "../../../../src/webrtc/stats/statsReportEmitter"; import { groupCallParticipantsFourOtherDevices } from "../../../test-utils/webrtc"; describe("SummaryStatsReportGatherer", () => { - let reporter: SummaryStatsReporter; + let reporter: SummaryStatsReportGatherer; let emitter: StatsReportEmitter; beforeEach(() => { emitter = new StatsReportEmitter(); emitter.emitSummaryStatsReport = jest.fn(); - reporter = new SummaryStatsReporter(emitter); + reporter = new SummaryStatsReportGatherer(emitter); }); describe("build Summary Stats Report", () => { @@ -659,7 +659,7 @@ describe("SummaryStatsReportGatherer", () => { peerConnections: 4, percentageConcealedAudio: 0, }; - SummaryStatsReporter.extendSummaryReport(summary, groupCallParticipantsFourOtherDevices); + SummaryStatsReportGatherer.extendSummaryReport(summary, groupCallParticipantsFourOtherDevices); expect(summary).toStrictEqual({ percentageReceivedMedia: 1, percentageReceivedAudioMedia: 1, @@ -684,7 +684,7 @@ describe("SummaryStatsReportGatherer", () => { peerConnections: 4, percentageConcealedAudio: 0, }; - SummaryStatsReporter.extendSummaryReport(summary, new Map()); + SummaryStatsReportGatherer.extendSummaryReport(summary, new Map()); expect(summary).toStrictEqual({ percentageReceivedMedia: 1, percentageReceivedAudioMedia: 1, diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 20ed9fe200c..eb776d5b93f 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -26,7 +26,7 @@ import { IScreensharingOpts } from "./mediaHandler"; import { mapsEqual } from "../utils"; import { GroupCallStats } from "./stats/groupCallStats"; import { ByteSentStatsReport, ConnectionStatsReport, StatsReport, SummaryStatsReport } from "./stats/statsReport"; -import { SummaryStatsReporter } from "./stats/summaryStatsReporter"; +import { SummaryStatsReportGatherer } from "./stats/SummaryStatsReportGatherer"; export enum GroupCallIntent { Ring = "m.ring", @@ -283,7 +283,7 @@ export class GroupCall extends TypedEventEmitter< }; private onSummaryStats = (report: SummaryStatsReport): void => { - SummaryStatsReporter.extendSummaryReport(report, this.participants); + SummaryStatsReportGatherer.extendSummaryReport(report, this.participants); // Final emit of the summary event, to be consumed by the client this.emit(GroupCallStatsReportEvent.SummaryStats, { report }); }; diff --git a/src/webrtc/stats/summaryStatsReporter.ts b/src/webrtc/stats/SummaryStatsReportGatherer.ts similarity index 99% rename from src/webrtc/stats/summaryStatsReporter.ts rename to src/webrtc/stats/SummaryStatsReportGatherer.ts index dae0abe77df..b0227670d98 100644 --- a/src/webrtc/stats/summaryStatsReporter.ts +++ b/src/webrtc/stats/SummaryStatsReportGatherer.ts @@ -24,7 +24,7 @@ interface CallStatsReportSummaryCounter { totalAudio: number; } -export class SummaryStatsReporter { +export class SummaryStatsReportGatherer { public constructor(private emitter: StatsReportEmitter) {} public build(allSummary: CallStatsReportSummary[]): void { diff --git a/src/webrtc/stats/groupCallStats.ts b/src/webrtc/stats/groupCallStats.ts index 7cdf395441f..1b9e37bf178 100644 --- a/src/webrtc/stats/groupCallStats.ts +++ b/src/webrtc/stats/groupCallStats.ts @@ -16,14 +16,14 @@ limitations under the License. import { CallStatsReportGatherer } from "./callStatsReportGatherer"; import { StatsReportEmitter } from "./statsReportEmitter"; import { CallStatsReportSummary } from "./callStatsReportSummary"; -import { SummaryStatsReporter } from "./summaryStatsReporter"; +import { SummaryStatsReportGatherer } from "./SummaryStatsReportGatherer"; import { logger } from "../../logger"; export class GroupCallStats { private timer: undefined | ReturnType; private readonly gatherers: Map = new Map(); public readonly reports = new StatsReportEmitter(); - private readonly summaryStatsReportGatherer = new SummaryStatsReporter(this.reports); + private readonly summaryStatsReportGatherer = new SummaryStatsReportGatherer(this.reports); public constructor(private groupCallId: string, private userId: string, private interval: number = 10000) {} From 0fd11acf615f1d3384104d3f83d41db01040763b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 2 Jun 2023 13:33:05 +0000 Subject: [PATCH 12/52] Update all non-major dependencies (#3433) * Update all non-major dependencies * Remove name wrap-ansi-cjs * Remove name string-width-cjs --------- Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Michael Telatynski <7t3chguy@gmail.com> --- package.json | 2 +- yarn.lock | 106 +++++++++++++++++++++++++++------------------------ 2 files changed, 57 insertions(+), 51 deletions(-) diff --git a/package.json b/package.json index a8128f9285d..ea5282525b4 100644 --- a/package.json +++ b/package.json @@ -101,7 +101,7 @@ "debug": "^4.3.4", "docdash": "^2.0.0", "domexception": "^4.0.0", - "eslint": "8.40.0", + "eslint": "8.41.0", "eslint-config-google": "^0.14.0", "eslint-config-prettier": "^8.5.0", "eslint-import-resolver-typescript": "^3.5.1", diff --git a/yarn.lock b/yarn.lock index 71b5d3eda4f..11717bdc1db 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1091,10 +1091,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@eslint/js@8.40.0": - version "8.40.0" - resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.40.0.tgz#3ba73359e11f5a7bd3e407f70b3528abfae69cec" - integrity sha512-ElyB54bJIhXQYVKjDSvCkPO1iU1tSAeVQJbllWJq1XQSmmA4dgFk8CbiBGpiOPxleE48vDogxCtmMYku4HSVLA== +"@eslint/js@8.41.0": + version "8.41.0" + resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.41.0.tgz#080321c3b68253522f7646b55b577dd99d2950b3" + integrity sha512-LxcyMGxwmTh2lY9FwHPGWOHmYFCZvbrFCBZL4FzSSsxsRPuhrYUg/49/0KDfW8tnIEaEHtfmn6+NPN+1DqaNmA== "@humanwhocodes/config-array@^0.11.8": version "0.11.8" @@ -1385,7 +1385,7 @@ resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.1.2.tgz#7c6cf998d6d20b914c0a55a91ae928ff25965e72" integrity sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw== -"@jridgewell/source-map@^0.3.2": +"@jridgewell/source-map@^0.3.3": version "0.3.3" resolved "https://registry.yarnpkg.com/@jridgewell/source-map/-/source-map-0.3.3.tgz#8108265659d4c33e72ffe14e33d6cc5eb59f2fda" integrity sha512-b+fsZXeLYi9fEULmfBrhxn4IrPlINf8fiNarzTof004v3lFdntdwa9PF7vFJqm3mg7s+ScJMxXaE3Acp1irZcg== @@ -2056,7 +2056,7 @@ acorn@^7.0.0: resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A== -acorn@^8.1.0, acorn@^8.4.1, acorn@^8.5.0, acorn@^8.8.0, acorn@^8.8.1: +acorn@^8.1.0, acorn@^8.4.1, acorn@^8.8.0, acorn@^8.8.1, acorn@^8.8.2: version "8.8.2" resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.2.tgz#1b2f25db02af965399b9776b0c2c391276d37c4a" integrity sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw== @@ -3704,15 +3704,15 @@ eslint-visitor-keys@^3.3.0, eslint-visitor-keys@^3.4.1: resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.4.1.tgz#c22c48f48942d08ca824cc526211ae400478a994" integrity sha512-pZnmmLwYzf+kWaM/Qgrvpen51upAktaaiI01nsJD/Yr3lMOdNtq0cxkrrg16w64VtisN6okbs7Q8AfGqj4c9fA== -eslint@8.40.0: - version "8.40.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.40.0.tgz#a564cd0099f38542c4e9a2f630fa45bf33bc42a4" - integrity sha512-bvR+TsP9EHL3TqNtj9sCNJVAFK3fBN8Q7g5waghxyRsPLIMwL73XSKnZFK0hk/O2ANC+iAoq6PWMQ+IfBAJIiQ== +eslint@8.41.0: + version "8.41.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.41.0.tgz#3062ca73363b4714b16dbc1e60f035e6134b6f1c" + integrity sha512-WQDQpzGBOP5IrXPo4Hc0814r4/v2rrIsB0rhT7jtunIalgg6gYXWhRMOejVO8yH21T/FGaxjmFjBMNqcIlmH1Q== dependencies: "@eslint-community/eslint-utils" "^4.2.0" "@eslint-community/regexpp" "^4.4.0" "@eslint/eslintrc" "^2.0.3" - "@eslint/js" "8.40.0" + "@eslint/js" "8.41.0" "@humanwhocodes/config-array" "^0.11.8" "@humanwhocodes/module-importer" "^1.0.1" "@nodelib/fs.walk" "^1.2.8" @@ -3732,13 +3732,12 @@ eslint@8.40.0: find-up "^5.0.0" glob-parent "^6.0.2" globals "^13.19.0" - grapheme-splitter "^1.0.4" + graphemer "^1.4.0" ignore "^5.2.0" import-fresh "^3.0.0" imurmurhash "^0.1.4" is-glob "^4.0.0" is-path-inside "^3.0.3" - js-sdsl "^4.1.4" js-yaml "^4.1.0" json-stable-stringify-without-jsonify "^1.0.1" levn "^0.4.1" @@ -4179,15 +4178,15 @@ glob-to-regexp@^0.4.0: resolved "https://registry.yarnpkg.com/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz#c75297087c851b9a578bd217dd59a92f59fe546e" integrity sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw== -glob@^10.0.0: - version "10.2.4" - resolved "https://registry.yarnpkg.com/glob/-/glob-10.2.4.tgz#f5bf7ddb080e3e9039b148a9e2aef3d5ebfc0a25" - integrity sha512-fDboBse/sl1oXSLhIp0FcCJgzW9KmhC/q8ULTKC82zc+DL3TL7FNb8qlt5qqXN53MsKEUSIcb+7DLmEygOE5Yw== +glob@^10.2.5: + version "10.2.6" + resolved "https://registry.yarnpkg.com/glob/-/glob-10.2.6.tgz#1e27edbb3bbac055cb97113e27a066c100a4e5e1" + integrity sha512-U/rnDpXJGF414QQQZv5uVsabTVxMSwzS5CH0p3DRCIV6ownl4f7PzGnkGmvlum2wB+9RlJWJZ6ACU1INnBqiPA== dependencies: foreground-child "^3.1.0" jackspeak "^2.0.3" - minimatch "^9.0.0" - minipass "^5.0.0 || ^6.0.0" + minimatch "^9.0.1" + minipass "^5.0.0 || ^6.0.2" path-scurry "^1.7.0" glob@^7.1.0, glob@^7.1.3, glob@^7.1.4, glob@^7.2.0: @@ -4269,6 +4268,11 @@ grapheme-splitter@^1.0.4: resolved "https://registry.yarnpkg.com/grapheme-splitter/-/grapheme-splitter-1.0.4.tgz#9cf3a665c6247479896834af35cf1dbb4400767e" integrity sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ== +graphemer@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/graphemer/-/graphemer-1.4.0.tgz#fb2f1d55e0e3a1849aeffc90c4fa0dd53a0e66c6" + integrity sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag== + has-bigints@^1.0.1, has-bigints@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/has-bigints/-/has-bigints-1.0.2.tgz#0871bd3e3d51626f6ca0966668ba35d5602d6eaa" @@ -4808,9 +4812,9 @@ istanbul-reports@^3.1.3, istanbul-reports@^3.1.4: istanbul-lib-report "^3.0.0" jackspeak@^2.0.3: - version "2.2.0" - resolved "https://registry.yarnpkg.com/jackspeak/-/jackspeak-2.2.0.tgz#497cbaedc902ec3f31d5d61be804d2364ff9ddad" - integrity sha512-r5XBrqIJfwRIjRt/Xr5fv9Wh09qyhHfKnYddDlpM+ibRR20qrYActpCAgU6U+d53EOEjzkvxPMVHSlgR7leXrQ== + version "2.2.1" + resolved "https://registry.yarnpkg.com/jackspeak/-/jackspeak-2.2.1.tgz#655e8cf025d872c9c03d3eb63e8f0c024fef16a6" + integrity sha512-MXbxovZ/Pm42f6cDIDkl3xpwv1AGwObKwfmjs2nQePiy85tP3fatofl3FC1aBsOtP/6fq5SbtgHwWcMsLP+bDw== dependencies: "@isaacs/cliui" "^8.0.2" optionalDependencies: @@ -5253,11 +5257,6 @@ jju@~1.4.0: resolved "https://registry.yarnpkg.com/jju/-/jju-1.4.0.tgz#a3abe2718af241a2b2904f84a625970f389ae32a" integrity sha512-8wb9Yw966OSxApiCt0K3yNJL8pnNeIv+OEq2YMidz4FKP6nonSRoOXc80iXY4JaN2FC11B9qsNmDsm+ZOfMROA== -js-sdsl@^4.1.4: - version "4.4.0" - resolved "https://registry.yarnpkg.com/js-sdsl/-/js-sdsl-4.4.0.tgz#8b437dbe642daa95760400b602378ed8ffea8430" - integrity sha512-FfVSdx6pJ41Oa+CF7RDaFmTnCaFhua+SNYQX74riGOpl96x+2jQCqEfQ2bnXu/5DPCqlRuiqyvTJM0Qjz26IVg== - js-stringify@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/js-stringify/-/js-stringify-1.0.2.tgz#1736fddfd9724f28a3682adc6230ae7e4e9679db" @@ -5547,9 +5546,9 @@ lru-cache@^6.0.0: yallist "^4.0.0" lru-cache@^9.1.1: - version "9.1.1" - resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-9.1.1.tgz#c58a93de58630b688de39ad04ef02ef26f1902f1" - integrity sha512-65/Jky17UwSb0BuB9V+MyDpsOtXKmYwzhyl+cOa9XUiI4uV2Ouy/2voFP3+al0BjZbJgMBD8FojMpAf+Z+qn4A== + version "9.1.2" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-9.1.2.tgz#255fdbc14b75589d6d0e73644ca167a8db506835" + integrity sha512-ERJq3FOzJTxBbFjZ7iDs+NiK4VI9Wz+RdrrAB8dio1oV+YvdPzUEE4QNiT2VD51DkIbCYRUUzCRkssXCHqSnKQ== lru-queue@^0.1.0: version "0.1.0" @@ -5723,6 +5722,13 @@ minimatch@^9.0.0: dependencies: brace-expansion "^2.0.1" +minimatch@^9.0.1: + version "9.0.1" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.1.tgz#8a555f541cf976c622daf078bb28f29fb927c253" + integrity sha512-0jWhJpD/MdhPXwPuiRkCbfYfSKp2qnn2eOc279qI7f+osl/l+prKSrvhg157zSYvx/1nmgn2NqdT6k2Z7zSH9w== + dependencies: + brace-expansion "^2.0.1" + minimist@0.0.5: version "0.0.5" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.5.tgz#d7aa327bcecf518f9106ac6b8f003fa3bcea8566" @@ -5733,10 +5739,10 @@ minimist@^1.1.0, minimist@^1.2.0, minimist@^1.2.5, minimist@^1.2.6: resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== -"minipass@^5.0.0 || ^6.0.0": - version "6.0.1" - resolved "https://registry.yarnpkg.com/minipass/-/minipass-6.0.1.tgz#315417c259cb32a1b2fc530c0e7f55c901a60a6d" - integrity sha512-Tenl5QPpgozlOGBiveNYHg2f6y+VpxsXRoIHFUVJuSmTonXRAE6q9b8Mp/O46762/2AlW4ye4Nkyvx0fgWDKbw== +"minipass@^5.0.0 || ^6.0.2": + version "6.0.2" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-6.0.2.tgz#542844b6c4ce95b202c0995b0a471f1229de4c81" + integrity sha512-MzWSV5nYVT7mVyWCwn2o7JH13w2TBRmmSqSRCKzTw+lmft9X4z+3wjvs06Tzijo5z4W/kahUCDpRXTF+ZrmF/w== mkdirp-classic@^0.5.2: version "0.5.3" @@ -6107,12 +6113,12 @@ path-platform@~0.11.15: integrity sha512-Y30dB6rab1A/nfEKsZxmr01nUotHX0c/ZiIAsCTatEe1CmS5Pm5He7fZ195bPT7RdquoaL8lLxFCMQi/bS7IJg== path-scurry@^1.7.0: - version "1.9.1" - resolved "https://registry.yarnpkg.com/path-scurry/-/path-scurry-1.9.1.tgz#838566bb22e38feaf80ecd49ae06cd12acd782ee" - integrity sha512-UgmoiySyjFxP6tscZDgWGEAgsW5ok8W3F5CJDnnH2pozwSTGE6eH7vwTotMwATWA2r5xqdkKdxYPkwlJjAI/3g== + version "1.9.2" + resolved "https://registry.yarnpkg.com/path-scurry/-/path-scurry-1.9.2.tgz#90f9d296ac5e37e608028e28a447b11d385b3f63" + integrity sha512-qSDLy2aGFPm8i4rsbHd4MNyTcrzHFsLQykrtbuGRknZZCBBVXSv2tSCDN2Cg6Rt/GFRw8GoW9y9Ecw5rIPG1sg== dependencies: lru-cache "^9.1.1" - minipass "^5.0.0 || ^6.0.0" + minipass "^5.0.0 || ^6.0.2" path-to-regexp@^2.2.1: version "2.4.0" @@ -6713,11 +6719,11 @@ rimraf@^3.0.2: glob "^7.1.3" rimraf@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-5.0.0.tgz#5bda14e410d7e4dd522154891395802ce032c2cb" - integrity sha512-Jf9llaP+RvaEVS5nPShYFhtXIrb3LRKP281ib3So0KkeZKo2wIKyq0Re7TOSwanasA423PSr6CCIL4bP6T040g== + version "5.0.1" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-5.0.1.tgz#0881323ab94ad45fec7c0221f27ea1a142f3f0d0" + integrity sha512-OfFZdwtd3lZ+XZzYP/6gTACubwFcHdLRqS9UX3UwpU2dnGQYkPFISRwvM3w9IiB2w7bW5qGo/uAwE4SmXXSKvg== dependencies: - glob "^10.0.0" + glob "^10.2.5" ripemd160@^2.0.0, ripemd160@^2.0.1: version "2.0.2" @@ -7082,9 +7088,9 @@ string_decoder@~1.1.1: ansi-regex "^5.0.1" strip-ansi@^7.0.1: - version "7.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2" - integrity sha512-cXNxvT8dFNRVfhVME3JAe98mkXDYN2O1l7jmcwMnOslDeESg1rF/OZMtK0nRAhiari1unG5cD4jG3rapUAkLbw== + version "7.1.0" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45" + integrity sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ== dependencies: ansi-regex "^6.0.1" @@ -7191,12 +7197,12 @@ tapable@^2.2.0: integrity sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ== terser@^5.5.1: - version "5.17.4" - resolved "https://registry.yarnpkg.com/terser/-/terser-5.17.4.tgz#b0c2d94897dfeba43213ed5f90ed117270a2c696" - integrity sha512-jcEKZw6UPrgugz/0Tuk/PVyLAPfMBJf5clnGueo45wTweoV8yh7Q7PEkhkJ5uuUbC7zAxEcG3tqNr1bstkQ8nw== + version "5.17.7" + resolved "https://registry.yarnpkg.com/terser/-/terser-5.17.7.tgz#2a8b134826fe179b711969fd9d9a0c2479b2a8c3" + integrity sha512-/bi0Zm2C6VAexlGgLlVxA0P2lru/sdLyfCVaRMfKVo9nWxbmz7f/sD8VPybPeSUJaJcwmCJis9pBIhcVcG1QcQ== dependencies: - "@jridgewell/source-map" "^0.3.2" - acorn "^8.5.0" + "@jridgewell/source-map" "^0.3.3" + acorn "^8.8.2" commander "^2.20.0" source-map-support "~0.5.20" From 600f26d24d62e1271b3bb3dd2de090b9aa1fdabd Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 2 Jun 2023 13:36:53 +0000 Subject: [PATCH 13/52] Update definitelyTyped (#3430) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Michael Telatynski <7t3chguy@gmail.com> --- yarn.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/yarn.lock b/yarn.lock index 11717bdc1db..975f9e111a4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1709,9 +1709,9 @@ integrity sha512-dgMN+syt1xb7Hk8LU6AODOfPlvz5z1CbXpPuJE5ZrX9STfBOIXF09pEB8N7a97WT9dbngt3ksDCm6GW6yMrxfQ== "@types/debug@^4.1.7": - version "4.1.7" - resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.7.tgz#7cc0ea761509124709b8b2d1090d8f6c17aadb82" - integrity sha512-9AonUzyTjXXhEOa0DnqpzZi6VHlqKMswga9EXjpXnnqxwLtdvPPtlO8evrI5D9S6asFRCQ6v+wpiUKbw+vKqyg== + version "4.1.8" + resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.8.tgz#cef723a5d0a90990313faec2d1e22aee5eecb317" + integrity sha512-/vPO1EPOs306Cvhwv7KfVfYvOJqA/S/AXjaHQiJboCZzcNDb+TIJFN9/2C9DZ//ijSKWioNyUxD792QmDJ+HKQ== dependencies: "@types/ms" "*" @@ -1791,9 +1791,9 @@ integrity sha512-JJulVEQXmiY9Px5axXHeYGLSjhkZEnD+MDPDGbCbIAbMslkKwmygtZFy1X6s/075Yo94sf8GuSlFfPzysQrWZQ== "@types/node@18": - version "18.16.10" - resolved "https://registry.yarnpkg.com/@types/node/-/node-18.16.10.tgz#9b16d918f4f6fec6cae4af34283a91d555b81519" - integrity sha512-sMo3EngB6QkMBlB9rBe1lFdKSLqljyWPPWv6/FzSxh/IDlyVWSzE9RiF4eAuerQHybrWdqBgAGb03PM89qOasA== + version "18.16.16" + resolved "https://registry.yarnpkg.com/@types/node/-/node-18.16.16.tgz#3b64862856c7874ccf7439e6bab872d245c86d8e" + integrity sha512-NpaM49IGQQAUlBhHMF82QH80J08os4ZmyF9MkpCzWAGuOHqE4gTEbhzd7L3l5LmWuZ6E0OiC1FweQ4tsiW35+g== "@types/normalize-package-data@^2.4.0": version "2.4.1" @@ -1801,9 +1801,9 @@ integrity sha512-Gj7cI7z+98M282Tqmp2K5EIsoouUEzbBJhQQzDE3jSIRk6r9gsz0oUokqIUR4u1R3dMHo0pDHM7sNOHyhulypw== "@types/prettier@^2.1.5": - version "2.7.2" - resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.7.2.tgz#6c2324641cc4ba050a8c710b2b251b377581fbf0" - integrity sha512-KufADq8uQqo1pYKVIYzfKbJfBAc0sOeXqGbFaSpv8MRmC/zXgowNZmFcbngndGk922QDmOASEXUZCaY48gs4cg== + version "2.7.3" + resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.7.3.tgz#3e51a17e291d01d17d3fc61422015a933af7a08f" + integrity sha512-+68kP9yzs4LMp7VNh8gdzMSPZFL44MLGqiHWvttYJe+6qnuVr4Ek9wSBQoveqY/r+LwjCcU29kNVkidwim+kYA== "@types/retry@0.12.0": version "0.12.0" From de34d240b8cd460320dd61dfa6f154ed83021346 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 2 Jun 2023 15:29:49 +0100 Subject: [PATCH 14/52] Export FALLBACK_ICE_SERVER (#3429) --- src/webrtc/call.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 875dd26996b..32b3aec8ef9 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -255,7 +255,7 @@ export enum CallErrorCode { const VOIP_PROTO_VERSION = "1"; /** The fallback ICE server to use for STUN or TURN protocols. */ -const FALLBACK_ICE_SERVER = "stun:turn.matrix.org"; +export const FALLBACK_ICE_SERVER = "stun:turn.matrix.org"; /** The length of time a call can be ringing for. */ const CALL_TIMEOUT_MS = 60 * 1000; // ms From 6b60287b19569b006f5269fd5c8f3ac328fbbc65 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 2 Jun 2023 16:01:21 +0100 Subject: [PATCH 15/52] Add an integration test for verification (#3436) * Move existing crypto integ tests into a subdirectory * Factor out some common bits from `crypto.spec.ts` * Integration test for device verification * Ignore generated file in prettier --- .prettierignore | 3 + spec/integ/{ => crypto}/cross-signing.spec.ts | 4 +- spec/integ/{ => crypto}/crypto.spec.ts | 32 +-- spec/integ/{ => crypto}/megolm-backup.spec.ts | 12 +- .../integ/{ => crypto}/olm-encryption-spec.ts | 16 +- spec/integ/{ => crypto}/rust-crypto.spec.ts | 2 +- spec/integ/crypto/verification.spec.ts | 255 ++++++++++++++++++ spec/test-utils/mockEndpoints.ts | 30 +++ spec/test-utils/test-data/.gitignore | 1 + .../test-data/generate-test-data.py | 117 ++++++++ spec/test-utils/test-data/index.ts | 33 +++ 11 files changed, 470 insertions(+), 35 deletions(-) rename spec/integ/{ => crypto}/cross-signing.spec.ts (98%) rename spec/integ/{ => crypto}/crypto.spec.ts (98%) rename spec/integ/{ => crypto}/megolm-backup.spec.ts (94%) rename spec/integ/{ => crypto}/olm-encryption-spec.ts (98%) rename spec/integ/{ => crypto}/rust-crypto.spec.ts (98%) create mode 100644 spec/integ/crypto/verification.spec.ts create mode 100644 spec/test-utils/mockEndpoints.ts create mode 100644 spec/test-utils/test-data/.gitignore create mode 100755 spec/test-utils/test-data/generate-test-data.py create mode 100644 spec/test-utils/test-data/index.ts diff --git a/.prettierignore b/.prettierignore index 6d56d969f02..3d2a9aded15 100644 --- a/.prettierignore +++ b/.prettierignore @@ -24,3 +24,6 @@ out # This file is owned, parsed, and generated by allchange, which doesn't comply with prettier /CHANGELOG.md + +# This file is also autogenerated +/spec/test-utils/test-data/index.ts diff --git a/spec/integ/cross-signing.spec.ts b/spec/integ/crypto/cross-signing.spec.ts similarity index 98% rename from spec/integ/cross-signing.spec.ts rename to spec/integ/crypto/cross-signing.spec.ts index ba227d3f89c..305bae222b9 100644 --- a/spec/integ/cross-signing.spec.ts +++ b/spec/integ/crypto/cross-signing.spec.ts @@ -18,8 +18,8 @@ import fetchMock from "fetch-mock-jest"; import "fake-indexeddb/auto"; import { IDBFactory } from "fake-indexeddb"; -import { CRYPTO_BACKENDS, InitCrypto } from "../test-utils/test-utils"; -import { createClient, MatrixClient, UIAuthCallback } from "../../src"; +import { CRYPTO_BACKENDS, InitCrypto } from "../../test-utils/test-utils"; +import { createClient, MatrixClient, UIAuthCallback } from "../../../src"; afterEach(() => { // reset fake-indexeddb after each test, to make sure we don't leak connections diff --git a/spec/integ/crypto.spec.ts b/spec/integ/crypto/crypto.spec.ts similarity index 98% rename from spec/integ/crypto.spec.ts rename to spec/integ/crypto/crypto.spec.ts index 951e853f90e..f4d7fc49f44 100644 --- a/spec/integ/crypto.spec.ts +++ b/spec/integ/crypto/crypto.spec.ts @@ -21,11 +21,11 @@ import "fake-indexeddb/auto"; import { IDBFactory } from "fake-indexeddb"; import { MockResponse, MockResponseFunction } from "fetch-mock"; -import type { IDeviceKeys } from "../../src/@types/crypto"; -import * as testUtils from "../test-utils/test-utils"; -import { CRYPTO_BACKENDS, InitCrypto, syncPromise } from "../test-utils/test-utils"; -import { TestClient } from "../TestClient"; -import { logger } from "../../src/logger"; +import type { IDeviceKeys } from "../../../src/@types/crypto"; +import * as testUtils from "../../test-utils/test-utils"; +import { CRYPTO_BACKENDS, InitCrypto, syncPromise } from "../../test-utils/test-utils"; +import { TestClient } from "../../TestClient"; +import { logger } from "../../../src/logger"; import { createClient, IClaimOTKsResult, @@ -43,13 +43,14 @@ import { Room, RoomMember, RoomStateEvent, -} from "../../src/matrix"; -import { DeviceInfo } from "../../src/crypto/deviceinfo"; -import { E2EKeyReceiver, IE2EKeyReceiver } from "../test-utils/E2EKeyReceiver"; -import { ISyncResponder, SyncResponder } from "../test-utils/SyncResponder"; -import { escapeRegExp } from "../../src/utils"; -import { downloadDeviceToJsDevice } from "../../src/rust-crypto/device-converter"; -import { flushPromises } from "../test-utils/flushPromises"; +} from "../../../src/matrix"; +import { DeviceInfo } from "../../../src/crypto/deviceinfo"; +import { E2EKeyReceiver, IE2EKeyReceiver } from "../../test-utils/E2EKeyReceiver"; +import { ISyncResponder, SyncResponder } from "../../test-utils/SyncResponder"; +import { escapeRegExp } from "../../../src/utils"; +import { downloadDeviceToJsDevice } from "../../../src/rust-crypto/device-converter"; +import { flushPromises } from "../../test-utils/flushPromises"; +import { mockInitialApiRequests } from "../../test-utils/mockEndpoints"; const ROOM_ID = "!room:id"; @@ -419,12 +420,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("crypto (%s)", (backend: string, async function startClientAndAwaitFirstSync(opts: IStartClientOpts = {}): Promise { logger.log(aliceClient.getUserId() + ": starting"); - const homeserverUrl = aliceClient.getHomeserverUrl(); - fetchMock.get(new URL("/_matrix/client/versions", homeserverUrl).toString(), { versions: ["r0.5.0"] }); - fetchMock.get(new URL("/_matrix/client/r0/pushrules/", homeserverUrl).toString(), {}); - fetchMock.post(new URL("/_matrix/client/r0/user/%40alice%3Alocalhost/filter", homeserverUrl).toString(), { - filter_id: "fid", - }); + mockInitialApiRequests(aliceClient.getHomeserverUrl()); // we let the client do a very basic initial sync, which it needs before // it will upload one-time keys. diff --git a/spec/integ/megolm-backup.spec.ts b/spec/integ/crypto/megolm-backup.spec.ts similarity index 94% rename from spec/integ/megolm-backup.spec.ts rename to spec/integ/crypto/megolm-backup.spec.ts index adbfb70d7fa..58adcbfc38f 100644 --- a/spec/integ/megolm-backup.spec.ts +++ b/spec/integ/crypto/megolm-backup.spec.ts @@ -16,12 +16,12 @@ limitations under the License. import { Account } from "@matrix-org/olm"; -import { logger } from "../../src/logger"; -import { decodeRecoveryKey } from "../../src/crypto/recoverykey"; -import { IKeyBackupInfo, IKeyBackupSession } from "../../src/crypto/keybackup"; -import { TestClient } from "../TestClient"; -import { IEvent } from "../../src"; -import { MatrixEvent, MatrixEventEvent } from "../../src/models/event"; +import { logger } from "../../../src/logger"; +import { decodeRecoveryKey } from "../../../src/crypto/recoverykey"; +import { IKeyBackupInfo, IKeyBackupSession } from "../../../src/crypto/keybackup"; +import { TestClient } from "../../TestClient"; +import { IEvent } from "../../../src"; +import { MatrixEvent, MatrixEventEvent } from "../../../src/models/event"; const ROOM_ID = "!ROOM:ID"; diff --git a/spec/integ/olm-encryption-spec.ts b/spec/integ/crypto/olm-encryption-spec.ts similarity index 98% rename from spec/integ/olm-encryption-spec.ts rename to spec/integ/crypto/olm-encryption-spec.ts index 171cb3fa0cc..aff150adb39 100644 --- a/spec/integ/olm-encryption-spec.ts +++ b/spec/integ/crypto/olm-encryption-spec.ts @@ -26,16 +26,16 @@ limitations under the License. */ // load olm before the sdk if possible -import "../olm-loader"; +import "../../olm-loader"; import type { Session } from "@matrix-org/olm"; -import type { IDeviceKeys, IOneTimeKey } from "../../src/@types/crypto"; -import { logger } from "../../src/logger"; -import * as testUtils from "../test-utils/test-utils"; -import { TestClient } from "../TestClient"; -import { CRYPTO_ENABLED, IClaimKeysRequest, IQueryKeysRequest, IUploadKeysRequest } from "../../src/client"; -import { ClientEvent, IContent, ISendEventResponse, MatrixClient, MatrixEvent } from "../../src/matrix"; -import { DeviceInfo } from "../../src/crypto/deviceinfo"; +import type { IDeviceKeys, IOneTimeKey } from "../../../src/@types/crypto"; +import { logger } from "../../../src/logger"; +import * as testUtils from "../../test-utils/test-utils"; +import { TestClient } from "../../TestClient"; +import { CRYPTO_ENABLED, IClaimKeysRequest, IQueryKeysRequest, IUploadKeysRequest } from "../../../src/client"; +import { ClientEvent, IContent, ISendEventResponse, MatrixClient, MatrixEvent } from "../../../src/matrix"; +import { DeviceInfo } from "../../../src/crypto/deviceinfo"; let aliTestClient: TestClient; const roomId = "!room:localhost"; diff --git a/spec/integ/rust-crypto.spec.ts b/spec/integ/crypto/rust-crypto.spec.ts similarity index 98% rename from spec/integ/rust-crypto.spec.ts rename to spec/integ/crypto/rust-crypto.spec.ts index e018c210268..6fa56b3d496 100644 --- a/spec/integ/rust-crypto.spec.ts +++ b/spec/integ/crypto/rust-crypto.spec.ts @@ -17,7 +17,7 @@ limitations under the License. import "fake-indexeddb/auto"; import { IDBFactory } from "fake-indexeddb"; -import { createClient } from "../../src"; +import { createClient } from "../../../src"; afterEach(() => { // reset fake-indexeddb after each test, to make sure we don't leak connections diff --git a/spec/integ/crypto/verification.spec.ts b/spec/integ/crypto/verification.spec.ts new file mode 100644 index 00000000000..64a43d1ce1a --- /dev/null +++ b/spec/integ/crypto/verification.spec.ts @@ -0,0 +1,255 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import fetchMock from "fetch-mock-jest"; +import { MockResponse } from "fetch-mock"; + +import { createClient, MatrixClient } from "../../../src"; +import { ShowSasCallbacks, VerifierEvent } from "../../../src/crypto-api/verification"; +import { escapeRegExp } from "../../../src/utils"; +import { VerificationBase } from "../../../src/crypto/verification/Base"; +import { CRYPTO_BACKENDS, InitCrypto } from "../../test-utils/test-utils"; +import { SyncResponder } from "../../test-utils/SyncResponder"; +import { + SIGNED_TEST_DEVICE_DATA, + TEST_DEVICE_ID, + TEST_DEVICE_PUBLIC_ED25519_KEY_BASE64, + TEST_USER_ID, +} from "../../test-utils/test-data"; +import { mockInitialApiRequests } from "../../test-utils/mockEndpoints"; +import { + Phase, + VerificationRequest, + VerificationRequestEvent, +} from "../../../src/crypto/verification/request/VerificationRequest"; + +// The verification flows use javascript timers to set timeouts. We tell jest to use mock timer implementations +// to ensure that we don't end up with dangling timeouts. +jest.useFakeTimers(); + +/** + * Integration tests for verification functionality. + * + * These tests work by intercepting HTTP requests via fetch-mock rather than mocking out bits of the client, so as + * to provide the most effective integration tests possible. + */ +describe.each(Object.entries(CRYPTO_BACKENDS))("verification (%s)", (backend: string, initCrypto: InitCrypto) => { + // oldBackendOnly is an alternative to `it` or `test` which will skip the test if we are running against the + // Rust backend. Once we have full support in the rust sdk, it will go away. + const oldBackendOnly = backend === "rust-sdk" ? test.skip : test; + + /** the client under test */ + let aliceClient: MatrixClient; + + /** an object which intercepts `/sync` requests from {@link #aliceClient} */ + let syncResponder: SyncResponder; + + beforeEach(async () => { + // anything that we don't have a specific matcher for silently returns a 404 + fetchMock.catch(404); + fetchMock.config.warnOnFallback = false; + + const homeserverUrl = "https://alice-server.com"; + aliceClient = createClient({ + baseUrl: homeserverUrl, + userId: TEST_USER_ID, + accessToken: "akjgkrgjs", + deviceId: "device_under_test", + }); + + await initCrypto(aliceClient); + }); + + afterEach(async () => { + await aliceClient.stopClient(); + fetchMock.mockReset(); + }); + + beforeEach(() => { + syncResponder = new SyncResponder(aliceClient.getHomeserverUrl()); + mockInitialApiRequests(aliceClient.getHomeserverUrl()); + aliceClient.startClient(); + }); + + oldBackendOnly("Outgoing verification: can verify another device via SAS", async () => { + // expect requests to download our own keys + fetchMock.post(new RegExp("/_matrix/client/(r0|v3)/keys/query"), { + device_keys: { + [TEST_USER_ID]: { + [TEST_DEVICE_ID]: SIGNED_TEST_DEVICE_DATA, + }, + }, + }); + + // have alice initiate a verification. She should send a m.key.verification.request + let [requestBody, request] = await Promise.all([ + expectSendToDeviceMessage("m.key.verification.request"), + aliceClient.requestVerification(TEST_USER_ID, [TEST_DEVICE_ID]), + ]); + const transactionId = request.channel.transactionId; + expect(transactionId).toBeDefined(); + expect(request.phase).toEqual(Phase.Requested); + + let toDeviceMessage = requestBody.messages[TEST_USER_ID][TEST_DEVICE_ID]; + expect(toDeviceMessage.methods).toContain("m.sas.v1"); + expect(toDeviceMessage.from_device).toEqual(aliceClient.deviceId); + expect(toDeviceMessage.transaction_id).toEqual(transactionId); + + // The dummy device replies with an m.key.verification.ready... + returnToDeviceMessageFromSync({ + type: "m.key.verification.ready", + content: { + from_device: TEST_DEVICE_ID, + methods: ["m.sas.v1"], + transaction_id: transactionId, + }, + }); + await waitForVerificationRequestChanged(request); + expect(request.phase).toEqual(Phase.Ready); + + // ... and picks a method with m.key.verification.start + returnToDeviceMessageFromSync({ + type: "m.key.verification.start", + content: { + from_device: TEST_DEVICE_ID, + method: "m.sas.v1", + transaction_id: transactionId, + hashes: ["sha256"], + key_agreement_protocols: ["curve25519"], + message_authentication_codes: ["hkdf-hmac-sha256.v2"], + short_authentication_string: ["emoji"], + }, + }); + await waitForVerificationRequestChanged(request); + expect(request.phase).toEqual(Phase.Started); + expect(request.chosenMethod).toEqual("m.sas.v1"); + + // there should now be a verifier + const verifier: VerificationBase = request.verifier!; + expect(verifier).toBeDefined(); + + // start off the verification process: alice will send an `accept` + const verificationPromise = verifier.verify(); + // advance the clock, because the devicelist likes to sleep for 5ms during key downloads + jest.advanceTimersByTime(10); + + requestBody = await expectSendToDeviceMessage("m.key.verification.accept"); + toDeviceMessage = requestBody.messages[TEST_USER_ID][TEST_DEVICE_ID]; + expect(toDeviceMessage.key_agreement_protocol).toEqual("curve25519"); + expect(toDeviceMessage.short_authentication_string).toEqual(["emoji"]); + expect(toDeviceMessage.transaction_id).toEqual(transactionId); + + // The dummy device makes up a curve25519 keypair and sends the public bit back in an `m.key.verification.key' + // We use the Curve25519, HMAC and HKDF implementations in libolm, for now + const olmSAS = new global.Olm.SAS(); + returnToDeviceMessageFromSync({ + type: "m.key.verification.key", + content: { + transaction_id: transactionId, + key: olmSAS.get_pubkey(), + }, + }); + + // alice responds with a 'key' ... + requestBody = await expectSendToDeviceMessage("m.key.verification.key"); + toDeviceMessage = requestBody.messages[TEST_USER_ID][TEST_DEVICE_ID]; + expect(toDeviceMessage.transaction_id).toEqual(transactionId); + const aliceDevicePubKeyBase64 = toDeviceMessage.key; + olmSAS.set_their_key(aliceDevicePubKeyBase64); + + // ... and the client is notified to show the emoji + const showSas = await new Promise((resolve) => { + verifier.once(VerifierEvent.ShowSas, resolve); + }); + + // user confirms that the emoji match, and alice sends a 'mac' + [requestBody] = await Promise.all([expectSendToDeviceMessage("m.key.verification.mac"), showSas.confirm()]); + toDeviceMessage = requestBody.messages[TEST_USER_ID][TEST_DEVICE_ID]; + expect(toDeviceMessage.transaction_id).toEqual(transactionId); + + // the dummy device also confirms that the emoji match, and sends a mac + const macInfoBase = `MATRIX_KEY_VERIFICATION_MAC${TEST_USER_ID}${TEST_DEVICE_ID}${TEST_USER_ID}${aliceClient.deviceId}${transactionId}`; + returnToDeviceMessageFromSync({ + type: "m.key.verification.mac", + content: { + keys: calculateMAC(olmSAS, `ed25519:${TEST_DEVICE_ID}`, `${macInfoBase}KEY_IDS`), + transaction_id: transactionId, + mac: { + [`ed25519:${TEST_DEVICE_ID}`]: calculateMAC( + olmSAS, + TEST_DEVICE_PUBLIC_ED25519_KEY_BASE64, + `${macInfoBase}ed25519:${TEST_DEVICE_ID}`, + ), + }, + }, + }); + + // that should satisfy Alice, who should reply with a 'done' + await expectSendToDeviceMessage("m.key.verification.done"); + + // ... and the whole thing should be done! + await verificationPromise; + expect(request.phase).toEqual(Phase.Done); + + // we're done with the temporary keypair + olmSAS.free(); + }); + + function returnToDeviceMessageFromSync(ev: { type: string; content: object; sender?: string }): void { + ev.sender ??= TEST_USER_ID; + syncResponder.sendOrQueueSyncResponse({ to_device: { events: [ev] } }); + } +}); + +/** + * Wait for the client under test to send a to-device message of the given type. + * + * @param msgtype - type of to-device message we expect + * @returns A Promise which resolves with the body of the HTTP request + */ +function expectSendToDeviceMessage(msgtype: string): Promise<{ messages: any }> { + return new Promise((resolve) => { + fetchMock.putOnce( + new RegExp(`/_matrix/client/(r0|v3)/sendToDevice/${escapeRegExp(msgtype)}`), + (url: string, opts: RequestInit): MockResponse => { + resolve(JSON.parse(opts.body as string)); + return {}; + }, + ); + }); +} + +/** wait for the verification request to emit a 'Change' event */ +function waitForVerificationRequestChanged(request: VerificationRequest): Promise { + return new Promise((resolve) => { + request.once(VerificationRequestEvent.Change, resolve); + }); +} + +/** Perform a MAC calculation on the given data + * + * Does an HKDR and HMAC as defined by the matrix spec (https://spec.matrix.org/v1.7/client-server-api/#mac-calculation, + * as amended by https://github.com/matrix-org/matrix-spec/issues/1553). + * + * @param olmSAS + * @param input + * @param info + */ +function calculateMAC(olmSAS: Olm.SAS, input: string, info: string): string { + const mac = olmSAS.calculate_mac_fixed_base64(input, info); + //console.info(`Test MAC: input:'${input}, info: '${info}' -> '${mac}`); + return mac; +} diff --git a/spec/test-utils/mockEndpoints.ts b/spec/test-utils/mockEndpoints.ts new file mode 100644 index 00000000000..a4c162867fe --- /dev/null +++ b/spec/test-utils/mockEndpoints.ts @@ -0,0 +1,30 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import fetchMock from "fetch-mock-jest"; + +/** + * Mock out the endpoints that the js-sdk calls when we call `MatrixClient.start()`. + * + * @param homeserverUrl - the homeserver url for the client under test + */ +export function mockInitialApiRequests(homeserverUrl: string) { + fetchMock.getOnce(new URL("/_matrix/client/versions", homeserverUrl).toString(), { versions: ["r0.5.0"] }); + fetchMock.getOnce(new URL("/_matrix/client/r0/pushrules/", homeserverUrl).toString(), {}); + fetchMock.postOnce(new URL("/_matrix/client/r0/user/%40alice%3Alocalhost/filter", homeserverUrl).toString(), { + filter_id: "fid", + }); +} diff --git a/spec/test-utils/test-data/.gitignore b/spec/test-utils/test-data/.gitignore new file mode 100644 index 00000000000..91078accddd --- /dev/null +++ b/spec/test-utils/test-data/.gitignore @@ -0,0 +1 @@ +/env diff --git a/spec/test-utils/test-data/generate-test-data.py b/spec/test-utils/test-data/generate-test-data.py new file mode 100755 index 00000000000..f5eae004e4a --- /dev/null +++ b/spec/test-utils/test-data/generate-test-data.py @@ -0,0 +1,117 @@ +#!/bin/env python +# +# Copyright 2023 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This file is a Python script to generate test data for crypto tests. + +To run it: + +python -m venv env +./env/bin/pip install cryptography canonicaljson +./env/bin/python generate-test-data.py > index.ts +""" + +import base64 +import json + +from canonicaljson import encode_canonical_json +from cryptography.hazmat.primitives.asymmetric import ed25519 +from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat + +# input data +TEST_USER_ID = "@alice:localhost" +TEST_DEVICE_ID = "test_device" +# any 32-byte string can be an ed25519 private key. +TEST_DEVICE_PRIVATE_KEY_BYTES = b"deadbeefdeadbeefdeadbeefdeadbeef" + + +def main() -> None: + private_key = ed25519.Ed25519PrivateKey.from_private_bytes( + TEST_DEVICE_PRIVATE_KEY_BYTES + ) + b64_public_key = encode_base64( + private_key.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw) + ) + + device_data = { + "algorithms": ["m.olm.v1.curve25519-aes-sha2", "m.megolm.v1.aes-sha2"], + "device_id": TEST_DEVICE_ID, + "keys": { + f"curve25519:{TEST_DEVICE_ID}": "F4uCNNlcbRvc7CfBz95ZGWBvY1ALniG1J8+6rhVoKS0", + f"ed25519:{TEST_DEVICE_ID}": b64_public_key, + }, + "signatures": {TEST_USER_ID: {}}, + "user_id": TEST_USER_ID, + } + + device_data["signatures"][TEST_USER_ID][ f"ed25519:{TEST_DEVICE_ID}"] = sign_json( + device_data, private_key + ) + + print( + f"""\ +/* Test data for cryptography tests + * + * Do not edit by hand! This file is generated by `./generate-test-data.py` + */ + +import {{ IDeviceKeys }} from "../../../src/@types/crypto"; + +/* eslint-disable comma-dangle */ + +export const TEST_USER_ID = "{TEST_USER_ID}"; +export const TEST_DEVICE_ID = "{TEST_DEVICE_ID}"; + +/** The base64-encoded public ed25519 key for this device */ +export const TEST_DEVICE_PUBLIC_ED25519_KEY_BASE64 = "{b64_public_key}"; + +/** Signed device data, suitable for returning from a `/keys/query` call */ +export const SIGNED_TEST_DEVICE_DATA: IDeviceKeys = {json.dumps(device_data, indent=4)}; +""", end='', + ) + + +def encode_base64(input_bytes: bytes) -> str: + """Encode with unpadded base64""" + output_bytes = base64.b64encode(input_bytes) + output_string = output_bytes.decode("ascii") + return output_string.rstrip("=") + + +def sign_json(json_object: dict, private_key: ed25519.Ed25519PrivateKey) -> str: + """ + Sign the given json object + + Returns the base64-encoded signature of signing `input` following the Matrix + JSON signature algorithm [1] + + [1]: https://spec.matrix.org/v1.7/appendices/#signing-details + """ + signatures = json_object.pop("signatures", {}) + unsigned = json_object.pop("unsigned", None) + + signature = private_key.sign(encode_canonical_json(json_object)) + signature_base64 = encode_base64(signature) + + json_object["signatures"] = signatures + if unsigned is not None: + json_object["unsigned"] = unsigned + + return signature_base64 + + +if __name__ == "__main__": + main() diff --git a/spec/test-utils/test-data/index.ts b/spec/test-utils/test-data/index.ts new file mode 100644 index 00000000000..fbb9a1c2bc7 --- /dev/null +++ b/spec/test-utils/test-data/index.ts @@ -0,0 +1,33 @@ +/* Test data for cryptography tests + * + * Do not edit by hand! This file is generated by `./generate-test-data.py` + */ + +import { IDeviceKeys } from "../../../src/@types/crypto"; + +/* eslint-disable comma-dangle */ + +export const TEST_USER_ID = "@alice:localhost"; +export const TEST_DEVICE_ID = "test_device"; + +/** The base64-encoded public ed25519 key for this device */ +export const TEST_DEVICE_PUBLIC_ED25519_KEY_BASE64 = "YI/7vbGVLpGdYtuceQR8MSsKB/QjgfMXM1xqnn+0NWU"; + +/** Signed device data, suitable for returning from a `/keys/query` call */ +export const SIGNED_TEST_DEVICE_DATA: IDeviceKeys = { + "algorithms": [ + "m.olm.v1.curve25519-aes-sha2", + "m.megolm.v1.aes-sha2" + ], + "device_id": "test_device", + "keys": { + "curve25519:test_device": "F4uCNNlcbRvc7CfBz95ZGWBvY1ALniG1J8+6rhVoKS0", + "ed25519:test_device": "YI/7vbGVLpGdYtuceQR8MSsKB/QjgfMXM1xqnn+0NWU" + }, + "user_id": "@alice:localhost", + "signatures": { + "@alice:localhost": { + "ed25519:test_device": "LmQC/yAUZJmkxZ+3L0nEwvtVWOzjqQqADWBhk+C47SPaFYHeV+E291mgXaSCJVeGltX+HC49Aw7nb6ga7sw0Aw" + } + } +}; From 44ed483e204b879b63b09138e74ba1ce211f4b77 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 2 Jun 2023 18:38:17 +0100 Subject: [PATCH 16/52] Always show a summary after Jest tests (#3440) ... because it is otherwise impossible to see what failed. --- jest.config.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/jest.config.ts b/jest.config.ts index 82a2ef4ee44..d541b67a06b 100644 --- a/jest.config.ts +++ b/jest.config.ts @@ -23,11 +23,19 @@ const config: Config = { collectCoverageFrom: ["/src/**/*.{js,ts}"], coverageReporters: ["text-summary", "lcov"], testResultsProcessor: "@casualbot/jest-sonar-reporter", + + // Always print out a summary if there are any failing tests. Normally + // a summary is only printed if there are more than 20 test *suites*. + reporters: [["default", { summaryThreshold: 0 }]], }; // if we're running under GHA, enable the GHA reporter if (env["GITHUB_ACTIONS"] !== undefined) { - const reporters: Config["reporters"] = [["github-actions", { silent: false }], "summary"]; + const reporters: Config["reporters"] = [ + ["github-actions", { silent: false }], + // as above: always show a summary if there were any failing tests. + ["summary", { summaryThreshold: 0 }], + ]; // if we're running against the develop branch, also enable the slow test reporter if (env["GITHUB_REF"] == "refs/heads/develop") { From dd23e4b72053b8deba64cb062954c91348280bd9 Mon Sep 17 00:00:00 2001 From: David Lee Date: Sat, 3 Jun 2023 10:56:12 -0400 Subject: [PATCH 17/52] Use correct /v3 prefix for /refresh (#3016) * Add tests to ensure /v3/refresh is called + automatic /v1 retry * Request /refresh with v3 prefix, and quietly fall back to v1 * Add tests checking re-raising errors * Update spec/unit/login.spec.ts * Update comment Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> --------- Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> --- spec/unit/login.spec.ts | 90 +++++++++++++++++++++++++++++++++++++++++ src/client.ts | 31 +++++++++----- 2 files changed, 111 insertions(+), 10 deletions(-) diff --git a/spec/unit/login.spec.ts b/spec/unit/login.spec.ts index a1951d18ca3..242594fc433 100644 --- a/spec/unit/login.spec.ts +++ b/spec/unit/login.spec.ts @@ -1,6 +1,15 @@ +import fetchMock from "fetch-mock-jest"; + +import { ClientPrefix, MatrixClient } from "../../src"; import { SSOAction } from "../../src/@types/auth"; import { TestClient } from "../TestClient"; +function createExampleMatrixClient(): MatrixClient { + return new MatrixClient({ + baseUrl: "https://example.com", + }); +} + describe("Login request", function () { let client: TestClient; @@ -57,3 +66,84 @@ describe("SSO login URL", function () { }); }); }); + +describe("refreshToken", () => { + afterEach(() => { + fetchMock.mockReset(); + }); + + it("requests the correctly-prefixed /refresh endpoint when server correctly accepts /v3", async () => { + const client = createExampleMatrixClient(); + + const response = { + access_token: "access_token", + refresh_token: "refresh_token", + expires_in_ms: 30000, + }; + + fetchMock.postOnce(client.http.getUrl("/refresh", undefined, ClientPrefix.V3).toString(), response); + fetchMock.postOnce(client.http.getUrl("/refresh", undefined, ClientPrefix.V1).toString(), () => { + throw new Error("/v1/refresh unexpectedly called"); + }); + + const refreshResult = await client.refreshToken("initial_refresh_token"); + expect(refreshResult).toEqual(response); + }); + + it("falls back to /v1 when server does not recognized /v3 refresh", async () => { + const client = createExampleMatrixClient(); + + const response = { + access_token: "access_token", + refresh_token: "refresh_token", + expires_in_ms: 30000, + }; + + fetchMock.postOnce(client.http.getUrl("/refresh", undefined, ClientPrefix.V3).toString(), { + status: 400, + body: { errcode: "M_UNRECOGNIZED" }, + }); + fetchMock.postOnce(client.http.getUrl("/refresh", undefined, ClientPrefix.V1).toString(), response); + + const refreshResult = await client.refreshToken("initial_refresh_token"); + expect(refreshResult).toEqual(response); + }); + + it("re-raises M_UNRECOGNIZED exceptions from /v1", async () => { + const client = createExampleMatrixClient(); + + fetchMock.postOnce(client.http.getUrl("/refresh", undefined, ClientPrefix.V3).toString(), { + status: 400, + body: { errcode: "M_UNRECOGNIZED" }, + }); + fetchMock.postOnce(client.http.getUrl("/refresh", undefined, ClientPrefix.V1).toString(), { + status: 400, + body: { errcode: "M_UNRECOGNIZED" }, + }); + + expect(client.refreshToken("initial_refresh_token")).rejects.toMatchObject({ errcode: "M_UNRECOGNIZED" }); + }); + + it("re-raises non-M_UNRECOGNIZED exceptions from /v3", async () => { + const client = createExampleMatrixClient(); + + fetchMock.postOnce(client.http.getUrl("/refresh", undefined, ClientPrefix.V3).toString(), 429); + fetchMock.postOnce(client.http.getUrl("/refresh", undefined, ClientPrefix.V1).toString(), () => { + throw new Error("/v1/refresh unexpectedly called"); + }); + + expect(client.refreshToken("initial_refresh_token")).rejects.toMatchObject({ httpStatus: 429 }); + }); + + it("re-raises non-M_UNRECOGNIZED exceptions from /v1", async () => { + const client = createExampleMatrixClient(); + + fetchMock.postOnce(client.http.getUrl("/refresh", undefined, ClientPrefix.V3).toString(), { + status: 400, + body: { errcode: "M_UNRECOGNIZED" }, + }); + fetchMock.postOnce(client.http.getUrl("/refresh", undefined, ClientPrefix.V1).toString(), 429); + + expect(client.refreshToken("initial_refresh_token")).rejects.toMatchObject({ httpStatus: 429 }); + }); +}); diff --git a/src/client.ts b/src/client.ts index 4937494b435..3351850548d 100644 --- a/src/client.ts +++ b/src/client.ts @@ -7689,16 +7689,27 @@ export class MatrixClient extends TypedEventEmitter { - return this.http.authedRequest( - Method.Post, - "/refresh", - undefined, - { refresh_token: refreshToken }, - { - prefix: ClientPrefix.V1, - inhibitLogoutEmit: true, // we don't want to cause logout loops - }, - ); + const performRefreshRequestWithPrefix = (prefix: ClientPrefix): Promise => + this.http.authedRequest( + Method.Post, + "/refresh", + undefined, + { refresh_token: refreshToken }, + { + prefix, + inhibitLogoutEmit: true, // we don't want to cause logout loops + }, + ); + + // First try with the (specced) /v3/ prefix. + // However, before Synapse 1.72.0, Synapse incorrectly required a /v1/ prefix, so we fall + // back to that if the request fails, for backwards compatibility. + return performRefreshRequestWithPrefix(ClientPrefix.V3).catch((e) => { + if (e.errcode === "M_UNRECOGNIZED") { + return performRefreshRequestWithPrefix(ClientPrefix.V1); + } + throw e; + }); } /** From 5976d6982242e7ec489c2bab9d6d8d3fd9bfc6ae Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Mon, 5 Jun 2023 10:23:44 +0200 Subject: [PATCH 18/52] Update Mutual Rooms (MSC2666) support (#3381) * update mutual rooms support * clarify docs and switch eslint comment with todo * please the holy linter * change query variable names around * add mock tests and fix issue * ye holy linter --- spec/unit/room-member.spec.ts | 134 +++++++++++++++++++++++++++++++++- src/client.ts | 78 ++++++++++++++++---- 2 files changed, 197 insertions(+), 15 deletions(-) diff --git a/spec/unit/room-member.spec.ts b/spec/unit/room-member.spec.ts index bc8a27344b2..99a2d589598 100644 --- a/spec/unit/room-member.spec.ts +++ b/spec/unit/room-member.spec.ts @@ -14,9 +14,19 @@ See the License for the specific language governing permissions and limitations under the License. */ +import fetchMock from "fetch-mock-jest"; + import * as utils from "../test-utils/test-utils"; import { RoomMember, RoomMemberEvent } from "../../src/models/room-member"; -import { EventType, RoomState } from "../../src"; +import { + createClient, + EventType, + MatrixClient, + RoomState, + UNSTABLE_MSC2666_MUTUAL_ROOMS, + UNSTABLE_MSC2666_QUERY_MUTUAL_ROOMS, + UNSTABLE_MSC2666_SHARED_ROOMS, +} from "../../src"; describe("RoomMember", function () { const roomId = "!foo:bar"; @@ -481,3 +491,125 @@ describe("RoomMember", function () { }); }); }); + +describe("MutualRooms", () => { + let client: MatrixClient; + const HS_URL = "https://example.com"; + const TEST_USER_ID = "@alice:localhost"; + const TEST_DEVICE_ID = "xzcvb"; + const QUERIED_USER = "@user:example.com"; + + beforeEach(async () => { + // anything that we don't have a specific matcher for silently returns a 404 + fetchMock.catch(404); + fetchMock.config.warnOnFallback = true; + + client = createClient({ + baseUrl: HS_URL, + userId: TEST_USER_ID, + accessToken: "akjgkrgjs", + deviceId: TEST_DEVICE_ID, + }); + }); + + afterEach(async () => { + await client.stopClient(); + fetchMock.mockReset(); + }); + + function enableFeature(feature: string) { + const mapping: Record = {}; + + mapping[feature] = true; + + fetchMock.get(`${HS_URL}/_matrix/client/versions`, { + unstable_features: mapping, + versions: ["v1.1"], + }); + } + + it("supports the initial MSC version (shared rooms)", async () => { + enableFeature(UNSTABLE_MSC2666_SHARED_ROOMS); + + fetchMock.get("express:/_matrix/client/unstable/uk.half-shot.msc2666/user/shared_rooms/:user_id", (rawUrl) => { + const segments = rawUrl.split("/"); + const lastSegment = decodeURIComponent(segments[segments.length - 1]); + + expect(lastSegment).toEqual(QUERIED_USER); + + return { + joined: ["!test:example.com"], + }; + }); + + const rooms = await client._unstable_getSharedRooms(QUERIED_USER); + + expect(rooms).toEqual(["!test:example.com"]); + }); + + it("supports the renaming MSC version (mutual rooms)", async () => { + enableFeature(UNSTABLE_MSC2666_MUTUAL_ROOMS); + + fetchMock.get("express:/_matrix/client/unstable/uk.half-shot.msc2666/user/mutual_rooms/:user_id", (rawUrl) => { + const segments = rawUrl.split("/"); + const lastSegment = decodeURIComponent(segments[segments.length - 1]); + + expect(lastSegment).toEqual(QUERIED_USER); + + return { + joined: ["!test2:example.com"], + }; + }); + + const rooms = await client._unstable_getSharedRooms(QUERIED_USER); + + expect(rooms).toEqual(["!test2:example.com"]); + }); + + describe("can work the latest MSC version (query mutual rooms)", () => { + beforeEach(() => { + enableFeature(UNSTABLE_MSC2666_QUERY_MUTUAL_ROOMS); + }); + + it("works with a simple response", async () => { + fetchMock.get("express:/_matrix/client/unstable/uk.half-shot.msc2666/user/mutual_rooms", (rawUrl) => { + const url = new URL(rawUrl); + + expect(url.searchParams.get("user_id")).toEqual(QUERIED_USER); + + return { + joined: ["!test3:example.com"], + }; + }); + + const rooms = await client._unstable_getSharedRooms(QUERIED_USER); + + expect(rooms).toEqual(["!test3:example.com"]); + }); + + it("works with a paginated response", async () => { + fetchMock.get("express:/_matrix/client/unstable/uk.half-shot.msc2666/user/mutual_rooms", (rawUrl) => { + const url = new URL(rawUrl); + + expect(url.searchParams.get("user_id")).toEqual(QUERIED_USER); + + const token = url.searchParams.get("batch_token"); + + if (token == "yahaha") { + return { + joined: ["!korok:example.com"], + }; + } else { + return { + joined: ["!rock:example.com"], + next_batch_token: "yahaha", + }; + } + }); + + const rooms = await client._unstable_getSharedRooms(QUERIED_USER); + + expect(rooms).toEqual(["!rock:example.com", "!korok:example.com"]); + }); + }); +}); diff --git a/src/client.ts b/src/client.ts index 3351850548d..caca1a89a82 100644 --- a/src/client.ts +++ b/src/client.ts @@ -500,6 +500,10 @@ export interface IMSC3882GetLoginTokenCapability extends ICapability {} export const UNSTABLE_MSC3882_CAPABILITY = new UnstableValue("m.get_login_token", "org.matrix.msc3882.get_login_token"); +export const UNSTABLE_MSC2666_SHARED_ROOMS = "uk.half-shot.msc2666"; +export const UNSTABLE_MSC2666_MUTUAL_ROOMS = "uk.half-shot.msc2666.mutual_rooms"; +export const UNSTABLE_MSC2666_QUERY_MUTUAL_ROOMS = "uk.half-shot.msc2666.query_mutual_rooms"; + /** * A representation of the capabilities advertised by a homeserver as defined by * [Capabilities negotiation](https://spec.matrix.org/v1.6/client-server-api/#get_matrixclientv3capabilities). @@ -7148,29 +7152,75 @@ export class MatrixClient extends TypedEventEmitter { - const sharedRoomsSupport = await this.doesServerSupportUnstableFeature("uk.half-shot.msc2666"); - const mutualRoomsSupport = await this.doesServerSupportUnstableFeature("uk.half-shot.msc2666.mutual_rooms"); + // Initial variant of the MSC + const sharedRoomsSupport = await this.doesServerSupportUnstableFeature(UNSTABLE_MSC2666_SHARED_ROOMS); - if (!sharedRoomsSupport && !mutualRoomsSupport) { - throw Error("Server does not support mutual_rooms API"); - } + // Newer variant that renamed shared rooms to mutual rooms + const mutualRoomsSupport = await this.doesServerSupportUnstableFeature(UNSTABLE_MSC2666_MUTUAL_ROOMS); - const path = utils.encodeUri( - `/uk.half-shot.msc2666/user/${mutualRoomsSupport ? "mutual_rooms" : "shared_rooms"}/$userId`, - { $userId: userId }, + // Latest variant that changed from path elements to query elements + const queryMutualRoomsSupport = await this.doesServerSupportUnstableFeature( + UNSTABLE_MSC2666_QUERY_MUTUAL_ROOMS, ); - const res = await this.http.authedRequest<{ joined: string[] }>(Method.Get, path, undefined, undefined, { - prefix: ClientPrefix.Unstable, - }); - return res.joined; + if (!sharedRoomsSupport && !mutualRoomsSupport && !queryMutualRoomsSupport) { + throw Error("Server does not support the Mutual Rooms API"); + } + + let path; + let query; + + // Cascading unstable support switching. + if (queryMutualRoomsSupport) { + path = "/uk.half-shot.msc2666/user/mutual_rooms"; + query = { user_id: userId }; + } else { + path = utils.encodeUri( + `/uk.half-shot.msc2666/user/${mutualRoomsSupport ? "mutual_rooms" : "shared_rooms"}/$userId`, + { $userId: userId }, + ); + query = {}; + } + + // Accumulated rooms + const rooms: string[] = []; + let token = null; + + do { + const tokenQuery: Record = {}; + if (token != null && queryMutualRoomsSupport) { + tokenQuery["batch_token"] = token; + } + + const res = await this.http.authedRequest<{ + joined: string[]; + next_batch_token?: string; + }>(Method.Get, path, { ...query, ...tokenQuery }, undefined, { + prefix: ClientPrefix.Unstable, + }); + + rooms.push(...res.joined); + + if (res.next_batch_token !== undefined) { + token = res.next_batch_token; + } else { + token = null; + } + } while (token != null); + + return rooms; } /** From 3895fbb9eaf365a4d1c1b4fac804072db148b77c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 5 Jun 2023 10:11:00 +0100 Subject: [PATCH 19/52] GHA: build and cypress-test a copy of element-web after each push (#3412) * Build a copy of element-web after each push * Run cypress after each build of element-web --- .github/workflows/cypress.yml | 29 ++++++++++++++++++++++ .github/workflows/downstream-artifacts.yml | 17 +++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 .github/workflows/cypress.yml create mode 100644 .github/workflows/downstream-artifacts.yml diff --git a/.github/workflows/cypress.yml b/.github/workflows/cypress.yml new file mode 100644 index 00000000000..09c0a303b34 --- /dev/null +++ b/.github/workflows/cypress.yml @@ -0,0 +1,29 @@ +# Triggers after the "Downstream artifacts" build has finished, to run the +# cypress tests (with access to repo secrets) + +name: matrix-react-sdk Cypress End to End Tests +on: + workflow_run: + workflows: ["Build downstream artifacts"] + types: + - completed + +concurrency: + group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch || github.run_id }} + cancel-in-progress: ${{ github.event.workflow_run.event == 'pull_request' }} + +jobs: + cypress: + name: Cypress + uses: matrix-org/matrix-react-sdk/.github/workflows/cypress.yaml@HEAD + permissions: + actions: read + issues: read + statuses: write + pull-requests: read + secrets: + # secrets are not automatically shared with called workflows, so share the cypress dashboard key + CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} + with: + react-sdk-repository: matrix-org/matrix-react-sdk + rust-crypto: true diff --git a/.github/workflows/downstream-artifacts.yml b/.github/workflows/downstream-artifacts.yml new file mode 100644 index 00000000000..d21ab7537ab --- /dev/null +++ b/.github/workflows/downstream-artifacts.yml @@ -0,0 +1,17 @@ +name: Build downstream artifacts +on: + pull_request: {} + merge_group: + types: [checks_requested] + push: + branches: [develop, master] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +jobs: + build-element-web: + name: Build element-web + uses: matrix-org/matrix-react-sdk/.github/workflows/element-web.yaml@HEAD + with: + matrix-js-sdk-sha: ${{ github.sha }} + react-sdk-repository: matrix-org/matrix-react-sdk From 6f79cd26d289e7f25a1fb623a7e18c334e7db935 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 5 Jun 2023 16:05:14 +0100 Subject: [PATCH 20/52] Fix downstream-artifacts build (#3443) * Fix downstream-artifacts build * Update cypress.yml --- .github/workflows/cypress.yml | 2 +- .github/workflows/downstream-artifacts.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cypress.yml b/.github/workflows/cypress.yml index 09c0a303b34..b9dd709d0bd 100644 --- a/.github/workflows/cypress.yml +++ b/.github/workflows/cypress.yml @@ -15,7 +15,7 @@ concurrency: jobs: cypress: name: Cypress - uses: matrix-org/matrix-react-sdk/.github/workflows/cypress.yaml@HEAD + uses: matrix-org/matrix-react-sdk/.github/workflows/cypress.yaml@develop permissions: actions: read issues: read diff --git a/.github/workflows/downstream-artifacts.yml b/.github/workflows/downstream-artifacts.yml index d21ab7537ab..48e958bec49 100644 --- a/.github/workflows/downstream-artifacts.yml +++ b/.github/workflows/downstream-artifacts.yml @@ -11,7 +11,7 @@ concurrency: jobs: build-element-web: name: Build element-web - uses: matrix-org/matrix-react-sdk/.github/workflows/element-web.yaml@HEAD + uses: matrix-org/matrix-react-sdk/.github/workflows/element-web.yaml@develop with: matrix-js-sdk-sha: ${{ github.sha }} react-sdk-repository: matrix-org/matrix-react-sdk From 1e2f701f4c15920c89cb5b13830459c33a774493 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Mon, 5 Jun 2023 16:18:56 +0100 Subject: [PATCH 21/52] Fix edge cases around 2nd order relations and threads (#3437) * Fix tests oversimplifying threads fixtures * Check for unsigned thread_id in MatrixEvent::threadRootId * Fix threads order being racy * Make Sonar happier * Iterate --- .../matrix-client-event-timeline.spec.ts | 9 +++--- spec/unit/event-timeline-set.spec.ts | 27 ++++++++++++----- src/models/event.ts | 15 ++++++++-- src/models/room.ts | 29 ++++++------------- 4 files changed, 45 insertions(+), 35 deletions(-) diff --git a/spec/integ/matrix-client-event-timeline.spec.ts b/spec/integ/matrix-client-event-timeline.spec.ts index 680e408380b..c7083db1968 100644 --- a/spec/integ/matrix-client-event-timeline.spec.ts +++ b/spec/integ/matrix-client-event-timeline.spec.ts @@ -1274,6 +1274,7 @@ describe("MatrixClient event timelines", function () { THREAD_ROOT.event_id, THREAD_REPLY.event_id, THREAD_REPLY2.getId(), + THREAD_ROOT_REACTION.getId(), THREAD_REPLY3.getId(), ]); }); @@ -1322,7 +1323,7 @@ describe("MatrixClient event timelines", function () { request.respond(200, function () { return { original_event: root, - chunk: [replies], + chunk: replies, // no next batch as this is the oldest end of the timeline }; }); @@ -1479,7 +1480,7 @@ describe("MatrixClient event timelines", function () { user: userId, type: "m.room.message", content: { - "body": "thread reply", + "body": "thread2 reply", "msgtype": "m.text", "m.relates_to": { // We can't use the const here because we change server support mode for test @@ -1499,7 +1500,7 @@ describe("MatrixClient event timelines", function () { user: userId, type: "m.room.message", content: { - "body": "thread reply", + "body": "thread reply2", "msgtype": "m.text", "m.relates_to": { // We can't use the const here because we change server support mode for test @@ -1567,7 +1568,7 @@ describe("MatrixClient event timelines", function () { // Test adding a second event to the first thread const thread = room.getThread(THREAD_ROOT.event_id!)!; thread.initialEventsFetched = true; - const prom = emitPromise(room, ThreadEvent.NewReply); + const prom = emitPromise(room, ThreadEvent.Update); respondToEvent(THREAD_ROOT_UPDATED); respondToEvent(THREAD_ROOT_UPDATED); respondToEvent(THREAD_ROOT_UPDATED); diff --git a/spec/unit/event-timeline-set.spec.ts b/spec/unit/event-timeline-set.spec.ts index 7afee718967..b5445a03397 100644 --- a/spec/unit/event-timeline-set.spec.ts +++ b/spec/unit/event-timeline-set.spec.ts @@ -142,13 +142,6 @@ describe("EventTimelineSet", () => { }); describe("addEventToTimeline", () => { - let thread: Thread; - - beforeEach(() => { - (client.supportsThreads as jest.Mock).mockReturnValue(true); - thread = new Thread("!thread_id:server", messageEvent, { room, client }); - }); - it("Adds event to timeline", () => { const liveTimeline = eventTimelineSet.getLiveTimeline(); expect(liveTimeline.getEvents().length).toStrictEqual(0); @@ -167,6 +160,15 @@ describe("EventTimelineSet", () => { eventTimelineSet.addEventToTimeline(messageEvent, liveTimeline, true, false); }).not.toThrow(); }); + }); + + describe("addEventToTimeline (thread timeline)", () => { + let thread: Thread; + + beforeEach(() => { + (client.supportsThreads as jest.Mock).mockReturnValue(true); + thread = new Thread("!thread_id:server", messageEvent, { room, client }); + }); it("should not add an event to a timeline that does not belong to the timelineSet", () => { const eventTimelineSet2 = new EventTimelineSet(room); @@ -197,7 +199,14 @@ describe("EventTimelineSet", () => { const liveTimeline = eventTimelineSetForThread.getLiveTimeline(); expect(liveTimeline.getEvents().length).toStrictEqual(0); - eventTimelineSetForThread.addEventToTimeline(messageEvent, liveTimeline, { + const normalMessage = utils.mkMessage({ + room: roomId, + user: userA, + msg: "Hello!", + event: true, + }); + + eventTimelineSetForThread.addEventToTimeline(normalMessage, liveTimeline, { toStartOfTimeline: true, }); expect(liveTimeline.getEvents().length).toStrictEqual(0); @@ -336,7 +345,9 @@ describe("EventTimelineSet", () => { }); it("should return true if the timeline set is not for a thread and the event is a thread root", () => { + const thread = new Thread(messageEvent.getId()!, messageEvent, { room, client }); const eventTimelineSet = new EventTimelineSet(room, {}, client); + messageEvent.setThread(thread); expect(eventTimelineSet.canContain(messageEvent)).toBeTruthy(); }); diff --git a/src/models/event.ts b/src/models/event.ts index feb21fbba74..3b372c776a9 100644 --- a/src/models/event.ts +++ b/src/models/event.ts @@ -579,9 +579,18 @@ export class MatrixEvent extends TypedEventEmitter { } } - this.on(ThreadEvent.NewReply, this.onThreadNewReply); + this.on(ThreadEvent.Update, this.onThreadUpdate); this.on(ThreadEvent.Delete, this.onThreadDelete); this.threadsReady = true; } @@ -2055,7 +2055,7 @@ export class Room extends ReadReceipt { } } - private onThreadNewReply(thread: Thread): void { + private onThreadUpdate(thread: Thread): void { this.updateThreadRootEvents(thread, false, true); } @@ -2113,12 +2113,13 @@ export class Room extends ReadReceipt { }; } - // A thread relation is always only shown in a thread - if (event.isRelation(THREAD_RELATION_TYPE.name)) { + // A thread relation (1st and 2nd order) is always only shown in a thread + const threadRootId = event.threadRootId; + if (threadRootId != undefined) { return { shouldLiveInRoom: false, shouldLiveInThread: true, - threadId: event.threadRootId, + threadId: threadRootId, }; } @@ -2149,15 +2150,6 @@ export class Room extends ReadReceipt { }; } - const unsigned = event.getUnsigned(); - if (typeof unsigned[UNSIGNED_THREAD_ID_FIELD.name] === "string") { - return { - shouldLiveInRoom: false, - shouldLiveInThread: true, - threadId: unsigned[UNSIGNED_THREAD_ID_FIELD.name], - }; - } - // We've exhausted all scenarios, // we cannot assume that it lives in the main timeline as this may be a relation for an unknown thread // adding the event in the wrong timeline causes stuck notifications and can break ability to send read receipts @@ -2890,12 +2882,9 @@ export class Room extends ReadReceipt { private findThreadRoots(events: MatrixEvent[]): Set { const threadRoots = new Set(); for (const event of events) { - if (event.isRelation(THREAD_RELATION_TYPE.name)) { - threadRoots.add(event.relationEventId ?? ""); - } - const unsigned = event.getUnsigned(); - if (typeof unsigned[UNSIGNED_THREAD_ID_FIELD.name] === "string") { - threadRoots.add(unsigned[UNSIGNED_THREAD_ID_FIELD.name]!); + const threadRootId = event.threadRootId; + if (threadRootId != undefined) { + threadRoots.add(threadRootId); } } return threadRoots; From d4e345260525136420874ec28ad4175129566185 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Mon, 5 Jun 2023 17:40:19 +0100 Subject: [PATCH 22/52] Make sliding sync linearize processing of sync requests (#3442) * Make sliding sync linearize processing of sync requests * Iterate * Iterate * Iterate * Iterate --- spec/integ/sliding-sync-sdk.spec.ts | 4 ++-- src/models/typed-event-emitter.ts | 18 ++++++++++++++++++ src/sliding-sync-sdk.ts | 4 ++-- src/sliding-sync.ts | 12 ++++++------ 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/spec/integ/sliding-sync-sdk.spec.ts b/spec/integ/sliding-sync-sdk.spec.ts index dfec79e1583..a55316b5fa5 100644 --- a/spec/integ/sliding-sync-sdk.spec.ts +++ b/spec/integ/sliding-sync-sdk.spec.ts @@ -20,7 +20,7 @@ import { fail } from "assert"; import { SlidingSync, SlidingSyncEvent, MSC3575RoomData, SlidingSyncState, Extension } from "../../src/sliding-sync"; import { TestClient } from "../TestClient"; -import { IRoomEvent, IStateEvent } from "../../src/sync-accumulator"; +import { IRoomEvent, IStateEvent } from "../../src"; import { MatrixClient, MatrixEvent, @@ -39,7 +39,7 @@ import { } from "../../src"; import { SlidingSyncSdk } from "../../src/sliding-sync-sdk"; import { SyncApiOptions, SyncState } from "../../src/sync"; -import { IStoredClientOpts } from "../../src/client"; +import { IStoredClientOpts } from "../../src"; import { logger } from "../../src/logger"; import { emitPromise } from "../test-utils/test-utils"; import { defer } from "../../src/utils"; diff --git a/src/models/typed-event-emitter.ts b/src/models/typed-event-emitter.ts index 7eac48b962b..de0dd2b8696 100644 --- a/src/models/typed-event-emitter.ts +++ b/src/models/typed-event-emitter.ts @@ -89,6 +89,24 @@ export class TypedEventEmitter< return super.emit(event, ...args); } + /** + * Similar to `emit` but calls all listeners within a `Promise.all` and returns the promise chain + * @param event - The name of the event to emit + * @param args - Arguments to pass to the listener + * @returns `true` if the event had listeners, `false` otherwise. + */ + public async emitPromised( + event: T, + ...args: Parameters + ): Promise; + public async emitPromised(event: T, ...args: Parameters): Promise; + public async emitPromised(event: T, ...args: any[]): Promise { + const listeners = this.listeners(event); + return Promise.allSettled(listeners.map((l) => l(...args))).then(() => { + return listeners.length > 0; + }); + } + /** * Returns the number of listeners listening to the event named `event`. * diff --git a/src/sliding-sync-sdk.ts b/src/sliding-sync-sdk.ts index 27eae2d94b9..36eed5d73e2 100644 --- a/src/sliding-sync-sdk.ts +++ b/src/sliding-sync-sdk.ts @@ -376,7 +376,7 @@ export class SlidingSyncSdk { }); } - private onRoomData(roomId: string, roomData: MSC3575RoomData): void { + private async onRoomData(roomId: string, roomData: MSC3575RoomData): Promise { let room = this.client.store.getRoom(roomId); if (!room) { if (!roomData.initial) { @@ -385,7 +385,7 @@ export class SlidingSyncSdk { } room = _createAndReEmitRoom(this.client, roomId, this.opts); } - this.processRoomData(this.client, room, roomData); + await this.processRoomData(this.client, room!, roomData); } private onLifecycle(state: SlidingSyncState, resp: MSC3575SlidingSyncResponse | null, err?: Error): void { diff --git a/src/sliding-sync.ts b/src/sliding-sync.ts index dde5f1be73b..a45a142d58e 100644 --- a/src/sliding-sync.ts +++ b/src/sliding-sync.ts @@ -326,7 +326,7 @@ export enum SlidingSyncEvent { } export type SlidingSyncEventHandlerMap = { - [SlidingSyncEvent.RoomData]: (roomId: string, roomData: MSC3575RoomData) => void; + [SlidingSyncEvent.RoomData]: (roomId: string, roomData: MSC3575RoomData) => Promise | void; [SlidingSyncEvent.Lifecycle]: ( state: SlidingSyncState, resp: MSC3575SlidingSyncResponse | null, @@ -567,14 +567,14 @@ export class SlidingSync extends TypedEventEmitter { if (!roomData.required_state) { roomData.required_state = []; } if (!roomData.timeline) { roomData.timeline = []; } - this.emit(SlidingSyncEvent.RoomData, roomId, roomData); + await this.emitPromised(SlidingSyncEvent.RoomData, roomId, roomData); } /** @@ -923,9 +923,9 @@ export class SlidingSync extends TypedEventEmitter { - this.invokeRoomDataListeners(roomId, resp!.rooms[roomId]); - }); + for (const roomId in resp.rooms) { + await this.invokeRoomDataListeners(roomId, resp!.rooms[roomId]); + } const listKeysWithUpdates: Set = new Set(); if (!doNotUpdateList) { From 8ccbb8d708a2ee87e259f818fbaa321a646718a0 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 5 Jun 2023 18:38:08 +0100 Subject: [PATCH 23/52] Disable downstream artifacts build for develop branch (#3444) --- .github/workflows/downstream-artifacts.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/downstream-artifacts.yml b/.github/workflows/downstream-artifacts.yml index 48e958bec49..b16528cb66c 100644 --- a/.github/workflows/downstream-artifacts.yml +++ b/.github/workflows/downstream-artifacts.yml @@ -3,8 +3,16 @@ on: pull_request: {} merge_group: types: [checks_requested] - push: - branches: [develop, master] + + # For now at least, we don't run this or the cypress-tests against pushes + # to develop or master. + # + # Note that if we later choose to do so, we'll need to find a way to stop + # the results in Cypress Cloud from clobbering those from the 'develop' + # branch of matrix-react-sdk. + # + #push: + # branches: [develop, master] concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true From e1ce16a5b0984d0cd638e12eb85f1c721c42d479 Mon Sep 17 00:00:00 2001 From: Stanislav Demydiuk Date: Tue, 6 Jun 2023 10:59:10 +0300 Subject: [PATCH 24/52] Export thread-related types from SDK (#3447) * Export thread-related types from SDK * address PR feedback --- src/matrix.ts | 1 + src/models/event.ts | 2 +- src/models/room.ts | 2 +- src/models/thread.ts | 13 +++++++++---- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/matrix.ts b/src/matrix.ts index e7fd2ad75f0..365bf6dfcf8 100644 --- a/src/matrix.ts +++ b/src/matrix.ts @@ -37,6 +37,7 @@ export * from "./models/event-timeline-set"; export * from "./models/poll"; export * from "./models/room-member"; export * from "./models/room-state"; +export { ThreadEvent, ThreadEmittedEvents, ThreadEventHandlerMap, Thread } from "./models/thread"; export * from "./models/typed-event-emitter"; export * from "./models/user"; export * from "./models/device"; diff --git a/src/models/event.ts b/src/models/event.ts index 3b372c776a9..1790a9836da 100644 --- a/src/models/event.ts +++ b/src/models/event.ts @@ -34,7 +34,7 @@ import { import { Crypto } from "../crypto"; import { deepSortedObjectEntries, internaliseString } from "../utils"; import { RoomMember } from "./room-member"; -import { Thread, ThreadEvent, EventHandlerMap as ThreadEventHandlerMap, THREAD_RELATION_TYPE } from "./thread"; +import { Thread, ThreadEvent, ThreadEventHandlerMap, THREAD_RELATION_TYPE } from "./thread"; import { IActionsObject } from "../pushprocessor"; import { TypedReEmitter } from "../ReEmitter"; import { MatrixError } from "../http-api"; diff --git a/src/models/room.ts b/src/models/room.ts index eb45f88100d..e64d9f938a2 100644 --- a/src/models/room.ts +++ b/src/models/room.ts @@ -49,7 +49,7 @@ import { BeaconEvent, BeaconEventHandlerMap } from "./beacon"; import { Thread, ThreadEvent, - EventHandlerMap as ThreadHandlerMap, + ThreadEventHandlerMap as ThreadHandlerMap, FILTER_RELATED_BY_REL_TYPES, THREAD_RELATION_TYPE, FILTER_RELATED_BY_SENDERS, diff --git a/src/models/thread.ts b/src/models/thread.ts index 3179860819d..33bd44ed8c7 100644 --- a/src/models/thread.ts +++ b/src/models/thread.ts @@ -38,15 +38,20 @@ export enum ThreadEvent { Delete = "Thread.delete", } -type EmittedEvents = Exclude | RoomEvent.Timeline | RoomEvent.TimelineReset; +export type ThreadEmittedEvents = Exclude | RoomEvent.Timeline | RoomEvent.TimelineReset; -export type EventHandlerMap = { +export type ThreadEventHandlerMap = { [ThreadEvent.Update]: (thread: Thread) => void; [ThreadEvent.NewReply]: (thread: Thread, event: MatrixEvent) => void; [ThreadEvent.ViewThread]: () => void; [ThreadEvent.Delete]: (thread: Thread) => void; } & EventTimelineSetHandlerMap; +/** + * @deprecated please use ThreadEventHandlerMap instead + */ +export type EventHandlerMap = ThreadEventHandlerMap; + interface IThreadOpts { room: Room; client: MatrixClient; @@ -70,7 +75,7 @@ export function determineFeatureSupport(stable: boolean, unstable: boolean): Fea } } -export class Thread extends ReadReceipt { +export class Thread extends ReadReceipt { public static hasServerSideSupport = FeatureSupport.None; public static hasServerSideListSupport = FeatureSupport.None; public static hasServerSideFwdPaginationSupport = FeatureSupport.None; @@ -83,7 +88,7 @@ export class Thread extends ReadReceipt { private _currentUserParticipated = false; - private reEmitter: TypedReEmitter; + private reEmitter: TypedReEmitter; private lastEvent: MatrixEvent | undefined; private replyCount = 0; From a21736c16ee27c9bee3a13bcd25af934ecc02f87 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 6 Jun 2023 11:22:12 +0100 Subject: [PATCH 25/52] Integration test for QR code verification (#3439) * Integration test for QR code verification Followup to https://github.com/matrix-org/matrix-js-sdk/pull/3436: another integration test, this time using the QR code flow * Use Object.defineProperty, and restore afterwards Apparently global.crypto exists in some environments * apply ts-ignore * remove stray comment * Update spec/integ/crypto/verification.spec.ts --- spec/integ/crypto/verification.spec.ts | 137 +++++++++++++++++- .../test-data/generate-test-data.py | 90 +++++++++++- spec/test-utils/test-data/index.ts | 51 +++++++ 3 files changed, 275 insertions(+), 3 deletions(-) diff --git a/spec/integ/crypto/verification.spec.ts b/spec/integ/crypto/verification.spec.ts index 64a43d1ce1a..fdcc4bf1a4c 100644 --- a/spec/integ/crypto/verification.spec.ts +++ b/spec/integ/crypto/verification.spec.ts @@ -18,12 +18,14 @@ import fetchMock from "fetch-mock-jest"; import { MockResponse } from "fetch-mock"; import { createClient, MatrixClient } from "../../../src"; -import { ShowSasCallbacks, VerifierEvent } from "../../../src/crypto-api/verification"; +import { ShowQrCodeCallbacks, ShowSasCallbacks, VerifierEvent } from "../../../src/crypto-api/verification"; import { escapeRegExp } from "../../../src/utils"; import { VerificationBase } from "../../../src/crypto/verification/Base"; import { CRYPTO_BACKENDS, InitCrypto } from "../../test-utils/test-utils"; import { SyncResponder } from "../../test-utils/SyncResponder"; import { + MASTER_CROSS_SIGNING_PUBLIC_KEY_BASE64, + SIGNED_CROSS_SIGNING_KEYS_DATA, SIGNED_TEST_DEVICE_DATA, TEST_DEVICE_ID, TEST_DEVICE_PUBLIC_ED25519_KEY_BASE64, @@ -40,6 +42,34 @@ import { // to ensure that we don't end up with dangling timeouts. jest.useFakeTimers(); +let previousCrypto: Crypto | undefined; + +beforeAll(() => { + // Stub out global.crypto + previousCrypto = global["crypto"]; + + Object.defineProperty(global, "crypto", { + value: { + getRandomValues: function (array: T): T { + array.fill(0x12); + return array; + }, + }, + }); +}); + +// restore the original global.crypto +afterAll(() => { + if (previousCrypto === undefined) { + // @ts-ignore deleting a non-optional property. It *is* optional really. + delete global.crypto; + } else { + Object.defineProperty(global, "crypto", { + value: previousCrypto, + }); + } +}); + /** * Integration tests for verification functionality. * @@ -208,6 +238,107 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("verification (%s)", (backend: st olmSAS.free(); }); + oldBackendOnly( + "Outgoing verification: can verify another device via QR code with an untrusted cross-signing key", + async () => { + // expect requests to download our own keys + fetchMock.post(new RegExp("/_matrix/client/(r0|v3)/keys/query"), { + device_keys: { + [TEST_USER_ID]: { + [TEST_DEVICE_ID]: SIGNED_TEST_DEVICE_DATA, + }, + }, + ...SIGNED_CROSS_SIGNING_KEYS_DATA, + }); + + // QRCode fails if we don't yet have the cross-signing keys, so make sure we have them now. + // + // Completing the initial sync will make the device list download outdated device lists (of which our own + // user will be one). + syncResponder.sendOrQueueSyncResponse({}); + // DeviceList has a sleep(5) which we need to make happen + await jest.advanceTimersByTimeAsync(10); + expect(aliceClient.getStoredCrossSigningForUser(TEST_USER_ID)).toBeTruthy(); + + // have alice initiate a verification. She should send a m.key.verification.request + const [requestBody, request] = await Promise.all([ + expectSendToDeviceMessage("m.key.verification.request"), + aliceClient.requestVerification(TEST_USER_ID, [TEST_DEVICE_ID]), + ]); + const transactionId = request.channel.transactionId; + + const toDeviceMessage = requestBody.messages[TEST_USER_ID][TEST_DEVICE_ID]; + expect(toDeviceMessage.methods).toContain("m.qr_code.show.v1"); + expect(toDeviceMessage.methods).toContain("m.qr_code.scan.v1"); + expect(toDeviceMessage.methods).toContain("m.reciprocate.v1"); + expect(toDeviceMessage.from_device).toEqual(aliceClient.deviceId); + expect(toDeviceMessage.transaction_id).toEqual(transactionId); + + // The dummy device replies with an m.key.verification.ready, with an indication we can scan the QR code + returnToDeviceMessageFromSync({ + type: "m.key.verification.ready", + content: { + from_device: TEST_DEVICE_ID, + methods: ["m.qr_code.scan.v1"], + transaction_id: transactionId, + }, + }); + await waitForVerificationRequestChanged(request); + expect(request.phase).toEqual(Phase.Ready); + + // we should now have QR data we can display + const qrCodeData = request.qrCodeData!; + expect(qrCodeData).toBeTruthy(); + const qrCodeBuffer = qrCodeData.getBuffer(); + // https://spec.matrix.org/v1.7/client-server-api/#qr-code-format + expect(qrCodeBuffer.subarray(0, 6).toString("latin1")).toEqual("MATRIX"); + expect(qrCodeBuffer.readUint8(6)).toEqual(0x02); // version + expect(qrCodeBuffer.readUint8(7)).toEqual(0x02); // mode + const txnIdLen = qrCodeBuffer.readUint16BE(8); + expect(qrCodeBuffer.subarray(10, 10 + txnIdLen).toString("utf-8")).toEqual(transactionId); + // Alice's device's public key comes next, but we have nothing to do with it here. + // const aliceDevicePubKey = qrCodeBuffer.subarray(10 + txnIdLen, 32 + 10 + txnIdLen); + expect(qrCodeBuffer.subarray(42 + txnIdLen, 32 + 42 + txnIdLen)).toEqual( + Buffer.from(MASTER_CROSS_SIGNING_PUBLIC_KEY_BASE64, "base64"), + ); + const sharedSecret = qrCodeBuffer.subarray(74 + txnIdLen); + + // the dummy device "scans" the displayed QR code and acknowledges it with a "m.key.verification.start" + returnToDeviceMessageFromSync({ + type: "m.key.verification.start", + content: { + from_device: TEST_DEVICE_ID, + method: "m.reciprocate.v1", + transaction_id: transactionId, + secret: encodeUnpaddedBase64(sharedSecret), + }, + }); + await waitForVerificationRequestChanged(request); + expect(request.phase).toEqual(Phase.Started); + expect(request.chosenMethod).toEqual("m.reciprocate.v1"); + + // there should now be a verifier + const verifier: VerificationBase = request.verifier!; + expect(verifier).toBeDefined(); + + // ... which we call .verify on, which emits a ShowReciprocateQr event + const verificationPromise = verifier.verify(); + const reciprocateQRCodeCallbacks = await new Promise((resolve) => { + verifier.once(VerifierEvent.ShowReciprocateQr, resolve); + }); + + // Alice confirms she is happy + reciprocateQRCodeCallbacks.confirm(); + + // that should satisfy Alice, who should reply with a 'done' + await expectSendToDeviceMessage("m.key.verification.done"); + + // ... and the whole thing should be done! + await verificationPromise; + expect(request.phase).toEqual(Phase.Done); + }, + ); + function returnToDeviceMessageFromSync(ev: { type: string; content: object; sender?: string }): void { ev.sender ??= TEST_USER_ID; syncResponder.sendOrQueueSyncResponse({ to_device: { events: [ev] } }); @@ -253,3 +384,7 @@ function calculateMAC(olmSAS: Olm.SAS, input: string, info: string): string { //console.info(`Test MAC: input:'${input}, info: '${info}' -> '${mac}`); return mac; } + +function encodeUnpaddedBase64(uint8Array: ArrayBuffer | Uint8Array): string { + return Buffer.from(uint8Array).toString("base64").replace(/=+$/g, ""); +} diff --git a/spec/test-utils/test-data/generate-test-data.py b/spec/test-utils/test-data/generate-test-data.py index f5eae004e4a..3ba7dd4a8b9 100755 --- a/spec/test-utils/test-data/generate-test-data.py +++ b/spec/test-utils/test-data/generate-test-data.py @@ -37,6 +37,10 @@ # any 32-byte string can be an ed25519 private key. TEST_DEVICE_PRIVATE_KEY_BYTES = b"deadbeefdeadbeefdeadbeefdeadbeef" +MASTER_CROSS_SIGNING_PRIVATE_KEY_BYTES = b"doyouspeakwhaaaaaaaaaaaaaaaaaale" +USER_CROSS_SIGNING_PRIVATE_KEY_BYTES = b"useruseruseruseruseruseruseruser" +SELF_CROSS_SIGNING_PRIVATE_KEY_BYTES = b"selfselfselfselfselfselfselfself" + def main() -> None: private_key = ed25519.Ed25519PrivateKey.from_private_bytes( @@ -57,10 +61,17 @@ def main() -> None: "user_id": TEST_USER_ID, } - device_data["signatures"][TEST_USER_ID][ f"ed25519:{TEST_DEVICE_ID}"] = sign_json( + device_data["signatures"][TEST_USER_ID][f"ed25519:{TEST_DEVICE_ID}"] = sign_json( device_data, private_key ) + master_private_key = ed25519.Ed25519PrivateKey.from_private_bytes( + MASTER_CROSS_SIGNING_PRIVATE_KEY_BYTES + ) + b64_master_public_key = encode_base64( + master_private_key.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw) + ) + print( f"""\ /* Test data for cryptography tests @@ -69,6 +80,7 @@ def main() -> None: */ import {{ IDeviceKeys }} from "../../../src/@types/crypto"; +import {{ IDownloadKeyResult }} from "../../../src"; /* eslint-disable comma-dangle */ @@ -80,8 +92,82 @@ def main() -> None: /** Signed device data, suitable for returning from a `/keys/query` call */ export const SIGNED_TEST_DEVICE_DATA: IDeviceKeys = {json.dumps(device_data, indent=4)}; -""", end='', + +/** base64-encoded public master cross-signing key */ +export const MASTER_CROSS_SIGNING_PUBLIC_KEY_BASE64 = "{b64_master_public_key}"; + +/** Signed cross-signing keys data, also suitable for returning from a `/keys/query` call */ +export const SIGNED_CROSS_SIGNING_KEYS_DATA: Partial = { + json.dumps(build_cross_signing_keys_data(), indent=4) +}; +""", + end="", + ) + + +def build_cross_signing_keys_data() -> dict: + """Build the signed cross-signing-keys data for return from /keys/query""" + master_private_key = ed25519.Ed25519PrivateKey.from_private_bytes( + MASTER_CROSS_SIGNING_PRIVATE_KEY_BYTES ) + b64_master_public_key = encode_base64( + master_private_key.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw) + ) + self_signing_private_key = ed25519.Ed25519PrivateKey.from_private_bytes( + SELF_CROSS_SIGNING_PRIVATE_KEY_BYTES + ) + b64_self_signing_public_key = encode_base64( + self_signing_private_key.public_key().public_bytes( + Encoding.Raw, PublicFormat.Raw + ) + ) + user_signing_private_key = ed25519.Ed25519PrivateKey.from_private_bytes( + USER_CROSS_SIGNING_PRIVATE_KEY_BYTES + ) + b64_user_signing_public_key = encode_base64( + user_signing_private_key.public_key().public_bytes( + Encoding.Raw, PublicFormat.Raw + ) + ) + # create without signatures initially + cross_signing_keys_data = { + "master_keys": { + TEST_USER_ID: { + "keys": { + f"ed25519:{b64_master_public_key}": b64_master_public_key, + }, + "user_id": TEST_USER_ID, + "usage": ["master"], + } + }, + "self_signing_keys": { + TEST_USER_ID: { + "keys": { + f"ed25519:{b64_self_signing_public_key}": b64_self_signing_public_key, + }, + "user_id": TEST_USER_ID, + "usage": ["self_signing"], + }, + }, + "user_signing_keys": { + TEST_USER_ID: { + "keys": { + f"ed25519:{b64_user_signing_public_key}": b64_user_signing_public_key, + }, + "user_id": TEST_USER_ID, + "usage": ["user_signing"], + }, + }, + } + # sign the sub-keys with the master + for k in ["self_signing_keys", "user_signing_keys"]: + to_sign = cross_signing_keys_data[k][TEST_USER_ID] + sig = sign_json(to_sign, master_private_key) + to_sign["signatures"] = { + TEST_USER_ID: {f"ed25519:{b64_master_public_key}": sig} + } + + return cross_signing_keys_data def encode_base64(input_bytes: bytes) -> str: diff --git a/spec/test-utils/test-data/index.ts b/spec/test-utils/test-data/index.ts index fbb9a1c2bc7..07ff1222526 100644 --- a/spec/test-utils/test-data/index.ts +++ b/spec/test-utils/test-data/index.ts @@ -4,6 +4,7 @@ */ import { IDeviceKeys } from "../../../src/@types/crypto"; +import { IDownloadKeyResult } from "../../../src"; /* eslint-disable comma-dangle */ @@ -31,3 +32,53 @@ export const SIGNED_TEST_DEVICE_DATA: IDeviceKeys = { } } }; + +/** base64-encoded public master cross-signing key */ +export const MASTER_CROSS_SIGNING_PUBLIC_KEY_BASE64 = "J+5An10v1vzZpAXTYFokD1/PEVccFnLC61EfRXit0UY"; + +/** Signed cross-signing keys data, also suitable for returning from a `/keys/query` call */ +export const SIGNED_CROSS_SIGNING_KEYS_DATA: Partial = { + "master_keys": { + "@alice:localhost": { + "keys": { + "ed25519:J+5An10v1vzZpAXTYFokD1/PEVccFnLC61EfRXit0UY": "J+5An10v1vzZpAXTYFokD1/PEVccFnLC61EfRXit0UY" + }, + "user_id": "@alice:localhost", + "usage": [ + "master" + ] + } + }, + "self_signing_keys": { + "@alice:localhost": { + "keys": { + "ed25519:aU2+2CyXQTCuDcmWW0EL2bhJ6PdjFW2LbAsbHqf02AY": "aU2+2CyXQTCuDcmWW0EL2bhJ6PdjFW2LbAsbHqf02AY" + }, + "user_id": "@alice:localhost", + "usage": [ + "self_signing" + ], + "signatures": { + "@alice:localhost": { + "ed25519:J+5An10v1vzZpAXTYFokD1/PEVccFnLC61EfRXit0UY": "XfhYEhZmOs8BJdb3viatILBZ/bElsHXEW28V4tIaY5CxrBR0YOym3yZHWmRmypXessHZAKOhZn3yBMXzdajyCw" + } + } + } + }, + "user_signing_keys": { + "@alice:localhost": { + "keys": { + "ed25519:g5TC/zjQXyZYuDLZv7a41z5fFVrXpYPypG//AFQj8hY": "g5TC/zjQXyZYuDLZv7a41z5fFVrXpYPypG//AFQj8hY" + }, + "user_id": "@alice:localhost", + "usage": [ + "user_signing" + ], + "signatures": { + "@alice:localhost": { + "ed25519:J+5An10v1vzZpAXTYFokD1/PEVccFnLC61EfRXit0UY": "6AkD1XM2H0/ebgP9oBdMKNeft7uxsrb0XN1CsjjHgeZCvCTMmv3BHlLiT/Hzy4fe8H+S1tr484dcXN/PIdnfDA" + } + } + } + } +}; From 9afad108eff9d90d7a7faa29104db72583261591 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 6 Jun 2023 12:16:19 +0100 Subject: [PATCH 26/52] Add `getShowSasCallbacks`, `getShowQrCodeCallbacks` to VerifierBase (#3422) * Add `getShowSasCallbacks`, `getShowQrCodeCallbacks` to VerifierBase ... to avoid some type-casting * Integration test for QR code verification Followup to https://github.com/matrix-org/matrix-js-sdk/pull/3436: another integration test, this time using the QR code flow * Rename method ... it turns out not to be used quite as I thought. * tests for new methods * Use Object.defineProperty, and restore afterwards Apparently global.crypto exists in some environments * apply ts-ignore * More test coverage * fix bad merge --- spec/integ/crypto/verification.spec.ts | 10 ++++++++++ src/crypto/verification/Base.ts | 27 +++++++++++++++++++++++++- src/crypto/verification/QRCode.ts | 4 ++++ src/crypto/verification/SAS.ts | 4 ++++ 4 files changed, 44 insertions(+), 1 deletion(-) diff --git a/spec/integ/crypto/verification.spec.ts b/spec/integ/crypto/verification.spec.ts index fdcc4bf1a4c..f4690294515 100644 --- a/spec/integ/crypto/verification.spec.ts +++ b/spec/integ/crypto/verification.spec.ts @@ -170,6 +170,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("verification (%s)", (backend: st // there should now be a verifier const verifier: VerificationBase = request.verifier!; expect(verifier).toBeDefined(); + expect(verifier.getShowSasCallbacks()).toBeNull(); // start off the verification process: alice will send an `accept` const verificationPromise = verifier.verify(); @@ -205,6 +206,10 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("verification (%s)", (backend: st verifier.once(VerifierEvent.ShowSas, resolve); }); + // `getShowSasCallbacks` is an alternative way to get the callbacks + expect(verifier.getShowSasCallbacks()).toBe(showSas); + expect(verifier.getReciprocateQrCodeCallbacks()).toBeNull(); + // user confirms that the emoji match, and alice sends a 'mac' [requestBody] = await Promise.all([expectSendToDeviceMessage("m.key.verification.mac"), showSas.confirm()]); toDeviceMessage = requestBody.messages[TEST_USER_ID][TEST_DEVICE_ID]; @@ -320,6 +325,7 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("verification (%s)", (backend: st // there should now be a verifier const verifier: VerificationBase = request.verifier!; expect(verifier).toBeDefined(); + expect(verifier.getReciprocateQrCodeCallbacks()).toBeNull(); // ... which we call .verify on, which emits a ShowReciprocateQr event const verificationPromise = verifier.verify(); @@ -327,6 +333,10 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("verification (%s)", (backend: st verifier.once(VerifierEvent.ShowReciprocateQr, resolve); }); + // getReciprocateQrCodeCallbacks() is an alternative way to get the callbacks + expect(verifier.getReciprocateQrCodeCallbacks()).toBe(reciprocateQRCodeCallbacks); + expect(verifier.getShowSasCallbacks()).toBeNull(); + // Alice confirms she is happy reciprocateQRCodeCallbacks.confirm(); diff --git a/src/crypto/verification/Base.ts b/src/crypto/verification/Base.ts index 89bead3aa3e..de6a0f836b0 100644 --- a/src/crypto/verification/Base.ts +++ b/src/crypto/verification/Base.ts @@ -29,7 +29,12 @@ import { IVerificationChannel } from "./request/Channel"; import { MatrixClient } from "../../client"; import { VerificationRequest } from "./request/VerificationRequest"; import { TypedEventEmitter } from "../../models/typed-event-emitter"; -import { VerifierEvent, VerifierEventHandlerMap } from "../../crypto-api/verification"; +import { + ShowQrCodeCallbacks, + ShowSasCallbacks, + VerifierEvent, + VerifierEventHandlerMap, +} from "../../crypto-api/verification"; const timeoutException = new Error("Verification timed out"); @@ -373,4 +378,24 @@ export class VerificationBase< public get events(): string[] | undefined { return undefined; } + + /** + * Get the details for an SAS verification, if one is in progress + * + * Returns `null`, unless this verifier is for a SAS-based verification and we are waiting for the user to confirm + * the SAS matches. + */ + public getShowSasCallbacks(): ShowSasCallbacks | null { + return null; + } + + /** + * Get the details for reciprocating QR code verification, if one is in progress + * + * Returns `null`, unless this verifier is for reciprocating a QR-code-based verification (ie, the other user has + * already scanned our QR code), and we are waiting for the user to confirm. + */ + public getReciprocateQrCodeCallbacks(): ShowQrCodeCallbacks | null { + return null; + } } diff --git a/src/crypto/verification/QRCode.ts b/src/crypto/verification/QRCode.ts index 38feade12db..a51db2e34e6 100644 --- a/src/crypto/verification/QRCode.ts +++ b/src/crypto/verification/QRCode.ts @@ -119,6 +119,10 @@ export class ReciprocateQRCode extends Base { } }); }; + + public getReciprocateQrCodeCallbacks(): ShowQrCodeCallbacks | null { + return this.reciprocateQREvent ?? null; + } } const CODE_VERSION = 0x02; // the version of binary QR codes we support diff --git a/src/crypto/verification/SAS.ts b/src/crypto/verification/SAS.ts index 61f096ca452..f15adf585f0 100644 --- a/src/crypto/verification/SAS.ts +++ b/src/crypto/verification/SAS.ts @@ -480,4 +480,8 @@ export class SAS extends Base { } }); } + + public getShowSasCallbacks(): ShowSasCallbacks | null { + return this.sasEvent ?? null; + } } From 2c94bae78559ffa5b66bdb8b82540703f1d7821f Mon Sep 17 00:00:00 2001 From: ElementRobot Date: Tue, 16 May 2023 13:36:40 +0100 Subject: [PATCH 27/52] Fix changelog_head.py script to be Python 3 compatible --- scripts/changelog_head.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/changelog_head.py b/scripts/changelog_head.py index c7c1c0aeaa5..dfe54b466d4 100755 --- a/scripts/changelog_head.py +++ b/scripts/changelog_head.py @@ -15,4 +15,4 @@ break found_first_header = True elif not re.match(r"^=+$", line) and len(line) > 0: - print line + print(line) From 8bab6cd7f07e80870b7414d70c196a43ae98c16f Mon Sep 17 00:00:00 2001 From: ElementRobot Date: Tue, 16 May 2023 13:37:17 +0100 Subject: [PATCH 28/52] Prepare changelog for v25.2.0-rc.1 --- CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e8005168f1..c013129e338 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,19 @@ +Changes in [25.2.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v25.2.0-rc.1) (2023-05-16) +============================================================================================================ + +## 🦖 Deprecations + * Deprecate device methods in MatrixClient ([\#3357](https://github.com/matrix-org/matrix-js-sdk/pull/3357)). + +## ✨ Features + * Total summary count ([\#3351](https://github.com/matrix-org/matrix-js-sdk/pull/3351)). Contributed by @toger5. + * Audio concealment ([\#3349](https://github.com/matrix-org/matrix-js-sdk/pull/3349)). Contributed by @toger5. + +## 🐛 Bug Fixes + * Correctly accumulate sync summaries. ([\#3366](https://github.com/matrix-org/matrix-js-sdk/pull/3366)). Fixes vector-im/element-web#23345. + * Keep measuring a call feed's volume after a stream replacement ([\#3361](https://github.com/matrix-org/matrix-js-sdk/pull/3361)). Fixes vector-im/element-call#1051. + * Element-R: Avoid uploading a new fallback key at every `/sync` ([\#3338](https://github.com/matrix-org/matrix-js-sdk/pull/3338)). Fixes vector-im/element-web#25215. + * Accumulate receipts for the main thread and unthreaded separately ([\#3339](https://github.com/matrix-org/matrix-js-sdk/pull/3339)). Fixes vector-im/element-web#24629. + Changes in [25.1.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v25.1.1) (2023-05-16) ================================================================================================== From 1aeaba53488499ff07e7d0b25f5796e8c5602bf6 Mon Sep 17 00:00:00 2001 From: ElementRobot Date: Tue, 16 May 2023 13:37:19 +0100 Subject: [PATCH 29/52] v25.2.0-rc.1 --- package.json | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index ea5282525b4..001ad3dd277 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "matrix-js-sdk", - "version": "25.1.1", + "version": "25.2.0-rc.1", "description": "Matrix Client-Server SDK for Javascript", "engines": { "node": ">=16.0.0" @@ -32,8 +32,8 @@ "keywords": [ "matrix-org" ], - "main": "./src/index.ts", - "browser": "./src/browser-index.ts", + "main": "./lib/index.js", + "browser": "./lib/browser-index.js", "matrix_src_main": "./src/index.ts", "matrix_src_browser": "./src/browser-index.ts", "matrix_lib_main": "./lib/index.js", @@ -156,5 +156,6 @@ "no-rust-crypto": { "src/rust-crypto/index.ts$": "./src/rust-crypto/browserify-index.ts" } - } + }, + "typings": "./lib/index.d.ts" } From b0a73fc85203a767fcae9ca2ec0dd0bd37bc5648 Mon Sep 17 00:00:00 2001 From: ElementRobot Date: Tue, 16 May 2023 13:58:05 +0100 Subject: [PATCH 30/52] Fix tsconfig-build.json --- tsconfig-build.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tsconfig-build.json b/tsconfig-build.json index 3108314a4d6..5437c65b2db 100644 --- a/tsconfig-build.json +++ b/tsconfig-build.json @@ -1,6 +1,7 @@ { "extends": "./tsconfig.json", "compilerOptions": { + "forceConsistentCasingInFileNames": true, "declarationMap": true, "sourceMap": true, "noEmit": false, From ec3fd91f73d6890ab78483e24c35206f2aaa5559 Mon Sep 17 00:00:00 2001 From: ElementRobot Date: Tue, 16 May 2023 13:58:33 +0100 Subject: [PATCH 31/52] Prepare changelog for v25.2.0-rc.2 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c013129e338..d55e6bdaad3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -Changes in [25.2.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v25.2.0-rc.1) (2023-05-16) +Changes in [25.2.0-rc.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v25.2.0-rc.2) (2023-05-16) ============================================================================================================ ## 🦖 Deprecations From 1c67b386806553be40486a996f24921b1335c250 Mon Sep 17 00:00:00 2001 From: ElementRobot Date: Tue, 16 May 2023 13:58:35 +0100 Subject: [PATCH 32/52] v25.2.0-rc.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 001ad3dd277..d7b9b44db19 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "matrix-js-sdk", - "version": "25.2.0-rc.1", + "version": "25.2.0-rc.2", "description": "Matrix Client-Server SDK for Javascript", "engines": { "node": ">=16.0.0" From a3c1656b355a62649e7aa6d3661fe69740124627 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Tue, 16 May 2023 14:15:27 +0100 Subject: [PATCH 33/52] Fix docs deployment --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8bf9a6489fa..0273854cf29 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,7 +32,7 @@ jobs: - name: 📖 Generate docs run: | - yarn tpv --yes --out _docs --stale --major 10 + yarn tpv purge --yes --out _docs --stale --major 10 yarn gendoc symlinks -rc _docs From 6969446f687163f3b47e4c6ea047674d2fcfbd50 Mon Sep 17 00:00:00 2001 From: ElementRobot Date: Tue, 16 May 2023 14:17:02 +0100 Subject: [PATCH 34/52] Prepare changelog for v25.2.0-rc.3 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d55e6bdaad3..c126333a6a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -Changes in [25.2.0-rc.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v25.2.0-rc.2) (2023-05-16) +Changes in [25.2.0-rc.3](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v25.2.0-rc.3) (2023-05-16) ============================================================================================================ ## 🦖 Deprecations From 9a795545cec817a5a61122d9233916e229f893aa Mon Sep 17 00:00:00 2001 From: ElementRobot Date: Tue, 16 May 2023 14:17:04 +0100 Subject: [PATCH 35/52] v25.2.0-rc.3 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d7b9b44db19..fdcb4260160 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "matrix-js-sdk", - "version": "25.2.0-rc.2", + "version": "25.2.0-rc.3", "description": "Matrix Client-Server SDK for Javascript", "engines": { "node": ">=16.0.0" From 6ad2cc32be39c5f26663df4815e35d63118ef57f Mon Sep 17 00:00:00 2001 From: ElementRobot Date: Tue, 16 May 2023 14:22:02 +0100 Subject: [PATCH 36/52] Prepare changelog for v25.2.0-rc.4 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c126333a6a4..22c776ccad4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -Changes in [25.2.0-rc.3](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v25.2.0-rc.3) (2023-05-16) +Changes in [25.2.0-rc.4](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v25.2.0-rc.4) (2023-05-16) ============================================================================================================ ## 🦖 Deprecations From 4c250b5ae1e093a721b0c223cf106782da81b301 Mon Sep 17 00:00:00 2001 From: ElementRobot Date: Tue, 16 May 2023 14:22:04 +0100 Subject: [PATCH 37/52] v25.2.0-rc.4 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index fdcb4260160..b9176c39b85 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "matrix-js-sdk", - "version": "25.2.0-rc.3", + "version": "25.2.0-rc.4", "description": "Matrix Client-Server SDK for Javascript", "engines": { "node": ">=16.0.0" From a8b327e5a42dd7eb26f84fd6e41e0a379d0bde1a Mon Sep 17 00:00:00 2001 From: ElementRobot Date: Fri, 19 May 2023 15:57:55 +0100 Subject: [PATCH 38/52] [Backport staging] Attempt a potential workaround for stuck notifs (#3387) Co-authored-by: Andy Balaam --- src/models/event-timeline.ts | 2 +- src/models/thread.ts | 23 +++++++++++------------ 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/models/event-timeline.ts b/src/models/event-timeline.ts index a20dac880e4..67be25fbc9c 100644 --- a/src/models/event-timeline.ts +++ b/src/models/event-timeline.ts @@ -458,7 +458,7 @@ export class EventTimeline { // member event, whereas we want to set the .sender value for the ACTUAL // member event itself. if (!event.sender || event.getType() === EventType.RoomMember) { - EventTimeline.setEventMetadata(event, roomState, false); + EventTimeline.setEventMetadata(event, roomState!, false); } } } diff --git a/src/models/thread.ts b/src/models/thread.ts index 33bd44ed8c7..d7569bfbba6 100644 --- a/src/models/thread.ts +++ b/src/models/thread.ts @@ -281,7 +281,6 @@ export class Thread extends ReadReceipt { + if (relations.events.length) { + event.makeReplaced(relations.events[0]); + this.insertEventIntoTimeline(event); + } + }) + .catch((e) => { + logger.error("Failed to load edits for encrypted thread event", e); + }); + }), ); } } From e255baba1bdfe922f2d52067276ea2615c19c9c7 Mon Sep 17 00:00:00 2001 From: ElementRobot Date: Fri, 19 May 2023 16:35:32 +0100 Subject: [PATCH 39/52] Prepare changelog for v25.2.0-rc.5 --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22c776ccad4..0d15a4cf7b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +Changes in [25.2.0-rc.5](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v25.2.0-rc.5) (2023-05-19) +============================================================================================================ + +## 🐛 Bug Fixes + * Attempt a potential workaround for stuck notifs ([\#3384](https://github.com/matrix-org/matrix-js-sdk/pull/3384)). Fixes vector-im/element-web#25406. Contributed by @andybalaam. + Changes in [25.2.0-rc.4](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v25.2.0-rc.4) (2023-05-16) ============================================================================================================ From bf107ac9b0033a78026e2f2a66541d3b7f6af4f4 Mon Sep 17 00:00:00 2001 From: ElementRobot Date: Fri, 19 May 2023 16:35:34 +0100 Subject: [PATCH 40/52] v25.2.0-rc.5 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b9176c39b85..4243fde6ee2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "matrix-js-sdk", - "version": "25.2.0-rc.4", + "version": "25.2.0-rc.5", "description": "Matrix Client-Server SDK for Javascript", "engines": { "node": ">=16.0.0" From e53368a9453ba75246acd97bcb054a5e5cc10c90 Mon Sep 17 00:00:00 2001 From: ElementRobot Date: Wed, 24 May 2023 09:02:18 +0100 Subject: [PATCH 41/52] [Backport staging] Fix mark as unread button (#3401) Co-authored-by: Michael Weimann --- spec/unit/room.spec.ts | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/spec/unit/room.spec.ts b/spec/unit/room.spec.ts index 2ed44dffd7b..add515e1936 100644 --- a/spec/unit/room.spec.ts +++ b/spec/unit/room.spec.ts @@ -172,9 +172,9 @@ describe("Room", function () { * @param timestamp - Timestamp of the message * @return The message event */ - const mkMessageInRoom = async (room: Room, timestamp: number) => { + const mkMessageInRoom = (room: Room, timestamp: number) => { const message = mkMessage({ ts: timestamp }); - await room.addLiveEvents([message]); + room.addLiveEvents([message]); return message; }; @@ -3528,8 +3528,8 @@ describe("Room", function () { expect(room.getLastLiveEvent()).toBeUndefined(); }); - it("when there is only an event in the main timeline and there are no threads, it should return the last event from the main timeline", async () => { - const lastEventInMainTimeline = await mkMessageInRoom(room, 23); + it("when there is only an event in the main timeline and there are no threads, it should return the last event from the main timeline", () => { + const lastEventInMainTimeline = mkMessageInRoom(room, 23); expect(room.getLastLiveEvent()).toBe(lastEventInMainTimeline); }); @@ -3544,29 +3544,29 @@ describe("Room", function () { }); describe("when there are events in both, the main timeline and threads", () => { - it("and the last event is in a thread, it should return the last event from the thread", async () => { - await mkMessageInRoom(room, 23); + it("and the last event is in a thread, it should return the last event from the thread", () => { + mkMessageInRoom(room, 23); const { thread } = mkThread({ room, length: 0 }); const lastEventInThread = mkMessageInThread(thread, 42); expect(room.getLastLiveEvent()).toBe(lastEventInThread); }); - it("and the last event is in the main timeline, it should return the last event from the main timeline", async () => { - const lastEventInMainTimeline = await mkMessageInRoom(room, 42); + it("and the last event is in the main timeline, it should return the last event from the main timeline", () => { + const lastEventInMainTimeline = mkMessageInRoom(room, 42); const { thread } = mkThread({ room, length: 0 }); mkMessageInThread(thread, 23); expect(room.getLastLiveEvent()).toBe(lastEventInMainTimeline); }); - it("and both events have the same timestamp, it should return the last event from the thread", async () => { - await mkMessageInRoom(room, 23); + it("and both events have the same timestamp, it should return the last event from the thread", () => { + mkMessageInRoom(room, 23); const { thread } = mkThread({ room, length: 0 }); const lastEventInThread = mkMessageInThread(thread, 23); expect(room.getLastLiveEvent()).toBe(lastEventInThread); }); - it("and there is a thread without any messages, it should return the last event from the main timeline", async () => { - const lastEventInMainTimeline = await mkMessageInRoom(room, 23); + it("and there is a thread without any messages, it should return the last event from the main timeline", () => { + const lastEventInMainTimeline = mkMessageInRoom(room, 23); mkThread({ room, length: 0 }); expect(room.getLastLiveEvent()).toBe(lastEventInMainTimeline); }); From 29b988951217b0082849829ffdbabac79f20a747 Mon Sep 17 00:00:00 2001 From: ElementRobot Date: Thu, 1 Jun 2023 16:54:22 +0100 Subject: [PATCH 42/52] Prepare changelog for v26.0.0-rc.1 --- CHANGELOG.md | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d15a4cf7b6..419960a769b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,16 +1,18 @@ -Changes in [25.2.0-rc.5](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v25.2.0-rc.5) (2023-05-19) +Changes in [26.0.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v26.0.0-rc.1) (2023-06-01) ============================================================================================================ -## 🐛 Bug Fixes - * Attempt a potential workaround for stuck notifs ([\#3384](https://github.com/matrix-org/matrix-js-sdk/pull/3384)). Fixes vector-im/element-web#25406. Contributed by @andybalaam. - -Changes in [25.2.0-rc.4](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v25.2.0-rc.4) (2023-05-16) -============================================================================================================ +## 🚨 BREAKING CHANGES + * Ensure we do not add relations to the wrong timeline ([\#3427](https://github.com/matrix-org/matrix-js-sdk/pull/3427)). Fixes vector-im/element-web#25450 and vector-im/element-web#25494. + * Deprecate `QrCodeEvent`, `SasEvent` and `VerificationEvent` ([\#3386](https://github.com/matrix-org/matrix-js-sdk/pull/3386)). ## 🦖 Deprecations - * Deprecate device methods in MatrixClient ([\#3357](https://github.com/matrix-org/matrix-js-sdk/pull/3357)). + * Move crypto classes into a separate namespace ([\#3385](https://github.com/matrix-org/matrix-js-sdk/pull/3385)). ## ✨ Features + * Mention deno support in the README ([\#3417](https://github.com/matrix-org/matrix-js-sdk/pull/3417)). Contributed by @sigmaSd. + * Mark room version 10 as safe ([\#3425](https://github.com/matrix-org/matrix-js-sdk/pull/3425)). + * Prioritise entirely supported flows for UIA ([\#3402](https://github.com/matrix-org/matrix-js-sdk/pull/3402)). + * Add methods to terminate idb worker ([\#3362](https://github.com/matrix-org/matrix-js-sdk/pull/3362)). * Total summary count ([\#3351](https://github.com/matrix-org/matrix-js-sdk/pull/3351)). Contributed by @toger5. * Audio concealment ([\#3349](https://github.com/matrix-org/matrix-js-sdk/pull/3349)). Contributed by @toger5. @@ -19,6 +21,11 @@ Changes in [25.2.0-rc.4](https://github.com/matrix-org/matrix-js-sdk/releases/ta * Keep measuring a call feed's volume after a stream replacement ([\#3361](https://github.com/matrix-org/matrix-js-sdk/pull/3361)). Fixes vector-im/element-call#1051. * Element-R: Avoid uploading a new fallback key at every `/sync` ([\#3338](https://github.com/matrix-org/matrix-js-sdk/pull/3338)). Fixes vector-im/element-web#25215. * Accumulate receipts for the main thread and unthreaded separately ([\#3339](https://github.com/matrix-org/matrix-js-sdk/pull/3339)). Fixes vector-im/element-web#24629. + * Remove spec non-compliant extended glob format ([\#3423](https://github.com/matrix-org/matrix-js-sdk/pull/3423)). Fixes vector-im/element-web#25474. + * Fix bug where original event was inserted into timeline instead of the edit event ([\#3398](https://github.com/matrix-org/matrix-js-sdk/pull/3398)). Contributed by @andybalaam. + * Only add a local receipt if it's after an existing receipt ([\#3399](https://github.com/matrix-org/matrix-js-sdk/pull/3399)). Contributed by @andybalaam. + * Attempt a potential workaround for stuck notifs ([\#3384](https://github.com/matrix-org/matrix-js-sdk/pull/3384)). Fixes vector-im/element-web#25406. Contributed by @andybalaam. + * Fix verification bug with `pendingEventOrdering: "chronological"` ([\#3382](https://github.com/matrix-org/matrix-js-sdk/pull/3382)). Changes in [25.1.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v25.1.1) (2023-05-16) ================================================================================================== From 144edca99a5f0eb8f407ad5b32da4d376f9fc302 Mon Sep 17 00:00:00 2001 From: ElementRobot Date: Thu, 1 Jun 2023 16:54:25 +0100 Subject: [PATCH 43/52] v26.0.0-rc.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4243fde6ee2..1b53fd6a386 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "matrix-js-sdk", - "version": "25.2.0-rc.5", + "version": "26.0.0-rc.1", "description": "Matrix Client-Server SDK for Javascript", "engines": { "node": ">=16.0.0" From e7a02b04a72379580f2dd233eab6b9303139ea63 Mon Sep 17 00:00:00 2001 From: RiotRobot Date: Tue, 6 Jun 2023 13:44:03 +0100 Subject: [PATCH 44/52] Prepare changelog for v26.0.0 --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 419960a769b..c538f868e8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ -Changes in [26.0.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v26.0.0-rc.1) (2023-06-01) -============================================================================================================ +Changes in [26.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v26.0.0) (2023-06-06) +================================================================================================== ## 🚨 BREAKING CHANGES * Ensure we do not add relations to the wrong timeline ([\#3427](https://github.com/matrix-org/matrix-js-sdk/pull/3427)). Fixes vector-im/element-web#25450 and vector-im/element-web#25494. From 3e1b3d92aca288b885defdf67f1fe9cf65cf30de Mon Sep 17 00:00:00 2001 From: RiotRobot Date: Tue, 6 Jun 2023 13:44:07 +0100 Subject: [PATCH 45/52] v26.0.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1b53fd6a386..0b8aa4751f8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "matrix-js-sdk", - "version": "26.0.0-rc.1", + "version": "26.0.0", "description": "Matrix Client-Server SDK for Javascript", "engines": { "node": ">=16.0.0" From 4c219cf95e4be224d964b84bf7b6e65028cf22ff Mon Sep 17 00:00:00 2001 From: RiotRobot Date: Tue, 6 Jun 2023 13:45:40 +0100 Subject: [PATCH 46/52] Resetting package fields for development --- package.json | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 0b8aa4751f8..04097307a53 100644 --- a/package.json +++ b/package.json @@ -32,8 +32,8 @@ "keywords": [ "matrix-org" ], - "main": "./lib/index.js", - "browser": "./lib/browser-index.js", + "main": "./src/index.ts", + "browser": "./src/browser-index.ts", "matrix_src_main": "./src/index.ts", "matrix_src_browser": "./src/browser-index.ts", "matrix_lib_main": "./lib/index.js", @@ -156,6 +156,5 @@ "no-rust-crypto": { "src/rust-crypto/index.ts$": "./src/rust-crypto/browserify-index.ts" } - }, - "typings": "./lib/index.d.ts" + } } From accd4a53c3b6d4e1096672c423d8088ec40f630e Mon Sep 17 00:00:00 2001 From: Kerry Date: Wed, 7 Jun 2023 09:52:55 +1200 Subject: [PATCH 47/52] use cli.canSupport to determine intentional mentions support (#3445) * use cli.canSupport to determine intentional mentions support * more specific comment * Update src/client.ts Co-authored-by: Michael Telatynski <7t3chguy@gmail.com> --------- Co-authored-by: Michael Telatynski <7t3chguy@gmail.com> --- src/client.ts | 9 ++------- src/feature.ts | 4 ++++ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/client.ts b/src/client.ts index caca1a89a82..3877cb713e8 100644 --- a/src/client.ts +++ b/src/client.ts @@ -469,11 +469,6 @@ export interface IStartClientOpts { * @experimental */ slidingSync?: SlidingSync; - - /** - * @experimental - */ - intentionalMentions?: boolean; } export interface IStoredClientOpts extends IStartClientOpts {} @@ -9635,11 +9630,11 @@ export class MatrixClient extends TypedEventEmitter = { [Feature.RelationsRecursion]: { unstablePrefixes: ["org.matrix.msc3981"], }, + [Feature.IntentionalMentions]: { + unstablePrefixes: ["org.matrix.msc3952_intentional_mentions"], + }, }; export async function buildFeatureSupportMap(versions: IServerVersions): Promise> { From 0fcfe9ce2e3338e1a65e878a93091daa076483bc Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 7 Jun 2023 11:56:48 +0200 Subject: [PATCH 48/52] git fixup Signed-off-by: Timo K --- spec/unit/room.spec.ts | 24 ++++++++++++------------ src/models/event-timeline.ts | 2 +- src/models/thread.ts | 23 ++++++++++++----------- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/spec/unit/room.spec.ts b/spec/unit/room.spec.ts index add515e1936..2ed44dffd7b 100644 --- a/spec/unit/room.spec.ts +++ b/spec/unit/room.spec.ts @@ -172,9 +172,9 @@ describe("Room", function () { * @param timestamp - Timestamp of the message * @return The message event */ - const mkMessageInRoom = (room: Room, timestamp: number) => { + const mkMessageInRoom = async (room: Room, timestamp: number) => { const message = mkMessage({ ts: timestamp }); - room.addLiveEvents([message]); + await room.addLiveEvents([message]); return message; }; @@ -3528,8 +3528,8 @@ describe("Room", function () { expect(room.getLastLiveEvent()).toBeUndefined(); }); - it("when there is only an event in the main timeline and there are no threads, it should return the last event from the main timeline", () => { - const lastEventInMainTimeline = mkMessageInRoom(room, 23); + it("when there is only an event in the main timeline and there are no threads, it should return the last event from the main timeline", async () => { + const lastEventInMainTimeline = await mkMessageInRoom(room, 23); expect(room.getLastLiveEvent()).toBe(lastEventInMainTimeline); }); @@ -3544,29 +3544,29 @@ describe("Room", function () { }); describe("when there are events in both, the main timeline and threads", () => { - it("and the last event is in a thread, it should return the last event from the thread", () => { - mkMessageInRoom(room, 23); + it("and the last event is in a thread, it should return the last event from the thread", async () => { + await mkMessageInRoom(room, 23); const { thread } = mkThread({ room, length: 0 }); const lastEventInThread = mkMessageInThread(thread, 42); expect(room.getLastLiveEvent()).toBe(lastEventInThread); }); - it("and the last event is in the main timeline, it should return the last event from the main timeline", () => { - const lastEventInMainTimeline = mkMessageInRoom(room, 42); + it("and the last event is in the main timeline, it should return the last event from the main timeline", async () => { + const lastEventInMainTimeline = await mkMessageInRoom(room, 42); const { thread } = mkThread({ room, length: 0 }); mkMessageInThread(thread, 23); expect(room.getLastLiveEvent()).toBe(lastEventInMainTimeline); }); - it("and both events have the same timestamp, it should return the last event from the thread", () => { - mkMessageInRoom(room, 23); + it("and both events have the same timestamp, it should return the last event from the thread", async () => { + await mkMessageInRoom(room, 23); const { thread } = mkThread({ room, length: 0 }); const lastEventInThread = mkMessageInThread(thread, 23); expect(room.getLastLiveEvent()).toBe(lastEventInThread); }); - it("and there is a thread without any messages, it should return the last event from the main timeline", () => { - const lastEventInMainTimeline = mkMessageInRoom(room, 23); + it("and there is a thread without any messages, it should return the last event from the main timeline", async () => { + const lastEventInMainTimeline = await mkMessageInRoom(room, 23); mkThread({ room, length: 0 }); expect(room.getLastLiveEvent()).toBe(lastEventInMainTimeline); }); diff --git a/src/models/event-timeline.ts b/src/models/event-timeline.ts index 67be25fbc9c..a20dac880e4 100644 --- a/src/models/event-timeline.ts +++ b/src/models/event-timeline.ts @@ -458,7 +458,7 @@ export class EventTimeline { // member event, whereas we want to set the .sender value for the ACTUAL // member event itself. if (!event.sender || event.getType() === EventType.RoomMember) { - EventTimeline.setEventMetadata(event, roomState!, false); + EventTimeline.setEventMetadata(event, roomState, false); } } } diff --git a/src/models/thread.ts b/src/models/thread.ts index d7569bfbba6..33bd44ed8c7 100644 --- a/src/models/thread.ts +++ b/src/models/thread.ts @@ -281,6 +281,7 @@ export class Thread extends ReadReceipt { - if (relations.events.length) { - event.makeReplaced(relations.events[0]); - this.insertEventIntoTimeline(event); - } - }) - .catch((e) => { - logger.error("Failed to load edits for encrypted thread event", e); - }); - }), + }, + ); + if (relations.events.length) { + const editEvent = relations.events[0]; + event.makeReplaced(editEvent); + this.insertEventIntoTimeline(editEvent); + } + } catch (e) { + logger.error("Failed to load edits for encrypted thread event", e); + } + }), ); } } From 4c09840b7f729f7e0181d64088668c3f2bd32769 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 7 Jun 2023 12:05:55 +0200 Subject: [PATCH 49/52] import updates Signed-off-by: Timo K --- spec/unit/webrtc/stats/summaryStatsReportGatherer.spec.ts | 2 +- src/webrtc/groupCall.ts | 2 +- src/webrtc/stats/groupCallStats.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/unit/webrtc/stats/summaryStatsReportGatherer.spec.ts b/spec/unit/webrtc/stats/summaryStatsReportGatherer.spec.ts index 2cf2460a3b9..aca482e219c 100644 --- a/spec/unit/webrtc/stats/summaryStatsReportGatherer.spec.ts +++ b/spec/unit/webrtc/stats/summaryStatsReportGatherer.spec.ts @@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -import { SummaryStatsReportGatherer } from "../../../../src/webrtc/stats/SummaryStatsReportGatherer"; +import { SummaryStatsReportGatherer } from "../../../../src/webrtc/stats/summaryStatsReportGatherer"; import { StatsReportEmitter } from "../../../../src/webrtc/stats/statsReportEmitter"; import { groupCallParticipantsFourOtherDevices } from "../../../test-utils/webrtc"; diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index eb776d5b93f..d6f7b4f1ecf 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -26,7 +26,7 @@ import { IScreensharingOpts } from "./mediaHandler"; import { mapsEqual } from "../utils"; import { GroupCallStats } from "./stats/groupCallStats"; import { ByteSentStatsReport, ConnectionStatsReport, StatsReport, SummaryStatsReport } from "./stats/statsReport"; -import { SummaryStatsReportGatherer } from "./stats/SummaryStatsReportGatherer"; +import { SummaryStatsReportGatherer } from "./stats/summaryStatsReportGatherer"; export enum GroupCallIntent { Ring = "m.ring", diff --git a/src/webrtc/stats/groupCallStats.ts b/src/webrtc/stats/groupCallStats.ts index 1b9e37bf178..de660f9909f 100644 --- a/src/webrtc/stats/groupCallStats.ts +++ b/src/webrtc/stats/groupCallStats.ts @@ -16,7 +16,7 @@ limitations under the License. import { CallStatsReportGatherer } from "./callStatsReportGatherer"; import { StatsReportEmitter } from "./statsReportEmitter"; import { CallStatsReportSummary } from "./callStatsReportSummary"; -import { SummaryStatsReportGatherer } from "./SummaryStatsReportGatherer"; +import { SummaryStatsReportGatherer } from "./summaryStatsReportGatherer"; import { logger } from "../../logger"; export class GroupCallStats { From 7a19ddffdac5015a24ff6d179758e1e5c9147ac1 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 7 Jun 2023 12:15:23 +0200 Subject: [PATCH 50/52] dont revert enricos change Signed-off-by: Timo K --- src/matrix.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix.ts b/src/matrix.ts index 365bf6dfcf8..7f2ca3f9d3f 100644 --- a/src/matrix.ts +++ b/src/matrix.ts @@ -37,7 +37,7 @@ export * from "./models/event-timeline-set"; export * from "./models/poll"; export * from "./models/room-member"; export * from "./models/room-state"; -export { ThreadEvent, ThreadEmittedEvents, ThreadEventHandlerMap, Thread } from "./models/thread"; +export * from "./models/thread"; export * from "./models/typed-event-emitter"; export * from "./models/user"; export * from "./models/device"; From d2c8b04482b88227b254665ad66eefdb9d134b22 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 7 Jun 2023 12:21:02 +0200 Subject: [PATCH 51/52] temp rename for lowercase --- ...aryStatsReportGatherer.ts => SummaryStatsReportGathererABC.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/webrtc/stats/{SummaryStatsReportGatherer.ts => SummaryStatsReportGathererABC.ts} (100%) diff --git a/src/webrtc/stats/SummaryStatsReportGatherer.ts b/src/webrtc/stats/SummaryStatsReportGathererABC.ts similarity index 100% rename from src/webrtc/stats/SummaryStatsReportGatherer.ts rename to src/webrtc/stats/SummaryStatsReportGathererABC.ts From 1eb1fdba9e81194c425e8bb496077ec12f23e3c4 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 7 Jun 2023 12:21:25 +0200 Subject: [PATCH 52/52] lowercase --- ...aryStatsReportGathererABC.ts => summaryStatsReportGatherer.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/webrtc/stats/{SummaryStatsReportGathererABC.ts => summaryStatsReportGatherer.ts} (100%) diff --git a/src/webrtc/stats/SummaryStatsReportGathererABC.ts b/src/webrtc/stats/summaryStatsReportGatherer.ts similarity index 100% rename from src/webrtc/stats/SummaryStatsReportGathererABC.ts rename to src/webrtc/stats/summaryStatsReportGatherer.ts