From 497b38b609a66a424f868539fe41ff7a41a1b8d0 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Mon, 19 Aug 2024 16:47:27 +0100 Subject: [PATCH 01/20] Prototyping for to-device key distribution Show participant ID and some encryption status message Allow encryption system to be chosen at point of room creation Send cryptoVersion platform data to Posthog Send key distribution stats to posthog Send encryption type for CallStarted and CallEnded events Update js-sdk --- src/analytics/PosthogAnalytics.ts | 4 ++ src/analytics/PosthogEvents.ts | 50 ++++++++++++++++++++++- src/home/RegisteredView.tsx | 47 +++++++++++++++------- src/home/UnauthenticatedView.tsx | 53 +++++++++++++++++++------ src/room/GroupCallView.tsx | 18 +++++---- src/rtcSessionHelper.test.ts | 3 +- src/rtcSessionHelpers.ts | 10 +++-- src/state/CallViewModel.ts | 42 +++++++++++++++++--- src/state/MediaViewModel.ts | 66 +++++++++++++++++++++++++++++-- src/tile/GridTile.tsx | 12 ++++++ src/tile/MediaView.module.css | 20 +++++++++- src/tile/MediaView.tsx | 27 +++++++++++-- src/tile/SpotlightTile.tsx | 18 +++++++++ src/utils/matrix.ts | 27 +++++++++++-- 14 files changed, 339 insertions(+), 58 deletions(-) diff --git a/src/analytics/PosthogAnalytics.ts b/src/analytics/PosthogAnalytics.ts index 05979a897..ca0df15fb 100644 --- a/src/analytics/PosthogAnalytics.ts +++ b/src/analytics/PosthogAnalytics.ts @@ -73,6 +73,7 @@ interface PlatformProperties { appVersion: string; matrixBackend: "embedded" | "jssdk"; callBackend: "livekit" | "full-mesh"; + cryptoVersion?: string; } interface PosthogSettings { @@ -193,6 +194,9 @@ export class PosthogAnalytics { appVersion, matrixBackend: widget ? "embedded" : "jssdk", callBackend: "livekit", + cryptoVersion: widget + ? undefined + : window.matrixclient.getCrypto()?.getVersion(), }; } diff --git a/src/analytics/PosthogEvents.ts b/src/analytics/PosthogEvents.ts index 778392bab..1776b4ed2 100644 --- a/src/analytics/PosthogEvents.ts +++ b/src/analytics/PosthogEvents.ts @@ -16,19 +16,40 @@ limitations under the License. import { DisconnectReason } from "livekit-client"; import { logger } from "matrix-js-sdk/src/logger"; +import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc"; import { IPosthogEvent, PosthogAnalytics, RegistrationType, } from "./PosthogAnalytics"; - +import { E2eeType } from "../e2ee/e2eeType"; + +type EncryptionScheme = "none" | "shared" | "per_sender"; + +function mapE2eeType(type: E2eeType): EncryptionScheme { + switch (type) { + case E2eeType.NONE: + return "none"; + case E2eeType.SHARED_KEY: + return "shared"; + case E2eeType.PER_PARTICIPANT: + return "per_sender"; + } +} interface CallEnded extends IPosthogEvent { eventName: "CallEnded"; callId: string; callParticipantsOnLeave: number; callParticipantsMax: number; callDuration: number; + encryption: EncryptionScheme; + toDeviceEncryptionKeysSent: number; + toDeviceEncryptionKeysReceived: number; + toDeviceEncryptionKeysReceivedAverageAge: number; + roomEventEncryptionKeysSent: number; + roomEventEncryptionKeysReceived: number; + roomEventEncryptionKeysReceivedAverageAge: number; } export class CallEndedTracker { @@ -51,6 +72,8 @@ export class CallEndedTracker { public track( callId: string, callParticipantsNow: number, + e2eeType: E2eeType, + rtcSession: MatrixRTCSession, sendInstantly: boolean, ): void { PosthogAnalytics.instance.trackEvent( @@ -60,6 +83,27 @@ export class CallEndedTracker { callParticipantsMax: this.cache.maxParticipantsCount, callParticipantsOnLeave: callParticipantsNow, callDuration: (Date.now() - this.cache.startTime.getTime()) / 1000, + encryption: mapE2eeType(e2eeType), + toDeviceEncryptionKeysSent: + rtcSession.statistics.counters.toDeviceEncryptionKeysSent, + toDeviceEncryptionKeysReceived: + rtcSession.statistics.counters.toDeviceEncryptionKeysReceived, + toDeviceEncryptionKeysReceivedAverageAge: + rtcSession.statistics.counters.toDeviceEncryptionKeysReceived > 0 + ? rtcSession.statistics.totals + .toDeviceEncryptionKeysReceivedTotalAge / + rtcSession.statistics.counters.toDeviceEncryptionKeysReceived + : 0, + roomEventEncryptionKeysSent: + rtcSession.statistics.counters.roomEventEncryptionKeysSent, + roomEventEncryptionKeysReceived: + rtcSession.statistics.counters.roomEventEncryptionKeysReceived, + roomEventEncryptionKeysReceivedAverageAge: + rtcSession.statistics.counters.roomEventEncryptionKeysReceived > 0 + ? rtcSession.statistics.totals + .roomEventEncryptionKeysReceivedTotalAge / + rtcSession.statistics.counters.roomEventEncryptionKeysReceived + : 0, }, { send_instantly: sendInstantly }, ); @@ -69,13 +113,15 @@ export class CallEndedTracker { interface CallStarted extends IPosthogEvent { eventName: "CallStarted"; callId: string; + encryption: EncryptionScheme; } export class CallStartedTracker { - public track(callId: string): void { + public track(callId: string, e2eeType: E2eeType): void { PosthogAnalytics.instance.trackEvent({ eventName: "CallStarted", callId: callId, + encryption: mapE2eeType(e2eeType), }); } } diff --git a/src/home/RegisteredView.tsx b/src/home/RegisteredView.tsx index 3335acd28..70aea754a 100644 --- a/src/home/RegisteredView.tsx +++ b/src/home/RegisteredView.tsx @@ -18,7 +18,7 @@ import { useState, useCallback, FormEvent, FormEventHandler, FC } from "react"; import { useHistory } from "react-router-dom"; import { MatrixClient } from "matrix-js-sdk/src/client"; import { useTranslation } from "react-i18next"; -import { Heading } from "@vector-im/compound-web"; +import { Dropdown, Heading } from "@vector-im/compound-web"; import { logger } from "matrix-js-sdk/src/logger"; import { Button } from "@vector-im/compound-web"; @@ -45,6 +45,17 @@ import { useOptInAnalytics } from "../settings/settings"; interface Props { client: MatrixClient; } +const encryptionOptions = { + shared: { + label: "Shared key", + e2eeType: E2eeType.SHARED_KEY, + }, + sender: { + label: "Per-participant key", + e2eeType: E2eeType.PER_PARTICIPANT, + }, + none: { label: "None", e2eeType: E2eeType.NONE }, +}; export const RegisteredView: FC = ({ client }) => { const [loading, setLoading] = useState(false); @@ -59,6 +70,9 @@ export const RegisteredView: FC = ({ client }) => { [setJoinExistingCallModalOpen], ); + const [encryption, setEncryption] = + useState("shared"); + const onSubmit: FormEventHandler = useCallback( (e: FormEvent) => { e.preventDefault(); @@ -73,21 +87,13 @@ export const RegisteredView: FC = ({ client }) => { setError(undefined); setLoading(true); - const createRoomResult = await createRoom( + const { roomId, encryptionSystem } = await createRoom( client, roomName, - E2eeType.SHARED_KEY, - ); - if (!createRoomResult.password) - throw new Error("Failed to create room with shared secret"); - - history.push( - getRelativeRoomUrl( - createRoomResult.roomId, - { kind: E2eeType.SHARED_KEY, secret: createRoomResult.password }, - roomName, - ), + encryptionOptions[encryption].e2eeType, ); + + history.push(getRelativeRoomUrl(roomId, encryptionSystem, roomName)); } submit().catch((error) => { @@ -103,7 +109,7 @@ export const RegisteredView: FC = ({ client }) => { } }); }, - [client, history, setJoinExistingCallModalOpen], + [client, history, setJoinExistingCallModalOpen, encryption], ); const recentRooms = useGroupCallRooms(client); @@ -142,6 +148,19 @@ export const RegisteredView: FC = ({ client }) => { data-testid="home_callName" /> + + setEncryption(x as keyof typeof encryptionOptions) + } + values={Object.keys(encryptionOptions).map((value) => [ + value, + encryptionOptions[value as keyof typeof encryptionOptions] + .label, + ])} + placeholder="" + /> )} + + {error && } + {sent && {t("settings.feedback_tab_thank_you")}} + ); diff --git a/src/settings/submit-rageshake.ts b/src/settings/submit-rageshake.ts index f9e10abae..10b31f6cd 100644 --- a/src/settings/submit-rageshake.ts +++ b/src/settings/submit-rageshake.ts @@ -270,11 +270,17 @@ export function useSubmitRageshake(): { ); } - await fetch(Config.get().rageshake!.submit_url, { + const res = await fetch(Config.get().rageshake!.submit_url, { method: "POST", body, }); + if (res.status !== 200) { + throw new Error( + `Failed to submit feedback: receive HTTP ${res.status} ${res.statusText}`, + ); + } + setState({ sending: false, sent: true, error: undefined }); } catch (error) { setState({ sending: false, sent: false, error: error as Error }); From 0c762fd510c02b15b9390358201e556fd4c9fb55 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Wed, 11 Sep 2024 10:08:35 +0100 Subject: [PATCH 17/20] Rageshake logging improvements Capture MatrixRTC related prefixed/child loggers from js-sdk. Add explicit prefix to livekit log entries. --- src/main.tsx | 9 ++++++--- src/settings/rageshake.ts | 30 +++++++++++++++++++++--------- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/main.tsx b/src/main.tsx index 263619c97..5ea00213a 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -18,7 +18,7 @@ import "./index.css"; import { logger } from "matrix-js-sdk/src/logger"; import { setLogExtension as setLKLogExtension, - setLogLevel, + setLogLevel as setLKLogLevel, } from "livekit-client"; import { App } from "./App"; @@ -26,8 +26,11 @@ import { init as initRageshake } from "./settings/rageshake"; import { Initializer } from "./initializer"; initRageshake(); -setLogLevel("debug"); -setLKLogExtension(global.mx_rage_logger.log); +setLKLogLevel("debug"); +setLKLogExtension((level, msg, context) => { + // we pass a synthetic logger name of "livekit" to the rageshake to make it easier to read + global.mx_rage_logger.log(level, "livekit", msg, context); +}); logger.info(`Element Call ${import.meta.env.VITE_APP_VERSION || "dev"}`); diff --git a/src/settings/rageshake.ts b/src/settings/rageshake.ts index 3d5103ace..cf1f116fd 100644 --- a/src/settings/rageshake.ts +++ b/src/settings/rageshake.ts @@ -29,9 +29,9 @@ Please see LICENSE in the repository root for full details. import EventEmitter from "events"; import { throttle } from "lodash"; -import { logger } from "matrix-js-sdk/src/logger"; +import { Logger, logger } from "matrix-js-sdk/src/logger"; import { randomString } from "matrix-js-sdk/src/randomstring"; -import { LoggingMethod } from "loglevel"; +import loglevel, { LoggingMethod } from "loglevel"; // the length of log data we keep in indexeddb (and include in the reports) const MAX_LOG_SIZE = 1024 * 1024 * 5; // 5 MB @@ -473,7 +473,12 @@ declare global { */ export function init(): Promise { global.mx_rage_logger = new ConsoleLogger(); - setLogExtension(global.mx_rage_logger.log); + setLogExtension(logger, global.mx_rage_logger.log); + // these are the child/prefixed loggers we want to capture from js-sdk + // there doesn't seem to be an easy way to capture all children + ["MatrixRTCSession", "MatrixRTCSessionManager"].forEach((loggerName) => { + setLogExtension(logger.getChild(loggerName), global.mx_rage_logger.log); + }); return tryInitStorage(); } @@ -586,10 +591,14 @@ type LogLevelString = keyof typeof LogLevel; * took loglevel's example honouring log levels). Adds a loglevel logging extension * in the recommended way. */ -export function setLogExtension(extension: LogExtensionFunc): void { - const originalFactory = logger.methodFactory; - - logger.methodFactory = function ( +function setLogExtension( + _loggerToExtend: Logger, + extension: LogExtensionFunc, +): void { + const loggerToExtend = _loggerToExtend as unknown as loglevel.Logger; + const originalFactory = loggerToExtend.methodFactory; + + loggerToExtend.methodFactory = function ( methodName, configLevel, loggerName, @@ -600,11 +609,14 @@ export function setLogExtension(extension: LogExtensionFunc): void { const needLog = logLevel >= configLevel && logLevel < LogLevel.silent; return (...args) => { + // we don't send the logger name to the raw method as some of them are already outputting the prefix rawMethod.apply(this, args); if (needLog) { - extension(logLevel, ...args); + // we prefix the logger name to the extension + // this makes sure that the rageshake contains the logger name + extension(logLevel, loggerName?.toString(), ...args); } }; }; - logger.setLevel(logger.getLevel()); // Be sure to call setLevel method in order to apply plugin + loggerToExtend.setLevel(loggerToExtend.getLevel()); // Be sure to call setLevel method in order to apply plugin } From 55ea373cd03cedcb9651055772bd084658343fe5 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Wed, 11 Sep 2024 12:20:30 +0100 Subject: [PATCH 18/20] Intercept matrix_sdk logging via console --- src/settings/rageshake.ts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/settings/rageshake.ts b/src/settings/rageshake.ts index ea9805dfb..ffd9f333b 100644 --- a/src/settings/rageshake.ts +++ b/src/settings/rageshake.ts @@ -467,6 +467,8 @@ declare global { */ export async function init(): Promise { global.mx_rage_logger = new ConsoleLogger(); + + // configure loglevel based loggers: setLogExtension(logger, global.mx_rage_logger.log); // these are the child/prefixed loggers we want to capture from js-sdk // there doesn't seem to be an easy way to capture all children @@ -474,6 +476,28 @@ export async function init(): Promise { setLogExtension(logger.getChild(loggerName), global.mx_rage_logger.log); }); + // intercept console logging so that we can get matrix_sdk logs: + // this is nasty, but no logging hooks are provided + ( + ["trace", "debug", "info", "warn", "error"] as ( + | "trace" + | "debug" + | "info" + | "warn" + | "error" + )[] + ).forEach((level) => { + if (!window.console[level]) return; + const prefix = `${level.toUpperCase()} matrix_sdk`; + const originalMethod = window.console[level]; + window.console[level] = (...args): void => { + originalMethod(...args); + if (typeof args[0] === "string" && args[0].startsWith(prefix)) { + global.mx_rage_logger.log(LogLevel[level], "matrix_sdk", ...args); + } + }; + }); + return tryInitStorage(); } From c1161ed047548d24f0443250b9e39b55e26030b2 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Wed, 11 Sep 2024 16:11:15 +0100 Subject: [PATCH 19/20] Bump js-sdk to fix embedded mode --- package.json | 2 +- yarn.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 71843ce8d..4355effc0 100644 --- a/package.json +++ b/package.json @@ -83,7 +83,7 @@ "livekit-client": "^2.0.2", "lodash": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "matrix-org/matrix-js-sdk#f1974b2090c10bdf51b717ce4995fc70cd482364", + "matrix-js-sdk": "matrix-org/matrix-js-sdk#b0174eccdb0e33f5df5d7b590938daf8ff5c7f7a", "matrix-widget-api": "^1.8.2", "normalize.css": "^8.0.1", "observable-hooks": "^4.2.3", diff --git a/yarn.lock b/yarn.lock index 7ec494888..13d88ea8a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5948,9 +5948,9 @@ matrix-events-sdk@0.0.1: resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-0.0.1.tgz#c8c38911e2cb29023b0bbac8d6f32e0de2c957dd" integrity sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA== -matrix-js-sdk@matrix-org/matrix-js-sdk#f1974b2090c10bdf51b717ce4995fc70cd482364: +matrix-js-sdk@matrix-org/matrix-js-sdk#b0174eccdb0e33f5df5d7b590938daf8ff5c7f7a: version "34.4.0" - resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/f1974b2090c10bdf51b717ce4995fc70cd482364" + resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/b0174eccdb0e33f5df5d7b590938daf8ff5c7f7a" dependencies: "@babel/runtime" "^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm" "https://floofy.netlify.app/matrix-org-matrix-sdk-crypto-wasm-v7.0.0-to-device.tgz" From 8f73f81bc7c1ecfbcd09d8ba94590b1c73054aad Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Wed, 11 Sep 2024 16:26:10 +0100 Subject: [PATCH 20/20] Request widget permission to send encryption key to device messages --- src/widget.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/widget.ts b/src/widget.ts index f08968b65..2540aa5bb 100644 --- a/src/widget.ts +++ b/src/widget.ts @@ -130,6 +130,7 @@ export const widget = ((): WidgetHelpers | null => { EventType.CallSDPStreamMetadataChanged, EventType.CallSDPStreamMetadataChangedPrefix, EventType.CallReplaces, + EventType.CallEncryptionKeysPrefix, ]; const client = createRoomWidgetClient(