diff --git a/.eslintrc.cjs b/.eslintrc.cjs index f6a2e5693..5970790f0 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -38,15 +38,6 @@ module.exports = { "jsx-a11y/media-has-caption": "off", // We should use the js-sdk logger, never console directly. "no-console": ["error"], - "no-restricted-imports": [ - "error", - { - name: "@react-rxjs/core", - importNames: ["Subscribe", "RemoveSubscribe"], - message: - "These components are easy to misuse, please use the 'subscribe' component wrapper instead", - }, - ], "react/display-name": "error", }, settings: { diff --git a/package.json b/package.json index 7c13619fa..58eb3ddb6 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,6 @@ "@react-aria/tabs": "^3.1.0", "@react-aria/tooltip": "^3.1.3", "@react-aria/utils": "^3.10.0", - "@react-rxjs/core": "^0.10.7", "@react-spring/web": "^9.4.4", "@react-stately/collections": "^3.3.4", "@react-stately/select": "^3.1.3", @@ -66,6 +65,7 @@ "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#238eea0ef5c82d0a11b8d5cc5c04104d6c94c4c1", "matrix-widget-api": "^1.3.1", "normalize.css": "^8.0.1", + "observable-hooks": "^4.2.3", "pako": "^2.0.4", "postcss-preset-env": "^9.0.0", "posthog-js": "^1.29.0", diff --git a/public/locales/en-GB/app.json b/public/locales/en-GB/app.json index 7fd742cb8..0dbebf381 100644 --- a/public/locales/en-GB/app.json +++ b/public/locales/en-GB/app.json @@ -41,6 +41,7 @@ "analytics": "Analytics", "audio": "Audio", "avatar": "Avatar", + "back": "Back", "camera": "Camera", "copied": "Copied!", "display_name": "Display name", @@ -49,6 +50,7 @@ "home": "Home", "loading": "Loading…", "microphone": "Microphone", + "next": "Next", "options": "Options", "password": "Password", "profile": "Profile", @@ -130,6 +132,7 @@ "developer_settings_label": "Developer Settings", "developer_settings_label_description": "Expose developer settings in the settings window.", "developer_tab_title": "Developer", + "duplicate_tiles_label": "Number of additional tile copies per participant", "feedback_tab_body": "If you are experiencing issues or simply would like to provide some feedback, please send us a short description below.", "feedback_tab_description_label": "Your feedback", "feedback_tab_h4": "Submit feedback", @@ -138,7 +141,6 @@ "feedback_tab_title": "Feedback", "more_tab_title": "More", "opt_in_description": "<0><1>You may withdraw consent by unchecking this box. If you are currently in a call, this setting will take effect at the end of the call.", - "show_connection_stats_label": "Show connection stats", "speaker_device_selection_label": "Speaker" }, "star_rating_input_label_one": "{{count}} stars", @@ -154,12 +156,12 @@ "unmute_microphone_button_label": "Unmute microphone", "version": "Version: {{version}}", "video_tile": { + "always_show": "Always show", "change_fit_contain": "Fit to frame", "exit_full_screen": "Exit full screen", "full_screen": "Full screen", "mute_for_me": "Mute for me", "sfu_participant_local": "You", "volume": "Volume" - }, - "waiting_for_participants": "Waiting for other participants…" + } } diff --git a/src/Header.module.css b/src/Header.module.css index 53f51d3a4..5a408bd3d 100644 --- a/src/Header.module.css +++ b/src/Header.module.css @@ -1,5 +1,5 @@ /* -Copyright 2022 New Vector Ltd +Copyright 2022-2024 New Vector Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/Header.tsx b/src/Header.tsx index 1bf8a4a72..e0fb92974 100644 --- a/src/Header.tsx +++ b/src/Header.tsx @@ -1,5 +1,5 @@ /* -Copyright 2022 New Vector Ltd +Copyright 2022-2024 New Vector Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ limitations under the License. */ import classNames from "classnames"; -import { FC, HTMLAttributes, ReactNode } from "react"; +import { FC, HTMLAttributes, ReactNode, forwardRef } from "react"; import { Link } from "react-router-dom"; import { useTranslation } from "react-i18next"; import { Heading, Text } from "@vector-im/compound-web"; @@ -32,13 +32,21 @@ interface HeaderProps extends HTMLAttributes { className?: string; } -export const Header: FC = ({ children, className, ...rest }) => { - return ( -
- {children} -
- ); -}; +export const Header = forwardRef( + ({ children, className, ...rest }, ref) => { + return ( +
+ {children} +
+ ); + }, +); + +Header.displayName = "Header"; interface LeftNavProps extends HTMLAttributes { children: ReactNode; diff --git a/src/Platform.ts b/src/Platform.ts index 0e5b71f1c..86f1bc57d 100644 --- a/src/Platform.ts +++ b/src/Platform.ts @@ -36,3 +36,8 @@ if (/android/i.test(navigator.userAgent)) { } else { platform = "desktop"; } + +export const isFirefox = (): boolean => { + const { userAgent } = navigator; + return userAgent.includes("Firefox"); +}; diff --git a/src/analytics/PosthogAnalytics.ts b/src/analytics/PosthogAnalytics.ts index cfeb1e7a1..5cbc552b4 100644 --- a/src/analytics/PosthogAnalytics.ts +++ b/src/analytics/PosthogAnalytics.ts @@ -20,7 +20,6 @@ import { MatrixClient } from "matrix-js-sdk"; import { Buffer } from "buffer"; import { widget } from "../widget"; -import { getSetting, setSetting, getSettingKey } from "../settings/useSetting"; import { CallEndedTracker, CallStartedTracker, @@ -35,7 +34,7 @@ import { } from "./PosthogEvents"; import { Config } from "../config/Config"; import { getUrlParams } from "../UrlParams"; -import { localStorageBus } from "../useLocalStorage"; +import { optInAnalytics } from "../settings/settings"; /* Posthog analytics tracking. * @@ -131,7 +130,7 @@ export class PosthogAnalytics { const { analyticsID } = getUrlParams(); // if the embedding platform (element web) already got approval to communicating with posthog // element call can also send events to posthog - setSetting("opt-in-analytics", Boolean(analyticsID)); + optInAnalytics.setValue(Boolean(analyticsID)); } this.posthog.init(posthogConfig.project_api_key, { @@ -151,9 +150,7 @@ export class PosthogAnalytics { ); this.enabled = false; } - this.startListeningToSettingsChanges(); - const optInAnalytics = getSetting("opt-in-analytics", false); - this.updateAnonymityAndIdentifyUser(optInAnalytics); + this.startListeningToSettingsChanges(); // Triggers maybeIdentifyUser } private sanitizeProperties = ( @@ -336,8 +333,7 @@ export class PosthogAnalytics { } public onLoginStatusChanged(): void { - const optInAnalytics = getSetting("opt-in-analytics", false); - this.updateAnonymityAndIdentifyUser(optInAnalytics); + this.maybeIdentifyUser(); } private updateSuperProperties(): void { @@ -360,20 +356,12 @@ export class PosthogAnalytics { return this.eventSignup.getSignupEndTime() > new Date(0); } - private async updateAnonymityAndIdentifyUser( - pseudonymousOptIn: boolean, - ): Promise { - // Update this.anonymity based on the user's analytics opt-in settings - const anonymity = pseudonymousOptIn - ? Anonymity.Pseudonymous - : Anonymity.Disabled; - this.setAnonymity(anonymity); - + private async maybeIdentifyUser(): Promise { // We may not yet have a Matrix client at this point, if not, bail. This should get // triggered again by onLoginStatusChanged once we do have a client. if (!window.matrixclient) return; - if (anonymity === Anonymity.Pseudonymous) { + if (this.anonymity === Anonymity.Pseudonymous) { this.setRegistrationType( window.matrixclient.isGuest() || window.passwordlessUser ? RegistrationType.Guest @@ -389,7 +377,7 @@ export class PosthogAnalytics { } } - if (anonymity !== Anonymity.Disabled) { + if (this.anonymity !== Anonymity.Disabled) { this.updateSuperProperties(); } } @@ -419,8 +407,9 @@ export class PosthogAnalytics { // * When the user changes their preferences on this device // Note that for new accounts, pseudonymousAnalyticsOptIn won't be set, so updateAnonymityFromSettings // won't be called (i.e. this.anonymity will be left as the default, until the setting changes) - localStorageBus.on(getSettingKey("opt-in-analytics"), (optInAnalytics) => { - this.updateAnonymityAndIdentifyUser(optInAnalytics); + optInAnalytics.value.subscribe((optIn) => { + this.setAnonymity(optIn ? Anonymity.Pseudonymous : Anonymity.Disabled); + this.maybeIdentifyUser(); }); } diff --git a/src/grid/CallLayout.ts b/src/grid/CallLayout.ts new file mode 100644 index 000000000..ecac04ad8 --- /dev/null +++ b/src/grid/CallLayout.ts @@ -0,0 +1,159 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { BehaviorSubject, Observable } from "rxjs"; +import { ComponentType } from "react"; + +import { MediaViewModel, UserMediaViewModel } from "../state/MediaViewModel"; +import { LayoutProps } from "./Grid"; + +export interface Bounds { + width: number; + height: number; +} + +export interface Alignment { + inline: "start" | "end"; + block: "start" | "end"; +} + +export const defaultSpotlightAlignment: Alignment = { + inline: "end", + block: "end", +}; +export const defaultPipAlignment: Alignment = { inline: "end", block: "start" }; + +export interface CallLayoutInputs { + /** + * The minimum bounds of the layout area. + */ + minBounds: Observable; + /** + * The alignment of the floating spotlight tile, if present. + */ + spotlightAlignment: BehaviorSubject; + /** + * The alignment of the small picture-in-picture tile, if present. + */ + pipAlignment: BehaviorSubject; +} + +export interface GridTileModel { + type: "grid"; + vm: UserMediaViewModel; +} + +export interface SpotlightTileModel { + type: "spotlight"; + vms: MediaViewModel[]; + maximised: boolean; +} + +export type TileModel = GridTileModel | SpotlightTileModel; + +export interface CallLayoutOutputs { + /** + * Whether the scrolling layer of the layout should appear on top. + */ + scrollingOnTop: boolean; + /** + * The visually fixed (non-scrolling) layer of the layout. + */ + fixed: ComponentType>; + /** + * The layer of the layout that can overflow and be scrolled. + */ + scrolling: ComponentType>; +} + +/** + * A layout system for media tiles. + */ +export type CallLayout = ( + inputs: CallLayoutInputs, +) => CallLayoutOutputs; + +export interface GridArrangement { + tileWidth: number; + tileHeight: number; + gap: number; + columns: number; +} + +const tileMinHeight = 130; +const tileMaxAspectRatio = 17 / 9; +const tileMinAspectRatio = 4 / 3; +const tileMobileMinAspectRatio = 2 / 3; + +/** + * Determine the ideal arrangement of tiles into a grid of a particular size. + */ +export function arrangeTiles( + width: number, + minHeight: number, + tileCount: number, +): GridArrangement { + // The goal here is to determine the grid size and padding that maximizes + // use of screen space for n tiles without making those tiles too small or + // too cropped (having an extreme aspect ratio) + const gap = width < 800 ? 16 : 20; + const tileMinWidth = width < 500 ? 150 : 180; + + let columns = Math.min( + // Don't create more columns than we have items for + tileCount, + // The ideal number of columns is given by a packing of equally-sized + // squares into a grid. + // width / column = height / row. + // columns * rows = number of squares. + // ∴ columns = sqrt(width / height * number of squares). + // Except we actually want 16:9-ish tiles rather than squares, so we + // divide the width-to-height ratio by the target aspect ratio. + Math.ceil(Math.sqrt((width / minHeight / tileMaxAspectRatio) * tileCount)), + ); + let rows = Math.ceil(tileCount / columns); + + let tileWidth = (width - (columns + 1) * gap) / columns; + let tileHeight = (minHeight - (rows - 1) * gap) / rows; + + // Impose a minimum width and height on the tiles + if (tileWidth < tileMinWidth) { + // In this case we want the tile width to determine the number of columns, + // not the other way around. If we take the above equation for the tile + // width (w = (W - (c - 1) * g) / c) and solve for c, we get + // c = (W + g) / (w + g). + columns = Math.floor((width + gap) / (tileMinWidth + gap)); + rows = Math.ceil(tileCount / columns); + tileWidth = (width - (columns + 1) * gap) / columns; + tileHeight = (minHeight - (rows - 1) * gap) / rows; + } + if (tileHeight < tileMinHeight) tileHeight = tileMinHeight; + + // Impose a minimum and maximum aspect ratio on the tiles + const tileAspectRatio = tileWidth / tileHeight; + // We enforce a different min aspect ratio in 1:1s on mobile + const minAspectRatio = + tileCount === 1 && width < 600 + ? tileMobileMinAspectRatio + : tileMinAspectRatio; + if (tileAspectRatio > tileMaxAspectRatio) + tileWidth = tileHeight * tileMaxAspectRatio; + else if (tileAspectRatio < minAspectRatio) + tileHeight = tileWidth / minAspectRatio; + // TODO: We might now be hitting the minimum height or width limit again + + return { tileWidth, tileHeight, gap, columns }; +} diff --git a/src/video-grid/NewVideoGrid.module.css b/src/grid/Grid.css similarity index 100% rename from src/video-grid/NewVideoGrid.module.css rename to src/grid/Grid.css diff --git a/src/video-grid/VideoGrid.module.css b/src/grid/Grid.module.css similarity index 75% rename from src/video-grid/VideoGrid.module.css rename to src/grid/Grid.module.css index df6e4fa77..33e593be2 100644 --- a/src/video-grid/VideoGrid.module.css +++ b/src/grid/Grid.module.css @@ -1,11 +1,11 @@ /* -Copyright 2022 New Vector Ltd +Copyright 2023-2024 New Vector Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at -http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -14,9 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -.videoGrid { - position: relative; - overflow: hidden; - flex: 1; - touch-action: none; +.grid { + contain: layout style; +} + +.slot { + contain: strict; } diff --git a/src/grid/Grid.tsx b/src/grid/Grid.tsx new file mode 100644 index 000000000..ea33a32d0 --- /dev/null +++ b/src/grid/Grid.tsx @@ -0,0 +1,481 @@ +/* +Copyright 2023-2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { + SpringRef, + TransitionFn, + animated, + useTransition, +} from "@react-spring/web"; +import { EventTypes, Handler, useScroll } from "@use-gesture/react"; +import { + CSSProperties, + ComponentProps, + ComponentType, + FC, + LegacyRef, + ReactNode, + useCallback, + useEffect, + useMemo, + useRef, + useState, +} from "react"; +import useMeasure from "react-use-measure"; +import classNames from "classnames"; + +import styles from "./Grid.module.css"; +import { useMergedRefs } from "../useMergedRefs"; +import { TileWrapper } from "./TileWrapper"; +import { usePrefersReducedMotion } from "../usePrefersReducedMotion"; +import { useInitial } from "../useInitial"; + +interface Rect { + x: number; + y: number; + width: number; + height: number; +} + +interface Tile { + id: string; + model: Model; + onDrag: DragCallback | undefined; +} + +type PlacedTile = Tile & Rect; + +interface TileSpring { + opacity: number; + scale: number; + zIndex: number; + x: number; + y: number; + width: number; + height: number; +} + +interface TileSpringUpdate extends Partial { + from?: Partial; + reset?: boolean; + immediate?: boolean | ((key: string) => boolean); + delay?: (key: string) => number; +} + +interface DragState { + tileId: string; + tileX: number; + tileY: number; + cursorX: number; + cursorY: number; +} + +interface SlotProps extends Omit, "onDrag"> { + id: string; + model: Model; + onDrag?: DragCallback; + style?: CSSProperties; + className?: string; +} + +interface Offset { + x: number; + y: number; +} + +/** + * Gets the offset of one element relative to an ancestor. + */ +function offset(element: HTMLElement, relativeTo: Element): Offset { + if ( + !(element.offsetParent instanceof HTMLElement) || + element.offsetParent === relativeTo + ) { + return { x: element.offsetLeft, y: element.offsetTop }; + } else { + const o = offset(element.offsetParent, relativeTo); + o.x += element.offsetLeft; + o.y += element.offsetTop; + return o; + } +} + +export interface LayoutProps { + ref: LegacyRef; + model: LayoutModel; + /** + * Component creating an invisible "slot" for a tile to go in. + */ + Slot: ComponentType>; +} + +export interface TileProps { + ref: LegacyRef; + className?: string; + style?: ComponentProps["style"]; + /** + * The width this tile will have once its animations have settled. + */ + targetWidth: number; + /** + * The height this tile will have once its animations have settled. + */ + targetHeight: number; + model: Model; +} + +interface Drag { + /** + * The X coordinate of the dragged tile in grid space. + */ + x: number; + /** + * The Y coordinate of the dragged tile in grid space. + */ + y: number; + /** + * The X coordinate of the dragged tile, as a scalar of the grid width. + */ + xRatio: number; + /** + * The Y coordinate of the dragged tile, as a scalar of the grid height. + */ + yRatio: number; +} + +export type DragCallback = (drag: Drag) => void; + +interface Props< + LayoutModel, + TileModel, + LayoutRef extends HTMLElement, + TileRef extends HTMLElement, +> { + /** + * Data with which to populate the layout. + */ + model: LayoutModel; + /** + * A component which creates an invisible layout grid of "slots" for tiles to + * go in. The root element must have a data-generation attribute which + * increments whenever the layout may have changed. + */ + Layout: ComponentType>; + /** + * The component used to render each tile in the layout. + */ + Tile: ComponentType>; + className?: string; + style?: CSSProperties; +} + +/** + * A grid of animated tiles. + */ +export function Grid< + LayoutModel, + TileModel, + LayoutRef extends HTMLElement, + TileRef extends HTMLElement, +>({ + model, + Layout, + Tile, + className, + style, +}: Props): ReactNode { + // Overview: This component places tiles by rendering an invisible layout grid + // of "slots" for tiles to go in. Once rendered, it uses the DOM API to get + // the dimensions of each slot, feeding these numbers back into react-spring + // to let the actual tiles move freely atop the layout. + + // To tell us when the layout has changed, the layout system increments its + // data-generation attribute, which we watch with a MutationObserver. + + const [gridRef1, gridBounds] = useMeasure(); + const [gridRoot, gridRef2] = useState(null); + const gridRef = useMergedRefs(gridRef1, gridRef2); + + const [layoutRoot, setLayoutRoot] = useState(null); + const [generation, setGeneration] = useState(null); + const tiles = useInitial(() => new Map>()); + const prefersReducedMotion = usePrefersReducedMotion(); + + const Slot: FC> = useMemo( + () => + function Slot({ id, model, onDrag, style, className, ...props }) { + const ref = useRef(null); + useEffect(() => { + tiles.set(id, { id, model, onDrag }); + return (): void => void tiles.delete(id); + }, [id, model, onDrag]); + + return ( +
+ ); + }, + [tiles], + ); + + const layoutRef = useCallback( + (e: HTMLElement | null) => { + setLayoutRoot(e); + if (e !== null) + setGeneration(parseInt(e.getAttribute("data-generation")!)); + }, + [setLayoutRoot, setGeneration], + ); + + useEffect(() => { + if (layoutRoot !== null) { + const observer = new MutationObserver((mutations) => { + if (mutations.some((m) => m.type === "attributes")) { + setGeneration(parseInt(layoutRoot.getAttribute("data-generation")!)); + } + }); + + observer.observe(layoutRoot, { attributes: true }); + return (): void => observer.disconnect(); + } + }, [layoutRoot, setGeneration]); + + // Combine the tile definitions and slots together to create placed tiles + const placedTiles = useMemo(() => { + const result: PlacedTile[] = []; + + if (gridRoot !== null && layoutRoot !== null) { + const slots = layoutRoot.getElementsByClassName( + styles.slot, + ) as HTMLCollectionOf; + for (const slot of slots) { + const id = slot.getAttribute("data-id")!; + if (slot.offsetWidth > 0 && slot.offsetHeight > 0) + result.push({ + ...tiles.get(id)!, + ...offset(slot, gridRoot), + width: slot.offsetWidth, + height: slot.offsetHeight, + }); + } + } + + return result; + // The rects may change due to the grid updating to a new generation, but + // eslint can't statically verify this + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [gridRoot, layoutRoot, tiles, generation]); + + // Drag state is stored in a ref rather than component state, because we use + // react-spring's imperative API during gestures to improve responsiveness + const dragState = useRef(null); + + const [tileTransitions, springRef] = useTransition( + placedTiles, + () => ({ + key: ({ id }: Tile): string => id, + from: ({ + x, + y, + width, + height, + }: PlacedTile): TileSpringUpdate => ({ + opacity: 0, + scale: 0, + zIndex: 1, + x, + y, + width, + height, + immediate: prefersReducedMotion, + }), + enter: { opacity: 1, scale: 1, immediate: prefersReducedMotion }, + update: ({ + id, + x, + y, + width, + height, + }: PlacedTile): TileSpringUpdate | null => + id === dragState.current?.tileId + ? null + : { + x, + y, + width, + height, + immediate: prefersReducedMotion, + }, + leave: { opacity: 0, scale: 0, immediate: prefersReducedMotion }, + config: { mass: 0.7, tension: 252, friction: 25 }, + }), + // react-spring's types are bugged and can't infer the spring type + ) as unknown as [ + TransitionFn, TileSpring>, + SpringRef, + ]; + + // Because we're using react-spring in imperative mode, we're responsible for + // firing animations manually whenever the tiles array updates + useEffect(() => { + springRef.start(); + }, [placedTiles, springRef]); + + const animateDraggedTile = ( + endOfGesture: boolean, + callback: DragCallback, + ): void => { + const { tileId, tileX, tileY } = dragState.current!; + const tile = placedTiles.find((t) => t.id === tileId)!; + + springRef.current + .find((c) => (c.item as Tile).id === tileId) + ?.start( + endOfGesture + ? { + scale: 1, + zIndex: 1, + x: tile.x, + y: tile.y, + width: tile.width, + height: tile.height, + immediate: + prefersReducedMotion || ((key): boolean => key === "zIndex"), + // Allow the tile's position to settle before pushing its + // z-index back down + delay: (key): number => (key === "zIndex" ? 500 : 0), + } + : { + scale: 1.1, + zIndex: 2, + x: tileX, + y: tileY, + immediate: + prefersReducedMotion || + ((key): boolean => + key === "zIndex" || key === "x" || key === "y"), + }, + ); + + if (endOfGesture) + callback({ + x: tileX, + y: tileY, + xRatio: tileX / (gridBounds.width - tile.width), + yRatio: tileY / (gridBounds.height - tile.height), + }); + }; + + // Callback for useDrag. We could call useDrag here, but the default + // pattern of spreading {...bind()} across the children to bind the gesture + // ends up breaking memoization and ruining this component's performance. + // Instead, we pass this callback to each tile via a ref, to let them bind the + // gesture using the much more sensible ref-based method. + const onTileDrag = ( + tileId: string, + + { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + tap, + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + initial: [initialX, initialY], + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + delta: [dx, dy], + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + last, + }: Parameters>[0], + ): void => { + if (!tap) { + const tileController = springRef.current.find( + (c) => (c.item as Tile).id === tileId, + )!; + const callback = tiles.get(tileController.item.id)!.onDrag; + + if (callback != null) { + if (dragState.current === null) { + const tileSpring = tileController.get(); + dragState.current = { + tileId, + tileX: tileSpring.x, + tileY: tileSpring.y, + cursorX: initialX - gridBounds.x, + cursorY: initialY - gridBounds.y + scrollOffset.current, + }; + } + + dragState.current.tileX += dx; + dragState.current.tileY += dy; + dragState.current.cursorX += dx; + dragState.current.cursorY += dy; + + animateDraggedTile(last, callback); + + if (last) dragState.current = null; + } + } + }; + + const onTileDragRef = useRef(onTileDrag); + onTileDragRef.current = onTileDrag; + + const scrollOffset = useRef(0); + + useScroll( + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + ({ xy: [, y], delta: [, dy] }) => { + scrollOffset.current = y; + + if (dragState.current !== null) { + dragState.current.tileY += dy; + dragState.current.cursorY += dy; + animateDraggedTile(false, tiles.get(dragState.current.tileId)!.onDrag!); + } + }, + { target: gridRoot ?? undefined }, + ); + + return ( +
+ + {tileTransitions((spring, { id, model, onDrag, width, height }) => ( + + ))} +
+ ); +} diff --git a/src/grid/GridLayout.module.css b/src/grid/GridLayout.module.css new file mode 100644 index 000000000..6838ae911 --- /dev/null +++ b/src/grid/GridLayout.module.css @@ -0,0 +1,60 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.fixed, +.scrolling { + block-size: 100%; +} + +.scrolling { + display: flex; + flex-wrap: wrap; + justify-content: center; + align-content: center; + gap: var(--gap); +} + +.scrolling > .slot { + width: var(--width); + height: var(--height); +} + +.fixed { + position: relative; +} + +.fixed > .slot { + position: absolute; + inline-size: 404px; + block-size: 233px; + inset: 0; +} + +.fixed > .slot[data-block-alignment="start"] { + inset-block-end: unset; +} + +.fixed > .slot[data-block-alignment="end"] { + inset-block-start: unset; +} + +.fixed > .slot[data-inline-alignment="start"] { + inset-inline-end: unset; +} + +.fixed > .slot[data-inline-alignment="end"] { + inset-inline-start: unset; +} diff --git a/src/grid/GridLayout.tsx b/src/grid/GridLayout.tsx new file mode 100644 index 000000000..18a10cbfc --- /dev/null +++ b/src/grid/GridLayout.tsx @@ -0,0 +1,139 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { CSSProperties, forwardRef, useCallback, useMemo } from "react"; +import { distinctUntilChanged } from "rxjs"; +import { useObservableEagerState } from "observable-hooks"; + +import { GridLayout as GridLayoutModel } from "../state/CallViewModel"; +import styles from "./GridLayout.module.css"; +import { useReactiveState } from "../useReactiveState"; +import { useInitial } from "../useInitial"; +import { + CallLayout, + GridTileModel, + TileModel, + arrangeTiles, +} from "./CallLayout"; +import { DragCallback } from "./Grid"; + +interface GridCSSProperties extends CSSProperties { + "--gap": string; + "--width": string; + "--height": string; +} + +/** + * An implementation of the "grid" layout, in which all participants are shown + * together in a scrolling grid. + */ +export const makeGridLayout: CallLayout = ({ + minBounds, + spotlightAlignment, +}) => ({ + scrollingOnTop: false, + + // The "fixed" (non-scrolling) part of the layout is where the spotlight tile + // lives + fixed: forwardRef(function GridLayoutFixed({ model, Slot }, ref) { + const { width, height } = useObservableEagerState(minBounds); + const alignment = useObservableEagerState( + useInitial(() => + spotlightAlignment.pipe( + distinctUntilChanged( + (a1, a2) => a1.block === a2.block && a1.inline === a2.inline, + ), + ), + ), + ); + const tileModel: TileModel | undefined = useMemo( + () => + model.spotlight && { + type: "spotlight", + vms: model.spotlight, + maximised: false, + }, + [model.spotlight], + ); + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [model.spotlight === undefined, width, height, alignment], + ); + + const onDragSpotlight: DragCallback = useCallback( + ({ xRatio, yRatio }) => + spotlightAlignment.next({ + block: yRatio < 0.5 ? "start" : "end", + inline: xRatio < 0.5 ? "start" : "end", + }), + [], + ); + + return ( +
+ {tileModel && ( + + )} +
+ ); + }), + + // The scrolling part of the layout is where all the grid tiles live + scrolling: forwardRef(function GridLayout({ model, Slot }, ref) { + const { width, height: minHeight } = useObservableEagerState(minBounds); + const { gap, tileWidth, tileHeight } = useMemo( + () => arrangeTiles(width, minHeight, model.grid.length), + [width, minHeight, model.grid.length], + ); + + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [model.grid, width, minHeight], + ); + + const tileModels: GridTileModel[] = useMemo( + () => model.grid.map((vm) => ({ type: "grid", vm })), + [model.grid], + ); + + return ( +
+ {tileModels.map((m) => ( + + ))} +
+ ); + }), +}); diff --git a/src/grid/OneOnOneLayout.module.css b/src/grid/OneOnOneLayout.module.css new file mode 100644 index 000000000..0c22b2536 --- /dev/null +++ b/src/grid/OneOnOneLayout.module.css @@ -0,0 +1,61 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.layer { + block-size: 100%; + display: grid; + place-items: center; +} + +.container { + position: relative; +} + +.local { + position: absolute; + inline-size: 135px; + block-size: 160px; + inset: var(--cpd-space-4x); +} + +@media (min-width: 600px) { + .local { + inline-size: 170px; + block-size: 110px; + } +} + +.spotlight { + position: absolute; + inline-size: 404px; + block-size: 233px; +} + +.slot[data-block-alignment="start"] { + inset-block-end: unset; +} + +.slot[data-block-alignment="end"] { + inset-block-start: unset; +} + +.slot[data-inline-alignment="start"] { + inset-inline-end: unset; +} + +.slot[data-inline-alignment="end"] { + inset-inline-start: unset; +} diff --git a/src/grid/OneOnOneLayout.tsx b/src/grid/OneOnOneLayout.tsx new file mode 100644 index 000000000..635c7898a --- /dev/null +++ b/src/grid/OneOnOneLayout.tsx @@ -0,0 +1,92 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { forwardRef, useCallback, useMemo } from "react"; +import { useObservableEagerState } from "observable-hooks"; +import classNames from "classnames"; + +import { OneOnOneLayout as OneOnOneLayoutModel } from "../state/CallViewModel"; +import { CallLayout, GridTileModel, arrangeTiles } from "./CallLayout"; +import { useReactiveState } from "../useReactiveState"; +import styles from "./OneOnOneLayout.module.css"; +import { DragCallback } from "./Grid"; + +/** + * An implementation of the "one-on-one" layout, in which the remote participant + * is shown at maximum size, overlaid by a small view of the local participant. + */ +export const makeOneOnOneLayout: CallLayout = ({ + minBounds, + pipAlignment, +}) => ({ + scrollingOnTop: false, + + fixed: forwardRef(function OneOnOneLayoutFixed(_props, ref) { + return
; + }), + + scrolling: forwardRef(function OneOnOneLayoutScrolling({ model, Slot }, ref) { + const { width, height } = useObservableEagerState(minBounds); + const pipAlignmentValue = useObservableEagerState(pipAlignment); + const { tileWidth, tileHeight } = useMemo( + () => arrangeTiles(width, height, 1), + [width, height], + ); + + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [width, height, pipAlignmentValue], + ); + + const remoteTileModel: GridTileModel = useMemo( + () => ({ type: "grid", vm: model.remote }), + [model.remote], + ); + const localTileModel: GridTileModel = useMemo( + () => ({ type: "grid", vm: model.local }), + [model.local], + ); + + const onDragLocalTile: DragCallback = useCallback( + ({ xRatio, yRatio }) => + pipAlignment.next({ + block: yRatio < 0.5 ? "start" : "end", + inline: xRatio < 0.5 ? "start" : "end", + }), + [], + ); + + return ( +
+ + + +
+ ); + }), +}); diff --git a/src/grid/SpotlightExpandedLayout.module.css b/src/grid/SpotlightExpandedLayout.module.css new file mode 100644 index 000000000..6556110e2 --- /dev/null +++ b/src/grid/SpotlightExpandedLayout.module.css @@ -0,0 +1,47 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.layer { + block-size: 100%; +} + +.spotlight { + block-size: 100%; + inline-size: 100%; +} + +.pip { + position: absolute; + inline-size: 180px; + block-size: 135px; + inset: var(--cpd-space-4x); +} + +.pip[data-block-alignment="start"] { + inset-block-end: unset; +} + +.pip[data-block-alignment="end"] { + inset-block-start: unset; +} + +.pip[data-inline-alignment="start"] { + inset-inline-end: unset; +} + +.pip[data-inline-alignment="end"] { + inset-inline-start: unset; +} diff --git a/src/grid/SpotlightExpandedLayout.tsx b/src/grid/SpotlightExpandedLayout.tsx new file mode 100644 index 000000000..b92460ee3 --- /dev/null +++ b/src/grid/SpotlightExpandedLayout.tsx @@ -0,0 +1,103 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { forwardRef, useCallback, useMemo } from "react"; +import { useObservableEagerState } from "observable-hooks"; + +import { SpotlightExpandedLayout as SpotlightExpandedLayoutModel } from "../state/CallViewModel"; +import { CallLayout, GridTileModel, SpotlightTileModel } from "./CallLayout"; +import { DragCallback } from "./Grid"; +import styles from "./SpotlightExpandedLayout.module.css"; +import { useReactiveState } from "../useReactiveState"; + +/** + * An implementation of the "expanded spotlight" layout, in which the spotlight + * tile stretches edge-to-edge and is overlaid by a picture-in-picture tile. + */ +export const makeSpotlightExpandedLayout: CallLayout< + SpotlightExpandedLayoutModel +> = ({ minBounds, pipAlignment }) => ({ + scrollingOnTop: true, + + fixed: forwardRef(function SpotlightExpandedLayoutFixed( + { model, Slot }, + ref, + ) { + const { width, height } = useObservableEagerState(minBounds); + + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [width, height], + ); + + const spotlightTileModel: SpotlightTileModel = useMemo( + () => ({ type: "spotlight", vms: model.spotlight, maximised: true }), + [model.spotlight], + ); + + return ( +
+ +
+ ); + }), + + scrolling: forwardRef(function SpotlightExpandedLayoutScrolling( + { model, Slot }, + ref, + ) { + const { width, height } = useObservableEagerState(minBounds); + const pipAlignmentValue = useObservableEagerState(pipAlignment); + + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [width, height, model.pip === undefined, pipAlignmentValue], + ); + + const pipTileModel: GridTileModel | undefined = useMemo( + () => model.pip && { type: "grid", vm: model.pip }, + [model.pip], + ); + + const onDragPip: DragCallback = useCallback( + ({ xRatio, yRatio }) => + pipAlignment.next({ + block: yRatio < 0.5 ? "start" : "end", + inline: xRatio < 0.5 ? "start" : "end", + }), + [], + ); + + return ( +
+ {pipTileModel && ( + + )} +
+ ); + }), +}); diff --git a/src/grid/SpotlightLandscapeLayout.module.css b/src/grid/SpotlightLandscapeLayout.module.css new file mode 100644 index 000000000..8ca91e108 --- /dev/null +++ b/src/grid/SpotlightLandscapeLayout.module.css @@ -0,0 +1,54 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.layer { + block-size: 100%; + display: grid; + --gap: 20px; + gap: var(--gap); + --grid-slot-width: 180px; + grid-template-columns: 1fr var(--grid-slot-width); + grid-template-rows: minmax(1fr, auto); + padding-inline: var(--gap); +} + +.spotlight { + container: spotlight / size; + display: grid; + place-items: center; +} + +/* CSS makes us put a condition here, even though all we want to do is +unconditionally select the container so we can use cq units */ +@container spotlight (width > 0) { + .spotlight > .slot { + inline-size: min(100cqi, 100cqb * (17 / 9)); + block-size: min(100cqb, 100cqi / (4 / 3)); + } +} + +.grid { + display: flex; + flex-wrap: wrap; + gap: var(--gap); + justify-content: center; + align-content: center; +} + +.grid > .slot { + inline-size: 180px; + block-size: 135px; +} diff --git a/src/grid/SpotlightLandscapeLayout.tsx b/src/grid/SpotlightLandscapeLayout.tsx new file mode 100644 index 000000000..e3ca1bf9a --- /dev/null +++ b/src/grid/SpotlightLandscapeLayout.tsx @@ -0,0 +1,98 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { forwardRef, useMemo } from "react"; +import { useObservableEagerState } from "observable-hooks"; +import classNames from "classnames"; + +import { CallLayout, GridTileModel, TileModel } from "./CallLayout"; +import { SpotlightLandscapeLayout as SpotlightLandscapeLayoutModel } from "../state/CallViewModel"; +import styles from "./SpotlightLandscapeLayout.module.css"; +import { useReactiveState } from "../useReactiveState"; + +/** + * An implementation of the "spotlight landscape" layout, in which the spotlight + * tile takes up most of the space on the left, and the grid of participants is + * shown as a scrolling rail on the right. + */ +export const makeSpotlightLandscapeLayout: CallLayout< + SpotlightLandscapeLayoutModel +> = ({ minBounds }) => ({ + scrollingOnTop: false, + + fixed: forwardRef(function SpotlightLandscapeLayoutFixed( + { model, Slot }, + ref, + ) { + const { width, height } = useObservableEagerState(minBounds); + const tileModel: TileModel = useMemo( + () => ({ + type: "spotlight", + vms: model.spotlight, + maximised: false, + }), + [model.spotlight], + ); + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [model.grid.length, width, height], + ); + + return ( +
+
+ +
+
+
+ ); + }), + + scrolling: forwardRef(function SpotlightLandscapeLayoutScrolling( + { model, Slot }, + ref, + ) { + const { width, height } = useObservableEagerState(minBounds); + const tileModels: GridTileModel[] = useMemo( + () => model.grid.map((vm) => ({ type: "grid", vm })), + [model.grid], + ); + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [model.spotlight.length, model.grid, width, height], + ); + + return ( +
+
1, + })} + /> +
+ {tileModels.map((m) => ( + + ))} +
+
+ ); + }), +}); diff --git a/src/grid/SpotlightPortraitLayout.module.css b/src/grid/SpotlightPortraitLayout.module.css new file mode 100644 index 000000000..1ee91334f --- /dev/null +++ b/src/grid/SpotlightPortraitLayout.module.css @@ -0,0 +1,56 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.layer { + block-size: 100%; + display: grid; + --gap: 20px; + gap: var(--gap); + margin-inline: 0; + display: block; +} + +.spotlight { + container: spotlight / size; + display: grid; + place-items: center; + inline-size: 100%; + aspect-ratio: 16 / 9; + margin-block-end: var(--cpd-space-4x); +} + +.spotlight.withIndicators { + margin-block-end: calc(2 * var(--cpd-space-4x) + 2px); +} + +.spotlight > .slot { + inline-size: 100%; + block-size: 100%; +} + +.grid { + display: flex; + flex-wrap: wrap; + gap: var(--grid-gap); + justify-content: center; + align-content: start; + padding-inline: var(--grid-gap); +} + +.grid > .slot { + inline-size: var(--grid-tile-width); + block-size: var(--grid-tile-height); +} diff --git a/src/grid/SpotlightPortraitLayout.tsx b/src/grid/SpotlightPortraitLayout.tsx new file mode 100644 index 000000000..5b82ca627 --- /dev/null +++ b/src/grid/SpotlightPortraitLayout.tsx @@ -0,0 +1,124 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { CSSProperties, forwardRef, useMemo } from "react"; +import { useObservableEagerState } from "observable-hooks"; +import classNames from "classnames"; + +import { + CallLayout, + GridTileModel, + TileModel, + arrangeTiles, +} from "./CallLayout"; +import { SpotlightPortraitLayout as SpotlightPortraitLayoutModel } from "../state/CallViewModel"; +import styles from "./SpotlightPortraitLayout.module.css"; +import { useReactiveState } from "../useReactiveState"; + +interface GridCSSProperties extends CSSProperties { + "--grid-gap": string; + "--grid-tile-width": string; + "--grid-tile-height": string; +} + +/** + * An implementation of the "spotlight portrait" layout, in which the spotlight + * tile is shown across the top of the screen, and the grid of participants + * scrolls behind it. + */ +export const makeSpotlightPortraitLayout: CallLayout< + SpotlightPortraitLayoutModel +> = ({ minBounds }) => ({ + scrollingOnTop: false, + + fixed: forwardRef(function SpotlightPortraitLayoutFixed( + { model, Slot }, + ref, + ) { + const { width, height } = useObservableEagerState(minBounds); + const tileModel: TileModel = useMemo( + () => ({ + type: "spotlight", + vms: model.spotlight, + maximised: true, + }), + [model.spotlight], + ); + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [model.grid.length, width, height], + ); + + return ( +
+
+ +
+
+ ); + }), + + scrolling: forwardRef(function SpotlightPortraitLayoutScrolling( + { model, Slot }, + ref, + ) { + const { width, height } = useObservableEagerState(minBounds); + const { gap, tileWidth, tileHeight } = arrangeTiles( + width, + 0, + model.grid.length, + ); + const tileModels: GridTileModel[] = useMemo( + () => model.grid.map((vm) => ({ type: "grid", vm })), + [model.grid], + ); + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [model.spotlight.length, model.grid, width, height], + ); + + return ( +
+
1, + })} + /> +
+ {tileModels.map((m) => ( + + ))} +
+
+ ); + }), +}); diff --git a/src/video-grid/BigGrid.module.css b/src/grid/TileWrapper.module.css similarity index 71% rename from src/video-grid/BigGrid.module.css rename to src/grid/TileWrapper.module.css index 2201295dc..ed3acda30 100644 --- a/src/video-grid/BigGrid.module.css +++ b/src/grid/TileWrapper.module.css @@ -1,5 +1,5 @@ /* -Copyright 2023 New Vector Ltd +Copyright 2024 New Vector Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,15 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -.bigGrid { - display: grid; - grid-auto-rows: 130px; - gap: var(--cpd-space-2x); +.tile.draggable { + cursor: grab; } -@media (min-width: 800px) { - .bigGrid { - grid-auto-rows: 135px; - gap: var(--cpd-space-5x); - } +.tile.draggable:active { + cursor: grabbing; } diff --git a/src/video-grid/TileWrapper.tsx b/src/grid/TileWrapper.tsx similarity index 58% rename from src/video-grid/TileWrapper.tsx rename to src/grid/TileWrapper.tsx index 5c771e6c9..dcb8e908a 100644 --- a/src/video-grid/TileWrapper.tsx +++ b/src/grid/TileWrapper.tsx @@ -1,5 +1,5 @@ /* -Copyright 2023 New Vector Ltd +Copyright 2023-2024 New Vector Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,83 +14,76 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { memo, ReactNode, RefObject, useRef } from "react"; +import { ComponentType, memo, RefObject, useRef } from "react"; import { EventTypes, Handler, useDrag } from "@use-gesture/react"; -import { SpringValue, to } from "@react-spring/web"; +import { SpringValue } from "@react-spring/web"; +import classNames from "classnames"; -import { ChildrenProperties } from "./VideoGrid"; +import { TileProps } from "./Grid"; +import styles from "./TileWrapper.module.css"; -interface Props { +interface Props { id: string; - onDragRef: RefObject< + onDrag: RefObject< ( tileId: string, state: Parameters>[0], ) => void - >; + > | null; targetWidth: number; targetHeight: number; - data: T; + model: M; + Tile: ComponentType>; opacity: SpringValue; scale: SpringValue; - shadow: SpringValue; - shadowSpread: SpringValue; zIndex: SpringValue; x: SpringValue; y: SpringValue; width: SpringValue; height: SpringValue; - children: (props: ChildrenProperties) => ReactNode; } const TileWrapper_ = memo( - ({ + ({ id, - onDragRef, + onDrag, targetWidth, targetHeight, - data, + model, + Tile, opacity, scale, - shadow, - shadowSpread, zIndex, x, y, width, height, - children, - }: Props) => { - const ref = useRef(null); + }: Props) => { + const ref = useRef(null); - useDrag((state) => onDragRef?.current!(id, state), { + useDrag((state) => onDrag?.current!(id, state), { target: ref, filterTaps: true, preventScroll: true, }); return ( - <> - {children({ - ref, - style: { - opacity, - scale, - zIndex, - x, - y, - width, - height, - boxShadow: to( - [shadow, shadowSpread], - (s, ss) => `rgba(0, 0, 0, 0.5) 0px ${s}px ${2 * s}px ${ss}px`, - ), - }, - targetWidth, - targetHeight, - data, - })} - + ); }, ); @@ -104,4 +97,6 @@ TileWrapper_.displayName = "TileWrapper"; // We pretend this component is a simple function rather than a // NamedExoticComponent, because that's the only way we can fit in a type // parameter -export const TileWrapper = TileWrapper_ as (props: Props) => JSX.Element; +export const TileWrapper = TileWrapper_ as ( + props: Props, +) => JSX.Element; diff --git a/src/home/RegisteredView.tsx b/src/home/RegisteredView.tsx index 35e958ab0..1c1b0d3af 100644 --- a/src/home/RegisteredView.tsx +++ b/src/home/RegisteredView.tsx @@ -38,9 +38,12 @@ import { UserMenuContainer } from "../UserMenuContainer"; import { JoinExistingCallModal } from "./JoinExistingCallModal"; import { Caption } from "../typography/Typography"; import { Form } from "../form/Form"; -import { useOptInAnalytics } from "../settings/useSetting"; import { AnalyticsNotice } from "../analytics/AnalyticsNotice"; import { E2eeType } from "../e2ee/e2eeType"; +import { + useSetting, + optInAnalytics as optInAnalyticsSetting, +} from "../settings/settings"; interface Props { client: MatrixClient; @@ -49,7 +52,7 @@ interface Props { export const RegisteredView: FC = ({ client }) => { const [loading, setLoading] = useState(false); const [error, setError] = useState(); - const [optInAnalytics] = useOptInAnalytics(); + const [optInAnalytics] = useSetting(optInAnalyticsSetting); const history = useHistory(); const { t } = useTranslation(); const [joinExistingCallModalOpen, setJoinExistingCallModalOpen] = diff --git a/src/home/UnauthenticatedView.tsx b/src/home/UnauthenticatedView.tsx index d5f00fea5..35cc832e2 100644 --- a/src/home/UnauthenticatedView.tsx +++ b/src/home/UnauthenticatedView.tsx @@ -41,15 +41,18 @@ import styles from "./UnauthenticatedView.module.css"; import commonStyles from "./common.module.css"; import { generateRandomName } from "../auth/generateRandomName"; import { AnalyticsNotice } from "../analytics/AnalyticsNotice"; -import { useOptInAnalytics } from "../settings/useSetting"; import { Config } from "../config/Config"; import { E2eeType } from "../e2ee/e2eeType"; +import { + useSetting, + optInAnalytics as optInAnalyticsSetting, +} from "../settings/settings"; export const UnauthenticatedView: FC = () => { const { setClient } = useClient(); const [loading, setLoading] = useState(false); const [error, setError] = useState(); - const [optInAnalytics] = useOptInAnalytics(); + const [optInAnalytics] = useSetting(optInAnalyticsSetting); const { recaptchaKey, register } = useInteractiveRegistration(); const { execute, reset, recaptchaId } = useRecaptcha(recaptchaKey); diff --git a/src/livekit/MediaDevicesContext.tsx b/src/livekit/MediaDevicesContext.tsx index b42a0afa7..2f6549466 100644 --- a/src/livekit/MediaDevicesContext.tsx +++ b/src/livekit/MediaDevicesContext.tsx @@ -29,11 +29,12 @@ import { Observable } from "rxjs"; import { logger } from "matrix-js-sdk/src/logger"; import { - isFirefox, - useAudioInput, - useAudioOutput, - useVideoInput, -} from "../settings/useSetting"; + useSetting, + audioInput as audioInputSetting, + audioOutput as audioOutputSetting, + videoInput as videoInputSetting, +} from "../settings/settings"; +import { isFirefox } from "../Platform"; export interface MediaDevice { available: MediaDeviceInfo[]; @@ -145,43 +146,36 @@ export const MediaDevicesProvider: FC = ({ children }) => { // for ouput devices because the selector wont be shown on FF. const useOutputNames = usingNames && !isFirefox(); - const [audioInputSetting, setAudioInputSetting] = useAudioInput(); - const [audioOutputSetting, setAudioOutputSetting] = useAudioOutput(); - const [videoInputSetting, setVideoInputSetting] = useVideoInput(); + const [storedAudioInput, setStoredAudioInput] = useSetting(audioInputSetting); + const [storedAudioOutput, setStoredAudioOutput] = + useSetting(audioOutputSetting); + const [storedVideoInput, setStoredVideoInput] = useSetting(videoInputSetting); - const audioInput = useMediaDevice( - "audioinput", - audioInputSetting, - usingNames, - ); + const audioInput = useMediaDevice("audioinput", storedAudioInput, usingNames); const audioOutput = useMediaDevice( "audiooutput", - audioOutputSetting, + storedAudioOutput, useOutputNames, alwaysUseDefaultAudio, ); - const videoInput = useMediaDevice( - "videoinput", - videoInputSetting, - usingNames, - ); + const videoInput = useMediaDevice("videoinput", storedVideoInput, usingNames); useEffect(() => { if (audioInput.selectedId !== undefined) - setAudioInputSetting(audioInput.selectedId); - }, [setAudioInputSetting, audioInput.selectedId]); + setStoredAudioInput(audioInput.selectedId); + }, [setStoredAudioInput, audioInput.selectedId]); useEffect(() => { // Skip setting state for ff output. Redundent since it is set to always return 'undefined' // but makes it clear while debugging that this is not happening on FF. + perf ;) if (audioOutput.selectedId !== undefined && !isFirefox()) - setAudioOutputSetting(audioOutput.selectedId); - }, [setAudioOutputSetting, audioOutput.selectedId]); + setStoredAudioOutput(audioOutput.selectedId); + }, [setStoredAudioOutput, audioOutput.selectedId]); useEffect(() => { if (videoInput.selectedId !== undefined) - setVideoInputSetting(videoInput.selectedId); - }, [setVideoInputSetting, videoInput.selectedId]); + setStoredVideoInput(videoInput.selectedId); + }, [setStoredVideoInput, videoInput.selectedId]); const startUsingDeviceNames = useCallback( () => setNumCallersUsingNames((n) => n + 1), diff --git a/src/observable-utils.ts b/src/observable-utils.ts index 1d6a59e87..bc0135c3b 100644 --- a/src/observable-utils.ts +++ b/src/observable-utils.ts @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { Observable, defer, finalize, tap } from "rxjs"; +import { Observable, defer, finalize, scan, startWith, tap } from "rxjs"; const nothing = Symbol("nothing"); @@ -35,3 +35,15 @@ export function finalizeValue(callback: (finalValue: T) => void) { ); }); } + +/** + * RxJS operator that accumulates a state from a source of events. This is like + * scan, except it emits an initial value immediately before any events arrive. + */ +export function accumulate( + initial: State, + update: (state: State, event: Event) => State, +) { + return (events: Observable): Observable => + events.pipe(scan(update, initial), startWith(initial)); +} diff --git a/src/room/InCallView.module.css b/src/room/InCallView.module.css index 4bbb6f33f..b8cf9f5e7 100644 --- a/src/room/InCallView.module.css +++ b/src/room/InCallView.module.css @@ -1,5 +1,5 @@ /* -Copyright 2021 New Vector Ltd +Copyright 2021-2024 New Vector Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ limitations under the License. flex-direction: column; height: 100%; width: 100%; + overflow-y: auto; } .controlsOverlay { @@ -46,9 +47,21 @@ limitations under the License. margin-bottom: 0; } +.header { + position: sticky; + inset-block-start: 0; + z-index: 1; + background: linear-gradient( + 0deg, + rgba(0, 0, 0, 0) 0%, + var(--cpd-color-bg-canvas-default) 100% + ); +} + .footer { position: sticky; inset-block-end: 0; + z-index: 1; display: grid; grid-template-columns: 1fr auto 1fr; grid-template-areas: "logo buttons layout"; @@ -109,3 +122,44 @@ limitations under the License. .footerHidden { display: none; } + +.footer.overlay { + position: absolute; + inset-block-end: 0; + inset-inline: 0; +} + +.fixedGrid { + position: absolute; + inline-size: 100%; + align-self: center; +} + +.scrollingGrid { + position: relative; + flex-grow: 1; + inline-size: 100%; + align-self: center; +} + +.fixedGrid, +.scrollingGrid { + /* Disable pointer events so the overlay doesn't block interaction with + elements behind it */ + pointer-events: none; +} + +.fixedGrid > :not(:first-child), +.scrollingGrid > :not(:first-child) { + pointer-events: initial; +} + +.tile { + position: absolute; + inset-block-start: 0; +} + +.tile.maximised { + position: relative; + flex-grow: 1; +} diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index d655c672d..43808b8e4 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -1,5 +1,5 @@ /* -Copyright 2022 - 2023 New Vector Ltd +Copyright 2022 - 2024 New Vector Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,31 +14,29 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { ResizeObserver } from "@juggle/resize-observer"; import { RoomAudioRenderer, RoomContext, useLocalParticipant, - useTracks, } from "@livekit/components-react"; import { usePreventScroll } from "@react-aria/overlays"; -import { ConnectionState, Room, Track } from "livekit-client"; +import { ConnectionState, Room } from "livekit-client"; import { MatrixClient } from "matrix-js-sdk/src/client"; import { FC, - ReactNode, - Ref, + PropsWithoutRef, + forwardRef, useCallback, useEffect, useMemo, useRef, useState, } from "react"; -import { useTranslation } from "react-i18next"; import useMeasure from "react-use-measure"; import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; import classNames from "classnames"; -import { useStateObservable } from "@react-rxjs/core"; +import { BehaviorSubject } from "rxjs"; +import { useObservableEagerState } from "observable-hooks"; import LogoMark from "../icons/LogoMark.svg?react"; import LogoType from "../icons/LogoType.svg?react"; @@ -51,21 +49,16 @@ import { SettingsButton, } from "../button"; import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header"; -import { useVideoGridLayout, VideoGrid } from "../video-grid/VideoGrid"; import { useUrlParams } from "../UrlParams"; import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts"; -import { usePrefersReducedMotion } from "../usePrefersReducedMotion"; import { ElementWidgetActions, widget } from "../widget"; import styles from "./InCallView.module.css"; -import { VideoTile } from "../video-grid/VideoTile"; -import { NewVideoGrid } from "../video-grid/NewVideoGrid"; +import { GridTile } from "../tile/GridTile"; import { OTelGroupCallMembership } from "../otel/OTelGroupCallMembership"; import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal"; import { useRageshakeRequestModal } from "../settings/submit-rageshake"; import { RageshakeRequestModal } from "./RageshakeRequestModal"; import { useLiveKit } from "../livekit/useLiveKit"; -import { useFullscreen } from "./useFullscreen"; -import { useLayoutStates } from "../video-grid/Layout"; import { useWakeLock } from "../useWakeLock"; import { useMergedRefs } from "../useMergedRefs"; import { MuteStates } from "./MuteStates"; @@ -74,13 +67,26 @@ import { InviteButton } from "../button/InviteButton"; import { LayoutToggle } from "./LayoutToggle"; import { ECConnectionState } from "../livekit/useECConnectionState"; import { useOpenIDSFU } from "../livekit/openIDSFU"; -import { useCallViewModel } from "../state/CallViewModel"; -import { subscribe } from "../state/subscribe"; +import { GridMode, Layout, useCallViewModel } from "../state/CallViewModel"; +import { Grid, TileProps } from "../grid/Grid"; +import { useObservable } from "../state/useObservable"; +import { useInitial } from "../useInitial"; +import { SpotlightTile } from "../tile/SpotlightTile"; import { EncryptionSystem } from "../e2ee/sharedKeyManagement"; import { E2eeType } from "../e2ee/e2eeType"; +import { makeGridLayout } from "../grid/GridLayout"; +import { + CallLayoutOutputs, + TileModel, + defaultPipAlignment, + defaultSpotlightAlignment, +} from "../grid/CallLayout"; +import { makeOneOnOneLayout } from "../grid/OneOnOneLayout"; +import { makeSpotlightExpandedLayout } from "../grid/SpotlightExpandedLayout"; +import { makeSpotlightLandscapeLayout } from "../grid/SpotlightLandscapeLayout"; +import { makeSpotlightPortraitLayout } from "../grid/SpotlightPortraitLayout"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); -const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); export interface ActiveCallProps extends Omit { @@ -126,342 +132,403 @@ export interface InCallViewProps { onShareClick: (() => void) | null; } -export const InCallView: FC = subscribe( - ({ - client, - matrixInfo, - rtcSession, +export const InCallView: FC = ({ + client, + matrixInfo, + rtcSession, + livekitRoom, + muteStates, + participantCount, + onLeave, + hideHeader, + connState, + onShareClick, +}) => { + usePreventScroll(); + useWakeLock(); + + useEffect(() => { + if (connState === ConnectionState.Disconnected) { + // annoyingly we don't get the disconnection reason this way, + // only by listening for the emitted event + onLeave(new Error("Disconnected from call server")); + } + }, [connState, onLeave]); + + const containerRef1 = useRef(null); + const [containerRef2, bounds] = useMeasure(); + const boundsValid = bounds.height > 0; + // Merge the refs so they can attach to the same element + const containerRef = useMergedRefs(containerRef1, containerRef2); + + const { hideScreensharing, showControls } = useUrlParams(); + + const { isScreenShareEnabled, localParticipant } = useLocalParticipant({ + room: livekitRoom, + }); + + const toggleMicrophone = useCallback( + () => muteStates.audio.setEnabled?.((e) => !e), + [muteStates], + ); + const toggleCamera = useCallback( + () => muteStates.video.setEnabled?.((e) => !e), + [muteStates], + ); + + // This function incorrectly assumes that there is a camera and microphone, which is not always the case. + // TODO: Make sure that this module is resilient when it comes to camera/microphone availability! + useCallViewKeyboardShortcuts( + containerRef1, + toggleMicrophone, + toggleCamera, + (muted) => muteStates.audio.setEnabled?.(!muted), + ); + + const mobile = boundsValid && bounds.width <= 660; + const reducedControls = boundsValid && bounds.width <= 340; + const noControls = reducedControls && bounds.height <= 400; + + const vm = useCallViewModel( + rtcSession.room, livekitRoom, - muteStates, - participantCount, - onLeave, - hideHeader, - otelGroupCallMembership, + matrixInfo.e2eeSystem.kind !== E2eeType.NONE, connState, - onShareClick, - }) => { - const { t } = useTranslation(); - usePreventScroll(); - useWakeLock(); - - useEffect(() => { - if (connState === ConnectionState.Disconnected) { - // annoyingly we don't get the disconnection reason this way, - // only by listening for the emitted event - onLeave(new Error("Disconnected from call server")); - } - }, [connState, onLeave]); - - const containerRef1 = useRef(null); - const [containerRef2, bounds] = useMeasure({ polyfill: ResizeObserver }); - const boundsValid = bounds.height > 0; - // Merge the refs so they can attach to the same element - const containerRef = useMergedRefs(containerRef1, containerRef2); - - const screenSharingTracks = useTracks( - [{ source: Track.Source.ScreenShare, withPlaceholder: false }], - { - room: livekitRoom, - }, - ); - const { layout, setLayout } = useVideoGridLayout( - screenSharingTracks.length > 0, - ); + ); + const windowMode = useObservableEagerState(vm.windowMode); + const layout = useObservableEagerState(vm.layout); + const gridMode = useObservableEagerState(vm.gridMode); - const { hideScreensharing, showControls } = useUrlParams(); + const [settingsModalOpen, setSettingsModalOpen] = useState(false); + const [settingsTab, setSettingsTab] = useState(defaultSettingsTab); - const { isScreenShareEnabled, localParticipant } = useLocalParticipant({ - room: livekitRoom, - }); + const openSettings = useCallback( + () => setSettingsModalOpen(true), + [setSettingsModalOpen], + ); + const closeSettings = useCallback( + () => setSettingsModalOpen(false), + [setSettingsModalOpen], + ); - const toggleMicrophone = useCallback( - () => muteStates.audio.setEnabled?.((e) => !e), - [muteStates], - ); - const toggleCamera = useCallback( - () => muteStates.video.setEnabled?.((e) => !e), - [muteStates], - ); + const openProfile = useCallback(() => { + setSettingsTab("profile"); + setSettingsModalOpen(true); + }, [setSettingsTab, setSettingsModalOpen]); + + const [headerRef, headerBounds] = useMeasure(); + const [footerRef, footerBounds] = useMeasure(); + + const gridBounds = useMemo( + () => ({ + width: bounds.width, + height: + bounds.height - + headerBounds.height - + (windowMode === "flat" ? 0 : footerBounds.height), + }), + [ + bounds.width, + bounds.height, + headerBounds.height, + footerBounds.height, + windowMode, + ], + ); + const gridBoundsObservable = useObservable(gridBounds); + + const spotlightAlignment = useInitial( + () => new BehaviorSubject(defaultSpotlightAlignment), + ); + const pipAlignment = useInitial( + () => new BehaviorSubject(defaultPipAlignment), + ); - // This function incorrectly assumes that there is a camera and microphone, which is not always the case. - // TODO: Make sure that this module is resilient when it comes to camera/microphone availability! - useCallViewKeyboardShortcuts( - containerRef1, - toggleMicrophone, - toggleCamera, - (muted) => muteStates.audio.setEnabled?.(!muted), + const setGridMode = useCallback( + (mode: GridMode) => vm.setGridMode(mode), + [vm], + ); + + useEffect(() => { + widget?.api.transport.send( + gridMode === "grid" + ? ElementWidgetActions.TileLayout + : ElementWidgetActions.SpotlightLayout, + {}, ); + }, [gridMode]); - useEffect(() => { - widget?.api.transport.send( - layout === "grid" - ? ElementWidgetActions.TileLayout - : ElementWidgetActions.SpotlightLayout, - {}, + useEffect(() => { + if (widget) { + const onTileLayout = (ev: CustomEvent): void => { + setGridMode("grid"); + widget!.api.transport.reply(ev.detail, {}); + }; + const onSpotlightLayout = (ev: CustomEvent): void => { + setGridMode("spotlight"); + widget!.api.transport.reply(ev.detail, {}); + }; + + widget.lazyActions.on(ElementWidgetActions.TileLayout, onTileLayout); + widget.lazyActions.on( + ElementWidgetActions.SpotlightLayout, + onSpotlightLayout, ); - }, [layout]); - - useEffect(() => { - if (widget) { - const onTileLayout = (ev: CustomEvent): void => { - setLayout("grid"); - widget!.api.transport.reply(ev.detail, {}); - }; - const onSpotlightLayout = ( - ev: CustomEvent, - ): void => { - setLayout("spotlight"); - widget!.api.transport.reply(ev.detail, {}); - }; - - widget.lazyActions.on(ElementWidgetActions.TileLayout, onTileLayout); - widget.lazyActions.on( + + return (): void => { + widget!.lazyActions.off(ElementWidgetActions.TileLayout, onTileLayout); + widget!.lazyActions.off( ElementWidgetActions.SpotlightLayout, onSpotlightLayout, ); + }; + } + }, [setGridMode]); - return (): void => { - widget!.lazyActions.off( - ElementWidgetActions.TileLayout, - onTileLayout, - ); - widget!.lazyActions.off( - ElementWidgetActions.SpotlightLayout, - onSpotlightLayout, - ); - }; - } - }, [setLayout]); - - const mobile = boundsValid && bounds.width <= 660; - const reducedControls = boundsValid && bounds.width <= 340; - const noControls = reducedControls && bounds.height <= 400; - - const vm = useCallViewModel( - rtcSession.room, - livekitRoom, - matrixInfo.e2eeSystem.kind !== E2eeType.NONE, - connState, - ); - const items = useStateObservable(vm.tiles); - const { fullscreenItem, toggleFullscreen, exitFullscreen } = - useFullscreen(items); - - // The maximised participant: either the participant that the user has - // manually put in fullscreen, or the focused (active) participant if the - // window is too small to show everyone - const maximisedParticipant = useMemo( - () => - fullscreenItem ?? - (noControls - ? (items.find((item) => item.isSpeaker) ?? items.at(0) ?? null) - : null), - [fullscreenItem, noControls, items], - ); - - const Grid = - items.length > 12 && layout === "grid" ? NewVideoGrid : VideoGrid; - - const prefersReducedMotion = usePrefersReducedMotion(); - - // This state is lifted out of NewVideoGrid so that layout states can be - // restored after a layout switch or upon exiting fullscreen - const layoutStates = useLayoutStates(); + const toggleSpotlightExpanded = useCallback( + () => vm.toggleSpotlightExpanded(), + [vm], + ); - const renderContent = (): JSX.Element => { - if (items.length === 0) { - return ( -
-

{t("waiting_for_participants")}

-
+ const Tile = useMemo( + () => + forwardRef< + HTMLDivElement, + PropsWithoutRef> + >(function Tile( + { className, style, targetWidth, targetHeight, model }, + ref, + ) { + const spotlightExpanded = useObservableEagerState(vm.spotlightExpanded); + const showSpeakingIndicatorsValue = useObservableEagerState( + vm.showSpeakingIndicators, ); - } - if (maximisedParticipant) { - return ( - + ) : ( + ); - } + }), + [vm, toggleSpotlightExpanded, openProfile], + ); + + const layouts = useMemo(() => { + const inputs = { + minBounds: gridBoundsObservable, + spotlightAlignment, + pipAlignment, + }; + return { + grid: makeGridLayout(inputs), + "spotlight-landscape": makeSpotlightLandscapeLayout(inputs), + "spotlight-portrait": makeSpotlightPortraitLayout(inputs), + "spotlight-expanded": makeSpotlightExpandedLayout(inputs), + "one-on-one": makeOneOnOneLayout(inputs), + }; + }, [gridBoundsObservable, spotlightAlignment, pipAlignment]); + const renderContent = (): JSX.Element => { + if (layout.type === "pip") { return ( - - {({ data: vm, ...props }): ReactNode => ( - 2} - onOpenProfile={openProfile} - {...props} - ref={props.ref as Ref} - /> - )} - + ); - }; + } - const rageshakeRequestModalProps = useRageshakeRequestModal( - rtcSession.room.roomId, + const layers = layouts[layout.type] as CallLayoutOutputs; + const fixedGrid = ( + ); - - const [settingsModalOpen, setSettingsModalOpen] = useState(false); - const [settingsTab, setSettingsTab] = useState(defaultSettingsTab); - - const openSettings = useCallback( - () => setSettingsModalOpen(true), - [setSettingsModalOpen], + const scrollingGrid = ( + ); - const closeSettings = useCallback( - () => setSettingsModalOpen(false), - [setSettingsModalOpen], + // The grid tiles go *under* the spotlight in the portrait layout, but + // *over* the spotlight in the expanded layout + return layout.type === "spotlight-expanded" ? ( + <> + {fixedGrid} + {scrollingGrid} + + ) : ( + <> + {scrollingGrid} + {fixedGrid} + ); + }; - const openProfile = useCallback(() => { - setSettingsTab("profile"); - setSettingsModalOpen(true); - }, [setSettingsTab, setSettingsModalOpen]); - - const toggleScreensharing = useCallback(async () => { - exitFullscreen(); - await localParticipant.setScreenShareEnabled(!isScreenShareEnabled, { - audio: true, - selfBrowserSurface: "include", - surfaceSwitching: "include", - systemAudio: "include", - }); - }, [localParticipant, isScreenShareEnabled, exitFullscreen]); - - let footer: JSX.Element | null; - - if (noControls) { - footer = null; - } else { - const buttons: JSX.Element[] = []; - - buttons.push( - , - , - ); + const rageshakeRequestModalProps = useRageshakeRequestModal( + rtcSession.room.roomId, + ); - if (!reducedControls) { - if (canScreenshare && !hideScreensharing) { - buttons.push( - , - ); - } - buttons.push(); - } + const toggleScreensharing = useCallback(async () => { + await localParticipant.setScreenShareEnabled(!isScreenShareEnabled, { + audio: true, + selfBrowserSurface: "include", + surfaceSwitching: "include", + systemAudio: "include", + }); + }, [localParticipant, isScreenShareEnabled]); + + let footer: JSX.Element | null; + + if (noControls) { + footer = null; + } else { + const buttons: JSX.Element[] = []; + + buttons.push( + , + , + ); - buttons.push( - , - ); - footer = ( -
- {!mobile && !hideHeader && ( -
- - -
- )} - {showControls &&
{buttons}
} - {!mobile && !hideHeader && showControls && ( - - )} -
- ); + if (!reducedControls) { + if (canScreenshare && !hideScreensharing) { + buttons.push( + , + ); + } + buttons.push(); } - return ( -
- {!hideHeader && maximisedParticipant === null && ( -
- - - - - {!reducedControls && showControls && onShareClick !== null && ( - - )} - -
+ buttons.push( + , + ); + footer = ( +
- - {renderContent()} - {footer} -
- {!noControls && ( - + > + {!mobile && !hideHeader && ( +
+ + +
+ )} + {showControls &&
{buttons}
} + {!mobile && !hideHeader && showControls && ( + )} -
); - }, -); + } + + return ( +
+ {!hideHeader && windowMode !== "pip" && windowMode !== "flat" && ( +
+ + + + + {!reducedControls && showControls && onShareClick !== null && ( + + )} + +
+ )} + + {renderContent()} + {footer} + {!noControls && } + +
+ ); +}; diff --git a/src/room/RoomPage.tsx b/src/room/RoomPage.tsx index 2f18cb638..1a14fb7ed 100644 --- a/src/room/RoomPage.tsx +++ b/src/room/RoomPage.tsx @@ -26,7 +26,6 @@ import { GroupCallLoader } from "./GroupCallLoader"; import { GroupCallView } from "./GroupCallView"; import { useRoomIdentifier, useUrlParams } from "../UrlParams"; import { useRegisterPasswordlessUser } from "../auth/useRegisterPasswordlessUser"; -import { useOptInAnalytics } from "../settings/useSetting"; import { HomePage } from "../home/HomePage"; import { platform } from "../Platform"; import { AppSelectionModal } from "./AppSelectionModal"; @@ -36,6 +35,10 @@ import { LobbyView } from "./LobbyView"; import { E2eeType } from "../e2ee/e2eeType"; import { useProfile } from "../profile/useProfile"; import { useMuteStates } from "./MuteStates"; +import { + useSetting, + optInAnalytics as optInAnalyticsSetting, +} from "../settings/settings"; export const RoomPage: FC = () => { const { @@ -80,7 +83,7 @@ export const RoomPage: FC = () => { registerPasswordlessUser, ]); - const [optInAnalytics, setOptInAnalytics] = useOptInAnalytics(); + const [optInAnalytics, setOptInAnalytics] = useSetting(optInAnalyticsSetting); useEffect(() => { // During the beta, opt into analytics by default if (optInAnalytics === null && setOptInAnalytics) setOptInAnalytics(true); diff --git a/src/room/VideoPreview.module.css b/src/room/VideoPreview.module.css index ad7b2671c..e858c3c27 100644 --- a/src/room/VideoPreview.module.css +++ b/src/room/VideoPreview.module.css @@ -18,20 +18,12 @@ limitations under the License. margin-inline: var(--inline-content-inset); min-block-size: 0; block-size: 50vh; -} - -.preview.content { - margin-inline: 0; -} - -.content { + border-radius: var(--cpd-space-4x); position: relative; - block-size: 100%; - inline-size: 100%; overflow: hidden; } -.content video { +.preview > video { width: 100%; height: 100%; object-fit: cover; @@ -69,12 +61,20 @@ limitations under the License. ); } -.preview.content .buttonBar { - padding-inline: var(--inline-content-inset); -} - @media (min-aspect-ratio: 1 / 1) { - .preview video { + .preview > video { aspect-ratio: 16 / 9; } } + +@media (max-width: 550px) { + .preview { + margin-inline: 0; + border-radius: 0; + block-size: 100%; + } + + .buttonBar { + padding-inline: var(--inline-content-inset); + } +} diff --git a/src/room/VideoPreview.tsx b/src/room/VideoPreview.tsx index 2f6dbbbc0..5899a8bfc 100644 --- a/src/room/VideoPreview.tsx +++ b/src/room/VideoPreview.tsx @@ -1,5 +1,5 @@ /* -Copyright 2022 - 2023 New Vector Ltd +Copyright 2022 - 2024 New Vector Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,20 +18,15 @@ import { useEffect, useMemo, useRef, FC, ReactNode, useCallback } from "react"; import useMeasure from "react-use-measure"; import { ResizeObserver } from "@juggle/resize-observer"; import { usePreviewTracks } from "@livekit/components-react"; -import { - CreateLocalTracksOptions, - LocalVideoTrack, - Track, -} from "livekit-client"; +import { LocalVideoTrack, Track } from "livekit-client"; import classNames from "classnames"; import { logger } from "matrix-js-sdk/src/logger"; -import { Glass } from "@vector-im/compound-web"; import { Avatar } from "../Avatar"; import styles from "./VideoPreview.module.css"; import { useMediaDevices } from "../livekit/MediaDevicesContext"; import { MuteStates } from "./MuteStates"; -import { useMediaQuery } from "../useMediaQuery"; +import { useInitial } from "../useInitial"; import { EncryptionSystem } from "../e2ee/sharedKeyManagement"; export type MatrixInfo = { @@ -63,10 +58,10 @@ export const VideoPreview: FC = ({ // Capture the audio options as they were when we first mounted, because // we're not doing anything with the audio anyway so we don't need to // re-open the devices when they change (see below). - const initialAudioOptions = useRef(); - initialAudioOptions.current ??= muteStates.audio.enabled && { - deviceId: devices.audioInput.selectedId, - }; + const initialAudioOptions = useInitial( + () => + muteStates.audio.enabled && { deviceId: devices.audioInput.selectedId }, + ); const localTrackOptions = useMemo( () => ({ @@ -76,12 +71,16 @@ export const VideoPreview: FC = ({ // reference the initial values here. // We also pass in a clone because livekit mutates the object passed in, // which would cause the devices to be re-opened on the next render. - audio: Object.assign({}, initialAudioOptions.current), + audio: Object.assign({}, initialAudioOptions), video: muteStates.video.enabled && { deviceId: devices.videoInput.selectedId, }, }), - [devices.videoInput.selectedId, muteStates.video.enabled], + [ + initialAudioOptions, + devices.videoInput.selectedId, + muteStates.video.enabled, + ], ); const onError = useCallback( @@ -115,8 +114,8 @@ export const VideoPreview: FC = ({ }; }, [videoTrack]); - const content = ( - <> + return ( +
)}
{children}
- - ); - - return useMediaQuery("(max-width: 550px)") ? ( -
- {content}
- ) : ( - -
- {content} -
- ); }; diff --git a/src/room/useFullscreen.ts b/src/room/useFullscreen.ts index 39ea6e789..a80b7ff56 100644 --- a/src/room/useFullscreen.ts +++ b/src/room/useFullscreen.ts @@ -20,7 +20,6 @@ import { useCallback, useLayoutEffect, useRef } from "react"; import { useReactiveState } from "../useReactiveState"; import { useEventTarget } from "../useEvents"; -import { TileDescriptor } from "../state/CallViewModel"; const isFullscreen = (): boolean => Boolean(document.fullscreenElement) || @@ -55,31 +54,30 @@ function useFullscreenChange(onFullscreenChange: () => void): void { * Provides callbacks for controlling the full-screen view, which can hold one * item at a time. */ -export function useFullscreen(items: TileDescriptor[]): { - fullscreenItem: TileDescriptor | null; +// TODO: Simplify this. Nowadays we only allow the spotlight to be fullscreen, +// so we don't need to bother with multiple items. +export function useFullscreen(items: string[]): { + fullscreenItem: string | null; toggleFullscreen: (itemId: string) => void; exitFullscreen: () => void; } { - const [fullscreenItem, setFullscreenItem] = - useReactiveState | null>( - (prevItem) => - prevItem == null - ? null - : (items.find((i) => i.id === prevItem.id) ?? null), - [items], - ); - - const latestItems = useRef[]>(items); + const [fullscreenItem, setFullscreenItem] = useReactiveState( + (prevItem) => + prevItem == null ? null : (items.find((i) => i === prevItem) ?? null), + [items], + ); + + const latestItems = useRef(items); latestItems.current = items; - const latestFullscreenItem = useRef | null>(fullscreenItem); + const latestFullscreenItem = useRef(fullscreenItem); latestFullscreenItem.current = fullscreenItem; const toggleFullscreen = useCallback( (itemId: string) => { setFullscreenItem( latestFullscreenItem.current === null - ? (latestItems.current.find((i) => i.id === itemId) ?? null) + ? (latestItems.current.find((i) => i === itemId) ?? null) : null, ); }, diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx index a86f20c53..d84300227 100644 --- a/src/settings/SettingsModal.tsx +++ b/src/settings/SettingsModal.tsx @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { ChangeEvent, FC, Key, ReactNode } from "react"; +import { ChangeEvent, FC, Key, ReactNode, useCallback } from "react"; import { Item } from "@react-stately/collections"; import { Trans, useTranslation } from "react-i18next"; import { MatrixClient } from "matrix-js-sdk"; @@ -29,12 +29,6 @@ import OverflowIcon from "../icons/Overflow.svg?react"; import UserIcon from "../icons/User.svg?react"; import FeedbackIcon from "../icons/Feedback.svg?react"; import { SelectInput } from "../input/SelectInput"; -import { - useOptInAnalytics, - useDeveloperSettingsTab, - useShowConnectionStats, - isFirefox, -} from "./useSetting"; import { FieldRow, InputField } from "../input/Input"; import { Body, Caption } from "../typography/Typography"; import { AnalyticsNotice } from "../analytics/AnalyticsNotice"; @@ -46,6 +40,13 @@ import { useMediaDeviceNames, } from "../livekit/MediaDevicesContext"; import { widget } from "../widget"; +import { + useSetting, + optInAnalytics as optInAnalyticsSetting, + developerSettingsTab as developerSettingsTabSetting, + duplicateTiles as duplicateTilesSetting, +} from "./settings"; +import { isFirefox } from "../Platform"; type SettingsTab = | "audio" @@ -76,11 +77,11 @@ export const SettingsModal: FC = ({ }) => { const { t } = useTranslation(); - const [optInAnalytics, setOptInAnalytics] = useOptInAnalytics(); - const [developerSettingsTab, setDeveloperSettingsTab] = - useDeveloperSettingsTab(); - const [showConnectionStats, setShowConnectionStats] = - useShowConnectionStats(); + const [optInAnalytics, setOptInAnalytics] = useSetting(optInAnalyticsSetting); + const [developerSettingsTab, setDeveloperSettingsTab] = useSetting( + developerSettingsTabSetting, + ); + const [duplicateTiles, setDuplicateTiles] = useSetting(duplicateTilesSetting); // Generate a `SelectInput` with a list of devices for a given device kind. const generateDeviceSelection = ( @@ -247,14 +248,16 @@ export const SettingsModal: FC = ({ ): void => - setShowConnectionStats(e.target.checked) - } + id="duplicateTiles" + type="number" + label={t("settings.duplicate_tiles_label")} + value={duplicateTiles.toString()} + onChange={useCallback( + (event: ChangeEvent): void => { + setDuplicateTiles(event.target.valueAsNumber); + }, + [setDuplicateTiles], + )} /> diff --git a/src/settings/settings.ts b/src/settings/settings.ts new file mode 100644 index 000000000..9c181ccf5 --- /dev/null +++ b/src/settings/settings.ts @@ -0,0 +1,94 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { logger } from "matrix-js-sdk/src/logger"; +import { BehaviorSubject, Observable } from "rxjs"; +import { useObservableEagerState } from "observable-hooks"; + +import { PosthogAnalytics } from "../analytics/PosthogAnalytics"; + +export class Setting { + public constructor(key: string, defaultValue: T) { + this.key = `matrix-setting-${key}`; + + const storedValue = localStorage.getItem(this.key); + let initialValue = defaultValue; + if (storedValue !== null) { + try { + initialValue = JSON.parse(storedValue); + } catch (e) { + logger.warn(`Invalid value stored for setting ${key}: ${storedValue}`); + } + } + + this._value = new BehaviorSubject(initialValue); + this.value = this._value; + } + + private readonly key: string; + + private readonly _value: BehaviorSubject; + public readonly value: Observable; + + public readonly setValue = (value: T): void => { + this._value.next(value); + localStorage.setItem(this.key, JSON.stringify(value)); + }; +} + +/** + * React hook that returns a settings's current value and a setter. + */ +export function useSetting(setting: Setting): [T, (value: T) => void] { + return [useObservableEagerState(setting.value), setting.setValue]; +} + +// null = undecided +export const optInAnalytics = new Setting( + "opt-in-analytics", + null, +); +// TODO: This setting can be disabled. Work out an approach to disableable +// settings thats works for Observables in addition to React. +export const useOptInAnalytics = (): [ + boolean | null, + ((value: boolean | null) => void) | null, +] => { + const setting = useSetting(optInAnalytics); + return PosthogAnalytics.instance.isEnabled() ? setting : [false, null]; +}; + +export const developerSettingsTab = new Setting( + "developer-settings-tab", + false, +); + +export const duplicateTiles = new Setting("duplicate-tiles", 0); + +export const audioInput = new Setting( + "audio-input", + undefined, +); +export const audioOutput = new Setting( + "audio-output", + undefined, +); +export const videoInput = new Setting( + "video-input", + undefined, +); + +export const alwaysShowSelf = new Setting("always-show-self", true); diff --git a/src/settings/useSetting.ts b/src/settings/useSetting.ts deleted file mode 100644 index a2733b98c..000000000 --- a/src/settings/useSetting.ts +++ /dev/null @@ -1,104 +0,0 @@ -/* -Copyright 2022 - 2023 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { useCallback, useMemo } from "react"; - -import { PosthogAnalytics } from "../analytics/PosthogAnalytics"; -import { - getLocalStorageItem, - setLocalStorageItem, - useLocalStorage, -} from "../useLocalStorage"; - -type Setting = [T, (value: T) => void]; -type DisableableSetting = [T, ((value: T) => void) | null]; - -export const getSettingKey = (name: string): string => { - return `matrix-setting-${name}`; -}; -// Like useState, but reads from and persists the value to localStorage -export const useSetting = (name: string, defaultValue: T): Setting => { - const key = useMemo(() => getSettingKey(name), [name]); - - const [item, setItem] = useLocalStorage(key); - - const value = useMemo( - () => (item == null ? defaultValue : JSON.parse(item)), - [item, defaultValue], - ); - const setValue = useCallback( - (value: T) => { - setItem(JSON.stringify(value)); - }, - [setItem], - ); - - return [value, setValue]; -}; - -export const getSetting = (name: string, defaultValue: T): T => { - const item = getLocalStorageItem(getSettingKey(name)); - return item === null ? defaultValue : JSON.parse(item); -}; - -export const setSetting = (name: string, newValue: T): void => - setLocalStorageItem(getSettingKey(name), JSON.stringify(newValue)); - -export const isFirefox = (): boolean => { - const { userAgent } = navigator; - return userAgent.includes("Firefox"); -}; - -const canEnableSpatialAudio = (): boolean => { - // Spatial audio means routing audio through audio contexts. On Chrome, - // this bypasses the AEC processor and so breaks echo cancellation. - // We only allow spatial audio to be enabled on Firefox which we know - // passes audio context audio through the AEC algorithm. - // https://bugs.chromium.org/p/chromium/issues/detail?id=687574 is the - // chrome bug for this: once this is fixed and the updated version is deployed - // widely enough, we can allow spatial audio everywhere. It's currently in a - // chrome flag, so we could enable this in Electron if we enabled the chrome flag - // in the Electron wrapper. - return isFirefox(); -}; - -export const useSpatialAudio = (): DisableableSetting => { - const settingVal = useSetting("spatial-audio", false); - if (canEnableSpatialAudio()) return settingVal; - - return [false, null]; -}; - -// null = undecided -export const useOptInAnalytics = (): DisableableSetting => { - const settingVal = useSetting("opt-in-analytics", null); - if (PosthogAnalytics.instance.isEnabled()) return settingVal; - - return [false, null]; -}; - -export const useDeveloperSettingsTab = (): Setting => - useSetting("developer-settings-tab", false); - -export const useShowConnectionStats = (): Setting => - useSetting("show-connection-stats", false); - -export const useAudioInput = (): Setting => - useSetting("audio-input", undefined); -export const useAudioOutput = (): Setting => - useSetting("audio-output", undefined); -export const useVideoInput = (): Setting => - useSetting("video-input", undefined); diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 4ad2f0242..54029b0b0 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -28,14 +28,15 @@ import { import { Room as MatrixRoom, RoomMember } from "matrix-js-sdk/src/matrix"; import { useEffect, useRef } from "react"; import { - BehaviorSubject, EMPTY, Observable, + Subject, audit, combineLatest, concat, distinctUntilChanged, filter, + fromEvent, map, merge, mergeAll, @@ -43,14 +44,13 @@ import { sample, scan, shareReplay, + skip, startWith, - switchAll, switchMap, throttleTime, timer, zip, } from "rxjs"; -import { StateObservable, state } from "@react-rxjs/core"; import { logger } from "matrix-js-sdk/src/logger"; import { ViewModel } from "./ViewModel"; @@ -61,32 +61,26 @@ import { } from "../livekit/useECConnectionState"; import { usePrevious } from "../usePrevious"; import { + LocalUserMediaViewModel, MediaViewModel, - UserMediaViewModel, + RemoteUserMediaViewModel, ScreenShareViewModel, + UserMediaViewModel, } from "./MediaViewModel"; -import { finalizeValue } from "../observable-utils"; +import { accumulate, finalizeValue } from "../observable-utils"; import { ObservableScope } from "./ObservableScope"; +import { duplicateTiles } from "../settings/settings"; // How long we wait after a focus switch before showing the real participant // list again const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000; -// Represents something that should get a tile on the layout, -// ie. a user's video feed or a screen share feed. -// TODO: This exposes too much information to the view layer, let's keep this -// information internal to the view model and switch to using Tile instead -export interface TileDescriptor { - id: string; - focused: boolean; - isPresenter: boolean; - isSpeaker: boolean; - hasVideo: boolean; - local: boolean; - largeBaseSize: boolean; - placeNear?: string; - data: T; -} +// This is the number of participants that we think constitutes a "large" grid. +// The hypothesis is that, after this many participants there's enough cognitive +// load that it makes sense to show the speaker in an easy-to-locate spotlight +// tile. We might change this to a scroll-based condition or do something else +// entirely with the spotlight tile, if we workshop this further. +const largeGridThreshold = 20; export interface GridLayout { type: "grid"; @@ -94,18 +88,30 @@ export interface GridLayout { grid: UserMediaViewModel[]; } -export interface SpotlightLayout { - type: "spotlight"; +export interface SpotlightLandscapeLayout { + type: "spotlight-landscape"; + spotlight: MediaViewModel[]; + grid: UserMediaViewModel[]; +} + +export interface SpotlightPortraitLayout { + type: "spotlight-portrait"; spotlight: MediaViewModel[]; grid: UserMediaViewModel[]; } -export interface FullScreenLayout { - type: "full screen"; +export interface SpotlightExpandedLayout { + type: "spotlight-expanded"; spotlight: MediaViewModel[]; pip?: UserMediaViewModel; } +export interface OneOnOneLayout { + type: "one-on-one"; + local: LocalUserMediaViewModel; + remote: RemoteUserMediaViewModel; +} + export interface PipLayout { type: "pip"; spotlight: MediaViewModel[]; @@ -117,26 +123,52 @@ export interface PipLayout { */ export type Layout = | GridLayout - | SpotlightLayout - | FullScreenLayout + | SpotlightLandscapeLayout + | SpotlightPortraitLayout + | SpotlightExpandedLayout + | OneOnOneLayout | PipLayout; export type GridMode = "grid" | "spotlight"; -export type WindowMode = "normal" | "full screen" | "pip"; +export type WindowMode = "normal" | "narrow" | "flat" | "pip"; /** * Sorting bins defining the order in which media tiles appear in the layout. */ enum SortingBin { - SelfStart, + /** + * Yourself, when the "always show self" option is on. + */ + SelfAlwaysShown, + /** + * Participants that are sharing their screen. + */ Presenters, + /** + * Participants that have been speaking recently. + */ Speakers, + /** + * Participants with both video and audio. + */ VideoAndAudio, + /** + * Participants with video but no audio. + */ Video, + /** + * Participants with audio but no video. + */ Audio, + /** + * Participants not sharing any media. + */ NoMedia, - SelfEnd, + /** + * Yourself, when the "always show self" option is off. + */ + SelfNotAlwaysShown, } class UserMedia { @@ -151,14 +183,17 @@ class UserMedia { participant: LocalParticipant | RemoteParticipant, callEncrypted: boolean, ) { - this.vm = new UserMediaViewModel(id, member, participant, callEncrypted); + this.vm = + participant instanceof LocalParticipant + ? new LocalUserMediaViewModel(id, member, participant, callEncrypted) + : new RemoteUserMediaViewModel(id, member, participant, callEncrypted); - this.speaker = this.vm.speaking.pipeState( - // Require 1 s of continuous speaking to become a speaker, and 10 s of + this.speaker = this.vm.speaking.pipe( + // Require 1 s of continuous speaking to become a speaker, and 60 s of // continuous silence to stop being considered a speaker audit((s) => merge( - timer(s ? 1000 : 10000), + timer(s ? 1000 : 60000), // If the speaking flag resets to its original value during this time, // end the silencing window to stick with that original value this.vm.speaking.pipe(filter((s1) => s1 !== s)), @@ -210,7 +245,8 @@ function findMatrixMember( room: MatrixRoom, id: string, ): RoomMember | undefined { - if (!id) return undefined; + if (id === "local") + return room.getMember(room.client.getUserId()!) ?? undefined; const parts = id.split(":"); // must be at least 3 parts because we know the first part is a userId which must necessarily contain a colon @@ -229,9 +265,9 @@ function findMatrixMember( // TODO: Move wayyyy more business logic from the call and lobby views into here export class CallViewModel extends ViewModel { - private readonly rawRemoteParticipants = state( - connectedParticipantsObserver(this.livekitRoom), - ); + private readonly rawRemoteParticipants = connectedParticipantsObserver( + this.livekitRoom, + ).pipe(shareReplay(1)); // Lists of participants to "hold" on display, even if LiveKit claims that // they've left @@ -271,16 +307,13 @@ export class CallViewModel extends ViewModel { }, ).pipe( mergeAll(), - // Aggregate the hold instructions into a single list showing which + // Accumulate the hold instructions into a single list showing which // participants are being held - scan( - (holds, instruction) => - "hold" in instruction - ? [instruction.hold, ...holds] - : holds.filter((h) => h !== instruction.unhold), - [] as RemoteParticipant[][], + accumulate([] as RemoteParticipant[][], (holds, instruction) => + "hold" in instruction + ? [instruction.hold, ...holds] + : holds.filter((h) => h !== instruction.unhold), ), - startWith([]), ); private readonly remoteParticipants: Observable = @@ -304,33 +337,30 @@ export class CallViewModel extends ViewModel { }, ); - private readonly mediaItems: StateObservable = state( - combineLatest([ - this.remoteParticipants, - observeParticipantMedia(this.livekitRoom.localParticipant), - ]).pipe( - scan( - ( - prevItems, - [remoteParticipants, { participant: localParticipant }], - ) => { - let allGhosts = true; - - const newItems = new Map( - function* (this: CallViewModel): Iterable<[string, MediaItem]> { - for (const p of [localParticipant, ...remoteParticipants]) { - const member = findMatrixMember(this.matrixRoom, p.identity); - allGhosts &&= member === undefined; - // We always start with a local participant with the empty string as - // their ID before we're connected, this is fine and we'll be in - // "all ghosts" mode. - if (p.identity !== "" && member === undefined) { - logger.warn( - `Ruh, roh! No matrix member found for SFU participant '${p.identity}': creating g-g-g-ghost!`, - ); - } + private readonly mediaItems: Observable = combineLatest([ + this.remoteParticipants, + observeParticipantMedia(this.livekitRoom.localParticipant), + duplicateTiles.value, + ]).pipe( + scan( + ( + prevItems, + [remoteParticipants, { participant: localParticipant }, duplicateTiles], + ) => { + const newItems = new Map( + function* (this: CallViewModel): Iterable<[string, MediaItem]> { + for (const p of [localParticipant, ...remoteParticipants]) { + const userMediaId = p === localParticipant ? "local" : p.identity; + const member = findMatrixMember(this.matrixRoom, userMediaId); + if (member === undefined) + logger.warn( + `Ruh, roh! No matrix member found for SFU participant '${p.identity}': creating g-g-g-ghost!`, + ); - const userMediaId = p.identity; + // Create as many tiles for this participant as called for by + // the duplicateTiles option + for (let i = 0; i < 1 + duplicateTiles; i++) { + const userMediaId = `${p.identity}:${i}`; yield [ userMediaId, prevItems.get(userMediaId) ?? @@ -346,69 +376,99 @@ export class CallViewModel extends ViewModel { ]; } } - }.bind(this)(), - ); - - for (const [id, t] of prevItems) if (!newItems.has(id)) t.destroy(); + } + }.bind(this)(), + ); - // If every item is a ghost, that probably means we're still connecting - // and shouldn't bother showing anything yet - return allGhosts ? new Map() : newItems; - }, - new Map(), - ), - map((ms) => [...ms.values()]), - finalizeValue((ts) => { - for (const t of ts) t.destroy(); - }), + for (const [id, t] of prevItems) if (!newItems.has(id)) t.destroy(); + return newItems; + }, + new Map(), ), + map((mediaItems) => [...mediaItems.values()]), + finalizeValue((ts) => { + for (const t of ts) t.destroy(); + }), + shareReplay(1), ); private readonly userMedia: Observable = this.mediaItems.pipe( - map((ms) => ms.filter((m): m is UserMedia => m instanceof UserMedia)), + map((mediaItems) => + mediaItems.filter((m): m is UserMedia => m instanceof UserMedia), + ), ); + private readonly localUserMedia: Observable = + this.mediaItems.pipe( + map((ms) => ms.find((m) => m.vm.local)!.vm as LocalUserMediaViewModel), + ); + private readonly screenShares: Observable = this.mediaItems.pipe( - map((ms) => ms.filter((m): m is ScreenShare => m instanceof ScreenShare)), + map((mediaItems) => + mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), + ), + shareReplay(1), + ); + + private readonly hasRemoteScreenShares: Observable = + this.screenShares.pipe( + map((ms) => ms.some((m) => !m.vm.local)), + distinctUntilChanged(), ); - private readonly spotlightSpeaker: Observable = + private readonly spotlightSpeaker: Observable = this.userMedia.pipe( - switchMap((ms) => - ms.length === 0 + switchMap((mediaItems) => + mediaItems.length === 0 ? of([]) : combineLatest( - ms.map((m) => m.vm.speaking.pipe(map((s) => [m, s] as const))), + mediaItems.map((m) => + m.vm.speaking.pipe(map((s) => [m, s] as const)), + ), ), ), - scan<(readonly [UserMedia, boolean])[], UserMedia | null, null>( - (prev, ms) => + scan<(readonly [UserMedia, boolean])[], UserMedia, null>( + (prev, mediaItems) => // Decide who to spotlight: - // If the previous speaker is still speaking, stick with them rather - // than switching eagerly to someone else - ms.find(([m, s]) => m === prev && s)?.[0] ?? - // Otherwise, select anyone who is speaking - ms.find(([, s]) => s)?.[0] ?? + // If the previous speaker (not the local user) is still speaking, + // stick with them rather than switching eagerly to someone else + (prev === null || prev.vm.local + ? null + : mediaItems.find(([m, s]) => m === prev && s)?.[0]) ?? + // Otherwise, select any remote user who is speaking + mediaItems.find(([m, s]) => !m.vm.local && s)?.[0] ?? // Otherwise, stick with the person who was last speaking prev ?? // Otherwise, spotlight the local user - ms.find(([m]) => m.vm.local)?.[0] ?? - null, + mediaItems.find(([m]) => m.vm.local)![0], null, ), distinctUntilChanged(), - throttleTime(800, undefined, { leading: true, trailing: true }), + map((speaker) => speaker.vm), + shareReplay(1), + throttleTime(1600, undefined, { leading: true, trailing: true }), ); private readonly grid: Observable = this.userMedia.pipe( - switchMap((ms) => { - const bins = ms.map((m) => + switchMap((mediaItems) => { + const bins = mediaItems.map((m) => combineLatest( - [m.speaker, m.presenter, m.vm.audioEnabled, m.vm.videoEnabled], - (speaker, presenter, audio, video) => { + [ + m.speaker, + m.presenter, + m.vm.audioEnabled, + m.vm.videoEnabled, + m.vm instanceof LocalUserMediaViewModel + ? m.vm.alwaysShow + : of(false), + ], + (speaker, presenter, audio, video, alwaysShow) => { let bin: SortingBin; - if (m.vm.local) bin = SortingBin.SelfStart; + if (m.vm.local) + bin = alwaysShow + ? SortingBin.SelfAlwaysShown + : SortingBin.SelfNotAlwaysShown; else if (presenter) bin = SortingBin.Presenters; else if (speaker) bin = SortingBin.Speakers; else if (video) @@ -428,153 +488,197 @@ export class CallViewModel extends ViewModel { }), ); - private readonly spotlight: Observable = combineLatest( - [this.screenShares, this.spotlightSpeaker], - (screenShares, spotlightSpeaker): MediaViewModel[] => + private readonly spotlightAndPip: Observable< + [Observable, Observable] + > = this.screenShares.pipe( + map((screenShares) => screenShares.length > 0 - ? screenShares.map((m) => m.vm) - : spotlightSpeaker === null - ? [] - : [spotlightSpeaker.vm], + ? ([of(screenShares.map((m) => m.vm)), this.spotlightSpeaker] as const) + : ([ + this.spotlightSpeaker.pipe(map((speaker) => [speaker!])), + this.localUserMedia.pipe( + switchMap((vm) => + vm.alwaysShow.pipe( + map((alwaysShow) => (alwaysShow ? vm : null)), + ), + ), + ), + ] as const), + ), + ); + + private readonly spotlight: Observable = + this.spotlightAndPip.pipe( + switchMap(([spotlight]) => spotlight), + shareReplay(1), + ); + + private readonly pip: Observable = + this.spotlightAndPip.pipe(switchMap(([, pip]) => pip)); + + /** + * The general shape of the window. + */ + public readonly windowMode: Observable = fromEvent( + window, + "resize", + ).pipe( + startWith(null), + map(() => { + const height = window.innerHeight; + const width = window.innerWidth; + if (height <= 400 && width <= 340) return "pip"; + if (width <= 660) return "narrow"; + if (height <= 660) return "flat"; + return "normal"; + }), + distinctUntilChanged(), + shareReplay(1), ); - // TODO: Make this react to changes in window dimensions and screen - // orientation - private readonly windowMode = of("normal"); + private readonly spotlightExpandedToggle = new Subject(); + public readonly spotlightExpanded: Observable = + this.spotlightExpandedToggle.pipe( + accumulate(false, (expanded) => !expanded), + shareReplay(1), + ); + + public toggleSpotlightExpanded(): void { + this.spotlightExpandedToggle.next(); + } - private readonly _gridMode = new BehaviorSubject("grid"); + private readonly gridModeUserSelection = new Subject(); /** * The layout mode of the media tile grid. */ - public readonly gridMode = state(this._gridMode); + public readonly gridMode: Observable = + // If the user hasn't selected spotlight and somebody starts screen sharing, + // automatically switch to spotlight mode and reset when screen sharing ends + this.gridModeUserSelection.pipe( + startWith(null), + switchMap((userSelection) => + (userSelection === "spotlight" + ? EMPTY + : combineLatest([this.hasRemoteScreenShares, this.windowMode]).pipe( + skip(userSelection === null ? 0 : 1), + map( + ([hasScreenShares, windowMode]): GridMode => + hasScreenShares || windowMode === "flat" + ? "spotlight" + : "grid", + ), + ) + ).pipe(startWith(userSelection ?? "grid")), + ), + distinctUntilChanged(), + shareReplay(1), + ); public setGridMode(value: GridMode): void { - this._gridMode.next(value); + this.gridModeUserSelection.next(value); } - public readonly layout: StateObservable = state( - combineLatest([this._gridMode, this.windowMode], (gridMode, windowMode) => { + public readonly layout: Observable = this.windowMode.pipe( + switchMap((windowMode) => { + const spotlightLandscapeLayout = combineLatest( + [this.grid, this.spotlight], + (grid, spotlight): Layout => ({ + type: "spotlight-landscape", + spotlight, + grid, + }), + ); + const spotlightExpandedLayout = combineLatest( + [this.spotlight, this.pip], + (spotlight, pip): Layout => ({ + type: "spotlight-expanded", + spotlight, + pip: pip ?? undefined, + }), + ); + switch (windowMode) { - case "full screen": - throw new Error("unimplemented"); + case "normal": + return this.gridMode.pipe( + switchMap((gridMode) => { + switch (gridMode) { + case "grid": + return combineLatest( + [this.grid, this.spotlight, this.screenShares], + (grid, spotlight, screenShares): Layout => + grid.length == 2 && screenShares.length === 0 + ? { + type: "one-on-one", + local: grid.find( + (vm) => vm.local, + ) as LocalUserMediaViewModel, + remote: grid.find( + (vm) => !vm.local, + ) as RemoteUserMediaViewModel, + } + : { + type: "grid", + spotlight: + screenShares.length > 0 || + grid.length > largeGridThreshold + ? spotlight + : undefined, + grid, + }, + ); + case "spotlight": + return this.spotlightExpanded.pipe( + switchMap((expanded) => + expanded + ? spotlightExpandedLayout + : spotlightLandscapeLayout, + ), + ); + } + }), + ); + case "narrow": + return combineLatest( + [this.grid, this.spotlight], + (grid, spotlight): Layout => ({ + type: "spotlight-portrait", + spotlight, + grid, + }), + ); + case "flat": + return this.gridMode.pipe( + switchMap((gridMode) => { + switch (gridMode) { + case "grid": + // Yes, grid mode actually gets you a "spotlight" layout in + // this window mode. + return spotlightLandscapeLayout; + case "spotlight": + return spotlightExpandedLayout; + } + }), + ); case "pip": - throw new Error("unimplemented"); - case "normal": { - switch (gridMode) { - case "grid": - return combineLatest( - [this.grid, this.spotlight, this.screenShares], - (grid, spotlight, screenShares): Layout => ({ - type: "grid", - spotlight: screenShares.length > 0 ? spotlight : undefined, - grid, - }), - ); - case "spotlight": - return combineLatest( - [this.grid, this.spotlight], - (grid, spotlight): Layout => ({ - type: "spotlight", - spotlight, - grid, - }), - ); - } - } + return this.spotlight.pipe( + map((spotlight): Layout => ({ type: "pip", spotlight })), + ); } - }).pipe(switchAll()), + }), + shareReplay(1), ); - /** - * The media tiles to be displayed in the call view. - */ - // TODO: Get rid of this field, replacing it with the 'layout' field above - // which keeps more details of the layout order internal to the view model - public readonly tiles: StateObservable[]> = - state( - combineLatest([ - this.remoteParticipants, - observeParticipantMedia(this.livekitRoom.localParticipant), - ]).pipe( - scan((ts, [remoteParticipants, { participant: localParticipant }]) => { - const ps = [localParticipant, ...remoteParticipants]; - const tilesById = new Map(ts.map((t) => [t.id, t])); - const now = Date.now(); - let allGhosts = true; - - const newTiles = ps.flatMap((p) => { - const userMediaId = p.identity; - const member = findMatrixMember(this.matrixRoom, userMediaId); - allGhosts &&= member === undefined; - const spokeRecently = - p.lastSpokeAt !== undefined && now - +p.lastSpokeAt <= 10000; - - // We always start with a local participant with the empty string as - // their ID before we're connected, this is fine and we'll be in - // "all ghosts" mode. - if (userMediaId !== "" && member === undefined) { - logger.warn( - `Ruh, roh! No matrix member found for SFU participant '${userMediaId}': creating g-g-g-ghost!`, - ); - } - - const userMediaVm = - tilesById.get(userMediaId)?.data ?? - new UserMediaViewModel(userMediaId, member, p, this.encrypted); - tilesById.delete(userMediaId); - - const userMediaTile: TileDescriptor = { - id: userMediaId, - focused: false, - isPresenter: p.isScreenShareEnabled, - isSpeaker: (p.isSpeaking || spokeRecently) && !p.isLocal, - hasVideo: p.isCameraEnabled, - local: p.isLocal, - largeBaseSize: false, - data: userMediaVm, - }; - - if (p.isScreenShareEnabled) { - const screenShareId = `${userMediaId}:screen-share`; - const screenShareVm = - tilesById.get(screenShareId)?.data ?? - new ScreenShareViewModel( - screenShareId, - member, - p, - this.encrypted, - ); - tilesById.delete(screenShareId); - - const screenShareTile: TileDescriptor = { - id: screenShareId, - focused: true, - isPresenter: false, - isSpeaker: false, - hasVideo: true, - local: p.isLocal, - largeBaseSize: true, - placeNear: userMediaId, - data: screenShareVm, - }; - return [userMediaTile, screenShareTile]; - } else { - return [userMediaTile]; - } - }); - - // Any tiles left in the map are unused and should be destroyed - for (const t of tilesById.values()) t.data.destroy(); + public showSpotlightIndicators: Observable = this.layout.pipe( + map((l) => l.type !== "grid"), + distinctUntilChanged(), + shareReplay(1), + ); - // If every item is a ghost, that probably means we're still connecting - // and shouldn't bother showing anything yet - return allGhosts ? [] : newTiles; - }, [] as TileDescriptor[]), - finalizeValue((ts) => { - for (const t of ts) t.data.destroy(); - }), - ), - ); + public showSpeakingIndicators: Observable = this.layout.pipe( + map((l) => l.type !== "one-on-one" && l.type !== "spotlight-expanded"), + distinctUntilChanged(), + shareReplay(1), + ); public constructor( // A call is permanently tied to a single Matrix room and LiveKit room diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index db11017e1..8ad565e42 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -21,7 +21,6 @@ import { observeParticipantEvents, observeParticipantMedia, } from "@livekit/components-core"; -import { StateObservable, state } from "@react-rxjs/core"; import { LocalParticipant, LocalTrack, @@ -32,34 +31,77 @@ import { TrackEvent, facingModeFromLocalTrack, } from "livekit-client"; -import { RoomMember } from "matrix-js-sdk/src/matrix"; +import { RoomMember, RoomMemberEvent } from "matrix-js-sdk/src/matrix"; import { BehaviorSubject, + Observable, combineLatest, distinctUntilChanged, distinctUntilKeyChanged, fromEvent, map, of, + shareReplay, startWith, switchMap, } from "rxjs"; +import { useTranslation } from "react-i18next"; +import { useEffect } from "react"; import { ViewModel } from "./ViewModel"; +import { useReactiveState } from "../useReactiveState"; +import { alwaysShowSelf } from "../settings/settings"; + +export interface NameData { + /** + * The display name of the participant. + */ + displayName: string; + /** + * The text to be shown on the participant's name tag. + */ + nameTag: string; +} + +// TODO: Move this naming logic into the view model +export function useNameData(vm: MediaViewModel): NameData { + const { t } = useTranslation(); + + const [displayName, setDisplayName] = useReactiveState( + () => vm.member?.rawDisplayName ?? "[👻]", + [vm.member], + ); + useEffect(() => { + if (vm.member) { + const updateName = (): void => { + setDisplayName(vm.member!.rawDisplayName); + }; + + vm.member!.on(RoomMemberEvent.Name, updateName); + return (): void => { + vm.member!.removeListener(RoomMemberEvent.Name, updateName); + }; + } + }, [vm.member, setDisplayName]); + const nameTag = vm.local + ? t("video_tile.sfu_participant_local") + : displayName; + + return { displayName, nameTag }; +} function observeTrackReference( participant: Participant, source: Track.Source, -): StateObservable { - return state( - observeParticipantMedia(participant).pipe( - map(() => ({ - participant, - publication: participant.getTrackPublication(source), - source, - })), - distinctUntilKeyChanged("publication"), - ), +): Observable { + return observeParticipantMedia(participant).pipe( + map(() => ({ + participant, + publication: participant.getTrackPublication(source), + source, + })), + distinctUntilKeyChanged("publication"), + shareReplay(1), ); } @@ -71,15 +113,16 @@ abstract class BaseMediaViewModel extends ViewModel { /** * The LiveKit video track for this media. */ - public readonly video: StateObservable; + public readonly video: Observable; /** * Whether there should be a warning that this media is unencrypted. */ - public readonly unencryptedWarning: StateObservable; + public readonly unencryptedWarning: Observable; public constructor( - // TODO: This is only needed for full screen toggling and can be removed as - // soon as that code is moved into the view models + /** + * An opaque identifier for this media. + */ public readonly id: string, /** * The Matrix room member to which this media belongs. @@ -95,15 +138,13 @@ abstract class BaseMediaViewModel extends ViewModel { super(); const audio = observeTrackReference(participant, audioSource); this.video = observeTrackReference(participant, videoSource); - this.unencryptedWarning = state( - combineLatest( - [audio, this.video], - (a, v) => - callEncrypted && - (a.publication?.isEncrypted === false || - v.publication?.isEncrypted === false), - ).pipe(distinctUntilChanged()), - ); + this.unencryptedWarning = combineLatest( + [audio, this.video], + (a, v) => + callEncrypted && + (a.publication?.isEncrypted === false || + v.publication?.isEncrypted === false), + ).pipe(distinctUntilChanged(), shareReplay(1)); } } @@ -111,66 +152,39 @@ abstract class BaseMediaViewModel extends ViewModel { * Some participant's media. */ export type MediaViewModel = UserMediaViewModel | ScreenShareViewModel; +export type UserMediaViewModel = + | LocalUserMediaViewModel + | RemoteUserMediaViewModel; /** * Some participant's user media. */ -export class UserMediaViewModel extends BaseMediaViewModel { - /** - * Whether the video should be mirrored. - */ - public readonly mirror = state( - this.video.pipe( - switchMap((v) => { - const track = v.publication?.track; - if (!(track instanceof LocalTrack)) return of(false); - // Watch for track restarts, because they indicate a camera switch - return fromEvent(track, TrackEvent.Restarted).pipe( - startWith(null), - // Mirror only front-facing cameras (those that face the user) - map(() => facingModeFromLocalTrack(track).facingMode === "user"), - ); - }), - ), - ); - +abstract class BaseUserMediaViewModel extends BaseMediaViewModel { /** * Whether the participant is speaking. */ - public readonly speaking = state( - observeParticipantEvents( - this.participant, - ParticipantEvent.IsSpeakingChanged, - ).pipe(map((p) => p.isSpeaking)), + public readonly speaking = observeParticipantEvents( + this.participant, + ParticipantEvent.IsSpeakingChanged, + ).pipe( + map((p) => p.isSpeaking), + shareReplay(1), ); - private readonly _locallyMuted = new BehaviorSubject(false); - /** - * Whether we've disabled this participant's audio. - */ - public readonly locallyMuted = state(this._locallyMuted); - - private readonly _localVolume = new BehaviorSubject(1); - /** - * The volume to which we've set this participant's audio, as a scalar - * multiplier. - */ - public readonly localVolume = state(this._localVolume); - /** * Whether this participant is sending audio (i.e. is unmuted on their side). */ - public readonly audioEnabled: StateObservable; + public readonly audioEnabled: Observable; /** * Whether this participant is sending video. */ - public readonly videoEnabled: StateObservable; + public readonly videoEnabled: Observable; private readonly _cropVideo = new BehaviorSubject(true); /** * Whether the tile video should be contained inside the tile or be cropped to fit. */ - public readonly cropVideo = state(this._cropVideo); + public readonly cropVideo: Observable = this._cropVideo; public constructor( id: string, @@ -187,33 +201,97 @@ export class UserMediaViewModel extends BaseMediaViewModel { Track.Source.Camera, ); - const media = observeParticipantMedia(participant); - this.audioEnabled = state( - media.pipe(map((m) => m.microphoneTrack?.isMuted === false)), + const media = observeParticipantMedia(participant).pipe(shareReplay(1)); + this.audioEnabled = media.pipe( + map((m) => m.microphoneTrack?.isMuted === false), ); - this.videoEnabled = state( - media.pipe(map((m) => m.cameraTrack?.isMuted === false)), + this.videoEnabled = media.pipe( + map((m) => m.cameraTrack?.isMuted === false), ); + } + + public toggleFitContain(): void { + this._cropVideo.next(!this._cropVideo.value); + } +} + +/** + * The local participant's user media. + */ +export class LocalUserMediaViewModel extends BaseUserMediaViewModel { + /** + * Whether the video should be mirrored. + */ + public readonly mirror = this.video.pipe( + switchMap((v) => { + const track = v.publication?.track; + if (!(track instanceof LocalTrack)) return of(false); + // Watch for track restarts, because they indicate a camera switch + return fromEvent(track, TrackEvent.Restarted).pipe( + startWith(null), + // Mirror only front-facing cameras (those that face the user) + map(() => facingModeFromLocalTrack(track).facingMode === "user"), + ); + }), + shareReplay(1), + ); + + /** + * Whether to show this tile in a highly visible location near the start of + * the grid. + */ + public readonly alwaysShow = alwaysShowSelf.value; + public readonly setAlwaysShow = alwaysShowSelf.setValue; + + public constructor( + id: string, + member: RoomMember | undefined, + participant: LocalParticipant, + callEncrypted: boolean, + ) { + super(id, member, participant, callEncrypted); + } +} + +/** + * A remote participant's user media. + */ +export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { + private readonly _locallyMuted = new BehaviorSubject(false); + /** + * Whether we've disabled this participant's audio. + */ + public readonly locallyMuted: Observable = this._locallyMuted; + + private readonly _localVolume = new BehaviorSubject(1); + /** + * The volume to which we've set this participant's audio, as a scalar + * multiplier. + */ + public readonly localVolume: Observable = this._localVolume; + + public constructor( + id: string, + member: RoomMember | undefined, + participant: RemoteParticipant, + callEncrypted: boolean, + ) { + super(id, member, participant, callEncrypted); // Sync the local mute state and volume with LiveKit - if (!this.local) - combineLatest([this._locallyMuted, this._localVolume], (muted, volume) => - muted ? 0 : volume, - ) - .pipe(this.scope.bind()) - .subscribe((volume) => { - (this.participant as RemoteParticipant).setVolume(volume); - }); + combineLatest([this._locallyMuted, this._localVolume], (muted, volume) => + muted ? 0 : volume, + ) + .pipe(this.scope.bind()) + .subscribe((volume) => { + (this.participant as RemoteParticipant).setVolume(volume); + }); } public toggleLocallyMuted(): void { this._locallyMuted.next(!this._locallyMuted.value); } - public toggleFitContain(): void { - this._cropVideo.next(!this._cropVideo.value); - } - public setLocalVolume(value: number): void { this._localVolume.next(value); } diff --git a/src/state/subscribe.tsx b/src/state/subscribe.tsx deleted file mode 100644 index e0441aeb7..000000000 --- a/src/state/subscribe.tsx +++ /dev/null @@ -1,49 +0,0 @@ -/* -Copyright 2023 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { - ForwardRefExoticComponent, - ForwardRefRenderFunction, - PropsWithoutRef, - RefAttributes, - forwardRef, -} from "react"; -// eslint-disable-next-line no-restricted-imports -import { Subscribe, RemoveSubscribe } from "@react-rxjs/core"; - -/** - * Wraps a React component that consumes Observables, resulting in a component - * that safely subscribes to its Observables before rendering. The component - * will return null until the subscriptions are created. - */ -export function subscribe( - render: ForwardRefRenderFunction, -): ForwardRefExoticComponent & RefAttributes> { - const Subscriber = forwardRef(({ p }, ref) => ( - {render(p, ref)} - )); - Subscriber.displayName = "Subscriber"; - - // eslint-disable-next-line react/display-name - const OuterComponent = forwardRef((p, ref) => ( - - - - )); - // Copy over the component's display name, default props, etc. - Object.assign(OuterComponent, render); - return OuterComponent; -} diff --git a/src/state/useObservable.ts b/src/state/useObservable.ts index 92210e345..037c3bd59 100644 --- a/src/state/useObservable.ts +++ b/src/state/useObservable.ts @@ -14,9 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { useRef } from "react"; +import { Ref, useCallback, useRef } from "react"; import { BehaviorSubject, Observable } from "rxjs"; +import { useInitial } from "../useInitial"; + /** * React hook that creates an Observable from a changing value. The Observable * replays its current value upon subscription and emits whenever the value @@ -28,3 +30,14 @@ export function useObservable(value: T): Observable { if (value !== subject.current.value) subject.current.next(value); return subject.current; } + +/** + * React hook that creates a ref and an Observable that emits any values + * stored in the ref. The Observable replays the value currently stored in the + * ref upon subscription. + */ +export function useObservableRef(initialValue: T): [Observable, Ref] { + const subject = useInitial(() => new BehaviorSubject(initialValue)); + const ref = useCallback((value: T) => subject.next(value), [subject]); + return [subject, ref]; +} diff --git a/src/tile/GridTile.module.css b/src/tile/GridTile.module.css new file mode 100644 index 000000000..ea015f435 --- /dev/null +++ b/src/tile/GridTile.module.css @@ -0,0 +1,72 @@ +/* +Copyright 2022-2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.tile { + --media-view-border-radius: var(--cpd-space-4x); + transition: outline-color ease 0.15s; + outline: var(--cpd-border-width-2) solid rgb(0 0 0 / 0); +} + +/* Use a pseudo-element to create the expressive speaking border, since CSS +borders don't support gradients */ +.tile::before { + content: ""; + position: absolute; + z-index: -1; /* Put it below the outline */ + opacity: 0; /* Hidden unless speaking */ + transition: opacity ease 0.15s; + inset: calc(-1 * var(--cpd-border-width-4)); + border-radius: var(--cpd-space-5x); + background: linear-gradient( + 119deg, + rgba(13, 92, 189, 0.7) 0%, + rgba(13, 189, 168, 0.7) 100% + ), + linear-gradient( + 180deg, + rgba(13, 92, 189, 0.9) 0%, + rgba(13, 189, 168, 0.9) 100% + ); + background-blend-mode: overlay, normal; +} + +.tile.speaking { + /* !important because speaking border should take priority over hover */ + outline: var(--cpd-border-width-1) solid var(--cpd-color-bg-canvas-default) !important; +} + +.tile.speaking::before { + opacity: 1; +} + +@media (hover: hover) { + .tile:hover { + outline: var(--cpd-border-width-2) solid + var(--cpd-color-border-interactive-hovered); + } +} + +.muteIcon[data-muted="true"] { + color: var(--cpd-color-icon-secondary); +} + +.muteIcon[data-muted="false"] { + color: var(--cpd-color-icon-primary); +} + +.volumeSlider { + width: 100%; +} diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx new file mode 100644 index 000000000..eb2625e82 --- /dev/null +++ b/src/tile/GridTile.tsx @@ -0,0 +1,301 @@ +/* +Copyright 2022-2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { + ComponentProps, + ReactNode, + forwardRef, + useCallback, + useState, +} from "react"; +import { animated } from "@react-spring/web"; +import classNames from "classnames"; +import { useTranslation } from "react-i18next"; +import MicOnSolidIcon from "@vector-im/compound-design-tokens/icons/mic-on-solid.svg?react"; +import MicOffSolidIcon from "@vector-im/compound-design-tokens/icons/mic-off-solid.svg?react"; +import MicOffIcon from "@vector-im/compound-design-tokens/icons/mic-off.svg?react"; +import OverflowHorizontalIcon from "@vector-im/compound-design-tokens/icons/overflow-horizontal.svg?react"; +import VolumeOnIcon from "@vector-im/compound-design-tokens/icons/volume-on.svg?react"; +import VolumeOffIcon from "@vector-im/compound-design-tokens/icons/volume-off.svg?react"; +import VisibilityOnIcon from "@vector-im/compound-design-tokens/icons/visibility-on.svg?react"; +import UserProfileIcon from "@vector-im/compound-design-tokens/icons/user-profile.svg?react"; +import ExpandIcon from "@vector-im/compound-design-tokens/icons/expand.svg?react"; +import { + ContextMenu, + MenuItem, + ToggleMenuItem, + Menu, +} from "@vector-im/compound-web"; +import { useObservableEagerState } from "observable-hooks"; + +import styles from "./GridTile.module.css"; +import { + UserMediaViewModel, + useNameData, + LocalUserMediaViewModel, + RemoteUserMediaViewModel, +} from "../state/MediaViewModel"; +import { Slider } from "../Slider"; +import { MediaView } from "./MediaView"; +import { useLatest } from "../useLatest"; + +interface TileProps { + className?: string; + style?: ComponentProps["style"]; + targetWidth: number; + targetHeight: number; + displayName: string; + nameTag: string; + showSpeakingIndicators: boolean; +} + +interface UserMediaTileProps extends TileProps { + vm: UserMediaViewModel; + mirror: boolean; + menuStart?: ReactNode; + menuEnd?: ReactNode; +} + +const UserMediaTile = forwardRef( + ( + { + vm, + showSpeakingIndicators, + menuStart, + menuEnd, + className, + nameTag, + ...props + }, + ref, + ) => { + const { t } = useTranslation(); + const video = useObservableEagerState(vm.video); + const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning); + const audioEnabled = useObservableEagerState(vm.audioEnabled); + const videoEnabled = useObservableEagerState(vm.videoEnabled); + const speaking = useObservableEagerState(vm.speaking); + const cropVideo = useObservableEagerState(vm.cropVideo); + const onChangeFitContain = useCallback(() => vm.toggleFitContain(), [vm]); + const onSelectFitContain = useCallback( + (e: Event) => e.preventDefault(), + [], + ); + + const MicIcon = audioEnabled ? MicOnSolidIcon : MicOffSolidIcon; + + const [menuOpen, setMenuOpen] = useState(false); + const menu = ( + <> + {menuStart} + + {menuEnd} + + ); + + const tile = ( + + } + nameTag={nameTag} + primaryButton={ + + + + } + side="left" + align="start" + > + {menu} + + } + {...props} + /> + ); + + return ( + + {menu} + + ); + }, +); + +UserMediaTile.displayName = "UserMediaTile"; + +interface LocalUserMediaTileProps extends TileProps { + vm: LocalUserMediaViewModel; + onOpenProfile: () => void; +} + +const LocalUserMediaTile = forwardRef( + ({ vm, onOpenProfile, ...props }, ref) => { + const { t } = useTranslation(); + const mirror = useObservableEagerState(vm.mirror); + const alwaysShow = useObservableEagerState(vm.alwaysShow); + const latestAlwaysShow = useLatest(alwaysShow); + const onSelectAlwaysShow = useCallback( + (e: Event) => e.preventDefault(), + [], + ); + const onChangeAlwaysShow = useCallback( + () => vm.setAlwaysShow(!latestAlwaysShow.current), + [vm, latestAlwaysShow], + ); + + return ( + + } + menuEnd={ + + } + {...props} + /> + ); + }, +); + +LocalUserMediaTile.displayName = "LocalUserMediaTile"; + +interface RemoteUserMediaTileProps extends TileProps { + vm: RemoteUserMediaViewModel; +} + +const RemoteUserMediaTile = forwardRef< + HTMLDivElement, + RemoteUserMediaTileProps +>(({ vm, ...props }, ref) => { + const { t } = useTranslation(); + const locallyMuted = useObservableEagerState(vm.locallyMuted); + const localVolume = useObservableEagerState(vm.localVolume); + const onChangeMute = useCallback(() => vm.toggleLocallyMuted(), [vm]); + const onSelectMute = useCallback((e: Event) => e.preventDefault(), []); + const onChangeLocalVolume = useCallback( + (v: number) => vm.setLocalVolume(v), + [vm], + ); + + const VolumeIcon = locallyMuted ? VolumeOffIcon : VolumeOnIcon; + + return ( + + + {/* TODO: Figure out how to make this slider keyboard accessible */} + + + + + } + {...props} + /> + ); +}); + +RemoteUserMediaTile.displayName = "RemoteUserMediaTile"; + +interface GridTileProps { + vm: UserMediaViewModel; + onOpenProfile: () => void; + targetWidth: number; + targetHeight: number; + className?: string; + style?: ComponentProps["style"]; + showSpeakingIndicators: boolean; +} + +export const GridTile = forwardRef( + ({ vm, onOpenProfile, ...props }, ref) => { + const nameData = useNameData(vm); + + if (vm instanceof LocalUserMediaViewModel) { + return ( + + ); + } else { + return ; + } + }, +); + +GridTile.displayName = "GridTile"; diff --git a/src/video-grid/VideoTile.module.css b/src/tile/MediaView.module.css similarity index 63% rename from src/video-grid/VideoTile.module.css rename to src/tile/MediaView.module.css index b4da6e5e6..e3622f4d2 100644 --- a/src/video-grid/VideoTile.module.css +++ b/src/tile/MediaView.module.css @@ -1,5 +1,5 @@ /* -Copyright 2022-2023 New Vector Ltd +Copyright 2022-2024 New Vector Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,63 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -.videoTile { - position: absolute; - top: 0; - container-name: videoTile; +.media { + container-name: mediaView; container-type: size; - border-radius: var(--cpd-space-4x); - transition: outline-color ease 0.15s; - outline: var(--cpd-border-width-2) solid rgb(0 0 0 / 0); -} - -/* Use a pseudo-element to create the expressive speaking border, since CSS -borders don't support gradients */ -.videoTile::before { - content: ""; - position: absolute; - z-index: -1; /* Put it below the outline */ - opacity: 0; /* Hidden unless speaking */ - transition: opacity ease 0.15s; - inset: calc(-1 * var(--cpd-border-width-4)); - border-radius: var(--cpd-space-5x); - background: linear-gradient( - 119deg, - rgba(13, 92, 189, 0.7) 0%, - rgba(13, 189, 168, 0.7) 100% - ), - linear-gradient( - 180deg, - rgba(13, 92, 189, 0.9) 0%, - rgba(13, 189, 168, 0.9) 100% - ); - background-blend-mode: overlay, normal; -} - -.videoTile.speaking { - /* !important because speaking border should take priority over hover */ - outline: var(--cpd-border-width-1) solid var(--cpd-color-bg-canvas-default) !important; -} - -.videoTile.speaking::before { - opacity: 1; -} - -@media (hover: hover) { - .videoTile:hover { - outline: var(--cpd-border-width-2) solid - var(--cpd-color-border-interactive-hovered); - } -} - -.videoTile.maximised { - position: relative; - border-radius: 0; - inline-size: 100%; - block-size: 100%; + border-radius: var(--media-view-border-radius); } -.videoTile video { +.media video { inline-size: 100%; block-size: 100%; object-fit: contain; @@ -81,19 +31,19 @@ borders don't support gradients */ transform: translate(0); } -.videoTile.mirror video { +.media.mirror video { transform: scaleX(-1); } -.videoTile.screenshare video { - object-fit: contain; +.media[data-video-fit="cover"] video { + object-fit: cover; } -.videoTile.cropVideo video { - object-fit: cover; +.media[data-video-fit="contain"] video { + object-fit: contain; } -.videoTile.videoMuted video { +.media.videoMuted video { display: none; } @@ -114,13 +64,13 @@ borders don't support gradients */ pointer-events: none; } -.videoTile.videoMuted .avatar { +.media.videoMuted .avatar { display: initial; } /* CSS makes us put a condition here, even though all we want to do is unconditionally select the container so we can use cqmin units */ -@container videoTile (width > 0) { +@container mediaView (width > 0) { .avatar { /* Half of the smallest dimension of the tile */ inline-size: 50cqmin; @@ -137,11 +87,14 @@ unconditionally select the container so we can use cqmin units */ .fg { position: absolute; - inset: var(--cpd-space-1x); + inset: var( + --media-view-fg-inset, + calc(var(--media-view-border-radius) - var(--cpd-space-3x)) + ); display: grid; grid-template-columns: 1fr auto; grid-template-rows: 1fr auto; - grid-template-areas: ". button2" "nameTag button1"; + grid-template-areas: ". ." "nameTag button"; gap: var(--cpd-space-1x); place-items: start; } @@ -167,14 +120,6 @@ unconditionally select the container so we can use cqmin units */ flex-shrink: 0; } -.muteIcon[data-muted="true"] { - color: var(--cpd-color-icon-secondary); -} - -.muteIcon[data-muted="false"] { - color: var(--cpd-color-icon-primary); -} - .nameTag > .name { text-overflow: ellipsis; overflow: hidden; @@ -200,8 +145,7 @@ unconditionally select the container so we can use cqmin units */ transition: opacity ease 0.15s; } -.fg > button:focus-visible, -.fg > :focus-visible ~ button, +.fg:has(:focus-visible) > button, .fg > button[data-enabled="true"], .fg > button[data-state="open"] { opacity: 1; @@ -231,13 +175,5 @@ unconditionally select the container so we can use cqmin units */ } .fg > button:first-of-type { - grid-area: button1; -} - -.fg > button:nth-of-type(2) { - grid-area: button2; -} - -.volumeSlider { - width: 100%; + grid-area: button; } diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx new file mode 100644 index 000000000..e34b4fdda --- /dev/null +++ b/src/tile/MediaView.tsx @@ -0,0 +1,127 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { TrackReferenceOrPlaceholder } from "@livekit/components-core"; +import { animated } from "@react-spring/web"; +import { RoomMember } from "matrix-js-sdk/src/matrix"; +import { ComponentProps, ReactNode, forwardRef } from "react"; +import { useTranslation } from "react-i18next"; +import classNames from "classnames"; +import { VideoTrack } from "@livekit/components-react"; +import { Text, Tooltip } from "@vector-im/compound-web"; +import ErrorIcon from "@vector-im/compound-design-tokens/icons/error.svg?react"; + +import styles from "./MediaView.module.css"; +import { Avatar } from "../Avatar"; + +interface Props extends ComponentProps { + className?: string; + style?: ComponentProps["style"]; + targetWidth: number; + targetHeight: number; + video: TrackReferenceOrPlaceholder; + videoFit: "cover" | "contain"; + mirror: boolean; + member: RoomMember | undefined; + videoEnabled: boolean; + unencryptedWarning: boolean; + nameTagLeadingIcon?: ReactNode; + nameTag: string; + displayName: string; + primaryButton?: ReactNode; +} + +export const MediaView = forwardRef( + ( + { + className, + style, + targetWidth, + targetHeight, + video, + videoFit, + mirror, + member, + videoEnabled, + unencryptedWarning, + nameTagLeadingIcon, + nameTag, + displayName, + primaryButton, + ...props + }, + ref, + ) => { + const { t } = useTranslation(); + + return ( + +
+ + {video.publication !== undefined && ( + + )} +
+
+
+ {nameTagLeadingIcon} + + {nameTag} + + {unencryptedWarning && ( + + + + )} +
+ {primaryButton} +
+ + ); + }, +); + +MediaView.displayName = "MediaView"; diff --git a/src/tile/SpotlightTile.module.css b/src/tile/SpotlightTile.module.css new file mode 100644 index 000000000..1aee4589f --- /dev/null +++ b/src/tile/SpotlightTile.module.css @@ -0,0 +1,167 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.tile { + display: flex; + border-radius: var(--cpd-space-6x); + contain: strict; + overflow-x: auto; + overflow-y: hidden; + scrollbar-width: none; + scroll-snap-type: inline mandatory; + scroll-snap-stop: always; + /* It would be nice to use smooth scrolling here, but Firefox has a bug where + it will not re-snap if the snapping point changes while it's smoothly + animating to another snapping point. + scroll-behavior: smooth; */ +} + +.tile.maximised { + border-radius: 0; +} + +.item { + height: 100%; + flex-basis: 100%; + flex-shrink: 0; + --media-view-fg-inset: 10px; +} + +.item.snap { + scroll-snap-align: start; +} + +.advance { + appearance: none; + cursor: pointer; + opacity: 0; + padding: calc(var(--cpd-space-3x) - var(--cpd-border-width-1)); + border: var(--cpd-border-width-1) solid + var(--cpd-color-border-interactive-secondary); + border-radius: var(--cpd-radius-pill-effect); + background: var(--cpd-color-alpha-gray-1400); + box-shadow: var(--small-drop-shadow); + transition-duration: 0.1s; + transition-property: opacity, background-color, border-color; + position: absolute; + z-index: 1; + /* Center the button vertically on the tile */ + top: 50%; + transform: translateY(-50%); +} + +.advance > svg { + display: block; + color: var(--cpd-color-icon-on-solid-primary); +} + +@media (hover) { + .advance:hover { + border-color: var(--cpd-color-bg-action-primary-hovered); + background: var(--cpd-color-bg-action-primary-hovered); + } +} + +.advance:active { + border-color: var(--cpd-color-bg-action-primary-pressed); + background: var(--cpd-color-bg-action-primary-pressed); +} + +.back { + inset-inline-start: var(--cpd-space-1x); +} + +.next { + inset-inline-end: var(--cpd-space-1x); +} + +.expand { + appearance: none; + cursor: pointer; + opacity: 0; + padding: var(--cpd-space-2x); + border: none; + border-radius: var(--cpd-radius-pill-effect); + background: var(--cpd-color-alpha-gray-1400); + box-shadow: var(--small-drop-shadow); + transition-duration: 0.1s; + transition-property: opacity, background-color; + position: absolute; + z-index: 1; + --inset: 6px; + inset-block-end: var(--inset); + inset-inline-end: var(--inset); +} + +.expand > svg { + display: block; + color: var(--cpd-color-icon-on-solid-primary); +} + +@media (hover) { + .expand:hover { + background: var(--cpd-color-bg-action-primary-hovered); + } +} + +.expand:active { + background: var(--cpd-color-bg-action-primary-pressed); +} + +@media (hover) { + .tile:hover > button { + opacity: 1; + } +} + +.tile:has(:focus-visible) > button { + opacity: 1; +} + +.indicators { + display: flex; + gap: var(--cpd-space-2x); + position: absolute; + inset-inline-start: 0; + inset-block-end: calc(-1 * var(--cpd-space-6x)); + width: 100%; + justify-content: start; + transition: opacity ease 0.15s; + opacity: 0; +} + +.indicators.show { + opacity: 1; +} + +.maximised .indicators { + inset-block-end: calc(-1 * var(--cpd-space-4x) - 2px); + justify-content: center; +} + +.indicators > .item { + inline-size: 32px; + block-size: 2px; + transition: background-color ease 0.15s; +} + +.indicators > .item[data-visible="false"] { + background: var(--cpd-color-alpha-gray-600); +} + +.indicators > .item[data-visible="true"] { + background: var(--cpd-color-gray-1400); +} diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx new file mode 100644 index 000000000..5407b1a75 --- /dev/null +++ b/src/tile/SpotlightTile.tsx @@ -0,0 +1,323 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { + ComponentProps, + RefAttributes, + forwardRef, + useCallback, + useEffect, + useRef, + useState, +} from "react"; +import ExpandIcon from "@vector-im/compound-design-tokens/icons/expand.svg?react"; +import CollapseIcon from "@vector-im/compound-design-tokens/icons/collapse.svg?react"; +import ChevronLeftIcon from "@vector-im/compound-design-tokens/icons/chevron-left.svg?react"; +import ChevronRightIcon from "@vector-im/compound-design-tokens/icons/chevron-right.svg?react"; +import { animated } from "@react-spring/web"; +import { Observable, map } from "rxjs"; +import { useObservableEagerState } from "observable-hooks"; +import { useTranslation } from "react-i18next"; +import classNames from "classnames"; +import { TrackReferenceOrPlaceholder } from "@livekit/components-core"; +import { RoomMember } from "matrix-js-sdk"; + +import { MediaView } from "./MediaView"; +import styles from "./SpotlightTile.module.css"; +import { + LocalUserMediaViewModel, + MediaViewModel, + ScreenShareViewModel, + UserMediaViewModel, + useNameData, +} from "../state/MediaViewModel"; +import { useInitial } from "../useInitial"; +import { useMergedRefs } from "../useMergedRefs"; +import { useObservableRef } from "../state/useObservable"; +import { useReactiveState } from "../useReactiveState"; +import { useLatest } from "../useLatest"; + +interface SpotlightItemBaseProps { + className?: string; + "data-id": string; + targetWidth: number; + targetHeight: number; + video: TrackReferenceOrPlaceholder; + member: RoomMember | undefined; + unencryptedWarning: boolean; + nameTag: string; + displayName: string; +} + +interface SpotlightUserMediaItemBaseProps extends SpotlightItemBaseProps { + videoEnabled: boolean; + videoFit: "contain" | "cover"; +} + +interface SpotlightLocalUserMediaItemProps + extends SpotlightUserMediaItemBaseProps { + vm: LocalUserMediaViewModel; +} + +const SpotlightLocalUserMediaItem = forwardRef< + HTMLDivElement, + SpotlightLocalUserMediaItemProps +>(({ vm, ...props }, ref) => { + const mirror = useObservableEagerState(vm.mirror); + return ; +}); + +SpotlightLocalUserMediaItem.displayName = "SpotlightLocalUserMediaItem"; + +interface SpotlightUserMediaItemProps extends SpotlightItemBaseProps { + vm: UserMediaViewModel; +} + +const SpotlightUserMediaItem = forwardRef< + HTMLDivElement, + SpotlightUserMediaItemProps +>(({ vm, ...props }, ref) => { + const videoEnabled = useObservableEagerState(vm.videoEnabled); + const cropVideo = useObservableEagerState(vm.cropVideo); + + const baseProps: SpotlightUserMediaItemBaseProps = { + videoEnabled, + videoFit: cropVideo ? "cover" : "contain", + ...props, + }; + + return vm instanceof LocalUserMediaViewModel ? ( + + ) : ( + + ); +}); + +SpotlightUserMediaItem.displayName = "SpotlightUserMediaItem"; + +interface SpotlightItemProps { + vm: MediaViewModel; + targetWidth: number; + targetHeight: number; + intersectionObserver: Observable; + /** + * Whether this item should act as a scroll snapping point. + */ + snap: boolean; +} + +const SpotlightItem = forwardRef( + ({ vm, targetWidth, targetHeight, intersectionObserver, snap }, theirRef) => { + const ourRef = useRef(null); + const ref = useMergedRefs(ourRef, theirRef); + const { displayName, nameTag } = useNameData(vm); + const video = useObservableEagerState(vm.video); + const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning); + + // Hook this item up to the intersection observer + useEffect(() => { + const element = ourRef.current!; + let prevIo: IntersectionObserver | null = null; + const subscription = intersectionObserver.subscribe((io) => { + prevIo?.unobserve(element); + io.observe(element); + prevIo = io; + }); + return (): void => { + subscription.unsubscribe(); + prevIo?.unobserve(element); + }; + }, [intersectionObserver]); + + const baseProps: SpotlightItemBaseProps & RefAttributes = { + ref, + "data-id": vm.id, + className: classNames(styles.item, { [styles.snap]: snap }), + targetWidth, + targetHeight, + video, + member: vm.member, + unencryptedWarning, + nameTag, + displayName, + }; + + return vm instanceof ScreenShareViewModel ? ( + + ) : ( + + ); + }, +); + +SpotlightItem.displayName = "SpotlightItem"; + +interface Props { + vms: MediaViewModel[]; + maximised: boolean; + expanded: boolean; + onToggleExpanded: (() => void) | null; + targetWidth: number; + targetHeight: number; + showIndicators: boolean; + className?: string; + style?: ComponentProps["style"]; +} + +export const SpotlightTile = forwardRef( + ( + { + vms, + maximised, + expanded, + onToggleExpanded, + targetWidth, + targetHeight, + showIndicators, + className, + style, + }, + theirRef, + ) => { + const { t } = useTranslation(); + const [root, ourRef] = useObservableRef(null); + const ref = useMergedRefs(ourRef, theirRef); + const [visibleId, setVisibleId] = useState(vms[0].id); + const latestVms = useLatest(vms); + const latestVisibleId = useLatest(visibleId); + const visibleIndex = vms.findIndex((vm) => vm.id === visibleId); + const canGoBack = visibleIndex > 0; + const canGoToNext = visibleIndex !== -1 && visibleIndex < vms.length - 1; + + // To keep track of which item is visible, we need an intersection observer + // hooked up to the root element and the items. Because the items will run + // their effects before their parent does, we need to do this dance with an + // Observable to actually give them the intersection observer. + const intersectionObserver = useInitial>( + () => + root.pipe( + map( + (r) => + new IntersectionObserver( + (entries) => { + const visible = entries.find((e) => e.isIntersecting); + if (visible !== undefined) + setVisibleId(visible.target.getAttribute("data-id")!); + }, + { root: r, threshold: 0.5 }, + ), + ), + ), + ); + + const [scrollToId, setScrollToId] = useReactiveState( + (prev) => + prev == null || prev === visibleId || vms.every((vm) => vm.id !== prev) + ? null + : prev, + [visibleId], + ); + + const onBackClick = useCallback(() => { + const vms = latestVms.current; + const visibleIndex = vms.findIndex( + (vm) => vm.id === latestVisibleId.current, + ); + if (visibleIndex > 0) setScrollToId(vms[visibleIndex - 1].id); + }, [latestVisibleId, latestVms, setScrollToId]); + + const onNextClick = useCallback(() => { + const vms = latestVms.current; + const visibleIndex = vms.findIndex( + (vm) => vm.id === latestVisibleId.current, + ); + if (visibleIndex !== -1 && visibleIndex !== vms.length - 1) + setScrollToId(vms[visibleIndex + 1].id); + }, [latestVisibleId, latestVms, setScrollToId]); + + const ToggleExpandIcon = expanded ? CollapseIcon : ExpandIcon; + + return ( + + {canGoBack && ( + + )} + {vms.map((vm) => ( + + ))} + {onToggleExpanded && ( + + )} + {canGoToNext && ( + + )} + {!expanded && ( +
1, + })} + > + {vms.map((vm) => ( +
+ ))} +
+ )} + + ); + }, +); + +SpotlightTile.displayName = "SpotlightTile"; diff --git a/src/useInitial.ts b/src/useInitial.ts new file mode 100644 index 000000000..3b794dd3d --- /dev/null +++ b/src/useInitial.ts @@ -0,0 +1,26 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { useRef } from "react"; + +/** + * React hook that returns the value given on the initial render. + */ +export function useInitial(getValue: () => T): T { + const ref = useRef<{ value: T }>(); + ref.current ??= { value: getValue() }; + return ref.current.value; +} diff --git a/src/useLatest.ts b/src/useLatest.ts new file mode 100644 index 000000000..a0e1ecc7a --- /dev/null +++ b/src/useLatest.ts @@ -0,0 +1,31 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { RefObject, useRef } from "react"; + +export interface LatestRef extends RefObject { + current: T; +} + +/** + * React hook that returns a ref containing the value given on the latest + * render. + */ +export function useLatest(value: T): LatestRef { + const ref = useRef(value); + ref.current = value; + return ref; +} diff --git a/src/useReactiveState.ts b/src/useReactiveState.ts index f5daa1fe0..afd509fb1 100644 --- a/src/useReactiveState.ts +++ b/src/useReactiveState.ts @@ -1,5 +1,5 @@ /* -Copyright 2023 New Vector Ltd +Copyright 2023-2024 New Vector Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -44,7 +44,8 @@ export const useReactiveState = ( if ( prevDeps.current === undefined || deps.length !== prevDeps.current.length || - deps.some((d, i) => d !== prevDeps.current![i]) + // Deps might be NaN, so we compare with Object.is rather than === + deps.some((d, i) => !Object.is(d, prevDeps.current![i])) ) { state.current = updateFn(state.current); } diff --git a/src/video-grid/BigGrid.tsx b/src/video-grid/BigGrid.tsx deleted file mode 100644 index bde7eda6d..000000000 --- a/src/video-grid/BigGrid.tsx +++ /dev/null @@ -1,1070 +0,0 @@ -/* -Copyright 2023 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import TinyQueue from "tinyqueue"; -import { RectReadOnly } from "react-use-measure"; -import { FC, memo, ReactNode } from "react"; -import { zip } from "lodash"; - -import { Slot } from "./NewVideoGrid"; -import { Layout } from "./Layout"; -import { count, findLastIndex } from "../array-utils"; -import styles from "./BigGrid.module.css"; -import { TileDescriptor } from "../state/CallViewModel"; - -/** - * A 1×1 cell in a grid which belongs to a tile. - */ -interface Cell { - /** - * The item displayed on the tile. - */ - readonly item: TileDescriptor; - /** - * Whether this cell is the origin (top left corner) of the tile. - */ - readonly origin: boolean; - /** - * The width, in columns, of the tile. - */ - readonly columns: number; - /** - * The height, in rows, of the tile. - */ - readonly rows: number; -} - -export interface Grid { - columns: number; - /** - * The cells of the grid, in left-to-right top-to-bottom order. - */ - cells: Cell[]; -} - -export interface SparseGrid { - columns: number; - /** - * The cells of the grid, in left-to-right top-to-bottom order. - * undefined = a gap in the grid. - */ - cells: (Cell | undefined)[]; -} - -/** - * Gets the paths that tiles should travel along in the grid to reach a - * particular destination. - * @param g The grid. - * @param dest The destination index. - * @param avoid A predicate defining the cells that paths should avoid going - * through. - * @returns An array in which each cell holds the index of the next cell to move - * to to reach the destination, or null if it is the destination or otherwise - * immovable. - */ -export function getPaths( - g: SparseGrid, - dest: number, - avoid: (cell: number) => boolean = (): boolean => false, -): (number | null)[] { - const destRow = row(dest, g); - const destColumn = column(dest, g); - - // This is Dijkstra's algorithm - - const distances = new Array(dest + 1).fill(Infinity); - distances[dest] = 0; - const edges = new Array(dest).fill(null); - edges[dest] = null; - const heap = new TinyQueue([dest], (i) => distances[i]); - - const visit = (curr: number, via: number, distanceVia: number): void => { - if (distanceVia < distances[curr]) { - distances[curr] = distanceVia; - edges[curr] = via; - heap.push(curr); - } - }; - - while (heap.length > 0) { - const via = heap.pop()!; - - if (!avoid(via)) { - const viaRow = row(via, g); - const viaColumn = column(via, g); - const viaCell = g.cells[via]; - const viaLargeTile = viaCell !== undefined && !is1By1(viaCell); - // Since it looks nicer to have paths go around large tiles, we impose an - // increased cost for moving through them - const distanceVia = distances[via] + (viaLargeTile ? 8 : 1); - - // Visit each neighbor - if (viaRow > 0) visit(via - g.columns, via, distanceVia); - if (viaColumn > 0) visit(via - 1, via, distanceVia); - if (viaColumn < (viaRow === destRow ? destColumn : g.columns - 1)) - visit(via + 1, via, distanceVia); - if ( - viaRow < destRow - 1 || - (viaRow === destRow - 1 && viaColumn <= destColumn) - ) - visit(via + g.columns, via, distanceVia); - } - } - - // The heap is empty, so we've generated all paths - return edges; -} - -const is1By1 = (c: Cell): boolean => c.columns === 1 && c.rows === 1; - -const findLast1By1Index = (g: SparseGrid): number | null => - findLastIndex(g.cells, (c) => c !== undefined && is1By1(c)); - -export function row(index: number, g: SparseGrid): number { - return Math.floor(index / g.columns); -} - -export function column(index: number, g: SparseGrid): number { - return ((index % g.columns) + g.columns) % g.columns; -} - -function inArea( - index: number, - start: number, - end: number, - g: SparseGrid, -): boolean { - const indexColumn = column(index, g); - const indexRow = row(index, g); - return ( - indexRow >= row(start, g) && - indexRow <= row(end, g) && - indexColumn >= column(start, g) && - indexColumn <= column(end, g) - ); -} - -function* cellsInArea( - start: number, - end: number, - g: SparseGrid, -): Generator { - const startColumn = column(start, g); - const endColumn = column(end, g); - for ( - let i = start; - i <= end; - i = - column(i, g) === endColumn - ? i + g.columns + startColumn - endColumn - : i + 1 - ) - yield i; -} - -export function forEachCellInArea( - start: number, - end: number, - g: G, - fn: (c: G["cells"][0], i: number) => void, -): void { - for (const i of cellsInArea(start, end, g)) fn(g.cells[i], i); -} - -function allCellsInArea( - start: number, - end: number, - g: G, - fn: (c: G["cells"][0], i: number) => boolean, -): boolean { - for (const i of cellsInArea(start, end, g)) { - if (!fn(g.cells[i], i)) return false; - } - - return true; -} - -/** - * Counts the number of cells in the area that satsify the given predicate. - */ -function countCellsInArea( - start: number, - end: number, - g: G, - predicate: (c: G["cells"][0], i: number) => boolean, -): number { - let count = 0; - for (const i of cellsInArea(start, end, g)) { - if (predicate(g.cells[i], i)) count++; - } - return count; -} - -const areaEnd = ( - start: number, - columns: number, - rows: number, - g: SparseGrid, -): number => start + columns - 1 + g.columns * (rows - 1); - -const cloneGrid = (g: G): G => ({ - ...g, - cells: [...g.cells], -}); - -/** - * Gets the index of the next gap in the grid that should be backfilled by 1×1 - * tiles. - */ -function getNextGap( - g: SparseGrid, - ignoreGap: (cell: number) => boolean, -): number | null { - const last1By1Index = findLast1By1Index(g); - if (last1By1Index === null) return null; - - for (let i = 0; i < last1By1Index; i++) { - // To make the backfilling process look natural when there are multiple - // gaps, we actually scan each row from right to left - const j = i; /* - (row(i, g) === row(last1By1Index, g) - ? last1By1Index - : (row(i, g) + 1) * g.columns) - - 1 - - column(i, g);*/ - - if (!ignoreGap(j) && g.cells[j] === undefined) return j; - } - - return null; -} - -/** - * Moves the tile at index "from" over to index "to", displacing other tiles - * along the way. - * Precondition: the destination area must consist of only 1×1 tiles. - */ -function moveTileUnchecked(g: SparseGrid, from: number, to: number): void { - const tile = g.cells[from]!; - const fromEnd = areaEnd(from, tile.columns, tile.rows, g); - const toEnd = areaEnd(to, tile.columns, tile.rows, g); - - const displacedTiles: Cell[] = []; - forEachCellInArea(to, toEnd, g, (c, i) => { - if (c !== undefined && !inArea(i, from, fromEnd, g)) - displacedTiles.push(c!); - }); - - const movingCells: Cell[] = []; - forEachCellInArea(from, fromEnd, g, (c, i) => { - movingCells.push(c!); - g.cells[i] = undefined; - }); - - forEachCellInArea( - to, - toEnd, - g, - (_c, i) => (g.cells[i] = movingCells.shift()), - ); - forEachCellInArea( - from, - fromEnd, - g, - (_c, i) => (g.cells[i] ??= displacedTiles.shift()), - ); -} - -/** - * Moves the tile at index "from" over to index "to", if there is space. - */ -export function moveTile( - g: G, - from: number, - to: number, -): G { - const tile = g.cells[from]!; - - if ( - to !== from && // Skip the operation if nothing would move - to >= 0 && - to < g.cells.length && - column(to, g) <= g.columns - tile.columns - ) { - const fromEnd = areaEnd(from, tile.columns, tile.rows, g); - const toEnd = areaEnd(to, tile.columns, tile.rows, g); - - // The contents of a given cell are 'displaceable' if it's empty, holds a - // 1×1 tile, or is part of the original tile we're trying to reposition - const displaceable = (c: Cell | undefined, i: number): boolean => - c === undefined || is1By1(c) || inArea(i, from, fromEnd, g); - - if (allCellsInArea(to, toEnd, g, displaceable)) { - // The target space is free; move - const gClone = cloneGrid(g); - moveTileUnchecked(gClone, from, to); - return gClone; - } - } - - // The target space isn't free; don't move - return g; -} - -/** - * Attempts to push a tile upwards by a number of rows, displacing 1×1 tiles. - * @returns The number of rows the tile was successfully pushed (may be less - * than requested if there are obstacles blocking movement). - */ -function pushTileUp( - g: SparseGrid, - from: number, - rows: number, - avoid: (cell: number) => boolean = (): boolean => false, -): number { - const tile = g.cells[from]!; - - for (let tryRows = rows; tryRows > 0; tryRows--) { - const to = from - tryRows * g.columns; - const toEnd = areaEnd(to, tile.columns, tile.rows, g); - - const cellsAboveAreDisplacable = - from - g.columns >= 0 && - allCellsInArea( - to, - Math.min(from - g.columns + tile.columns - 1, toEnd), - g, - (c, i) => (c === undefined || is1By1(c)) && !avoid(i), - ); - - if (cellsAboveAreDisplacable) { - moveTileUnchecked(g, from, to); - return tryRows; - } - } - - return 0; -} - -function trimTrailingGaps(g: SparseGrid): void { - // Shrink the array to remove trailing gaps - const newLength = (findLastIndex(g.cells, (c) => c !== undefined) ?? -1) + 1; - if (newLength !== g.cells.length) g.cells = g.cells.slice(0, newLength); -} - -/** - * Determines whether the given area is sufficiently clear of obstacles for - * vacateArea to work. - */ -function canVacateArea(g: SparseGrid, start: number, end: number): boolean { - const newCellCount = countCellsInArea(start, end, g, (c) => c !== undefined); - const newFullRows = Math.floor(newCellCount / g.columns); - return allCellsInArea( - start, - end - newFullRows * g.columns, - g, - (c) => c === undefined || is1By1(c), - ); -} - -/** - * Clears away all the tiles in a given area by pushing them elsewhere. - * Precondition: the area must first be checked with canVacateArea, and the only - * gaps in the given grid must lie either within the area being cleared, or - * after the last 1×1 tile. - */ -function vacateArea(g: SparseGrid, start: number, end: number): SparseGrid { - const newCellCount = countCellsInArea( - start, - end, - g, - (c, i) => c !== undefined || i >= g.cells.length, - ); - const newFullRows = Math.floor(newCellCount / g.columns); - const endRow = row(end, g); - - // To avoid subverting users' expectations, this operation should be the exact - // inverse of fillGaps. We do this by reverse-engineering a grid G with the - // area cleared out and structured such that fillGaps(G) = g. - - // A grid that will have the same structure as the final result, but be filled - // with fake data - const outputStructure: SparseGrid = { - columns: g.columns, - cells: new Array(g.cells.length + newCellCount), - }; - - // The first step in populating outputStructure is to copy over all the large - // tiles, pushing those tiles downwards that fillGaps would push upwards - g.cells.forEach((cell, fromStart) => { - if (cell?.origin && !is1By1(cell)) { - const fromEnd = areaEnd(fromStart, cell.columns, cell.rows, g); - const offset = - row(fromStart, g) + newFullRows > endRow ? newFullRows * g.columns : 0; - forEachCellInArea(fromStart, fromEnd, g, (c, i) => { - outputStructure.cells[i + offset] = c; - }); - } - }); - - // Then, we need to fill it in with the same number of 1×1 tiles as appear in - // the input - const oneByOneTileCount = count(g.cells, (c) => c !== undefined && is1By1(c)); - let oneByOneTilesDistributed = 0; - - for (let i = 0; i < outputStructure.cells.length; i++) { - if (outputStructure.cells[i] === undefined) { - if (inArea(i, start, end, g)) { - // Leave the requested area clear - outputStructure.cells[i] = undefined; - } else if (oneByOneTilesDistributed < oneByOneTileCount) { - outputStructure.cells[i] = { - // Fake data because we only care about the grid's structure - item: {} as unknown as TileDescriptor, - origin: true, - columns: 1, - rows: 1, - }; - oneByOneTilesDistributed++; - } - } - } - - // Lastly, handle the edge case where there were gaps in the input after the - // last 1×1 tile by resizing the cells array to delete these gaps - trimTrailingGaps(outputStructure); - - // outputStructure is now fully populated, and so running fillGaps on it - // should produce a grid with the same structure as the input - const inputStructure = fillGaps( - outputStructure, - false, - (i) => inArea(i, start, end, g) && g.cells[i] === undefined, - ); - - // We exploit the fact that g and inputStructure have the same structure to - // create a mapping between cells in the structure grids and cells in g - const structureMapping = new Map(zip(inputStructure.cells, g.cells)); - - // And finally, we can use that mapping to swap the fake data in - // outputStructure with the real thing - return { - columns: g.columns, - cells: outputStructure.cells.map((placeholder) => - structureMapping.get(placeholder), - ), - }; -} - -/** - * Backfill any gaps in the grid. - */ -export function fillGaps( - g: SparseGrid, - packLargeTiles?: true, - ignoreGap?: () => false, -): Grid; -export function fillGaps( - g: SparseGrid, - packLargeTiles?: boolean, - ignoreGap?: (cell: number) => boolean, -): SparseGrid; -export function fillGaps( - g: SparseGrid, - packLargeTiles = true, - ignoreGap: (cell: number) => boolean = (): boolean => false, -): SparseGrid { - const lastGap = findLastIndex( - g.cells, - (c, i) => c === undefined && !ignoreGap(i), - ); - if (lastGap === null) return g; // There are no gaps to fill - const lastGapRow = row(lastGap, g); - - const result = cloneGrid(g); - - // This will be the size of the grid after we're done here (assuming we're - // allowed to pack the large tiles into the rest of the grid as necessary) - let idealLength = count( - result.cells, - (c, i) => c !== undefined || ignoreGap(i), - ); - const fullRowsRemoved = Math.floor( - (g.cells.length - idealLength) / g.columns, - ); - - // Step 1: Push all large tiles below the last gap upwards, so that they move - // roughly the same distance that we're expecting 1×1 tiles to move - if (fullRowsRemoved > 0) { - for ( - let i = (lastGapRow + 1) * result.columns; - i < result.cells.length; - i++ - ) { - const cell = result.cells[i]; - if (cell?.origin && !is1By1(cell)) - pushTileUp(result, i, fullRowsRemoved, ignoreGap); - } - } - - // Step 2: Deal with any large tiles that are still hanging off the bottom - if (packLargeTiles) { - for (let i = result.cells.length - 1; i >= idealLength; i--) { - const cell = result.cells[i]; - if (cell !== undefined && !is1By1(cell)) { - // First, try to just push it upwards a bit more - const originIndex = - i - (cell.columns - 1) - result.columns * (cell.rows - 1); - const pushed = pushTileUp(result, originIndex, 1, ignoreGap) === 1; - - // If that failed, collapse the tile to 1×1 so it can be dealt with in - // step 3 - if (!pushed) { - const collapsedTile: Cell = { - item: cell.item, - origin: true, - columns: 1, - rows: 1, - }; - forEachCellInArea(originIndex, i, result, (_c, j) => { - result.cells[j] = undefined; - }); - result.cells[i] = collapsedTile; - // Collapsing the tile makes the final grid size smaller - idealLength -= cell.columns * cell.rows - 1; - } - } - } - } - - // Step 3: Fill all remaining gaps with 1×1 tiles - let gap = getNextGap(result, ignoreGap); - - if (gap !== null) { - const pathsToEnd = getPaths(result, findLast1By1Index(result)!, ignoreGap); - - do { - let filled = false; - let to = gap; - let from = pathsToEnd[gap]; - - // First, attempt to fill the gap by moving 1×1 tiles backwards from the - // end of the grid along a set path - while (from !== null) { - const toCell = result.cells[to] as Cell | undefined; - const fromCell = result.cells[from] as Cell | undefined; - - // Skip over slots that are already full - if (toCell !== undefined) { - to = pathsToEnd[to]!; - // Skip over large tiles. Also, we might run into gaps along the path - // created during the filling of previous gaps. Skip over those too; - // they'll be picked up on the next iteration of the outer loop. - } else if (fromCell === undefined || !is1By1(fromCell)) { - from = pathsToEnd[from]; - } else { - result.cells[to] = result.cells[from]; - result.cells[from] = undefined; - filled = true; - to = pathsToEnd[to]!; - from = pathsToEnd[from]; - } - } - - // In case the path approach failed, fall back to taking the very last 1×1 - // tile, and just dropping it into place - if (!filled) { - const last1By1Index = findLast1By1Index(result)!; - result.cells[gap] = result.cells[last1By1Index]; - result.cells[last1By1Index] = undefined; - } - - gap = getNextGap(result, ignoreGap); - } while (gap !== null); - } - - trimTrailingGaps(result); - return result; -} - -// TODO: replace all usages of this function with vacateArea, as this results in -// somewhat unpredictable movement -function createRows(g: SparseGrid, count: number, atRow: number): SparseGrid { - const result = { - columns: g.columns, - cells: new Array(g.cells.length + g.columns * count), - }; - const offsetAfterNewRows = g.columns * count; - - // Copy tiles from the original grid to the new one, with the new rows - // inserted at the target location - g.cells.forEach((c, from) => { - if (c?.origin) { - const offset = row(from, g) >= atRow ? offsetAfterNewRows : 0; - forEachCellInArea( - from, - areaEnd(from, c.columns, c.rows, g), - g, - (c, i) => { - result.cells[i + offset] = c; - }, - ); - } - }); - - return result; -} - -/** - * Adds a set of new items into the grid. - */ -export function addItems( - items: TileDescriptor[], - g: SparseGrid, -): SparseGrid { - let result: SparseGrid = cloneGrid(g); - - for (const item of items) { - const cell = { - item, - origin: true, - columns: 1, - rows: 1, - }; - - let placeAt: number; - - if (item.placeNear === undefined) { - // This item has no special placement requests, so let's put it - // uneventfully at the end of the grid - placeAt = result.cells.length; - } else { - // This item wants to be placed near another; let's put it on a row - // directly below the related tile - const placeNear = result.cells.findIndex( - (c) => c?.item.id === item.placeNear, - ); - if (placeNear === -1) { - // Can't find the related tile, so let's give up and place it at the end - placeAt = result.cells.length; - } else { - const placeNearCell = result.cells[placeNear]!; - const placeNearEnd = areaEnd( - placeNear, - placeNearCell.columns, - placeNearCell.rows, - result, - ); - - result = createRows(result, 1, row(placeNearEnd, result) + 1); - placeAt = - placeNear + - Math.floor(placeNearCell.columns / 2) + - result.columns * placeNearCell.rows; - } - } - - result.cells[placeAt] = cell; - - if (item.largeBaseSize) { - // Cycle the tile size once to set up the tile with its larger base size - // This also fills any gaps in the grid, hence no extra call to fillGaps - result = cycleTileSize(result, item); - } - } - - return result; -} - -const largeTileDimensions = (g: SparseGrid): [number, number] => [ - Math.min(3, Math.max(2, g.columns - 1)), - 2, -]; - -const extraLargeTileDimensions = (g: SparseGrid): [number, number] => - g.columns > 3 ? [4, 3] : [g.columns, 2]; - -export function cycleTileSize( - g: G, - tile: TileDescriptor, -): G { - const from = g.cells.findIndex((c) => c?.item === tile); - if (from === -1) return g; // Tile removed, no change - const fromCell = g.cells[from]!; - const fromWidth = fromCell.columns; - const fromHeight = fromCell.rows; - - const [baseDimensions, enlargedDimensions] = fromCell.item.largeBaseSize - ? [largeTileDimensions(g), extraLargeTileDimensions(g)] - : [[1, 1], largeTileDimensions(g)]; - // The target dimensions, which toggle between the base and enlarged sizes - const [toWidth, toHeight] = - fromWidth === baseDimensions[0] && fromHeight === baseDimensions[1] - ? enlargedDimensions - : baseDimensions; - - return setTileSize(g, from, toWidth, toHeight); -} - -/** - * Finds the cell nearest to 'nearestTo' that satsifies the given predicate. - * @param shouldScan A predicate constraining the bounds of the search. - */ -function findNearestCell( - g: G, - nearestTo: number, - shouldScan: (index: number) => boolean, - predicate: (cell: G["cells"][0], index: number) => boolean, -): number | null { - const scanLocations = new Set([nearestTo]); - - for (const scanLocation of scanLocations) { - if (shouldScan(scanLocation)) { - if (predicate(g.cells[scanLocation], scanLocation)) return scanLocation; - - // Scan outwards in all directions - const scanColumn = column(scanLocation, g); - const scanRow = row(scanLocation, g); - if (scanColumn > 0) scanLocations.add(scanLocation - 1); - if (scanColumn < g.columns - 1) scanLocations.add(scanLocation + 1); - if (scanRow > 0) scanLocations.add(scanLocation - g.columns); - scanLocations.add(scanLocation + g.columns); - } - } - - return null; -} - -/** - * Changes the size of a tile, rearranging the grid to make space. - * @param tileId The ID of the tile to modify. - * @param g The grid. - * @returns The updated grid. - */ -export function setTileSize( - g: G, - from: number, - toWidth: number, - toHeight: number, -): G { - const fromCell = g.cells[from]!; - const fromWidth = fromCell.columns; - const fromHeight = fromCell.rows; - const fromEnd = areaEnd(from, fromWidth, fromHeight, g); - const newGridSize = - g.cells.length + toWidth * toHeight - fromWidth * fromHeight; - - const toColumn = Math.max( - 0, - Math.min( - g.columns - toWidth, - column(from, g) + Math.trunc((fromWidth - toWidth) / 2), - ), - ); - const toRow = Math.max( - 0, - row(from, g) + Math.trunc((fromHeight - toHeight) / 2), - ); - const targetDest = toColumn + toRow * g.columns; - - const gridWithoutTile = cloneGrid(g); - forEachCellInArea(from, fromEnd, gridWithoutTile, (_c, i) => { - gridWithoutTile.cells[i] = undefined; - }); - - const placeTile = ( - to: number, - toEnd: number, - grid: Grid | SparseGrid, - ): void => { - forEachCellInArea(to, toEnd, grid, (_c, i) => { - grid.cells[i] = { - item: fromCell.item, - origin: i === to, - columns: toWidth, - rows: toHeight, - }; - }); - }; - - if (toWidth <= fromWidth && toHeight <= fromHeight) { - // The tile is shrinking, which can always happen in-place - const to = targetDest; - const toEnd = areaEnd(to, toWidth, toHeight, g); - - const result: SparseGrid = gridWithoutTile; - placeTile(to, toEnd, result); - return fillGaps(result, true, (i: number) => inArea(i, to, toEnd, g)) as G; - } else if (toWidth >= fromWidth && toHeight >= fromHeight) { - // The tile is growing, which might be able to happen in-place - const to = findNearestCell( - gridWithoutTile, - targetDest, - (i) => { - const end = areaEnd(i, toWidth, toHeight, g); - return ( - column(i, g) + toWidth - 1 < g.columns && - inArea(from, i, end, g) && - inArea(fromEnd, i, end, g) - ); - }, - (_c, i) => { - const end = areaEnd(i, toWidth, toHeight, g); - return end < newGridSize && canVacateArea(gridWithoutTile, i, end); - }, - ); - - if (to !== null) { - const toEnd = areaEnd(to, toWidth, toHeight, g); - const result = vacateArea(gridWithoutTile, to, toEnd); - - placeTile(to, toEnd, result); - return result as G; - } - } - - // Catch-all path for when the tile is neither strictly shrinking nor - // growing, or when there's not enough space for it to grow in-place - - const packedGridWithoutTile = fillGaps(gridWithoutTile, false); - - const to = findNearestCell( - packedGridWithoutTile, - targetDest, - (i) => i < newGridSize && column(i, g) + toWidth - 1 < g.columns, - (_c, i) => { - const end = areaEnd(i, toWidth, toHeight, g); - return end < newGridSize && canVacateArea(packedGridWithoutTile, i, end); - }, - ); - - if (to === null) return g; // There's no space anywhere; give up - - const toEnd = areaEnd(to, toWidth, toHeight, g); - const result = vacateArea(packedGridWithoutTile, to, toEnd); - - placeTile(to, toEnd, result); - return result as G; -} - -/** - * Resizes the grid to a new column width. - */ -export function resize(g: Grid, columns: number): Grid { - const result: SparseGrid = { columns, cells: [] }; - const [largeColumns, largeRows] = largeTileDimensions(result); - - // Copy each tile from the old grid to the resized one in the same order - - // The next index in the result grid to copy a tile to - let next = 0; - - for (const cell of g.cells) { - if (cell.origin) { - // TODO make aware of extra large tiles - const [nextColumns, nextRows] = is1By1(cell) - ? [1, 1] - : [largeColumns, largeRows]; - - // If there isn't enough space left on this row, jump to the next row - if (columns - column(next, result) < nextColumns) - next = columns * (Math.floor(next / columns) + 1); - const nextEnd = areaEnd(next, nextColumns, nextRows, result); - - // Expand the cells array as necessary - if (result.cells.length <= nextEnd) - result.cells.push(...new Array(nextEnd + 1 - result.cells.length)); - - // Copy the tile into place - forEachCellInArea(next, nextEnd, result, (_c, i) => { - result.cells[i] = { - item: cell.item, - origin: i === next, - columns: nextColumns, - rows: nextRows, - }; - }); - - next = nextEnd + 1; - } - } - - return fillGaps(result); -} - -/** - * Promotes speakers to the first page of the grid. - */ -export function promoteSpeakers(g: SparseGrid): void { - // This is all a bit of a hack right now, because we don't know if the designs - // will stick with this approach in the long run - // We assume that 4 rows are probably about 1 page - const firstPageEnd = g.columns * 4; - - for (let from = firstPageEnd; from < g.cells.length; from++) { - const fromCell = g.cells[from]; - // Don't bother trying to promote enlarged tiles - if (fromCell?.item.isSpeaker && is1By1(fromCell)) { - // Promote this tile by making 10 attempts to place it on the first page - for (let j = 0; j < 10; j++) { - const to = Math.floor(Math.random() * firstPageEnd); - const toCell = g.cells[to]; - if (toCell === undefined || is1By1(toCell)) { - moveTileUnchecked(g, from, to); - break; - } - } - } - } -} - -/** - * The algorithm for updating a grid with a new set of tiles. - */ -function updateTiles(g: Grid, tiles: TileDescriptor[]): Grid { - // Step 1: Update tiles that still exist, and remove tiles that have left - // the grid - const itemsById = new Map(tiles.map((i) => [i.id, i])); - const grid1: SparseGrid = { - ...g, - cells: g.cells.map((c) => { - if (c === undefined) return undefined; - const item = itemsById.get(c.item.id); - return item === undefined ? undefined : { ...c, item }; - }), - }; - - // Step 2: Add new tiles - const existingItemIds = new Set( - grid1.cells.filter((c) => c !== undefined).map((c) => c!.item.id), - ); - const newItems = tiles.filter((i) => !existingItemIds.has(i.id)); - const grid2 = addItems(newItems, grid1); - - // Step 3: Promote speakers to the top - promoteSpeakers(grid2); - - return fillGaps(grid2); -} - -function updateBounds(g: Grid, bounds: RectReadOnly): Grid { - const columns = Math.max(2, Math.floor(bounds.width * 0.0055)); - return columns === g.columns ? g : resize(g, columns); -} - -const Slots: FC<{ s: Grid }> = memo(({ s: g }) => { - const areas = new Array<(number | null)[]>( - Math.ceil(g.cells.length / g.columns), - ); - for (let i = 0; i < areas.length; i++) - areas[i] = new Array(g.columns).fill(null); - - let slotCount = 0; - for (let i = 0; i < g.cells.length; i++) { - const cell = g.cells[i]; - if (cell.origin) { - const slotEnd = i + cell.columns - 1 + g.columns * (cell.rows - 1); - forEachCellInArea( - i, - slotEnd, - g, - (_c, j) => (areas[row(j, g)][column(j, g)] = slotCount), - ); - slotCount++; - } - } - - const style = { - gridTemplateAreas: areas - .map( - (row) => - `'${row - .map((slotId) => (slotId === null ? "." : `s${slotId}`)) - .join(" ")}'`, - ) - .join(" "), - gridTemplateColumns: `repeat(${g.columns}, 1fr)`, - }; - - const slots = new Array(slotCount); - for (let i = 0; i < slotCount; i++) - slots[i] = ; - - return ( -
- {slots} -
- ); -}); - -Slots.displayName = "Slots"; - -/** - * Given a tile and numbers in the range [0, 1) describing a position within the - * tile, this returns the index of the specific cell in which that position - * lies. - */ -function positionOnTileToCell( - g: SparseGrid, - tileOriginIndex: number, - xPositionOnTile: number, - yPositionOnTile: number, -): number { - const tileOrigin = g.cells[tileOriginIndex]!; - const columnOnTile = Math.floor(xPositionOnTile * tileOrigin.columns); - const rowOnTile = Math.floor(yPositionOnTile * tileOrigin.rows); - return tileOriginIndex + columnOnTile + g.columns * rowOnTile; -} - -function dragTile( - g: Grid, - from: TileDescriptor, - to: TileDescriptor, - xPositionOnFrom: number, - yPositionOnFrom: number, - xPositionOnTo: number, - yPositionOnTo: number, -): Grid { - const fromOrigin = g.cells.findIndex((c) => c.item === from); - const toOrigin = g.cells.findIndex((c) => c.item === to); - const fromCell = positionOnTileToCell( - g, - fromOrigin, - xPositionOnFrom, - yPositionOnFrom, - ); - const toCell = positionOnTileToCell( - g, - toOrigin, - xPositionOnTo, - yPositionOnTo, - ); - - return moveTile(g, fromOrigin, fromOrigin + toCell - fromCell); -} - -export const BigGrid: Layout = { - emptyState: { columns: 4, cells: [] }, - updateTiles, - updateBounds, - getTiles: (g: Grid) => - g.cells.filter((c) => c.origin).map((c) => c!.item as T), - canDragTile: () => true, - dragTile, - toggleFocus: cycleTileSize, - Slots, - rememberState: false, -}; diff --git a/src/video-grid/Layout.tsx b/src/video-grid/Layout.tsx deleted file mode 100644 index b540cbe17..000000000 --- a/src/video-grid/Layout.tsx +++ /dev/null @@ -1,195 +0,0 @@ -/* -Copyright 2023 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { ComponentType, ReactNode, useCallback, useMemo, useRef } from "react"; - -import type { RectReadOnly } from "react-use-measure"; -import { useReactiveState } from "../useReactiveState"; -import { TileDescriptor } from "../state/CallViewModel"; - -/** - * A video grid layout system with concrete states of type State. - */ -// Ideally State would be parameterized by the tile data type, but then that -// makes Layout a higher-kinded type, which isn't achievable in TypeScript -// (unless you invoke some dark type-level computation magic… 😏) -// So we're stuck with these types being a little too strong. -export interface Layout { - /** - * The layout state for zero tiles. - */ - readonly emptyState: State; - /** - * Updates/adds/removes tiles in a way that looks natural in the context of - * the given initial state. - */ - readonly updateTiles: (s: State, tiles: TileDescriptor[]) => State; - /** - * Adapts the layout to a new container size. - */ - readonly updateBounds: (s: State, bounds: RectReadOnly) => State; - /** - * Gets tiles in the order created by the layout. - */ - readonly getTiles: (s: State) => TileDescriptor[]; - /** - * Determines whether a tile is draggable. - */ - readonly canDragTile: (s: State, tile: TileDescriptor) => boolean; - /** - * Drags the tile 'from' to the location of the tile 'to' (if possible). - * The position parameters are numbers in the range [0, 1) describing the - * specific positions on 'from' and 'to' that the drag gesture is targeting. - */ - readonly dragTile: ( - s: State, - from: TileDescriptor, - to: TileDescriptor, - xPositionOnFrom: number, - yPositionOnFrom: number, - xPositionOnTo: number, - yPositionOnTo: number, - ) => State; - /** - * Toggles the focus of the given tile (if this layout has the concept of - * focus). - */ - readonly toggleFocus?: (s: State, tile: TileDescriptor) => State; - /** - * A React component generating the slot elements for a given layout state. - */ - readonly Slots: ComponentType<{ s: State }>; - /** - * Whether the state of this layout should be remembered even while a - * different layout is active. - */ - readonly rememberState: boolean; -} - -/** - * A version of Map with stronger types that allow us to save layout states in a - * type-safe way. - */ -export interface LayoutStatesMap { - get(layout: Layout): State | undefined; - set(layout: Layout, state: State): LayoutStatesMap; - delete(layout: Layout): boolean; -} - -/** - * Hook creating a Map to store layout states in. - */ -export const useLayoutStates = (): LayoutStatesMap => { - const layoutStates = useRef>(); - if (layoutStates.current === undefined) layoutStates.current = new Map(); - return layoutStates.current as LayoutStatesMap; -}; - -interface UseLayout { - state: State; - orderedItems: TileDescriptor[]; - generation: number; - canDragTile: (tile: TileDescriptor) => boolean; - dragTile: ( - from: TileDescriptor, - to: TileDescriptor, - xPositionOnFrom: number, - yPositionOnFrom: number, - xPositionOnTo: number, - yPositionOnTo: number, - ) => void; - toggleFocus: ((tile: TileDescriptor) => void) | undefined; - slots: ReactNode; -} - -/** - * Hook which uses the provided layout system to arrange a set of items into a - * concrete layout state, and provides callbacks for user interaction. - */ -export function useLayout( - layout: Layout, - items: TileDescriptor[], - bounds: RectReadOnly, - layoutStates: LayoutStatesMap, -): UseLayout { - const prevLayout = useRef>(); - const prevState = layoutStates.get(layout); - - const [state, setState] = useReactiveState(() => { - // If the bounds aren't known yet, don't add anything to the layout - if (bounds.width === 0) { - return layout.emptyState; - } else { - if ( - prevLayout.current !== undefined && - layout !== prevLayout.current && - !prevLayout.current.rememberState - ) - layoutStates.delete(prevLayout.current); - - const baseState = layoutStates.get(layout) ?? layout.emptyState; - return layout.updateTiles(layout.updateBounds(baseState, bounds), items); - } - }, [layout, items, bounds]); - - const generation = useRef(0); - if (state !== prevState) generation.current++; - - prevLayout.current = layout as Layout; - // No point in remembering an empty state, plus it would end up clobbering the - // real saved state while restoring a layout - if (state !== layout.emptyState) layoutStates.set(layout, state); - - return { - state, - orderedItems: useMemo(() => layout.getTiles(state), [layout, state]), - generation: generation.current, - canDragTile: useCallback( - (tile: TileDescriptor) => layout.canDragTile(state, tile), - [layout, state], - ), - dragTile: useCallback( - ( - from: TileDescriptor, - to: TileDescriptor, - xPositionOnFrom: number, - yPositionOnFrom: number, - xPositionOnTo: number, - yPositionOnTo: number, - ) => - setState((s) => - layout.dragTile( - s, - from, - to, - xPositionOnFrom, - yPositionOnFrom, - xPositionOnTo, - yPositionOnTo, - ), - ), - [layout, setState], - ), - toggleFocus: useMemo( - () => - layout.toggleFocus && - ((tile: TileDescriptor): void => - setState((s) => layout.toggleFocus!(s, tile))), - [layout, setState], - ), - slots: , - }; -} diff --git a/src/video-grid/NewVideoGrid.tsx b/src/video-grid/NewVideoGrid.tsx deleted file mode 100644 index 61813125b..000000000 --- a/src/video-grid/NewVideoGrid.tsx +++ /dev/null @@ -1,389 +0,0 @@ -/* -Copyright 2023 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { SpringRef, TransitionFn, useTransition } from "@react-spring/web"; -import { EventTypes, Handler, useScroll } from "@use-gesture/react"; -import { - CSSProperties, - FC, - ReactNode, - useEffect, - useMemo, - useRef, - useState, -} from "react"; -import useMeasure from "react-use-measure"; -import { zip } from "lodash"; - -import styles from "./NewVideoGrid.module.css"; -import { - VideoGridProps as Props, - TileSpring, - ChildrenProperties, - TileSpringUpdate, -} from "./VideoGrid"; -import { useReactiveState } from "../useReactiveState"; -import { useMergedRefs } from "../useMergedRefs"; -import { TileWrapper } from "./TileWrapper"; -import { BigGrid } from "./BigGrid"; -import { useLayout } from "./Layout"; -import { TileDescriptor } from "../state/CallViewModel"; - -interface Rect { - x: number; - y: number; - width: number; - height: number; -} - -interface Tile extends Rect { - item: TileDescriptor; -} - -interface DragState { - tileId: string; - tileX: number; - tileY: number; - cursorX: number; - cursorY: number; -} - -interface TapData { - tileId: string; - ts: number; -} - -interface SlotProps { - style?: CSSProperties; -} - -export const Slot: FC = ({ style }) => ( -
-); - -/** - * An interactive, animated grid of video tiles. - */ -export function NewVideoGrid({ - items, - disableAnimations, - layoutStates, - children, -}: Props): ReactNode { - // Overview: This component lays out tiles by rendering an invisible template - // grid of "slots" for tiles to go in. Once rendered, it uses the DOM API to - // get the dimensions of each slot, feeding these numbers back into - // react-spring to let the actual tiles move freely atop the template. - - // To know when the rendered grid becomes consistent with the layout we've - // requested, we give it a data-generation attribute which holds the ID of the - // most recently rendered generation of the grid, and watch it with a - // MutationObserver. - - const [slotsRoot, setSlotsRoot] = useState(null); - const [renderedGeneration, setRenderedGeneration] = useState(0); - - useEffect(() => { - if (slotsRoot !== null) { - setRenderedGeneration( - parseInt(slotsRoot.getAttribute("data-generation")!), - ); - - const observer = new MutationObserver((mutations) => { - if (mutations.some((m) => m.type === "attributes")) { - setRenderedGeneration( - parseInt(slotsRoot.getAttribute("data-generation")!), - ); - } - }); - - observer.observe(slotsRoot, { attributes: true }); - return (): void => observer.disconnect(); - } - }, [slotsRoot, setRenderedGeneration]); - - const [gridRef1, gridBounds] = useMeasure(); - const gridRef2 = useRef(null); - const gridRef = useMergedRefs(gridRef1, gridRef2); - - const slotRects = useMemo(() => { - if (slotsRoot === null) return []; - - const slots = slotsRoot.getElementsByClassName(styles.slot); - const rects = new Array(slots.length); - for (let i = 0; i < slots.length; i++) { - const slot = slots[i] as HTMLElement; - rects[i] = { - x: slot.offsetLeft, - y: slot.offsetTop, - width: slot.offsetWidth, - height: slot.offsetHeight, - }; - } - - return rects; - // The rects may change due to the grid being resized or rerendered, but - // eslint can't statically verify this - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [slotsRoot, renderedGeneration, gridBounds]); - - // TODO: Implement more layouts and select the right one here - const layout = BigGrid; - const { - state: grid, - orderedItems, - generation, - canDragTile, - dragTile, - toggleFocus, - slots, - } = useLayout(layout, items, gridBounds, layoutStates); - - const [tiles] = useReactiveState[]>( - (prevTiles) => { - // If React hasn't yet rendered the current generation of the grid, skip - // the update, because grid and slotRects will be out of sync - if (renderedGeneration !== generation) return prevTiles ?? []; - - const tileRects = new Map( - zip(orderedItems, slotRects) as [TileDescriptor, Rect][], - ); - // In order to not break drag gestures, it's critical that we render tiles - // in a stable order (that of 'items') - return items.map((item) => ({ ...tileRects.get(item)!, item })); - }, - [slotRects, grid, renderedGeneration], - ); - - // Drag state is stored in a ref rather than component state, because we use - // react-spring's imperative API during gestures to improve responsiveness - const dragState = useRef(null); - - const [tileTransitions, springRef] = useTransition( - tiles, - () => ({ - key: ({ item }: Tile): string => item.id, - from: ({ x, y, width, height }: Tile): TileSpringUpdate => ({ - opacity: 0, - scale: 0, - shadow: 0, - shadowSpread: 0, - zIndex: 1, - x, - y, - width, - height, - immediate: disableAnimations, - }), - enter: { opacity: 1, scale: 1, immediate: disableAnimations }, - update: ({ - item, - x, - y, - width, - height, - }: Tile): TileSpringUpdate | null => - item.id === dragState.current?.tileId - ? null - : { - x, - y, - width, - height, - immediate: disableAnimations, - }, - leave: { opacity: 0, scale: 0, immediate: disableAnimations }, - config: { mass: 0.7, tension: 252, friction: 25 }, - }), - // react-spring's types are bugged and can't infer the spring type - ) as unknown as [TransitionFn, TileSpring>, SpringRef]; - - // Because we're using react-spring in imperative mode, we're responsible for - // firing animations manually whenever the tiles array updates - useEffect(() => { - springRef.start(); - }, [tiles, springRef]); - - const animateDraggedTile = (endOfGesture: boolean): void => { - const { tileId, tileX, tileY, cursorX, cursorY } = dragState.current!; - const tile = tiles.find((t) => t.item.id === tileId)!; - - springRef.current - .find((c) => (c.item as Tile).item.id === tileId) - ?.start( - endOfGesture - ? { - scale: 1, - zIndex: 1, - shadow: 0, - x: tile.x, - y: tile.y, - width: tile.width, - height: tile.height, - immediate: - disableAnimations || ((key): boolean => key === "zIndex"), - // Allow the tile's position to settle before pushing its - // z-index back down - delay: (key): number => (key === "zIndex" ? 500 : 0), - } - : { - scale: 1.1, - zIndex: 2, - shadow: 15, - x: tileX, - y: tileY, - immediate: - disableAnimations || - ((key): boolean => - key === "zIndex" || key === "x" || key === "y"), - }, - ); - - const overTile = tiles.find( - (t) => - cursorX >= t.x && - cursorX < t.x + t.width && - cursorY >= t.y && - cursorY < t.y + t.height, - ); - - if (overTile !== undefined) - dragTile( - tile.item, - overTile.item, - (cursorX - tileX) / tile.width, - (cursorY - tileY) / tile.height, - (cursorX - overTile.x) / overTile.width, - (cursorY - overTile.y) / overTile.height, - ); - }; - - const lastTap = useRef(null); - - // Callback for useDrag. We could call useDrag here, but the default - // pattern of spreading {...bind()} across the children to bind the gesture - // ends up breaking memoization and ruining this component's performance. - // Instead, we pass this callback to each tile via a ref, to let them bind the - // gesture using the much more sensible ref-based method. - const onTileDrag = ( - tileId: string, - - { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - tap, - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - initial: [initialX, initialY], - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - delta: [dx, dy], - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - last, - }: Parameters>[0], - ): void => { - if (tap) { - const now = Date.now(); - - if ( - tileId === lastTap.current?.tileId && - now - lastTap.current.ts < 500 - ) { - toggleFocus?.(items.find((i) => i.id === tileId)!); - lastTap.current = null; - } else { - lastTap.current = { tileId, ts: now }; - } - } else { - const tileController = springRef.current.find( - (c) => (c.item as Tile).item.id === tileId, - )!; - - if (canDragTile((tileController.item as Tile).item)) { - if (dragState.current === null) { - const tileSpring = tileController.get(); - dragState.current = { - tileId, - tileX: tileSpring.x, - tileY: tileSpring.y, - cursorX: initialX - gridBounds.x, - cursorY: initialY - gridBounds.y + scrollOffset.current, - }; - } - - dragState.current.tileX += dx; - dragState.current.tileY += dy; - dragState.current.cursorX += dx; - dragState.current.cursorY += dy; - - animateDraggedTile(last); - - if (last) dragState.current = null; - } - } - }; - - const onTileDragRef = useRef(onTileDrag); - onTileDragRef.current = onTileDrag; - - const scrollOffset = useRef(0); - - useScroll( - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - ({ xy: [, y], delta: [, dy] }) => { - scrollOffset.current = y; - - if (dragState.current !== null) { - dragState.current.tileY += dy; - dragState.current.cursorY += dy; - animateDraggedTile(false); - } - }, - { target: gridRef2 }, - ); - - // Render nothing if the grid has yet to be generated - if (grid === null) { - return
; - } - - return ( -
-
- {slots} -
- {tileTransitions((spring, tile) => ( - - {children as (props: ChildrenProperties) => ReactNode} - - ))} -
- ); -} diff --git a/src/video-grid/VideoGrid.tsx b/src/video-grid/VideoGrid.tsx deleted file mode 100644 index fa56bd88b..000000000 --- a/src/video-grid/VideoGrid.tsx +++ /dev/null @@ -1,1406 +0,0 @@ -/* -Copyright 2022-2023 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { - ComponentProps, - MutableRefObject, - ReactNode, - Ref, - useCallback, - useEffect, - useRef, - useState, -} from "react"; -import { - EventTypes, - FullGestureState, - Handler, - useGesture, -} from "@use-gesture/react"; -import { - animated, - SpringRef, - SpringValues, - useSprings, -} from "@react-spring/web"; -import useMeasure from "react-use-measure"; -import { ResizeObserver as JuggleResizeObserver } from "@juggle/resize-observer"; -import { logger } from "matrix-js-sdk/src/logger"; - -import styles from "./VideoGrid.module.css"; -import { Layout } from "../room/LayoutToggle"; -import { TileWrapper } from "./TileWrapper"; -import { LayoutStatesMap } from "./Layout"; -import { TileDescriptor } from "../state/CallViewModel"; - -interface TilePosition { - x: number; - y: number; - width: number; - height: number; - zIndex: number; -} - -export interface Tile { - key: string; - order: number; - item: TileDescriptor; - remove: boolean; - focused: boolean; - isPresenter: boolean; - isSpeaker: boolean; - hasVideo: boolean; -} - -export interface TileSpring { - opacity: number; - scale: number; - shadow: number; - shadowSpread: number; - zIndex: number; - x: number; - y: number; - width: number; - height: number; -} - -export interface TileSpringUpdate extends Partial { - from?: Partial; - reset?: boolean; - immediate?: boolean | ((key: string) => boolean); - delay?: (key: string) => number; -} - -type LayoutDirection = "vertical" | "horizontal"; - -export function useVideoGridLayout(hasScreenshareFeeds: boolean): { - layout: Layout; - setLayout: (layout: Layout) => void; -} { - const layoutRef = useRef("grid"); - const revertLayoutRef = useRef("grid"); - const prevHasScreenshareFeeds = useRef(hasScreenshareFeeds); - const [, forceUpdate] = useState({}); - - const setLayout = useCallback((layout: Layout) => { - // Store the user's set layout to revert to after a screenshare is finished - revertLayoutRef.current = layout; - layoutRef.current = layout; - forceUpdate({}); - }, []); - - // Note: We need the returned layout to update synchronously with a change in hasScreenshareFeeds - // so use refs and avoid useEffect. - if (prevHasScreenshareFeeds.current !== hasScreenshareFeeds) { - if (hasScreenshareFeeds) { - // Automatically switch to spotlight layout when there's a screenshare - layoutRef.current = "spotlight"; - } else { - // When the screenshares have ended, revert to the previous layout - layoutRef.current = revertLayoutRef.current; - } - } - - prevHasScreenshareFeeds.current = hasScreenshareFeeds; - - return { layout: layoutRef.current, setLayout }; -} - -const GAP = 20; - -function useIsMounted(): MutableRefObject { - const isMountedRef = useRef(false); - - useEffect(() => { - isMountedRef.current = true; - - return (): void => { - isMountedRef.current = false; - }; - }, []); - - return isMountedRef; -} - -function isInside([x, y]: number[], targetTile: TilePosition): boolean { - const left = targetTile.x; - const top = targetTile.y; - const bottom = targetTile.y + targetTile.height; - const right = targetTile.x + targetTile.width; - - if (x < left || x > right || y < top || y > bottom) { - return false; - } - - return true; -} - -const getPipGap = (gridAspectRatio: number, gridWidth: number): number => - gridAspectRatio < 1 || gridWidth < 700 ? 12 : 24; - -function getTilePositions( - tileCount: number, - focusedTileCount: number, - gridWidth: number, - gridHeight: number, - pipXRatio: number, - pipYRatio: number, - layout: Layout, -): TilePosition[] { - if (layout === "grid") { - if (tileCount === 2 && focusedTileCount === 0) { - return getOneOnOneLayoutTilePositions( - gridWidth, - gridHeight, - pipXRatio, - pipYRatio, - ); - } - - return getFreedomLayoutTilePositions( - tileCount, - focusedTileCount, - gridWidth, - gridHeight, - ); - } else { - return getSpotlightLayoutTilePositions(tileCount, gridWidth, gridHeight); - } -} - -function getOneOnOneLayoutTilePositions( - gridWidth: number, - gridHeight: number, - pipXRatio: number, - pipYRatio: number, -): TilePosition[] { - const [remotePosition] = getFreedomLayoutTilePositions( - 1, - 0, - gridWidth, - gridHeight, - ); - - const gridAspectRatio = gridWidth / gridHeight; - - const smallPip = gridAspectRatio < 1 || gridWidth < 700; - const maxPipWidth = smallPip ? 114 : 230; - const maxPipHeight = smallPip ? 163 : 155; - // Cap the PiP size at 1/3 the remote tile size, preserving aspect ratio - const pipScaleFactor = Math.min( - 1, - remotePosition.width / 3 / maxPipWidth, - remotePosition.height / 3 / maxPipHeight, - ); - const pipWidth = maxPipWidth * pipScaleFactor; - const pipHeight = maxPipHeight * pipScaleFactor; - const pipGap = getPipGap(gridAspectRatio, gridWidth); - - const pipMinX = remotePosition.x + pipGap; - const pipMinY = remotePosition.y + pipGap; - const pipMaxX = remotePosition.x + remotePosition.width - pipWidth - pipGap; - const pipMaxY = remotePosition.y + remotePosition.height - pipHeight - pipGap; - - return [ - { - // Apply the PiP position as a proportion of the available space - x: pipMinX + pipXRatio * (pipMaxX - pipMinX), - y: pipMinY + pipYRatio * (pipMaxY - pipMinY), - width: pipWidth, - height: pipHeight, - zIndex: 1, - }, - remotePosition, - ]; -} - -function getSpotlightLayoutTilePositions( - tileCount: number, - gridWidth: number, - gridHeight: number, -): TilePosition[] { - const tilePositions: TilePosition[] = []; - - const gridAspectRatio = gridWidth / gridHeight; - - if (gridAspectRatio < 1) { - // Vertical layout (mobile) - const spotlightTileHeight = - tileCount > 1 ? (gridHeight - GAP * 3) * (4 / 5) : gridHeight - GAP * 2; - const spectatorTileSize = - tileCount > 1 ? gridHeight - GAP * 3 - spotlightTileHeight : 0; - - for (let i = 0; i < tileCount; i++) { - if (i === 0) { - // Spotlight tile - tilePositions.push({ - x: GAP, - y: GAP, - width: gridWidth - GAP * 2, - height: spotlightTileHeight, - zIndex: 0, - }); - } else { - // Spectator tile - tilePositions.push({ - x: (GAP + spectatorTileSize) * (i - 1) + GAP, - y: spotlightTileHeight + GAP * 2, - width: spectatorTileSize, - height: spectatorTileSize, - zIndex: 0, - }); - } - } - } else { - // Horizontal layout (desktop) - const spotlightTileWidth = - tileCount > 1 ? ((gridWidth - GAP * 3) * 4) / 5 : gridWidth - GAP * 2; - const spectatorTileWidth = - tileCount > 1 ? gridWidth - GAP * 3 - spotlightTileWidth : 0; - const spectatorTileHeight = spectatorTileWidth * (9 / 16); - - for (let i = 0; i < tileCount; i++) { - if (i === 0) { - tilePositions.push({ - x: GAP, - y: GAP, - width: spotlightTileWidth, - height: gridHeight - GAP * 2, - zIndex: 0, - }); - } else { - tilePositions.push({ - x: GAP * 2 + spotlightTileWidth, - y: (GAP + spectatorTileHeight) * (i - 1) + GAP, - width: spectatorTileWidth, - height: spectatorTileHeight, - zIndex: 0, - }); - } - } - } - - return tilePositions; -} - -function getFreedomLayoutTilePositions( - tileCount: number, - focusedTileCount: number, - gridWidth: number, - gridHeight: number, -): TilePosition[] { - if (tileCount === 0) { - return []; - } - - if (tileCount > 12) { - logger.warn("Over 12 tiles is not currently supported"); - } - - const { layoutDirection, itemGridRatio } = getGridLayout( - tileCount, - focusedTileCount, - gridWidth, - gridHeight, - ); - - let itemGridWidth; - let itemGridHeight; - - if (layoutDirection === "vertical") { - itemGridWidth = gridWidth; - itemGridHeight = Math.round(gridHeight * itemGridRatio); - } else { - itemGridWidth = Math.round(gridWidth * itemGridRatio); - itemGridHeight = gridHeight; - } - - const itemTileCount = tileCount - focusedTileCount; - - const { - columnCount: itemColumnCount, - rowCount: itemRowCount, - tileAspectRatio: itemTileAspectRatio, - } = getSubGridLayout(itemTileCount, itemGridWidth, itemGridHeight); - - const itemGridPositions = getSubGridPositions( - itemTileCount, - itemColumnCount, - itemRowCount, - itemTileAspectRatio, - itemGridWidth, - itemGridHeight, - ); - const itemGridBounds = getSubGridBoundingBox(itemGridPositions); - - let focusedGridWidth: number; - let focusedGridHeight: number; - - if (focusedTileCount === 0) { - focusedGridWidth = 0; - focusedGridHeight = 0; - } else if (layoutDirection === "vertical") { - focusedGridWidth = gridWidth; - focusedGridHeight = - gridHeight - (itemGridBounds.height + (itemTileCount ? GAP * 2 : 0)); - } else { - focusedGridWidth = - gridWidth - (itemGridBounds.width + (itemTileCount ? GAP * 2 : 0)); - focusedGridHeight = gridHeight; - } - - const { - columnCount: focusedColumnCount, - rowCount: focusedRowCount, - tileAspectRatio: focusedTileAspectRatio, - } = getSubGridLayout(focusedTileCount, focusedGridWidth, focusedGridHeight); - - const focusedGridPositions = getSubGridPositions( - focusedTileCount, - focusedColumnCount, - focusedRowCount, - focusedTileAspectRatio, - focusedGridWidth, - focusedGridHeight, - ); - - const tilePositions = [...focusedGridPositions, ...itemGridPositions]; - - centerTiles(focusedGridPositions, focusedGridWidth, focusedGridHeight, 0, 0); - - if (layoutDirection === "vertical") { - centerTiles( - itemGridPositions, - gridWidth, - gridHeight - focusedGridHeight, - 0, - focusedGridHeight, - ); - } else { - centerTiles( - itemGridPositions, - gridWidth - focusedGridWidth, - gridHeight, - focusedGridWidth, - 0, - ); - } - - return tilePositions; -} - -function getSubGridBoundingBox(positions: TilePosition[]): { - left: number; - right: number; - top: number; - bottom: number; - width: number; - height: number; -} { - let left = 0; - let right = 0; - let top = 0; - let bottom = 0; - - for (let i = 0; i < positions.length; i++) { - const { x, y, width, height } = positions[i]; - - if (i === 0) { - left = x; - right = x + width; - top = y; - bottom = y + height; - } else { - if (x < left) { - left = x; - } - - if (y < top) { - top = y; - } - - if (x + width > right) { - right = x + width; - } - - if (y + height > bottom) { - bottom = y + height; - } - } - } - - return { - left, - right, - top, - bottom, - width: right - left, - height: bottom - top, - }; -} - -function isMobileBreakpoint(gridWidth: number, gridHeight: number): boolean { - const gridAspectRatio = gridWidth / gridHeight; - return gridAspectRatio < 1; -} - -function getGridLayout( - tileCount: number, - focusedTileCount: number, - gridWidth: number, - gridHeight: number, -): { itemGridRatio: number; layoutDirection: LayoutDirection } { - let layoutDirection: LayoutDirection = "horizontal"; - let itemGridRatio = 1; - - if (focusedTileCount === 0) { - return { itemGridRatio, layoutDirection }; - } - - if (isMobileBreakpoint(gridWidth, gridHeight)) { - layoutDirection = "vertical"; - itemGridRatio = 1 / 3; - } else { - layoutDirection = "horizontal"; - itemGridRatio = 1 / 3; - } - - return { itemGridRatio, layoutDirection }; -} - -function centerTiles( - positions: TilePosition[], - gridWidth: number, - gridHeight: number, - offsetLeft: number, - offsetTop: number, -): TilePosition[] { - const bounds = getSubGridBoundingBox(positions); - - const leftOffset = Math.round((gridWidth - bounds.width) / 2) + offsetLeft; - const topOffset = Math.round((gridHeight - bounds.height) / 2) + offsetTop; - - applyTileOffsets(positions, leftOffset, topOffset); - - return positions; -} - -function applyTileOffsets( - positions: TilePosition[], - leftOffset: number, - topOffset: number, -): TilePosition[] { - for (const position of positions) { - position.x += leftOffset; - position.y += topOffset; - } - - return positions; -} - -function getSubGridLayout( - tileCount: number, - gridWidth: number, - gridHeight: number, -): { columnCount: number; rowCount: number; tileAspectRatio: number } { - const gridAspectRatio = gridWidth / gridHeight; - - let columnCount: number; - let rowCount: number; - let tileAspectRatio: number = 16 / 9; - - if (gridAspectRatio < 3 / 4) { - // Phone - if (tileCount === 1) { - columnCount = 1; - rowCount = 1; - tileAspectRatio = 0; - } else if (tileCount <= 4) { - columnCount = 1; - rowCount = tileCount; - } else if (tileCount <= 12) { - columnCount = 2; - rowCount = Math.ceil(tileCount / columnCount); - tileAspectRatio = 0; - } else { - // Unsupported - columnCount = 3; - rowCount = Math.ceil(tileCount / columnCount); - tileAspectRatio = 1; - } - } else if (gridAspectRatio < 1) { - // Tablet - if (tileCount === 1) { - columnCount = 1; - rowCount = 1; - tileAspectRatio = 0; - } else if (tileCount <= 4) { - columnCount = 1; - rowCount = tileCount; - } else if (tileCount <= 12) { - columnCount = 2; - rowCount = Math.ceil(tileCount / columnCount); - } else { - // Unsupported - columnCount = 3; - rowCount = Math.ceil(tileCount / columnCount); - tileAspectRatio = 1; - } - } else if (gridAspectRatio < 17 / 9) { - // Computer - if (tileCount === 1) { - columnCount = 1; - rowCount = 1; - } else if (tileCount === 2) { - columnCount = 2; - rowCount = 1; - } else if (tileCount <= 4) { - columnCount = 2; - rowCount = 2; - } else if (tileCount <= 6) { - columnCount = 3; - rowCount = 2; - } else if (tileCount <= 8) { - columnCount = 4; - rowCount = 2; - tileAspectRatio = 1; - } else if (tileCount <= 12) { - columnCount = 4; - rowCount = 3; - tileAspectRatio = 1; - } else { - // Unsupported - columnCount = 4; - rowCount = 4; - } - } else if (gridAspectRatio <= 32 / 9) { - // Ultrawide - if (tileCount === 1) { - columnCount = 1; - rowCount = 1; - } else if (tileCount === 2) { - columnCount = 2; - rowCount = 1; - } else if (tileCount <= 4) { - columnCount = 2; - rowCount = 2; - } else if (tileCount <= 6) { - columnCount = 3; - rowCount = 2; - } else if (tileCount <= 8) { - columnCount = 4; - rowCount = 2; - } else if (tileCount <= 12) { - columnCount = 4; - rowCount = 3; - } else { - // Unsupported - columnCount = 4; - rowCount = 4; - } - } else { - // Super Ultrawide - if (tileCount <= 6) { - columnCount = tileCount; - rowCount = 1; - } else { - columnCount = Math.ceil(tileCount / 2); - rowCount = 2; - } - } - - return { columnCount, rowCount, tileAspectRatio }; -} - -function getSubGridPositions( - tileCount: number, - columnCount: number, - rowCount: number, - tileAspectRatio: number, - gridWidth: number, - gridHeight: number, -): TilePosition[] { - if (tileCount === 0) { - return []; - } - - const newTilePositions: TilePosition[] = []; - - const boxWidth = Math.round( - (gridWidth - GAP * (columnCount + 1)) / columnCount, - ); - const boxHeight = Math.round((gridHeight - GAP * (rowCount + 1)) / rowCount); - - let tileWidth: number; - let tileHeight: number; - - if (tileAspectRatio) { - const boxAspectRatio = boxWidth / boxHeight; - - if (boxAspectRatio > tileAspectRatio) { - tileWidth = boxHeight * tileAspectRatio; - tileHeight = boxHeight; - } else { - tileWidth = boxWidth; - tileHeight = boxWidth / tileAspectRatio; - } - } else { - tileWidth = boxWidth; - tileHeight = boxHeight; - } - - for (let i = 0; i < tileCount; i++) { - const verticalIndex = Math.floor(i / columnCount); - const top = verticalIndex * GAP + verticalIndex * tileHeight; - - let rowItemCount: number; - - if (verticalIndex + 1 === rowCount && tileCount % columnCount !== 0) { - rowItemCount = tileCount % columnCount; - } else { - rowItemCount = columnCount; - } - - const horizontalIndex = i % columnCount; - - let centeringPadding = 0; - - if (rowItemCount < columnCount) { - const subgridWidth = tileWidth * columnCount + (GAP * columnCount - 1); - centeringPadding = Math.round( - (subgridWidth - (tileWidth * rowItemCount + (GAP * rowItemCount - 1))) / - 2, - ); - } - - const left = - centeringPadding + GAP * horizontalIndex + tileWidth * horizontalIndex; - - newTilePositions.push({ - width: tileWidth, - height: tileHeight, - x: left, - y: top, - zIndex: 0, - }); - } - - return newTilePositions; -} - -// Calculates the number of possible tiles that can be displayed -function displayedTileCount( - layout: Layout, - tileCount: number, - gridWidth: number, - gridHeight: number, -): number { - let displayedTile = -1; - if (layout === "grid") { - return displayedTile; - } - if (tileCount < 2) { - return displayedTile; - } - - const gridAspectRatio = gridWidth / gridHeight; - - if (gridAspectRatio < 1) { - // Vertical layout (mobile) - const spotlightTileHeight = (gridHeight - GAP * 3) * (4 / 5); - const spectatorTileSize = gridHeight - GAP * 3 - spotlightTileHeight; - displayedTile = Math.round(gridWidth / spectatorTileSize); - } else { - const spotlightTileWidth = ((gridWidth - GAP * 3) * 4) / 5; - const spectatorTileWidth = gridWidth - GAP * 3 - spotlightTileWidth; - const spectatorTileHeight = spectatorTileWidth * (9 / 16); - displayedTile = Math.round(gridHeight / spectatorTileHeight); - } - - return displayedTile; -} - -// Sets the 'order' property on tiles based on the layout param and -// other properties of the tiles, eg. 'focused' and 'presenter' -export function reorderTiles( - tiles: Tile[], - layout: Layout, - displayedTile = -1, -): void { - // We use a special layout for 1:1 to always put the local tile first. - // We only do this if there are two tiles (obviously) and exactly one - // of them is local: during startup we can have tiles from other users - // but not our own, due to the order they're added, so without this we - // can assign multiple remote tiles order '1' and this persists through - // subsequent reorders because we preserve the order of the tiles. - if ( - layout === "grid" && - tiles.length === 2 && - tiles.filter((t) => t.item.local).length === 1 && - !tiles.some((t) => t.focused) - ) { - // 1:1 layout - tiles.forEach((tile) => (tile.order = tile.item.local ? 0 : 1)); - } else { - const focusedTiles: Tile[] = []; - const presenterTiles: Tile[] = []; - const onlyVideoTiles: Tile[] = []; - const otherTiles: Tile[] = []; - - const orderedTiles: Tile[] = new Array(tiles.length); - tiles.forEach((tile) => (orderedTiles[tile.order] = tile)); - - let firstLocalTile: Tile | undefined; - orderedTiles.forEach((tile) => { - if (tile.focused) { - focusedTiles.push(tile); - } else if (tile.isPresenter) { - presenterTiles.push(tile); - } else if (tile.hasVideo) { - if (tile.order === 0 && tile.item.local) { - firstLocalTile = tile; - } else { - onlyVideoTiles.push(tile); - } - } else { - if (tile.order === 0 && tile.item.local) { - firstLocalTile = tile; - } else { - otherTiles.push(tile); - } - } - }); - - if (firstLocalTile) { - if (firstLocalTile.hasVideo) { - onlyVideoTiles.push(firstLocalTile); - } else { - otherTiles.push(firstLocalTile); - } - } - - const reorderedTiles = [ - ...focusedTiles, - ...presenterTiles, - ...onlyVideoTiles, - ...otherTiles, - ]; - let nextSpeakerTileIndex = focusedTiles.length + presenterTiles.length; - - reorderedTiles.forEach((tile, i) => { - // If a speaker's natural ordering would place it outside the default - // visible area, promote them to the section dedicated to speakers - if (tile.isSpeaker && displayedTile <= i && nextSpeakerTileIndex < i) { - // Remove the tile from its current section - reorderedTiles.splice(i, 1); - // Insert it into the speaker section - reorderedTiles.splice(nextSpeakerTileIndex, 0, tile); - nextSpeakerTileIndex++; - } - }); - - reorderedTiles.forEach((tile, i) => (tile.order = i)); - } -} - -interface DragTileData { - offsetX: number; - offsetY: number; - key: string; - x: number; - y: number; -} - -export interface ChildrenProperties { - ref: Ref; - style: ComponentProps["style"]; - /** - * The width this tile will have once its animations have settled. - */ - targetWidth: number; - /** - * The height this tile will have once its animations have settled. - */ - targetHeight: number; - data: T; -} - -export interface VideoGridProps { - items: TileDescriptor[]; - layout: Layout; - disableAnimations: boolean; - layoutStates: LayoutStatesMap; - children: (props: ChildrenProperties) => ReactNode; -} - -export function VideoGrid({ - items, - layout, - disableAnimations, - children, -}: VideoGridProps): ReactNode { - // Place the PiP in the bottom right corner by default - const [pipXRatio, setPipXRatio] = useState(1); - const [pipYRatio, setPipYRatio] = useState(1); - - const [{ tiles, tilePositions }, setTileState] = useState<{ - tiles: Tile[]; - tilePositions: TilePosition[]; - }>({ - tiles: [], - tilePositions: [], - }); - const [scrollPosition, setScrollPosition] = useState(0); - const draggingTileRef = useRef(null); - const lastTappedRef = useRef<{ [index: string]: number }>({}); - const lastLayoutRef = useRef(layout); - const isMounted = useIsMounted(); - - // The 'polyfill' argument to useMeasure is not a polyfill at all but is the impl that is always used - // if passed, whether the browser has native support or not, so pass in either the browser native - // version or the ponyfill (yes, pony) because Juggle's resizeobserver ponyfill is being weirdly - // buggy for me on my dev env my never updating the size until the window resizes. - const [gridRef, gridBounds] = useMeasure({ - polyfill: window.ResizeObserver ?? JuggleResizeObserver, - }); - - useEffect(() => { - setTileState(({ tiles, ...rest }) => { - const newTiles: Tile[] = []; - const removedTileKeys: Set = new Set(); - - for (const tile of tiles) { - let item = items.find((item) => item.id === tile.key); - - let remove = false; - - if (!item) { - remove = true; - item = tile.item; - removedTileKeys.add(tile.key); - } - - let focused: boolean; - let isSpeaker: boolean; - let isPresenter: boolean; - let hasVideo: boolean; - if (layout === "spotlight") { - focused = item.focused; - isPresenter = item.isPresenter; - isSpeaker = item.isSpeaker; - hasVideo = item.hasVideo; - } else { - focused = layout === lastLayoutRef.current ? tile.focused : false; - isPresenter = false; - isSpeaker = false; - hasVideo = false; - } - - newTiles.push({ - key: item.id, - order: tile.order, - item, - remove, - focused, - isSpeaker: isSpeaker, - isPresenter: isPresenter, - hasVideo: hasVideo, - }); - } - - for (const item of items) { - const existingTileIndex = newTiles.findIndex( - ({ key }) => item.id === key, - ); - - const existingTile = newTiles[existingTileIndex]; - - if (existingTile && !existingTile.remove) { - continue; - } - - const newTile: Tile = { - key: item.id, - order: existingTile?.order ?? newTiles.length, - item, - remove: false, - focused: layout === "spotlight" && item.focused, - isPresenter: item.isPresenter, - isSpeaker: item.isSpeaker, - hasVideo: item.hasVideo, - }; - - if (existingTile) { - // Replace an existing tile - newTiles.splice(existingTileIndex, 1, newTile); - } else { - // Added tiles - newTiles.push(newTile); - } - } - - const presenter = newTiles.find((t) => t.isPresenter); - let displayedTile = -1; - // Only on screen share we will not move active displayed speaker - if (presenter !== undefined) { - displayedTile = displayedTileCount( - layout, - newTiles.length, - gridBounds.width, - gridBounds.height, - ); - } - - reorderTiles(newTiles, layout, displayedTile); - - if (removedTileKeys.size > 0) { - setTimeout(() => { - if (!isMounted.current) { - return; - } - - setTileState(({ tiles, ...rest }) => { - const newTiles: Tile[] = tiles - .filter((tile) => !removedTileKeys.has(tile.key)) - .map((tile) => ({ ...tile })); // clone before reordering - reorderTiles(newTiles, layout); - - const focusedTileCount = newTiles.reduce( - (count, tile) => count + (tile.focused ? 1 : 0), - 0, - ); - - return { - ...rest, - tiles: newTiles, - tilePositions: getTilePositions( - newTiles.length, - focusedTileCount, - gridBounds.width, - gridBounds.height, - pipXRatio, - pipYRatio, - layout, - ), - }; - }); - }, 250); - } - - const focusedTileCount = newTiles.reduce( - (count, tile) => count + (tile.focused ? 1 : 0), - 0, - ); - - lastLayoutRef.current = layout; - - return { - ...rest, - tiles: newTiles, - tilePositions: getTilePositions( - newTiles.length, - focusedTileCount, - gridBounds.width, - gridBounds.height, - pipXRatio, - pipYRatio, - layout, - ), - }; - }); - }, [items, gridBounds, layout, isMounted, pipXRatio, pipYRatio]); - - const tilePositionsValid = useRef(false); - - const animate = useCallback( - (tiles: Tile[]) => { - // Whether the tile positions were valid at the time of the previous - // animation - const tilePositionsWereValid = tilePositionsValid.current; - const oneOnOneLayout = - tiles.length === 2 && !tiles.some((t) => t.focused); - - return (tileIndex: number): TileSpringUpdate => { - const tile = tiles[tileIndex]; - const tilePosition = tilePositions[tile.order]; - const draggingTile = draggingTileRef.current; - const dragging = draggingTile && tile.key === draggingTile.key; - const remove = tile.remove; - tilePositionsValid.current = tilePosition.height > 0; - - if (dragging) { - return { - width: tilePosition.width, - height: tilePosition.height, - x: draggingTile.offsetX + draggingTile.x, - y: draggingTile.offsetY + draggingTile.y, - scale: 1.1, - opacity: 1, - zIndex: 2, - shadow: 15, - shadowSpread: 0, - immediate: (key: string): boolean => - disableAnimations || - key === "zIndex" || - key === "x" || - key === "y" || - key === "shadow" || - key === "shadowSpread", - from: { - shadow: 0, - scale: 0, - opacity: 0, - zIndex: 0, - }, - reset: false, - }; - } else { - const isMobile = isMobileBreakpoint( - gridBounds.width, - gridBounds.height, - ); - - const x = - tilePosition.x + - (layout === "spotlight" && tile.order !== 0 && isMobile - ? scrollPosition - : 0); - const y = - tilePosition.y + - (layout === "spotlight" && tile.order !== 0 && !isMobile - ? scrollPosition - : 0); - const from: { - shadow: number; - scale: number; - opacity: number; - zIndex?: number; - x?: number; - y?: number; - width?: number; - height?: number; - } = { shadow: 0, scale: 0, opacity: 0 }; - let reset = false; - - if (!tilePositionsWereValid) { - // This indicates that the component just mounted. We discard the - // previous keyframe by resetting the tile's position, so that it - // animates in from the right place on screen, rather than wherever - // the zero-height grid placed it. - from.x = x; - from.y = y; - from.width = tilePosition.width; - from.height = tilePosition.height; - reset = true; - } - - return { - x, - y, - width: tilePosition.width, - height: tilePosition.height, - scale: remove ? 0 : 1, - opacity: remove ? 0 : 1, - zIndex: tilePosition.zIndex, - shadow: oneOnOneLayout && tile.item.local ? 1 : 0, - shadowSpread: oneOnOneLayout && tile.item.local ? 1 : 0, - from, - reset, - immediate: (key: string): boolean => - disableAnimations || - key === "zIndex" || - key === "shadow" || - key === "shadowSpread", - // If we just stopped dragging a tile, give it time for the - // animation to settle before pushing its z-index back down - delay: (key: string): number => (key === "zIndex" ? 500 : 0), - }; - } - }; - }, - [tilePositions, disableAnimations, scrollPosition, layout, gridBounds], - ); - - const [springs, api] = useSprings(tiles.length, animate(tiles), [ - tilePositions, - tiles, - scrollPosition, - // react-spring's types are bugged and can't infer the spring type - ]) as unknown as [SpringValues[], SpringRef]; - - const onTap = useCallback( - (tileKey: string) => { - const lastTapped = lastTappedRef.current[tileKey]; - - if (!lastTapped || Date.now() - lastTapped > 500) { - lastTappedRef.current[tileKey] = Date.now(); - return; - } - - lastTappedRef.current[tileKey] = 0; - - const tile = tiles.find((tile) => tile.key === tileKey); - if (!tile || layout !== "grid") return; - const item = tile.item; - - setTileState(({ tiles, ...state }) => { - let focusedTileCount = 0; - const newTiles = tiles.map((tile) => { - const newTile = { ...tile }; // clone before reordering - - if (tile.item === item) { - newTile.focused = !tile.focused; - } - if (newTile.focused) { - focusedTileCount++; - } - - return newTile; - }); - - reorderTiles(newTiles, layout); - - return { - ...state, - tiles: newTiles, - tilePositions: getTilePositions( - newTiles.length, - focusedTileCount, - gridBounds.width, - gridBounds.height, - pipXRatio, - pipYRatio, - layout, - ), - }; - }); - }, - [tiles, layout, gridBounds.width, gridBounds.height, pipXRatio, pipYRatio], - ); - - // Callback for useDrag. We could call useDrag here, but the default - // pattern of spreading {...bind()} across the children to bind the gesture - // ends up breaking memoization and ruining this component's performance. - // Instead, we pass this callback to each tile via a ref, to let them bind the - // gesture using the much more sensible ref-based method. - const onTileDrag = ( - tileId: string, - { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - active, - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - xy, - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - movement, - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - tap, - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - last, - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - event, - }: Parameters>[0], - ): void => { - event.preventDefault(); - - if (tap) { - onTap(tileId); - return; - } - - if (layout !== "grid") return; - - const dragTileIndex = tiles.findIndex((tile) => tile.key === tileId); - const dragTile = tiles[dragTileIndex]; - const dragTilePosition = tilePositions[dragTile.order]; - - const cursorPosition = [xy[0] - gridBounds.left, xy[1] - gridBounds.top]; - - let newTiles = tiles; - - if (tiles.length === 2 && !tiles.some((t) => t.focused)) { - // We're in 1:1 mode, so only the local tile should be draggable - if (!dragTile.item.local) return; - - // Position should only update on the very last event, to avoid - // compounding the offset on every drag event - if (last) { - const remotePosition = tilePositions[1]; - - const pipGap = getPipGap( - gridBounds.width / gridBounds.height, - gridBounds.width, - ); - const pipMinX = remotePosition.x + pipGap; - const pipMinY = remotePosition.y + pipGap; - const pipMaxX = - remotePosition.x + - remotePosition.width - - dragTilePosition.width - - pipGap; - const pipMaxY = - remotePosition.y + - remotePosition.height - - dragTilePosition.height - - pipGap; - - const newPipXRatio = - (dragTilePosition.x + movement[0] - pipMinX) / (pipMaxX - pipMinX); - const newPipYRatio = - (dragTilePosition.y + movement[1] - pipMinY) / (pipMaxY - pipMinY); - - setPipXRatio(Math.max(0, Math.min(1, newPipXRatio))); - setPipYRatio(Math.max(0, Math.min(1, newPipYRatio))); - } - } else { - const hoverTile = tiles.find( - (tile) => - tile.key !== tileId && - isInside(cursorPosition, tilePositions[tile.order]), - ); - - if (hoverTile) { - // Shift the tiles into their new order - newTiles = newTiles.map((tile) => { - let order = tile.order; - if (order < dragTile.order) { - if (order >= hoverTile.order) order++; - } else if (order > dragTile.order) { - if (order <= hoverTile.order) order--; - } else { - order = hoverTile.order; - } - - let focused; - if (tile === hoverTile) { - focused = dragTile.focused; - } else if (tile === dragTile) { - focused = hoverTile.focused; - } else { - focused = tile.focused; - } - - return { ...tile, order, focused }; - }); - - reorderTiles(newTiles, layout); - - setTileState((state) => ({ ...state, tiles: newTiles })); - } - } - - if (active) { - if (!draggingTileRef.current) { - draggingTileRef.current = { - key: dragTile.key, - offsetX: dragTilePosition.x, - offsetY: dragTilePosition.y, - x: movement[0], - y: movement[1], - }; - } else { - draggingTileRef.current.x = movement[0]; - draggingTileRef.current.y = movement[1]; - } - } else { - draggingTileRef.current = null; - } - - api.start(animate(newTiles)); - }; - - const onTileDragRef = useRef(onTileDrag); - onTileDragRef.current = onTileDrag; - - const onGridGesture = useCallback( - ( - e: - | Omit, "event"> - | Omit, "event">, - isWheel: boolean, - ) => { - if (layout !== "spotlight") { - return; - } - - const isMobile = isMobileBreakpoint(gridBounds.width, gridBounds.height); - - let movement = e.delta[isMobile ? 0 : 1]; - - if (isWheel) { - movement = -movement; - } - - let min = 0; - - if (tilePositions.length > 1) { - const lastTile = tilePositions[tilePositions.length - 1]; - min = isMobile - ? gridBounds.width - lastTile.x - lastTile.width - GAP - : gridBounds.height - lastTile.y - lastTile.height - GAP; - } - - setScrollPosition((scrollPosition) => - Math.min(Math.max(movement + scrollPosition, min), 0), - ); - }, - [layout, gridBounds, tilePositions], - ); - - const bindGrid = useGesture( - { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - onWheel: (e) => onGridGesture(e, true), - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - onDrag: (e) => onGridGesture(e, false), - }, - {}, - ); - - return ( -
- {springs.map((spring, i) => { - const tile = tiles[i]; - const tilePosition = tilePositions[tile.order]; - - return ( - - {children as (props: ChildrenProperties) => ReactNode} - - ); - })} -
- ); -} - -VideoGrid.defaultProps = { - layout: "grid", -}; diff --git a/src/video-grid/VideoTile.tsx b/src/video-grid/VideoTile.tsx deleted file mode 100644 index d4a7442ec..000000000 --- a/src/video-grid/VideoTile.tsx +++ /dev/null @@ -1,494 +0,0 @@ -/* -Copyright 2022-2023 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { - ComponentProps, - ForwardedRef, - ReactNode, - forwardRef, - useCallback, - useEffect, - useState, -} from "react"; -import { animated } from "@react-spring/web"; -import classNames from "classnames"; -import { useTranslation } from "react-i18next"; -import { - TrackReferenceOrPlaceholder, - VideoTrack, -} from "@livekit/components-react"; -import { - RoomMember, - RoomMemberEvent, -} from "matrix-js-sdk/src/models/room-member"; -import MicOnSolidIcon from "@vector-im/compound-design-tokens/icons/mic-on-solid.svg?react"; -import MicOffSolidIcon from "@vector-im/compound-design-tokens/icons/mic-off-solid.svg?react"; -import ErrorIcon from "@vector-im/compound-design-tokens/icons/error.svg?react"; -import MicOffIcon from "@vector-im/compound-design-tokens/icons/mic-off.svg?react"; -import OverflowHorizontalIcon from "@vector-im/compound-design-tokens/icons/overflow-horizontal.svg?react"; -import VolumeOnIcon from "@vector-im/compound-design-tokens/icons/volume-on.svg?react"; -import VolumeOffIcon from "@vector-im/compound-design-tokens/icons/volume-off.svg?react"; -import UserProfileIcon from "@vector-im/compound-design-tokens/icons/user-profile.svg?react"; -import ExpandIcon from "@vector-im/compound-design-tokens/icons/expand.svg?react"; -import CollapseIcon from "@vector-im/compound-design-tokens/icons/collapse.svg?react"; -import { - Text, - Tooltip, - ContextMenu, - MenuItem, - ToggleMenuItem, - Menu, -} from "@vector-im/compound-web"; -import { useStateObservable } from "@react-rxjs/core"; - -import { Avatar } from "../Avatar"; -import styles from "./VideoTile.module.css"; -import { useReactiveState } from "../useReactiveState"; -import { - ScreenShareViewModel, - MediaViewModel, - UserMediaViewModel, -} from "../state/MediaViewModel"; -import { subscribe } from "../state/subscribe"; -import { useMergedRefs } from "../useMergedRefs"; -import { Slider } from "../Slider"; - -interface TileProps { - tileRef?: ForwardedRef; - className?: string; - style?: ComponentProps["style"]; - targetWidth: number; - targetHeight: number; - video: TrackReferenceOrPlaceholder; - member: RoomMember | undefined; - videoEnabled: boolean; - maximised: boolean; - unencryptedWarning: boolean; - nameTagLeadingIcon?: ReactNode; - nameTag: string; - displayName: string; - primaryButton: ReactNode; - secondaryButton?: ReactNode; - [k: string]: unknown; -} - -const Tile = forwardRef( - ( - { - tileRef = null, - className, - style, - targetWidth, - targetHeight, - video, - member, - videoEnabled, - maximised, - unencryptedWarning, - nameTagLeadingIcon, - nameTag, - displayName, - primaryButton, - secondaryButton, - ...props - }, - ref, - ) => { - const { t } = useTranslation(); - const mergedRef = useMergedRefs(tileRef, ref); - - return ( - -
- - {video.publication !== undefined && ( - - )} -
-
-
- {nameTagLeadingIcon} - - {nameTag} - - {unencryptedWarning && ( - - - - )} -
- {primaryButton} - {secondaryButton} -
- - ); - }, -); - -Tile.displayName = "Tile"; - -interface UserMediaTileProps { - vm: UserMediaViewModel; - className?: string; - style?: ComponentProps["style"]; - targetWidth: number; - targetHeight: number; - nameTag: string; - displayName: string; - maximised: boolean; - onOpenProfile: () => void; - showSpeakingIndicator: boolean; -} - -const UserMediaTile = subscribe( - ( - { - vm, - className, - style, - targetWidth, - targetHeight, - nameTag, - displayName, - maximised, - onOpenProfile, - showSpeakingIndicator, - }, - ref, - ) => { - const { t } = useTranslation(); - const video = useStateObservable(vm.video); - const audioEnabled = useStateObservable(vm.audioEnabled); - const videoEnabled = useStateObservable(vm.videoEnabled); - const unencryptedWarning = useStateObservable(vm.unencryptedWarning); - const mirror = useStateObservable(vm.mirror); - const speaking = useStateObservable(vm.speaking); - const locallyMuted = useStateObservable(vm.locallyMuted); - const cropVideo = useStateObservable(vm.cropVideo); - const localVolume = useStateObservable(vm.localVolume); - const onChangeMute = useCallback(() => vm.toggleLocallyMuted(), [vm]); - const onChangeFitContain = useCallback(() => vm.toggleFitContain(), [vm]); - const onSelectMute = useCallback((e: Event) => e.preventDefault(), []); - const onSelectFitContain = useCallback( - (e: Event) => e.preventDefault(), - [], - ); - - const onChangeLocalVolume = useCallback( - (v: number) => vm.setLocalVolume(v), - [vm], - ); - - const MicIcon = audioEnabled ? MicOnSolidIcon : MicOffSolidIcon; - const VolumeIcon = locallyMuted ? VolumeOffIcon : VolumeOnIcon; - - const [menuOpen, setMenuOpen] = useState(false); - const menu = vm.local ? ( - <> - - - - ) : ( - <> - - - {/* TODO: Figure out how to make this slider keyboard accessible */} - - - - - ); - - const tile = ( - - } - nameTag={nameTag} - displayName={displayName} - primaryButton={ - - - - } - side="left" - align="start" - > - {menu} - - } - /> - ); - - return ( - - {menu} - - ); - }, -); - -UserMediaTile.displayName = "UserMediaTile"; - -interface ScreenShareTileProps { - vm: ScreenShareViewModel; - className?: string; - style?: ComponentProps["style"]; - targetWidth: number; - targetHeight: number; - nameTag: string; - displayName: string; - maximised: boolean; - fullscreen: boolean; - onToggleFullscreen: (itemId: string) => void; -} - -const ScreenShareTile = subscribe( - ( - { - vm, - className, - style, - targetWidth, - targetHeight, - nameTag, - displayName, - maximised, - fullscreen, - onToggleFullscreen, - }, - ref, - ) => { - const { t } = useTranslation(); - const video = useStateObservable(vm.video); - const unencryptedWarning = useStateObservable(vm.unencryptedWarning); - const onClickFullScreen = useCallback( - () => onToggleFullscreen(vm.id), - [onToggleFullscreen, vm], - ); - - const FullScreenIcon = fullscreen ? CollapseIcon : ExpandIcon; - - return ( - - - - ) - } - /> - ); - }, -); - -ScreenShareTile.displayName = "ScreenShareTile"; - -interface Props { - vm: MediaViewModel; - maximised: boolean; - fullscreen: boolean; - onToggleFullscreen: (itemId: string) => void; - onOpenProfile: () => void; - targetWidth: number; - targetHeight: number; - className?: string; - style?: ComponentProps["style"]; - showSpeakingIndicator: boolean; -} - -export const VideoTile = forwardRef( - ( - { - vm, - maximised, - fullscreen, - onToggleFullscreen, - onOpenProfile, - className, - style, - targetWidth, - targetHeight, - showSpeakingIndicator, - }, - ref, - ) => { - const { t } = useTranslation(); - - // Handle display name changes. - // TODO: Move this into the view model - const [displayName, setDisplayName] = useReactiveState( - () => vm.member?.rawDisplayName ?? "[👻]", - [vm.member], - ); - useEffect(() => { - if (vm.member) { - const updateName = (): void => { - setDisplayName(vm.member!.rawDisplayName); - }; - - vm.member!.on(RoomMemberEvent.Name, updateName); - return (): void => { - vm.member!.removeListener(RoomMemberEvent.Name, updateName); - }; - } - }, [vm.member, setDisplayName]); - const nameTag = vm.local - ? t("video_tile.sfu_participant_local") - : displayName; - - if (vm instanceof UserMediaViewModel) { - return ( - - ); - } else { - return ( - - ); - } - }, -); - -VideoTile.displayName = "VideoTile"; diff --git a/test/video-grid/BigGrid-test.ts b/test/video-grid/BigGrid-test.ts deleted file mode 100644 index 3d29db6cc..000000000 --- a/test/video-grid/BigGrid-test.ts +++ /dev/null @@ -1,493 +0,0 @@ -/* -Copyright 2023 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { TileDescriptor } from "../../src/state/CallViewModel"; -import { - addItems, - column, - cycleTileSize, - fillGaps, - forEachCellInArea, - Grid, - SparseGrid, - resize, - row, - moveTile, -} from "../../src/video-grid/BigGrid"; - -/** - * Builds a grid from a string specifying the contents of each cell as a letter. - */ -function mkGrid(spec: string): Grid { - const secondNewline = spec.indexOf("\n", 1); - const columns = secondNewline === -1 ? spec.length : secondNewline - 1; - const cells = spec.match(/[a-z ]/g) ?? ([] as string[]); - const areas = new Set(cells); - areas.delete(" "); // Space represents an empty cell, not an area - const grid: Grid = { columns, cells: new Array(cells.length) }; - - for (const area of areas) { - const start = cells.indexOf(area); - const end = cells.lastIndexOf(area); - const rows = row(end, grid) - row(start, grid) + 1; - const columns = column(end, grid) - column(start, grid) + 1; - - forEachCellInArea(start, end, grid, (_c, i) => { - grid.cells[i] = { - item: { id: area } as unknown as TileDescriptor, - origin: i === start, - rows, - columns, - }; - }); - } - - return grid; -} - -/** - * Turns a grid into a string showing the contents of each cell as a letter. - */ -function showGrid(g: Grid): string { - let result = "\n"; - for (let i = 0; i < g.cells.length; i++) { - if (i > 0 && i % g.columns == 0) result += "\n"; - result += g.cells[i]?.item.id ?? " "; - } - return result; -} - -function testFillGaps(title: string, input: string, output: string): void { - test(`fillGaps ${title}`, () => { - expect(showGrid(fillGaps(mkGrid(input)))).toBe(output); - }); -} - -testFillGaps( - "does nothing on an empty grid", - ` -`, - ` -`, -); - -testFillGaps( - "does nothing if there are no gaps", - ` -ab -cd -ef`, - ` -ab -cd -ef`, -); - -testFillGaps( - "fills a gap", - ` -a b -cde -f`, - ` -cab -fde`, -); - -testFillGaps( - "fills multiple gaps", - ` -a bc -defgh - ijkl -mno`, - ` -aebch -difgl -mjnok`, -); - -testFillGaps( - "fills a big gap with 1×1 tiles", - ` -abcd -e f -g h -ijkl`, - ` -abcd -ehkf -glji`, -); - -testFillGaps( - "fills a big gap with a large tile", - ` - -aa -bc`, - ` -aa -cb`, -); - -testFillGaps( - "prefers moving around large tiles", - ` -a bc -ddde -dddf -ghij -k`, - ` -abce -dddf -dddj -kghi`, -); - -testFillGaps( - "moves through large tiles if necessary", - ` -a bc -dddd -efgh -i`, - ` -afbc -dddd -iegh`, -); - -testFillGaps( - "keeps a large tile from hanging off the bottom", - ` -abcd -efgh - -ii -ii`, - ` -abcd -iigh -iief`, -); - -testFillGaps( - "collapses large tiles trapped at the bottom", - ` -abcd -e fg -hh -hh - ii - ii`, - ` -abcd -hhfg -hhie`, -); - -testFillGaps( - "gives up on pushing large tiles upwards when not possible", - ` -aa -aa -bccd -eccf -ghij`, - ` -aadf -aaji -bcch -eccg`, -); - -function testCycleTileSize( - title: string, - tileId: string, - input: string, - output: string, -): void { - test(`cycleTileSize ${title}`, () => { - const grid = mkGrid(input); - const tile = grid.cells.find((c) => c?.item.id === tileId)!.item; - expect(showGrid(cycleTileSize(grid, tile))).toBe(output); - }); -} - -testCycleTileSize( - "expands a tile to 2×2 in a 3 column layout", - "c", - ` -abc -def -ghi`, - ` -acc -dcc -gbe -ifh`, -); - -testCycleTileSize( - "expands a tile to 3×3 in a 4 column layout", - "g", - ` -abcd -efgh`, - ` -acdh -bggg -fggg -e`, -); - -testCycleTileSize( - "restores a tile to 1×1", - "b", - ` -abbc -dbbe -fghi -jk`, - ` -abhc -djge -fik`, -); - -testCycleTileSize( - "expands a tile even in a crowded grid", - "c", - ` -abb -cbb -dde -ddf -ghi -klm`, - ` -abb -gbb -dde -ddf -ccm -cch -lik`, -); - -testCycleTileSize( - "does nothing if the tile has no room to expand", - "c", - ` -abb -cbb -dde -ddf`, - ` -abb -cbb -dde -ddf`, -); - -test("cycleTileSize is its own inverse", () => { - const input = ` -abc -def -ghi -jk`; - - const grid = mkGrid(input); - let gridAfter = grid; - - const toggle = (tileId: string): void => { - const tile = grid.cells.find((c) => c?.item.id === tileId)!.item; - gridAfter = cycleTileSize(gridAfter, tile); - }; - - // Toggle a series of tiles - toggle("j"); - toggle("h"); - toggle("a"); - // Now do the same thing in reverse - toggle("a"); - toggle("h"); - toggle("j"); - - // The grid should be back to its original state - expect(showGrid(gridAfter)).toBe(input); -}); - -function testAddItems( - title: string, - items: TileDescriptor[], - input: string, - output: string, -): void { - test(`addItems ${title}`, () => { - expect(showGrid(addItems(items, mkGrid(input) as SparseGrid) as Grid)).toBe( - output, - ); - }); -} - -testAddItems( - "appends 1×1 tiles", - ["e", "f"].map((i) => ({ id: i }) as unknown as TileDescriptor), - ` -aab -aac -d`, - ` -aab -aac -def`, -); - -testAddItems( - "places one tile near another on request", - [{ id: "g", placeNear: "b" } as unknown as TileDescriptor], - ` -abc -def`, - ` -abc - g -def`, -); - -testAddItems( - "places items with a large base size", - [{ id: "g", largeBaseSize: true } as unknown as TileDescriptor], - ` -abc -def`, - ` -abc -ggf -gge -d`, -); - -function testMoveTile( - title: string, - from: number, - to: number, - input: string, - output: string, -): void { - test(`moveTile ${title}`, () => { - expect(showGrid(moveTile(mkGrid(input), from, to))).toBe(output); - }); -} - -testMoveTile( - "refuses to move a tile too far to the left", - 1, - -1, - ` -abc`, - ` -abc`, -); - -testMoveTile( - "refuses to move a tile too far to the right", - 1, - 3, - ` -abc`, - ` -abc`, -); - -testMoveTile( - "moves a large tile to an unoccupied space", - 3, - 1, - ` -a b -ccd -cce`, - ` -acc -bcc -d e`, -); - -testMoveTile( - "refuses to move a large tile to an occupied space", - 3, - 1, - ` -abb -ccd -cce`, - ` -abb -ccd -cce`, -); - -function testResize( - title: string, - columns: number, - input: string, - output: string, -): void { - test(`resize ${title}`, () => { - expect(showGrid(resize(mkGrid(input), columns))).toBe(output); - }); -} - -testResize( - "contracts the grid", - 2, - ` -abbb -cbbb -ddde -dddf -gh`, - ` -af -bb -bb -dd -dd -ch -eg`, -); - -testResize( - "expands the grid", - 4, - ` -af -bb -bb -ch -dd -dd -eg`, - ` -afcd -bbbg -bbbe -h`, -); diff --git a/test/video-grid/VideoGrid-test.ts b/test/video-grid/VideoGrid-test.ts deleted file mode 100644 index cf15c0221..000000000 --- a/test/video-grid/VideoGrid-test.ts +++ /dev/null @@ -1,69 +0,0 @@ -/* -Copyright 2023 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { TileDescriptor } from "../../src/state/CallViewModel"; -import { Tile, reorderTiles } from "../../src/video-grid/VideoGrid"; - -const alice: Tile = { - key: "alice", - order: 0, - item: { local: false } as unknown as TileDescriptor, - remove: false, - focused: false, - isPresenter: false, - isSpeaker: false, - hasVideo: true, -}; -const bob: Tile = { - key: "bob", - order: 1, - item: { local: false } as unknown as TileDescriptor, - remove: false, - focused: false, - isPresenter: false, - isSpeaker: false, - hasVideo: false, -}; - -test("reorderTiles does not promote a non-speaker", () => { - const tiles = [{ ...alice }, { ...bob }]; - reorderTiles(tiles, "spotlight", 1); - expect(tiles).toEqual([ - expect.objectContaining({ key: "alice", order: 0 }), - expect.objectContaining({ key: "bob", order: 1 }), - ]); -}); - -test("reorderTiles promotes a speaker into the visible area", () => { - const tiles = [{ ...alice }, { ...bob, isSpeaker: true }]; - reorderTiles(tiles, "spotlight", 1); - expect(tiles).toEqual([ - expect.objectContaining({ key: "alice", order: 1 }), - expect.objectContaining({ key: "bob", order: 0 }), - ]); -}); - -test("reorderTiles keeps a promoted speaker in the visible area", () => { - const tiles = [ - { ...alice, order: 1 }, - { ...bob, isSpeaker: true, order: 0 }, - ]; - reorderTiles(tiles, "spotlight", 1); - expect(tiles).toEqual([ - expect.objectContaining({ key: "alice", order: 1 }), - expect.objectContaining({ key: "bob", order: 0 }), - ]); -}); diff --git a/yarn.lock b/yarn.lock index a76332cc1..1cf6e488d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2880,14 +2880,6 @@ "@react-aria/utils" "^3.13.1" clsx "^1.1.1" -"@react-rxjs/core@^0.10.7": - version "0.10.7" - resolved "https://registry.yarnpkg.com/@react-rxjs/core/-/core-0.10.7.tgz#09951f43a6c80892526ac13d51859098b0e74993" - integrity sha512-dornp8pUs9OcdqFKKRh9+I2FVe21gWufNun6RYU1ddts7kUy9i4Thvl0iqcPFbGY61cJQMAJF7dxixWMSD/A/A== - dependencies: - "@rx-state/core" "0.1.4" - use-sync-external-store "^1.0.0" - "@react-spring/animated@~9.7.3": version "9.7.3" resolved "https://registry.yarnpkg.com/@react-spring/animated/-/animated-9.7.3.tgz#4211b1a6d48da0ff474a125e93c0f460ff816e0f" @@ -3227,11 +3219,6 @@ resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.18.1.tgz#0cb240c147c0dfd0e3eaff4cc060a772d39e155c" integrity sha512-yjk2MAkQmoaPYCSu35RLJ62+dz358nE83VfTePJRp8CG7aMg25mEJYpXFiD+NcevhX8LxD5OP5tktPXnXN7GDw== -"@rx-state/core@0.1.4": - version "0.1.4" - resolved "https://registry.yarnpkg.com/@rx-state/core/-/core-0.1.4.tgz#586dde80be9dbdac31844006a0dcaa2bc7f35a5c" - integrity sha512-Z+3hjU2xh1HisLxt+W5hlYX/eGSDaXXP+ns82gq/PLZpkXLu0uwcNUh9RLY3Clq4zT+hSsA3vcpIGt6+UAb8rQ== - "@sentry-internal/browser-utils@8.18.0": version "8.18.0" resolved "https://registry.yarnpkg.com/@sentry-internal/browser-utils/-/browser-utils-8.18.0.tgz#b3d06a77bf80e8d00e4cd8fc11a242cb4e9fa534" @@ -6988,6 +6975,11 @@ object.values@^1.1.7: define-properties "^1.2.0" es-abstract "^1.22.1" +observable-hooks@^4.2.3: + version "4.2.3" + resolved "https://registry.yarnpkg.com/observable-hooks/-/observable-hooks-4.2.3.tgz#69e3353caafd7887ad9030bd440b053304e8d2d1" + integrity sha512-d6fYTIU+9sg1V+CT0GhgoE/ntjIqcy9DGaYGE6ELGVP4ojaWIEsaLvL/05hLOM+AL7aySN4DCTLvj6dDF9T8XA== + oidc-client-ts@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/oidc-client-ts/-/oidc-client-ts-3.0.1.tgz#be264fb87c89f74f73863646431c32cd06f5ceb7" @@ -8770,11 +8762,6 @@ use-sidecar@^1.1.2: detect-node-es "^1.1.0" tslib "^2.0.0" -use-sync-external-store@^1.0.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/use-sync-external-store/-/use-sync-external-store-1.2.0.tgz#7dbefd6ef3fe4e767a0cf5d7287aacfb5846928a" - integrity sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA== - usehooks-ts@3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/usehooks-ts/-/usehooks-ts-3.1.0.tgz#156119f36efc85f1b1952616c02580f140950eca"