From 25bcbfa072ec04bd1ae6e052b801decb8cdf4b62 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 17 May 2024 16:38:00 -0400 Subject: [PATCH] Implement the new spotlight layout --- src/grid/CallLayout.ts | 55 ++++++++++++++++++ src/grid/GridLayout.tsx | 25 +++------ src/grid/SpotlightLayout.module.css | 63 +++++++++++++++++++++ src/grid/SpotlightLayout.tsx | 87 +++++++++++++++++++++++++++++ src/room/InCallView.tsx | 62 ++++++++++++-------- src/state/CallViewModel.ts | 32 +++++++++-- src/tile/GridTile.tsx | 11 ++-- 7 files changed, 286 insertions(+), 49 deletions(-) create mode 100644 src/grid/CallLayout.ts create mode 100644 src/grid/SpotlightLayout.module.css create mode 100644 src/grid/SpotlightLayout.tsx diff --git a/src/grid/CallLayout.ts b/src/grid/CallLayout.ts new file mode 100644 index 000000000..287f116d4 --- /dev/null +++ b/src/grid/CallLayout.ts @@ -0,0 +1,55 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { BehaviorSubject, Observable } from "rxjs"; + +import { MediaViewModel } from "../state/MediaViewModel"; +import { LayoutSystem } from "./Grid"; +import { Alignment } from "../room/InCallView"; + +export interface Bounds { + width: number; + height: number; +} + +export interface CallLayoutInputs { + /** + * The minimum bounds of the layout area. + */ + minBounds: Observable; + /** + * The alignment of the floating tile, if any. + */ + floatingAlignment: BehaviorSubject; +} + +export interface CallLayoutOutputs { + /** + * The visually fixed (non-scrolling) layer of the layout. + */ + fixed: LayoutSystem; + /** + * The layer of the layout that can overflow and be scrolled. + */ + scrolling: LayoutSystem; +} + +/** + * A layout system for media tiles. + */ +export type CallLayout = ( + inputs: CallLayoutInputs, +) => CallLayoutOutputs; diff --git a/src/grid/GridLayout.tsx b/src/grid/GridLayout.tsx index 6b673e644..75f1e7268 100644 --- a/src/grid/GridLayout.tsx +++ b/src/grid/GridLayout.tsx @@ -15,21 +15,15 @@ limitations under the License. */ import { CSSProperties, forwardRef, useMemo } from "react"; -import { BehaviorSubject, Observable, distinctUntilChanged } from "rxjs"; +import { distinctUntilChanged } from "rxjs"; import { useObservableEagerState } from "observable-hooks"; import { GridLayout as GridLayoutModel } from "../state/CallViewModel"; -import { MediaViewModel } from "../state/MediaViewModel"; -import { LayoutSystem, Slot } from "./Grid"; +import { Slot } from "./Grid"; import styles from "./GridLayout.module.css"; import { useReactiveState } from "../useReactiveState"; -import { Alignment } from "../room/InCallView"; import { useInitial } from "../useInitial"; - -export interface Bounds { - width: number; - height: number; -} +import { CallLayout } from "./CallLayout"; interface GridCSSProperties extends CSSProperties { "--gap": string; @@ -37,19 +31,14 @@ interface GridCSSProperties extends CSSProperties { "--height": string; } -interface GridLayoutSystems { - scrolling: LayoutSystem; - fixed: LayoutSystem; -} - const slotMinHeight = 130; const slotMaxAspectRatio = 17 / 9; const slotMinAspectRatio = 4 / 3; -export const gridLayoutSystems = ( - minBounds: Observable, - floatingAlignment: BehaviorSubject, -): GridLayoutSystems => ({ +export const makeGridLayout: CallLayout = ({ + minBounds, + floatingAlignment, +}) => ({ // The "fixed" (non-scrolling) part of the layout is where the spotlight tile // lives fixed: { diff --git a/src/grid/SpotlightLayout.module.css b/src/grid/SpotlightLayout.module.css new file mode 100644 index 000000000..aec6be86f --- /dev/null +++ b/src/grid/SpotlightLayout.module.css @@ -0,0 +1,63 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.fixed, +.scrolling { + display: grid; + --grid-slot-width: 180px; + --grid-gap: 20px; + grid-template-columns: 1fr calc( + var(--grid-columns) * var(--grid-slot-width) + (var(--grid-columns) - 1) * + var(--grid-gap) + ); + grid-template-rows: minmax(1fr, auto); + grid-template-areas: "spotlight grid"; + gap: 30px; +} + +.scrolling { + block-size: 100%; +} + +.spotlight { + grid-area: spotlight; + container: spotlight / size; + display: grid; + place-items: center; +} + +/* CSS makes us put a condition here, even though all we want to do is +unconditionally select the container so we can use cq units */ +@container spotlight (width > 0) { + .spotlight > .slot { + inline-size: min(100cqi, 100cqb * (17 / 9)); + block-size: min(100cqb, 100cqi / (4 / 3)); + } +} + +.grid { + grid-area: grid; + display: flex; + flex-wrap: wrap; + gap: var(--grid-gap); + justify-content: center; + align-content: center; +} + +.grid > .slot { + inline-size: var(--grid-slot-width); + block-size: 135px; +} diff --git a/src/grid/SpotlightLayout.tsx b/src/grid/SpotlightLayout.tsx new file mode 100644 index 000000000..8958dcd2a --- /dev/null +++ b/src/grid/SpotlightLayout.tsx @@ -0,0 +1,87 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { CSSProperties, forwardRef } from "react"; +import { useObservableEagerState } from "observable-hooks"; + +import { CallLayout } from "./CallLayout"; +import { SpotlightLayout as SpotlightLayoutModel } from "../state/CallViewModel"; +import { useReactiveState } from "../useReactiveState"; +import styles from "./SpotlightLayout.module.css"; +import { Slot } from "./Grid"; + +interface GridCSSProperties extends CSSProperties { + "--grid-columns": number; +} + +const getGridColumns = (gridLength: number): number => + gridLength > 20 ? 2 : 1; + +export const makeSpotlightLayout: CallLayout = ({ + minBounds, +}) => ({ + fixed: { + tiles: (model) => new Map([["spotlight", model.spotlight]]), + Layout: forwardRef(function SpotlightLayoutFixed({ model }, ref) { + const { width, height } = useObservableEagerState(minBounds); + const gridColumns = getGridColumns(model.grid.length); + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [model.grid.length, width, height], + ); + + return ( +
+
+ +
+
+ ); + }), + }, + + scrolling: { + tiles: (model) => new Map(model.grid.map((tile) => [tile.id, tile])), + Layout: forwardRef(function SpotlightLayoutScrolling({ model }, ref) { + const { width, height } = useObservableEagerState(minBounds); + const gridColumns = getGridColumns(model.grid.length); + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [model.grid, width, height], + ); + + return ( +
+
+ {model.grid.map((tile) => ( + + ))} +
+
+ ); + }), + }, +}); diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 995bb6b75..1abcfd946 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -35,7 +35,7 @@ import { import useMeasure from "react-use-measure"; import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; import classNames from "classnames"; -import { BehaviorSubject } from "rxjs"; +import { BehaviorSubject, map } from "rxjs"; import { useObservableEagerState } from "observable-hooks"; import { useTranslation } from "react-i18next"; @@ -73,17 +73,20 @@ import { ECConnectionState } from "../livekit/useECConnectionState"; import { useOpenIDSFU } from "../livekit/openIDSFU"; import { GridMode, + Layout, TileDescriptor, useCallViewModel, } from "../state/CallViewModel"; import { Grid, TileProps } from "../grid/Grid"; import { MediaViewModel } from "../state/MediaViewModel"; -import { gridLayoutSystems } from "../grid/GridLayout"; import { useObservable } from "../state/useObservable"; import { useInitial } from "../useInitial"; import { SpotlightTile } from "../tile/SpotlightTile"; import { EncryptionSystem } from "../e2ee/sharedKeyManagement"; import { E2eeType } from "../e2ee/e2eeType"; +import { makeGridLayout } from "../grid/GridLayout"; +import { makeSpotlightLayout } from "../grid/SpotlightLayout"; +import { CallLayout } from "../grid/CallLayout"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); @@ -302,6 +305,7 @@ export const InCallView: FC = ({ const [headerRef, headerBounds] = useMeasure(); const [footerRef, footerBounds] = useMeasure(); + const gridBounds = useMemo( () => ({ width: footerBounds.width, @@ -315,11 +319,29 @@ export const InCallView: FC = ({ ], ); const gridBoundsObservable = useObservable(gridBounds); + const floatingAlignment = useInitial( () => new BehaviorSubject(defaultAlignment), ); - const { fixed, scrolling } = useInitial(() => - gridLayoutSystems(gridBoundsObservable, floatingAlignment), + + const layoutSystem = useObservableEagerState( + useInitial(() => + vm.layout.pipe( + map((l) => { + let makeLayout: CallLayout; + if (l.type === "grid" && l.grid.length !== 2) + makeLayout = makeGridLayout as CallLayout; + else if (l.type === "spotlight") + makeLayout = makeSpotlightLayout as CallLayout; + else return null; // Not yet implemented + + return makeLayout({ + minBounds: gridBoundsObservable, + floatingAlignment, + }); + }), + ), + ), ); const setGridMode = useCallback( @@ -423,39 +445,35 @@ export const InCallView: FC = ({ ); } - // The only new layout we've implemented so far is grid layout for non-1:1 - // calls. All other layouts use the legacy grid system for now. - if ( - legacyLayout === "grid" && - layout.type === "grid" && - !(layout.grid.length === 2 && layout.spotlight === undefined) - ) { + if (layoutSystem === null) { + // This new layout doesn't yet have an implemented layout system, so fall + // back to the legacy grid system + return ( + + ); + } else { return ( <> ); - } else { - return ( - - ); } }; diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 6b4219113..0d2f34a78 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -28,12 +28,13 @@ import { import { Room as MatrixRoom, RoomMember } from "matrix-js-sdk/src/matrix"; import { useEffect, useRef } from "react"; import { - BehaviorSubject, EMPTY, Observable, + Subject, audit, combineLatest, concat, + concatMap, distinctUntilChanged, filter, map, @@ -48,6 +49,7 @@ import { switchMap, throttleTime, timer, + withLatestFrom, zip, } from "rxjs"; import { logger } from "matrix-js-sdk/src/logger"; @@ -372,6 +374,13 @@ export class CallViewModel extends ViewModel { private readonly screenShares: Observable = this.mediaItems.pipe( map((ms) => ms.filter((m): m is ScreenShare => m instanceof ScreenShare)), + shareReplay(1), + ); + + private readonly hasScreenShares: Observable = + this.screenShares.pipe( + map((ms) => ms.length > 0), + distinctUntilChanged(), ); private readonly spotlightSpeaker: Observable = @@ -456,18 +465,31 @@ export class CallViewModel extends ViewModel { // orientation private readonly windowMode = of("normal"); - private readonly _gridMode = new BehaviorSubject("grid"); + private readonly gridModeUserSelection = new Subject(); /** * The layout mode of the media tile grid. */ - public readonly gridMode: Observable = this._gridMode; + public readonly gridMode: Observable = merge( + // Always honor a manual user selection + this.gridModeUserSelection, + // If the user hasn't selected spotlight and somebody starts screen sharing, + // automatically switch to spotlight mode and reset when screen sharing ends + this.hasScreenShares.pipe( + withLatestFrom(this.gridModeUserSelection.pipe(startWith(null))), + concatMap(([hasScreenShares, userSelection]) => + userSelection === "spotlight" + ? EMPTY + : of(hasScreenShares ? "spotlight" : "grid"), + ), + ), + ).pipe(distinctUntilChanged(), shareReplay(1)); public setGridMode(value: GridMode): void { - this._gridMode.next(value); + this.gridModeUserSelection.next(value); } public readonly layout: Observable = combineLatest( - [this._gridMode, this.windowMode], + [this.gridMode, this.windowMode], (gridMode, windowMode) => { switch (windowMode) { case "full screen": diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 3a12e6e91..9fbb11965 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -73,6 +73,7 @@ interface MediaTileProps vm: MediaViewModel; videoEnabled: boolean; videoFit: "contain" | "cover"; + mirror: boolean; nameTagLeadingIcon?: ReactNode; primaryButton: ReactNode; secondaryButton?: ReactNode; @@ -89,7 +90,6 @@ const MediaTile = forwardRef( className={classNames(className, styles.tile)} data-maximised={maximised} video={video} - mirror={false} member={vm.member} unencryptedWarning={unencryptedWarning} {...props} @@ -102,6 +102,7 @@ MediaTile.displayName = "MediaTile"; interface UserMediaTileProps extends TileProps { vm: UserMediaViewModel; + mirror: boolean; showSpeakingIndicators: boolean; menuStart?: ReactNode; menuEnd?: ReactNode; @@ -205,7 +206,7 @@ interface LocalUserMediaTileProps extends TileProps { } const LocalUserMediaTile = forwardRef( - ({ vm, onOpenProfile, className, ...props }, ref) => { + ({ vm, onOpenProfile, ...props }, ref) => { const { t } = useTranslation(); const mirror = useObservableEagerState(vm.mirror); const alwaysShow = useObservableEagerState(vm.alwaysShow); @@ -223,6 +224,7 @@ const LocalUserMediaTile = forwardRef( ( onSelect={onOpenProfile} /> } - className={classNames(className, { [styles.mirror]: mirror })} {...props} /> ); @@ -277,6 +278,7 @@ const RemoteUserMediaTile = forwardRef< ( ( ) } - videoEnabled {...props} /> );