From 34c45cb5e244052ac8857e346be38ac697c950dd Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 21 May 2024 17:05:37 -0400 Subject: [PATCH 1/8] Get the right grid offset even when offsetParent is a layout element --- src/grid/Grid.tsx | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/src/grid/Grid.tsx b/src/grid/Grid.tsx index 9aa7f95f8..b6cf8fcdb 100644 --- a/src/grid/Grid.tsx +++ b/src/grid/Grid.tsx @@ -91,6 +91,28 @@ export const Slot: FC = ({ tile, style, className, ...props }) => ( /> ); +interface Offset { + x: number; + y: number; +} + +/** + * Gets the offset of one element relative to an ancestor. + */ +function offset(element: HTMLElement, relativeTo: Element): Offset { + if ( + !(element.offsetParent instanceof HTMLElement) || + element.offsetParent === relativeTo + ) { + return { x: element.offsetLeft, y: element.offsetTop }; + } else { + const o = offset(element.offsetParent, relativeTo); + o.x += element.offsetLeft; + o.y += element.offsetTop; + return o; + } +} + export interface LayoutProps { ref: LegacyRef; model: Model; @@ -228,24 +250,23 @@ export function Grid< const slotRects = useMemo(() => { const rects = new Map(); - if (layoutRoot !== null) { + if (gridRoot !== null && layoutRoot !== null) { const slots = layoutRoot.getElementsByClassName( styles.slot, ) as HTMLCollectionOf; for (const slot of slots) rects.set(slot.getAttribute("data-tile")!, { - x: slot.offsetLeft, - y: slot.offsetTop, + ...offset(slot, gridRoot), width: slot.offsetWidth, height: slot.offsetHeight, }); } return rects; - // The rects may change due to the grid being resized or rerendered, but + // The rects may change due to the grid updating to a new generation, but // eslint can't statically verify this // eslint-disable-next-line react-hooks/exhaustive-deps - }, [layoutRoot, generation]); + }, [gridRoot, layoutRoot, generation]); const tileModels = useMemo( () => getTileModels(model), From ffbbc74a9654f8703a502760abd7db9b859e3e68 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 17 May 2024 16:38:00 -0400 Subject: [PATCH 2/8] Implement the new spotlight layout --- src/grid/CallLayout.ts | 55 ++++++++++++++++++ src/grid/GridLayout.module.css | 6 +- src/grid/GridLayout.tsx | 25 +++----- src/grid/SpotlightLayout.module.css | 89 +++++++++++++++++++++++++++++ src/grid/SpotlightLayout.tsx | 89 +++++++++++++++++++++++++++++ src/room/InCallView.module.css | 4 +- src/room/InCallView.tsx | 62 +++++++++++++------- src/state/CallViewModel.ts | 47 +++++++++++---- src/tile/GridTile.tsx | 11 ++-- 9 files changed, 330 insertions(+), 58 deletions(-) create mode 100644 src/grid/CallLayout.ts create mode 100644 src/grid/SpotlightLayout.module.css create mode 100644 src/grid/SpotlightLayout.tsx diff --git a/src/grid/CallLayout.ts b/src/grid/CallLayout.ts new file mode 100644 index 000000000..287f116d4 --- /dev/null +++ b/src/grid/CallLayout.ts @@ -0,0 +1,55 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { BehaviorSubject, Observable } from "rxjs"; + +import { MediaViewModel } from "../state/MediaViewModel"; +import { LayoutSystem } from "./Grid"; +import { Alignment } from "../room/InCallView"; + +export interface Bounds { + width: number; + height: number; +} + +export interface CallLayoutInputs { + /** + * The minimum bounds of the layout area. + */ + minBounds: Observable; + /** + * The alignment of the floating tile, if any. + */ + floatingAlignment: BehaviorSubject; +} + +export interface CallLayoutOutputs { + /** + * The visually fixed (non-scrolling) layer of the layout. + */ + fixed: LayoutSystem; + /** + * The layer of the layout that can overflow and be scrolled. + */ + scrolling: LayoutSystem; +} + +/** + * A layout system for media tiles. + */ +export type CallLayout = ( + inputs: CallLayoutInputs, +) => CallLayoutOutputs; diff --git a/src/grid/GridLayout.module.css b/src/grid/GridLayout.module.css index ef234b337..084b56bd2 100644 --- a/src/grid/GridLayout.module.css +++ b/src/grid/GridLayout.module.css @@ -14,6 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */ +.fixed, +.scrolling { + margin-inline: var(--inline-content-inset); +} + .scrolling { box-sizing: border-box; block-size: 100%; @@ -22,7 +27,6 @@ limitations under the License. justify-content: center; align-content: center; gap: var(--gap); - box-sizing: border-box; } .scrolling > .slot { diff --git a/src/grid/GridLayout.tsx b/src/grid/GridLayout.tsx index 6b673e644..75f1e7268 100644 --- a/src/grid/GridLayout.tsx +++ b/src/grid/GridLayout.tsx @@ -15,21 +15,15 @@ limitations under the License. */ import { CSSProperties, forwardRef, useMemo } from "react"; -import { BehaviorSubject, Observable, distinctUntilChanged } from "rxjs"; +import { distinctUntilChanged } from "rxjs"; import { useObservableEagerState } from "observable-hooks"; import { GridLayout as GridLayoutModel } from "../state/CallViewModel"; -import { MediaViewModel } from "../state/MediaViewModel"; -import { LayoutSystem, Slot } from "./Grid"; +import { Slot } from "./Grid"; import styles from "./GridLayout.module.css"; import { useReactiveState } from "../useReactiveState"; -import { Alignment } from "../room/InCallView"; import { useInitial } from "../useInitial"; - -export interface Bounds { - width: number; - height: number; -} +import { CallLayout } from "./CallLayout"; interface GridCSSProperties extends CSSProperties { "--gap": string; @@ -37,19 +31,14 @@ interface GridCSSProperties extends CSSProperties { "--height": string; } -interface GridLayoutSystems { - scrolling: LayoutSystem; - fixed: LayoutSystem; -} - const slotMinHeight = 130; const slotMaxAspectRatio = 17 / 9; const slotMinAspectRatio = 4 / 3; -export const gridLayoutSystems = ( - minBounds: Observable, - floatingAlignment: BehaviorSubject, -): GridLayoutSystems => ({ +export const makeGridLayout: CallLayout = ({ + minBounds, + floatingAlignment, +}) => ({ // The "fixed" (non-scrolling) part of the layout is where the spotlight tile // lives fixed: { diff --git a/src/grid/SpotlightLayout.module.css b/src/grid/SpotlightLayout.module.css new file mode 100644 index 000000000..bbce45cfe --- /dev/null +++ b/src/grid/SpotlightLayout.module.css @@ -0,0 +1,89 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.fixed, +.scrolling { + margin-inline: var(--inline-content-inset); + display: grid; + --grid-slot-width: 180px; + --grid-gap: 20px; + grid-template-columns: 1fr calc( + var(--grid-columns) * var(--grid-slot-width) + (var(--grid-columns) - 1) * + var(--grid-gap) + ); + grid-template-rows: minmax(1fr, auto); + gap: 30px; +} + +.scrolling { + block-size: 100%; +} + +.spotlight { + container: spotlight / size; + display: grid; + place-items: center; +} + +/* CSS makes us put a condition here, even though all we want to do is +unconditionally select the container so we can use cq units */ +@container spotlight (width > 0) { + .spotlight > .slot { + inline-size: min(100cqi, 100cqb * (17 / 9)); + block-size: min(100cqb, 100cqi / (4 / 3)); + } +} + +.grid { + display: flex; + flex-wrap: wrap; + gap: var(--grid-gap); + justify-content: center; + align-content: center; +} + +.grid > .slot { + inline-size: var(--grid-slot-width); + block-size: 135px; +} + +@media (max-width: 600px) { + .fixed, + .scrolling { + margin-inline: 0; + display: block; + } + + .spotlight { + inline-size: 100%; + aspect-ratio: 16 / 9; + margin-block-end: var(--cpd-space-4x); + } + + .grid { + margin-inline: var(--inline-content-inset); + align-content: start; + } + + .grid > .slot { + --grid-columns: 2; + --grid-slot-width: calc( + (100% - (var(--grid-columns) - 1) * var(--grid-gap)) / var(--grid-columns) + ); + block-size: unset; + aspect-ratio: 4 / 3; + } +} diff --git a/src/grid/SpotlightLayout.tsx b/src/grid/SpotlightLayout.tsx new file mode 100644 index 000000000..38bc6e374 --- /dev/null +++ b/src/grid/SpotlightLayout.tsx @@ -0,0 +1,89 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { CSSProperties, forwardRef } from "react"; +import { useObservableEagerState } from "observable-hooks"; + +import { CallLayout } from "./CallLayout"; +import { SpotlightLayout as SpotlightLayoutModel } from "../state/CallViewModel"; +import { useReactiveState } from "../useReactiveState"; +import styles from "./SpotlightLayout.module.css"; +import { Slot } from "./Grid"; + +interface GridCSSProperties extends CSSProperties { + "--grid-columns": number; +} + +const getGridColumns = (gridLength: number): number => + gridLength > 20 ? 2 : 1; + +export const makeSpotlightLayout: CallLayout = ({ + minBounds, +}) => ({ + fixed: { + tiles: (model) => new Map([["spotlight", model.spotlight]]), + Layout: forwardRef(function SpotlightLayoutFixed({ model }, ref) { + const { width, height } = useObservableEagerState(minBounds); + const gridColumns = getGridColumns(model.grid.length); + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [model.grid.length, width, height], + ); + + return ( +
+
+ +
+
+
+ ); + }), + }, + + scrolling: { + tiles: (model) => new Map(model.grid.map((tile) => [tile.id, tile])), + Layout: forwardRef(function SpotlightLayoutScrolling({ model }, ref) { + const { width, height } = useObservableEagerState(minBounds); + const gridColumns = getGridColumns(model.grid.length); + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [model.grid, width, height], + ); + + return ( +
+
+
+ {model.grid.map((tile) => ( + + ))} +
+
+ ); + }), + }, +}); diff --git a/src/room/InCallView.module.css b/src/room/InCallView.module.css index f53ba0255..76dc9bae6 100644 --- a/src/room/InCallView.module.css +++ b/src/room/InCallView.module.css @@ -125,7 +125,7 @@ limitations under the License. .fixedGrid { position: absolute; - inline-size: calc(100% - 2 * var(--inline-content-inset)); + inline-size: 100%; align-self: center; /* Disable pointer events so the overlay doesn't block interaction with elements behind it */ @@ -139,6 +139,6 @@ limitations under the License. .scrollingGrid { position: relative; flex-grow: 1; - inline-size: calc(100% - 2 * var(--inline-content-inset)); + inline-size: 100%; align-self: center; } diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index dba01e9d9..b2aeb9d23 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -35,7 +35,7 @@ import { import useMeasure from "react-use-measure"; import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; import classNames from "classnames"; -import { BehaviorSubject } from "rxjs"; +import { BehaviorSubject, map } from "rxjs"; import { useObservableEagerState } from "observable-hooks"; import { useTranslation } from "react-i18next"; @@ -73,17 +73,20 @@ import { ECConnectionState } from "../livekit/useECConnectionState"; import { useOpenIDSFU } from "../livekit/openIDSFU"; import { GridMode, + Layout, TileDescriptor, useCallViewModel, } from "../state/CallViewModel"; import { Grid, TileProps } from "../grid/Grid"; import { MediaViewModel } from "../state/MediaViewModel"; -import { gridLayoutSystems } from "../grid/GridLayout"; import { useObservable } from "../state/useObservable"; import { useInitial } from "../useInitial"; import { SpotlightTile } from "../tile/SpotlightTile"; import { EncryptionSystem } from "../e2ee/sharedKeyManagement"; import { E2eeType } from "../e2ee/e2eeType"; +import { makeGridLayout } from "../grid/GridLayout"; +import { makeSpotlightLayout } from "../grid/SpotlightLayout"; +import { CallLayout } from "../grid/CallLayout"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); @@ -302,6 +305,7 @@ export const InCallView: FC = ({ const [headerRef, headerBounds] = useMeasure(); const [footerRef, footerBounds] = useMeasure(); + const gridBounds = useMemo( () => ({ width: footerBounds.width, @@ -315,11 +319,29 @@ export const InCallView: FC = ({ ], ); const gridBoundsObservable = useObservable(gridBounds); + const floatingAlignment = useInitial( () => new BehaviorSubject(defaultAlignment), ); - const { fixed, scrolling } = useInitial(() => - gridLayoutSystems(gridBoundsObservable, floatingAlignment), + + const layoutSystem = useObservableEagerState( + useInitial(() => + vm.layout.pipe( + map((l) => { + let makeLayout: CallLayout; + if (l.type === "grid" && l.grid.length !== 2) + makeLayout = makeGridLayout as CallLayout; + else if (l.type === "spotlight") + makeLayout = makeSpotlightLayout as CallLayout; + else return null; // Not yet implemented + + return makeLayout({ + minBounds: gridBoundsObservable, + floatingAlignment, + }); + }), + ), + ), ); const setGridMode = useCallback( @@ -423,39 +445,35 @@ export const InCallView: FC = ({ ); } - // The only new layout we've implemented so far is grid layout for non-1:1 - // calls. All other layouts use the legacy grid system for now. - if ( - legacyLayout === "grid" && - layout.type === "grid" && - !(layout.grid.length === 2 && layout.spotlight === undefined) - ) { + if (layoutSystem === null) { + // This new layout doesn't yet have an implemented layout system, so fall + // back to the legacy grid system + return ( + + ); + } else { return ( <> ); - } else { - return ( - - ); } }; diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index b0816dc2d..cc5afc463 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -28,12 +28,13 @@ import { import { Room as MatrixRoom, RoomMember } from "matrix-js-sdk/src/matrix"; import { useEffect, useRef } from "react"; import { - BehaviorSubject, EMPTY, Observable, + Subject, audit, combineLatest, concat, + concatMap, distinctUntilChanged, filter, map, @@ -48,6 +49,7 @@ import { switchMap, throttleTime, timer, + withLatestFrom, zip, } from "rxjs"; import { logger } from "matrix-js-sdk/src/logger"; @@ -371,6 +373,13 @@ export class CallViewModel extends ViewModel { private readonly screenShares: Observable = this.mediaItems.pipe( map((ms) => ms.filter((m): m is ScreenShare => m instanceof ScreenShare)), + shareReplay(1), + ); + + private readonly hasScreenShares: Observable = + this.screenShares.pipe( + map((ms) => ms.length > 0), + distinctUntilChanged(), ); private readonly spotlightSpeaker: Observable = @@ -385,11 +394,13 @@ export class CallViewModel extends ViewModel { scan<(readonly [UserMedia, boolean])[], UserMedia | null, null>( (prev, ms) => // Decide who to spotlight: - // If the previous speaker is still speaking, stick with them rather - // than switching eagerly to someone else - ms.find(([m, s]) => m === prev && s)?.[0] ?? - // Otherwise, select anyone who is speaking - ms.find(([, s]) => s)?.[0] ?? + // If the previous speaker (not the local user) is still speaking, + // stick with them rather than switching eagerly to someone else + (prev === null || prev.vm.local + ? null + : ms.find(([m, s]) => m === prev && s)?.[0]) ?? + // Otherwise, select any remote user who is speaking + ms.find(([m, s]) => !m.vm.local && s)?.[0] ?? // Otherwise, stick with the person who was last speaking prev ?? // Otherwise, spotlight the local user @@ -398,7 +409,8 @@ export class CallViewModel extends ViewModel { null, ), distinctUntilChanged(), - throttleTime(800, undefined, { leading: true, trailing: true }), + shareReplay(1), + throttleTime(1600, undefined, { leading: true, trailing: true }), ); private readonly grid: Observable = this.userMedia.pipe( @@ -453,18 +465,31 @@ export class CallViewModel extends ViewModel { // orientation private readonly windowMode = of("normal"); - private readonly _gridMode = new BehaviorSubject("grid"); + private readonly gridModeUserSelection = new Subject(); /** * The layout mode of the media tile grid. */ - public readonly gridMode: Observable = this._gridMode; + public readonly gridMode: Observable = merge( + // Always honor a manual user selection + this.gridModeUserSelection, + // If the user hasn't selected spotlight and somebody starts screen sharing, + // automatically switch to spotlight mode and reset when screen sharing ends + this.hasScreenShares.pipe( + withLatestFrom(this.gridModeUserSelection.pipe(startWith(null))), + concatMap(([hasScreenShares, userSelection]) => + userSelection === "spotlight" + ? EMPTY + : of(hasScreenShares ? "spotlight" : "grid"), + ), + ), + ).pipe(distinctUntilChanged(), shareReplay(1)); public setGridMode(value: GridMode): void { - this._gridMode.next(value); + this.gridModeUserSelection.next(value); } public readonly layout: Observable = combineLatest( - [this._gridMode, this.windowMode], + [this.gridMode, this.windowMode], (gridMode, windowMode) => { switch (windowMode) { case "full screen": diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index ba6159530..14f858315 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -71,6 +71,7 @@ interface MediaTileProps vm: MediaViewModel; videoEnabled: boolean; videoFit: "contain" | "cover"; + mirror: boolean; nameTagLeadingIcon?: ReactNode; primaryButton: ReactNode; secondaryButton?: ReactNode; @@ -87,7 +88,6 @@ const MediaTile = forwardRef( className={classNames(className, styles.tile)} data-maximised={maximised} video={video} - mirror={false} member={vm.member} unencryptedWarning={unencryptedWarning} {...props} @@ -100,6 +100,7 @@ MediaTile.displayName = "MediaTile"; interface UserMediaTileProps extends TileProps { vm: UserMediaViewModel; + mirror: boolean; showSpeakingIndicators: boolean; menuStart?: ReactNode; menuEnd?: ReactNode; @@ -202,7 +203,7 @@ interface LocalUserMediaTileProps extends TileProps { } const LocalUserMediaTile = forwardRef( - ({ vm, onOpenProfile, className, ...props }, ref) => { + ({ vm, onOpenProfile, ...props }, ref) => { const { t } = useTranslation(); const mirror = useObservableEagerState(vm.mirror); const alwaysShow = useObservableEagerState(vm.alwaysShow); @@ -220,6 +221,7 @@ const LocalUserMediaTile = forwardRef( ( onSelect={onOpenProfile} /> } - className={classNames(className, { [styles.mirror]: mirror })} {...props} /> ); @@ -270,6 +271,7 @@ const RemoteUserMediaTile = forwardRef< ( ( ) } - videoEnabled {...props} /> ); From 54c22f4ab28c987a994ba901045179e4f3807883 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 28 May 2024 13:57:23 -0400 Subject: [PATCH 3/8] Clean up spotlight tile code --- src/tile/SpotlightTile.tsx | 120 +++++++++++++++++++++++++------------ 1 file changed, 83 insertions(+), 37 deletions(-) diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index 77a3526f5..fccc52356 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -16,6 +16,7 @@ limitations under the License. import { ComponentProps, + RefAttributes, forwardRef, useCallback, useEffect, @@ -28,17 +29,20 @@ import CollapseIcon from "@vector-im/compound-design-tokens/icons/collapse.svg?r import ChevronLeftIcon from "@vector-im/compound-design-tokens/icons/chevron-left.svg?react"; import ChevronRightIcon from "@vector-im/compound-design-tokens/icons/chevron-right.svg?react"; import { animated } from "@react-spring/web"; -import { Observable, map, of } from "rxjs"; +import { Observable, map } from "rxjs"; import { useObservableEagerState } from "observable-hooks"; import { useTranslation } from "react-i18next"; import classNames from "classnames"; +import { TrackReferenceOrPlaceholder } from "@livekit/components-core"; +import { RoomMember } from "matrix-js-sdk"; import { MediaView } from "./MediaView"; import styles from "./SpotlightTile.module.css"; import { LocalUserMediaViewModel, MediaViewModel, - RemoteUserMediaViewModel, + ScreenShareViewModel, + UserMediaViewModel, useNameData, } from "../state/MediaViewModel"; import { useInitial } from "../useInitial"; @@ -47,12 +51,63 @@ import { useObservableRef } from "../state/useObservable"; import { useReactiveState } from "../useReactiveState"; import { useLatest } from "../useLatest"; -// Screen share video is always enabled -const videoEnabledDefault = of(true); -// Never mirror screen share video -const mirrorDefault = of(false); -// Never crop screen share video -const cropVideoDefault = of(false); +interface SpotlightItemBaseProps { + className?: string; + "data-id": string; + targetWidth: number; + targetHeight: number; + video: TrackReferenceOrPlaceholder; + member: RoomMember | undefined; + unencryptedWarning: boolean; + nameTag: string; + displayName: string; +} + +interface SpotlightUserMediaItemBaseProps extends SpotlightItemBaseProps { + videoEnabled: boolean; + videoFit: "contain" | "cover"; +} + +interface SpotlightLocalUserMediaItemProps + extends SpotlightUserMediaItemBaseProps { + vm: LocalUserMediaViewModel; +} + +const SpotlightLocalUserMediaItem = forwardRef< + HTMLDivElement, + SpotlightLocalUserMediaItemProps +>(({ vm, ...props }, ref) => { + const mirror = useObservableEagerState(vm.mirror); + return ; +}); + +SpotlightLocalUserMediaItem.displayName = "SpotlightLocalUserMediaItem"; + +interface SpotlightUserMediaItemProps extends SpotlightItemBaseProps { + vm: UserMediaViewModel; +} + +const SpotlightUserMediaItem = forwardRef< + HTMLDivElement, + SpotlightUserMediaItemProps +>(({ vm, ...props }, ref) => { + const videoEnabled = useObservableEagerState(vm.videoEnabled); + const cropVideo = useObservableEagerState(vm.cropVideo); + + const baseProps: SpotlightUserMediaItemBaseProps = { + videoEnabled, + videoFit: cropVideo ? "cover" : "contain", + ...props, + }; + + return vm instanceof LocalUserMediaViewModel ? ( + + ) : ( + + ); +}); + +SpotlightUserMediaItem.displayName = "SpotlightUserMediaItem"; interface SpotlightItemProps { vm: MediaViewModel; @@ -71,21 +126,6 @@ const SpotlightItem = forwardRef( const ref = useMergedRefs(ourRef, theirRef); const { displayName, nameTag } = useNameData(vm); const video = useObservableEagerState(vm.video); - const videoEnabled = useObservableEagerState( - vm instanceof LocalUserMediaViewModel || - vm instanceof RemoteUserMediaViewModel - ? vm.videoEnabled - : videoEnabledDefault, - ); - const mirror = useObservableEagerState( - vm instanceof LocalUserMediaViewModel ? vm.mirror : mirrorDefault, - ); - const cropVideo = useObservableEagerState( - vm instanceof LocalUserMediaViewModel || - vm instanceof RemoteUserMediaViewModel - ? vm.cropVideo - : cropVideoDefault, - ); const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning); // Hook this item up to the intersection observer @@ -103,22 +143,28 @@ const SpotlightItem = forwardRef( }; }, [intersectionObserver]); - return ( + const baseProps: SpotlightItemBaseProps & RefAttributes = { + ref, + "data-id": vm.id, + className: classNames(styles.item, { [styles.snap]: snap }), + targetWidth, + targetHeight, + video, + member: vm.member, + unencryptedWarning, + nameTag, + displayName, + }; + + return vm instanceof ScreenShareViewModel ? ( + ) : ( + ); }, ); From ec1b020d4eaf5eb6adac96802f1208b8d381979d Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 30 May 2024 13:06:24 -0400 Subject: [PATCH 4/8] Add indicators to spotlight tile and make spotlight layout responsive --- src/grid/CallLayout.ts | 20 ++- src/grid/Grid.tsx | 152 ++++++++--------- src/grid/GridLayout.module.css | 4 + src/grid/GridLayout.tsx | 253 ++++++++++++++-------------- src/grid/SpotlightLayout.module.css | 70 ++++---- src/grid/SpotlightLayout.tsx | 143 ++++++++++------ src/room/InCallView.module.css | 10 ++ src/room/InCallView.tsx | 139 +++++++++------ src/tile/GridTile.module.css | 4 - src/tile/SpotlightTile.module.css | 43 ++++- src/tile/SpotlightTile.tsx | 11 ++ 11 files changed, 494 insertions(+), 355 deletions(-) diff --git a/src/grid/CallLayout.ts b/src/grid/CallLayout.ts index 287f116d4..c4412677b 100644 --- a/src/grid/CallLayout.ts +++ b/src/grid/CallLayout.ts @@ -15,9 +15,10 @@ limitations under the License. */ import { BehaviorSubject, Observable } from "rxjs"; +import { ComponentType } from "react"; import { MediaViewModel } from "../state/MediaViewModel"; -import { LayoutSystem } from "./Grid"; +import { LayoutProps } from "./Grid"; import { Alignment } from "../room/InCallView"; export interface Bounds { @@ -36,15 +37,28 @@ export interface CallLayoutInputs { floatingAlignment: BehaviorSubject; } +export interface GridTileModel { + type: "grid"; + vm: MediaViewModel; +} + +export interface SpotlightTileModel { + type: "spotlight"; + vms: MediaViewModel[]; + maximised: boolean; +} + +export type TileModel = GridTileModel | SpotlightTileModel; + export interface CallLayoutOutputs { /** * The visually fixed (non-scrolling) layer of the layout. */ - fixed: LayoutSystem; + fixed: ComponentType>; /** * The layer of the layout that can overflow and be scrolled. */ - scrolling: LayoutSystem; + scrolling: ComponentType>; } /** diff --git a/src/grid/Grid.tsx b/src/grid/Grid.tsx index b6cf8fcdb..2e6a48aeb 100644 --- a/src/grid/Grid.tsx +++ b/src/grid/Grid.tsx @@ -42,6 +42,7 @@ import { useMergedRefs } from "../useMergedRefs"; import { TileWrapper } from "./TileWrapper"; import { usePrefersReducedMotion } from "../usePrefersReducedMotion"; import { TileSpringUpdate } from "./LegacyGrid"; +import { useInitial } from "../useInitial"; interface Rect { x: number; @@ -50,11 +51,14 @@ interface Rect { height: number; } -interface Tile extends Rect { +interface Tile { id: string; model: Model; + onDrag: DragCallback | undefined; } +type PlacedTile = Tile & Rect; + interface TileSpring { opacity: number; scale: number; @@ -73,24 +77,14 @@ interface DragState { cursorY: number; } -interface SlotProps extends ComponentProps<"div"> { - tile: string; +interface SlotProps extends Omit, "onDrag"> { + id: string; + model: Model; + onDrag?: DragCallback; style?: CSSProperties; className?: string; } -/** - * An invisible "slot" for a tile to go in. - */ -export const Slot: FC = ({ tile, style, className, ...props }) => ( -
-); - interface Offset { x: number; y: number; @@ -113,9 +107,13 @@ function offset(element: HTMLElement, relativeTo: Element): Offset { } } -export interface LayoutProps { +export interface LayoutProps { ref: LegacyRef; - model: Model; + model: LayoutModel; + /** + * Component creating an invisible "slot" for a tile to go in. + */ + Slot: ComponentType>; } export interface TileProps { @@ -152,25 +150,7 @@ interface Drag { yRatio: number; } -type DragCallback = (drag: Drag) => void; - -export interface LayoutSystem { - /** - * Defines the ID and model of each tile present in the layout. - */ - tiles: (model: LayoutModel) => Map; - /** - * A component which creates an invisible layout grid of "slots" for tiles to - * go in. The root element must have a data-generation attribute which - * increments whenever the layout may have changed. - */ - Layout: ComponentType>; - /** - * Gets a drag callback for the tile with the given ID. If this is not - * provided or it returns null, the tile is not draggable. - */ - onDrag?: (model: LayoutModel, tile: string) => DragCallback | null; -} +export type DragCallback = (drag: Drag) => void; interface Props< LayoutModel, @@ -183,9 +163,11 @@ interface Props< */ model: LayoutModel; /** - * The system by which to arrange the layout and respond to interactions. + * A component which creates an invisible layout grid of "slots" for tiles to + * go in. The root element must have a data-generation attribute which + * increments whenever the layout may have changed. */ - system: LayoutSystem; + Layout: ComponentType>; /** * The component used to render each tile in the layout. */ @@ -204,7 +186,7 @@ export function Grid< TileRef extends HTMLElement, >({ model, - system: { tiles: getTileModels, Layout, onDrag }, + Layout, Tile, className, style, @@ -223,8 +205,31 @@ export function Grid< const [layoutRoot, setLayoutRoot] = useState(null); const [generation, setGeneration] = useState(null); + const tiles = useInitial(() => new Map>()); const prefersReducedMotion = usePrefersReducedMotion(); + const Slot: FC> = useMemo( + () => + function Slot({ id, model, onDrag, style, className, ...props }) { + const ref = useRef(null); + useEffect(() => { + tiles.set(id, { id, model, onDrag }); + return (): void => void tiles.delete(id); + }, [id, model, onDrag]); + + return ( +
+ ); + }, + [tiles], + ); + const layoutRef = useCallback( (e: HTMLElement | null) => { setLayoutRoot(e); @@ -247,62 +252,45 @@ export function Grid< } }, [layoutRoot, setGeneration]); - const slotRects = useMemo(() => { - const rects = new Map(); + // Combine the tile definitions and slots together to create placed tiles + const placedTiles = useMemo(() => { + const result: PlacedTile[] = []; if (gridRoot !== null && layoutRoot !== null) { const slots = layoutRoot.getElementsByClassName( styles.slot, ) as HTMLCollectionOf; - for (const slot of slots) - rects.set(slot.getAttribute("data-tile")!, { + for (const slot of slots) { + const id = slot.getAttribute("data-id")!; + result.push({ + ...tiles.get(id)!, ...offset(slot, gridRoot), width: slot.offsetWidth, height: slot.offsetHeight, }); + } } - return rects; + return result; // The rects may change due to the grid updating to a new generation, but // eslint can't statically verify this // eslint-disable-next-line react-hooks/exhaustive-deps - }, [gridRoot, layoutRoot, generation]); - - const tileModels = useMemo( - () => getTileModels(model), - [getTileModels, model], - ); - - // Combine the tile models and slots together to create placed tiles - const tiles = useMemo[]>(() => { - const items: Tile[] = []; - for (const [id, model] of tileModels) { - const rect = slotRects.get(id); - if (rect !== undefined) items.push({ id, model, ...rect }); - } - return items; - }, [slotRects, tileModels]); - - const dragCallbacks = useMemo( - () => - new Map( - (function* (): Iterable<[string, DragCallback | null]> { - if (onDrag !== undefined) - for (const id of tileModels.keys()) yield [id, onDrag(model, id)]; - })(), - ), - [onDrag, tileModels, model], - ); + }, [gridRoot, layoutRoot, tiles, generation]); // Drag state is stored in a ref rather than component state, because we use // react-spring's imperative API during gestures to improve responsiveness const dragState = useRef(null); const [tileTransitions, springRef] = useTransition( - tiles, + placedTiles, () => ({ key: ({ id }: Tile): string => id, - from: ({ x, y, width, height }: Tile): TileSpringUpdate => ({ + from: ({ + x, + y, + width, + height, + }: PlacedTile): TileSpringUpdate => ({ opacity: 0, scale: 0, zIndex: 1, @@ -319,7 +307,7 @@ export function Grid< y, width, height, - }: Tile): TileSpringUpdate | null => + }: PlacedTile): TileSpringUpdate | null => id === dragState.current?.tileId ? null : { @@ -334,7 +322,7 @@ export function Grid< }), // react-spring's types are bugged and can't infer the spring type ) as unknown as [ - TransitionFn, TileSpring>, + TransitionFn, TileSpring>, SpringRef, ]; @@ -342,14 +330,14 @@ export function Grid< // firing animations manually whenever the tiles array updates useEffect(() => { springRef.start(); - }, [tiles, springRef]); + }, [placedTiles, springRef]); const animateDraggedTile = ( endOfGesture: boolean, callback: DragCallback, ): void => { const { tileId, tileX, tileY } = dragState.current!; - const tile = tiles.find((t) => t.id === tileId)!; + const tile = placedTiles.find((t) => t.id === tileId)!; springRef.current .find((c) => (c.item as Tile).id === tileId) @@ -416,7 +404,7 @@ export function Grid< const tileController = springRef.current.find( (c) => (c.item as Tile).id === tileId, )!; - const callback = dragCallbacks.get(tileController.item.id); + const callback = tiles.get(tileController.item.id)!.onDrag; if (callback != null) { if (dragState.current === null) { @@ -456,7 +444,7 @@ export function Grid< if (dragState.current !== null) { dragState.current.tileY += dy; dragState.current.cursorY += dy; - animateDraggedTile(false, onDrag!(model, dragState.current.tileId)!); + animateDraggedTile(false, tiles.get(dragState.current.tileId)!.onDrag!); } }, { target: gridRoot ?? undefined }, @@ -468,12 +456,12 @@ export function Grid< className={classNames(className, styles.grid)} style={style} > - - {tileTransitions((spring, { id, model, width, height }) => ( + + {tileTransitions((spring, { id, model, onDrag, width, height }) => ( .slot { position: absolute; inline-size: 404px; diff --git a/src/grid/GridLayout.tsx b/src/grid/GridLayout.tsx index 75f1e7268..3861457e8 100644 --- a/src/grid/GridLayout.tsx +++ b/src/grid/GridLayout.tsx @@ -14,16 +14,16 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { CSSProperties, forwardRef, useMemo } from "react"; +import { CSSProperties, forwardRef, useCallback, useMemo } from "react"; import { distinctUntilChanged } from "rxjs"; import { useObservableEagerState } from "observable-hooks"; import { GridLayout as GridLayoutModel } from "../state/CallViewModel"; -import { Slot } from "./Grid"; import styles from "./GridLayout.module.css"; import { useReactiveState } from "../useReactiveState"; import { useInitial } from "../useInitial"; -import { CallLayout } from "./CallLayout"; +import { CallLayout, GridTileModel, TileModel } from "./CallLayout"; +import { DragCallback } from "./Grid"; interface GridCSSProperties extends CSSProperties { "--gap": string; @@ -41,135 +41,144 @@ export const makeGridLayout: CallLayout = ({ }) => ({ // The "fixed" (non-scrolling) part of the layout is where the spotlight tile // lives - fixed: { - tiles: (model) => - new Map( - model.spotlight === undefined ? [] : [["spotlight", model.spotlight]], - ), - Layout: forwardRef(function GridLayoutFixed({ model }, ref) { - const { width, height } = useObservableEagerState(minBounds); - const alignment = useObservableEagerState( - useInitial(() => - floatingAlignment.pipe( - distinctUntilChanged( - (a1, a2) => a1.block === a2.block && a1.inline === a2.inline, - ), + fixed: forwardRef(function GridLayoutFixed({ model, Slot }, ref) { + const { width, height } = useObservableEagerState(minBounds); + const alignment = useObservableEagerState( + useInitial(() => + floatingAlignment.pipe( + distinctUntilChanged( + (a1, a2) => a1.block === a2.block && a1.inline === a2.inline, ), ), - ); - const [generation] = useReactiveState( - (prev) => (prev === undefined ? 0 : prev + 1), - [model.spotlight === undefined, width, height, alignment], - ); - - return ( -
- {model.spotlight && ( - - )} -
- ); - }), - onDrag: + ), + ); + const tileModel: TileModel | undefined = useMemo( () => + model.spotlight && { + type: "spotlight", + vms: model.spotlight, + maximised: false, + }, + [model.spotlight], + ); + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [model.spotlight === undefined, width, height, alignment], + ); + + const onDragSpotlight: DragCallback = useCallback( ({ xRatio, yRatio }) => floatingAlignment.next({ block: yRatio < 0.5 ? "start" : "end", inline: xRatio < 0.5 ? "start" : "end", }), - }, + [], + ); + + return ( +
+ {tileModel && ( + + )} +
+ ); + }), // The scrolling part of the layout is where all the grid tiles live - scrolling: { - tiles: (model) => new Map(model.grid.map((tile) => [tile.id, tile])), - Layout: forwardRef(function GridLayout({ model }, ref) { - const { width, height: minHeight } = useObservableEagerState(minBounds); - - // The goal here is to determine the grid size and padding that maximizes - // use of screen space for n tiles without making those tiles too small or - // too cropped (having an extreme aspect ratio) - const [gap, slotWidth, slotHeight] = useMemo(() => { - const gap = width < 800 ? 16 : 20; - const slotMinWidth = width < 500 ? 150 : 180; - - let columns = Math.min( - // Don't create more columns than we have items for - model.grid.length, - // The ideal number of columns is given by a packing of equally-sized - // squares into a grid. - // width / column = height / row. - // columns * rows = number of squares. - // ∴ columns = sqrt(width / height * number of squares). - // Except we actually want 16:9-ish slots rather than squares, so we - // divide the width-to-height ratio by the target aspect ratio. - Math.ceil( - Math.sqrt( - (width / minHeight / slotMaxAspectRatio) * model.grid.length, - ), + scrolling: forwardRef(function GridLayout({ model, Slot }, ref) { + const { width, height: minHeight } = useObservableEagerState(minBounds); + + // The goal here is to determine the grid size and padding that maximizes + // use of screen space for n tiles without making those tiles too small or + // too cropped (having an extreme aspect ratio) + const [gap, slotWidth, slotHeight] = useMemo(() => { + const gap = width < 800 ? 16 : 20; + const slotMinWidth = width < 500 ? 150 : 180; + + let columns = Math.min( + // Don't create more columns than we have items for + model.grid.length, + // The ideal number of columns is given by a packing of equally-sized + // squares into a grid. + // width / column = height / row. + // columns * rows = number of squares. + // ∴ columns = sqrt(width / height * number of squares). + // Except we actually want 16:9-ish slots rather than squares, so we + // divide the width-to-height ratio by the target aspect ratio. + Math.ceil( + Math.sqrt( + (width / minHeight / slotMaxAspectRatio) * model.grid.length, ), - ); - let rows = Math.ceil(model.grid.length / columns); - - let slotWidth = (width - (columns - 1) * gap) / columns; - let slotHeight = (minHeight - (rows - 1) * gap) / rows; - - // Impose a minimum width and height on the slots - if (slotWidth < slotMinWidth) { - // In this case we want the slot width to determine the number of columns, - // not the other way around. If we take the above equation for the slot - // width (w = (W - (c - 1) * g) / c) and solve for c, we get - // c = (W + g) / (w + g). - columns = Math.floor((width + gap) / (slotMinWidth + gap)); - rows = Math.ceil(model.grid.length / columns); - slotWidth = (width - (columns - 1) * gap) / columns; - slotHeight = (minHeight - (rows - 1) * gap) / rows; - } - if (slotHeight < slotMinHeight) slotHeight = slotMinHeight; - // Impose a minimum and maximum aspect ratio on the slots - const slotAspectRatio = slotWidth / slotHeight; - if (slotAspectRatio > slotMaxAspectRatio) - slotWidth = slotHeight * slotMaxAspectRatio; - else if (slotAspectRatio < slotMinAspectRatio) - slotHeight = slotWidth / slotMinAspectRatio; - // TODO: We might now be hitting the minimum height or width limit again - - return [gap, slotWidth, slotHeight]; - }, [width, minHeight, model.grid.length]); - - const [generation] = useReactiveState( - (prev) => (prev === undefined ? 0 : prev + 1), - [model.grid, width, minHeight], - ); - - return ( -
- {model.grid.map((tile) => ( - - ))} -
+ ), ); - }), - }, + let rows = Math.ceil(model.grid.length / columns); + + let slotWidth = (width - (columns - 1) * gap) / columns; + let slotHeight = (minHeight - (rows - 1) * gap) / rows; + + // Impose a minimum width and height on the slots + if (slotWidth < slotMinWidth) { + // In this case we want the slot width to determine the number of columns, + // not the other way around. If we take the above equation for the slot + // width (w = (W - (c - 1) * g) / c) and solve for c, we get + // c = (W + g) / (w + g). + columns = Math.floor((width + gap) / (slotMinWidth + gap)); + rows = Math.ceil(model.grid.length / columns); + slotWidth = (width - (columns - 1) * gap) / columns; + slotHeight = (minHeight - (rows - 1) * gap) / rows; + } + if (slotHeight < slotMinHeight) slotHeight = slotMinHeight; + // Impose a minimum and maximum aspect ratio on the slots + const slotAspectRatio = slotWidth / slotHeight; + if (slotAspectRatio > slotMaxAspectRatio) + slotWidth = slotHeight * slotMaxAspectRatio; + else if (slotAspectRatio < slotMinAspectRatio) + slotHeight = slotWidth / slotMinAspectRatio; + // TODO: We might now be hitting the minimum height or width limit again + + return [gap, slotWidth, slotHeight]; + }, [width, minHeight, model.grid.length]); + + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [model.grid, width, minHeight], + ); + + const tileModels: GridTileModel[] = useMemo( + () => model.grid.map((vm) => ({ type: "grid", vm })), + [model.grid], + ); + + return ( +
+ {tileModels.map((m) => ( + + ))} +
+ ); + }), }); diff --git a/src/grid/SpotlightLayout.module.css b/src/grid/SpotlightLayout.module.css index bbce45cfe..af43216c7 100644 --- a/src/grid/SpotlightLayout.module.css +++ b/src/grid/SpotlightLayout.module.css @@ -14,18 +14,20 @@ See the License for the specific language governing permissions and limitations under the License. */ -.fixed, -.scrolling { +.layer { margin-inline: var(--inline-content-inset); display: grid; - --grid-slot-width: 180px; --grid-gap: 20px; + gap: 30px; +} + +.layer[data-orientation="landscape"] { + --grid-slot-width: 180px; grid-template-columns: 1fr calc( var(--grid-columns) * var(--grid-slot-width) + (var(--grid-columns) - 1) * var(--grid-gap) ); grid-template-rows: minmax(1fr, auto); - gap: 30px; } .scrolling { @@ -41,7 +43,7 @@ limitations under the License. /* CSS makes us put a condition here, even though all we want to do is unconditionally select the container so we can use cq units */ @container spotlight (width > 0) { - .spotlight > .slot { + .layer[data-orientation="landscape"] > .spotlight > .slot { inline-size: min(100cqi, 100cqb * (17 / 9)); block-size: min(100cqb, 100cqi / (4 / 3)); } @@ -52,38 +54,48 @@ unconditionally select the container so we can use cq units */ flex-wrap: wrap; gap: var(--grid-gap); justify-content: center; +} + +.layer[data-orientation="landscape"] > .grid { align-content: center; } -.grid > .slot { +.layer > .grid > .slot { inline-size: var(--grid-slot-width); +} + +.layer[data-orientation="landscape"] > .grid > .slot { block-size: 135px; } -@media (max-width: 600px) { - .fixed, - .scrolling { - margin-inline: 0; - display: block; - } +.layer[data-orientation="portrait"] { + margin-inline: 0; + display: block; +} - .spotlight { - inline-size: 100%; - aspect-ratio: 16 / 9; - margin-block-end: var(--cpd-space-4x); - } +.layer[data-orientation="portrait"] > .spotlight { + inline-size: 100%; + aspect-ratio: 16 / 9; + margin-block-end: var(--cpd-space-4x); +} - .grid { - margin-inline: var(--inline-content-inset); - align-content: start; - } +.layer[data-orientation="portrait"] > .spotlight.withIndicators { + margin-block-end: calc(2 * var(--cpd-space-4x) + 2px); +} - .grid > .slot { - --grid-columns: 2; - --grid-slot-width: calc( - (100% - (var(--grid-columns) - 1) * var(--grid-gap)) / var(--grid-columns) - ); - block-size: unset; - aspect-ratio: 4 / 3; - } +.layer[data-orientation="portrait"] > .spotlight > .slot { + inline-size: 100%; + block-size: 100%; +} + +.layer[data-orientation="portrait"] > .grid { + margin-inline: var(--inline-content-inset); + align-content: start; +} + +.layer[data-orientation="portrait"] > .grid > .slot { + --grid-slot-width: calc( + (100% - (var(--grid-columns) - 1) * var(--grid-gap)) / var(--grid-columns) + ); + aspect-ratio: 4 / 3; } diff --git a/src/grid/SpotlightLayout.tsx b/src/grid/SpotlightLayout.tsx index 38bc6e374..3e07a0b2f 100644 --- a/src/grid/SpotlightLayout.tsx +++ b/src/grid/SpotlightLayout.tsx @@ -14,76 +14,113 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { CSSProperties, forwardRef } from "react"; +import { CSSProperties, forwardRef, useMemo } from "react"; import { useObservableEagerState } from "observable-hooks"; +import classNames from "classnames"; -import { CallLayout } from "./CallLayout"; +import { CallLayout, GridTileModel, TileModel } from "./CallLayout"; import { SpotlightLayout as SpotlightLayoutModel } from "../state/CallViewModel"; -import { useReactiveState } from "../useReactiveState"; import styles from "./SpotlightLayout.module.css"; -import { Slot } from "./Grid"; +import { useReactiveState } from "../useReactiveState"; interface GridCSSProperties extends CSSProperties { "--grid-columns": number; } -const getGridColumns = (gridLength: number): number => - gridLength > 20 ? 2 : 1; +interface Layout { + orientation: "portrait" | "landscape"; + gridColumns: number; +} + +function getLayout(gridLength: number, width: number): Layout { + const orientation = width < 800 ? "portrait" : "landscape"; + return { + orientation, + gridColumns: + orientation === "portrait" + ? Math.floor(width / 190) + : gridLength > 20 + ? 2 + : 1, + }; +} export const makeSpotlightLayout: CallLayout = ({ minBounds, }) => ({ - fixed: { - tiles: (model) => new Map([["spotlight", model.spotlight]]), - Layout: forwardRef(function SpotlightLayoutFixed({ model }, ref) { - const { width, height } = useObservableEagerState(minBounds); - const gridColumns = getGridColumns(model.grid.length); - const [generation] = useReactiveState( - (prev) => (prev === undefined ? 0 : prev + 1), - [model.grid.length, width, height], - ); + fixed: forwardRef(function SpotlightLayoutFixed({ model, Slot }, ref) { + const { width, height } = useObservableEagerState(minBounds); + const layout = getLayout(model.grid.length, width); + const tileModel: TileModel = useMemo( + () => ({ + type: "spotlight", + vms: model.spotlight, + maximised: layout.orientation === "portrait", + }), + [model.spotlight, layout.orientation], + ); + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [model.grid.length, width, height], + ); - return ( -
-
- -
-
+ return ( +
+
+
- ); - }), - }, +
+
+ ); + }), - scrolling: { - tiles: (model) => new Map(model.grid.map((tile) => [tile.id, tile])), - Layout: forwardRef(function SpotlightLayoutScrolling({ model }, ref) { - const { width, height } = useObservableEagerState(minBounds); - const gridColumns = getGridColumns(model.grid.length); - const [generation] = useReactiveState( - (prev) => (prev === undefined ? 0 : prev + 1), - [model.grid, width, height], - ); + scrolling: forwardRef(function SpotlightLayoutScrolling( + { model, Slot }, + ref, + ) { + const { width, height } = useObservableEagerState(minBounds); + const layout = getLayout(model.grid.length, width); + const tileModels: GridTileModel[] = useMemo( + () => model.grid.map((vm) => ({ type: "grid", vm })), + [model.grid], + ); + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [model.spotlight.length, model.grid, width, height], + ); - return ( + return ( +
-
-
- {model.grid.map((tile) => ( - - ))} -
+ className={classNames(styles.spotlight, { + [styles.withIndicators]: model.spotlight.length > 1, + })} + /> +
+ {tileModels.map((m) => ( + + ))}
- ); - }), - }, +
+ ); + }), }); diff --git a/src/room/InCallView.module.css b/src/room/InCallView.module.css index 76dc9bae6..60c46aa62 100644 --- a/src/room/InCallView.module.css +++ b/src/room/InCallView.module.css @@ -142,3 +142,13 @@ limitations under the License. inline-size: 100%; align-self: center; } + +.tile { + position: absolute; + inset-block-start: 0; +} + +.tile.maximised { + position: relative; + flex-grow: 1; +} diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index b2aeb9d23..3a16e1c67 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -25,6 +25,7 @@ import { ConnectionState, Room, Track } from "livekit-client"; import { MatrixClient } from "matrix-js-sdk/src/client"; import { FC, + PropsWithoutRef, forwardRef, useCallback, useEffect, @@ -86,7 +87,7 @@ import { EncryptionSystem } from "../e2ee/sharedKeyManagement"; import { E2eeType } from "../e2ee/e2eeType"; import { makeGridLayout } from "../grid/GridLayout"; import { makeSpotlightLayout } from "../grid/SpotlightLayout"; -import { CallLayout } from "../grid/CallLayout"; +import { CallLayout, GridTileModel, TileModel } from "../grid/CallLayout"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); @@ -329,7 +330,10 @@ export const InCallView: FC = ({ vm.layout.pipe( map((l) => { let makeLayout: CallLayout; - if (l.type === "grid" && l.grid.length !== 2) + if ( + l.type === "grid" && + !(l.grid.length === 2 && l.spotlight === undefined) + ) makeLayout = makeGridLayout as CallLayout; else if (l.type === "spotlight") makeLayout = makeSpotlightLayout as CallLayout; @@ -352,59 +356,79 @@ export const InCallView: FC = ({ [setLegacyLayout, vm], ); - const showSpeakingIndicators = + const showSpotlightIndicators = useObservable(layout.type === "spotlight"); + const showSpeakingIndicators = useObservable( layout.type === "spotlight" || - (layout.type === "grid" && layout.grid.length > 2); + (layout.type === "grid" && layout.grid.length > 2), + ); - const SpotlightTileView = useMemo( + const Tile = useMemo( () => - forwardRef>( - function SpotlightTileView( - { className, style, targetWidth, targetHeight, model }, - ref, - ) { - return ( - - ); - }, - ), - [toggleSpotlightFullscreen], + forwardRef< + HTMLDivElement, + PropsWithoutRef> + >(function Tile( + { className, style, targetWidth, targetHeight, model }, + ref, + ) { + const showSpeakingIndicatorsValue = useObservableEagerState( + showSpeakingIndicators, + ); + const showSpotlightIndicatorsValue = useObservableEagerState( + showSpotlightIndicators, + ); + + return model.type === "grid" ? ( + + ) : ( + + ); + }), + [ + toggleFullscreen, + toggleSpotlightFullscreen, + openProfile, + showSpeakingIndicators, + showSpotlightIndicators, + ], ); - const GridTileView = useMemo( + + const LegacyTile = useMemo( () => - forwardRef>( - function GridTileView( - { className, style, targetWidth, targetHeight, model }, - ref, - ) { - return ( - - ); - }, - ), - [toggleFullscreen, openProfile, showSpeakingIndicators], + forwardRef< + HTMLDivElement, + PropsWithoutRef> + >(function LegacyTile({ model: legacyModel, ...props }, ref) { + const model: GridTileModel = useMemo( + () => ({ type: "grid", vm: legacyModel }), + [legacyModel], + ); + return ; + }), + [Tile], ); const renderContent = (): JSX.Element => { @@ -421,17 +445,20 @@ export const InCallView: FC = ({ if (maximisedParticipant.id === "spotlight") { return ( ); } return ( = ({ items={items} layout={legacyLayout} disableAnimations={prefersReducedMotion} - Tile={GridTileView} + Tile={LegacyTile} /> ); } else { @@ -462,15 +489,15 @@ export const InCallView: FC = ({ ); diff --git a/src/tile/GridTile.module.css b/src/tile/GridTile.module.css index 923c76338..7ef66d8d5 100644 --- a/src/tile/GridTile.module.css +++ b/src/tile/GridTile.module.css @@ -15,8 +15,6 @@ limitations under the License. */ .tile { - position: absolute; - top: 0; --media-view-border-radius: var(--cpd-space-4x); transition: outline-color ease 0.15s; outline: var(--cpd-border-width-2) solid rgb(0 0 0 / 0); @@ -62,8 +60,6 @@ borders don't support gradients */ } .tile[data-maximised="true"] { - position: relative; - flex-grow: 1; --media-view-border-radius: 0; --media-view-fg-inset: 10px; } diff --git a/src/tile/SpotlightTile.module.css b/src/tile/SpotlightTile.module.css index 9d772c1d7..cc591fee3 100644 --- a/src/tile/SpotlightTile.module.css +++ b/src/tile/SpotlightTile.module.css @@ -15,14 +15,10 @@ limitations under the License. */ .tile { - position: absolute; - top: 0; --border-width: var(--cpd-space-3x); } .tile.maximised { - position: relative; - flex-grow: 1; --border-width: 0px; } @@ -54,14 +50,14 @@ limitations under the License. border-radius: 0; } -.item { +.contents > .item { height: 100%; flex-basis: 100%; flex-shrink: 0; --media-view-fg-inset: 10px; } -.item.snap { +.contents > .item.snap { scroll-snap-align: start; } @@ -151,3 +147,38 @@ limitations under the License. .tile:has(:focus-visible) > button { opacity: 1; } + +.indicators { + display: flex; + gap: var(--cpd-space-2x); + position: absolute; + inset-inline-start: 0; + inset-block-end: calc(-1 * var(--cpd-space-6x)); + width: 100%; + justify-content: start; + transition: opacity ease 0.15s; + opacity: 0; +} + +.indicators.show { + opacity: 1; +} + +.maximised .indicators { + inset-block-end: calc(-1 * var(--cpd-space-4x) - 2px); + justify-content: center; +} + +.indicators > .item { + inline-size: 32px; + block-size: 2px; + transition: background-color ease 0.15s; +} + +.indicators > .item[data-visible="false"] { + background: var(--cpd-color-alpha-gray-600); +} + +.indicators > .item[data-visible="true"] { + background: var(--cpd-color-gray-1400); +} diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index fccc52356..f77ce4cf9 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -178,6 +178,7 @@ interface Props { onToggleFullscreen: () => void; targetWidth: number; targetHeight: number; + showIndicators: boolean; className?: string; style?: ComponentProps["style"]; } @@ -191,6 +192,7 @@ export const SpotlightTile = forwardRef( onToggleFullscreen, targetWidth, targetHeight, + showIndicators, className, style, }, @@ -307,6 +309,15 @@ export const SpotlightTile = forwardRef( )} +
1, + })} + > + {vms.map((vm) => ( +
+ ))} +
); }, From 7f40ce8dde4d6b5a915eaf6a349a4b0482419a06 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 31 May 2024 10:54:17 -0400 Subject: [PATCH 5/8] Fix advance buttons showing up for the spotlight speaker --- src/tile/SpotlightTile.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index f77ce4cf9..a171fe4fd 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -204,8 +204,9 @@ export const SpotlightTile = forwardRef( const [visibleId, setVisibleId] = useState(vms[0].id); const latestVms = useLatest(vms); const latestVisibleId = useLatest(visibleId); - const canGoBack = visibleId !== vms[0].id; - const canGoToNext = visibleId !== vms[vms.length - 1].id; + const visibleIndex = vms.findIndex((vm) => vm.id === visibleId); + const canGoBack = visibleIndex > 0; + const canGoToNext = visibleIndex !== -1 && visibleIndex < vms.length - 1; // To keep track of which item is visible, we need an intersection observer // hooked up to the root element and the items. Because the items will run From dfda7539d6b592b8ae7eef5edcf35d0cdbaaff61 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 4 Jun 2024 16:06:40 -0400 Subject: [PATCH 6/8] Only switch to spotlight for remote screen shares --- src/state/CallViewModel.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index cc5afc463..3bbc0a2d7 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -376,9 +376,9 @@ export class CallViewModel extends ViewModel { shareReplay(1), ); - private readonly hasScreenShares: Observable = + private readonly hasRemoteScreenShares: Observable = this.screenShares.pipe( - map((ms) => ms.length > 0), + map((ms) => ms.find((m) => !m.vm.local) !== undefined), distinctUntilChanged(), ); @@ -474,7 +474,7 @@ export class CallViewModel extends ViewModel { this.gridModeUserSelection, // If the user hasn't selected spotlight and somebody starts screen sharing, // automatically switch to spotlight mode and reset when screen sharing ends - this.hasScreenShares.pipe( + this.hasRemoteScreenShares.pipe( withLatestFrom(this.gridModeUserSelection.pipe(startWith(null))), concatMap(([hasScreenShares, userSelection]) => userSelection === "spotlight" From 12b719da95ac8221f5aa9e8f44f8a0797af4041e Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 4 Jun 2024 16:07:07 -0400 Subject: [PATCH 7/8] Make layout reactivity a little more fine-grained --- src/state/CallViewModel.ts | 59 ++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 3bbc0a2d7..31a1fdb37 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -45,7 +45,6 @@ import { scan, shareReplay, startWith, - switchAll, switchMap, throttleTime, timer, @@ -488,39 +487,43 @@ export class CallViewModel extends ViewModel { this.gridModeUserSelection.next(value); } - public readonly layout: Observable = combineLatest( - [this.gridMode, this.windowMode], - (gridMode, windowMode) => { + public readonly layout: Observable = this.windowMode.pipe( + switchMap((windowMode) => { switch (windowMode) { case "full screen": throw new Error("unimplemented"); case "pip": throw new Error("unimplemented"); - case "normal": { - switch (gridMode) { - case "grid": - return combineLatest( - [this.grid, this.spotlight, this.screenShares], - (grid, spotlight, screenShares): Layout => ({ - type: "grid", - spotlight: screenShares.length > 0 ? spotlight : undefined, - grid, - }), - ); - case "spotlight": - return combineLatest( - [this.grid, this.spotlight], - (grid, spotlight): Layout => ({ - type: "spotlight", - spotlight, - grid, - }), - ); - } - } + case "normal": + return this.gridMode.pipe( + switchMap((gridMode) => { + switch (gridMode) { + case "grid": + return combineLatest( + [this.grid, this.spotlight, this.screenShares], + (grid, spotlight, screenShares): Layout => ({ + type: "grid", + spotlight: + screenShares.length > 0 ? spotlight : undefined, + grid, + }), + ); + case "spotlight": + return combineLatest( + [this.grid, this.spotlight], + (grid, spotlight): Layout => ({ + type: "spotlight", + spotlight, + grid, + }), + ); + } + }), + ); } - }, - ).pipe(switchAll(), shareReplay(1)); + }), + shareReplay(1), + ); /** * The media tiles to be displayed in the call view. From 1efa594430d6d64d5507adbf22b3f2559d70b507 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 17 Jul 2024 16:06:48 -0400 Subject: [PATCH 8/8] Use Array.some where it's appropriate --- src/state/CallViewModel.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 31a1fdb37..62d71048a 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -377,7 +377,7 @@ export class CallViewModel extends ViewModel { private readonly hasRemoteScreenShares: Observable = this.screenShares.pipe( - map((ms) => ms.find((m) => !m.vm.local) !== undefined), + map((ms) => ms.some((m) => !m.vm.local)), distinctUntilChanged(), );