diff --git a/src/button/Button.tsx b/src/button/Button.tsx index 4caa9c4f1..c11c92dde 100644 --- a/src/button/Button.tsx +++ b/src/button/Button.tsx @@ -16,7 +16,6 @@ import { EndCallIcon, ShareScreenSolidIcon, SettingsSolidIcon, - SwitchCameraSolidIcon, } from "@vector-im/compound-design-tokens/assets/web/icons"; import styles from "./Button.module.css"; @@ -67,23 +66,6 @@ export const VideoButton: FC = ({ muted, ...props }) => { ); }; -export const SwitchCameraButton: FC> = ( - props, -) => { - const { t } = useTranslation(); - - return ( - - - - ); -}; - interface ShareScreenButtonProps extends ComponentPropsWithoutRef<"button"> { enabled: boolean; } diff --git a/src/grid/TileWrapper.tsx b/src/grid/TileWrapper.tsx index 9e58fd7c2..1bed08daa 100644 --- a/src/grid/TileWrapper.tsx +++ b/src/grid/TileWrapper.tsx @@ -61,7 +61,12 @@ const TileWrapper_ = memo( useDrag((state) => onDrag?.current!(id, state), { target: ref, filterTaps: true, - preventScroll: true, + // Previous designs, which allowed tiles to be dragged and dropped around + // the scrolling grid, required us to set preventScroll to true here. But + // our designs no longer call for this, and meanwhile there's a bug in + // use-gesture that causes filterTaps + preventScroll to break buttons + // within tiles (like the 'switch camera' button) on mobile. + // https://github.com/pmndrs/use-gesture/issues/593 }); return ( diff --git a/src/room/InCallView.module.css b/src/room/InCallView.module.css index 24dfbe5c9..96b8a368f 100644 --- a/src/room/InCallView.module.css +++ b/src/room/InCallView.module.css @@ -108,14 +108,6 @@ Please see LICENSE in the repository root for full details. } @media (max-width: 370px) { - .raiseHand { - display: none; - } -} - -@media (max-width: 340px) { - .invite, - .switchCamera, .shareScreen { display: none; } @@ -127,6 +119,13 @@ Please see LICENSE in the repository root for full details. } } +@media (max-width: 320px) { + .invite, + .raiseHand { + display: none; + } +} + @media (max-height: 400px) { .footer { padding-block: var(--cpd-space-4x); diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 452e8572c..53fc06674 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -44,7 +44,6 @@ import { ShareScreenButton, SettingsButton, ReactionToggleButton, - SwitchCameraButton, } from "../button"; import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header"; import { type HeaderStyle, useUrlParams } from "../UrlParams"; @@ -94,7 +93,6 @@ import { useReactionsSender, } from "../reactions/useReactionsSender"; import { ReactionsAudioRenderer } from "./ReactionAudioRenderer"; -import { useSwitchCamera } from "./useSwitchCamera"; import { ReactionsOverlay } from "./ReactionsOverlay"; import { CallEventAudioRenderer } from "./CallEventAudioRenderer"; import { @@ -311,7 +309,6 @@ export const InCallView: FC = ({ const showFooter = useObservableEagerState(vm.showFooter$); const earpieceMode = useObservableEagerState(vm.earpieceMode$); const audioOutputSwitcher = useObservableEagerState(vm.audioOutputSwitcher$); - const switchCamera = useSwitchCamera(vm.localVideo$); // Ideally we could detect taps by listening for click events and checking // that the pointerType of the event is "touch", but this isn't yet supported @@ -672,15 +669,6 @@ export const InCallView: FC = ({ data-testid="incall_videomute" />, ); - if (switchCamera !== null) - buttons.push( - , - ); if (canScreenshare && !hideScreensharing) { buttons.push( = ({ }, [devices, videoInputId, videoTrack]); useTrackProcessorSync(videoTrack); - const showSwitchCamera = useShowSwitchCamera( - useObservable( - (inputs$) => inputs$.pipe(map(([video]) => video)), - [videoTrack], - ), - ); // TODO: Unify this component with InCallView, so we can get slick joining // animations and don't have to feel bad about reusing its CSS @@ -257,9 +248,6 @@ export const LobbyView: FC = ({ onClick={onVideoPress} disabled={muteStates.video.setEnabled === null} /> - {showSwitchCamera && ( - - )} {!confineToRoom && } diff --git a/src/room/useSwitchCamera.ts b/src/room/useSwitchCamera.ts deleted file mode 100644 index 975776ae7..000000000 --- a/src/room/useSwitchCamera.ts +++ /dev/null @@ -1,93 +0,0 @@ -/* -Copyright 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - fromEvent, - map, - merge, - type Observable, - of, - startWith, - switchMap, -} from "rxjs"; -import { - facingModeFromLocalTrack, - type LocalVideoTrack, - TrackEvent, -} from "livekit-client"; -import { useObservable, useObservableEagerState } from "observable-hooks"; -import { logger } from "matrix-js-sdk/lib/logger"; - -import { useMediaDevices } from "../MediaDevicesContext"; -import { platform } from "../Platform"; -import { useLatest } from "../useLatest"; - -/** - * Determines whether the user should be shown a button to switch their camera, - * producing a callback if so. - */ -export function useSwitchCamera( - video$: Observable, -): (() => void) | null { - const mediaDevices = useMediaDevices(); - const setVideoInput = useLatest(mediaDevices.videoInput.select); - - // Produce an observable like the input 'video' observable, except make it - // emit whenever the track is muted or the device changes - const videoTrack$: Observable = useObservable( - (inputs$) => - inputs$.pipe( - switchMap(([video$]) => video$), - switchMap((video) => { - if (video === null) return of(null); - return merge( - fromEvent(video, TrackEvent.Restarted).pipe( - startWith(null), - map(() => video), - ), - fromEvent(video, TrackEvent.Muted).pipe(map(() => null)), - ); - }), - ), - [video$], - ); - - const switchCamera$: Observable<(() => void) | null> = useObservable( - (inputs$) => - platform === "desktop" - ? of(null) - : inputs$.pipe( - switchMap(([track$]) => track$), - map((track) => { - if (track === null) return null; - const facingMode = facingModeFromLocalTrack(track).facingMode; - // If the camera isn't front or back-facing, don't provide a switch - // camera shortcut at all - if (facingMode !== "user" && facingMode !== "environment") - return null; - // Restart the track with a camera facing the opposite direction - return (): void => - void track - .restartTrack({ - facingMode: facingMode === "user" ? "environment" : "user", - }) - .then(() => { - // Inform the MediaDeviceContext which camera was chosen - const deviceId = - track.mediaStreamTrack.getSettings().deviceId; - if (deviceId !== undefined) setVideoInput.current(deviceId); - }) - .catch((e) => - logger.error("Failed to switch camera", facingMode, e), - ); - }), - ), - [videoTrack$], - ); - - return useObservableEagerState(switchCamera$); -} diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index fc1222c44..9dd2c3f08 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -13,10 +13,8 @@ import { import { type Room as LivekitRoom, type LocalParticipant, - LocalVideoTrack, ParticipantEvent, type RemoteParticipant, - Track, } from "livekit-client"; import { RoomStateEvent, type Room, type RoomMember } from "matrix-js-sdk"; import { @@ -60,7 +58,6 @@ import { import { LocalUserMediaViewModel, type MediaViewModel, - observeTrackReference$, RemoteUserMediaViewModel, ScreenShareViewModel, type UserMediaViewModel, @@ -258,6 +255,7 @@ class UserMedia { participant: LocalParticipant | RemoteParticipant | undefined, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, + mediaDevices: MediaDevices, displayname$: Observable, handRaised$: Observable, reaction$: Observable, @@ -271,6 +269,7 @@ class UserMedia { this.participant$.asObservable() as Observable, encryptionSystem, livekitRoom, + mediaDevices, displayname$, handRaised$, reaction$, @@ -382,17 +381,6 @@ function getRoomMemberFromRtcMember( // TODO: Move wayyyy more business logic from the call and lobby views into here export class CallViewModel extends ViewModel { - public readonly localVideo$: Observable = - observeTrackReference$( - of(this.livekitRoom.localParticipant), - Track.Source.Camera, - ).pipe( - map((trackRef) => { - const track = trackRef?.publication?.track; - return track instanceof LocalVideoTrack ? track : null; - }), - ); - /** * The raw list of RemoteParticipants as reported by LiveKit */ @@ -579,6 +567,7 @@ export class CallViewModel extends ViewModel { participant, this.encryptionSystem, this.livekitRoom, + this.mediaDevices, this.memberDisplaynames$.pipe( map((m) => m.get(matrixIdentifier) ?? "[👻]"), ), @@ -643,6 +632,7 @@ export class CallViewModel extends ViewModel { participant, this.encryptionSystem, this.livekitRoom, + this.mediaDevices, this.memberDisplaynames$.pipe( map((m) => m.get(participant.identity) ?? "[👻]"), ), diff --git a/src/state/MediaViewModel.test.ts b/src/state/MediaViewModel.test.ts index 601133acf..8de550d45 100644 --- a/src/state/MediaViewModel.test.ts +++ b/src/state/MediaViewModel.test.ts @@ -5,14 +5,40 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { expect, test, vi } from "vitest"; +import { expect, onTestFinished, test, vi } from "vitest"; +import { of } from "rxjs"; +import { + type LocalTrackPublication, + LocalVideoTrack, + TrackEvent, +} from "livekit-client"; +import { waitFor } from "@testing-library/dom"; import { + mockLocalParticipant, + mockMediaDevices, mockRtcMembership, withLocalMedia, withRemoteMedia, withTestScheduler, } from "../utils/test"; +import { getValue } from "../utils/observable"; + +global.MediaStreamTrack = class {} as unknown as { + new (): MediaStreamTrack; + prototype: MediaStreamTrack; +}; +global.MediaStream = class {} as unknown as { + new (): MediaStream; + prototype: MediaStream; +}; + +const platformMock = vi.hoisted(() => vi.fn(() => "desktop")); +vi.mock("../Platform", () => ({ + get platform(): string { + return platformMock(); + }, +})); const rtcMembership = mockRtcMembership("@alice:example.org", "AAAA"); @@ -79,17 +105,23 @@ test("toggle fit/contain for a participant's video", async () => { }); test("local media remembers whether it should always be shown", async () => { - await withLocalMedia(rtcMembership, {}, (vm) => - withTestScheduler(({ expectObservable, schedule }) => { - schedule("-a|", { a: () => vm.setAlwaysShow(false) }); - expectObservable(vm.alwaysShow$).toBe("ab", { a: true, b: false }); - }), + await withLocalMedia( + rtcMembership, + {}, + mockLocalParticipant({}), + mockMediaDevices({}), + (vm) => + withTestScheduler(({ expectObservable, schedule }) => { + schedule("-a|", { a: () => vm.setAlwaysShow(false) }); + expectObservable(vm.alwaysShow$).toBe("ab", { a: true, b: false }); + }), ); // Next local media should start out *not* always shown await withLocalMedia( rtcMembership, - {}, + mockLocalParticipant({}), + mockMediaDevices({}), (vm) => withTestScheduler(({ expectObservable, schedule }) => { schedule("-a|", { a: () => vm.setAlwaysShow(true) }); @@ -97,3 +129,76 @@ test("local media remembers whether it should always be shown", async () => { }), ); }); + +test("switch cameras", async () => { + // Camera switching is only available on mobile + platformMock.mockReturnValue("android"); + onTestFinished(() => void platformMock.mockReset()); + + // Construct a mock video track which knows how to be restarted + const track = new LocalVideoTrack({ + getConstraints() {}, + addEventListener() {}, + removeEventListener() {}, + } as unknown as MediaStreamTrack); + + let deviceId = "front camera"; + const restartTrack = vi.fn(async ({ facingMode }) => { + deviceId = facingMode === "user" ? "front camera" : "back camera"; + track.emit(TrackEvent.Restarted); + return Promise.resolve(); + }); + track.restartTrack = restartTrack; + + Object.defineProperty(track, "mediaStreamTrack", { + get() { + return { + label: "Video", + getSettings: (): object => ({ + deviceId, + facingMode: deviceId === "front camera" ? "user" : "environment", + }), + }; + }, + }); + + const selectVideoInput = vi.fn(); + + await withLocalMedia( + rtcMembership, + {}, + mockLocalParticipant({ + getTrackPublication() { + return { track } as unknown as LocalTrackPublication; + }, + }), + mockMediaDevices({ + videoInput: { + available$: of(new Map()), + selected$: of(undefined), + select: selectVideoInput, + }, + }), + async (vm) => { + // Switch to back camera + getValue(vm.switchCamera$)!(); + expect(restartTrack).toHaveBeenCalledTimes(1); + expect(restartTrack).toHaveBeenCalledWith({ facingMode: "environment" }); + await waitFor(() => { + expect(selectVideoInput).toHaveBeenCalledTimes(1); + expect(selectVideoInput).toHaveBeenCalledWith("back camera"); + }); + expect(deviceId).toBe("back camera"); + + // Switch to front camera + getValue(vm.switchCamera$)!(); + expect(restartTrack).toHaveBeenCalledTimes(2); + expect(restartTrack).toHaveBeenLastCalledWith({ facingMode: "user" }); + await waitFor(() => { + expect(selectVideoInput).toHaveBeenCalledTimes(2); + expect(selectVideoInput).toHaveBeenLastCalledWith("front camera"); + }); + expect(deviceId).toBe("front camera"); + }, + ); +}); diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index 424d003e5..920b6ef39 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -16,6 +16,7 @@ import { import { type LocalParticipant, LocalTrack, + LocalVideoTrack, type Participant, ParticipantEvent, type RemoteParticipant, @@ -27,6 +28,7 @@ import { RemoteTrack, } from "livekit-client"; import { type RoomMember } from "matrix-js-sdk"; +import { logger } from "matrix-js-sdk/lib/logger"; import { BehaviorSubject, type Observable, @@ -51,6 +53,8 @@ import { accumulate } from "../utils/observable"; import { type EncryptionSystem } from "../e2ee/sharedKeyManagement"; import { E2eeType } from "../e2ee/e2eeType"; import { type ReactionOption } from "../reactions"; +import { platform } from "../Platform"; +import { type MediaDevices } from "./MediaDevices"; export function observeTrackReference$( participant$: Observable, @@ -433,20 +437,35 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { * The local participant's user media. */ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { + /** + * The local video track as an observable that emits whenever the track + * changes, the camera is switched, or the track is muted. + */ + private readonly videoTrack$: Observable = + this.video$.pipe( + switchMap((v) => { + const track = v?.publication?.track; + if (!(track instanceof LocalVideoTrack)) return of(null); + return merge( + // Watch for track restarts because they indicate a camera switch + fromEvent(track, TrackEvent.Restarted).pipe( + startWith(null), + map(() => track), + ), + fromEvent(track, TrackEvent.Muted).pipe(map(() => null)), + ); + }), + ); + /** * Whether the video should be mirrored. */ - public readonly mirror$ = this.video$.pipe( - switchMap((v) => { - const track = v?.publication?.track; - if (!(track instanceof LocalTrack)) return of(false); - // Watch for track restarts, because they indicate a camera switch - return fromEvent(track, TrackEvent.Restarted).pipe( - startWith(null), - // Mirror only front-facing cameras (those that face the user) - map(() => facingModeFromLocalTrack(track).facingMode === "user"), - ); - }), + public readonly mirror$ = this.videoTrack$.pipe( + // Mirror only front-facing cameras (those that face the user) + map( + (track) => + track !== null && facingModeFromLocalTrack(track).facingMode === "user", + ), this.scope.state(), ); @@ -457,12 +476,46 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { public readonly alwaysShow$ = alwaysShowSelf.value$; public readonly setAlwaysShow = alwaysShowSelf.setValue; + /** + * Callback for switching between the front and back cameras. + */ + public readonly switchCamera$: Observable<(() => void) | null> = + platform === "desktop" + ? of(null) + : this.videoTrack$.pipe( + map((track) => { + if (track === null) return null; + const facingMode = facingModeFromLocalTrack(track).facingMode; + // If the camera isn't front or back-facing, don't provide a switch + // camera shortcut at all + if (facingMode !== "user" && facingMode !== "environment") + return null; + // Restart the track with a camera facing the opposite direction + return (): void => + void track + .restartTrack({ + facingMode: facingMode === "user" ? "environment" : "user", + }) + .then(() => { + // Inform the MediaDevices which camera was chosen + const deviceId = + track.mediaStreamTrack.getSettings().deviceId; + if (deviceId !== undefined) + this.mediaDevices.videoInput.select(deviceId); + }) + .catch((e) => + logger.error("Failed to switch camera", facingMode, e), + ); + }), + ); + public constructor( id: string, member: RoomMember | undefined, participant$: Observable, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, + private readonly mediaDevices: MediaDevices, displayname$: Observable, handRaised$: Observable, reaction$: Observable, diff --git a/src/tile/GridTile.module.css b/src/tile/GridTile.module.css index 867c1cefc..ee605e46b 100644 --- a/src/tile/GridTile.module.css +++ b/src/tile/GridTile.module.css @@ -83,3 +83,25 @@ borders don't support gradients */ .volumeSlider { width: 100%; } + +.tile .switchCamera { + opacity: 1; + background: var(--cpd-color-bg-action-secondary-rest); + border: 1px solid var(--cpd-color-border-interactive-secondary); +} + +.tile .switchCamera > svg { + color: var(--cpd-color-icon-primary); +} + +@media (hover) { + .tile .switchCamera:hover { + background: var(--cpd-color-bg-subtle-secondary); + border-color: var(--cpd-color-border-interactive-hovered); + } +} + +.tile .switchCamera:active { + background: var(--cpd-color-bg-subtle-primary); + border-color: var(--cpd-color-border-interactive-hovered); +} diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 1e363b182..78534daea 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -28,6 +28,7 @@ import { UserProfileIcon, ExpandIcon, VolumeOffSolidIcon, + SwitchCameraSolidIcon, } from "@vector-im/compound-design-tokens/assets/web/icons"; import { ContextMenu, @@ -64,6 +65,7 @@ interface UserMediaTileProps extends TileProps { vm: UserMediaViewModel; mirror: boolean; locallyMuted: boolean; + primaryButton?: ReactNode; menuStart?: ReactNode; menuEnd?: ReactNode; } @@ -73,6 +75,7 @@ const UserMediaTile: FC = ({ vm, showSpeakingIndicators, locallyMuted, + primaryButton, menuStart, menuEnd, className, @@ -159,20 +162,22 @@ const UserMediaTile: FC = ({ } displayName={displayName} primaryButton={ - - - - } - side="left" - align="start" - > - {menu} - + primaryButton ?? ( + + + + } + side="left" + align="start" + > + {menu} + + ) } raisedHandTime={handRaised ?? undefined} currentReaction={reaction ?? undefined} @@ -207,6 +212,8 @@ const LocalUserMediaTile: FC = ({ const { t } = useTranslation(); const mirror = useObservableEagerState(vm.mirror$); const alwaysShow = useObservableEagerState(vm.alwaysShow$); + const switchCamera = useObservableEagerState(vm.switchCamera$); + const latestAlwaysShow = useLatest(alwaysShow); const onSelectAlwaysShow = useCallback( (e: Event) => { @@ -222,6 +229,17 @@ const LocalUserMediaTile: FC = ({ vm={vm} locallyMuted={false} mirror={mirror} + primaryButton={ + switchCamera === null ? undefined : ( + + ) + } menuStart={ button:active { - background: var(--cpd-color-bg-action-primary-pressed) !important; + background: var(--cpd-color-bg-action-primary-pressed); } .fg > button[data-state="open"] { diff --git a/src/tile/SpotlightTile.test.tsx b/src/tile/SpotlightTile.test.tsx index 1f3b7a864..3e039bca5 100644 --- a/src/tile/SpotlightTile.test.tsx +++ b/src/tile/SpotlightTile.test.tsx @@ -13,6 +13,8 @@ import { of } from "rxjs"; import { SpotlightTile } from "./SpotlightTile"; import { + mockLocalParticipant, + mockMediaDevices, mockRtcMembership, withLocalMedia, withRemoteMedia, @@ -39,6 +41,8 @@ test("SpotlightTile is accessible", async () => { rawDisplayName: "Bob", getMxcAvatarUrl: () => "mxc://dlskf", }, + mockLocalParticipant({}), + mockMediaDevices({}), async (vm2) => { const user = userEvent.setup(); const toggleExpanded = vi.fn(); diff --git a/src/utils/test.ts b/src/utils/test.ts index 8f8b19a38..a2234f577 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -205,9 +205,10 @@ export function mockLocalParticipant( export async function withLocalMedia( localRtcMember: CallMembership, roomMember: Partial, + localParticipant: LocalParticipant, + mediaDevices: MediaDevices, continuation: (vm: LocalUserMediaViewModel) => void | Promise, ): Promise { - const localParticipant = mockLocalParticipant({}); const vm = new LocalUserMediaViewModel( "local", mockMatrixRoomMember(localRtcMember, roomMember), @@ -216,6 +217,7 @@ export async function withLocalMedia( kind: E2eeType.PER_PARTICIPANT, }, mockLivekitRoom({ localParticipant }), + mediaDevices, of(roomMember.rawDisplayName ?? "nodisplayname"), of(null), of(null),