Add camera switching to the media view model

Author: robintown

src/room/InCallView.tsx

@@ -103,6 +103,7 @@ import { useTypedEventEmitter } from "../useEvents.ts";
 import { MatrixAudioRenderer } from "../livekit/MatrixAudioRenderer.tsx";
 import { muteAllAudio$ } from "../state/MuteAllAudioModel.ts";
 import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMemberships.ts";
+import { useMediaDevices } from "../MediaDevicesContext.ts";
 
 const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
 
@@ -114,6 +115,7 @@ export interface ActiveCallProps
 }
 
 export const ActiveCall: FC<ActiveCallProps> = (props) => {
+  const mediaDevices = useMediaDevices();
   const sfuConfig = useOpenIDSFU(props.client, props.rtcSession);
   const { livekitRoom, connState } = useLivekit(
     props.rtcSession,
@@ -154,6 +156,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
       const vm = new CallViewModel(
         props.rtcSession,
         livekitRoom,
+        mediaDevices,
         props.e2eeSystem,
         connStateObservable$,
         reactionsReader.raisedHands$,
@@ -165,7 +168,13 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
         reactionsReader.destroy();
       };
     }
-  }, [props.rtcSession, livekitRoom, props.e2eeSystem, connStateObservable$]);
+  }, [
+    props.rtcSession,
+    livekitRoom,
+    mediaDevices,
+    props.e2eeSystem,
+    connStateObservable$,
+  ]);
 
   if (livekitRoom === undefined || vm === null) return null;
 

src/state/CallViewModel.test.ts

@@ -71,6 +71,7 @@ import {
   localId,
   localRtcMember,
 } from "../utils/test-fixtures";
+import { type MediaDevices } from "./MediaDevices";
 
 vi.mock("@livekit/components-core");
 
@@ -262,6 +263,7 @@ function withCallViewModel(
   const vm = new CallViewModel(
     rtcSession as unknown as MatrixRTCSession,
     liveKitRoom,
+    {} as unknown as MediaDevices,
     {
       kind: E2eeType.PER_PARTICIPANT,
     },

src/state/CallViewModel.ts

@@ -90,6 +90,7 @@ import {
 import { observeSpeaker$ } from "./observeSpeaker";
 import { shallowEquals } from "../utils/array";
 import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname";
+import { type MediaDevices } from "./MediaDevices";
 
 // How long we wait after a focus switch before showing the real participant
 // list again
@@ -254,6 +255,7 @@ class UserMedia {
     participant: LocalParticipant | RemoteParticipant | undefined,
     encryptionSystem: EncryptionSystem,
     livekitRoom: LivekitRoom,
+    mediaDevices: MediaDevices,
     displayname$: Observable<string>,
     handRaised$: Observable<Date | null>,
     reaction$: Observable<ReactionOption | null>,
@@ -267,6 +269,7 @@ class UserMedia {
         this.participant$.asObservable() as Observable<LocalParticipant>,
         encryptionSystem,
         livekitRoom,
+        mediaDevices,
         displayname$,
         handRaised$,
         reaction$,
@@ -564,6 +567,7 @@ export class CallViewModel extends ViewModel {
                       participant,
                       this.encryptionSystem,
                       this.livekitRoom,
+                      this.mediaDevices,
                       this.memberDisplaynames$.pipe(
                         map((m) => m.get(matrixIdentifier) ?? "[👻]"),
                       ),
@@ -628,6 +632,7 @@ export class CallViewModel extends ViewModel {
                             participant,
                             this.encryptionSystem,
                             this.livekitRoom,
+                            this.mediaDevices,
                             this.memberDisplaynames$.pipe(
                               map((m) => m.get(participant.identity) ?? "[👻]"),
                             ),
@@ -1322,6 +1327,7 @@ export class CallViewModel extends ViewModel {
     // A call is permanently tied to a single Matrix room and LiveKit room
     private readonly matrixRTCSession: MatrixRTCSession,
     private readonly livekitRoom: LivekitRoom,
+    private readonly mediaDevices: MediaDevices,
     private readonly encryptionSystem: EncryptionSystem,
     private readonly connectionState$: Observable<ECConnectionState>,
     private readonly handsRaisedSubject$: Observable<

src/state/MediaViewModel.ts

@@ -16,6 +16,7 @@ import {
 import {
   type LocalParticipant,
   LocalTrack,
+  LocalVideoTrack,
   type Participant,
   ParticipantEvent,
   type RemoteParticipant,
@@ -27,6 +28,7 @@ import {
   RemoteTrack,
 } from "livekit-client";
 import { type RoomMember } from "matrix-js-sdk";
+import { logger } from "matrix-js-sdk/lib/logger";
 import {
   BehaviorSubject,
   type Observable,
@@ -51,6 +53,8 @@ import { accumulate } from "../utils/observable";
 import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
 import { E2eeType } from "../e2ee/e2eeType";
 import { type ReactionOption } from "../reactions";
+import { platform } from "../Platform";
+import { type MediaDevices } from "./MediaDevices";
 
 export function observeTrackReference$(
   participant$: Observable<Participant | undefined>,
@@ -433,20 +437,35 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
  * The local participant's user media.
  */
 export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
+  /**
+   * The local video track as an observable that emits whenever the track
+   * changes, the camera is switched, or the track is muted.
+   */
+  private readonly videoTrack$: Observable<LocalVideoTrack | null> =
+    this.video$.pipe(
+      switchMap((v) => {
+        const track = v?.publication?.track;
+        if (!(track instanceof LocalVideoTrack)) return of(null);
+        return merge(
+          // Watch for track restarts because they indicate a camera switch
+          fromEvent(track, TrackEvent.Restarted).pipe(
+            startWith(null),
+            map(() => track),
+          ),
+          fromEvent(track, TrackEvent.Muted).pipe(map(() => null)),
+        );
+      }),
+    );
+
   /**
    * Whether the video should be mirrored.
    */
-  public readonly mirror$ = this.video$.pipe(
-    switchMap((v) => {
-      const track = v?.publication?.track;
-      if (!(track instanceof LocalTrack)) return of(false);
-      // Watch for track restarts, because they indicate a camera switch
-      return fromEvent(track, TrackEvent.Restarted).pipe(
-        startWith(null),
-        // Mirror only front-facing cameras (those that face the user)
-        map(() => facingModeFromLocalTrack(track).facingMode === "user"),
-      );
-    }),
+  public readonly mirror$ = this.videoTrack$.pipe(
+    // Mirror only front-facing cameras (those that face the user)
+    map(
+      (track) =>
+        track !== null && facingModeFromLocalTrack(track).facingMode === "user",
+    ),
     this.scope.state(),
   );
 
@@ -457,12 +476,46 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
   public readonly alwaysShow$ = alwaysShowSelf.value$;
   public readonly setAlwaysShow = alwaysShowSelf.setValue;
 
+  /**
+   * Callback for switching between the front and back cameras.
+   */
+  public readonly switchCamera$: Observable<(() => void) | null> =
+    platform === "desktop"
+      ? of(null)
+      : this.videoTrack$.pipe(
+          map((track) => {
+            if (track === null) return null;
+            const facingMode = facingModeFromLocalTrack(track).facingMode;
+            // If the camera isn't front or back-facing, don't provide a switch
+            // camera shortcut at all
+            if (facingMode !== "user" && facingMode !== "environment")
+              return null;
+            // Restart the track with a camera facing the opposite direction
+            return (): void =>
+              void track
+                .restartTrack({
+                  facingMode: facingMode === "user" ? "environment" : "user",
+                })
+                .then(() => {
+                  // Inform the MediaDevices which camera was chosen
+                  const deviceId =
+                    track.mediaStreamTrack.getSettings().deviceId;
+                  if (deviceId !== undefined)
+                    this.mediaDevices.videoInput.select(deviceId);
+                })
+                .catch((e) =>
+                  logger.error("Failed to switch camera", facingMode, e),
+                );
+          }),
+        );
+
   public constructor(
     id: string,
     member: RoomMember | undefined,
     participant$: Observable<LocalParticipant | undefined>,
     encryptionSystem: EncryptionSystem,
     livekitRoom: LivekitRoom,
+    private readonly mediaDevices: MediaDevices,
     displayname$: Observable<string>,
     handRaised$: Observable<Date | null>,
     reaction$: Observable<ReactionOption | null>,

src/utils/test-viewmodel.ts

@@ -26,6 +26,7 @@ import {
   localRtcMember,
 } from "./test-fixtures";
 import { type RaisedHandInfo, type ReactionInfo } from "../reactions";
+import { type MediaDevices } from "../state/MediaDevices";
 
 export function getBasicRTCSession(
   members: RoomMember[],
@@ -132,6 +133,7 @@ export function getBasicCallViewModelEnvironment(
   const vm = new CallViewModel(
     rtcSession as unknown as MatrixRTCSession,
     liveKitRoom,
+    {} as unknown as MediaDevices,
     {
       kind: E2eeType.PER_PARTICIPANT,
     },

src/utils/test.ts

@@ -216,6 +216,7 @@ export async function withLocalMedia(
       kind: E2eeType.PER_PARTICIPANT,
     },
     mockLivekitRoom({ localParticipant }),
+    {} as unknown as MediaDevices,
     of(roomMember.rawDisplayName ?? "nodisplayname"),
     of(null),
     of(null),