Merge pull request #987 from vector-im/calls-mute-1-fix-camera-lights

Calls: Switch camera lights off when muting

src/domain/session/room/CallViewModel.ts

@@ -136,7 +136,7 @@ class OwnMemberViewModel extends ViewModel<Options> implements IStreamViewModel
     }
 
     get stream(): Stream | undefined {
-        return this.call.localMedia?.userMediaPreview;
+        return this.call.localPreviewMedia?.userMedia;
     }
 
     private get call(): GroupCall {

src/matrix/calls/LocalMedia.ts

@@ -20,24 +20,12 @@ import {SDPStreamMetadata} from "./callEventTypes";
 import {getStreamVideoTrack, getStreamAudioTrack} from "./common";
 
 export class LocalMedia {
-    // the userMedia stream without audio, to play in the UI
-    // without our own audio being played back to us
-    public readonly userMediaPreview?: Stream;
 
     constructor(
         public readonly userMedia?: Stream,
         public readonly screenShare?: Stream,
         public readonly dataChannelOptions?: RTCDataChannelInit,
-    ) {
+    ) {}
-        if (userMedia && userMedia.getVideoTracks().length > 0) {
-            this.userMediaPreview = userMedia.clone();
-            const audioTrack = getStreamAudioTrack(this.userMediaPreview);
-            if (audioTrack) {
-                audioTrack.stop();
-                this.userMediaPreview.removeTrack(audioTrack);
-            }
-        }
-    }
 
     withUserMedia(stream: Stream) {
         return new LocalMedia(stream, this.screenShare, this.dataChannelOptions);
@@ -51,6 +39,22 @@ export class LocalMedia {
         return new LocalMedia(this.userMedia, this.screenShare, options);
     }
 
+    /**
+     * Create an instance of LocalMedia without audio track (for user preview)
+     */
+    asPreview(): LocalMedia {
+        const media = new LocalMedia(this.userMedia, this.screenShare, this.dataChannelOptions);
+        const userMedia = media.userMedia;
+        if (userMedia && userMedia.getVideoTracks().length > 0) {
+            const audioTrack = getStreamAudioTrack(userMedia);
+            if (audioTrack) {
+                audioTrack.stop();
+                userMedia.removeTrack(audioTrack);
+            }
+        }
+        return media;
+    }
+
     /** @internal */
     replaceClone(oldClone: LocalMedia | undefined, oldOriginal: LocalMedia | undefined): LocalMedia {
         const cloneOrAdoptStream = (oldOriginalStream: Stream | undefined, oldCloneStream: Stream | undefined, newStream: Stream | undefined): Stream | undefined => {
@@ -76,7 +80,6 @@ export class LocalMedia {
     dispose() {
         getStreamAudioTrack(this.userMedia)?.stop();
         getStreamVideoTrack(this.userMedia)?.stop();
-        getStreamVideoTrack(this.userMediaPreview)?.stop();
         getStreamVideoTrack(this.screenShare)?.stop();
     }
 }

src/matrix/calls/PeerCall.ts

@@ -20,7 +20,7 @@ import {recursivelyAssign} from "../../utils/recursivelyAssign";
 import {Disposables, Disposable, IDisposable} from "../../utils/Disposables";
 import {WebRTC, PeerConnection, Transceiver, TransceiverDirection, Sender, Receiver, PeerConnectionEventMap} from "../../platform/types/WebRTC";
 import {MediaDevices, Track, TrackKind, Stream, StreamTrackEvent} from "../../platform/types/MediaDevices";
-import {getStreamVideoTrack, getStreamAudioTrack, MuteSettings} from "./common";
+import {getStreamVideoTrack, getStreamAudioTrack, MuteSettings, mute} from "./common";
 import {
     SDPStreamMetadataKey,
     SDPStreamMetadataPurpose,
@@ -266,14 +266,7 @@ export class PeerCall implements IDisposable {
             log.set("microphoneMuted", localMuteSettings.microphone);
 
             if (this.localMedia) {
-                const userMediaAudio = getStreamAudioTrack(this.localMedia.userMedia);
+                mute(this.localMedia, localMuteSettings, log);
-                if (userMediaAudio) {
-                    this.muteTrack(userMediaAudio, this.localMuteSettings.microphone, log);
-                }
-                const userMediaVideo = getStreamVideoTrack(this.localMedia.userMedia);
-                if (userMediaVideo) {
-                    this.muteTrack(userMediaVideo, this.localMuteSettings.camera, log);
-                }
                 const content: MCallSDPStreamMetadataChanged<MCallBase> = {
                     call_id: this.callId,
                     version: 1,
@@ -290,22 +283,6 @@ export class PeerCall implements IDisposable {
         });
     }
 
-    private muteTrack(track: Track, muted: boolean, log: ILogItem): void {
-        log.wrap({l: "track", kind: track.kind, id: track.id}, log => {
-            const enabled = !muted;
-            log.set("enabled", enabled);
-            const transceiver = this.findTransceiverForTrack(track);
-            if (transceiver) {
-                if (transceiver.sender.track) {
-                    transceiver.sender.track.enabled = enabled;
-                }
-                log.set("fromDirection", transceiver.direction);
-                // enableSenderOnTransceiver(transceiver, enabled);
-                log.set("toDirection", transceiver.direction);
-            }
-        });
-    }
-
     private async _hangup(errorCode: CallErrorCode, log: ILogItem): Promise<void> {
         if (this._state === CallState.Ended) {
             return;

src/matrix/calls/common.ts

@@ -14,7 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+import {ILogItem} from "../../logging/types";
 import type {Track, Stream} from "../../platform/types/MediaDevices";
+import {LocalMedia} from "./LocalMedia";
 
 export function getStreamAudioTrack(stream: Stream | undefined): Track | undefined {
     return stream?.getAudioTracks()[0];
@@ -24,6 +26,29 @@ export function getStreamVideoTrack(stream: Stream | undefined): Track | undefin
     return stream?.getVideoTracks()[0];
 }
 
+export function mute(localMedia: LocalMedia, localMuteSettings: MuteSettings, log: ILogItem) {
+        return log.wrap("mute", log => {
+            log.set("cameraMuted", localMuteSettings.camera);
+            log.set("microphoneMuted", localMuteSettings.microphone);
+
+            // Mute audio
+            const userMediaAudio = getStreamAudioTrack(localMedia.userMedia);
+            if (userMediaAudio) {
+                const enabled = !localMuteSettings.microphone;
+                log.set("microphone enabled", enabled);
+                userMediaAudio.enabled = enabled;
+            }
+
+            // Mute video
+            const userMediaVideo = getStreamVideoTrack(localMedia.userMedia);
+            if (userMediaVideo) {
+                const enabled = !localMuteSettings.camera;
+                log.set("camera enabled", enabled);
+                userMediaVideo.enabled = enabled;
+            }
+        });
+}
+
 export class MuteSettings {
     constructor (
         private readonly isMicrophoneMuted: boolean = false,

src/matrix/calls/group/GroupCall.ts

@@ -17,7 +17,7 @@ limitations under the License.
 import {ObservableMap} from "../../../observable/map/ObservableMap";
 import {Member, isMemberExpired, memberExpiresAt} from "./Member";
 import {LocalMedia} from "../LocalMedia";
-import {MuteSettings, CALL_LOG_TYPE, CALL_MEMBER_VALIDITY_PERIOD_MS} from "../common";
+import {MuteSettings, CALL_LOG_TYPE, CALL_MEMBER_VALIDITY_PERIOD_MS, mute} from "../common";
 import {MemberChange, RoomMember} from "../../room/members/RoomMember";
 import {EventEmitter} from "../../../utils/EventEmitter";
 import {EventType, CallIntent} from "../callEventTypes";
@@ -72,12 +72,14 @@ class JoinedData {
         public readonly logItem: ILogItem,
         public readonly membersLogItem: ILogItem,
         public localMedia: LocalMedia,
+        public localPreviewMedia: LocalMedia,
         public localMuteSettings: MuteSettings,
         public readonly turnServer: BaseObservableValue<RTCIceServer>
     ) {}
 
     dispose() {
         this.localMedia.dispose();
+        this.localPreviewMedia.dispose();
         this.logItem.finish();
         this.renewMembershipTimeout?.dispose();
     }
@@ -125,6 +127,7 @@ export class GroupCall extends EventEmitter<{change: never}> {
     }
 
     get localMedia(): LocalMedia | undefined { return this.joinedData?.localMedia; }
+    get localPreviewMedia(): LocalMedia | undefined { return this.joinedData?.localPreviewMedia; }
     get members(): BaseObservableMap<string, Member> { return this._members; }
 
     get isTerminated(): boolean {
@@ -165,10 +168,12 @@ export class GroupCall extends EventEmitter<{change: never}> {
         const membersLogItem = logItem.child("member connections");
         const localMuteSettings = new MuteSettings();
         localMuteSettings.updateTrackInfo(localMedia.userMedia);
+        const localPreviewMedia = localMedia.asPreview();
         const joinedData = new JoinedData(
             logItem,
             membersLogItem,
             localMedia,
+            localPreviewMedia,
             localMuteSettings,
             turnServer
         );
@@ -195,6 +200,8 @@ export class GroupCall extends EventEmitter<{change: never}> {
         if ((this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) && this.joinedData) {
             const oldMedia = this.joinedData.localMedia;
             this.joinedData.localMedia = localMedia;
+            this.joinedData.localPreviewMedia?.dispose();
+            this.joinedData.localPreviewMedia = localMedia.asPreview();
             // reflect the fact we gained or lost local tracks in the local mute settings
             // and update the track info so PeerCall can use it to send up to date metadata,
             this.joinedData.localMuteSettings.updateTrackInfo(localMedia.userMedia);
@@ -219,6 +226,14 @@ export class GroupCall extends EventEmitter<{change: never}> {
         muteSettings.updateTrackInfo(joinedData.localMedia.userMedia);
         joinedData.localMuteSettings = muteSettings;
         if (!prevMuteSettings.equals(muteSettings)) {
+            // Mute our copies of LocalMedias;
+            // otherwise the camera lights will still be on.
+            if (this.localPreviewMedia) {
+                mute(this.localPreviewMedia, muteSettings, this.joinedData!.logItem);
+            }
+            if (this.localMedia) {
+                mute(this.localMedia, muteSettings, this.joinedData!.logItem);
+            }
             await Promise.all(Array.from(this._members.values()).map(m => {
                 return m.setMuted(joinedData.localMuteSettings);
             }));