From f6c94ecc5aa514d5d1d42557a36e97579daebb2a Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Mon, 16 Jan 2023 13:41:47 +0530 Subject: [PATCH 1/4] WIP --- src/domain/session/room/CallViewModel.ts | 2 +- src/matrix/calls/LocalMedia.ts | 31 +++++++++++++----------- src/matrix/calls/PeerCall.ts | 27 ++------------------- src/matrix/calls/common.ts | 25 +++++++++++++++++++ src/matrix/calls/group/GroupCall.ts | 15 +++++++++++- 5 files changed, 59 insertions(+), 41 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 9c0eae9a..37f30840 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -136,7 +136,7 @@ class OwnMemberViewModel extends ViewModel implements IStreamViewModel } get stream(): Stream | undefined { - return this.call.localMedia?.userMediaPreview; + return this.call.localPreviewMedia?.userMedia; } private get call(): GroupCall { diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 25d6862f..7a673ba9 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -20,24 +20,12 @@ import {SDPStreamMetadata} from "./callEventTypes"; import {getStreamVideoTrack, getStreamAudioTrack} from "./common"; export class LocalMedia { - // the userMedia stream without audio, to play in the UI - // without our own audio being played back to us - public readonly userMediaPreview?: Stream; constructor( public readonly userMedia?: Stream, public readonly screenShare?: Stream, public readonly dataChannelOptions?: RTCDataChannelInit, - ) { - if (userMedia && userMedia.getVideoTracks().length > 0) { - this.userMediaPreview = userMedia.clone(); - const audioTrack = getStreamAudioTrack(this.userMediaPreview); - if (audioTrack) { - audioTrack.stop(); - this.userMediaPreview.removeTrack(audioTrack); - } - } - } + ) {} withUserMedia(stream: Stream) { return new LocalMedia(stream, this.screenShare, this.dataChannelOptions); @@ -51,6 +39,22 @@ export class LocalMedia { return new LocalMedia(this.userMedia, this.screenShare, options); } + /** + * Create an instance of LocalMedia without audio track (for user preview) + */ + asPreview(): LocalMedia { + const media = new LocalMedia(this.userMedia, this.screenShare, this.dataChannelOptions); + const userMedia = media.userMedia; + if (userMedia && userMedia.getVideoTracks().length > 0) { + const audioTrack = getStreamAudioTrack(userMedia); + if (audioTrack) { + audioTrack.stop(); + userMedia.removeTrack(audioTrack); + } + } + return media; + } + /** @internal */ replaceClone(oldClone: LocalMedia | undefined, oldOriginal: LocalMedia | undefined): LocalMedia { const cloneOrAdoptStream = (oldOriginalStream: Stream | undefined, oldCloneStream: Stream | undefined, newStream: Stream | undefined): Stream | undefined => { @@ -76,7 +80,6 @@ export class LocalMedia { dispose() { getStreamAudioTrack(this.userMedia)?.stop(); getStreamVideoTrack(this.userMedia)?.stop(); - getStreamVideoTrack(this.userMediaPreview)?.stop(); getStreamVideoTrack(this.screenShare)?.stop(); } } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index a64a1524..ae200ea2 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -20,7 +20,7 @@ import {recursivelyAssign} from "../../utils/recursivelyAssign"; import {Disposables, Disposable, IDisposable} from "../../utils/Disposables"; import {WebRTC, PeerConnection, Transceiver, TransceiverDirection, Sender, Receiver, PeerConnectionEventMap} from "../../platform/types/WebRTC"; import {MediaDevices, Track, TrackKind, Stream, StreamTrackEvent} from "../../platform/types/MediaDevices"; -import {getStreamVideoTrack, getStreamAudioTrack, MuteSettings} from "./common"; +import {getStreamVideoTrack, getStreamAudioTrack, MuteSettings, mute} from "./common"; import { SDPStreamMetadataKey, SDPStreamMetadataPurpose, @@ -266,14 +266,7 @@ export class PeerCall implements IDisposable { log.set("microphoneMuted", localMuteSettings.microphone); if (this.localMedia) { - const userMediaAudio = getStreamAudioTrack(this.localMedia.userMedia); - if (userMediaAudio) { - this.muteTrack(userMediaAudio, this.localMuteSettings.microphone, log); - } - const userMediaVideo = getStreamVideoTrack(this.localMedia.userMedia); - if (userMediaVideo) { - this.muteTrack(userMediaVideo, this.localMuteSettings.camera, log); - } + mute(this.localMedia, localMuteSettings, log); const content: MCallSDPStreamMetadataChanged = { call_id: this.callId, version: 1, @@ -290,22 +283,6 @@ export class PeerCall implements IDisposable { }); } - private muteTrack(track: Track, muted: boolean, log: ILogItem): void { - log.wrap({l: "track", kind: track.kind, id: track.id}, log => { - const enabled = !muted; - log.set("enabled", enabled); - const transceiver = this.findTransceiverForTrack(track); - if (transceiver) { - if (transceiver.sender.track) { - transceiver.sender.track.enabled = enabled; - } - log.set("fromDirection", transceiver.direction); - // enableSenderOnTransceiver(transceiver, enabled); - log.set("toDirection", transceiver.direction); - } - }); - } - private async _hangup(errorCode: CallErrorCode, log: ILogItem): Promise { if (this._state === CallState.Ended) { return; diff --git a/src/matrix/calls/common.ts b/src/matrix/calls/common.ts index 015e0df8..b99ed303 100644 --- a/src/matrix/calls/common.ts +++ b/src/matrix/calls/common.ts @@ -14,7 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */ +import {ILogItem} from "../../logging/types"; import type {Track, Stream} from "../../platform/types/MediaDevices"; +import {LocalMedia} from "./LocalMedia"; export function getStreamAudioTrack(stream: Stream | undefined): Track | undefined { return stream?.getAudioTracks()[0]; @@ -24,6 +26,29 @@ export function getStreamVideoTrack(stream: Stream | undefined): Track | undefin return stream?.getVideoTracks()[0]; } +export function mute(localMedia: LocalMedia, localMuteSettings: MuteSettings, log: ILogItem) { + return log.wrap("setMuted", log => { + log.set("cameraMuted", localMuteSettings.camera); + log.set("microphoneMuted", localMuteSettings.microphone); + + // Mute audio + const userMediaAudio = getStreamAudioTrack(localMedia.userMedia); + if (userMediaAudio) { + const enabled = !localMuteSettings.microphone; + log.set("microphone enabled", enabled); + userMediaAudio.enabled = enabled; + } + + // Mute video + const userMediaVideo = getStreamVideoTrack(localMedia.userMedia); + if (userMediaVideo) { + const enabled = !localMuteSettings.camera; + log.set("camera enabled", enabled); + userMediaVideo.enabled = enabled; + } + }); +} + export class MuteSettings { constructor ( private readonly isMicrophoneMuted: boolean = false, diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index d65ea838..79f0e66c 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -17,7 +17,7 @@ limitations under the License. import {ObservableMap} from "../../../observable/map/ObservableMap"; import {Member, isMemberExpired, memberExpiresAt} from "./Member"; import {LocalMedia} from "../LocalMedia"; -import {MuteSettings, CALL_LOG_TYPE, CALL_MEMBER_VALIDITY_PERIOD_MS} from "../common"; +import {MuteSettings, CALL_LOG_TYPE, CALL_MEMBER_VALIDITY_PERIOD_MS, mute} from "../common"; import {MemberChange, RoomMember} from "../../room/members/RoomMember"; import {EventEmitter} from "../../../utils/EventEmitter"; import {EventType, CallIntent} from "../callEventTypes"; @@ -72,12 +72,14 @@ class JoinedData { public readonly logItem: ILogItem, public readonly membersLogItem: ILogItem, public localMedia: LocalMedia, + public localPreviewMedia: LocalMedia, public localMuteSettings: MuteSettings, public readonly turnServer: BaseObservableValue ) {} dispose() { this.localMedia.dispose(); + this.localPreviewMedia.dispose(); this.logItem.finish(); this.renewMembershipTimeout?.dispose(); } @@ -125,6 +127,7 @@ export class GroupCall extends EventEmitter<{change: never}> { } get localMedia(): LocalMedia | undefined { return this.joinedData?.localMedia; } + get localPreviewMedia(): LocalMedia | undefined { return this.joinedData?.localPreviewMedia; } get members(): BaseObservableMap { return this._members; } get isTerminated(): boolean { @@ -165,10 +168,12 @@ export class GroupCall extends EventEmitter<{change: never}> { const membersLogItem = logItem.child("member connections"); const localMuteSettings = new MuteSettings(); localMuteSettings.updateTrackInfo(localMedia.userMedia); + const localPreviewMedia = localMedia.asPreview(); const joinedData = new JoinedData( logItem, membersLogItem, localMedia, + localPreviewMedia, localMuteSettings, turnServer ); @@ -219,6 +224,14 @@ export class GroupCall extends EventEmitter<{change: never}> { muteSettings.updateTrackInfo(joinedData.localMedia.userMedia); joinedData.localMuteSettings = muteSettings; if (!prevMuteSettings.equals(muteSettings)) { + // Mute our copies of LocalMedias; + // otherwise the camera lights will still be on. + if (this.localPreviewMedia) { + mute(this.localPreviewMedia, muteSettings, this.joinedData!.logItem); + } + if (this.localMedia) { + mute(this.localMedia, muteSettings, this.joinedData!.logItem); + } await Promise.all(Array.from(this._members.values()).map(m => { return m.setMuted(joinedData.localMuteSettings); })); From c064336e359a2d271a30a6af3aa73e8eeb83c035 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Mon, 16 Jan 2023 15:28:50 +0530 Subject: [PATCH 2/4] Create localPreviewMedia in GroupCall.setMedia --- src/matrix/calls/group/GroupCall.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 79f0e66c..f3f37a2b 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -200,6 +200,8 @@ export class GroupCall extends EventEmitter<{change: never}> { if ((this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) && this.joinedData) { const oldMedia = this.joinedData.localMedia; this.joinedData.localMedia = localMedia; + this.localPreviewMedia?.dispose(); + this.joinedData.localPreviewMedia = localMedia.asPreview(); // reflect the fact we gained or lost local tracks in the local mute settings // and update the track info so PeerCall can use it to send up to date metadata, this.joinedData.localMuteSettings.updateTrackInfo(localMedia.userMedia); From ce5c5779bdf59f4ab13182c7c7b7b81b4d6af556 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 17 Jan 2023 14:24:20 +0530 Subject: [PATCH 3/4] Set mute -> mute --- src/matrix/calls/common.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/common.ts b/src/matrix/calls/common.ts index b99ed303..5036eae4 100644 --- a/src/matrix/calls/common.ts +++ b/src/matrix/calls/common.ts @@ -27,7 +27,7 @@ export function getStreamVideoTrack(stream: Stream | undefined): Track | undefin } export function mute(localMedia: LocalMedia, localMuteSettings: MuteSettings, log: ILogItem) { - return log.wrap("setMuted", log => { + return log.wrap("mute", log => { log.set("cameraMuted", localMuteSettings.camera); log.set("microphoneMuted", localMuteSettings.microphone); From 1cd6540476dfa80b9b6e53d7ed83213d4079fe64 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 17 Jan 2023 14:32:02 +0530 Subject: [PATCH 4/4] Don't use getter here --- src/matrix/calls/group/GroupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index f3f37a2b..cda5a40f 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -200,7 +200,7 @@ export class GroupCall extends EventEmitter<{change: never}> { if ((this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) && this.joinedData) { const oldMedia = this.joinedData.localMedia; this.joinedData.localMedia = localMedia; - this.localPreviewMedia?.dispose(); + this.joinedData.localPreviewMedia?.dispose(); this.joinedData.localPreviewMedia = localMedia.asPreview(); // reflect the fact we gained or lost local tracks in the local mute settings // and update the track info so PeerCall can use it to send up to date metadata,