Merge branch 'bwindels/calls' into thirdroom/dev

src/domain/session/room/CallViewModel.ts

@@ -67,14 +67,6 @@ export class CallViewModel extends ViewModel<Options> {
         }
     }
 
-    get isCameraMuted(): boolean {
-        return this.call.muteSettings.camera;
-    }
-
-    get isMicrophoneMuted(): boolean {
-        return this.call.muteSettings.microphone;
-    }
-
     async toggleVideo() {
         this.call.setMuted(this.call.muteSettings.toggleCamera());
     }
@@ -95,11 +87,11 @@ class OwnMemberViewModel extends ViewModel<OwnMemberOptions> implements IStreamV
     }
 
     get isCameraMuted(): boolean {
-        return this.call.muteSettings.camera ?? !!getStreamVideoTrack(this.stream);
+        return isMuted(this.call.muteSettings.camera, !!getStreamVideoTrack(this.stream));
     }
 
     get isMicrophoneMuted(): boolean {
-        return this.call.muteSettings.microphone ?? !!getStreamAudioTrack(this.stream);
+        return isMuted(this.call.muteSettings.microphone, !!getStreamAudioTrack(this.stream));
     }
 
     get avatarLetter(): string {
@@ -135,11 +127,11 @@ export class CallMemberViewModel extends ViewModel<MemberOptions> implements ISt
     }
 
     get isCameraMuted(): boolean {
-        return this.member.remoteMuteSettings?.camera ?? !getStreamVideoTrack(this.stream);
+        return isMuted(this.member.remoteMuteSettings?.camera, !!getStreamVideoTrack(this.stream));
     }
 
     get isMicrophoneMuted(): boolean {
-        return this.member.remoteMuteSettings?.microphone ?? !getStreamAudioTrack(this.stream);
+        return isMuted(this.member.remoteMuteSettings?.microphone, !!getStreamAudioTrack(this.stream));
     }
 
     get avatarLetter(): string {
@@ -178,3 +170,11 @@ export interface IStreamViewModel extends AvatarSource, ViewModel {
     get isCameraMuted(): boolean;
     get isMicrophoneMuted(): boolean;
 }
+
+function isMuted(muted: boolean | undefined, hasTrack: boolean) {
+    if (muted) {
+        return true;
+    } else {
+        return !hasTrack;
+    }
+}

src/matrix/calls/PeerCall.ts

@@ -184,6 +184,7 @@ export class PeerCall implements IDisposable {
             }
             // after adding the local tracks, and wait for handleNegotiation to be called,
             // or invite glare where we give up our invite and answer instead
+            // TODO: we don't actually use this
             await this.waitForState([CallState.InviteSent, CallState.CreateAnswer]);
         });
     }
@@ -902,6 +903,9 @@ export class PeerCall implements IDisposable {
 
 
     private onRemoteTrack(track: Track, streams: ReadonlyArray<Stream>, log: ILogItem) {
+        log.set("kind", track.kind);
+        log.set("id", track.id);
+        log.set("streams", streams.map(s => s.id));
         if (streams.length === 0) {
             log.log({l: `ignoring ${track.kind} streamless track`, id: track.id});
             return;
@@ -932,39 +936,49 @@ export class PeerCall implements IDisposable {
                 disposeListener,
                 stream
             });
-            this.updateRemoteMedia(log);
         }
+        this.updateRemoteMedia(log);
     }
 
     private updateRemoteMedia(log: ILogItem): void {
-        this._remoteMedia.userMedia = undefined;
+        log.wrap("reevaluating remote media", log => {
-        this._remoteMedia.screenShare = undefined;
+            this._remoteMedia.userMedia = undefined;
-        if (this.remoteSDPStreamMetadata) {
+            this._remoteMedia.screenShare = undefined;
-            for (const streamDetails of this._remoteStreams.values()) {
+            if (this.remoteSDPStreamMetadata) {
-                const {stream} = streamDetails;
+                for (const streamDetails of this._remoteStreams.values()) {
-                const metaData = this.remoteSDPStreamMetadata[stream.id];
+                    const {stream} = streamDetails;
-                if (metaData) {
+                    const metaData = this.remoteSDPStreamMetadata[stream.id];
-                    if (metaData.purpose === SDPStreamMetadataPurpose.Usermedia) {
+                    if (metaData) {
-                        this._remoteMedia.userMedia = stream;
+                        if (metaData.purpose === SDPStreamMetadataPurpose.Usermedia) {
-                        const audioReceiver = this.findReceiverForStream(TrackKind.Audio, stream.id);
+                            this._remoteMedia.userMedia = stream;
-                        if (audioReceiver) {
+                            const audioReceiver = this.findReceiverForStream(TrackKind.Audio, stream.id);
-                            audioReceiver.track.enabled = !metaData.audio_muted;
+                            if (audioReceiver) {
+                                audioReceiver.track.enabled = !metaData.audio_muted;
+                            }
+                            const videoReceiver = this.findReceiverForStream(TrackKind.Video, stream.id);
+                            if (videoReceiver) {
+                                videoReceiver.track.enabled = !metaData.video_muted;
+                            }
+                            this._remoteMuteSettings = new MuteSettings(
+                                metaData.audio_muted || !audioReceiver?.track,
+                                metaData.video_muted || !videoReceiver?.track
+                            );
+                            log.log({
+                                l: "setting userMedia",
+                                micMuted: this._remoteMuteSettings.microphone,
+                                cameraMuted: this._remoteMuteSettings.camera
+                            });
+                        } else if (metaData.purpose === SDPStreamMetadataPurpose.Screenshare) {
+                            this._remoteMedia.screenShare = stream;
+                            log.log("setting screenShare");
                         }
-                        const videoReceiver = this.findReceiverForStream(TrackKind.Video, stream.id);
+                    } else {
-                        if (videoReceiver) {
+                        log.log({l: "no metadata yet for stream, ignoring for now", id: stream.id});
-                            videoReceiver.track.enabled = !metaData.video_muted;
-                        }
-                        this._remoteMuteSettings = new MuteSettings(
-                            metaData.audio_muted || !audioReceiver?.track,
-                            metaData.video_muted || !videoReceiver?.track
-                        );
-                    } else if (metaData.purpose === SDPStreamMetadataPurpose.Screenshare) {
-                        this._remoteMedia.screenShare = stream;
                     }
                 }
             }
-        }
+            this.options.emitUpdate(this, undefined);
-        this.options.emitUpdate(this, undefined);
+        });
     }
 
     private updateLocalMedia(localMedia: LocalMedia, logItem: ILogItem): Promise<void> {

src/matrix/calls/TODO.md

@@ -14,7 +14,7 @@
  - implement muting tracks with m.call.sdp_stream_metadata_changed
  - implement renegotiation
  - making logging better
- - finish session id support
+ - DONE: finish session id support
     - call peers are essentially identified by (userid, deviceid, sessionid). If see a new session id, we first disconnect from the current member so we're ready to connect with a clean slate again (in a member event, also in to_device? no harm I suppose, given olm encryption ensures you can't spoof the deviceid).
  - implement to_device messages arriving before m.call(.member) state event
  - reeable crypto & implement fetching olm keys before sending encrypted signalling message

src/matrix/calls/group/GroupCall.ts

@@ -65,6 +65,7 @@ export class GroupCall extends EventEmitter<{change: never}> {
     private _memberOptions: MemberOptions;
     private _state: GroupCallState;
     private localMuteSettings: MuteSettings = new MuteSettings(false, false);
+    private bufferedDeviceMessages = new Map<string, Set<SignallingMessage<MGroupCallBase>>>();
 
     private _deviceIndex?: number;
     private _eventTimestamp?: number;
@@ -235,7 +236,7 @@ export class GroupCall extends EventEmitter<{change: never}> {
                 const device = devices[deviceIndex];
                 const deviceId = device.device_id;
                 const memberKey = getMemberKey(userId, deviceId);
-                log.wrap({l: "update device member", id: memberKey}, log => {
+                log.wrap({l: "update device member", id: memberKey, sessionId: device.session_id}, log => {
                     if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) {
 
                         this._deviceIndex = deviceIndex;
@@ -250,7 +251,7 @@ export class GroupCall extends EventEmitter<{change: never}> {
                         let member = this._members.get(memberKey);
                         if (member) {
                             log.set("update", true);
-                            member!.updateCallInfo(device, deviceIndex, eventTimestamp);
+                            member!.updateCallInfo(device, deviceIndex, eventTimestamp, log);
                         } else {
                             const logItem = this.logItem.child({l: "member", id: memberKey});
                             log.set("add", true);
@@ -265,6 +266,9 @@ export class GroupCall extends EventEmitter<{change: never}> {
                                 member.connect(this._localMedia!.clone(), this.localMuteSettings);
                             }
                         }
+                        // flush pending messages, either after having created the member,
+                        // or updated the session id with updateCallInfo
+                        this.flushPendingDeviceMessages(member, log);
                     }
                 });
             }
@@ -298,6 +302,23 @@ export class GroupCall extends EventEmitter<{change: never}> {
         });
     }
 
+    private flushPendingDeviceMessages(member: Member, log: ILogItem) {
+        const memberKey = getMemberKey(member.userId, member.deviceId);
+        const bufferedMessages = this.bufferedDeviceMessages.get(memberKey);
+        // check if we have any pending message for the member with (userid, deviceid, sessionid)
+        if (bufferedMessages) {
+            for (const message of bufferedMessages) {
+                if (message.content.sender_session_id === member.sessionId) {
+                    member.handleDeviceMessage(message, log);
+                    bufferedMessages.delete(message);
+                }
+            }
+            if (bufferedMessages.size === 0) {
+                this.bufferedDeviceMessages.delete(memberKey);
+            }
+        }
+    }
+
     private getDeviceIdsForUserId(userId: string): string[] {
         return Array.from(this._members.keys())
             .filter(key => memberKeyIsForUser(key, userId))
@@ -338,13 +359,27 @@ export class GroupCall extends EventEmitter<{change: never}> {
     /** @internal */
     handleDeviceMessage(message: SignallingMessage<MGroupCallBase>, userId: string, deviceId: string, syncLog: ILogItem) {
         // TODO: return if we are not membering to the call
-        let member = this._members.get(getMemberKey(userId, deviceId));
+        const key = getMemberKey(userId, deviceId);
-        if (member) {
+        let member = this._members.get(key);
-            member.handleDeviceMessage(message, deviceId, syncLog);
+        if (member && message.content.sender_session_id === member.sessionId) {
+            member.handleDeviceMessage(message, syncLog);
         } else {
-            const item = this.logItem.log({l: "could not find member for signalling message", userId, deviceId});
+            const item = this.logItem.log({
+                l: "buffering to_device message, member not found",
+                userId,
+                deviceId,
+                sessionId: message.content.sender_session_id,
+                type: message.type
+            });
             syncLog.refDetached(item);
-            // we haven't received the m.call.member yet for this caller. buffer the device messages or create the member/call anyway?
+            // we haven't received the m.call.member yet for this caller (or with this session id).
+            // buffer the device messages or create the member/call as it should arrive in a moment
+            let messages = this.bufferedDeviceMessages.get(key);
+            if (!messages) {
+                messages = new Set();
+                this.bufferedDeviceMessages.set(key, messages);
+            }
+            messages.add(message);
         }
     }
 

src/matrix/calls/group/Member.ts

@@ -33,6 +33,7 @@ export type Options = Omit<PeerCallOptions, "emitUpdate" | "sendSignallingMessag
     confId: string,
     ownUserId: string,
     ownDeviceId: string,
+    // local session id of our client
     sessionId: string,
     hsApi: HomeServerApi,
     encryptDeviceMessage: (userId: string, message: SignallingMessage<MGroupCallBase>, log: ILogItem) => Promise<EncryptedMessage>,
@@ -83,6 +84,11 @@ export class Member {
         return this.callDeviceMembership.device_id;
     }
 
+    /** session id of the member */
+    get sessionId(): string {
+        return this.callDeviceMembership.session_id;
+    }
+
     get dataChannel(): any | undefined {
         return this.peerCall?.dataChannel;
     }
@@ -127,14 +133,35 @@ export class Member {
             this.peerCall = undefined;
             this.localMedia?.dispose();
             this.localMedia = undefined;
+            this.retryCount = 0;
         });
     }
 
     /** @internal */
-    updateCallInfo(callDeviceMembership: CallDeviceMembership, deviceIndex: number, eventTimestamp: number) {
+    updateCallInfo(callDeviceMembership: CallDeviceMembership, deviceIndex: number, eventTimestamp: number,  log: ILogItem) {
-        this.callDeviceMembership = callDeviceMembership;
+        log.wrap({l: "updateing device membership", deviceId: this.deviceId}, log => {
-        this._deviceIndex = deviceIndex;
+            // session id is changing, disconnect so we start with a new slate for the new session
-        this._eventTimestamp = eventTimestamp;
+            if (callDeviceMembership.session_id !== this.sessionId) {
+                log.wrap({
+                    l: "member event changes session id",
+                    oldSessionId: this.sessionId,
+                    newSessionId: callDeviceMembership.session_id
+                }, log => {
+                    // prevent localMedia from being stopped
+                    // as connect won't be called again when reconnecting
+                    // to the new session
+                    const localMedia = this.localMedia;
+                    this.localMedia = undefined;
+                    this.disconnect(false);
+                    // connect again, as the other side might be waiting for our invite
+                    // after refreshing
+                    this.connect(localMedia!, this.localMuteSettings!);
+                });
+            }
+            this.callDeviceMembership = callDeviceMembership;
+            this._deviceIndex = deviceIndex;
+            this._eventTimestamp = eventTimestamp;
+        });
     }
 
     /** @internal */
@@ -163,7 +190,7 @@ export class Member {
         groupMessage.content.device_id = this.options.ownDeviceId;
         groupMessage.content.party_id = this.options.ownDeviceId;
         groupMessage.content.sender_session_id = this.options.sessionId;
-        groupMessage.content.dest_session_id = this.callDeviceMembership.session_id;
+        groupMessage.content.dest_session_id = this.sessionId;
         // const encryptedMessages = await this.options.encryptDeviceMessage(this.member.userId, groupMessage, log);
         // const payload = formatToDeviceMessagesPayload(encryptedMessages);
         const payload = {
@@ -186,7 +213,7 @@ export class Member {
     }
 
     /** @internal */
-    handleDeviceMessage(message: SignallingMessage<MGroupCallBase>, deviceId: string, syncLog: ILogItem) {
+    handleDeviceMessage(message: SignallingMessage<MGroupCallBase>, syncLog: ILogItem): void {
         syncLog.refDetached(this.logItem);
         const destSessionId = message.content.dest_session_id;
         if (destSessionId !== this.options.sessionId) {
@@ -197,7 +224,7 @@ export class Member {
             this.peerCall = this._createPeerCall(message.content.call_id);
         }
         if (this.peerCall) {
-            this.peerCall.handleIncomingSignallingMessage(message, deviceId);
+            this.peerCall.handleIncomingSignallingMessage(message, this.deviceId);
         } else {
             // TODO: need to buffer events until invite comes?
         }

src/platform/web/ui/css/themes/element/theme.css

@@ -1197,6 +1197,7 @@ button.RoomDetailsView_row::after {
 .StreamView_muteStatus {
     align-self: end;
     justify-self: start;
+    color: var(--text-color--lighter-80);
 }
 
 .StreamView_muteStatus.microphoneMuted::before {