From 46ebd55092ad26dfbb822a2abd4b4f70a5f491cf Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Mon, 14 Feb 2022 17:14:21 +0100 Subject: [PATCH 001/323] WIP --- src/matrix/DeviceMessageHandler.js | 26 ++++- src/matrix/Session.js | 6 +- src/matrix/calls/CallHandler.ts | 156 ++++++++++++++++++++++++++++ src/matrix/e2ee/DecryptionResult.ts | 8 ++ src/matrix/room/Room.js | 22 ++++ src/utils/LRUCache.ts | 2 +- 6 files changed, 216 insertions(+), 4 deletions(-) create mode 100644 src/matrix/calls/CallHandler.ts diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index 6ac5ac07..c6bce31e 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -16,12 +16,15 @@ limitations under the License. import {OLM_ALGORITHM} from "./e2ee/common.js"; import {countBy, groupBy} from "../utils/groupBy"; +import {LRUCache} from "../../utils/LRUCache"; export class DeviceMessageHandler { - constructor({storage}) { + constructor({storage, callHandler}) { this._storage = storage; this._olmDecryption = null; this._megolmDecryption = null; + this._callHandler = callHandler; + this._senderDeviceCache = new LRUCache(10, di => di.curve25519Key); } enableEncryption({olmDecryption, megolmDecryption}) { @@ -49,6 +52,15 @@ export class DeviceMessageHandler { log.child("decrypt_error").catch(err); } const newRoomKeys = this._megolmDecryption.roomKeysFromDeviceMessages(olmDecryptChanges.results, log); + const callMessages = olmDecryptChanges.results.filter(dr => this._callHandler.handlesDeviceMessageEventType(dr.event?.type)); + await Promise.all(callMessages.map(async dr => { + dr.setDevice(await this._getDevice(dr.senderCurve25519Key, txn)); + this._callHandler.handleDeviceMessage(dr.device.userId, dr.device.deviceId, dr.event.type, dr.event.content, log); + })); + // TODO: somehow include rooms that received a call to_device message in the sync state? + // or have updates flow through event emitter? + // well, we don't really need to update the room other then when a call starts or stops + // any changes within the call will be emitted on the call object? return new SyncPreparation(olmDecryptChanges, newRoomKeys); } } @@ -60,6 +72,18 @@ export class DeviceMessageHandler { const didWriteValues = await Promise.all(prep.newRoomKeys.map(key => this._megolmDecryption.writeRoomKey(key, txn))); return didWriteValues.some(didWrite => !!didWrite); } + + + async _getDevice(senderKey, txn) { + let device = this._senderDeviceCache.get(senderKey); + if (!device) { + device = await txn.deviceIdentities.getByCurve25519Key(senderKey); + if (device) { + this._senderDeviceCache.set(device); + } + } + return device; + } } class SyncPreparation { diff --git a/src/matrix/Session.js b/src/matrix/Session.js index ae1dea61..3d9b13c8 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -73,7 +73,7 @@ export class Session { }; this._roomsBeingCreated = new ObservableMap(); this._user = new User(sessionInfo.userId); - this._deviceMessageHandler = new DeviceMessageHandler({storage}); + this._deviceMessageHandler = new DeviceMessageHandler({storage, callHandler: this._callHandler}); this._olm = olm; this._olmUtil = null; this._e2eeAccount = null; @@ -100,6 +100,7 @@ export class Session { this._createRoomEncryption = this._createRoomEncryption.bind(this); this._forgetArchivedRoom = this._forgetArchivedRoom.bind(this); this.needsKeyBackup = new ObservableValue(false); + this._callHandler = new CallHandler(this._platform, this._hsApi); } get fingerprintKey() { @@ -562,7 +563,8 @@ export class Session { pendingEvents, user: this._user, createRoomEncryption: this._createRoomEncryption, - platform: this._platform + platform: this._platform, + callHandler: this._callHandler }); } diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts new file mode 100644 index 00000000..55571c5a --- /dev/null +++ b/src/matrix/calls/CallHandler.ts @@ -0,0 +1,156 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {ObservableMap} from "../../observable/map/ObservableMap"; + +import type {Room} from "../room/Room"; +import type {StateEvent} from "../storage/types"; +import type {ILogItem} from "../../logging/types"; + +const GROUP_CALL_TYPE = "m.call"; +const GROUP_CALL_MEMBER_TYPE = "m.call.member"; + +enum CallSetupMessageType { + Invite = "m.call.invite", + Answer = "m.call.answer", + Candidates = "m.call.candidates", + Hangup = "m.call.hangup", +} + +const CALL_ID = "m.call_id"; +const CALL_TERMINATED = "m.terminated"; + +export class CallHandler { + public readonly groupCalls: ObservableMap = new ObservableMap(); + + constructor() { + + } + + handleRoomState(room: Room, events: StateEvent[], log: ILogItem) { + // first update call events + for (const event of events) { + if (event.type === GROUP_CALL_TYPE) { + const callId = event.state_key; + let call = this.groupCalls.get(callId); + if (call) { + call.updateCallEvent(event); + if (call.isTerminated) { + this.groupCalls.remove(call.id); + } + } else { + call = new GroupCall(event, room); + this.groupCalls.set(call.id, call); + } + } + } + // then update participants + for (const event of events) { + if (event.type === GROUP_CALL_MEMBER_TYPE) { + const participant = event.state_key; + const sources = event.content["m.sources"]; + for (const source of sources) { + const call = this.groupCalls.get(source[CALL_ID]); + if (call && !call.isTerminated) { + call.addParticipant(participant, source); + } + } + } + } + } + + handlesDeviceMessageEventType(eventType: string | undefined): boolean { + return eventType === CallSetupMessageType.Invite || + eventType === CallSetupMessageType.Candidates || + eventType === CallSetupMessageType.Answer || + eventType === CallSetupMessageType.Hangup; + } + + handleDeviceMessage(senderUserId: string, senderDeviceId: string, eventType: string, content: Record, log: ILogItem) { + const callId = content[CALL_ID]; + const call = this.groupCalls.get(callId); + call?.handleDeviceMessage(senderUserId, senderDeviceId, eventType, content, log); + } +} + +function peerCallKey(senderUserId: string, senderDeviceId: string) { + return JSON.stringify(senderUserId) + JSON.stringify(senderDeviceId); +} + +class GroupCall { + private peerCalls: Map + + constructor(private callEvent: StateEvent, private readonly room: Room) { + + } + + updateCallEvent(callEvent: StateEvent) { + this.callEvent = callEvent; + } + + addParticipant(userId, source) { + + } + + handleDeviceMessage(senderUserId: string, senderDeviceId: string, eventType: string, content: Record, log: ILogItem) { + const peerCall = this.peerCalls.get(peerCallKey(senderUserId, senderDeviceId)); + peerCall?.handleIncomingSignallingMessage() + } + + get id(): string { + return this.callEvent.state_key; + } + + get isTerminated(): boolean { + return !!this.callEvent.content[CALL_TERMINATED]; + } + + private createPeerCall(userId: string, deviceId: string): PeerCall { + + } +} + +/** + * Does WebRTC signalling for a single PeerConnection, and deals with WebRTC wrappers from platform + * */ + + +// when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already +// do for sharing keys will be best as that already deals with room tracking. +type SendSignallingMessageCallback = (type: CallSetupMessageType, content: Record) => Promise; + +class PeerCall { + constructor(private readonly sendSignallingMessage: SendSignallingMessageCallback) { + + } + + handleIncomingSignallingMessage(type: CallSetupMessageType, content: Record) { + switch (type) { + case CallSetupMessageType.Invite: + case CallSetupMessageType.Answer: + case CallSetupMessageType.Candidates: + case CallSetupMessageType.Hangup: + } + } +} + +class MediaSource { + +} + +class PeerConnection { + +} diff --git a/src/matrix/e2ee/DecryptionResult.ts b/src/matrix/e2ee/DecryptionResult.ts index 7735856a..8fe1a6ed 100644 --- a/src/matrix/e2ee/DecryptionResult.ts +++ b/src/matrix/e2ee/DecryptionResult.ts @@ -69,6 +69,14 @@ export class DecryptionResult { } } + get userId(): string | undefined { + return this.device?.userId; + } + + get deviceId(): string | undefined { + return this.device?.deviceId; + } + get isVerificationUnknown(): boolean { // verification is unknown if we haven't yet fetched the devices for the room return !this.device && !this.roomTracked; diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js index 12c17580..b9ec82a3 100644 --- a/src/matrix/room/Room.js +++ b/src/matrix/room/Room.js @@ -30,6 +30,7 @@ const EVENT_ENCRYPTED_TYPE = "m.room.encrypted"; export class Room extends BaseRoom { constructor(options) { super(options); + this._callHandler = options.callHandler; // TODO: pass pendingEvents to start like pendingOperations? const {pendingEvents} = options; const relationWriter = new RelationWriter({ @@ -92,6 +93,8 @@ export class Room extends BaseRoom { } } + this._updateCallHandler(roomResponse); + return { roomEncryption, summaryChanges, @@ -442,6 +445,25 @@ export class Room extends BaseRoom { return this._sendQueue.pendingEvents; } + _updateCallHandler(roomResponse) { + if (this._callHandler) { + const stateEvents = roomResponse.state?.events; + if (stateEvents) { + for (const e of stateEvents) { + this._callHandler.handleRoomState(this, e); + } + } + let timelineEvents = roomResponse.timeline?.events; + if (timelineEvents) { + for (const e of timelineEvents) { + if (typeof e.state_key === "string") { + this._callHandler.handleRoomState(this, e); + } + } + } + } + } + /** @package */ writeIsTrackingMembers(value, txn) { return this._summary.writeIsTrackingMembers(value, txn); diff --git a/src/utils/LRUCache.ts b/src/utils/LRUCache.ts index c5a7cd06..bab9bf51 100644 --- a/src/utils/LRUCache.ts +++ b/src/utils/LRUCache.ts @@ -71,7 +71,7 @@ export class BaseLRUCache { export class LRUCache extends BaseLRUCache { private _keyFn: (T) => K; - constructor(limit, keyFn: (T) => K) { + constructor(limit: number, keyFn: (T) => K) { super(limit); this._keyFn = keyFn; } From b12bc52c4a96448e0332f58ede8c57526e233a33 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Tue, 15 Feb 2022 17:05:20 +0100 Subject: [PATCH 002/323] WIP2 --- src/matrix/calls/Call.ts | 2493 ++++++++++++++++++++++++++++ src/matrix/calls/CallFeed.ts | 274 +++ src/matrix/calls/CallHandler.ts | 1 + src/matrix/calls/callEventTypes.ts | 92 + 4 files changed, 2860 insertions(+) create mode 100644 src/matrix/calls/Call.ts create mode 100644 src/matrix/calls/CallFeed.ts create mode 100644 src/matrix/calls/callEventTypes.ts diff --git a/src/matrix/calls/Call.ts b/src/matrix/calls/Call.ts new file mode 100644 index 00000000..d6ac4612 --- /dev/null +++ b/src/matrix/calls/Call.ts @@ -0,0 +1,2493 @@ +/* +Copyright 2015, 2016 OpenMarket Ltd +Copyright 2017 New Vector Ltd +Copyright 2019, 2020 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { + MCallReplacesEvent, + MCallAnswer, + MCallInviteNegotiate, + CallCapabilities, + SDPStreamMetadataPurpose, + SDPStreamMetadata, + SDPStreamMetadataKey, + MCallSDPStreamMetadataChanged, + MCallSelectAnswer, + MCAllAssertedIdentity, + MCallCandidates, + MCallBase, + MCallHangupReject, +} from './callEventTypes'; +import { CallFeed } from './CallFeed'; + +// events: hangup, error(err), replaced(call), state(state, oldState) + +/** + * Fires whenever an error occurs when call.js encounters an issue with setting up the call. + *

+ * The error given will have a code equal to either `MatrixCall.ERR_LOCAL_OFFER_FAILED` or + * `MatrixCall.ERR_NO_USER_MEDIA`. `ERR_LOCAL_OFFER_FAILED` is emitted when the local client + * fails to create an offer. `ERR_NO_USER_MEDIA` is emitted when the user has denied access + * to their audio/video hardware. + * + * @event module:webrtc/call~MatrixCall#"error" + * @param {Error} err The error raised by MatrixCall. + * @example + * matrixCall.on("error", function(err){ + * console.error(err.code, err); + * }); + */ + +interface CallOpts { + roomId?: string; + invitee?: string; + client?: any; // Fix when client is TSified + forceTURN?: boolean; + turnServers?: Array; + opponentDeviceId?: string; + opponentSessionId?: string; + groupCallId?: string; +} + +interface TurnServer { + urls: Array; + username?: string; + password?: string; + ttl?: number; +} + +interface AssertedIdentity { + id: string; + displayName: string; +} + +export enum CallState { + Fledgling = 'fledgling', + InviteSent = 'invite_sent', + WaitLocalMedia = 'wait_local_media', + CreateOffer = 'create_offer', + CreateAnswer = 'create_answer', + Connecting = 'connecting', + Connected = 'connected', + Ringing = 'ringing', + Ended = 'ended', +} + +export enum CallType { + Voice = 'voice', + Video = 'video', +} + +export enum CallDirection { + Inbound = 'inbound', + Outbound = 'outbound', +} + +export enum CallParty { + Local = 'local', + Remote = 'remote', +} + +export enum CallEvent { + Hangup = 'hangup', + State = 'state', + Error = 'error', + Replaced = 'replaced', + + // The value of isLocalOnHold() has changed + LocalHoldUnhold = 'local_hold_unhold', + // The value of isRemoteOnHold() has changed + RemoteHoldUnhold = 'remote_hold_unhold', + // backwards compat alias for LocalHoldUnhold: remove in a major version bump + HoldUnhold = 'hold_unhold', + // Feeds have changed + FeedsChanged = 'feeds_changed', + + AssertedIdentityChanged = 'asserted_identity_changed', + + LengthChanged = 'length_changed', + + DataChannel = 'datachannel', + + SendVoipEvent = "send_voip_event", +} + +export enum CallErrorCode { + /** The user chose to end the call */ + UserHangup = 'user_hangup', + + /** An error code when the local client failed to create an offer. */ + LocalOfferFailed = 'local_offer_failed', + /** + * An error code when there is no local mic/camera to use. This may be because + * the hardware isn't plugged in, or the user has explicitly denied access. + */ + NoUserMedia = 'no_user_media', + + /** + * Error code used when a call event failed to send + * because unknown devices were present in the room + */ + UnknownDevices = 'unknown_devices', + + /** + * Error code used when we fail to send the invite + * for some reason other than there being unknown devices + */ + SendInvite = 'send_invite', + + /** + * An answer could not be created + */ + CreateAnswer = 'create_answer', + + /** + * Error code used when we fail to send the answer + * for some reason other than there being unknown devices + */ + SendAnswer = 'send_answer', + + /** + * The session description from the other side could not be set + */ + SetRemoteDescription = 'set_remote_description', + + /** + * The session description from this side could not be set + */ + SetLocalDescription = 'set_local_description', + + /** + * A different device answered the call + */ + AnsweredElsewhere = 'answered_elsewhere', + + /** + * No media connection could be established to the other party + */ + IceFailed = 'ice_failed', + + /** + * The invite timed out whilst waiting for an answer + */ + InviteTimeout = 'invite_timeout', + + /** + * The call was replaced by another call + */ + Replaced = 'replaced', + + /** + * Signalling for the call could not be sent (other than the initial invite) + */ + SignallingFailed = 'signalling_timeout', + + /** + * The remote party is busy + */ + UserBusy = 'user_busy', + + /** + * We transferred the call off to somewhere else + */ + Transfered = 'transferred', + + /** + * A call from the same user was found with a new session id + */ + NewSession = 'new_session', +} + +/** + * The version field that we set in m.call.* events + */ +const VOIP_PROTO_VERSION = 1; + +/** The fallback ICE server to use for STUN or TURN protocols. */ +const FALLBACK_ICE_SERVER = 'stun:turn.matrix.org'; + +/** The length of time a call can be ringing for. */ +const CALL_TIMEOUT_MS = 60000; + +const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); + +export class CallError extends Error { + code: string; + + constructor(code: CallErrorCode, msg: string, err: Error) { + // Still don't think there's any way to have proper nested errors + super(msg + ": " + err); + + this.code = code; + } +} + +export function genCallID(): string { + return Date.now().toString() + randomString(16); +} + +/** + * Construct a new Matrix Call. + * @constructor + * @param {Object} opts Config options. + * @param {string} opts.roomId The room ID for this call. + * @param {Object} opts.webRtc The WebRTC globals from the browser. + * @param {boolean} opts.forceTURN whether relay through TURN should be forced. + * @param {Object} opts.URL The URL global. + * @param {Array} opts.turnServers Optional. A list of TURN servers. + * @param {MatrixClient} opts.client The Matrix Client instance to send events to. + */ +export class MatrixCall extends EventEmitter { + public roomId: string; + public callId: string; + public invitee?: string; + public state = CallState.Fledgling; + public hangupParty: CallParty; + public hangupReason: string; + public direction: CallDirection; + public ourPartyId: string; + public peerConn?: RTCPeerConnection; + + private client: MatrixClient; + private forceTURN: boolean; + private turnServers: Array; + // A queue for candidates waiting to go out. + // We try to amalgamate candidates into a single candidate message where + // possible + private candidateSendQueue: Array = []; + private candidateSendTries = 0; + private sentEndOfCandidates = false; + private feeds: Array = []; + private usermediaSenders: Array = []; + private screensharingSenders: Array = []; + private inviteOrAnswerSent = false; + private waitForLocalAVStream: boolean; + private successor: MatrixCall; + private opponentMember: RoomMember; + private opponentVersion: number | string; + // The party ID of the other side: undefined if we haven't chosen a partner + // yet, null if we have but they didn't send a party ID. + private opponentPartyId: string; + private opponentCaps: CallCapabilities; + private inviteTimeout: number; + private iceDisconnectedTimeout: number; + + // The logic of when & if a call is on hold is nontrivial and explained in is*OnHold + // This flag represents whether we want the other party to be on hold + private remoteOnHold = false; + + // the stats for the call at the point it ended. We can't get these after we + // tear the call down, so we just grab a snapshot before we stop the call. + // The typescript definitions have this type as 'any' :( + private callStatsAtEnd: any[]; + + // Perfect negotiation state: https://www.w3.org/TR/webrtc/#perfect-negotiation-example + private makingOffer = false; + private ignoreOffer: boolean; + + private responsePromiseChain?: Promise; + + // If candidates arrive before we've picked an opponent (which, in particular, + // will happen if the opponent sends candidates eagerly before the user answers + // the call) we buffer them up here so we can then add the ones from the party we pick + private remoteCandidateBuffer = new Map(); + + private remoteAssertedIdentity: AssertedIdentity; + + private remoteSDPStreamMetadata: SDPStreamMetadata; + + private callLengthInterval: number; + private callLength = 0; + + private opponentDeviceId: string; + private opponentSessionId: string; + public groupCallId: string; + + constructor(opts: CallOpts) { + super(); + this.roomId = opts.roomId; + this.invitee = opts.invitee; + this.client = opts.client; + this.forceTURN = opts.forceTURN; + this.ourPartyId = this.client.deviceId; + this.opponentDeviceId = opts.opponentDeviceId; + this.opponentSessionId = opts.opponentSessionId; + this.groupCallId = opts.groupCallId; + // Array of Objects with urls, username, credential keys + this.turnServers = opts.turnServers || []; + if (this.turnServers.length === 0 && this.client.isFallbackICEServerAllowed()) { + this.turnServers.push({ + urls: [FALLBACK_ICE_SERVER], + }); + } + for (const server of this.turnServers) { + utils.checkObjectHasKeys(server, ["urls"]); + } + this.callId = genCallID(); + } + + /** + * Place a voice call to this room. + * @throws If you have not specified a listener for 'error' events. + */ + public async placeVoiceCall(): Promise { + await this.placeCall(true, false); + } + + /** + * Place a video call to this room. + * @throws If you have not specified a listener for 'error' events. + */ + public async placeVideoCall(): Promise { + await this.placeCall(true, true); + } + + /** + * Create a datachannel using this call's peer connection. + * @param label A human readable label for this datachannel + * @param options An object providing configuration options for the data channel. + */ + public createDataChannel(label: string, options: RTCDataChannelInit) { + const dataChannel = this.peerConn.createDataChannel(label, options); + this.emit(CallEvent.DataChannel, dataChannel); + return dataChannel; + } + + public getOpponentMember(): RoomMember { + return this.opponentMember; + } + + public getOpponentSessionId(): string { + return this.opponentSessionId; + } + + public opponentCanBeTransferred(): boolean { + return Boolean(this.opponentCaps && this.opponentCaps["m.call.transferee"]); + } + + public opponentSupportsDTMF(): boolean { + return Boolean(this.opponentCaps && this.opponentCaps["m.call.dtmf"]); + } + + public getRemoteAssertedIdentity(): AssertedIdentity { + return this.remoteAssertedIdentity; + } + + public get type(): CallType { + return (this.hasLocalUserMediaVideoTrack || this.hasRemoteUserMediaVideoTrack) + ? CallType.Video + : CallType.Voice; + } + + public get hasLocalUserMediaVideoTrack(): boolean { + return this.localUsermediaStream?.getVideoTracks().length > 0; + } + + public get hasRemoteUserMediaVideoTrack(): boolean { + return this.getRemoteFeeds().some((feed) => { + return ( + feed.purpose === SDPStreamMetadataPurpose.Usermedia && + feed.stream.getVideoTracks().length > 0 + ); + }); + } + + public get hasLocalUserMediaAudioTrack(): boolean { + return this.localUsermediaStream?.getAudioTracks().length > 0; + } + + public get hasRemoteUserMediaAudioTrack(): boolean { + return this.getRemoteFeeds().some((feed) => { + return ( + feed.purpose === SDPStreamMetadataPurpose.Usermedia && + feed.stream.getAudioTracks().length > 0 + ); + }); + } + + public get localUsermediaFeed(): CallFeed { + return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); + } + + public get localScreensharingFeed(): CallFeed { + return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); + } + + public get localUsermediaStream(): MediaStream { + return this.localUsermediaFeed?.stream; + } + + public get localScreensharingStream(): MediaStream { + return this.localScreensharingFeed?.stream; + } + + public get remoteUsermediaFeed(): CallFeed { + return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); + } + + public get remoteScreensharingFeed(): CallFeed { + return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); + } + + public get remoteUsermediaStream(): MediaStream { + return this.remoteUsermediaFeed?.stream; + } + + public get remoteScreensharingStream(): MediaStream { + return this.remoteScreensharingFeed?.stream; + } + + private getFeedByStreamId(streamId: string): CallFeed { + return this.getFeeds().find((feed) => feed.stream.id === streamId); + } + + /** + * Returns an array of all CallFeeds + * @returns {Array} CallFeeds + */ + public getFeeds(): Array { + return this.feeds; + } + + /** + * Returns an array of all local CallFeeds + * @returns {Array} local CallFeeds + */ + public getLocalFeeds(): Array { + return this.feeds.filter((feed) => feed.isLocal()); + } + + /** + * Returns an array of all remote CallFeeds + * @returns {Array} remote CallFeeds + */ + public getRemoteFeeds(): Array { + return this.feeds.filter((feed) => !feed.isLocal()); + } + + /** + * Generates and returns localSDPStreamMetadata + * @returns {SDPStreamMetadata} localSDPStreamMetadata + */ + private getLocalSDPStreamMetadata(updateStreamIds = false): SDPStreamMetadata { + const metadata: SDPStreamMetadata = {}; + for (const localFeed of this.getLocalFeeds()) { + if (updateStreamIds) { + localFeed.sdpMetadataStreamId = localFeed.stream.id; + } + + metadata[localFeed.sdpMetadataStreamId] = { + purpose: localFeed.purpose, + audio_muted: localFeed.isAudioMuted(), + video_muted: localFeed.isVideoMuted(), + }; + } + return metadata; + } + + /** + * Returns true if there are no incoming feeds, + * otherwise returns false + * @returns {boolean} no incoming feeds + */ + public noIncomingFeeds(): boolean { + return !this.feeds.some((feed) => !feed.isLocal()); + } + + private pushRemoteFeed(stream: MediaStream): void { + // Fallback to old behavior if the other side doesn't support SDPStreamMetadata + if (!this.opponentSupportsSDPStreamMetadata()) { + this.pushRemoteFeedWithoutMetadata(stream); + return; + } + + const userId = this.getOpponentMember().userId; + const purpose = this.remoteSDPStreamMetadata[stream.id].purpose; + const audioMuted = this.remoteSDPStreamMetadata[stream.id].audio_muted; + const videoMuted = this.remoteSDPStreamMetadata[stream.id].video_muted; + + if (!purpose) { + logger.warn(`Ignoring stream with id ${stream.id} because we didn't get any metadata about it`); + return; + } + + // Try to find a feed with the same purpose as the new stream, + // if we find it replace the old stream with the new one + const existingFeed = this.getRemoteFeeds().find((feed) => feed.purpose === purpose); + if (existingFeed) { + existingFeed.setNewStream(stream); + } else { + this.feeds.push(new CallFeed({ + client: this.client, + roomId: this.roomId, + userId, + stream, + purpose, + audioMuted, + videoMuted, + })); + this.emit(CallEvent.FeedsChanged, this.feeds); + } + + logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}", purpose=${purpose})`); + } + + /** + * This method is used ONLY if the other client doesn't support sending SDPStreamMetadata + */ + private pushRemoteFeedWithoutMetadata(stream: MediaStream): void { + const userId = this.getOpponentMember().userId; + // We can guess the purpose here since the other client can only send one stream + const purpose = SDPStreamMetadataPurpose.Usermedia; + const oldRemoteStream = this.feeds.find((feed) => !feed.isLocal())?.stream; + + // Note that we check by ID and always set the remote stream: Chrome appears + // to make new stream objects when transceiver directionality is changed and the 'active' + // status of streams change - Dave + // If we already have a stream, check this stream has the same id + if (oldRemoteStream && stream.id !== oldRemoteStream.id) { + logger.warn(`Ignoring new stream ID ${stream.id}: we already have stream ID ${oldRemoteStream.id}`); + return; + } + + // Try to find a feed with the same stream id as the new stream, + // if we find it replace the old stream with the new one + const feed = this.getFeedByStreamId(stream.id); + if (feed) { + feed.setNewStream(stream); + } else { + this.feeds.push(new CallFeed({ + client: this.client, + roomId: this.roomId, + audioMuted: false, + videoMuted: false, + userId, + stream, + purpose, + })); + this.emit(CallEvent.FeedsChanged, this.feeds); + } + + logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}")`); + } + + private pushNewLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose, addToPeerConnection = true): void { + const userId = this.client.getUserId(); + + // TODO: Find out what is going on here + // why do we enable audio (and only audio) tracks here? -- matthew + setTracksEnabled(stream.getAudioTracks(), true); + + // We try to replace an existing feed if there already is one with the same purpose + const existingFeed = this.getLocalFeeds().find((feed) => feed.purpose === purpose); + if (existingFeed) { + existingFeed.setNewStream(stream); + } else { + this.pushLocalFeed( + new CallFeed({ + client: this.client, + roomId: this.roomId, + audioMuted: stream.getAudioTracks().length === 0, + videoMuted: stream.getVideoTracks().length === 0, + userId, + stream, + purpose, + }), + addToPeerConnection, + ); + this.emit(CallEvent.FeedsChanged, this.feeds); + } + } + + /** + * Pushes supplied feed to the call + * @param {CallFeed} callFeed to push + * @param {boolean} addToPeerConnection whether to add the tracks to the peer connection + */ + public pushLocalFeed(callFeed: CallFeed, addToPeerConnection = true): void { + if (this.feeds.some((feed) => callFeed.stream.id === feed.stream.id)) { + logger.info(`Ignoring duplicate local stream ${callFeed.stream.id} in call ${this.callId}`); + return; + } + + this.feeds.push(callFeed); + + if (addToPeerConnection) { + const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia ? + this.usermediaSenders : this.screensharingSenders; + // Empty the array + senderArray.splice(0, senderArray.length); + + for (const track of callFeed.stream.getTracks()) { + logger.info( + `Adding track (` + + `id="${track.id}", ` + + `kind="${track.kind}", ` + + `streamId="${callFeed.stream.id}", ` + + `streamPurpose="${callFeed.purpose}"` + + `) to peer connection`, + ); + senderArray.push(this.peerConn.addTrack(track, callFeed.stream)); + } + } + + logger.info( + `Pushed local stream `+ + `(id="${callFeed.stream.id}", `+ + `active="${callFeed.stream.active}", `+ + `purpose="${callFeed.purpose}")`, + ); + + this.emit(CallEvent.FeedsChanged, this.feeds); + } + + /** + * Removes local call feed from the call and its tracks from the peer + * connection + * @param callFeed to remove + */ + public removeLocalFeed(callFeed: CallFeed): void { + const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia + ? this.usermediaSenders + : this.screensharingSenders; + + for (const sender of senderArray) { + this.peerConn.removeTrack(sender); + } + + if (callFeed.purpose === SDPStreamMetadataPurpose.Screenshare) { + this.client.getMediaHandler().stopScreensharingStream(callFeed.stream); + } + + // Empty the array + senderArray.splice(0, senderArray.length); + this.deleteFeed(callFeed); + } + + private deleteAllFeeds(): void { + for (const feed of this.feeds) { + if (!feed.isLocal() || !this.groupCallId) { + feed.dispose(); + } + } + + this.feeds = []; + this.emit(CallEvent.FeedsChanged, this.feeds); + } + + private deleteFeedByStream(stream: MediaStream): void { + const feed = this.getFeedByStreamId(stream.id); + if (!feed) { + logger.warn(`Didn't find the feed with stream id ${stream.id} to delete`); + return; + } + this.deleteFeed(feed); + } + + private deleteFeed(feed: CallFeed): void { + feed.dispose(); + this.feeds.splice(this.feeds.indexOf(feed), 1); + this.emit(CallEvent.FeedsChanged, this.feeds); + } + + // The typescript definitions have this type as 'any' :( + public async getCurrentCallStats(): Promise { + if (this.callHasEnded()) { + return this.callStatsAtEnd; + } + + return this.collectCallStats(); + } + + private async collectCallStats(): Promise { + // This happens when the call fails before it starts. + // For example when we fail to get capture sources + if (!this.peerConn) return; + + const statsReport = await this.peerConn.getStats(); + const stats = []; + for (const item of statsReport) { + stats.push(item[1]); + } + + return stats; + } + + /** + * Configure this call from an invite event. Used by MatrixClient. + * @param {MatrixEvent} event The m.call.invite event + */ + public async initWithInvite(event: MatrixEvent): Promise { + const invite = event.getContent(); + this.direction = CallDirection.Inbound; + + // make sure we have valid turn creds. Unless something's gone wrong, it should + // poll and keep the credentials valid so this should be instant. + const haveTurnCreds = await this.client.checkTurnServers(); + if (!haveTurnCreds) { + logger.warn("Failed to get TURN credentials! Proceeding with call anyway..."); + } + + const sdpStreamMetadata = invite[SDPStreamMetadataKey]; + if (sdpStreamMetadata) { + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); + } else { + logger.debug("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); + } + + this.peerConn = this.createPeerConnection(); + // we must set the party ID before await-ing on anything: the call event + // handler will start giving us more call events (eg. candidates) so if + // we haven't set the party ID, we'll ignore them. + this.chooseOpponent(event); + try { + await this.peerConn.setRemoteDescription(invite.offer); + await this.addBufferedIceCandidates(); + } catch (e) { + logger.debug("Failed to set remote description", e); + this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); + return; + } + + const remoteStream = this.feeds.find((feed) => !feed.isLocal())?.stream; + + // According to previous comments in this file, firefox at some point did not + // add streams until media started arriving on them. Testing latest firefox + // (81 at time of writing), this is no longer a problem, so let's do it the correct way. + if (!remoteStream || remoteStream.getTracks().length === 0) { + logger.error("No remote stream or no tracks after setting remote description!"); + this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); + return; + } + + this.setState(CallState.Ringing); + + if (event.getLocalAge()) { + setTimeout(() => { + if (this.state == CallState.Ringing) { + logger.debug("Call invite has expired. Hanging up."); + this.hangupParty = CallParty.Remote; // effectively + this.setState(CallState.Ended); + this.stopAllMedia(); + if (this.peerConn.signalingState != 'closed') { + this.peerConn.close(); + } + this.emit(CallEvent.Hangup); + } + }, invite.lifetime - event.getLocalAge()); + } + } + + /** + * Configure this call from a hangup or reject event. Used by MatrixClient. + * @param {MatrixEvent} event The m.call.hangup event + */ + public initWithHangup(event: MatrixEvent): void { + // perverse as it may seem, sometimes we want to instantiate a call with a + // hangup message (because when getting the state of the room on load, events + // come in reverse order and we want to remember that a call has been hung up) + this.setState(CallState.Ended); + } + + private shouldAnswerWithMediaType( + wantedValue: boolean | undefined, valueOfTheOtherSide: boolean | undefined, type: "audio" | "video", + ): boolean { + if (wantedValue && !valueOfTheOtherSide) { + // TODO: Figure out how to do this + logger.warn(`Unable to answer with ${type} because the other side isn't sending it either.`); + return false; + } else if ( + !utils.isNullOrUndefined(wantedValue) && + wantedValue !== valueOfTheOtherSide && + !this.opponentSupportsSDPStreamMetadata() + ) { + logger.warn( + `Unable to answer with ${type}=${wantedValue} because the other side doesn't support it. ` + + `Answering with ${type}=${valueOfTheOtherSide}.`, + ); + return valueOfTheOtherSide; + } + return wantedValue ?? valueOfTheOtherSide; + } + + /** + * Answer a call. + */ + public async answer(audio?: boolean, video?: boolean): Promise { + if (this.inviteOrAnswerSent) return; + // TODO: Figure out how to do this + if (audio === false && video === false) throw new Error("You CANNOT answer a call without media"); + + if (!this.localUsermediaStream && !this.waitForLocalAVStream) { + const prevState = this.state; + const answerWithAudio = this.shouldAnswerWithMediaType(audio, this.hasRemoteUserMediaAudioTrack, "audio"); + const answerWithVideo = this.shouldAnswerWithMediaType(video, this.hasRemoteUserMediaVideoTrack, "video"); + + this.setState(CallState.WaitLocalMedia); + this.waitForLocalAVStream = true; + + try { + const stream = await this.client.getMediaHandler().getUserMediaStream( + answerWithAudio, answerWithVideo, + ); + this.waitForLocalAVStream = false; + const usermediaFeed = new CallFeed({ + client: this.client, + roomId: this.roomId, + userId: this.client.getUserId(), + stream, + purpose: SDPStreamMetadataPurpose.Usermedia, + audioMuted: stream.getAudioTracks().length === 0, + videoMuted: stream.getVideoTracks().length === 0, + }); + + const feeds = [usermediaFeed]; + + if (this.localScreensharingFeed) { + feeds.push(this.localScreensharingFeed); + } + + this.answerWithCallFeeds(feeds); + } catch (e) { + if (answerWithVideo) { + // Try to answer without video + logger.warn("Failed to getUserMedia(), trying to getUserMedia() without video"); + this.setState(prevState); + this.waitForLocalAVStream = false; + await this.answer(answerWithAudio, false); + } else { + this.getUserMediaFailed(e); + return; + } + } + } else if (this.waitForLocalAVStream) { + this.setState(CallState.WaitLocalMedia); + } + } + + public answerWithCallFeeds(callFeeds: CallFeed[]): void { + if (this.inviteOrAnswerSent) return; + + logger.debug(`Answering call ${this.callId}`); + + this.queueGotCallFeedsForAnswer(callFeeds); + } + + /** + * Replace this call with a new call, e.g. for glare resolution. Used by + * MatrixClient. + * @param {MatrixCall} newCall The new call. + */ + public replacedBy(newCall: MatrixCall): void { + if (this.state === CallState.WaitLocalMedia) { + logger.debug("Telling new call to wait for local media"); + newCall.waitForLocalAVStream = true; + } else if ([CallState.CreateOffer, CallState.InviteSent].includes(this.state)) { + if (newCall.direction === CallDirection.Outbound) { + newCall.queueGotCallFeedsForAnswer([]); + } else { + logger.debug("Handing local stream to new call"); + newCall.queueGotCallFeedsForAnswer(this.getLocalFeeds().map(feed => feed.clone())); + } + } + this.successor = newCall; + this.emit(CallEvent.Replaced, newCall); + this.hangup(CallErrorCode.Replaced, true); + } + + /** + * Hangup a call. + * @param {string} reason The reason why the call is being hung up. + * @param {boolean} suppressEvent True to suppress emitting an event. + */ + public hangup(reason: CallErrorCode, suppressEvent: boolean): void { + if (this.callHasEnded()) return; + + logger.debug("Ending call " + this.callId); + this.terminate(CallParty.Local, reason, !suppressEvent); + // We don't want to send hangup here if we didn't even get to sending an invite + if (this.state === CallState.WaitLocalMedia) return; + const content = {}; + // Don't send UserHangup reason to older clients + if ((this.opponentVersion && this.opponentVersion >= 1) || reason !== CallErrorCode.UserHangup) { + content["reason"] = reason; + } + this.sendVoipEvent(EventType.CallHangup, content); + } + + /** + * Reject a call + * This used to be done by calling hangup, but is a separate method and protocol + * event as of MSC2746. + */ + public reject(): void { + if (this.state !== CallState.Ringing) { + throw Error("Call must be in 'ringing' state to reject!"); + } + + if (this.opponentVersion < 1) { + logger.info( + `Opponent version is less than 1 (${this.opponentVersion}): sending hangup instead of reject`, + ); + this.hangup(CallErrorCode.UserHangup, true); + return; + } + + logger.debug("Rejecting call: " + this.callId); + this.terminate(CallParty.Local, CallErrorCode.UserHangup, true); + this.sendVoipEvent(EventType.CallReject, {}); + } + + /** + * Adds an audio and/or video track - upgrades the call + * @param {boolean} audio should add an audio track + * @param {boolean} video should add an video track + */ + private async upgradeCall( + audio: boolean, video: boolean, + ): Promise { + // We don't do call downgrades + if (!audio && !video) return; + if (!this.opponentSupportsSDPStreamMetadata()) return; + + try { + const upgradeAudio = audio && !this.hasLocalUserMediaAudioTrack; + const upgradeVideo = video && !this.hasLocalUserMediaVideoTrack; + logger.debug(`Upgrading call: audio?=${upgradeAudio} video?=${upgradeVideo}`); + + const stream = await this.client.getMediaHandler().getUserMediaStream(upgradeAudio, upgradeVideo); + if (upgradeAudio && upgradeVideo) { + if (this.hasLocalUserMediaAudioTrack) return; + if (this.hasLocalUserMediaVideoTrack) return; + + this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); + } else if (upgradeAudio) { + if (this.hasLocalUserMediaAudioTrack) return; + + const audioTrack = stream.getAudioTracks()[0]; + this.localUsermediaStream.addTrack(audioTrack); + this.peerConn.addTrack(audioTrack, this.localUsermediaStream); + } else if (upgradeVideo) { + if (this.hasLocalUserMediaVideoTrack) return; + + const videoTrack = stream.getVideoTracks()[0]; + this.localUsermediaStream.addTrack(videoTrack); + this.peerConn.addTrack(videoTrack, this.localUsermediaStream); + } + } catch (error) { + logger.error("Failed to upgrade the call", error); + this.emit(CallEvent.Error, + new CallError(CallErrorCode.NoUserMedia, "Failed to get camera access: ", error), + ); + } + } + + /** + * Returns true if this.remoteSDPStreamMetadata is defined, otherwise returns false + * @returns {boolean} can screenshare + */ + public opponentSupportsSDPStreamMetadata(): boolean { + return Boolean(this.remoteSDPStreamMetadata); + } + + /** + * If there is a screensharing stream returns true, otherwise returns false + * @returns {boolean} is screensharing + */ + public isScreensharing(): boolean { + return Boolean(this.localScreensharingStream); + } + + /** + * Starts/stops screensharing + * @param enabled the desired screensharing state + * @param {string} desktopCapturerSourceId optional id of the desktop capturer source to use + * @returns {boolean} new screensharing state + */ + public async setScreensharingEnabled( + enabled: boolean, desktopCapturerSourceId?: string, + ): Promise { + // Skip if there is nothing to do + if (enabled && this.isScreensharing()) { + logger.warn(`There is already a screensharing stream - there is nothing to do!`); + return true; + } else if (!enabled && !this.isScreensharing()) { + logger.warn(`There already isn't a screensharing stream - there is nothing to do!`); + return false; + } + + // Fallback to replaceTrack() + if (!this.opponentSupportsSDPStreamMetadata()) { + return await this.setScreensharingEnabledWithoutMetadataSupport(enabled, desktopCapturerSourceId); + } + + logger.debug(`Set screensharing enabled? ${enabled}`); + if (enabled) { + try { + const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId); + if (!stream) return false; + this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare); + return true; + } catch (err) { + logger.error("Failed to get screen-sharing stream:", err); + return false; + } + } else { + for (const sender of this.screensharingSenders) { + this.peerConn.removeTrack(sender); + } + this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream); + this.deleteFeedByStream(this.localScreensharingStream); + return false; + } + } + + /** + * Starts/stops screensharing + * Should be used ONLY if the opponent doesn't support SDPStreamMetadata + * @param enabled the desired screensharing state + * @param {string} desktopCapturerSourceId optional id of the desktop capturer source to use + * @returns {boolean} new screensharing state + */ + private async setScreensharingEnabledWithoutMetadataSupport( + enabled: boolean, desktopCapturerSourceId?: string, + ): Promise { + logger.debug(`Set screensharing enabled? ${enabled} using replaceTrack()`); + if (enabled) { + try { + const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId); + if (!stream) return false; + + const track = stream.getTracks().find((track) => { + return track.kind === "video"; + }); + const sender = this.usermediaSenders.find((sender) => { + return sender.track?.kind === "video"; + }); + sender.replaceTrack(track); + + this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare, false); + + return true; + } catch (err) { + logger.error("Failed to get screen-sharing stream:", err); + return false; + } + } else { + const track = this.localUsermediaStream.getTracks().find((track) => { + return track.kind === "video"; + }); + const sender = this.usermediaSenders.find((sender) => { + return sender.track?.kind === "video"; + }); + sender.replaceTrack(track); + + this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream); + this.deleteFeedByStream(this.localScreensharingStream); + + return false; + } + } + + /** + * Request a new local usermedia stream with the current device id. + */ + public async updateLocalUsermediaStream(stream: MediaStream) { + const callFeed = this.localUsermediaFeed; + callFeed.setNewStream(stream); + const micShouldBeMuted = callFeed.isAudioMuted() || this.remoteOnHold; + const vidShouldBeMuted = callFeed.isVideoMuted() || this.remoteOnHold; + setTracksEnabled(stream.getAudioTracks(), !micShouldBeMuted); + setTracksEnabled(stream.getVideoTracks(), !vidShouldBeMuted); + + const newSenders = []; + + for (const track of stream.getTracks()) { + const oldSender = this.usermediaSenders.find((sender) => { + return sender.track?.kind === track.kind; + }); + + let newSender: RTCRtpSender; + + try { + logger.info( + `Replacing track (` + + `id="${track.id}", ` + + `kind="${track.kind}", ` + + `streamId="${stream.id}", ` + + `streamPurpose="${callFeed.purpose}"` + + `) to peer connection`, + ); + await oldSender.replaceTrack(track); + newSender = oldSender; + } catch (error) { + logger.info( + `Adding track (` + + `id="${track.id}", ` + + `kind="${track.kind}", ` + + `streamId="${stream.id}", ` + + `streamPurpose="${callFeed.purpose}"` + + `) to peer connection`, + ); + newSender = this.peerConn.addTrack(track, stream); + } + + newSenders.push(newSender); + } + + this.usermediaSenders = newSenders; + } + + /** + * Set whether our outbound video should be muted or not. + * @param {boolean} muted True to mute the outbound video. + * @returns the new mute state + */ + public async setLocalVideoMuted(muted: boolean): Promise { + if (!await this.client.getMediaHandler().hasVideoDevice()) { + return this.isLocalVideoMuted(); + } + + if (!this.hasLocalUserMediaVideoTrack && !muted) { + await this.upgradeCall(false, true); + return this.isLocalVideoMuted(); + } + this.localUsermediaFeed?.setVideoMuted(muted); + this.updateMuteStatus(); + return this.isLocalVideoMuted(); + } + + /** + * Check if local video is muted. + * + * If there are multiple video tracks, all of the tracks need to be muted + * for this to return true. This means if there are no video tracks, this will + * return true. + * @return {Boolean} True if the local preview video is muted, else false + * (including if the call is not set up yet). + */ + public isLocalVideoMuted(): boolean { + return this.localUsermediaFeed?.isVideoMuted(); + } + + /** + * Set whether the microphone should be muted or not. + * @param {boolean} muted True to mute the mic. + * @returns the new mute state + */ + public async setMicrophoneMuted(muted: boolean): Promise { + if (!await this.client.getMediaHandler().hasAudioDevice()) { + return this.isMicrophoneMuted(); + } + + if (!this.hasLocalUserMediaAudioTrack && !muted) { + await this.upgradeCall(true, false); + return this.isMicrophoneMuted(); + } + this.localUsermediaFeed?.setAudioMuted(muted); + this.updateMuteStatus(); + return this.isMicrophoneMuted(); + } + + /** + * Check if the microphone is muted. + * + * If there are multiple audio tracks, all of the tracks need to be muted + * for this to return true. This means if there are no audio tracks, this will + * return true. + * @return {Boolean} True if the mic is muted, else false (including if the call + * is not set up yet). + */ + public isMicrophoneMuted(): boolean { + return this.localUsermediaFeed?.isAudioMuted(); + } + + /** + * @returns true if we have put the party on the other side of the call on hold + * (that is, we are signalling to them that we are not listening) + */ + public isRemoteOnHold(): boolean { + return this.remoteOnHold; + } + + public setRemoteOnHold(onHold: boolean): void { + if (this.isRemoteOnHold() === onHold) return; + this.remoteOnHold = onHold; + + for (const transceiver of this.peerConn.getTransceivers()) { + // We don't send hold music or anything so we're not actually + // sending anything, but sendrecv is fairly standard for hold and + // it makes it a lot easier to figure out who's put who on hold. + transceiver.direction = onHold ? 'sendonly' : 'sendrecv'; + } + this.updateMuteStatus(); + + this.emit(CallEvent.RemoteHoldUnhold, this.remoteOnHold); + } + + /** + * Indicates whether we are 'on hold' to the remote party (ie. if true, + * they cannot hear us). + * @returns true if the other party has put us on hold + */ + public isLocalOnHold(): boolean { + if (this.state !== CallState.Connected) return false; + + let callOnHold = true; + + // We consider a call to be on hold only if *all* the tracks are on hold + // (is this the right thing to do?) + for (const transceiver of this.peerConn.getTransceivers()) { + const trackOnHold = ['inactive', 'recvonly'].includes(transceiver.currentDirection); + + if (!trackOnHold) callOnHold = false; + } + + return callOnHold; + } + + /** + * Sends a DTMF digit to the other party + * @param digit The digit (nb. string - '#' and '*' are dtmf too) + */ + public sendDtmfDigit(digit: string): void { + for (const sender of this.peerConn.getSenders()) { + if (sender.track.kind === 'audio' && sender.dtmf) { + sender.dtmf.insertDTMF(digit); + return; + } + } + + throw new Error("Unable to find a track to send DTMF on"); + } + + private updateMuteStatus(): void { + this.sendVoipEvent(EventType.CallSDPStreamMetadataChangedPrefix, { + [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(), + }); + + const micShouldBeMuted = this.localUsermediaFeed?.isAudioMuted() || this.remoteOnHold; + const vidShouldBeMuted = this.localUsermediaFeed?.isVideoMuted() || this.remoteOnHold; + + setTracksEnabled(this.localUsermediaStream.getAudioTracks(), !micShouldBeMuted); + setTracksEnabled(this.localUsermediaStream.getVideoTracks(), !vidShouldBeMuted); + } + + private gotCallFeedsForInvite(callFeeds: CallFeed[], requestScreenshareFeed = false): void { + if (this.successor) { + this.successor.queueGotCallFeedsForAnswer(callFeeds); + return; + } + if (this.callHasEnded()) { + this.stopAllMedia(); + return; + } + + for (const feed of callFeeds) { + this.pushLocalFeed(feed); + } + + if (requestScreenshareFeed) { + this.peerConn.addTransceiver("video", { + direction: "recvonly", + }); + } + + this.setState(CallState.CreateOffer); + + logger.debug("gotUserMediaForInvite"); + // Now we wait for the negotiationneeded event + } + + private async sendAnswer(): Promise { + const answerContent = { + answer: { + sdp: this.peerConn.localDescription.sdp, + // type is now deprecated as of Matrix VoIP v1, but + // required to still be sent for backwards compat + type: this.peerConn.localDescription.type, + }, + [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(true), + } as MCallAnswer; + + answerContent.capabilities = { + 'm.call.transferee': this.client.supportsCallTransfer, + 'm.call.dtmf': false, + }; + + // We have just taken the local description from the peerConn which will + // contain all the local candidates added so far, so we can discard any candidates + // we had queued up because they'll be in the answer. + logger.info(`Discarding ${this.candidateSendQueue.length} candidates that will be sent in answer`); + this.candidateSendQueue = []; + + try { + await this.sendVoipEvent(EventType.CallAnswer, answerContent); + // If this isn't the first time we've tried to send the answer, + // we may have candidates queued up, so send them now. + this.inviteOrAnswerSent = true; + } catch (error) { + // We've failed to answer: back to the ringing state + this.setState(CallState.Ringing); + this.client.cancelPendingEvent(error.event); + + let code = CallErrorCode.SendAnswer; + let message = "Failed to send answer"; + if (error.name == 'UnknownDeviceError') { + code = CallErrorCode.UnknownDevices; + message = "Unknown devices present in the room"; + } + this.emit(CallEvent.Error, new CallError(code, message, error)); + throw error; + } + + // error handler re-throws so this won't happen on error, but + // we don't want the same error handling on the candidate queue + this.sendCandidateQueue(); + } + + private queueGotCallFeedsForAnswer(callFeeds: CallFeed[]): void { + // Ensure only one negotiate/answer event is being processed at a time. + if (this.responsePromiseChain) { + this.responsePromiseChain = + this.responsePromiseChain.then(() => this.gotCallFeedsForAnswer(callFeeds)); + } else { + this.responsePromiseChain = this.gotCallFeedsForAnswer(callFeeds); + } + } + + private async gotCallFeedsForAnswer(callFeeds: CallFeed[]): Promise { + if (this.callHasEnded()) return; + + this.waitForLocalAVStream = false; + + for (const feed of callFeeds) { + this.pushLocalFeed(feed); + } + + this.setState(CallState.CreateAnswer); + + let myAnswer; + try { + this.getRidOfRTXCodecs(); + myAnswer = await this.peerConn.createAnswer(); + } catch (err) { + logger.debug("Failed to create answer: ", err); + this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true); + return; + } + + try { + await this.peerConn.setLocalDescription(myAnswer); + this.setState(CallState.Connecting); + + // Allow a short time for initial candidates to be gathered + await new Promise(resolve => { + setTimeout(resolve, 200); + }); + + this.sendAnswer(); + } catch (err) { + logger.debug("Error setting local description!", err); + this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); + return; + } + + // HACK: Safari doesn't like it when we reuse MediaStreams. In most cases + // we can get around this by calling MediaStream.clone(), however inbound + // calls seem to still be broken unless we getUserMedia again and replace + // all MediaStreams using sender.replaceTrack + if (isSafari) { + await new Promise(resolve => { + setTimeout(resolve, 200); + }); + + if (this.state === CallState.Ended) { + return; + } + + const callFeed = this.localUsermediaFeed; + const stream = callFeed.stream; + + if (!stream.active) { + throw new Error(`Call ${this.callId} has an inactive stream ${ + stream.id} and its tracks cannot be replaced`); + } + + const newSenders = []; + + for (const track of this.localUsermediaStream.getTracks()) { + const oldSender = this.usermediaSenders.find((sender) => { + return sender.track?.kind === track.kind; + }); + + if (track.readyState === "ended") { + throw new Error(`Call ${this.callId} tried to replace track ${track.id} in the ended state`); + } + + let newSender: RTCRtpSender; + + try { + logger.info( + `Replacing track (` + + `id="${track.id}", ` + + `kind="${track.kind}", ` + + `streamId="${stream.id}", ` + + `streamPurpose="${callFeed.purpose}"` + + `) to peer connection`, + ); + await oldSender.replaceTrack(track); + newSender = oldSender; + } catch (error) { + logger.info( + `Adding track (` + + `id="${track.id}", ` + + `kind="${track.kind}", ` + + `streamId="${stream.id}", ` + + `streamPurpose="${callFeed.purpose}"` + + `) to peer connection`, + ); + newSender = this.peerConn.addTrack(track, stream); + } + + newSenders.push(newSender); + } + + this.usermediaSenders = newSenders; + } + } + + /** + * Internal + * @param {Object} event + */ + private gotLocalIceCandidate = (event: RTCPeerConnectionIceEvent): Promise => { + if (event.candidate) { + logger.debug( + "Call " + this.callId + " got local ICE " + event.candidate.sdpMid + " candidate: " + + event.candidate.candidate, + ); + + if (this.callHasEnded()) return; + + // As with the offer, note we need to make a copy of this object, not + // pass the original: that broke in Chrome ~m43. + if (event.candidate.candidate !== '' || !this.sentEndOfCandidates) { + this.queueCandidate(event.candidate); + + if (event.candidate.candidate === '') this.sentEndOfCandidates = true; + } + } + }; + + private onIceGatheringStateChange = (event: Event): void => { + logger.debug("ice gathering state changed to " + this.peerConn.iceGatheringState); + if (this.peerConn.iceGatheringState === 'complete' && !this.sentEndOfCandidates) { + // If we didn't get an empty-string candidate to signal the end of candidates, + // create one ourselves now gathering has finished. + // We cast because the interface lists all the properties as required but we + // only want to send 'candidate' + // XXX: We probably want to send either sdpMid or sdpMLineIndex, as it's not strictly + // correct to have a candidate that lacks both of these. We'd have to figure out what + // previous candidates had been sent with and copy them. + const c = { + candidate: '', + } as RTCIceCandidate; + this.queueCandidate(c); + this.sentEndOfCandidates = true; + } + }; + + public async onRemoteIceCandidatesReceived(ev: MatrixEvent): Promise { + if (this.callHasEnded()) { + //debuglog("Ignoring remote ICE candidate because call has ended"); + return; + } + + const content = ev.getContent(); + const candidates = content.candidates; + if (!candidates) { + logger.info("Ignoring candidates event with no candidates!"); + return; + } + + const fromPartyId = content.version === 0 ? null : content.party_id || null; + + if (this.opponentPartyId === undefined) { + // we haven't picked an opponent yet so save the candidates + logger.info(`Buffering ${candidates.length} candidates until we pick an opponent`); + const bufferedCandidates = this.remoteCandidateBuffer.get(fromPartyId) || []; + bufferedCandidates.push(...candidates); + this.remoteCandidateBuffer.set(fromPartyId, bufferedCandidates); + return; + } + + if (!this.partyIdMatches(content)) { + logger.info( + `Ignoring candidates from party ID ${content.party_id}: ` + + `we have chosen party ID ${this.opponentPartyId}`, + ); + + return; + } + + await this.addIceCandidates(candidates); + } + + /** + * Used by MatrixClient. + * @param {Object} msg + */ + public async onAnswerReceived(event: MatrixEvent): Promise { + const content = event.getContent(); + logger.debug(`Got answer for call ID ${this.callId} from party ID ${content.party_id}`); + + if (this.callHasEnded()) { + logger.debug(`Ignoring answer because call ID ${this.callId} has ended`); + return; + } + + if (this.opponentPartyId !== undefined) { + logger.info( + `Ignoring answer from party ID ${content.party_id}: ` + + `we already have an answer/reject from ${this.opponentPartyId}`, + ); + return; + } + + this.chooseOpponent(event); + await this.addBufferedIceCandidates(); + + this.setState(CallState.Connecting); + + const sdpStreamMetadata = content[SDPStreamMetadataKey]; + if (sdpStreamMetadata) { + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); + } else { + logger.warn("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); + } + + try { + await this.peerConn.setRemoteDescription(content.answer); + } catch (e) { + logger.debug("Failed to set remote description", e); + this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); + return; + } + + // If the answer we selected has a party_id, send a select_answer event + // We do this after setting the remote description since otherwise we'd block + // call setup on it + if (this.opponentPartyId !== null) { + try { + await this.sendVoipEvent(EventType.CallSelectAnswer, { + selected_party_id: this.opponentPartyId, + }); + } catch (err) { + // This isn't fatal, and will just mean that if another party has raced to answer + // the call, they won't know they got rejected, so we carry on & don't retry. + logger.warn("Failed to send select_answer event", err); + } + } + } + + public async onSelectAnswerReceived(event: MatrixEvent): Promise { + if (this.direction !== CallDirection.Inbound) { + logger.warn("Got select_answer for an outbound call: ignoring"); + return; + } + + const selectedPartyId = event.getContent().selected_party_id; + + if (selectedPartyId === undefined || selectedPartyId === null) { + logger.warn("Got nonsensical select_answer with null/undefined selected_party_id: ignoring"); + return; + } + + if (selectedPartyId !== this.ourPartyId) { + logger.info(`Got select_answer for party ID ${selectedPartyId}: we are party ID ${this.ourPartyId}.`); + // The other party has picked somebody else's answer + this.terminate(CallParty.Remote, CallErrorCode.AnsweredElsewhere, true); + } + } + + public async onNegotiateReceived(event: MatrixEvent): Promise { + const content = event.getContent(); + const description = content.description; + if (!description || !description.sdp || !description.type) { + logger.info("Ignoring invalid m.call.negotiate event"); + return; + } + // Politeness always follows the direction of the call: in a glare situation, + // we pick either the inbound or outbound call, so one side will always be + // inbound and one outbound + const polite = this.direction === CallDirection.Inbound; + + // Here we follow the perfect negotiation logic from + // https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation + const offerCollision = ( + (description.type === 'offer') && + (this.makingOffer || this.peerConn.signalingState !== 'stable') + ); + + this.ignoreOffer = !polite && offerCollision; + if (this.ignoreOffer) { + logger.info("Ignoring colliding negotiate event because we're impolite"); + return; + } + + const prevLocalOnHold = this.isLocalOnHold(); + + const sdpStreamMetadata = content[SDPStreamMetadataKey]; + if (sdpStreamMetadata) { + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); + } else { + logger.warn("Received negotiation event without SDPStreamMetadata!"); + } + + try { + await this.peerConn.setRemoteDescription(description); + + if (description.type === 'offer') { + this.getRidOfRTXCodecs(); + await this.peerConn.setLocalDescription(); + + this.sendVoipEvent(EventType.CallNegotiate, { + description: this.peerConn.localDescription, + [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(true), + }); + } + } catch (err) { + logger.warn("Failed to complete negotiation", err); + } + + const newLocalOnHold = this.isLocalOnHold(); + if (prevLocalOnHold !== newLocalOnHold) { + this.emit(CallEvent.LocalHoldUnhold, newLocalOnHold); + // also this one for backwards compat + this.emit(CallEvent.HoldUnhold, newLocalOnHold); + } + } + + private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { + this.remoteSDPStreamMetadata = utils.recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); + for (const feed of this.getRemoteFeeds()) { + const streamId = feed.stream.id; + feed.setAudioMuted(this.remoteSDPStreamMetadata[streamId]?.audio_muted); + feed.setVideoMuted(this.remoteSDPStreamMetadata[streamId]?.video_muted); + feed.purpose = this.remoteSDPStreamMetadata[streamId]?.purpose; + } + } + + public onSDPStreamMetadataChangedReceived(event: MatrixEvent): void { + const content = event.getContent(); + const metadata = content[SDPStreamMetadataKey]; + this.updateRemoteSDPStreamMetadata(metadata); + } + + public async onAssertedIdentityReceived(event: MatrixEvent): Promise { + const content = event.getContent(); + if (!content.asserted_identity) return; + + this.remoteAssertedIdentity = { + id: content.asserted_identity.id, + displayName: content.asserted_identity.display_name, + }; + this.emit(CallEvent.AssertedIdentityChanged); + } + + private callHasEnded(): boolean { + // This exists as workaround to typescript trying to be clever and erroring + // when putting if (this.state === CallState.Ended) return; twice in the same + // function, even though that function is async. + return this.state === CallState.Ended; + } + + private queueGotLocalOffer(): void { + // Ensure only one negotiate/answer event is being processed at a time. + if (this.responsePromiseChain) { + this.responsePromiseChain = + this.responsePromiseChain.then(() => this.wrappedGotLocalOffer()); + } else { + this.responsePromiseChain = this.wrappedGotLocalOffer(); + } + } + + private async wrappedGotLocalOffer(): Promise { + this.makingOffer = true; + try { + this.getRidOfRTXCodecs(); + await this.gotLocalOffer(); + } catch (e) { + this.getLocalOfferFailed(e); + return; + } finally { + this.makingOffer = false; + } + } + + private async gotLocalOffer(): Promise { + logger.debug("Setting local description"); + + if (this.callHasEnded()) { + logger.debug("Ignoring newly created offer on call ID " + this.callId + + " because the call has ended"); + return; + } + + try { + await this.peerConn.setLocalDescription(); + } catch (err) { + logger.debug("Error setting local description!", err); + this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); + return; + } + + if (this.peerConn.iceGatheringState === 'gathering') { + // Allow a short time for initial candidates to be gathered + await new Promise(resolve => { + setTimeout(resolve, 200); + }); + } + + if (this.callHasEnded()) return; + + const eventType = this.state === CallState.CreateOffer ? EventType.CallInvite : EventType.CallNegotiate; + + const content = { + lifetime: CALL_TIMEOUT_MS, + } as MCallInviteNegotiate; + + if (eventType === EventType.CallInvite && this.invitee) { + content.invitee = this.invitee; + } + + // clunky because TypeScript can't follow the types through if we use an expression as the key + if (this.state === CallState.CreateOffer) { + content.offer = this.peerConn.localDescription; + } else { + content.description = this.peerConn.localDescription; + } + + content.capabilities = { + 'm.call.transferee': this.client.supportsCallTransfer, + 'm.call.dtmf': false, + }; + + content[SDPStreamMetadataKey] = this.getLocalSDPStreamMetadata(true); + + // Get rid of any candidates waiting to be sent: they'll be included in the local + // description we just got and will send in the offer. + logger.info(`Discarding ${this.candidateSendQueue.length} candidates that will be sent in offer`); + this.candidateSendQueue = []; + + try { + await this.sendVoipEvent(eventType, content); + } catch (error) { + logger.error("Failed to send invite", error); + if (error.event) this.client.cancelPendingEvent(error.event); + + let code = CallErrorCode.SignallingFailed; + let message = "Signalling failed"; + if (this.state === CallState.CreateOffer) { + code = CallErrorCode.SendInvite; + message = "Failed to send invite"; + } + if (error.name == 'UnknownDeviceError') { + code = CallErrorCode.UnknownDevices; + message = "Unknown devices present in the room"; + } + + this.emit(CallEvent.Error, new CallError(code, message, error)); + this.terminate(CallParty.Local, code, false); + + // no need to carry on & send the candidate queue, but we also + // don't want to rethrow the error + return; + } + + this.sendCandidateQueue(); + if (this.state === CallState.CreateOffer) { + this.inviteOrAnswerSent = true; + this.setState(CallState.InviteSent); + this.inviteTimeout = setTimeout(() => { + this.inviteTimeout = null; + if (this.state === CallState.InviteSent) { + this.hangup(CallErrorCode.InviteTimeout, false); + } + }, CALL_TIMEOUT_MS); + } + } + + private getLocalOfferFailed = (err: Error): void => { + logger.error("Failed to get local offer", err); + + this.emit( + CallEvent.Error, + new CallError( + CallErrorCode.LocalOfferFailed, + "Failed to get local offer!", err, + ), + ); + this.terminate(CallParty.Local, CallErrorCode.LocalOfferFailed, false); + }; + + private getUserMediaFailed = (err: Error): void => { + if (this.successor) { + this.successor.getUserMediaFailed(err); + return; + } + + logger.warn("Failed to get user media - ending call", err); + + this.emit( + CallEvent.Error, + new CallError( + CallErrorCode.NoUserMedia, + "Couldn't start capturing media! Is your microphone set up and " + + "does this app have permission?", err, + ), + ); + this.terminate(CallParty.Local, CallErrorCode.NoUserMedia, false); + }; + + private onIceConnectionStateChanged = (): void => { + if (this.callHasEnded()) { + return; // because ICE can still complete as we're ending the call + } + logger.debug( + "Call ID " + this.callId + ": ICE connection state changed to: " + this.peerConn.iceConnectionState, + ); + // ideally we'd consider the call to be connected when we get media but + // chrome doesn't implement any of the 'onstarted' events yet + if (this.peerConn.iceConnectionState == 'connected') { + clearTimeout(this.iceDisconnectedTimeout); + this.setState(CallState.Connected); + + if (!this.callLengthInterval) { + this.callLengthInterval = setInterval(() => { + this.callLength++; + this.emit(CallEvent.LengthChanged, this.callLength); + }, 1000); + } + } else if (this.peerConn.iceConnectionState == 'failed') { + // Firefox for Android does not yet have support for restartIce() + if (this.peerConn.restartIce) { + this.peerConn.restartIce(); + } else { + this.hangup(CallErrorCode.IceFailed, false); + } + } else if (this.peerConn.iceConnectionState == 'disconnected') { + this.iceDisconnectedTimeout = setTimeout(() => { + this.hangup(CallErrorCode.IceFailed, false); + }, 30 * 1000); + } + }; + + private onSignallingStateChanged = (): void => { + logger.debug( + "call " + this.callId + ": Signalling state changed to: " + + this.peerConn.signalingState, + ); + }; + + private onTrack = (ev: RTCTrackEvent): void => { + if (ev.streams.length === 0) { + logger.warn(`Streamless ${ev.track.kind} found: ignoring.`); + return; + } + + const stream = ev.streams[0]; + this.pushRemoteFeed(stream); + stream.addEventListener("removetrack", () => { + logger.log(`Removing track streamId: ${stream.id}`); + this.deleteFeedByStream(stream); + }); + }; + + private onDataChannel = (ev: RTCDataChannelEvent): void => { + this.emit(CallEvent.DataChannel, ev.channel); + }; + + /** + * This method removes all video/rtx codecs from screensharing video + * transceivers. This is necessary since they can cause problems. Without + * this the following steps should produce an error: + * Chromium calls Firefox + * Firefox answers + * Firefox starts screen-sharing + * Chromium starts screen-sharing + * Call crashes for Chromium with: + * [96685:23:0518/162603.933321:ERROR:webrtc_video_engine.cc(3296)] RTX codec (PT=97) mapped to PT=96 which is not in the codec list. + * [96685:23:0518/162603.933377:ERROR:webrtc_video_engine.cc(1171)] GetChangedRecvParameters called without any video codecs. + * [96685:23:0518/162603.933430:ERROR:sdp_offer_answer.cc(4302)] Failed to set local video description recv parameters for m-section with mid='2'. (INVALID_PARAMETER) + */ + private getRidOfRTXCodecs(): void { + // RTCRtpReceiver.getCapabilities and RTCRtpSender.getCapabilities don't seem to be supported on FF + if (!RTCRtpReceiver.getCapabilities || !RTCRtpSender.getCapabilities) return; + + const recvCodecs = RTCRtpReceiver.getCapabilities("video").codecs; + const sendCodecs = RTCRtpSender.getCapabilities("video").codecs; + const codecs = [...sendCodecs, ...recvCodecs]; + + for (const codec of codecs) { + if (codec.mimeType === "video/rtx") { + const rtxCodecIndex = codecs.indexOf(codec); + codecs.splice(rtxCodecIndex, 1); + } + } + + for (const trans of this.peerConn.getTransceivers()) { + if ( + this.screensharingSenders.includes(trans.sender) && + ( + trans.sender.track?.kind === "video" || + trans.receiver.track?.kind === "video" + ) + ) { + trans.setCodecPreferences(codecs); + } + } + } + + private onNegotiationNeeded = async (): Promise => { + logger.info("Negotiation is needed!"); + + if (this.state !== CallState.CreateOffer && this.opponentVersion === 0) { + logger.info("Opponent does not support renegotiation: ignoring negotiationneeded event"); + return; + } + + this.queueGotLocalOffer(); + }; + + public onHangupReceived = (msg: MCallHangupReject): void => { + logger.debug("Hangup received for call ID " + this.callId); + + // party ID must match (our chosen partner hanging up the call) or be undefined (we haven't chosen + // a partner yet but we're treating the hangup as a reject as per VoIP v0) + if (this.partyIdMatches(msg) || this.state === CallState.Ringing) { + // default reason is user_hangup + this.terminate(CallParty.Remote, msg.reason || CallErrorCode.UserHangup, true); + } else { + logger.info(`Ignoring message from party ID ${msg.party_id}: our partner is ${this.opponentPartyId}`); + } + }; + + public onRejectReceived = (msg: MCallHangupReject): void => { + logger.debug("Reject received for call ID " + this.callId); + + // No need to check party_id for reject because if we'd received either + // an answer or reject, we wouldn't be in state InviteSent + + const shouldTerminate = ( + // reject events also end the call if it's ringing: it's another of + // our devices rejecting the call. + ([CallState.InviteSent, CallState.Ringing].includes(this.state)) || + // also if we're in the init state and it's an inbound call, since + // this means we just haven't entered the ringing state yet + this.state === CallState.Fledgling && this.direction === CallDirection.Inbound + ); + + if (shouldTerminate) { + this.terminate(CallParty.Remote, msg.reason || CallErrorCode.UserHangup, true); + } else { + logger.debug(`Call is in state: ${this.state}: ignoring reject`); + } + }; + + public onAnsweredElsewhere = (msg: MCallAnswer): void => { + logger.debug("Call ID " + this.callId + " answered elsewhere"); + this.terminate(CallParty.Remote, CallErrorCode.AnsweredElsewhere, true); + }; + + private setState(state: CallState): void { + const oldState = this.state; + this.state = state; + this.emit(CallEvent.State, state, oldState); + } + + /** + * Internal + * @param {string} eventType + * @param {Object} content + * @return {Promise} + */ + private sendVoipEvent(eventType: string, content: object): Promise { + const realContent = Object.assign({}, content, { + version: VOIP_PROTO_VERSION, + call_id: this.callId, + party_id: this.ourPartyId, + conf_id: this.groupCallId, + }); + + if (this.opponentDeviceId) { + this.emit(CallEvent.SendVoipEvent, { + type: "toDevice", + eventType, + userId: this.invitee || this.getOpponentMember().userId, + opponentDeviceId: this.opponentDeviceId, + content: { + ...realContent, + device_id: this.client.deviceId, + sender_session_id: this.client.getSessionId(), + dest_session_id: this.opponentSessionId, + }, + }); + + return this.client.sendToDevice(eventType, { + [this.invitee || this.getOpponentMember().userId]: { + [this.opponentDeviceId]: { + ...realContent, + device_id: this.client.deviceId, + sender_session_id: this.client.getSessionId(), + dest_session_id: this.opponentSessionId, + }, + }, + }); + } else { + this.emit(CallEvent.SendVoipEvent, { + type: "sendEvent", + eventType, + roomId: this.roomId, + content: realContent, + userId: this.invitee || this.getOpponentMember().userId, + }); + + return this.client.sendEvent(this.roomId, eventType, realContent); + } + } + + private queueCandidate(content: RTCIceCandidate): void { + // We partially de-trickle candidates by waiting for `delay` before sending them + // amalgamated, in order to avoid sending too many m.call.candidates events and hitting + // rate limits in Matrix. + // In practice, it'd be better to remove rate limits for m.call.* + + // N.B. this deliberately lets you queue and send blank candidates, which MSC2746 + // currently proposes as the way to indicate that candidate gathering is complete. + // This will hopefully be changed to an explicit rather than implicit notification + // shortly. + this.candidateSendQueue.push(content); + + // Don't send the ICE candidates yet if the call is in the ringing state: this + // means we tried to pick (ie. started generating candidates) and then failed to + // send the answer and went back to the ringing state. Queue up the candidates + // to send if we successfully send the answer. + // Equally don't send if we haven't yet sent the answer because we can send the + // first batch of candidates along with the answer + if (this.state === CallState.Ringing || !this.inviteOrAnswerSent) return; + + // MSC2746 recommends these values (can be quite long when calling because the + // callee will need a while to answer the call) + const delay = this.direction === CallDirection.Inbound ? 500 : 2000; + + if (this.candidateSendTries === 0) { + setTimeout(() => { + this.sendCandidateQueue(); + }, delay); + } + } + + /* + * Transfers this call to another user + */ + public async transfer(targetUserId: string): Promise { + // Fetch the target user's global profile info: their room avatar / displayname + // could be different in whatever room we share with them. + const profileInfo = await this.client.getProfileInfo(targetUserId); + + const replacementId = genCallID(); + + const body = { + replacement_id: genCallID(), + target_user: { + id: targetUserId, + display_name: profileInfo.displayname, + avatar_url: profileInfo.avatar_url, + }, + create_call: replacementId, + } as MCallReplacesEvent; + + await this.sendVoipEvent(EventType.CallReplaces, body); + + await this.terminate(CallParty.Local, CallErrorCode.Transfered, true); + } + + /* + * Transfers this call to the target call, effectively 'joining' the + * two calls (so the remote parties on each call are connected together). + */ + public async transferToCall(transferTargetCall?: MatrixCall): Promise { + const targetProfileInfo = await this.client.getProfileInfo(transferTargetCall.getOpponentMember().userId); + const transfereeProfileInfo = await this.client.getProfileInfo(this.getOpponentMember().userId); + + const newCallId = genCallID(); + + const bodyToTransferTarget = { + // the replacements on each side have their own ID, and it's distinct from the + // ID of the new call (but we can use the same function to generate it) + replacement_id: genCallID(), + target_user: { + id: this.getOpponentMember().userId, + display_name: transfereeProfileInfo.displayname, + avatar_url: transfereeProfileInfo.avatar_url, + }, + await_call: newCallId, + } as MCallReplacesEvent; + + await transferTargetCall.sendVoipEvent(EventType.CallReplaces, bodyToTransferTarget); + + const bodyToTransferee = { + replacement_id: genCallID(), + target_user: { + id: transferTargetCall.getOpponentMember().userId, + display_name: targetProfileInfo.displayname, + avatar_url: targetProfileInfo.avatar_url, + }, + create_call: newCallId, + } as MCallReplacesEvent; + + await this.sendVoipEvent(EventType.CallReplaces, bodyToTransferee); + + await this.terminate(CallParty.Local, CallErrorCode.Replaced, true); + await transferTargetCall.terminate(CallParty.Local, CallErrorCode.Transfered, true); + } + + private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean): Promise { + if (this.callHasEnded()) return; + + this.hangupParty = hangupParty; + this.hangupReason = hangupReason; + this.setState(CallState.Ended); + + if (this.inviteTimeout) { + clearTimeout(this.inviteTimeout); + this.inviteTimeout = null; + } + if (this.callLengthInterval) { + clearInterval(this.callLengthInterval); + this.callLengthInterval = null; + } + + this.callStatsAtEnd = await this.collectCallStats(); + + // Order is important here: first we stopAllMedia() and only then we can deleteAllFeeds() + this.stopAllMedia(); + this.deleteAllFeeds(); + + if (this.peerConn && this.peerConn.signalingState !== 'closed') { + this.peerConn.close(); + } + if (shouldEmit) { + this.emit(CallEvent.Hangup, this); + } + + this.client.callEventHandler.calls.delete(this.callId); + } + + private stopAllMedia(): void { + logger.debug(!this.groupCallId ? "Stopping all media" : "Stopping all media except local feeds" ); + for (const feed of this.feeds) { + if ( + feed.isLocal() && + feed.purpose === SDPStreamMetadataPurpose.Usermedia && + !this.groupCallId + ) { + this.client.getMediaHandler().stopUserMediaStream(feed.stream); + } else if ( + feed.isLocal() && + feed.purpose === SDPStreamMetadataPurpose.Screenshare && + !this.groupCallId + ) { + this.client.getMediaHandler().stopScreensharingStream(feed.stream); + } else if (!feed.isLocal() || !this.groupCallId) { + for (const track of feed.stream.getTracks()) { + track.stop(); + } + } + } + } + + private checkForErrorListener(): void { + if (this.listeners("error").length === 0) { + throw new Error( + "You MUST attach an error listener using call.on('error', function() {})", + ); + } + } + + private async sendCandidateQueue(): Promise { + if (this.candidateSendQueue.length === 0 || this.callHasEnded()) { + return; + } + + const candidates = this.candidateSendQueue; + this.candidateSendQueue = []; + ++this.candidateSendTries; + const content = { + candidates: candidates, + }; + logger.debug("Attempting to send " + candidates.length + " candidates"); + try { + await this.sendVoipEvent(EventType.CallCandidates, content); + // reset our retry count if we have successfully sent our candidates + // otherwise queueCandidate() will refuse to try to flush the queue + this.candidateSendTries = 0; + + // Try to send candidates again just in case we received more candidates while sending. + this.sendCandidateQueue(); + } catch (error) { + // don't retry this event: we'll send another one later as we might + // have more candidates by then. + if (error.event) this.client.cancelPendingEvent(error.event); + + // put all the candidates we failed to send back in the queue + this.candidateSendQueue.push(...candidates); + + if (this.candidateSendTries > 5) { + logger.debug( + "Failed to send candidates on attempt " + this.candidateSendTries + + ". Giving up on this call.", error, + ); + + const code = CallErrorCode.SignallingFailed; + const message = "Signalling failed"; + + this.emit(CallEvent.Error, new CallError(code, message, error)); + this.hangup(code, false); + + return; + } + + const delayMs = 500 * Math.pow(2, this.candidateSendTries); + ++this.candidateSendTries; + logger.debug("Failed to send candidates. Retrying in " + delayMs + "ms", error); + setTimeout(() => { + this.sendCandidateQueue(); + }, delayMs); + } + } + + /** + * Place a call to this room. + * @throws if you have not specified a listener for 'error' events. + * @throws if have passed audio=false. + */ + public async placeCall(audio: boolean, video: boolean): Promise { + if (!audio) { + throw new Error("You CANNOT start a call without audio"); + } + this.setState(CallState.WaitLocalMedia); + + try { + const stream = await this.client.getMediaHandler().getUserMediaStream(audio, video); + const callFeed = new CallFeed({ + client: this.client, + roomId: this.roomId, + userId: this.client.getUserId(), + stream, + purpose: SDPStreamMetadataPurpose.Usermedia, + audioMuted: stream.getAudioTracks().length === 0, + videoMuted: stream.getVideoTracks().length === 0, + }); + await this.placeCallWithCallFeeds([callFeed]); + } catch (e) { + this.getUserMediaFailed(e); + return; + } + } + + /** + * Place a call to this room with call feed. + * @param {CallFeed[]} callFeeds to use + * @throws if you have not specified a listener for 'error' events. + * @throws if have passed audio=false. + */ + public async placeCallWithCallFeeds(callFeeds: CallFeed[], requestScreenshareFeed = false): Promise { + this.checkForErrorListener(); + this.direction = CallDirection.Outbound; + + // XXX Find a better way to do this + this.client.callEventHandler.calls.set(this.callId, this); + + // make sure we have valid turn creds. Unless something's gone wrong, it should + // poll and keep the credentials valid so this should be instant. + const haveTurnCreds = await this.client.checkTurnServers(); + if (!haveTurnCreds) { + logger.warn("Failed to get TURN credentials! Proceeding with call anyway..."); + } + + // create the peer connection now so it can be gathering candidates while we get user + // media (assuming a candidate pool size is configured) + this.peerConn = this.createPeerConnection(); + this.gotCallFeedsForInvite(callFeeds, requestScreenshareFeed); + } + + private createPeerConnection(): RTCPeerConnection { + const pc = new window.RTCPeerConnection({ + iceTransportPolicy: this.forceTURN ? 'relay' : undefined, + iceServers: this.turnServers, + iceCandidatePoolSize: this.client.iceCandidatePoolSize, + }); + + // 'connectionstatechange' would be better, but firefox doesn't implement that. + pc.addEventListener('iceconnectionstatechange', this.onIceConnectionStateChanged); + pc.addEventListener('signalingstatechange', this.onSignallingStateChanged); + pc.addEventListener('icecandidate', this.gotLocalIceCandidate); + pc.addEventListener('icegatheringstatechange', this.onIceGatheringStateChange); + pc.addEventListener('track', this.onTrack); + pc.addEventListener('negotiationneeded', this.onNegotiationNeeded); + pc.addEventListener('datachannel', this.onDataChannel); + + return pc; + } + + private partyIdMatches(msg: MCallBase): boolean { + // They must either match or both be absent (in which case opponentPartyId will be null) + // Also we ignore party IDs on the invite/offer if the version is 0, so we must do the same + // here and use null if the version is 0 (woe betide any opponent sending messages in the + // same call with different versions) + const msgPartyId = msg.version === 0 ? null : msg.party_id || null; + return msgPartyId === this.opponentPartyId; + } + + // Commits to an opponent for the call + // ev: An invite or answer event + private chooseOpponent(ev: MatrixEvent): void { + // I choo-choo-choose you + const msg = ev.getContent(); + + logger.debug(`Choosing party ID ${msg.party_id} for call ID ${this.callId}`); + + this.opponentVersion = msg.version; + if (this.opponentVersion === 0) { + // set to null to indicate that we've chosen an opponent, but because + // they're v0 they have no party ID (even if they sent one, we're ignoring it) + this.opponentPartyId = null; + } else { + // set to their party ID, or if they're naughty and didn't send one despite + // not being v0, set it to null to indicate we picked an opponent with no + // party ID + this.opponentPartyId = msg.party_id || null; + } + this.opponentCaps = msg.capabilities || {} as CallCapabilities; + this.opponentMember = this.client.getRoom(this.roomId).getMember(ev.getSender()); + } + + private async addBufferedIceCandidates(): Promise { + const bufferedCandidates = this.remoteCandidateBuffer.get(this.opponentPartyId); + if (bufferedCandidates) { + logger.info(`Adding ${bufferedCandidates.length} buffered candidates for opponent ${this.opponentPartyId}`); + await this.addIceCandidates(bufferedCandidates); + } + this.remoteCandidateBuffer = null; + } + + private async addIceCandidates(candidates: RTCIceCandidate[]): Promise { + for (const candidate of candidates) { + if ( + (candidate.sdpMid === null || candidate.sdpMid === undefined) && + (candidate.sdpMLineIndex === null || candidate.sdpMLineIndex === undefined) + ) { + logger.debug("Ignoring remote ICE candidate with no sdpMid or sdpMLineIndex"); + continue; + } + logger.debug( + "Call " + this.callId + " got remote ICE " + candidate.sdpMid + " candidate: " + candidate.candidate, + ); + try { + await this.peerConn.addIceCandidate(candidate); + } catch (err) { + if (!this.ignoreOffer) { + logger.info("Failed to add remote ICE candidate", err); + } + } + } + } + + public get hasPeerConnection(): boolean { + return Boolean(this.peerConn); + } +} + +export function setTracksEnabled(tracks: Array, enabled: boolean): void { + for (let i = 0; i < tracks.length; i++) { + tracks[i].enabled = enabled; + } +} + +/** + * DEPRECATED + * Use client.createCall() + * + * Create a new Matrix call for the browser. + * @param {MatrixClient} client The client instance to use. + * @param {string} roomId The room the call is in. + * @param {Object?} options DEPRECATED optional options map. + * @param {boolean} options.forceTURN DEPRECATED whether relay through TURN should be + * forced. This option is deprecated - use opts.forceTURN when creating the matrix client + * since it's only possible to set this option on outbound calls. + * @return {MatrixCall} the call or null if the browser doesn't support calling. + */ +export function createNewMatrixCall(client: any, roomId: string, options?: CallOpts): MatrixCall { + // typeof prevents Node from erroring on an undefined reference + if (typeof(window) === 'undefined' || typeof(document) === 'undefined') { + // NB. We don't log here as apps try to create a call object as a test for + // whether calls are supported, so we shouldn't fill the logs up. + return null; + } + + // Firefox throws on so little as accessing the RTCPeerConnection when operating in + // a secure mode. There's some information at https://bugzilla.mozilla.org/show_bug.cgi?id=1542616 + // though the concern is that the browser throwing a SecurityError will brick the + // client creation process. + try { + const supported = Boolean( + window.RTCPeerConnection || window.RTCSessionDescription || + window.RTCIceCandidate || navigator.mediaDevices, + ); + if (!supported) { + // Adds a lot of noise to test runs, so disable logging there. + if (process.env.NODE_ENV !== "test") { + logger.error("WebRTC is not supported in this browser / environment"); + } + return null; + } + } catch (e) { + logger.error("Exception thrown when trying to access WebRTC", e); + return null; + } + + const optionsForceTURN = options ? options.forceTURN : false; + + const opts: CallOpts = { + client: client, + roomId: roomId, + invitee: options?.invitee, + turnServers: client.getTurnServers(), + // call level options + forceTURN: client.forceTURN || optionsForceTURN, + opponentDeviceId: options?.opponentDeviceId, + opponentSessionId: options?.opponentSessionId, + groupCallId: options?.groupCallId, + }; + const call = new MatrixCall(opts); + + client.reEmitter.reEmit(call, Object.values(CallEvent)); + + return call; +} diff --git a/src/matrix/calls/CallFeed.ts b/src/matrix/calls/CallFeed.ts new file mode 100644 index 00000000..c8cc8662 --- /dev/null +++ b/src/matrix/calls/CallFeed.ts @@ -0,0 +1,274 @@ +/* +Copyright 2021 Å imon Brandner + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { SDPStreamMetadataPurpose } from "./callEventTypes"; + +const POLLING_INTERVAL = 200; // ms +export const SPEAKING_THRESHOLD = -60; // dB +const SPEAKING_SAMPLE_COUNT = 8; // samples + +export interface ICallFeedOpts { + client: MatrixClient; + roomId: string; + userId: string; + stream: MediaStream; + purpose: SDPStreamMetadataPurpose; + audioMuted: boolean; + videoMuted: boolean; +} + +export enum CallFeedEvent { + NewStream = "new_stream", + MuteStateChanged = "mute_state_changed", + VolumeChanged = "volume_changed", + Speaking = "speaking", +} + +export class CallFeed extends EventEmitter { + public stream: MediaStream; + public sdpMetadataStreamId: string; + public userId: string; + public purpose: SDPStreamMetadataPurpose; + public speakingVolumeSamples: number[]; + + private client: MatrixClient; + private roomId: string; + private audioMuted: boolean; + private videoMuted: boolean; + private measuringVolumeActivity = false; + private audioContext: AudioContext; + private analyser: AnalyserNode; + private frequencyBinCount: Float32Array; + private speakingThreshold = SPEAKING_THRESHOLD; + private speaking = false; + private volumeLooperTimeout: number; + + constructor(opts: ICallFeedOpts) { + super(); + + this.client = opts.client; + this.roomId = opts.roomId; + this.userId = opts.userId; + this.purpose = opts.purpose; + this.audioMuted = opts.audioMuted; + this.videoMuted = opts.videoMuted; + this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); + this.sdpMetadataStreamId = opts.stream.id; + + this.updateStream(null, opts.stream); + + if (this.hasAudioTrack) { + this.initVolumeMeasuring(); + } + } + + private get hasAudioTrack(): boolean { + return this.stream.getAudioTracks().length > 0; + } + + private updateStream(oldStream: MediaStream, newStream: MediaStream): void { + if (newStream === oldStream) return; + + if (oldStream) { + oldStream.removeEventListener("addtrack", this.onAddTrack); + this.measureVolumeActivity(false); + } + if (newStream) { + this.stream = newStream; + newStream.addEventListener("addtrack", this.onAddTrack); + + if (this.hasAudioTrack) { + this.initVolumeMeasuring(); + } else { + this.measureVolumeActivity(false); + } + } + + this.emit(CallFeedEvent.NewStream, this.stream); + } + + private initVolumeMeasuring(): void { + const AudioContext = window.AudioContext || window.webkitAudioContext; + if (!this.hasAudioTrack || !AudioContext) return; + + this.audioContext = new AudioContext(); + + this.analyser = this.audioContext.createAnalyser(); + this.analyser.fftSize = 512; + this.analyser.smoothingTimeConstant = 0.1; + + const mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.stream); + mediaStreamAudioSourceNode.connect(this.analyser); + + this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount); + } + + private onAddTrack = (): void => { + this.emit(CallFeedEvent.NewStream, this.stream); + }; + + /** + * Returns callRoom member + * @returns member of the callRoom + */ + public getMember(): RoomMember { + const callRoom = this.client.getRoom(this.roomId); + return callRoom.getMember(this.userId); + } + + /** + * Returns true if CallFeed is local, otherwise returns false + * @returns {boolean} is local? + */ + public isLocal(): boolean { + return this.userId === this.client.getUserId(); + } + + /** + * Returns true if audio is muted or if there are no audio + * tracks, otherwise returns false + * @returns {boolean} is audio muted? + */ + public isAudioMuted(): boolean { + return this.stream.getAudioTracks().length === 0 || this.audioMuted; + } + + /** + * Returns true video is muted or if there are no video + * tracks, otherwise returns false + * @returns {boolean} is video muted? + */ + public isVideoMuted(): boolean { + // We assume only one video track + return this.stream.getVideoTracks().length === 0 || this.videoMuted; + } + + public isSpeaking(): boolean { + return this.speaking; + } + + /** + * Replaces the current MediaStream with a new one. + * This method should be only used by MatrixCall. + * @param newStream new stream with which to replace the current one + */ + public setNewStream(newStream: MediaStream): void { + this.updateStream(this.stream, newStream); + } + + /** + * Set feed's internal audio mute state + * @param muted is the feed's audio muted? + */ + public setAudioMuted(muted: boolean): void { + this.audioMuted = muted; + this.speakingVolumeSamples.fill(-Infinity); + this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); + } + + /** + * Set feed's internal video mute state + * @param muted is the feed's video muted? + */ + public setVideoMuted(muted: boolean): void { + this.videoMuted = muted; + this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); + } + + /** + * Starts emitting volume_changed events where the emitter value is in decibels + * @param enabled emit volume changes + */ + public measureVolumeActivity(enabled: boolean): void { + if (enabled) { + if (!this.audioContext || !this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return; + + this.measuringVolumeActivity = true; + this.volumeLooper(); + } else { + this.measuringVolumeActivity = false; + this.speakingVolumeSamples.fill(-Infinity); + this.emit(CallFeedEvent.VolumeChanged, -Infinity); + } + } + + public setSpeakingThreshold(threshold: number) { + this.speakingThreshold = threshold; + } + + private volumeLooper = () => { + if (!this.analyser) return; + + if (!this.measuringVolumeActivity) return; + + this.analyser.getFloatFrequencyData(this.frequencyBinCount); + + let maxVolume = -Infinity; + for (let i = 0; i < this.frequencyBinCount.length; i++) { + if (this.frequencyBinCount[i] > maxVolume) { + maxVolume = this.frequencyBinCount[i]; + } + } + + this.speakingVolumeSamples.shift(); + this.speakingVolumeSamples.push(maxVolume); + + this.emit(CallFeedEvent.VolumeChanged, maxVolume); + + let newSpeaking = false; + + for (let i = 0; i < this.speakingVolumeSamples.length; i++) { + const volume = this.speakingVolumeSamples[i]; + + if (volume > this.speakingThreshold) { + newSpeaking = true; + break; + } + } + + if (this.speaking !== newSpeaking) { + this.speaking = newSpeaking; + this.emit(CallFeedEvent.Speaking, this.speaking); + } + + this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL); + }; + + public clone(): CallFeed { + const mediaHandler = this.client.getMediaHandler(); + const stream = this.stream.clone(); + + if (this.purpose === SDPStreamMetadataPurpose.Usermedia) { + mediaHandler.userMediaStreams.push(stream); + } else { + mediaHandler.screensharingStreams.push(stream); + } + + return new CallFeed({ + client: this.client, + roomId: this.roomId, + userId: this.userId, + stream, + purpose: this.purpose, + audioMuted: this.audioMuted, + videoMuted: this.videoMuted, + }); + } + + public dispose(): void { + clearTimeout(this.volumeLooperTimeout); + } +} diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 55571c5a..b88c3ce2 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -34,6 +34,7 @@ const CALL_ID = "m.call_id"; const CALL_TERMINATED = "m.terminated"; export class CallHandler { + // group calls by call id public readonly groupCalls: ObservableMap = new ObservableMap(); constructor() { diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts new file mode 100644 index 00000000..cd0fbb9f --- /dev/null +++ b/src/matrix/calls/callEventTypes.ts @@ -0,0 +1,92 @@ +// allow non-camelcase as these are events type that go onto the wire +/* eslint-disable camelcase */ + +import { CallErrorCode } from "./Call"; + +// TODO: Change to "sdp_stream_metadata" when MSC3077 is merged +export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata"; + +export enum SDPStreamMetadataPurpose { + Usermedia = "m.usermedia", + Screenshare = "m.screenshare", +} + +export interface SDPStreamMetadataObject { + purpose: SDPStreamMetadataPurpose; + audio_muted: boolean; + video_muted: boolean; +} + +export interface SDPStreamMetadata { + [key: string]: SDPStreamMetadataObject; +} + +export interface CallCapabilities { + 'm.call.transferee': boolean; + 'm.call.dtmf': boolean; +} + +export interface CallReplacesTarget { + id: string; + display_name: string; + avatar_url: string; +} + +export interface MCallBase { + call_id: string; + version: string | number; + party_id?: string; + sender_session_id?: string; + dest_session_id?: string; +} + +export interface MCallAnswer extends MCallBase { + answer: RTCSessionDescription; + capabilities?: CallCapabilities; + [SDPStreamMetadataKey]: SDPStreamMetadata; +} + +export interface MCallSelectAnswer extends MCallBase { + selected_party_id: string; +} + +export interface MCallInviteNegotiate extends MCallBase { + offer: RTCSessionDescription; + description: RTCSessionDescription; + lifetime: number; + capabilities?: CallCapabilities; + invitee?: string; + sender_session_id?: string; + dest_session_id?: string; + [SDPStreamMetadataKey]: SDPStreamMetadata; +} + +export interface MCallSDPStreamMetadataChanged extends MCallBase { + [SDPStreamMetadataKey]: SDPStreamMetadata; +} + +export interface MCallReplacesEvent extends MCallBase { + replacement_id: string; + target_user: CallReplacesTarget; + create_call: string; + await_call: string; + target_room: string; +} + +export interface MCAllAssertedIdentity extends MCallBase { + asserted_identity: { + id: string; + display_name: string; + avatar_url: string; + }; +} + +export interface MCallCandidates extends MCallBase { + candidates: RTCIceCandidate[]; +} + +export interface MCallHangupReject extends MCallBase { + reason?: CallErrorCode; +} + +/* eslint-enable camelcase */ From 468841eceae09904ac05a638d64e48c509626a34 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 16 Feb 2022 17:01:02 +0100 Subject: [PATCH 003/323] WIP3 --- src/matrix/calls/CallHandler.ts | 97 +++++- src/platform/types/MediaDevices.ts | 42 +++ src/platform/types/WebRTC.ts | 55 ++++ src/platform/web/dom/MediaDevices.ts | 457 +++++++++++++++++++++++++++ src/platform/web/dom/WebRTC.ts | 414 ++++++++++++++++++++++++ 5 files changed, 1056 insertions(+), 9 deletions(-) create mode 100644 src/platform/types/MediaDevices.ts create mode 100644 src/platform/types/WebRTC.ts create mode 100644 src/platform/web/dom/MediaDevices.ts create mode 100644 src/platform/web/dom/WebRTC.ts diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index b88c3ce2..0425304e 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -20,6 +20,9 @@ import type {Room} from "../room/Room"; import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; +import {WebRTC, PeerConnection, PeerConnectionHandler, StreamPurpose} from "../../platform/types/WebRTC"; +import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; + const GROUP_CALL_TYPE = "m.call"; const GROUP_CALL_MEMBER_TYPE = "m.call.member"; @@ -129,13 +132,96 @@ class GroupCall { * */ +class LocalMedia { + private tracks = new Map(); + + setTracks(tracks: Track[]) { + for (const track of tracks) { + this.setTrack(track); + } + } + + setTrack(track: Track) { + let cameraAndMicStreamDontMatch = false; + if (track.type === TrackType.Microphone) { + const {cameraTrack} = this; + if (cameraTrack && track.streamId !== cameraTrack.streamId) { + cameraAndMicStreamDontMatch = true; + } + } + if (track.type === TrackType.Camera) { + const {microphoneTrack} = this; + if (microphoneTrack && track.streamId !== microphoneTrack.streamId) { + cameraAndMicStreamDontMatch = true; + } + } + if (cameraAndMicStreamDontMatch) { + throw new Error("The camera and audio track should have the same stream id"); + } + this.tracks.set(track.type, track); + } + + public get cameraTrack(): Track | undefined { return this.tracks.get(TrackType.Camera); }; + public get screenShareTrack(): Track | undefined { return this.tracks.get(TrackType.ScreenShare); }; + public get microphoneTrack(): AudioTrack | undefined { return this.tracks.get(TrackType.Microphone) as (AudioTrack | undefined); }; + + getSDPMetadata(): any { + const metadata = {}; + const userMediaTrack = this.microphoneTrack ?? this.cameraTrack; + if (userMediaTrack) { + metadata[userMediaTrack.streamId] = { + purpose: StreamPurpose.UserMedia + }; + } + if (this.screenShareTrack) { + metadata[this.screenShareTrack.streamId] = { + purpose: StreamPurpose.ScreenShare + }; + } + return metadata; + } +} + // when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already // do for sharing keys will be best as that already deals with room tracking. type SendSignallingMessageCallback = (type: CallSetupMessageType, content: Record) => Promise; -class PeerCall { - constructor(private readonly sendSignallingMessage: SendSignallingMessageCallback) { +/** Implements a call between two peers with the signalling state keeping, while still delegating the signalling message sending. Used by GroupCall.*/ +class PeerCall implements PeerConnectionHandler { + private readonly peerConnection: PeerConnection; + constructor( + private readonly sendSignallingMessage: SendSignallingMessageCallback, + private localMedia: LocalMedia, + webRTC: WebRTC + ) { + this.peerConnection = webRTC.createPeerConnection(this); + } + + onIceConnectionStateChange(state: RTCIceConnectionState) {} + onLocalIceCandidate(candidate: RTCIceCandidate) {} + onIceGatheringStateChange(state: RTCIceGatheringState) {} + onRemoteTracksChanged(tracks: Track[]) {} + onDataChannelChanged(dataChannel: DataChannel | undefined) {} + onNegotiationNeeded() { + const message = { + offer: this.peerConnection.createOffer(), + sdp_stream_metadata: this.localMedia.getSDPMetadata(), + version: 1 + } + this.sendSignallingMessage(CallSetupMessageType.Invite, message); + } + + setLocalMedia(localMedia: LocalMedia) { + this.localMedia = localMedia; + // TODO: send new metadata + } + + + // request the type of incoming track + getPurposeForStreamId(streamId: string): StreamPurpose { + // look up stream purpose + return StreamPurpose.UserMedia; } handleIncomingSignallingMessage(type: CallSetupMessageType, content: Record) { @@ -148,10 +234,3 @@ class PeerCall { } } -class MediaSource { - -} - -class PeerConnection { - -} diff --git a/src/platform/types/MediaDevices.ts b/src/platform/types/MediaDevices.ts new file mode 100644 index 00000000..ec4895fa --- /dev/null +++ b/src/platform/types/MediaDevices.ts @@ -0,0 +1,42 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +export interface MediaDevices { + // filter out audiooutput + enumerate(): Promise; + getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise; + getScreenShareTrack(): Promise; +} + +export enum TrackType { + ScreenShare, + Camera, + Microphone, +} + +export interface Track { + get type(): TrackType; + get label(): string; + get id(): string; + get streamId(): string; + get muted(): boolean; + setMuted(muted: boolean): void; +} + +export interface AudioTrack extends Track { + get isSpeaking(): boolean; +} + diff --git a/src/platform/types/WebRTC.ts b/src/platform/types/WebRTC.ts new file mode 100644 index 00000000..b34e4214 --- /dev/null +++ b/src/platform/types/WebRTC.ts @@ -0,0 +1,55 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {Track, TrackType} from "./MediaDevices"; + +export enum StreamPurpose { + UserMedia = "m.usermedia", + ScreenShare = "m.screenshare" +} + +export interface WebRTC { + createPeerConnection(handler: PeerConnectionHandler): PeerConnection; +} + +export interface PeerConnectionHandler { + onIceConnectionStateChange(state: RTCIceConnectionState); + onLocalIceCandidate(candidate: RTCIceCandidate); + onIceGatheringStateChange(state: RTCIceGatheringState); + onRemoteTracksChanged(tracks: Track[]); + onDataChannelChanged(dataChannel: DataChannel | undefined); + onNegotiationNeeded(); + // request the type of incoming stream + getPurposeForStreamId(trackId: string): StreamPurpose; +} +// does it make sense to wrap this? +export interface DataChannel { + close(); + send(); +} + +export interface PeerConnection { + get remoteTracks(): Track[] | undefined; + get dataChannel(): DataChannel | undefined; + createOffer(): Promise; + createAnswer(): Promise; + setLocalDescription(description: RTCSessionDescriptionInit); + setRemoteDescription(description: RTCSessionDescriptionInit); + addTrack(track: Track): void; + removeTrack(track: Track): boolean; + replaceTrack(oldTrack: Track, newTrack: Track): Promise; + createDataChannel(): DataChannel; +} diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts new file mode 100644 index 00000000..08ccc636 --- /dev/null +++ b/src/platform/web/dom/MediaDevices.ts @@ -0,0 +1,457 @@ +/* +Copyright 2021 Å imon Brandner +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {MediaDevices as IMediaDevices, TrackType, Track, AudioTrack} from "../../types/MediaDevices"; + +const POLLING_INTERVAL = 200; // ms +export const SPEAKING_THRESHOLD = -60; // dB +const SPEAKING_SAMPLE_COUNT = 8; // samples + +class MediaDevicesWrapper implements IMediaDevices { + constructor(private readonly mediaDevices: MediaDevices) {} + + enumerate(): Promise { + return this.mediaDevices.enumerateDevices(); + } + + async getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise { + const stream = await this.mediaDevices.getUserMedia(this.getUserMediaContraints(audio, video)); + const tracks = stream.getTracks().map(t => { + const type = t.kind === "audio" ? TrackType.Microphone : TrackType.Camera; + return wrapTrack(t, stream, type); + }); + return tracks; + } + + async getScreenShareTrack(): Promise { + const stream = await this.mediaDevices.getDisplayMedia(this.getScreenshareContraints()); + const videoTrack = stream.getTracks().find(t => t.kind === "video"); + if (videoTrack) { + return wrapTrack(videoTrack, stream, TrackType.ScreenShare); + } + return; + } + + private getUserMediaContraints(audio: boolean | MediaDeviceInfo, video: boolean | MediaDeviceInfo): MediaStreamConstraints { + const isWebkit = !!navigator["webkitGetUserMedia"]; + + return { + audio: audio + ? { + deviceId: typeof audio !== "boolean" ? { ideal: audio.deviceId } : undefined, + } + : false, + video: video + ? { + deviceId: typeof video !== "boolean" ? { ideal: video.deviceId } : undefined, + /* We want 640x360. Chrome will give it only if we ask exactly, + FF refuses entirely if we ask exactly, so have to ask for ideal + instead + XXX: Is this still true? + */ + width: isWebkit ? { exact: 640 } : { ideal: 640 }, + height: isWebkit ? { exact: 360 } : { ideal: 360 }, + } + : false, + }; + } + + private getScreenshareContraints(): DisplayMediaStreamConstraints { + return { + audio: false, + video: true, + }; + } +} + +export function wrapTrack(track: MediaStreamTrack, stream: MediaStream, type: TrackType) { + if (track.kind === "audio") { + return new AudioTrackWrapper(track, stream, type); + } else { + return new TrackWrapper(track, stream, type); + } +} + +export class TrackWrapper implements Track { + constructor( + public readonly track: MediaStreamTrack, + public readonly stream: MediaStream, + public readonly type: TrackType + ) {} + + get label(): string { return this.track.label; } + get id(): string { return this.track.id; } + get streamId(): string { return this.stream.id; } + get muted(): boolean { return this.track.muted; } + + setMuted(muted: boolean): void { + this.track.enabled = !muted; + } +} + +export class AudioTrackWrapper extends TrackWrapper { + private measuringVolumeActivity = false; + private audioContext?: AudioContext; + private analyser: AnalyserNode; + private frequencyBinCount: Float32Array; + private speakingThreshold = SPEAKING_THRESHOLD; + private speaking = false; + private volumeLooperTimeout: number; + private speakingVolumeSamples: number[]; + + constructor(track: MediaStreamTrack, stream: MediaStream, type: TrackType) { + super(track, stream, type); + this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); + this.initVolumeMeasuring(); + this.measureVolumeActivity(true); + } + + get isSpeaking(): boolean { return this.speaking; } + /** + * Starts emitting volume_changed events where the emitter value is in decibels + * @param enabled emit volume changes + */ + private measureVolumeActivity(enabled: boolean): void { + if (enabled) { + if (!this.audioContext || !this.analyser || !this.frequencyBinCount) return; + + this.measuringVolumeActivity = true; + this.volumeLooper(); + } else { + this.measuringVolumeActivity = false; + this.speakingVolumeSamples.fill(-Infinity); + this.emit(CallFeedEvent.VolumeChanged, -Infinity); + } + } + + private initVolumeMeasuring(): void { + const AudioContext = window.AudioContext || window["webkitAudioContext"] as undefined | typeof window.AudioContext; + if (!AudioContext) return; + + this.audioContext = new AudioContext(); + + this.analyser = this.audioContext.createAnalyser(); + this.analyser.fftSize = 512; + this.analyser.smoothingTimeConstant = 0.1; + + const mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.stream); + mediaStreamAudioSourceNode.connect(this.analyser); + + this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount); + } + + + public setSpeakingThreshold(threshold: number) { + this.speakingThreshold = threshold; + } + + private volumeLooper = () => { + if (!this.analyser) return; + + if (!this.measuringVolumeActivity) return; + + this.analyser.getFloatFrequencyData(this.frequencyBinCount); + + let maxVolume = -Infinity; + for (let i = 0; i < this.frequencyBinCount.length; i++) { + if (this.frequencyBinCount[i] > maxVolume) { + maxVolume = this.frequencyBinCount[i]; + } + } + + this.speakingVolumeSamples.shift(); + this.speakingVolumeSamples.push(maxVolume); + + this.emit(CallFeedEvent.VolumeChanged, maxVolume); + + let newSpeaking = false; + + for (let i = 0; i < this.speakingVolumeSamples.length; i++) { + const volume = this.speakingVolumeSamples[i]; + + if (volume > this.speakingThreshold) { + newSpeaking = true; + break; + } + } + + if (this.speaking !== newSpeaking) { + this.speaking = newSpeaking; + this.emit(CallFeedEvent.Speaking, this.speaking); + } + + this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL) as unknown as number; + }; + + public dispose(): void { + clearTimeout(this.volumeLooperTimeout); + } +} + +// export interface ICallFeedOpts { +// client: MatrixClient; +// roomId: string; +// userId: string; +// stream: MediaStream; +// purpose: SDPStreamMetadataPurpose; +// audioMuted: boolean; +// videoMuted: boolean; +// } + +// export enum CallFeedEvent { +// NewStream = "new_stream", +// MuteStateChanged = "mute_state_changed", +// VolumeChanged = "volume_changed", +// Speaking = "speaking", +// } + +// export class CallFeed extends EventEmitter { +// public stream: MediaStream; +// public sdpMetadataStreamId: string; +// public userId: string; +// public purpose: SDPStreamMetadataPurpose; +// public speakingVolumeSamples: number[]; + +// private client: MatrixClient; +// private roomId: string; +// private audioMuted: boolean; +// private videoMuted: boolean; +// private measuringVolumeActivity = false; +// private audioContext: AudioContext; +// private analyser: AnalyserNode; +// private frequencyBinCount: Float32Array; +// private speakingThreshold = SPEAKING_THRESHOLD; +// private speaking = false; +// private volumeLooperTimeout: number; + +// constructor(opts: ICallFeedOpts) { +// super(); + +// this.client = opts.client; +// this.roomId = opts.roomId; +// this.userId = opts.userId; +// this.purpose = opts.purpose; +// this.audioMuted = opts.audioMuted; +// this.videoMuted = opts.videoMuted; +// this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); +// this.sdpMetadataStreamId = opts.stream.id; + +// this.updateStream(null, opts.stream); + +// if (this.hasAudioTrack) { +// this.initVolumeMeasuring(); +// } +// } + +// private get hasAudioTrack(): boolean { +// return this.stream.getAudioTracks().length > 0; +// } + +// private updateStream(oldStream: MediaStream, newStream: MediaStream): void { +// if (newStream === oldStream) return; + +// if (oldStream) { +// oldStream.removeEventListener("addtrack", this.onAddTrack); +// this.measureVolumeActivity(false); +// } +// if (newStream) { +// this.stream = newStream; +// newStream.addEventListener("addtrack", this.onAddTrack); + +// if (this.hasAudioTrack) { +// this.initVolumeMeasuring(); +// } else { +// this.measureVolumeActivity(false); +// } +// } + +// this.emit(CallFeedEvent.NewStream, this.stream); +// } + +// private initVolumeMeasuring(): void { +// const AudioContext = window.AudioContext || window.webkitAudioContext; +// if (!this.hasAudioTrack || !AudioContext) return; + +// this.audioContext = new AudioContext(); + +// this.analyser = this.audioContext.createAnalyser(); +// this.analyser.fftSize = 512; +// this.analyser.smoothingTimeConstant = 0.1; + +// const mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.stream); +// mediaStreamAudioSourceNode.connect(this.analyser); + +// this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount); +// } + +// private onAddTrack = (): void => { +// this.emit(CallFeedEvent.NewStream, this.stream); +// }; + +// /** +// * Returns callRoom member +// * @returns member of the callRoom +// */ +// public getMember(): RoomMember { +// const callRoom = this.client.getRoom(this.roomId); +// return callRoom.getMember(this.userId); +// } + +// /** +// * Returns true if CallFeed is local, otherwise returns false +// * @returns {boolean} is local? +// */ +// public isLocal(): boolean { +// return this.userId === this.client.getUserId(); +// } + +// /** +// * Returns true if audio is muted or if there are no audio +// * tracks, otherwise returns false +// * @returns {boolean} is audio muted? +// */ +// public isAudioMuted(): boolean { +// return this.stream.getAudioTracks().length === 0 || this.audioMuted; +// } + +// * +// * Returns true video is muted or if there are no video +// * tracks, otherwise returns false +// * @returns {boolean} is video muted? + +// public isVideoMuted(): boolean { +// // We assume only one video track +// return this.stream.getVideoTracks().length === 0 || this.videoMuted; +// } + +// public isSpeaking(): boolean { +// return this.speaking; +// } + +// /** +// * Replaces the current MediaStream with a new one. +// * This method should be only used by MatrixCall. +// * @param newStream new stream with which to replace the current one +// */ +// public setNewStream(newStream: MediaStream): void { +// this.updateStream(this.stream, newStream); +// } + +// /** +// * Set feed's internal audio mute state +// * @param muted is the feed's audio muted? +// */ +// public setAudioMuted(muted: boolean): void { +// this.audioMuted = muted; +// this.speakingVolumeSamples.fill(-Infinity); +// this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); +// } + +// /** +// * Set feed's internal video mute state +// * @param muted is the feed's video muted? +// */ +// public setVideoMuted(muted: boolean): void { +// this.videoMuted = muted; +// this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); +// } + +// /** +// * Starts emitting volume_changed events where the emitter value is in decibels +// * @param enabled emit volume changes +// */ +// public measureVolumeActivity(enabled: boolean): void { +// if (enabled) { +// if (!this.audioContext || !this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return; + +// this.measuringVolumeActivity = true; +// this.volumeLooper(); +// } else { +// this.measuringVolumeActivity = false; +// this.speakingVolumeSamples.fill(-Infinity); +// this.emit(CallFeedEvent.VolumeChanged, -Infinity); +// } +// } + +// public setSpeakingThreshold(threshold: number) { +// this.speakingThreshold = threshold; +// } + +// private volumeLooper = () => { +// if (!this.analyser) return; + +// if (!this.measuringVolumeActivity) return; + +// this.analyser.getFloatFrequencyData(this.frequencyBinCount); + +// let maxVolume = -Infinity; +// for (let i = 0; i < this.frequencyBinCount.length; i++) { +// if (this.frequencyBinCount[i] > maxVolume) { +// maxVolume = this.frequencyBinCount[i]; +// } +// } + +// this.speakingVolumeSamples.shift(); +// this.speakingVolumeSamples.push(maxVolume); + +// this.emit(CallFeedEvent.VolumeChanged, maxVolume); + +// let newSpeaking = false; + +// for (let i = 0; i < this.speakingVolumeSamples.length; i++) { +// const volume = this.speakingVolumeSamples[i]; + +// if (volume > this.speakingThreshold) { +// newSpeaking = true; +// break; +// } +// } + +// if (this.speaking !== newSpeaking) { +// this.speaking = newSpeaking; +// this.emit(CallFeedEvent.Speaking, this.speaking); +// } + +// this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL); +// }; + +// public clone(): CallFeed { +// const mediaHandler = this.client.getMediaHandler(); +// const stream = this.stream.clone(); + +// if (this.purpose === SDPStreamMetadataPurpose.Usermedia) { +// mediaHandler.userMediaStreams.push(stream); +// } else { +// mediaHandler.screensharingStreams.push(stream); +// } + +// return new CallFeed({ +// client: this.client, +// roomId: this.roomId, +// userId: this.userId, +// stream, +// purpose: this.purpose, +// audioMuted: this.audioMuted, +// videoMuted: this.videoMuted, +// }); +// } + +// public dispose(): void { +// clearTimeout(this.volumeLooperTimeout); +// this.measureVolumeActivity(false); +// } +// } diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts new file mode 100644 index 00000000..712e8cc6 --- /dev/null +++ b/src/platform/web/dom/WebRTC.ts @@ -0,0 +1,414 @@ +/* +Copyright 2021 Å imon Brandner + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {TrackWrapper, wrapTrack} from "./MediaDevices"; +import {Track} from "../../types/MediaDevices"; +import {WebRTC, PeerConnectionHandler, DataChannel, PeerConnection, StreamPurpose} from "../../types/WebRTC"; + +const POLLING_INTERVAL = 200; // ms +export const SPEAKING_THRESHOLD = -60; // dB +const SPEAKING_SAMPLE_COUNT = 8; // samples + +class DOMPeerConnection implements PeerConnection { + private readonly peerConnection: RTCPeerConnection; + private readonly handler: PeerConnectionHandler; + private dataChannelWrapper?: DOMDataChannel; + private _remoteTracks: TrackWrapper[]; + + constructor(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize = 0) { + this.handler = handler; + this.peerConnection = new RTCPeerConnection({ + iceTransportPolicy: forceTURN ? 'relay' : undefined, + iceServers: turnServers, + iceCandidatePoolSize: iceCandidatePoolSize, + }); + this.registerHandler(); + } + + get remoteTracks(): Track[] { return this._remoteTracks; } + get dataChannel(): DataChannel | undefined { return this.dataChannelWrapper; } + + createOffer(): Promise { + return this.peerConnection.createOffer(); + } + + createAnswer(): Promise { + return this.peerConnection.createAnswer(); + } + + setLocalDescription(description: RTCSessionDescriptionInit) { + this.peerConnection.setLocalDescription(description); + } + + setRemoteDescription(description: RTCSessionDescriptionInit) { + this.peerConnection.setRemoteDescription(description); + } + + addTrack(track: Track): void { + if (!(track instanceof TrackWrapper)) { + throw new Error("Not a TrackWrapper"); + } + this.peerConnection.addTrack(track.track, track.stream); + } + + removeTrack(track: Track): boolean { + if (!(track instanceof TrackWrapper)) { + throw new Error("Not a TrackWrapper"); + } + const sender = this.peerConnection.getSenders().find(s => s.track === track.track); + if (sender) { + this.peerConnection.removeTrack(sender); + return true; + } + return false; + } + + async replaceTrack(oldTrack: Track, newTrack: Track): Promise { + if (!(oldTrack instanceof TrackWrapper) || !(newTrack instanceof TrackWrapper)) { + throw new Error("Not a TrackWrapper"); + } + const sender = this.peerConnection.getSenders().find(s => s.track === oldTrack.track); + if (sender) { + await sender.replaceTrack(newTrack.track); + return true; + } + return false; + } + createDataChannel(): DataChannel { + return new DataChannel(this.peerConnection.createDataChannel()); + } + + private registerHandler() { + const pc = this.peerConnection; + pc.addEventListener('negotiationneeded', this); + pc.addEventListener('icecandidate', this); + pc.addEventListener('iceconnectionstatechange', this); + pc.addEventListener('icegatheringstatechange', this); + pc.addEventListener('signalingstatechange', this); + pc.addEventListener('track', this); + pc.addEventListener('datachannel', this); + } + + /** @internal */ + handleEvent(evt: Event) { + switch (evt.type) { + case "iceconnectionstatechange": + this.handleIceConnectionStateChange(); + break; + case "icecandidate": + this.handleLocalIceCandidate(evt as RTCPeerConnectionIceEvent); + break; + case "icegatheringstatechange": + this.handler.onIceGatheringStateChange(this.peerConnection.iceGatheringState); + break; + case "track": + this.handleRemoteTrack(evt as RTCTrackEvent); + break; + case "negotiationneeded": + this.handler.onNegotiationNeeded(); + break; + case "datachannel": + break; + } + } + + private handleLocalIceCandidate(event: RTCPeerConnectionIceEvent) { + if (event.candidate) { + this.handler.onLocalIceCandidate(event.candidate); + } + }; + + private handleIceConnectionStateChange() { + const {iceConnectionState} = this.peerConnection; + if (iceConnectionState === "failed" && this.peerConnection.restartIce) { + this.peerConnection.restartIce(); + } else { + this.handler.onIceConnectionStateChange(iceConnectionState); + } + } + + private handleRemoteTrack(evt: RTCTrackEvent) { + const updatedTracks = evt.streams.flatMap(stream => stream.getTracks().map(track => {return {stream, track};})); + const withoutRemovedTracks = this._remoteTracks.filter(t => !updatedTracks.some(ut => t.track == ut.track)); + const addedTracks = updatedTracks.filter(ut => !this._remoteTracks.some(t => t.track === ut.track)); + const wrappedAddedTracks = addedTracks.map(t => this.wrapRemoteTrack(t.track, t.stream)); + this._remoteTracks = withoutRemovedTracks.concat(...wrappedAddedTracks); + this.handler.onRemoteTracksChanged(this.remoteTracks); + } + private wrapRemoteTrack(track: MediaStreamTrack, stream: MediaStream): TrackWrapper { + let type: TrackType; + if (track.kind === "video") { + const purpose = this.handler.getPurposeForStreamId(stream.id); + type = purpose === StreamPurpose.Usermedia ? TrackType.Camera : TrackType.ScreenShare; + } else { + type = TrackType.Microphone; + } + return wrapTrack(track, stream, type); + } +} + +export interface ICallFeedOpts { + client: MatrixClient; + roomId: string; + userId: string; + stream: MediaStream; + purpose: SDPStreamMetadataPurpose; + audioMuted: boolean; + videoMuted: boolean; +} + +export enum CallFeedEvent { + NewStream = "new_stream", + MuteStateChanged = "mute_state_changed", + VolumeChanged = "volume_changed", + Speaking = "speaking", +} + +export class CallFeed extends EventEmitter { + public stream: MediaStream; + public sdpMetadataStreamId: string; + public userId: string; + public purpose: SDPStreamMetadataPurpose; + public speakingVolumeSamples: number[]; + + private client: MatrixClient; + private roomId: string; + private audioMuted: boolean; + private videoMuted: boolean; + private measuringVolumeActivity = false; + private audioContext: AudioContext; + private analyser: AnalyserNode; + private frequencyBinCount: Float32Array; + private speakingThreshold = SPEAKING_THRESHOLD; + private speaking = false; + private volumeLooperTimeout: number; + + constructor(opts: ICallFeedOpts) { + super(); + + this.client = opts.client; + this.roomId = opts.roomId; + this.userId = opts.userId; + this.purpose = opts.purpose; + this.audioMuted = opts.audioMuted; + this.videoMuted = opts.videoMuted; + this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); + this.sdpMetadataStreamId = opts.stream.id; + + this.updateStream(null, opts.stream); + + if (this.hasAudioTrack) { + this.initVolumeMeasuring(); + } + } + + private get hasAudioTrack(): boolean { + return this.stream.getAudioTracks().length > 0; + } + + private updateStream(oldStream: MediaStream, newStream: MediaStream): void { + if (newStream === oldStream) return; + + if (oldStream) { + oldStream.removeEventListener("addtrack", this.onAddTrack); + this.measureVolumeActivity(false); + } + if (newStream) { + this.stream = newStream; + newStream.addEventListener("addtrack", this.onAddTrack); + + if (this.hasAudioTrack) { + this.initVolumeMeasuring(); + } else { + this.measureVolumeActivity(false); + } + } + + this.emit(CallFeedEvent.NewStream, this.stream); + } + + private initVolumeMeasuring(): void { + const AudioContext = window.AudioContext || window.webkitAudioContext; + if (!this.hasAudioTrack || !AudioContext) return; + + this.audioContext = new AudioContext(); + + this.analyser = this.audioContext.createAnalyser(); + this.analyser.fftSize = 512; + this.analyser.smoothingTimeConstant = 0.1; + + const mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.stream); + mediaStreamAudioSourceNode.connect(this.analyser); + + this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount); + } + + private onAddTrack = (): void => { + this.emit(CallFeedEvent.NewStream, this.stream); + }; + + /** + * Returns callRoom member + * @returns member of the callRoom + */ + public getMember(): RoomMember { + const callRoom = this.client.getRoom(this.roomId); + return callRoom.getMember(this.userId); + } + + /** + * Returns true if CallFeed is local, otherwise returns false + * @returns {boolean} is local? + */ + public isLocal(): boolean { + return this.userId === this.client.getUserId(); + } + + /** + * Returns true if audio is muted or if there are no audio + * tracks, otherwise returns false + * @returns {boolean} is audio muted? + */ + public isAudioMuted(): boolean { + return this.stream.getAudioTracks().length === 0 || this.audioMuted; + } + + /** + * Returns true video is muted or if there are no video + * tracks, otherwise returns false + * @returns {boolean} is video muted? + */ + public isVideoMuted(): boolean { + // We assume only one video track + return this.stream.getVideoTracks().length === 0 || this.videoMuted; + } + + public isSpeaking(): boolean { + return this.speaking; + } + + /** + * Replaces the current MediaStream with a new one. + * This method should be only used by MatrixCall. + * @param newStream new stream with which to replace the current one + */ + public setNewStream(newStream: MediaStream): void { + this.updateStream(this.stream, newStream); + } + + /** + * Set feed's internal audio mute state + * @param muted is the feed's audio muted? + */ + public setAudioMuted(muted: boolean): void { + this.audioMuted = muted; + this.speakingVolumeSamples.fill(-Infinity); + this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); + } + + /** + * Set feed's internal video mute state + * @param muted is the feed's video muted? + */ + public setVideoMuted(muted: boolean): void { + this.videoMuted = muted; + this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); + } + + /** + * Starts emitting volume_changed events where the emitter value is in decibels + * @param enabled emit volume changes + */ + public measureVolumeActivity(enabled: boolean): void { + if (enabled) { + if (!this.audioContext || !this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return; + + this.measuringVolumeActivity = true; + this.volumeLooper(); + } else { + this.measuringVolumeActivity = false; + this.speakingVolumeSamples.fill(-Infinity); + this.emit(CallFeedEvent.VolumeChanged, -Infinity); + } + } + + public setSpeakingThreshold(threshold: number) { + this.speakingThreshold = threshold; + } + + private volumeLooper = () => { + if (!this.analyser) return; + + if (!this.measuringVolumeActivity) return; + + this.analyser.getFloatFrequencyData(this.frequencyBinCount); + + let maxVolume = -Infinity; + for (let i = 0; i < this.frequencyBinCount.length; i++) { + if (this.frequencyBinCount[i] > maxVolume) { + maxVolume = this.frequencyBinCount[i]; + } + } + + this.speakingVolumeSamples.shift(); + this.speakingVolumeSamples.push(maxVolume); + + this.emit(CallFeedEvent.VolumeChanged, maxVolume); + + let newSpeaking = false; + + for (let i = 0; i < this.speakingVolumeSamples.length; i++) { + const volume = this.speakingVolumeSamples[i]; + + if (volume > this.speakingThreshold) { + newSpeaking = true; + break; + } + } + + if (this.speaking !== newSpeaking) { + this.speaking = newSpeaking; + this.emit(CallFeedEvent.Speaking, this.speaking); + } + + this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL); + }; + + public clone(): CallFeed { + const mediaHandler = this.client.getMediaHandler(); + const stream = this.stream.clone(); + + if (this.purpose === SDPStreamMetadataPurpose.Usermedia) { + mediaHandler.userMediaStreams.push(stream); + } else { + mediaHandler.screensharingStreams.push(stream); + } + + return new CallFeed({ + client: this.client, + roomId: this.roomId, + userId: this.userId, + stream, + purpose: this.purpose, + audioMuted: this.audioMuted, + videoMuted: this.videoMuted, + }); + } + + public dispose(): void { + clearTimeout(this.volumeLooperTimeout); + } +} From e5f44aecfbe2fbcc53e3a7fac75b517baa776c7d Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Thu, 17 Feb 2022 16:58:44 +0100 Subject: [PATCH 004/323] WIP4 --- src/matrix/calls/CallHandler.ts | 207 +- src/matrix/calls/PeerCall.ts | 2757 +++++++++++++++++++++++++ src/matrix/calls/TODO.md | 43 + src/matrix/calls/group/Call.ts | 88 + src/matrix/calls/group/Participant.ts | 46 + src/platform/types/MediaDevices.ts | 1 + src/platform/types/WebRTC.ts | 3 +- src/platform/web/dom/MediaDevices.ts | 1 + src/platform/web/dom/WebRTC.ts | 261 +-- 9 files changed, 3026 insertions(+), 381 deletions(-) create mode 100644 src/matrix/calls/PeerCall.ts create mode 100644 src/matrix/calls/TODO.md create mode 100644 src/matrix/calls/group/Call.ts create mode 100644 src/matrix/calls/group/Participant.ts diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 0425304e..a31ff666 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -36,9 +36,9 @@ enum CallSetupMessageType { const CALL_ID = "m.call_id"; const CALL_TERMINATED = "m.terminated"; -export class CallHandler { +export class GroupCallHandler { // group calls by call id - public readonly groupCalls: ObservableMap = new ObservableMap(); + public readonly calls: ObservableMap = new ObservableMap(); constructor() { @@ -49,15 +49,15 @@ export class CallHandler { for (const event of events) { if (event.type === GROUP_CALL_TYPE) { const callId = event.state_key; - let call = this.groupCalls.get(callId); + let call = this.calls.get(callId); if (call) { call.updateCallEvent(event); if (call.isTerminated) { - this.groupCalls.remove(call.id); + this.calls.remove(call.id); } } else { call = new GroupCall(event, room); - this.groupCalls.set(call.id, call); + this.calls.set(call.id, call); } } } @@ -67,7 +67,7 @@ export class CallHandler { const participant = event.state_key; const sources = event.content["m.sources"]; for (const source of sources) { - const call = this.groupCalls.get(source[CALL_ID]); + const call = this.calls.get(source[CALL_ID]); if (call && !call.isTerminated) { call.addParticipant(participant, source); } @@ -85,33 +85,102 @@ export class CallHandler { handleDeviceMessage(senderUserId: string, senderDeviceId: string, eventType: string, content: Record, log: ILogItem) { const callId = content[CALL_ID]; - const call = this.groupCalls.get(callId); + const call = this.calls.get(callId); call?.handleDeviceMessage(senderUserId, senderDeviceId, eventType, content, log); } } -function peerCallKey(senderUserId: string, senderDeviceId: string) { +function participantId(senderUserId: string, senderDeviceId: string | null) { return JSON.stringify(senderUserId) + JSON.stringify(senderDeviceId); } +class GroupParticipant implements PeerCallHandler { + private peerCall?: PeerCall; + + constructor( + private readonly userId: string, + private readonly deviceId: string, + private localMedia: LocalMedia | undefined, + private readonly webRTC: WebRTC, + private readonly hsApi: HomeServerApi + ) {} + + sendInvite() { + this.peerCall = new PeerCall(this, this.webRTC); + this.peerCall.setLocalMedia(this.localMedia); + this.peerCall.sendOffer(); + } + + /** From PeerCallHandler + * @internal */ + override emitUpdate() { + + } + + /** From PeerCallHandler + * @internal */ + override onSendSignallingMessage() { + // TODO: this needs to be encrypted with olm first + this.hsApi.sendToDevice(type, {[this.userId]: {[this.deviceId ?? "*"]: content}}); + } +} + class GroupCall { - private peerCalls: Map + private readonly participants: ObservableMap = new ObservableMap(); + private localMedia?: LocalMedia; - constructor(private callEvent: StateEvent, private readonly room: Room) { + constructor(private readonly ownUserId: string, private callEvent: StateEvent, private readonly room: Room, private readonly webRTC: WebRTC) { } + get id(): string { return this.callEvent.state_key; } + + async participate(tracks: Track[]) { + this.localMedia = LocalMedia.fromTracks(tracks); + for (const [,participant] of this.participants) { + participant.setLocalMedia(this.localMedia.clone()); + } + // send m.call.member state event + + // send invite to all participants that are < my userId + for (const [,participant] of this.participants) { + if (participant.userId < this.ownUserId) { + participant.sendInvite(); + } + } + } + updateCallEvent(callEvent: StateEvent) { this.callEvent = callEvent; } addParticipant(userId, source) { - + const participantId = getParticipantId(userId, source.device_id); + const participant = this.participants.get(participantId); + if (participant) { + participant.updateSource(source); + } else { + participant.add(participantId, new GroupParticipant(userId, source.device_id, this.localMedia?.clone(), this.webRTC)); + } } handleDeviceMessage(senderUserId: string, senderDeviceId: string, eventType: string, content: Record, log: ILogItem) { - const peerCall = this.peerCalls.get(peerCallKey(senderUserId, senderDeviceId)); - peerCall?.handleIncomingSignallingMessage() + const participantId = getParticipantId(senderUserId, senderDeviceId); + let peerCall = this.participants.get(participantId); + let hasDeviceInKey = true; + if (!peerCall) { + hasDeviceInKey = false; + peerCall = this.participants.get(getParticipantId(senderUserId, null)) + } + if (peerCall) { + peerCall.handleIncomingSignallingMessage(eventType, content, senderDeviceId); + if (!hasDeviceInKey && peerCall.opponentPartyId) { + this.participants.delete(getParticipantId(senderUserId, null)); + this.participants.add(getParticipantId(senderUserId, peerCall.opponentPartyId)); + } + } else { + // create peerCall + } } get id(): string { @@ -121,116 +190,4 @@ class GroupCall { get isTerminated(): boolean { return !!this.callEvent.content[CALL_TERMINATED]; } - - private createPeerCall(userId: string, deviceId: string): PeerCall { - - } } - -/** - * Does WebRTC signalling for a single PeerConnection, and deals with WebRTC wrappers from platform - * */ - - -class LocalMedia { - private tracks = new Map(); - - setTracks(tracks: Track[]) { - for (const track of tracks) { - this.setTrack(track); - } - } - - setTrack(track: Track) { - let cameraAndMicStreamDontMatch = false; - if (track.type === TrackType.Microphone) { - const {cameraTrack} = this; - if (cameraTrack && track.streamId !== cameraTrack.streamId) { - cameraAndMicStreamDontMatch = true; - } - } - if (track.type === TrackType.Camera) { - const {microphoneTrack} = this; - if (microphoneTrack && track.streamId !== microphoneTrack.streamId) { - cameraAndMicStreamDontMatch = true; - } - } - if (cameraAndMicStreamDontMatch) { - throw new Error("The camera and audio track should have the same stream id"); - } - this.tracks.set(track.type, track); - } - - public get cameraTrack(): Track | undefined { return this.tracks.get(TrackType.Camera); }; - public get screenShareTrack(): Track | undefined { return this.tracks.get(TrackType.ScreenShare); }; - public get microphoneTrack(): AudioTrack | undefined { return this.tracks.get(TrackType.Microphone) as (AudioTrack | undefined); }; - - getSDPMetadata(): any { - const metadata = {}; - const userMediaTrack = this.microphoneTrack ?? this.cameraTrack; - if (userMediaTrack) { - metadata[userMediaTrack.streamId] = { - purpose: StreamPurpose.UserMedia - }; - } - if (this.screenShareTrack) { - metadata[this.screenShareTrack.streamId] = { - purpose: StreamPurpose.ScreenShare - }; - } - return metadata; - } -} - -// when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already -// do for sharing keys will be best as that already deals with room tracking. -type SendSignallingMessageCallback = (type: CallSetupMessageType, content: Record) => Promise; - -/** Implements a call between two peers with the signalling state keeping, while still delegating the signalling message sending. Used by GroupCall.*/ -class PeerCall implements PeerConnectionHandler { - private readonly peerConnection: PeerConnection; - - constructor( - private readonly sendSignallingMessage: SendSignallingMessageCallback, - private localMedia: LocalMedia, - webRTC: WebRTC - ) { - this.peerConnection = webRTC.createPeerConnection(this); - } - - onIceConnectionStateChange(state: RTCIceConnectionState) {} - onLocalIceCandidate(candidate: RTCIceCandidate) {} - onIceGatheringStateChange(state: RTCIceGatheringState) {} - onRemoteTracksChanged(tracks: Track[]) {} - onDataChannelChanged(dataChannel: DataChannel | undefined) {} - onNegotiationNeeded() { - const message = { - offer: this.peerConnection.createOffer(), - sdp_stream_metadata: this.localMedia.getSDPMetadata(), - version: 1 - } - this.sendSignallingMessage(CallSetupMessageType.Invite, message); - } - - setLocalMedia(localMedia: LocalMedia) { - this.localMedia = localMedia; - // TODO: send new metadata - } - - - // request the type of incoming track - getPurposeForStreamId(streamId: string): StreamPurpose { - // look up stream purpose - return StreamPurpose.UserMedia; - } - - handleIncomingSignallingMessage(type: CallSetupMessageType, content: Record) { - switch (type) { - case CallSetupMessageType.Invite: - case CallSetupMessageType.Answer: - case CallSetupMessageType.Candidates: - case CallSetupMessageType.Hangup: - } - } -} - diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts new file mode 100644 index 00000000..8780258e --- /dev/null +++ b/src/matrix/calls/PeerCall.ts @@ -0,0 +1,2757 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {ObservableMap} from "../../observable/map/ObservableMap"; + +import type {Room} from "../room/Room"; +import type {StateEvent} from "../storage/types"; +import type {ILogItem} from "../../logging/types"; + +import {WebRTC, PeerConnection, PeerConnectionHandler, StreamPurpose} from "../../platform/types/WebRTC"; +import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; + +import { randomString } from '../randomstring'; +import { + MCallReplacesEvent, + MCallAnswer, + MCallInviteNegotiate, + CallCapabilities, + SDPStreamMetadataPurpose, + SDPStreamMetadata, + SDPStreamMetadataKey, + MCallSDPStreamMetadataChanged, + MCallSelectAnswer, + MCAllAssertedIdentity, + MCallCandidates, + MCallBase, + MCallHangupReject, +} from './callEventTypes'; + + +const GROUP_CALL_TYPE = "m.call"; +const GROUP_CALL_MEMBER_TYPE = "m.call.member"; + + +/** + * Fires whenever an error occurs when call.js encounters an issue with setting up the call. + *

+ * The error given will have a code equal to either `MatrixCall.ERR_LOCAL_OFFER_FAILED` or + * `MatrixCall.ERR_NO_USER_MEDIA`. `ERR_LOCAL_OFFER_FAILED` is emitted when the local client + * fails to create an offer. `ERR_NO_USER_MEDIA` is emitted when the user has denied access + * to their audio/video hardware. + * + * @event module:webrtc/call~MatrixCall#"error" + * @param {Error} err The error raised by MatrixCall. + * @example + * matrixCall.on("error", function(err){ + * console.error(err.code, err); + * }); + */ + +// null is used as a special value meaning that the we're in a legacy 1:1 call +// without MSC2746 that doesn't provide an id which device sent the message. +type PartyId = string | null; + +interface CallOpts { + roomId?: string; + invitee?: string; + client?: any; // Fix when client is TSified + forceTURN?: boolean; + turnServers?: Array; + opponentDeviceId?: string; + opponentSessionId?: string; + groupCallId?: string; +} + +interface TurnServer { + urls: Array; + username?: string; + password?: string; + ttl?: number; +} + +interface AssertedIdentity { + id: string; + displayName: string; +} + +export enum CallState { + Fledgling = 'fledgling', + InviteSent = 'invite_sent', + WaitLocalMedia = 'wait_local_media', + CreateOffer = 'create_offer', + CreateAnswer = 'create_answer', + Connecting = 'connecting', + Connected = 'connected', + Ringing = 'ringing', + Ended = 'ended', +} + +export enum CallType { + Voice = 'voice', + Video = 'video', +} + +export enum CallDirection { + Inbound = 'inbound', + Outbound = 'outbound', +} + +export enum CallParty { + Local = 'local', + Remote = 'remote', +} + +export enum CallEvent { + Hangup = 'hangup', + State = 'state', + Error = 'error', + Replaced = 'replaced', + + // The value of isLocalOnHold() has changed + LocalHoldUnhold = 'local_hold_unhold', + // The value of isRemoteOnHold() has changed + RemoteHoldUnhold = 'remote_hold_unhold', + // backwards compat alias for LocalHoldUnhold: remove in a major version bump + HoldUnhold = 'hold_unhold', + // Feeds have changed + FeedsChanged = 'feeds_changed', + + AssertedIdentityChanged = 'asserted_identity_changed', + + LengthChanged = 'length_changed', + + DataChannel = 'datachannel', + + SendVoipEvent = "send_voip_event", +} + +export enum CallErrorCode { + /** The user chose to end the call */ + UserHangup = 'user_hangup', + + /** An error code when the local client failed to create an offer. */ + LocalOfferFailed = 'local_offer_failed', + /** + * An error code when there is no local mic/camera to use. This may be because + * the hardware isn't plugged in, or the user has explicitly denied access. + */ + NoUserMedia = 'no_user_media', + + /** + * Error code used when a call event failed to send + * because unknown devices were present in the room + */ + UnknownDevices = 'unknown_devices', + + /** + * Error code used when we fail to send the invite + * for some reason other than there being unknown devices + */ + SendInvite = 'send_invite', + + /** + * An answer could not be created + */ + CreateAnswer = 'create_answer', + + /** + * Error code used when we fail to send the answer + * for some reason other than there being unknown devices + */ + SendAnswer = 'send_answer', + + /** + * The session description from the other side could not be set + */ + SetRemoteDescription = 'set_remote_description', + + /** + * The session description from this side could not be set + */ + SetLocalDescription = 'set_local_description', + + /** + * A different device answered the call + */ + AnsweredElsewhere = 'answered_elsewhere', + + /** + * No media connection could be established to the other party + */ + IceFailed = 'ice_failed', + + /** + * The invite timed out whilst waiting for an answer + */ + InviteTimeout = 'invite_timeout', + + /** + * The call was replaced by another call + */ + Replaced = 'replaced', + + /** + * Signalling for the call could not be sent (other than the initial invite) + */ + SignallingFailed = 'signalling_timeout', + + /** + * The remote party is busy + */ + UserBusy = 'user_busy', + + /** + * We transferred the call off to somewhere else + */ + Transfered = 'transferred', + + /** + * A call from the same user was found with a new session id + */ + NewSession = 'new_session', +} + +/** + * The version field that we set in m.call.* events + */ +const VOIP_PROTO_VERSION = 1; + +/** The fallback ICE server to use for STUN or TURN protocols. */ +const FALLBACK_ICE_SERVER = 'stun:turn.matrix.org'; + +/** The length of time a call can be ringing for. */ +const CALL_TIMEOUT_MS = 60000; + +const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); + +export class CallError extends Error { + code: string; + + constructor(code: CallErrorCode, msg: string, err: Error) { + // Still don't think there's any way to have proper nested errors + super(msg + ": " + err); + + this.code = code; + } +} + +export function genCallID(): string { + return Date.now().toString() + randomString(16); +} + +enum CallSetupMessageType { + Invite = "m.call.invite", + Answer = "m.call.answer", + Candidates = "m.call.candidates", + Hangup = "m.call.hangup", +} + +const CALL_ID = "m.call_id"; +const CALL_TERMINATED = "m.terminated"; + +class LocalMedia { + constructor( + public readonly cameraTrack?: Track, + public readonly screenShareTrack?: Track, + public readonly microphoneTrack?: AudioTrack + ) {} + + withTracks(tracks: Track[]) { + const cameraTrack = tracks.find(t => t.type === TrackType.Camera) ?? this.cameraTrack; + const screenShareTrack = tracks.find(t => t.type === TrackType.ScreenShare) ?? this.screenShareTrack; + const microphoneTrack = tracks.find(t => t.type === TrackType.Microphone) ?? this.microphoneTrack; + if (cameraTrack && microphoneTrack && cameraTrack.streamId !== microphoneTrack.streamId) { + throw new Error("The camera and audio track should have the same stream id"); + } + return new LocalMedia(cameraTrack, screenShareTrack, microphoneTrack as AudioTrack); + } + + getSDPMetadata(): any { + const metadata = {}; + const userMediaTrack = this.microphoneTrack ?? this.cameraTrack; + if (userMediaTrack) { + metadata[userMediaTrack.streamId] = { + purpose: StreamPurpose.UserMedia, + audio_muted: this.microphoneTrack?.muted ?? false, + video_muted: this.cameraTrack?.muted ?? false, + }; + } + if (this.screenShareTrack) { + metadata[this.screenShareTrack.streamId] = { + purpose: StreamPurpose.ScreenShare + }; + } + return metadata; + } +} + +interface PeerCallHandler { + emitUpdate(); + sendSignallingMessage(type: string, content: Record); +} + +// when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already +// do for sharing keys will be best as that already deals with room tracking. +type SendSignallingMessageCallback = (type: CallSetupMessageType, content: Record) => Promise; +/** + * Does WebRTC signalling for a single PeerConnection, and deals with WebRTC wrappers from platform + * */ +/** Implements a call between two peers with the signalling state keeping, while still delegating the signalling message sending. Used by GroupCall.*/ +class PeerCall implements PeerConnectionHandler { + private readonly peerConnection: PeerConnection; + + + public state = CallState.Fledgling; + public hangupParty: CallParty; + public hangupReason: string; + public direction: CallDirection; + public peerConn?: RTCPeerConnection; + + // A queue for candidates waiting to go out. + // We try to amalgamate candidates into a single candidate message where + // possible + private candidateSendQueue: Array = []; + private candidateSendTries = 0; + private sentEndOfCandidates = false; + + private inviteOrAnswerSent = false; + private waitForLocalAVStream: boolean; + private successor: MatrixCall; + private opponentVersion: number | string; + // The party ID of the other side: undefined if we haven't chosen a partner + // yet, null if we have but they didn't send a party ID. + private opponentPartyId: PartyId; + private opponentCaps: CallCapabilities; + private inviteTimeout: number; + private iceDisconnectedTimeout: number; + + // The logic of when & if a call is on hold is nontrivial and explained in is*OnHold + // This flag represents whether we want the other party to be on hold + private remoteOnHold = false; + + // the stats for the call at the point it ended. We can't get these after we + // tear the call down, so we just grab a snapshot before we stop the call. + // The typescript definitions have this type as 'any' :( + private callStatsAtEnd: any[]; + + // Perfect negotiation state: https://www.w3.org/TR/webrtc/#perfect-negotiation-example + private makingOffer = false; + private ignoreOffer: boolean; + + private responsePromiseChain?: Promise; + + // If candidates arrive before we've picked an opponent (which, in particular, + // will happen if the opponent sends candidates eagerly before the user answers + // the call) we buffer them up here so we can then add the ones from the party we pick + private remoteCandidateBuffer: Map = []; + + private remoteAssertedIdentity: AssertedIdentity; + + private remoteSDPStreamMetadata?: SDPStreamMetadata; + + + constructor( + private readonly handler: PeerCallHandler, + private localMedia: LocalMedia, + webRTC: WebRTC + ) { + this.peerConnection = webRTC.createPeerConnection(this); + } + + onIceConnectionStateChange(state: RTCIceConnectionState) {} + onLocalIceCandidate(candidate: RTCIceCandidate) {} + onIceGatheringStateChange(state: RTCIceGatheringState) {} + onRemoteTracksChanged(tracks: Track[]) {} + onDataChannelChanged(dataChannel: DataChannel | undefined) {} + onNegotiationNeeded() { + const message = { + offer: this.peerConnection.createOffer(), + sdp_stream_metadata: this.localMedia.getSDPMetadata(), + version: 1 + } + this.handler.sendSignallingMessage(CallSetupMessageType.Invite, message); + } + + setLocalMedia(localMedia: LocalMedia) { + this.localMedia = localMedia; + // TODO: send new metadata + } + + // request the type of incoming track + getPurposeForStreamId(streamId: string): StreamPurpose { + // TODO: should we return a promise here for the case where the metadata hasn't arrived yet? + const metaData = this.remoteSDPStreamMetadata[streamId]; + return metadata?.purpose as StreamPurpose ?? StreamPurpose.UserMedia; + } + + private setState(state: CallState): void { + const oldState = this.state; + this.state = state; + this.handler.emitUpdate(); + } + + handleIncomingSignallingMessage(type: CallSetupMessageType, content: Record, partyId: PartyId) { + switch (type) { + case CallSetupMessageType.Invite: + case CallSetupMessageType.Answer: + this.handleAnswer(content); + break; + case CallSetupMessageType.Candidates: + this.handleRemoteIceCandidates(content); + break; + case CallSetupMessageType.Hangup: + } + } + + private async handleAnswer(content: MCallAnswer, partyId: PartyId) { + // add buffered ice candidates to peerConnection + if (this.opponentPartyId !== undefined) { + return; + } + this.opponentPartyId = partyId; + const bufferedCandidates = this.remoteCandidateBuffer?.get(partyId); + if (bufferedCandidates) { + this.addIceCandidates(bufferedCandidates); + } + this.remoteCandidateBuffer = undefined; + + this.setState(CallState.Connecting); + + const sdpStreamMetadata = content[SDPStreamMetadataKey]; + if (sdpStreamMetadata) { + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); + } else { + logger.warn("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); + } + + try { + await this.peerConnection.setRemoteDescription(content.answer); + } catch (e) { + logger.debug("Failed to set remote description", e); + this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); + return; + } + + // If the answer we selected has a party_id, send a select_answer event + // We do this after setting the remote description since otherwise we'd block + // call setup on it + if (this.opponentPartyId !== null) { + try { + await this.sendVoipEvent(EventType.CallSelectAnswer, { + selected_party_id: this.opponentPartyId, + }); + } catch (err) { + // This isn't fatal, and will just mean that if another party has raced to answer + // the call, they won't know they got rejected, so we carry on & don't retry. + logger.warn("Failed to send select_answer event", err); + } + } + } + + private handleRemoteIceCandidates(content: Record) { + if (this.state === CallState.Ended) { + return; + } + const candidates = content.candidates; + if (!candidates) { + return; + } + if (this.opponentPartyId === undefined) { + if (!this.remoteCandidateBuffer) { + this.remoteCandidateBuffer = new Map(); + } + const bufferedCandidates = this.remoteCandidateBuffer.get(fromPartyId) || []; + bufferedCandidates.push(...candidates); + this.remoteCandidateBuffer.set(fromPartyId, bufferedCandidates); + } else { + this.addIceCandidates(candidates); + } + } + + private async addIceCandidates(candidates: RTCIceCandidate[]): Promise { + for (const candidate of candidates) { + if ( + (candidate.sdpMid === null || candidate.sdpMid === undefined) && + (candidate.sdpMLineIndex === null || candidate.sdpMLineIndex === undefined) + ) { + logger.debug("Ignoring remote ICE candidate with no sdpMid or sdpMLineIndex"); + continue; + } + logger.debug( + "Call " + this.callId + " got remote ICE " + candidate.sdpMid + " candidate: " + candidate.candidate, + ); + try { + await this.peerConnection.addIceCandidate(candidate); + } catch (err) { + if (!this.ignoreOffer) { + logger.info("Failed to add remote ICE candidate", err); + } + } + } + } +} + +/** + * Construct a new Matrix Call. + * @constructor + * @param {Object} opts Config options. + * @param {string} opts.roomId The room ID for this call. + * @param {Object} opts.webRtc The WebRTC globals from the browser. + * @param {boolean} opts.forceTURN whether relay through TURN should be forced. + * @param {Object} opts.URL The URL global. + * @param {Array} opts.turnServers Optional. A list of TURN servers. + * @param {MatrixClient} opts.client The Matrix Client instance to send events to. + */ +export class MatrixCall extends EventEmitter { + // should go into DirectCall/GroupCall class + public roomId: string; + public callId: string; + public invitee?: string; + public ourPartyId: string; + private opponentMember: RoomMember; + private opponentDeviceId: string; + private opponentSessionId: string; + public groupCallId: string; + private callLengthInterval: number; + private callLength = 0; + + + + public state = CallState.Fledgling; + public hangupParty: CallParty; + public hangupReason: string; + public direction: CallDirection; + public peerConn?: RTCPeerConnection; + + // A queue for candidates waiting to go out. + // We try to amalgamate candidates into a single candidate message where + // possible + private candidateSendQueue: Array = []; + private candidateSendTries = 0; + private sentEndOfCandidates = false; + + private inviteOrAnswerSent = false; + private waitForLocalAVStream: boolean; + private successor: MatrixCall; + private opponentVersion: number | string; + // The party ID of the other side: undefined if we haven't chosen a partner + // yet, null if we have but they didn't send a party ID. + private opponentPartyId: string; + private opponentCaps: CallCapabilities; + private inviteTimeout: number; + private iceDisconnectedTimeout: number; + + // The logic of when & if a call is on hold is nontrivial and explained in is*OnHold + // This flag represents whether we want the other party to be on hold + private remoteOnHold = false; + + // the stats for the call at the point it ended. We can't get these after we + // tear the call down, so we just grab a snapshot before we stop the call. + // The typescript definitions have this type as 'any' :( + private callStatsAtEnd: any[]; + + // Perfect negotiation state: https://www.w3.org/TR/webrtc/#perfect-negotiation-example + private makingOffer = false; + private ignoreOffer: boolean; + + private responsePromiseChain?: Promise; + + // If candidates arrive before we've picked an opponent (which, in particular, + // will happen if the opponent sends candidates eagerly before the user answers + // the call) we buffer them up here so we can then add the ones from the party we pick + private remoteCandidateBuffer: RTCIceCandidate[] = []; + + private remoteAssertedIdentity: AssertedIdentity; + + private remoteSDPStreamMetadata: SDPStreamMetadata; + + constructor(opts: CallOpts) { + super(); + this.roomId = opts.roomId; + this.invitee = opts.invitee; + this.client = opts.client; + this.forceTURN = opts.forceTURN; + this.ourPartyId = this.client.deviceId; + this.opponentDeviceId = opts.opponentDeviceId; + this.opponentSessionId = opts.opponentSessionId; + this.groupCallId = opts.groupCallId; + // Array of Objects with urls, username, credential keys + this.turnServers = opts.turnServers || []; + if (this.turnServers.length === 0 && this.client.isFallbackICEServerAllowed()) { + this.turnServers.push({ + urls: [FALLBACK_ICE_SERVER], + }); + } + for (const server of this.turnServers) { + utils.checkObjectHasKeys(server, ["urls"]); + } + this.callId = genCallID(); + } + + /** + * Place a voice call to this room. + * @throws If you have not specified a listener for 'error' events. + */ + public async placeVoiceCall(): Promise { + await this.placeCall(true, false); + } + + /** + * Place a video call to this room. + * @throws If you have not specified a listener for 'error' events. + */ + public async placeVideoCall(): Promise { + await this.placeCall(true, true); + } + + /** + * Create a datachannel using this call's peer connection. + * @param label A human readable label for this datachannel + * @param options An object providing configuration options for the data channel. + */ + public createDataChannel(label: string, options: RTCDataChannelInit) { + const dataChannel = this.peerConn.createDataChannel(label, options); + this.emit(CallEvent.DataChannel, dataChannel); + return dataChannel; + } + + public getOpponentMember(): RoomMember { + return this.opponentMember; + } + + public getOpponentSessionId(): string { + return this.opponentSessionId; + } + + public opponentCanBeTransferred(): boolean { + return Boolean(this.opponentCaps && this.opponentCaps["m.call.transferee"]); + } + + public opponentSupportsDTMF(): boolean { + return Boolean(this.opponentCaps && this.opponentCaps["m.call.dtmf"]); + } + + public getRemoteAssertedIdentity(): AssertedIdentity { + return this.remoteAssertedIdentity; + } + + public get type(): CallType { + return (this.hasLocalUserMediaVideoTrack || this.hasRemoteUserMediaVideoTrack) + ? CallType.Video + : CallType.Voice; + } + + public get hasLocalUserMediaVideoTrack(): boolean { + return this.localUsermediaStream?.getVideoTracks().length > 0; + } + + public get hasRemoteUserMediaVideoTrack(): boolean { + return this.getRemoteFeeds().some((feed) => { + return ( + feed.purpose === SDPStreamMetadataPurpose.Usermedia && + feed.stream.getVideoTracks().length > 0 + ); + }); + } + + public get hasLocalUserMediaAudioTrack(): boolean { + return this.localUsermediaStream?.getAudioTracks().length > 0; + } + + public get hasRemoteUserMediaAudioTrack(): boolean { + return this.getRemoteFeeds().some((feed) => { + return ( + feed.purpose === SDPStreamMetadataPurpose.Usermedia && + feed.stream.getAudioTracks().length > 0 + ); + }); + } + + public get localUsermediaFeed(): CallFeed { + return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); + } + + public get localScreensharingFeed(): CallFeed { + return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); + } + + public get localUsermediaStream(): MediaStream { + return this.localUsermediaFeed?.stream; + } + + public get localScreensharingStream(): MediaStream { + return this.localScreensharingFeed?.stream; + } + + public get remoteUsermediaFeed(): CallFeed { + return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); + } + + public get remoteScreensharingFeed(): CallFeed { + return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); + } + + public get remoteUsermediaStream(): MediaStream { + return this.remoteUsermediaFeed?.stream; + } + + public get remoteScreensharingStream(): MediaStream { + return this.remoteScreensharingFeed?.stream; + } + + private getFeedByStreamId(streamId: string): CallFeed { + return this.getFeeds().find((feed) => feed.stream.id === streamId); + } + + /** + * Returns an array of all CallFeeds + * @returns {Array} CallFeeds + */ + public getFeeds(): Array { + return this.feeds; + } + + /** + * Returns an array of all local CallFeeds + * @returns {Array} local CallFeeds + */ + public getLocalFeeds(): Array { + return this.feeds.filter((feed) => feed.isLocal()); + } + + /** + * Returns an array of all remote CallFeeds + * @returns {Array} remote CallFeeds + */ + public getRemoteFeeds(): Array { + return this.feeds.filter((feed) => !feed.isLocal()); + } + + /** + * Generates and returns localSDPStreamMetadata + * @returns {SDPStreamMetadata} localSDPStreamMetadata + */ + private getLocalSDPStreamMetadata(updateStreamIds = false): SDPStreamMetadata { + const metadata: SDPStreamMetadata = {}; + for (const localFeed of this.getLocalFeeds()) { + if (updateStreamIds) { + localFeed.sdpMetadataStreamId = localFeed.stream.id; + } + + metadata[localFeed.sdpMetadataStreamId] = { + purpose: localFeed.purpose, + audio_muted: localFeed.isAudioMuted(), + video_muted: localFeed.isVideoMuted(), + }; + } + return metadata; + } + + /** + * Returns true if there are no incoming feeds, + * otherwise returns false + * @returns {boolean} no incoming feeds + */ + public noIncomingFeeds(): boolean { + return !this.feeds.some((feed) => !feed.isLocal()); + } + + private pushRemoteFeed(stream: MediaStream): void { + // Fallback to old behavior if the other side doesn't support SDPStreamMetadata + if (!this.opponentSupportsSDPStreamMetadata()) { + this.pushRemoteFeedWithoutMetadata(stream); + return; + } + + const userId = this.getOpponentMember().userId; + const purpose = this.remoteSDPStreamMetadata[stream.id].purpose; + const audioMuted = this.remoteSDPStreamMetadata[stream.id].audio_muted; + const videoMuted = this.remoteSDPStreamMetadata[stream.id].video_muted; + + if (!purpose) { + logger.warn(`Ignoring stream with id ${stream.id} because we didn't get any metadata about it`); + return; + } + + // Try to find a feed with the same purpose as the new stream, + // if we find it replace the old stream with the new one + const existingFeed = this.getRemoteFeeds().find((feed) => feed.purpose === purpose); + if (existingFeed) { + existingFeed.setNewStream(stream); + } else { + this.feeds.push(new CallFeed({ + client: this.client, + roomId: this.roomId, + userId, + stream, + purpose, + audioMuted, + videoMuted, + })); + this.emit(CallEvent.FeedsChanged, this.feeds); + } + + logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}", purpose=${purpose})`); + } + + /** + * This method is used ONLY if the other client doesn't support sending SDPStreamMetadata + */ + private pushRemoteFeedWithoutMetadata(stream: MediaStream): void { + const userId = this.getOpponentMember().userId; + // We can guess the purpose here since the other client can only send one stream + const purpose = SDPStreamMetadataPurpose.Usermedia; + const oldRemoteStream = this.feeds.find((feed) => !feed.isLocal())?.stream; + + // Note that we check by ID and always set the remote stream: Chrome appears + // to make new stream objects when transceiver directionality is changed and the 'active' + // status of streams change - Dave + // If we already have a stream, check this stream has the same id + if (oldRemoteStream && stream.id !== oldRemoteStream.id) { + logger.warn(`Ignoring new stream ID ${stream.id}: we already have stream ID ${oldRemoteStream.id}`); + return; + } + + // Try to find a feed with the same stream id as the new stream, + // if we find it replace the old stream with the new one + const feed = this.getFeedByStreamId(stream.id); + if (feed) { + feed.setNewStream(stream); + } else { + this.feeds.push(new CallFeed({ + client: this.client, + roomId: this.roomId, + audioMuted: false, + videoMuted: false, + userId, + stream, + purpose, + })); + this.emit(CallEvent.FeedsChanged, this.feeds); + } + + logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}")`); + } + + private pushNewLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose, addToPeerConnection = true): void { + const userId = this.client.getUserId(); + + // TODO: Find out what is going on here + // why do we enable audio (and only audio) tracks here? -- matthew + setTracksEnabled(stream.getAudioTracks(), true); + + // We try to replace an existing feed if there already is one with the same purpose + const existingFeed = this.getLocalFeeds().find((feed) => feed.purpose === purpose); + if (existingFeed) { + existingFeed.setNewStream(stream); + } else { + this.pushLocalFeed( + new CallFeed({ + client: this.client, + roomId: this.roomId, + audioMuted: stream.getAudioTracks().length === 0, + videoMuted: stream.getVideoTracks().length === 0, + userId, + stream, + purpose, + }), + addToPeerConnection, + ); + this.emit(CallEvent.FeedsChanged, this.feeds); + } + } + + /** + * Pushes supplied feed to the call + * @param {CallFeed} callFeed to push + * @param {boolean} addToPeerConnection whether to add the tracks to the peer connection + */ + public pushLocalFeed(callFeed: CallFeed, addToPeerConnection = true): void { + if (this.feeds.some((feed) => callFeed.stream.id === feed.stream.id)) { + logger.info(`Ignoring duplicate local stream ${callFeed.stream.id} in call ${this.callId}`); + return; + } + + this.feeds.push(callFeed); + + if (addToPeerConnection) { + const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia ? + this.usermediaSenders : this.screensharingSenders; + // Empty the array + senderArray.splice(0, senderArray.length); + + for (const track of callFeed.stream.getTracks()) { + logger.info( + `Adding track (` + + `id="${track.id}", ` + + `kind="${track.kind}", ` + + `streamId="${callFeed.stream.id}", ` + + `streamPurpose="${callFeed.purpose}"` + + `) to peer connection`, + ); + senderArray.push(this.peerConn.addTrack(track, callFeed.stream)); + } + } + + logger.info( + `Pushed local stream `+ + `(id="${callFeed.stream.id}", `+ + `active="${callFeed.stream.active}", `+ + `purpose="${callFeed.purpose}")`, + ); + + this.emit(CallEvent.FeedsChanged, this.feeds); + } + + /** + * Removes local call feed from the call and its tracks from the peer + * connection + * @param callFeed to remove + */ + public removeLocalFeed(callFeed: CallFeed): void { + const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia + ? this.usermediaSenders + : this.screensharingSenders; + + for (const sender of senderArray) { + this.peerConn.removeTrack(sender); + } + + if (callFeed.purpose === SDPStreamMetadataPurpose.Screenshare) { + this.client.getMediaHandler().stopScreensharingStream(callFeed.stream); + } + + // Empty the array + senderArray.splice(0, senderArray.length); + this.deleteFeed(callFeed); + } + + private deleteAllFeeds(): void { + for (const feed of this.feeds) { + if (!feed.isLocal() || !this.groupCallId) { + feed.dispose(); + } + } + + this.feeds = []; + this.emit(CallEvent.FeedsChanged, this.feeds); + } + + private deleteFeedByStream(stream: MediaStream): void { + const feed = this.getFeedByStreamId(stream.id); + if (!feed) { + logger.warn(`Didn't find the feed with stream id ${stream.id} to delete`); + return; + } + this.deleteFeed(feed); + } + + private deleteFeed(feed: CallFeed): void { + feed.dispose(); + this.feeds.splice(this.feeds.indexOf(feed), 1); + this.emit(CallEvent.FeedsChanged, this.feeds); + } + + // The typescript definitions have this type as 'any' :( + public async getCurrentCallStats(): Promise { + if (this.callHasEnded()) { + return this.callStatsAtEnd; + } + + return this.collectCallStats(); + } + + private async collectCallStats(): Promise { + // This happens when the call fails before it starts. + // For example when we fail to get capture sources + if (!this.peerConn) return; + + const statsReport = await this.peerConn.getStats(); + const stats = []; + for (const item of statsReport) { + stats.push(item[1]); + } + + return stats; + } + + /** + * Configure this call from an invite event. Used by MatrixClient. + * @param {MatrixEvent} event The m.call.invite event + */ + public async initWithInvite(event: MatrixEvent): Promise { + const invite = event.getContent(); + this.direction = CallDirection.Inbound; + + // make sure we have valid turn creds. Unless something's gone wrong, it should + // poll and keep the credentials valid so this should be instant. + const haveTurnCreds = await this.client.checkTurnServers(); + if (!haveTurnCreds) { + logger.warn("Failed to get TURN credentials! Proceeding with call anyway..."); + } + + const sdpStreamMetadata = invite[SDPStreamMetadataKey]; + if (sdpStreamMetadata) { + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); + } else { + logger.debug("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); + } + + this.peerConn = this.createPeerConnection(); + // we must set the party ID before await-ing on anything: the call event + // handler will start giving us more call events (eg. candidates) so if + // we haven't set the party ID, we'll ignore them. + this.chooseOpponent(event); + try { + await this.peerConn.setRemoteDescription(invite.offer); + await this.addBufferedIceCandidates(); + } catch (e) { + logger.debug("Failed to set remote description", e); + this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); + return; + } + + const remoteStream = this.feeds.find((feed) => !feed.isLocal())?.stream; + + // According to previous comments in this file, firefox at some point did not + // add streams until media started arriving on them. Testing latest firefox + // (81 at time of writing), this is no longer a problem, so let's do it the correct way. + if (!remoteStream || remoteStream.getTracks().length === 0) { + logger.error("No remote stream or no tracks after setting remote description!"); + this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); + return; + } + + this.setState(CallState.Ringing); + + if (event.getLocalAge()) { + setTimeout(() => { + if (this.state == CallState.Ringing) { + logger.debug("Call invite has expired. Hanging up."); + this.hangupParty = CallParty.Remote; // effectively + this.setState(CallState.Ended); + this.stopAllMedia(); + if (this.peerConn.signalingState != 'closed') { + this.peerConn.close(); + } + this.emit(CallEvent.Hangup); + } + }, invite.lifetime - event.getLocalAge()); + } + } + + /** + * Configure this call from a hangup or reject event. Used by MatrixClient. + * @param {MatrixEvent} event The m.call.hangup event + */ + public initWithHangup(event: MatrixEvent): void { + // perverse as it may seem, sometimes we want to instantiate a call with a + // hangup message (because when getting the state of the room on load, events + // come in reverse order and we want to remember that a call has been hung up) + this.setState(CallState.Ended); + } + + private shouldAnswerWithMediaType( + wantedValue: boolean | undefined, valueOfTheOtherSide: boolean | undefined, type: "audio" | "video", + ): boolean { + if (wantedValue && !valueOfTheOtherSide) { + // TODO: Figure out how to do this + logger.warn(`Unable to answer with ${type} because the other side isn't sending it either.`); + return false; + } else if ( + !utils.isNullOrUndefined(wantedValue) && + wantedValue !== valueOfTheOtherSide && + !this.opponentSupportsSDPStreamMetadata() + ) { + logger.warn( + `Unable to answer with ${type}=${wantedValue} because the other side doesn't support it. ` + + `Answering with ${type}=${valueOfTheOtherSide}.`, + ); + return valueOfTheOtherSide; + } + return wantedValue ?? valueOfTheOtherSide; + } + + /** + * Answer a call. + */ + public async answer(audio?: boolean, video?: boolean): Promise { + if (this.inviteOrAnswerSent) return; + // TODO: Figure out how to do this + if (audio === false && video === false) throw new Error("You CANNOT answer a call without media"); + + if (!this.localUsermediaStream && !this.waitForLocalAVStream) { + const prevState = this.state; + const answerWithAudio = this.shouldAnswerWithMediaType(audio, this.hasRemoteUserMediaAudioTrack, "audio"); + const answerWithVideo = this.shouldAnswerWithMediaType(video, this.hasRemoteUserMediaVideoTrack, "video"); + + this.setState(CallState.WaitLocalMedia); + this.waitForLocalAVStream = true; + + try { + const stream = await this.client.getMediaHandler().getUserMediaStream( + answerWithAudio, answerWithVideo, + ); + this.waitForLocalAVStream = false; + const usermediaFeed = new CallFeed({ + client: this.client, + roomId: this.roomId, + userId: this.client.getUserId(), + stream, + purpose: SDPStreamMetadataPurpose.Usermedia, + audioMuted: stream.getAudioTracks().length === 0, + videoMuted: stream.getVideoTracks().length === 0, + }); + + const feeds = [usermediaFeed]; + + if (this.localScreensharingFeed) { + feeds.push(this.localScreensharingFeed); + } + + this.answerWithCallFeeds(feeds); + } catch (e) { + if (answerWithVideo) { + // Try to answer without video + logger.warn("Failed to getUserMedia(), trying to getUserMedia() without video"); + this.setState(prevState); + this.waitForLocalAVStream = false; + await this.answer(answerWithAudio, false); + } else { + this.getUserMediaFailed(e); + return; + } + } + } else if (this.waitForLocalAVStream) { + this.setState(CallState.WaitLocalMedia); + } + } + + public answerWithCallFeeds(callFeeds: CallFeed[]): void { + if (this.inviteOrAnswerSent) return; + + logger.debug(`Answering call ${this.callId}`); + + this.queueGotCallFeedsForAnswer(callFeeds); + } + + /** + * Replace this call with a new call, e.g. for glare resolution. Used by + * MatrixClient. + * @param {MatrixCall} newCall The new call. + */ + public replacedBy(newCall: MatrixCall): void { + if (this.state === CallState.WaitLocalMedia) { + logger.debug("Telling new call to wait for local media"); + newCall.waitForLocalAVStream = true; + } else if ([CallState.CreateOffer, CallState.InviteSent].includes(this.state)) { + if (newCall.direction === CallDirection.Outbound) { + newCall.queueGotCallFeedsForAnswer([]); + } else { + logger.debug("Handing local stream to new call"); + newCall.queueGotCallFeedsForAnswer(this.getLocalFeeds().map(feed => feed.clone())); + } + } + this.successor = newCall; + this.emit(CallEvent.Replaced, newCall); + this.hangup(CallErrorCode.Replaced, true); + } + + /** + * Hangup a call. + * @param {string} reason The reason why the call is being hung up. + * @param {boolean} suppressEvent True to suppress emitting an event. + */ + public hangup(reason: CallErrorCode, suppressEvent: boolean): void { + if (this.callHasEnded()) return; + + logger.debug("Ending call " + this.callId); + this.terminate(CallParty.Local, reason, !suppressEvent); + // We don't want to send hangup here if we didn't even get to sending an invite + if (this.state === CallState.WaitLocalMedia) return; + const content = {}; + // Don't send UserHangup reason to older clients + if ((this.opponentVersion && this.opponentVersion >= 1) || reason !== CallErrorCode.UserHangup) { + content["reason"] = reason; + } + this.sendVoipEvent(EventType.CallHangup, content); + } + + /** + * Reject a call + * This used to be done by calling hangup, but is a separate method and protocol + * event as of MSC2746. + */ + public reject(): void { + if (this.state !== CallState.Ringing) { + throw Error("Call must be in 'ringing' state to reject!"); + } + + if (this.opponentVersion < 1) { + logger.info( + `Opponent version is less than 1 (${this.opponentVersion}): sending hangup instead of reject`, + ); + this.hangup(CallErrorCode.UserHangup, true); + return; + } + + logger.debug("Rejecting call: " + this.callId); + this.terminate(CallParty.Local, CallErrorCode.UserHangup, true); + this.sendVoipEvent(EventType.CallReject, {}); + } + + /** + * Adds an audio and/or video track - upgrades the call + * @param {boolean} audio should add an audio track + * @param {boolean} video should add an video track + */ + private async upgradeCall( + audio: boolean, video: boolean, + ): Promise { + // We don't do call downgrades + if (!audio && !video) return; + if (!this.opponentSupportsSDPStreamMetadata()) return; + + try { + const upgradeAudio = audio && !this.hasLocalUserMediaAudioTrack; + const upgradeVideo = video && !this.hasLocalUserMediaVideoTrack; + logger.debug(`Upgrading call: audio?=${upgradeAudio} video?=${upgradeVideo}`); + + const stream = await this.client.getMediaHandler().getUserMediaStream(upgradeAudio, upgradeVideo); + if (upgradeAudio && upgradeVideo) { + if (this.hasLocalUserMediaAudioTrack) return; + if (this.hasLocalUserMediaVideoTrack) return; + + this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); + } else if (upgradeAudio) { + if (this.hasLocalUserMediaAudioTrack) return; + + const audioTrack = stream.getAudioTracks()[0]; + this.localUsermediaStream.addTrack(audioTrack); + this.peerConn.addTrack(audioTrack, this.localUsermediaStream); + } else if (upgradeVideo) { + if (this.hasLocalUserMediaVideoTrack) return; + + const videoTrack = stream.getVideoTracks()[0]; + this.localUsermediaStream.addTrack(videoTrack); + this.peerConn.addTrack(videoTrack, this.localUsermediaStream); + } + } catch (error) { + logger.error("Failed to upgrade the call", error); + this.emit(CallEvent.Error, + new CallError(CallErrorCode.NoUserMedia, "Failed to get camera access: ", error), + ); + } + } + + /** + * Returns true if this.remoteSDPStreamMetadata is defined, otherwise returns false + * @returns {boolean} can screenshare + */ + public opponentSupportsSDPStreamMetadata(): boolean { + return Boolean(this.remoteSDPStreamMetadata); + } + + /** + * If there is a screensharing stream returns true, otherwise returns false + * @returns {boolean} is screensharing + */ + public isScreensharing(): boolean { + return Boolean(this.localScreensharingStream); + } + + /** + * Starts/stops screensharing + * @param enabled the desired screensharing state + * @param {string} desktopCapturerSourceId optional id of the desktop capturer source to use + * @returns {boolean} new screensharing state + */ + public async setScreensharingEnabled( + enabled: boolean, desktopCapturerSourceId?: string, + ): Promise { + // Skip if there is nothing to do + if (enabled && this.isScreensharing()) { + logger.warn(`There is already a screensharing stream - there is nothing to do!`); + return true; + } else if (!enabled && !this.isScreensharing()) { + logger.warn(`There already isn't a screensharing stream - there is nothing to do!`); + return false; + } + + // Fallback to replaceTrack() + if (!this.opponentSupportsSDPStreamMetadata()) { + return await this.setScreensharingEnabledWithoutMetadataSupport(enabled, desktopCapturerSourceId); + } + + logger.debug(`Set screensharing enabled? ${enabled}`); + if (enabled) { + try { + const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId); + if (!stream) return false; + this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare); + return true; + } catch (err) { + logger.error("Failed to get screen-sharing stream:", err); + return false; + } + } else { + for (const sender of this.screensharingSenders) { + this.peerConn.removeTrack(sender); + } + this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream); + this.deleteFeedByStream(this.localScreensharingStream); + return false; + } + } + + /** + * Starts/stops screensharing + * Should be used ONLY if the opponent doesn't support SDPStreamMetadata + * @param enabled the desired screensharing state + * @param {string} desktopCapturerSourceId optional id of the desktop capturer source to use + * @returns {boolean} new screensharing state + */ + private async setScreensharingEnabledWithoutMetadataSupport( + enabled: boolean, desktopCapturerSourceId?: string, + ): Promise { + logger.debug(`Set screensharing enabled? ${enabled} using replaceTrack()`); + if (enabled) { + try { + const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId); + if (!stream) return false; + + const track = stream.getTracks().find((track) => { + return track.kind === "video"; + }); + const sender = this.usermediaSenders.find((sender) => { + return sender.track?.kind === "video"; + }); + sender.replaceTrack(track); + + this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare, false); + + return true; + } catch (err) { + logger.error("Failed to get screen-sharing stream:", err); + return false; + } + } else { + const track = this.localUsermediaStream.getTracks().find((track) => { + return track.kind === "video"; + }); + const sender = this.usermediaSenders.find((sender) => { + return sender.track?.kind === "video"; + }); + sender.replaceTrack(track); + + this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream); + this.deleteFeedByStream(this.localScreensharingStream); + + return false; + } + } + + /** + * Request a new local usermedia stream with the current device id. + */ + public async updateLocalUsermediaStream(stream: MediaStream) { + const callFeed = this.localUsermediaFeed; + callFeed.setNewStream(stream); + const micShouldBeMuted = callFeed.isAudioMuted() || this.remoteOnHold; + const vidShouldBeMuted = callFeed.isVideoMuted() || this.remoteOnHold; + setTracksEnabled(stream.getAudioTracks(), !micShouldBeMuted); + setTracksEnabled(stream.getVideoTracks(), !vidShouldBeMuted); + + const newSenders = []; + + for (const track of stream.getTracks()) { + const oldSender = this.usermediaSenders.find((sender) => { + return sender.track?.kind === track.kind; + }); + + let newSender: RTCRtpSender; + + try { + logger.info( + `Replacing track (` + + `id="${track.id}", ` + + `kind="${track.kind}", ` + + `streamId="${stream.id}", ` + + `streamPurpose="${callFeed.purpose}"` + + `) to peer connection`, + ); + await oldSender.replaceTrack(track); + newSender = oldSender; + } catch (error) { + logger.info( + `Adding track (` + + `id="${track.id}", ` + + `kind="${track.kind}", ` + + `streamId="${stream.id}", ` + + `streamPurpose="${callFeed.purpose}"` + + `) to peer connection`, + ); + newSender = this.peerConn.addTrack(track, stream); + } + + newSenders.push(newSender); + } + + this.usermediaSenders = newSenders; + } + + /** + * Set whether our outbound video should be muted or not. + * @param {boolean} muted True to mute the outbound video. + * @returns the new mute state + */ + public async setLocalVideoMuted(muted: boolean): Promise { + if (!await this.client.getMediaHandler().hasVideoDevice()) { + return this.isLocalVideoMuted(); + } + + if (!this.hasLocalUserMediaVideoTrack && !muted) { + await this.upgradeCall(false, true); + return this.isLocalVideoMuted(); + } + this.localUsermediaFeed?.setVideoMuted(muted); + this.updateMuteStatus(); + return this.isLocalVideoMuted(); + } + + /** + * Check if local video is muted. + * + * If there are multiple video tracks, all of the tracks need to be muted + * for this to return true. This means if there are no video tracks, this will + * return true. + * @return {Boolean} True if the local preview video is muted, else false + * (including if the call is not set up yet). + */ + public isLocalVideoMuted(): boolean { + return this.localUsermediaFeed?.isVideoMuted(); + } + + /** + * Set whether the microphone should be muted or not. + * @param {boolean} muted True to mute the mic. + * @returns the new mute state + */ + public async setMicrophoneMuted(muted: boolean): Promise { + if (!await this.client.getMediaHandler().hasAudioDevice()) { + return this.isMicrophoneMuted(); + } + + if (!this.hasLocalUserMediaAudioTrack && !muted) { + await this.upgradeCall(true, false); + return this.isMicrophoneMuted(); + } + this.localUsermediaFeed?.setAudioMuted(muted); + this.updateMuteStatus(); + return this.isMicrophoneMuted(); + } + + /** + * Check if the microphone is muted. + * + * If there are multiple audio tracks, all of the tracks need to be muted + * for this to return true. This means if there are no audio tracks, this will + * return true. + * @return {Boolean} True if the mic is muted, else false (including if the call + * is not set up yet). + */ + public isMicrophoneMuted(): boolean { + return this.localUsermediaFeed?.isAudioMuted(); + } + + /** + * @returns true if we have put the party on the other side of the call on hold + * (that is, we are signalling to them that we are not listening) + */ + public isRemoteOnHold(): boolean { + return this.remoteOnHold; + } + + public setRemoteOnHold(onHold: boolean): void { + if (this.isRemoteOnHold() === onHold) return; + this.remoteOnHold = onHold; + + for (const transceiver of this.peerConn.getTransceivers()) { + // We don't send hold music or anything so we're not actually + // sending anything, but sendrecv is fairly standard for hold and + // it makes it a lot easier to figure out who's put who on hold. + transceiver.direction = onHold ? 'sendonly' : 'sendrecv'; + } + this.updateMuteStatus(); + + this.emit(CallEvent.RemoteHoldUnhold, this.remoteOnHold); + } + + /** + * Indicates whether we are 'on hold' to the remote party (ie. if true, + * they cannot hear us). + * @returns true if the other party has put us on hold + */ + public isLocalOnHold(): boolean { + if (this.state !== CallState.Connected) return false; + + let callOnHold = true; + + // We consider a call to be on hold only if *all* the tracks are on hold + // (is this the right thing to do?) + for (const transceiver of this.peerConn.getTransceivers()) { + const trackOnHold = ['inactive', 'recvonly'].includes(transceiver.currentDirection); + + if (!trackOnHold) callOnHold = false; + } + + return callOnHold; + } + + /** + * Sends a DTMF digit to the other party + * @param digit The digit (nb. string - '#' and '*' are dtmf too) + */ + public sendDtmfDigit(digit: string): void { + for (const sender of this.peerConn.getSenders()) { + if (sender.track.kind === 'audio' && sender.dtmf) { + sender.dtmf.insertDTMF(digit); + return; + } + } + + throw new Error("Unable to find a track to send DTMF on"); + } + + private updateMuteStatus(): void { + this.sendVoipEvent(EventType.CallSDPStreamMetadataChangedPrefix, { + [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(), + }); + + const micShouldBeMuted = this.localUsermediaFeed?.isAudioMuted() || this.remoteOnHold; + const vidShouldBeMuted = this.localUsermediaFeed?.isVideoMuted() || this.remoteOnHold; + + setTracksEnabled(this.localUsermediaStream.getAudioTracks(), !micShouldBeMuted); + setTracksEnabled(this.localUsermediaStream.getVideoTracks(), !vidShouldBeMuted); + } + + private gotCallFeedsForInvite(callFeeds: CallFeed[], requestScreenshareFeed = false): void { + if (this.successor) { + this.successor.queueGotCallFeedsForAnswer(callFeeds); + return; + } + if (this.callHasEnded()) { + this.stopAllMedia(); + return; + } + + for (const feed of callFeeds) { + this.pushLocalFeed(feed); + } + + if (requestScreenshareFeed) { + this.peerConn.addTransceiver("video", { + direction: "recvonly", + }); + } + + this.setState(CallState.CreateOffer); + + logger.debug("gotUserMediaForInvite"); + // Now we wait for the negotiationneeded event + } + + private async sendAnswer(): Promise { + const answerContent = { + answer: { + sdp: this.peerConn.localDescription.sdp, + // type is now deprecated as of Matrix VoIP v1, but + // required to still be sent for backwards compat + type: this.peerConn.localDescription.type, + }, + [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(true), + } as MCallAnswer; + + answerContent.capabilities = { + 'm.call.transferee': this.client.supportsCallTransfer, + 'm.call.dtmf': false, + }; + + // We have just taken the local description from the peerConn which will + // contain all the local candidates added so far, so we can discard any candidates + // we had queued up because they'll be in the answer. + logger.info(`Discarding ${this.candidateSendQueue.length} candidates that will be sent in answer`); + this.candidateSendQueue = []; + + try { + await this.sendVoipEvent(EventType.CallAnswer, answerContent); + // If this isn't the first time we've tried to send the answer, + // we may have candidates queued up, so send them now. + this.inviteOrAnswerSent = true; + } catch (error) { + // We've failed to answer: back to the ringing state + this.setState(CallState.Ringing); + this.client.cancelPendingEvent(error.event); + + let code = CallErrorCode.SendAnswer; + let message = "Failed to send answer"; + if (error.name == 'UnknownDeviceError') { + code = CallErrorCode.UnknownDevices; + message = "Unknown devices present in the room"; + } + this.emit(CallEvent.Error, new CallError(code, message, error)); + throw error; + } + + // error handler re-throws so this won't happen on error, but + // we don't want the same error handling on the candidate queue + this.sendCandidateQueue(); + } + + private queueGotCallFeedsForAnswer(callFeeds: CallFeed[]): void { + // Ensure only one negotiate/answer event is being processed at a time. + if (this.responsePromiseChain) { + this.responsePromiseChain = + this.responsePromiseChain.then(() => this.gotCallFeedsForAnswer(callFeeds)); + } else { + this.responsePromiseChain = this.gotCallFeedsForAnswer(callFeeds); + } + } + + private async gotCallFeedsForAnswer(callFeeds: CallFeed[]): Promise { + if (this.callHasEnded()) return; + + this.waitForLocalAVStream = false; + + for (const feed of callFeeds) { + this.pushLocalFeed(feed); + } + + this.setState(CallState.CreateAnswer); + + let myAnswer; + try { + this.getRidOfRTXCodecs(); + myAnswer = await this.peerConn.createAnswer(); + } catch (err) { + logger.debug("Failed to create answer: ", err); + this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true); + return; + } + + try { + await this.peerConn.setLocalDescription(myAnswer); + this.setState(CallState.Connecting); + + // Allow a short time for initial candidates to be gathered + await new Promise(resolve => { + setTimeout(resolve, 200); + }); + + this.sendAnswer(); + } catch (err) { + logger.debug("Error setting local description!", err); + this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); + return; + } + + // HACK: Safari doesn't like it when we reuse MediaStreams. In most cases + // we can get around this by calling MediaStream.clone(), however inbound + // calls seem to still be broken unless we getUserMedia again and replace + // all MediaStreams using sender.replaceTrack + if (isSafari) { + await new Promise(resolve => { + setTimeout(resolve, 200); + }); + + if (this.state === CallState.Ended) { + return; + } + + const callFeed = this.localUsermediaFeed; + const stream = callFeed.stream; + + if (!stream.active) { + throw new Error(`Call ${this.callId} has an inactive stream ${ + stream.id} and its tracks cannot be replaced`); + } + + const newSenders = []; + + for (const track of this.localUsermediaStream.getTracks()) { + const oldSender = this.usermediaSenders.find((sender) => { + return sender.track?.kind === track.kind; + }); + + if (track.readyState === "ended") { + throw new Error(`Call ${this.callId} tried to replace track ${track.id} in the ended state`); + } + + let newSender: RTCRtpSender; + + try { + logger.info( + `Replacing track (` + + `id="${track.id}", ` + + `kind="${track.kind}", ` + + `streamId="${stream.id}", ` + + `streamPurpose="${callFeed.purpose}"` + + `) to peer connection`, + ); + await oldSender.replaceTrack(track); + newSender = oldSender; + } catch (error) { + logger.info( + `Adding track (` + + `id="${track.id}", ` + + `kind="${track.kind}", ` + + `streamId="${stream.id}", ` + + `streamPurpose="${callFeed.purpose}"` + + `) to peer connection`, + ); + newSender = this.peerConn.addTrack(track, stream); + } + + newSenders.push(newSender); + } + + this.usermediaSenders = newSenders; + } + } + + /** + * Internal + * @param {Object} event + */ + private gotLocalIceCandidate = (event: RTCPeerConnectionIceEvent): Promise => { + if (event.candidate) { + logger.debug( + "Call " + this.callId + " got local ICE " + event.candidate.sdpMid + " candidate: " + + event.candidate.candidate, + ); + + if (this.callHasEnded()) return; + + // As with the offer, note we need to make a copy of this object, not + // pass the original: that broke in Chrome ~m43. + if (event.candidate.candidate !== '' || !this.sentEndOfCandidates) { + this.queueCandidate(event.candidate); + + if (event.candidate.candidate === '') this.sentEndOfCandidates = true; + } + } + }; + + private onIceGatheringStateChange = (event: Event): void => { + logger.debug("ice gathering state changed to " + this.peerConn.iceGatheringState); + if (this.peerConn.iceGatheringState === 'complete' && !this.sentEndOfCandidates) { + // If we didn't get an empty-string candidate to signal the end of candidates, + // create one ourselves now gathering has finished. + // We cast because the interface lists all the properties as required but we + // only want to send 'candidate' + // XXX: We probably want to send either sdpMid or sdpMLineIndex, as it's not strictly + // correct to have a candidate that lacks both of these. We'd have to figure out what + // previous candidates had been sent with and copy them. + const c = { + candidate: '', + } as RTCIceCandidate; + this.queueCandidate(c); + this.sentEndOfCandidates = true; + } + }; + + public async onRemoteIceCandidatesReceived(ev: MatrixEvent): Promise { + if (this.callHasEnded()) { + //debuglog("Ignoring remote ICE candidate because call has ended"); + return; + } + + const content = ev.getContent(); + const candidates = content.candidates; + if (!candidates) { + logger.info("Ignoring candidates event with no candidates!"); + return; + } + + const fromPartyId = content.version === 0 ? null : content.party_id || null; + + if (this.opponentPartyId === undefined) { + // we haven't picked an opponent yet so save the candidates + logger.info(`Buffering ${candidates.length} candidates until we pick an opponent`); + const bufferedCandidates = this.remoteCandidateBuffer.get(fromPartyId) || []; + bufferedCandidates.push(...candidates); + this.remoteCandidateBuffer.set(fromPartyId, bufferedCandidates); + return; + } + + if (!this.partyIdMatches(content)) { + logger.info( + `Ignoring candidates from party ID ${content.party_id}: ` + + `we have chosen party ID ${this.opponentPartyId}`, + ); + + return; + } + + await this.addIceCandidates(candidates); + } + + /** + * Used by MatrixClient. + * @param {Object} msg + */ + public async onAnswerReceived(event: MatrixEvent): Promise { + const content = event.getContent(); + logger.debug(`Got answer for call ID ${this.callId} from party ID ${content.party_id}`); + + if (this.callHasEnded()) { + logger.debug(`Ignoring answer because call ID ${this.callId} has ended`); + return; + } + + if (this.opponentPartyId !== undefined) { + logger.info( + `Ignoring answer from party ID ${content.party_id}: ` + + `we already have an answer/reject from ${this.opponentPartyId}`, + ); + return; + } + + this.chooseOpponent(event); + await this.addBufferedIceCandidates(); + + this.setState(CallState.Connecting); + + const sdpStreamMetadata = content[SDPStreamMetadataKey]; + if (sdpStreamMetadata) { + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); + } else { + logger.warn("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); + } + + try { + await this.peerConn.setRemoteDescription(content.answer); + } catch (e) { + logger.debug("Failed to set remote description", e); + this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); + return; + } + + // If the answer we selected has a party_id, send a select_answer event + // We do this after setting the remote description since otherwise we'd block + // call setup on it + if (this.opponentPartyId !== null) { + try { + await this.sendVoipEvent(EventType.CallSelectAnswer, { + selected_party_id: this.opponentPartyId, + }); + } catch (err) { + // This isn't fatal, and will just mean that if another party has raced to answer + // the call, they won't know they got rejected, so we carry on & don't retry. + logger.warn("Failed to send select_answer event", err); + } + } + } + + public async onSelectAnswerReceived(event: MatrixEvent): Promise { + if (this.direction !== CallDirection.Inbound) { + logger.warn("Got select_answer for an outbound call: ignoring"); + return; + } + + const selectedPartyId = event.getContent().selected_party_id; + + if (selectedPartyId === undefined || selectedPartyId === null) { + logger.warn("Got nonsensical select_answer with null/undefined selected_party_id: ignoring"); + return; + } + + if (selectedPartyId !== this.ourPartyId) { + logger.info(`Got select_answer for party ID ${selectedPartyId}: we are party ID ${this.ourPartyId}.`); + // The other party has picked somebody else's answer + this.terminate(CallParty.Remote, CallErrorCode.AnsweredElsewhere, true); + } + } + + public async onNegotiateReceived(event: MatrixEvent): Promise { + const content = event.getContent(); + const description = content.description; + if (!description || !description.sdp || !description.type) { + logger.info("Ignoring invalid m.call.negotiate event"); + return; + } + // Politeness always follows the direction of the call: in a glare situation, + // we pick either the inbound or outbound call, so one side will always be + // inbound and one outbound + const polite = this.direction === CallDirection.Inbound; + + // Here we follow the perfect negotiation logic from + // https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation + const offerCollision = ( + (description.type === 'offer') && + (this.makingOffer || this.peerConn.signalingState !== 'stable') + ); + + this.ignoreOffer = !polite && offerCollision; + if (this.ignoreOffer) { + logger.info("Ignoring colliding negotiate event because we're impolite"); + return; + } + + const prevLocalOnHold = this.isLocalOnHold(); + + const sdpStreamMetadata = content[SDPStreamMetadataKey]; + if (sdpStreamMetadata) { + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); + } else { + logger.warn("Received negotiation event without SDPStreamMetadata!"); + } + + try { + await this.peerConn.setRemoteDescription(description); + + if (description.type === 'offer') { + this.getRidOfRTXCodecs(); + await this.peerConn.setLocalDescription(); + + this.sendVoipEvent(EventType.CallNegotiate, { + description: this.peerConn.localDescription, + [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(true), + }); + } + } catch (err) { + logger.warn("Failed to complete negotiation", err); + } + + const newLocalOnHold = this.isLocalOnHold(); + if (prevLocalOnHold !== newLocalOnHold) { + this.emit(CallEvent.LocalHoldUnhold, newLocalOnHold); + // also this one for backwards compat + this.emit(CallEvent.HoldUnhold, newLocalOnHold); + } + } + + private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { + this.remoteSDPStreamMetadata = utils.recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); + for (const feed of this.getRemoteFeeds()) { + const streamId = feed.stream.id; + feed.setAudioMuted(this.remoteSDPStreamMetadata[streamId]?.audio_muted); + feed.setVideoMuted(this.remoteSDPStreamMetadata[streamId]?.video_muted); + feed.purpose = this.remoteSDPStreamMetadata[streamId]?.purpose; + } + } + + public onSDPStreamMetadataChangedReceived(event: MatrixEvent): void { + const content = event.getContent(); + const metadata = content[SDPStreamMetadataKey]; + this.updateRemoteSDPStreamMetadata(metadata); + } + + public async onAssertedIdentityReceived(event: MatrixEvent): Promise { + const content = event.getContent(); + if (!content.asserted_identity) return; + + this.remoteAssertedIdentity = { + id: content.asserted_identity.id, + displayName: content.asserted_identity.display_name, + }; + this.emit(CallEvent.AssertedIdentityChanged); + } + + private callHasEnded(): boolean { + // This exists as workaround to typescript trying to be clever and erroring + // when putting if (this.state === CallState.Ended) return; twice in the same + // function, even though that function is async. + return this.state === CallState.Ended; + } + + private queueGotLocalOffer(): void { + // Ensure only one negotiate/answer event is being processed at a time. + if (this.responsePromiseChain) { + this.responsePromiseChain = + this.responsePromiseChain.then(() => this.wrappedGotLocalOffer()); + } else { + this.responsePromiseChain = this.wrappedGotLocalOffer(); + } + } + + private async wrappedGotLocalOffer(): Promise { + this.makingOffer = true; + try { + this.getRidOfRTXCodecs(); + await this.gotLocalOffer(); + } catch (e) { + this.getLocalOfferFailed(e); + return; + } finally { + this.makingOffer = false; + } + } + + private async gotLocalOffer(): Promise { + logger.debug("Setting local description"); + + if (this.callHasEnded()) { + logger.debug("Ignoring newly created offer on call ID " + this.callId + + " because the call has ended"); + return; + } + + try { + await this.peerConn.setLocalDescription(); + } catch (err) { + logger.debug("Error setting local description!", err); + this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); + return; + } + + if (this.peerConn.iceGatheringState === 'gathering') { + // Allow a short time for initial candidates to be gathered + await new Promise(resolve => { + setTimeout(resolve, 200); + }); + } + + if (this.callHasEnded()) return; + + const eventType = this.state === CallState.CreateOffer ? EventType.CallInvite : EventType.CallNegotiate; + + const content = { + lifetime: CALL_TIMEOUT_MS, + } as MCallInviteNegotiate; + + if (eventType === EventType.CallInvite && this.invitee) { + content.invitee = this.invitee; + } + + // clunky because TypeScript can't follow the types through if we use an expression as the key + if (this.state === CallState.CreateOffer) { + content.offer = this.peerConn.localDescription; + } else { + content.description = this.peerConn.localDescription; + } + + content.capabilities = { + 'm.call.transferee': this.client.supportsCallTransfer, + 'm.call.dtmf': false, + }; + + content[SDPStreamMetadataKey] = this.getLocalSDPStreamMetadata(true); + + // Get rid of any candidates waiting to be sent: they'll be included in the local + // description we just got and will send in the offer. + logger.info(`Discarding ${this.candidateSendQueue.length} candidates that will be sent in offer`); + this.candidateSendQueue = []; + + try { + await this.sendVoipEvent(eventType, content); + } catch (error) { + logger.error("Failed to send invite", error); + if (error.event) this.client.cancelPendingEvent(error.event); + + let code = CallErrorCode.SignallingFailed; + let message = "Signalling failed"; + if (this.state === CallState.CreateOffer) { + code = CallErrorCode.SendInvite; + message = "Failed to send invite"; + } + if (error.name == 'UnknownDeviceError') { + code = CallErrorCode.UnknownDevices; + message = "Unknown devices present in the room"; + } + + this.emit(CallEvent.Error, new CallError(code, message, error)); + this.terminate(CallParty.Local, code, false); + + // no need to carry on & send the candidate queue, but we also + // don't want to rethrow the error + return; + } + + this.sendCandidateQueue(); + if (this.state === CallState.CreateOffer) { + this.inviteOrAnswerSent = true; + this.setState(CallState.InviteSent); + this.inviteTimeout = setTimeout(() => { + this.inviteTimeout = null; + if (this.state === CallState.InviteSent) { + this.hangup(CallErrorCode.InviteTimeout, false); + } + }, CALL_TIMEOUT_MS); + } + } + + private getLocalOfferFailed = (err: Error): void => { + logger.error("Failed to get local offer", err); + + this.emit( + CallEvent.Error, + new CallError( + CallErrorCode.LocalOfferFailed, + "Failed to get local offer!", err, + ), + ); + this.terminate(CallParty.Local, CallErrorCode.LocalOfferFailed, false); + }; + + private getUserMediaFailed = (err: Error): void => { + if (this.successor) { + this.successor.getUserMediaFailed(err); + return; + } + + logger.warn("Failed to get user media - ending call", err); + + this.emit( + CallEvent.Error, + new CallError( + CallErrorCode.NoUserMedia, + "Couldn't start capturing media! Is your microphone set up and " + + "does this app have permission?", err, + ), + ); + this.terminate(CallParty.Local, CallErrorCode.NoUserMedia, false); + }; + + private onIceConnectionStateChanged = (): void => { + if (this.callHasEnded()) { + return; // because ICE can still complete as we're ending the call + } + logger.debug( + "Call ID " + this.callId + ": ICE connection state changed to: " + this.peerConn.iceConnectionState, + ); + // ideally we'd consider the call to be connected when we get media but + // chrome doesn't implement any of the 'onstarted' events yet + if (this.peerConn.iceConnectionState == 'connected') { + clearTimeout(this.iceDisconnectedTimeout); + this.setState(CallState.Connected); + + if (!this.callLengthInterval) { + this.callLengthInterval = setInterval(() => { + this.callLength++; + this.emit(CallEvent.LengthChanged, this.callLength); + }, 1000); + } + } else if (this.peerConn.iceConnectionState == 'failed') { + // Firefox for Android does not yet have support for restartIce() + if (this.peerConn.restartIce) { + this.peerConn.restartIce(); + } else { + this.hangup(CallErrorCode.IceFailed, false); + } + } else if (this.peerConn.iceConnectionState == 'disconnected') { + this.iceDisconnectedTimeout = setTimeout(() => { + this.hangup(CallErrorCode.IceFailed, false); + }, 30 * 1000); + } + }; + + private onSignallingStateChanged = (): void => { + logger.debug( + "call " + this.callId + ": Signalling state changed to: " + + this.peerConn.signalingState, + ); + }; + + private onTrack = (ev: RTCTrackEvent): void => { + if (ev.streams.length === 0) { + logger.warn(`Streamless ${ev.track.kind} found: ignoring.`); + return; + } + + const stream = ev.streams[0]; + this.pushRemoteFeed(stream); + stream.addEventListener("removetrack", () => { + logger.log(`Removing track streamId: ${stream.id}`); + this.deleteFeedByStream(stream); + }); + }; + + private onDataChannel = (ev: RTCDataChannelEvent): void => { + this.emit(CallEvent.DataChannel, ev.channel); + }; + + /** + * This method removes all video/rtx codecs from screensharing video + * transceivers. This is necessary since they can cause problems. Without + * this the following steps should produce an error: + * Chromium calls Firefox + * Firefox answers + * Firefox starts screen-sharing + * Chromium starts screen-sharing + * Call crashes for Chromium with: + * [96685:23:0518/162603.933321:ERROR:webrtc_video_engine.cc(3296)] RTX codec (PT=97) mapped to PT=96 which is not in the codec list. + * [96685:23:0518/162603.933377:ERROR:webrtc_video_engine.cc(1171)] GetChangedRecvParameters called without any video codecs. + * [96685:23:0518/162603.933430:ERROR:sdp_offer_answer.cc(4302)] Failed to set local video description recv parameters for m-section with mid='2'. (INVALID_PARAMETER) + */ + private getRidOfRTXCodecs(): void { + // RTCRtpReceiver.getCapabilities and RTCRtpSender.getCapabilities don't seem to be supported on FF + if (!RTCRtpReceiver.getCapabilities || !RTCRtpSender.getCapabilities) return; + + const recvCodecs = RTCRtpReceiver.getCapabilities("video").codecs; + const sendCodecs = RTCRtpSender.getCapabilities("video").codecs; + const codecs = [...sendCodecs, ...recvCodecs]; + + for (const codec of codecs) { + if (codec.mimeType === "video/rtx") { + const rtxCodecIndex = codecs.indexOf(codec); + codecs.splice(rtxCodecIndex, 1); + } + } + + for (const trans of this.peerConn.getTransceivers()) { + if ( + this.screensharingSenders.includes(trans.sender) && + ( + trans.sender.track?.kind === "video" || + trans.receiver.track?.kind === "video" + ) + ) { + trans.setCodecPreferences(codecs); + } + } + } + + private onNegotiationNeeded = async (): Promise => { + logger.info("Negotiation is needed!"); + + if (this.state !== CallState.CreateOffer && this.opponentVersion === 0) { + logger.info("Opponent does not support renegotiation: ignoring negotiationneeded event"); + return; + } + + this.queueGotLocalOffer(); + }; + + public onHangupReceived = (msg: MCallHangupReject): void => { + logger.debug("Hangup received for call ID " + this.callId); + + // party ID must match (our chosen partner hanging up the call) or be undefined (we haven't chosen + // a partner yet but we're treating the hangup as a reject as per VoIP v0) + if (this.partyIdMatches(msg) || this.state === CallState.Ringing) { + // default reason is user_hangup + this.terminate(CallParty.Remote, msg.reason || CallErrorCode.UserHangup, true); + } else { + logger.info(`Ignoring message from party ID ${msg.party_id}: our partner is ${this.opponentPartyId}`); + } + }; + + public onRejectReceived = (msg: MCallHangupReject): void => { + logger.debug("Reject received for call ID " + this.callId); + + // No need to check party_id for reject because if we'd received either + // an answer or reject, we wouldn't be in state InviteSent + + const shouldTerminate = ( + // reject events also end the call if it's ringing: it's another of + // our devices rejecting the call. + ([CallState.InviteSent, CallState.Ringing].includes(this.state)) || + // also if we're in the init state and it's an inbound call, since + // this means we just haven't entered the ringing state yet + this.state === CallState.Fledgling && this.direction === CallDirection.Inbound + ); + + if (shouldTerminate) { + this.terminate(CallParty.Remote, msg.reason || CallErrorCode.UserHangup, true); + } else { + logger.debug(`Call is in state: ${this.state}: ignoring reject`); + } + }; + + public onAnsweredElsewhere = (msg: MCallAnswer): void => { + logger.debug("Call ID " + this.callId + " answered elsewhere"); + this.terminate(CallParty.Remote, CallErrorCode.AnsweredElsewhere, true); + }; + + private setState(state: CallState): void { + const oldState = this.state; + this.state = state; + this.emit(CallEvent.State, state, oldState); + } + + /** + * Internal + * @param {string} eventType + * @param {Object} content + * @return {Promise} + */ + private sendVoipEvent(eventType: string, content: object): Promise { + const realContent = Object.assign({}, content, { + version: VOIP_PROTO_VERSION, + call_id: this.callId, + party_id: this.ourPartyId, + conf_id: this.groupCallId, + }); + + if (this.opponentDeviceId) { + this.emit(CallEvent.SendVoipEvent, { + type: "toDevice", + eventType, + userId: this.invitee || this.getOpponentMember().userId, + opponentDeviceId: this.opponentDeviceId, + content: { + ...realContent, + device_id: this.client.deviceId, + sender_session_id: this.client.getSessionId(), + dest_session_id: this.opponentSessionId, + }, + }); + + return this.client.sendToDevice(eventType, { + [this.invitee || this.getOpponentMember().userId]: { + [this.opponentDeviceId]: { + ...realContent, + device_id: this.client.deviceId, + sender_session_id: this.client.getSessionId(), + dest_session_id: this.opponentSessionId, + }, + }, + }); + } else { + this.emit(CallEvent.SendVoipEvent, { + type: "sendEvent", + eventType, + roomId: this.roomId, + content: realContent, + userId: this.invitee || this.getOpponentMember().userId, + }); + + return this.client.sendEvent(this.roomId, eventType, realContent); + } + } + + private queueCandidate(content: RTCIceCandidate): void { + // We partially de-trickle candidates by waiting for `delay` before sending them + // amalgamated, in order to avoid sending too many m.call.candidates events and hitting + // rate limits in Matrix. + // In practice, it'd be better to remove rate limits for m.call.* + + // N.B. this deliberately lets you queue and send blank candidates, which MSC2746 + // currently proposes as the way to indicate that candidate gathering is complete. + // This will hopefully be changed to an explicit rather than implicit notification + // shortly. + this.candidateSendQueue.push(content); + + // Don't send the ICE candidates yet if the call is in the ringing state: this + // means we tried to pick (ie. started generating candidates) and then failed to + // send the answer and went back to the ringing state. Queue up the candidates + // to send if we successfully send the answer. + // Equally don't send if we haven't yet sent the answer because we can send the + // first batch of candidates along with the answer + if (this.state === CallState.Ringing || !this.inviteOrAnswerSent) return; + + // MSC2746 recommends these values (can be quite long when calling because the + // callee will need a while to answer the call) + const delay = this.direction === CallDirection.Inbound ? 500 : 2000; + + if (this.candidateSendTries === 0) { + setTimeout(() => { + this.sendCandidateQueue(); + }, delay); + } + } + + /* + * Transfers this call to another user + */ + public async transfer(targetUserId: string): Promise { + // Fetch the target user's global profile info: their room avatar / displayname + // could be different in whatever room we share with them. + const profileInfo = await this.client.getProfileInfo(targetUserId); + + const replacementId = genCallID(); + + const body = { + replacement_id: genCallID(), + target_user: { + id: targetUserId, + display_name: profileInfo.displayname, + avatar_url: profileInfo.avatar_url, + }, + create_call: replacementId, + } as MCallReplacesEvent; + + await this.sendVoipEvent(EventType.CallReplaces, body); + + await this.terminate(CallParty.Local, CallErrorCode.Transfered, true); + } + + /* + * Transfers this call to the target call, effectively 'joining' the + * two calls (so the remote parties on each call are connected together). + */ + public async transferToCall(transferTargetCall?: MatrixCall): Promise { + const targetProfileInfo = await this.client.getProfileInfo(transferTargetCall.getOpponentMember().userId); + const transfereeProfileInfo = await this.client.getProfileInfo(this.getOpponentMember().userId); + + const newCallId = genCallID(); + + const bodyToTransferTarget = { + // the replacements on each side have their own ID, and it's distinct from the + // ID of the new call (but we can use the same function to generate it) + replacement_id: genCallID(), + target_user: { + id: this.getOpponentMember().userId, + display_name: transfereeProfileInfo.displayname, + avatar_url: transfereeProfileInfo.avatar_url, + }, + await_call: newCallId, + } as MCallReplacesEvent; + + await transferTargetCall.sendVoipEvent(EventType.CallReplaces, bodyToTransferTarget); + + const bodyToTransferee = { + replacement_id: genCallID(), + target_user: { + id: transferTargetCall.getOpponentMember().userId, + display_name: targetProfileInfo.displayname, + avatar_url: targetProfileInfo.avatar_url, + }, + create_call: newCallId, + } as MCallReplacesEvent; + + await this.sendVoipEvent(EventType.CallReplaces, bodyToTransferee); + + await this.terminate(CallParty.Local, CallErrorCode.Replaced, true); + await transferTargetCall.terminate(CallParty.Local, CallErrorCode.Transfered, true); + } + + private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean): Promise { + if (this.callHasEnded()) return; + + this.hangupParty = hangupParty; + this.hangupReason = hangupReason; + this.setState(CallState.Ended); + + if (this.inviteTimeout) { + clearTimeout(this.inviteTimeout); + this.inviteTimeout = null; + } + if (this.callLengthInterval) { + clearInterval(this.callLengthInterval); + this.callLengthInterval = null; + } + + this.callStatsAtEnd = await this.collectCallStats(); + + // Order is important here: first we stopAllMedia() and only then we can deleteAllFeeds() + this.stopAllMedia(); + this.deleteAllFeeds(); + + if (this.peerConn && this.peerConn.signalingState !== 'closed') { + this.peerConn.close(); + } + if (shouldEmit) { + this.emit(CallEvent.Hangup, this); + } + + this.client.callEventHandler.calls.delete(this.callId); + } + + private stopAllMedia(): void { + logger.debug(!this.groupCallId ? "Stopping all media" : "Stopping all media except local feeds" ); + for (const feed of this.feeds) { + if ( + feed.isLocal() && + feed.purpose === SDPStreamMetadataPurpose.Usermedia && + !this.groupCallId + ) { + this.client.getMediaHandler().stopUserMediaStream(feed.stream); + } else if ( + feed.isLocal() && + feed.purpose === SDPStreamMetadataPurpose.Screenshare && + !this.groupCallId + ) { + this.client.getMediaHandler().stopScreensharingStream(feed.stream); + } else if (!feed.isLocal() || !this.groupCallId) { + for (const track of feed.stream.getTracks()) { + track.stop(); + } + } + } + } + + private checkForErrorListener(): void { + if (this.listeners("error").length === 0) { + throw new Error( + "You MUST attach an error listener using call.on('error', function() {})", + ); + } + } + + private async sendCandidateQueue(): Promise { + if (this.candidateSendQueue.length === 0 || this.callHasEnded()) { + return; + } + + const candidates = this.candidateSendQueue; + this.candidateSendQueue = []; + ++this.candidateSendTries; + const content = { + candidates: candidates, + }; + logger.debug("Attempting to send " + candidates.length + " candidates"); + try { + await this.sendVoipEvent(EventType.CallCandidates, content); + // reset our retry count if we have successfully sent our candidates + // otherwise queueCandidate() will refuse to try to flush the queue + this.candidateSendTries = 0; + + // Try to send candidates again just in case we received more candidates while sending. + this.sendCandidateQueue(); + } catch (error) { + // don't retry this event: we'll send another one later as we might + // have more candidates by then. + if (error.event) this.client.cancelPendingEvent(error.event); + + // put all the candidates we failed to send back in the queue + this.candidateSendQueue.push(...candidates); + + if (this.candidateSendTries > 5) { + logger.debug( + "Failed to send candidates on attempt " + this.candidateSendTries + + ". Giving up on this call.", error, + ); + + const code = CallErrorCode.SignallingFailed; + const message = "Signalling failed"; + + this.emit(CallEvent.Error, new CallError(code, message, error)); + this.hangup(code, false); + + return; + } + + const delayMs = 500 * Math.pow(2, this.candidateSendTries); + ++this.candidateSendTries; + logger.debug("Failed to send candidates. Retrying in " + delayMs + "ms", error); + setTimeout(() => { + this.sendCandidateQueue(); + }, delayMs); + } + } + + /** + * Place a call to this room. + * @throws if you have not specified a listener for 'error' events. + * @throws if have passed audio=false. + */ + public async placeCall(audio: boolean, video: boolean): Promise { + if (!audio) { + throw new Error("You CANNOT start a call without audio"); + } + this.setState(CallState.WaitLocalMedia); + + try { + const stream = await this.client.getMediaHandler().getUserMediaStream(audio, video); + const callFeed = new CallFeed({ + client: this.client, + roomId: this.roomId, + userId: this.client.getUserId(), + stream, + purpose: SDPStreamMetadataPurpose.Usermedia, + audioMuted: stream.getAudioTracks().length === 0, + videoMuted: stream.getVideoTracks().length === 0, + }); + await this.placeCallWithCallFeeds([callFeed]); + } catch (e) { + this.getUserMediaFailed(e); + return; + } + } + + /** + * Place a call to this room with call feed. + * @param {CallFeed[]} callFeeds to use + * @throws if you have not specified a listener for 'error' events. + * @throws if have passed audio=false. + */ + public async placeCallWithCallFeeds(callFeeds: CallFeed[], requestScreenshareFeed = false): Promise { + this.checkForErrorListener(); + this.direction = CallDirection.Outbound; + + // XXX Find a better way to do this + this.client.callEventHandler.calls.set(this.callId, this); + + // make sure we have valid turn creds. Unless something's gone wrong, it should + // poll and keep the credentials valid so this should be instant. + const haveTurnCreds = await this.client.checkTurnServers(); + if (!haveTurnCreds) { + logger.warn("Failed to get TURN credentials! Proceeding with call anyway..."); + } + + // create the peer connection now so it can be gathering candidates while we get user + // media (assuming a candidate pool size is configured) + this.peerConn = this.createPeerConnection(); + this.gotCallFeedsForInvite(callFeeds, requestScreenshareFeed); + } + + private createPeerConnection(): RTCPeerConnection { + const pc = new window.RTCPeerConnection({ + iceTransportPolicy: this.forceTURN ? 'relay' : undefined, + iceServers: this.turnServers, + iceCandidatePoolSize: this.client.iceCandidatePoolSize, + }); + + // 'connectionstatechange' would be better, but firefox doesn't implement that. + pc.addEventListener('iceconnectionstatechange', this.onIceConnectionStateChanged); + pc.addEventListener('signalingstatechange', this.onSignallingStateChanged); + pc.addEventListener('icecandidate', this.gotLocalIceCandidate); + pc.addEventListener('icegatheringstatechange', this.onIceGatheringStateChange); + pc.addEventListener('track', this.onTrack); + pc.addEventListener('negotiationneeded', this.onNegotiationNeeded); + pc.addEventListener('datachannel', this.onDataChannel); + + return pc; + } + + private partyIdMatches(msg: MCallBase): boolean { + // They must either match or both be absent (in which case opponentPartyId will be null) + // Also we ignore party IDs on the invite/offer if the version is 0, so we must do the same + // here and use null if the version is 0 (woe betide any opponent sending messages in the + // same call with different versions) + const msgPartyId = msg.version === 0 ? null : msg.party_id || null; + return msgPartyId === this.opponentPartyId; + } + + // Commits to an opponent for the call + // ev: An invite or answer event + private chooseOpponent(ev: MatrixEvent): void { + // I choo-choo-choose you + const msg = ev.getContent(); + + logger.debug(`Choosing party ID ${msg.party_id} for call ID ${this.callId}`); + + this.opponentVersion = msg.version; + if (this.opponentVersion === 0) { + // set to null to indicate that we've chosen an opponent, but because + // they're v0 they have no party ID (even if they sent one, we're ignoring it) + this.opponentPartyId = null; + } else { + // set to their party ID, or if they're naughty and didn't send one despite + // not being v0, set it to null to indicate we picked an opponent with no + // party ID + this.opponentPartyId = msg.party_id || null; + } + this.opponentCaps = msg.capabilities || {} as CallCapabilities; + this.opponentMember = this.client.getRoom(this.roomId).getMember(ev.getSender()); + } + + private async addBufferedIceCandidates(): Promise { + const bufferedCandidates = this.remoteCandidateBuffer.get(this.opponentPartyId); + if (bufferedCandidates) { + logger.info(`Adding ${bufferedCandidates.length} buffered candidates for opponent ${this.opponentPartyId}`); + await this.addIceCandidates(bufferedCandidates); + } + this.remoteCandidateBuffer = null; + } + + private async addIceCandidates(candidates: RTCIceCandidate[]): Promise { + for (const candidate of candidates) { + if ( + (candidate.sdpMid === null || candidate.sdpMid === undefined) && + (candidate.sdpMLineIndex === null || candidate.sdpMLineIndex === undefined) + ) { + logger.debug("Ignoring remote ICE candidate with no sdpMid or sdpMLineIndex"); + continue; + } + logger.debug( + "Call " + this.callId + " got remote ICE " + candidate.sdpMid + " candidate: " + candidate.candidate, + ); + try { + await this.peerConn.addIceCandidate(candidate); + } catch (err) { + if (!this.ignoreOffer) { + logger.info("Failed to add remote ICE candidate", err); + } + } + } + } + + public get hasPeerConnection(): boolean { + return Boolean(this.peerConn); + } +} + +export function setTracksEnabled(tracks: Array, enabled: boolean): void { + for (let i = 0; i < tracks.length; i++) { + tracks[i].enabled = enabled; + } +} + +/** + * DEPRECATED + * Use client.createCall() + * + * Create a new Matrix call for the browser. + * @param {MatrixClient} client The client instance to use. + * @param {string} roomId The room the call is in. + * @param {Object?} options DEPRECATED optional options map. + * @param {boolean} options.forceTURN DEPRECATED whether relay through TURN should be + * forced. This option is deprecated - use opts.forceTURN when creating the matrix client + * since it's only possible to set this option on outbound calls. + * @return {MatrixCall} the call or null if the browser doesn't support calling. + */ +export function createNewMatrixCall(client: any, roomId: string, options?: CallOpts): MatrixCall { + // typeof prevents Node from erroring on an undefined reference + if (typeof(window) === 'undefined' || typeof(document) === 'undefined') { + // NB. We don't log here as apps try to create a call object as a test for + // whether calls are supported, so we shouldn't fill the logs up. + return null; + } + + // Firefox throws on so little as accessing the RTCPeerConnection when operating in + // a secure mode. There's some information at https://bugzilla.mozilla.org/show_bug.cgi?id=1542616 + // though the concern is that the browser throwing a SecurityError will brick the + // client creation process. + try { + const supported = Boolean( + window.RTCPeerConnection || window.RTCSessionDescription || + window.RTCIceCandidate || navigator.mediaDevices, + ); + if (!supported) { + // Adds a lot of noise to test runs, so disable logging there. + if (process.env.NODE_ENV !== "test") { + logger.error("WebRTC is not supported in this browser / environment"); + } + return null; + } + } catch (e) { + logger.error("Exception thrown when trying to access WebRTC", e); + return null; + } + + const optionsForceTURN = options ? options.forceTURN : false; + + const opts: CallOpts = { + client: client, + roomId: roomId, + invitee: options?.invitee, + turnServers: client.getTurnServers(), + // call level options + forceTURN: client.forceTURN || optionsForceTURN, + opponentDeviceId: options?.opponentDeviceId, + opponentSessionId: options?.opponentSessionId, + groupCallId: options?.groupCallId, + }; + const call = new MatrixCall(opts); + + client.reEmitter.reEmit(call, Object.values(CallEvent)); + + return call; +} + diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md new file mode 100644 index 00000000..33b408f6 --- /dev/null +++ b/src/matrix/calls/TODO.md @@ -0,0 +1,43 @@ +## Store ongoing calls + +Add store with all ongoing calls so when we quit and start again, we don't have to go through all the past calls to know which ones might still be ongoing. + + +## Notes + +we send m.call as state event in room + +we add m.call.participant for our own device + +we wait for other participants to add their user and device (in the sources) + +for each (userid, deviceid) + - if userId < ourUserId + - we setup a peer connection + - we wait for negotation event to get sdp + - we send an m.call.invite + - else + - wait for invite from other side + + +in some cases, we will actually send the invite to all devices (e.g. SFU), so +we probably still need to handle multiple anwsers? + +so we would send an invite to multiple devices and pick the one for which we +received the anwser first. between invite and anwser, we could already receive +ice candidates that we need to buffer. + +should a PeerCall only exist after we've received an answer? +Before that, we could have a PeerCallInvite + + + +updating the metadata: + +if we're renegotiating: use m.call.negotatie +if just muting: use m.call.sdp_stream_metadata_changed + + +party identification + - for 1:1 calls, we identify with a party_id + - for group calls, we identify with a device_id diff --git a/src/matrix/calls/group/Call.ts b/src/matrix/calls/group/Call.ts new file mode 100644 index 00000000..9abef197 --- /dev/null +++ b/src/matrix/calls/group/Call.ts @@ -0,0 +1,88 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {ObservableMap} from "../../../observable/map/ObservableMap"; + +function participantId(senderUserId: string, senderDeviceId: string | null) { + return JSON.stringify(senderUserId) + JSON.stringify(senderDeviceId); +} + +class Call { + private readonly participants: ObservableMap = new ObservableMap(); + private localMedia?: LocalMedia; + + constructor(private readonly ownUserId: string, private callEvent: StateEvent, private readonly room: Room, private readonly webRTC: WebRTC) { + + } + + get id(): string { return this.callEvent.state_key; } + + async participate(tracks: Track[]) { + this.localMedia = LocalMedia.fromTracks(tracks); + for (const [,participant] of this.participants) { + participant.setLocalMedia(this.localMedia.clone()); + } + // send m.call.member state event + + // send invite to all participants that are < my userId + for (const [,participant] of this.participants) { + if (participant.userId < this.ownUserId) { + participant.sendInvite(); + } + } + } + + updateCallEvent(callEvent: StateEvent) { + this.callEvent = callEvent; + } + + addParticipant(userId, source) { + const participantId = getParticipantId(userId, source.device_id); + const participant = this.participants.get(participantId); + if (participant) { + participant.updateSource(source); + } else { + participant.add(participantId, new Participant(userId, source.device_id, this.localMedia?.clone(), this.webRTC)); + } + } + + handleDeviceMessage(senderUserId: string, senderDeviceId: string, eventType: string, content: Record, log: ILogItem) { + const participantId = getParticipantId(senderUserId, senderDeviceId); + let peerCall = this.participants.get(participantId); + let hasDeviceInKey = true; + if (!peerCall) { + hasDeviceInKey = false; + peerCall = this.participants.get(getParticipantId(senderUserId, null)) + } + if (peerCall) { + peerCall.handleIncomingSignallingMessage(eventType, content, senderDeviceId); + if (!hasDeviceInKey && peerCall.opponentPartyId) { + this.participants.delete(getParticipantId(senderUserId, null)); + this.participants.add(getParticipantId(senderUserId, peerCall.opponentPartyId)); + } + } else { + // create peerCall + } + } + + get id(): string { + return this.callEvent.state_key; + } + + get isTerminated(): boolean { + return !!this.callEvent.content[CALL_TERMINATED]; + } +} diff --git a/src/matrix/calls/group/Participant.ts b/src/matrix/calls/group/Participant.ts new file mode 100644 index 00000000..5abdaf63 --- /dev/null +++ b/src/matrix/calls/group/Participant.ts @@ -0,0 +1,46 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +class Participant implements PeerCallHandler { + private peerCall?: PeerCall; + + constructor( + private readonly userId: string, + private readonly deviceId: string, + private localMedia: LocalMedia | undefined, + private readonly webRTC: WebRTC, + private readonly hsApi: HomeServerApi + ) {} + + sendInvite() { + this.peerCall = new PeerCall(this, this.webRTC); + this.peerCall.setLocalMedia(this.localMedia); + this.peerCall.sendOffer(); + } + + /** From PeerCallHandler + * @internal */ + override emitUpdate() { + + } + + /** From PeerCallHandler + * @internal */ + override onSendSignallingMessage() { + // TODO: this needs to be encrypted with olm first + this.hsApi.sendToDevice(type, {[this.userId]: {[this.deviceId ?? "*"]: content}}); + } +} diff --git a/src/platform/types/MediaDevices.ts b/src/platform/types/MediaDevices.ts index ec4895fa..3dda3059 100644 --- a/src/platform/types/MediaDevices.ts +++ b/src/platform/types/MediaDevices.ts @@ -32,6 +32,7 @@ export interface Track { get label(): string; get id(): string; get streamId(): string; + get settings(): MediaTrackSettings; get muted(): boolean; setMuted(muted: boolean): void; } diff --git a/src/platform/types/WebRTC.ts b/src/platform/types/WebRTC.ts index b34e4214..9a10ce5e 100644 --- a/src/platform/types/WebRTC.ts +++ b/src/platform/types/WebRTC.ts @@ -47,7 +47,8 @@ export interface PeerConnection { createOffer(): Promise; createAnswer(): Promise; setLocalDescription(description: RTCSessionDescriptionInit); - setRemoteDescription(description: RTCSessionDescriptionInit); + setRemoteDescription(description: RTCSessionDescriptionInit): Promise; + addIceCandidate(candidate: RTCIceCandidate): Promise; addTrack(track: Track): void; removeTrack(track: Track): boolean; replaceTrack(oldTrack: Track, newTrack: Track): Promise; diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index 08ccc636..7d5c8f4f 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -97,6 +97,7 @@ export class TrackWrapper implements Track { get id(): string { return this.track.id; } get streamId(): string { return this.stream.id; } get muted(): boolean { return this.track.muted; } + get settings(): MediaTrackSettings { return this.track.getSettings(); } setMuted(muted: boolean): void { this.track.enabled = !muted; diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 712e8cc6..36d5eb75 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -15,7 +15,7 @@ limitations under the License. */ import {TrackWrapper, wrapTrack} from "./MediaDevices"; -import {Track} from "../../types/MediaDevices"; +import {Track, TrackType} from "../../types/MediaDevices"; import {WebRTC, PeerConnectionHandler, DataChannel, PeerConnection, StreamPurpose} from "../../types/WebRTC"; const POLLING_INTERVAL = 200; // ms @@ -57,6 +57,10 @@ class DOMPeerConnection implements PeerConnection { this.peerConnection.setRemoteDescription(description); } + addIceCandidate(candidate: RTCIceCandidate): Promise { + return this.peerConnection.addIceCandidate(candidate); + } + addTrack(track: Track): void { if (!(track instanceof TrackWrapper)) { throw new Error("Not a TrackWrapper"); @@ -152,263 +156,10 @@ class DOMPeerConnection implements PeerConnection { let type: TrackType; if (track.kind === "video") { const purpose = this.handler.getPurposeForStreamId(stream.id); - type = purpose === StreamPurpose.Usermedia ? TrackType.Camera : TrackType.ScreenShare; + type = purpose === StreamPurpose.UserMedia ? TrackType.Camera : TrackType.ScreenShare; } else { type = TrackType.Microphone; } return wrapTrack(track, stream, type); } } - -export interface ICallFeedOpts { - client: MatrixClient; - roomId: string; - userId: string; - stream: MediaStream; - purpose: SDPStreamMetadataPurpose; - audioMuted: boolean; - videoMuted: boolean; -} - -export enum CallFeedEvent { - NewStream = "new_stream", - MuteStateChanged = "mute_state_changed", - VolumeChanged = "volume_changed", - Speaking = "speaking", -} - -export class CallFeed extends EventEmitter { - public stream: MediaStream; - public sdpMetadataStreamId: string; - public userId: string; - public purpose: SDPStreamMetadataPurpose; - public speakingVolumeSamples: number[]; - - private client: MatrixClient; - private roomId: string; - private audioMuted: boolean; - private videoMuted: boolean; - private measuringVolumeActivity = false; - private audioContext: AudioContext; - private analyser: AnalyserNode; - private frequencyBinCount: Float32Array; - private speakingThreshold = SPEAKING_THRESHOLD; - private speaking = false; - private volumeLooperTimeout: number; - - constructor(opts: ICallFeedOpts) { - super(); - - this.client = opts.client; - this.roomId = opts.roomId; - this.userId = opts.userId; - this.purpose = opts.purpose; - this.audioMuted = opts.audioMuted; - this.videoMuted = opts.videoMuted; - this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); - this.sdpMetadataStreamId = opts.stream.id; - - this.updateStream(null, opts.stream); - - if (this.hasAudioTrack) { - this.initVolumeMeasuring(); - } - } - - private get hasAudioTrack(): boolean { - return this.stream.getAudioTracks().length > 0; - } - - private updateStream(oldStream: MediaStream, newStream: MediaStream): void { - if (newStream === oldStream) return; - - if (oldStream) { - oldStream.removeEventListener("addtrack", this.onAddTrack); - this.measureVolumeActivity(false); - } - if (newStream) { - this.stream = newStream; - newStream.addEventListener("addtrack", this.onAddTrack); - - if (this.hasAudioTrack) { - this.initVolumeMeasuring(); - } else { - this.measureVolumeActivity(false); - } - } - - this.emit(CallFeedEvent.NewStream, this.stream); - } - - private initVolumeMeasuring(): void { - const AudioContext = window.AudioContext || window.webkitAudioContext; - if (!this.hasAudioTrack || !AudioContext) return; - - this.audioContext = new AudioContext(); - - this.analyser = this.audioContext.createAnalyser(); - this.analyser.fftSize = 512; - this.analyser.smoothingTimeConstant = 0.1; - - const mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.stream); - mediaStreamAudioSourceNode.connect(this.analyser); - - this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount); - } - - private onAddTrack = (): void => { - this.emit(CallFeedEvent.NewStream, this.stream); - }; - - /** - * Returns callRoom member - * @returns member of the callRoom - */ - public getMember(): RoomMember { - const callRoom = this.client.getRoom(this.roomId); - return callRoom.getMember(this.userId); - } - - /** - * Returns true if CallFeed is local, otherwise returns false - * @returns {boolean} is local? - */ - public isLocal(): boolean { - return this.userId === this.client.getUserId(); - } - - /** - * Returns true if audio is muted or if there are no audio - * tracks, otherwise returns false - * @returns {boolean} is audio muted? - */ - public isAudioMuted(): boolean { - return this.stream.getAudioTracks().length === 0 || this.audioMuted; - } - - /** - * Returns true video is muted or if there are no video - * tracks, otherwise returns false - * @returns {boolean} is video muted? - */ - public isVideoMuted(): boolean { - // We assume only one video track - return this.stream.getVideoTracks().length === 0 || this.videoMuted; - } - - public isSpeaking(): boolean { - return this.speaking; - } - - /** - * Replaces the current MediaStream with a new one. - * This method should be only used by MatrixCall. - * @param newStream new stream with which to replace the current one - */ - public setNewStream(newStream: MediaStream): void { - this.updateStream(this.stream, newStream); - } - - /** - * Set feed's internal audio mute state - * @param muted is the feed's audio muted? - */ - public setAudioMuted(muted: boolean): void { - this.audioMuted = muted; - this.speakingVolumeSamples.fill(-Infinity); - this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); - } - - /** - * Set feed's internal video mute state - * @param muted is the feed's video muted? - */ - public setVideoMuted(muted: boolean): void { - this.videoMuted = muted; - this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); - } - - /** - * Starts emitting volume_changed events where the emitter value is in decibels - * @param enabled emit volume changes - */ - public measureVolumeActivity(enabled: boolean): void { - if (enabled) { - if (!this.audioContext || !this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return; - - this.measuringVolumeActivity = true; - this.volumeLooper(); - } else { - this.measuringVolumeActivity = false; - this.speakingVolumeSamples.fill(-Infinity); - this.emit(CallFeedEvent.VolumeChanged, -Infinity); - } - } - - public setSpeakingThreshold(threshold: number) { - this.speakingThreshold = threshold; - } - - private volumeLooper = () => { - if (!this.analyser) return; - - if (!this.measuringVolumeActivity) return; - - this.analyser.getFloatFrequencyData(this.frequencyBinCount); - - let maxVolume = -Infinity; - for (let i = 0; i < this.frequencyBinCount.length; i++) { - if (this.frequencyBinCount[i] > maxVolume) { - maxVolume = this.frequencyBinCount[i]; - } - } - - this.speakingVolumeSamples.shift(); - this.speakingVolumeSamples.push(maxVolume); - - this.emit(CallFeedEvent.VolumeChanged, maxVolume); - - let newSpeaking = false; - - for (let i = 0; i < this.speakingVolumeSamples.length; i++) { - const volume = this.speakingVolumeSamples[i]; - - if (volume > this.speakingThreshold) { - newSpeaking = true; - break; - } - } - - if (this.speaking !== newSpeaking) { - this.speaking = newSpeaking; - this.emit(CallFeedEvent.Speaking, this.speaking); - } - - this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL); - }; - - public clone(): CallFeed { - const mediaHandler = this.client.getMediaHandler(); - const stream = this.stream.clone(); - - if (this.purpose === SDPStreamMetadataPurpose.Usermedia) { - mediaHandler.userMediaStreams.push(stream); - } else { - mediaHandler.screensharingStreams.push(stream); - } - - return new CallFeed({ - client: this.client, - roomId: this.roomId, - userId: this.userId, - stream, - purpose: this.purpose, - audioMuted: this.audioMuted, - videoMuted: this.videoMuted, - }); - } - - public dispose(): void { - clearTimeout(this.volumeLooperTimeout); - } -} From 98e1dcf79920b4b7d2064b4b8c33d2e521ceefb8 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Fri, 18 Feb 2022 16:38:10 +0100 Subject: [PATCH 005/323] WIP5 --- src/matrix/calls/PeerCall.ts | 2523 +++---------------------- src/matrix/calls/TODO.md | 23 + src/matrix/calls/group/Participant.ts | 10 +- src/platform/types/WebRTC.ts | 5 +- src/platform/web/dom/WebRTC.ts | 78 +- src/utils/AsyncQueue.ts | 52 + 6 files changed, 404 insertions(+), 2287 deletions(-) create mode 100644 src/utils/AsyncQueue.ts diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 8780258e..47dfaa3e 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -16,6 +16,7 @@ limitations under the License. import {ObservableMap} from "../../observable/map/ObservableMap"; +import {AsyncQueue} from "../../utils/AsyncQueue"; import type {Room} from "../room/Room"; import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; @@ -65,17 +66,6 @@ const GROUP_CALL_MEMBER_TYPE = "m.call.member"; // without MSC2746 that doesn't provide an id which device sent the message. type PartyId = string | null; -interface CallOpts { - roomId?: string; - invitee?: string; - client?: any; // Fix when client is TSified - forceTURN?: boolean; - turnServers?: Array; - opponentDeviceId?: string; - opponentSessionId?: string; - groupCallId?: string; -} - interface TurnServer { urls: Array; username?: string; @@ -115,6 +105,21 @@ export enum CallParty { Remote = 'remote', } +export enum EventType { + Invite = "m.call.invite", + Candidates = "m.call.candidates", + Answer = "m.call.answer", + Hangup = "m.call.hangup", + Reject = "m.call.reject", + SelectAnswer = "m.call.select_answer", + Negotiate = "m.call.negotiate", + SDPStreamMetadataChanged = "m.call.sdp_stream_metadata_changed", + SDPStreamMetadataChangedPrefix = "org.matrix.call.sdp_stream_metadata_changed", + Replaces = "m.call.replaces", + AssertedIdentity = "m.call.asserted_identity", + AssertedIdentityPrefix = "org.matrix.call.asserted_identity", +} + export enum CallEvent { Hangup = 'hangup', State = 'state', @@ -280,6 +285,8 @@ class LocalMedia { return new LocalMedia(cameraTrack, screenShareTrack, microphoneTrack as AudioTrack); } + get tracks(): Track[] { return []; } + getSDPMetadata(): any { const metadata = {}; const userMediaTrack = this.microphoneTrack ?? this.cameraTrack; @@ -299,14 +306,20 @@ class LocalMedia { } } -interface PeerCallHandler { - emitUpdate(); - sendSignallingMessage(type: string, content: Record); +export type InviteMessage = { + type: EventType.Invite, + content: { + version: number + } +} + +export interface PeerCallHandler { + emitUpdate(peerCall: PeerCall, params: any); + sendSignallingMessage(type: EventType, content: Record); } // when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already // do for sharing keys will be best as that already deals with room tracking. -type SendSignallingMessageCallback = (type: CallSetupMessageType, content: Record) => Promise; /** * Does WebRTC signalling for a single PeerConnection, and deals with WebRTC wrappers from platform * */ @@ -330,7 +343,6 @@ class PeerCall implements PeerConnectionHandler { private inviteOrAnswerSent = false; private waitForLocalAVStream: boolean; - private successor: MatrixCall; private opponentVersion: number | string; // The party ID of the other side: undefined if we haven't chosen a partner // yet, null if we have but they didn't send a party ID. @@ -352,17 +364,16 @@ class PeerCall implements PeerConnectionHandler { private makingOffer = false; private ignoreOffer: boolean; - private responsePromiseChain?: Promise; - // If candidates arrive before we've picked an opponent (which, in particular, // will happen if the opponent sends candidates eagerly before the user answers // the call) we buffer them up here so we can then add the ones from the party we pick - private remoteCandidateBuffer: Map = []; + private remoteCandidateBuffer: Map; private remoteAssertedIdentity: AssertedIdentity; private remoteSDPStreamMetadata?: SDPStreamMetadata; + private negotiationQueue: AsyncQueue; constructor( private readonly handler: PeerCallHandler, @@ -370,25 +381,233 @@ class PeerCall implements PeerConnectionHandler { webRTC: WebRTC ) { this.peerConnection = webRTC.createPeerConnection(this); + // TODO: should we use this to serialize all state changes? + this.negotiationQueue = new AsyncQueue(this.handleNegotiation, void); } + // PeerConnectionHandler method onIceConnectionStateChange(state: RTCIceConnectionState) {} + // PeerConnectionHandler method onLocalIceCandidate(candidate: RTCIceCandidate) {} + // PeerConnectionHandler method onIceGatheringStateChange(state: RTCIceGatheringState) {} + // PeerConnectionHandler method onRemoteTracksChanged(tracks: Track[]) {} + // PeerConnectionHandler method onDataChannelChanged(dataChannel: DataChannel | undefined) {} + // PeerConnectionHandler method onNegotiationNeeded() { + // trigger handleNegotiation + this.negotiationQueue.push(void); + } + + // calls are serialized and deduplicated by negotiationQueue + private handleNegotiation = async (): Promise => { + const offer = await this.peerConnection.createOffer(); + this.peerConnection.setLocalDescription(offer); + // need to queue this const message = { - offer: this.peerConnection.createOffer(), + offer, sdp_stream_metadata: this.localMedia.getSDPMetadata(), version: 1 } - this.handler.sendSignallingMessage(CallSetupMessageType.Invite, message); + if (this.state === CallState.Fledgling) { + const sendPromise = this.handler.sendSignallingMessage(EventType.Invite, message); + this.setState(CallState.InviteSent); + } else { + await this.handler.sendSignallingMessage(EventType.Negotiate, message); + } + }; + + async sendInvite(localMediaPromise: Promise): Promise { + if (this.state !== CallState.Fledgling) { + return; + } + this.setState(CallState.WaitLocalMedia); + this.localMedia = await localMediaPromise; + // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called + for (const t of this.localMedia.tracks) { + this.peerConnection.addTrack(t); + } + await this.waitForState(CallState.Ended, CallState.InviteSent); } - setLocalMedia(localMedia: LocalMedia) { - this.localMedia = localMedia; - // TODO: send new metadata + async sendAnswer(localMediaPromise: Promise): Promise { + if (this.callHasEnded()) return; + + if (this.state !== CallState.Ringing) { + return; + } + + this.setState(CallState.WaitLocalMedia); + this.waitForLocalAVStream = true; + this.localMedia = await localMediaPromise; + this.waitForLocalAVStream = false; + + // enqueue the following + + // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called + for (const t of this.localMedia.tracks) { + this.peerConnection.addTrack(t); + } + + this.setState(CallState.CreateAnswer); + + let myAnswer; + try { + myAnswer = await this.peerConn.createAnswer(); + } catch (err) { + logger.debug("Failed to create answer: ", err); + this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true); + return; + } + + try { + await this.peerConn.setLocalDescription(myAnswer); + this.setState(CallState.Connecting); + + // Allow a short time for initial candidates to be gathered + await new Promise(resolve => { + setTimeout(resolve, 200); + }); + // inlined sendAnswer + const answerContent = { + answer: { + sdp: this.peerConn.localDescription.sdp, + // type is now deprecated as of Matrix VoIP v1, but + // required to still be sent for backwards compat + type: this.peerConn.localDescription.type, + }, + [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(true), + } as MCallAnswer; + + answerContent.capabilities = { + 'm.call.transferee': this.client.supportsCallTransfer, + 'm.call.dtmf': false, + }; + + // We have just taken the local description from the peerConn which will + // contain all the local candidates added so far, so we can discard any candidates + // we had queued up because they'll be in the answer. + logger.info(`Discarding ${this.candidateSendQueue.length} candidates that will be sent in answer`); + this.candidateSendQueue = []; + + try { + await this.sendVoipEvent(EventType.CallAnswer, answerContent); + // If this isn't the first time we've tried to send the answer, + // we may have candidates queued up, so send them now. + this.inviteOrAnswerSent = true; + } catch (error) { + // We've failed to answer: back to the ringing state + this.setState(CallState.Ringing); + this.client.cancelPendingEvent(error.event); + + let code = CallErrorCode.SendAnswer; + let message = "Failed to send answer"; + if (error.name == 'UnknownDeviceError') { + code = CallErrorCode.UnknownDevices; + message = "Unknown devices present in the room"; + } + this.emit(CallEvent.Error, new CallError(code, message, error)); + throw error; + } + + // error handler re-throws so this won't happen on error, but + // we don't want the same error handling on the candidate queue + this.sendCandidateQueue(); + } catch (err) { + logger.debug("Error setting local description!", err); + this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); + return; + } + } + + async updateLocalMedia(localMediaPromise: Promise) { + const oldMedia = this.localMedia; + this.localMedia = await localMediaPromise; + + const applyTrack = (selectTrack: (media: LocalMedia) => Track | undefined) => { + const oldTrack = selectTrack(oldMedia); + const newTrack = selectTrack(this.localMedia); + if (oldTrack && newTrack) { + this.peerConnection.replaceTrack(oldTrack, newTrack); + } else if (oldTrack) { + this.peerConnection.removeTrack(oldTrack); + } else if (newTrack) { + this.peerConnection.addTrack(newTrack); + } + }; + + // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called + applyTrack(m => m.microphoneTrack); + applyTrack(m => m.cameraTrack); + applyTrack(m => m.screenShareTrack); + } + + + /** + * Replace this call with a new call, e.g. for glare resolution. Used by + * MatrixClient. + * @param {MatrixCall} newCall The new call. + */ + public replacedBy(newCall: MatrixCall): void { + if (this.state === CallState.WaitLocalMedia) { + logger.debug("Telling new call to wait for local media"); + newCall.waitForLocalAVStream = true; + } else if ([CallState.CreateOffer, CallState.InviteSent].includes(this.state)) { + if (newCall.direction === CallDirection.Outbound) { + newCall.queueGotCallFeedsForAnswer([]); + } else { + logger.debug("Handing local stream to new call"); + newCall.queueGotCallFeedsForAnswer(this.getLocalFeeds().map(feed => feed.clone())); + } + } + this.successor = newCall; + this.emit(CallEvent.Replaced, newCall); + this.hangup(CallErrorCode.Replaced, true); + } + + /** + * Hangup a call. + * @param {string} reason The reason why the call is being hung up. + * @param {boolean} suppressEvent True to suppress emitting an event. + */ + public hangup(reason: CallErrorCode, suppressEvent: boolean): void { + if (this.callHasEnded()) return; + + logger.debug("Ending call " + this.callId); + this.terminate(CallParty.Local, reason, !suppressEvent); + // We don't want to send hangup here if we didn't even get to sending an invite + if (this.state === CallState.WaitLocalMedia) return; + const content = {}; + // Don't send UserHangup reason to older clients + if ((this.opponentVersion && this.opponentVersion >= 1) || reason !== CallErrorCode.UserHangup) { + content["reason"] = reason; + } + this.sendVoipEvent(EventType.CallHangup, content); + } + + /** + * Reject a call + * This used to be done by calling hangup, but is a separate method and protocol + * event as of MSC2746. + */ + public reject(): void { + if (this.state !== CallState.Ringing) { + throw Error("Call must be in 'ringing' state to reject!"); + } + + if (this.opponentVersion < 1) { + logger.info( + `Opponent version is less than 1 (${this.opponentVersion}): sending hangup instead of reject`, + ); + this.hangup(CallErrorCode.UserHangup, true); + return; + } + + logger.debug("Rejecting call: " + this.callId); + this.terminate(CallParty.Local, CallErrorCode.UserHangup, true); + this.sendVoipEvent(EventType.CallReject, {}); } // request the type of incoming track @@ -402,6 +621,11 @@ class PeerCall implements PeerConnectionHandler { const oldState = this.state; this.state = state; this.handler.emitUpdate(); + if (this.inviteDeferred) { + if (this.state === CallState.InviteSent) { + this.inviteDeferred.resolve(); + } + } } handleIncomingSignallingMessage(type: CallSetupMessageType, content: Record, partyId: PartyId) { @@ -504,2254 +728,3 @@ class PeerCall implements PeerConnectionHandler { } } } - -/** - * Construct a new Matrix Call. - * @constructor - * @param {Object} opts Config options. - * @param {string} opts.roomId The room ID for this call. - * @param {Object} opts.webRtc The WebRTC globals from the browser. - * @param {boolean} opts.forceTURN whether relay through TURN should be forced. - * @param {Object} opts.URL The URL global. - * @param {Array} opts.turnServers Optional. A list of TURN servers. - * @param {MatrixClient} opts.client The Matrix Client instance to send events to. - */ -export class MatrixCall extends EventEmitter { - // should go into DirectCall/GroupCall class - public roomId: string; - public callId: string; - public invitee?: string; - public ourPartyId: string; - private opponentMember: RoomMember; - private opponentDeviceId: string; - private opponentSessionId: string; - public groupCallId: string; - private callLengthInterval: number; - private callLength = 0; - - - - public state = CallState.Fledgling; - public hangupParty: CallParty; - public hangupReason: string; - public direction: CallDirection; - public peerConn?: RTCPeerConnection; - - // A queue for candidates waiting to go out. - // We try to amalgamate candidates into a single candidate message where - // possible - private candidateSendQueue: Array = []; - private candidateSendTries = 0; - private sentEndOfCandidates = false; - - private inviteOrAnswerSent = false; - private waitForLocalAVStream: boolean; - private successor: MatrixCall; - private opponentVersion: number | string; - // The party ID of the other side: undefined if we haven't chosen a partner - // yet, null if we have but they didn't send a party ID. - private opponentPartyId: string; - private opponentCaps: CallCapabilities; - private inviteTimeout: number; - private iceDisconnectedTimeout: number; - - // The logic of when & if a call is on hold is nontrivial and explained in is*OnHold - // This flag represents whether we want the other party to be on hold - private remoteOnHold = false; - - // the stats for the call at the point it ended. We can't get these after we - // tear the call down, so we just grab a snapshot before we stop the call. - // The typescript definitions have this type as 'any' :( - private callStatsAtEnd: any[]; - - // Perfect negotiation state: https://www.w3.org/TR/webrtc/#perfect-negotiation-example - private makingOffer = false; - private ignoreOffer: boolean; - - private responsePromiseChain?: Promise; - - // If candidates arrive before we've picked an opponent (which, in particular, - // will happen if the opponent sends candidates eagerly before the user answers - // the call) we buffer them up here so we can then add the ones from the party we pick - private remoteCandidateBuffer: RTCIceCandidate[] = []; - - private remoteAssertedIdentity: AssertedIdentity; - - private remoteSDPStreamMetadata: SDPStreamMetadata; - - constructor(opts: CallOpts) { - super(); - this.roomId = opts.roomId; - this.invitee = opts.invitee; - this.client = opts.client; - this.forceTURN = opts.forceTURN; - this.ourPartyId = this.client.deviceId; - this.opponentDeviceId = opts.opponentDeviceId; - this.opponentSessionId = opts.opponentSessionId; - this.groupCallId = opts.groupCallId; - // Array of Objects with urls, username, credential keys - this.turnServers = opts.turnServers || []; - if (this.turnServers.length === 0 && this.client.isFallbackICEServerAllowed()) { - this.turnServers.push({ - urls: [FALLBACK_ICE_SERVER], - }); - } - for (const server of this.turnServers) { - utils.checkObjectHasKeys(server, ["urls"]); - } - this.callId = genCallID(); - } - - /** - * Place a voice call to this room. - * @throws If you have not specified a listener for 'error' events. - */ - public async placeVoiceCall(): Promise { - await this.placeCall(true, false); - } - - /** - * Place a video call to this room. - * @throws If you have not specified a listener for 'error' events. - */ - public async placeVideoCall(): Promise { - await this.placeCall(true, true); - } - - /** - * Create a datachannel using this call's peer connection. - * @param label A human readable label for this datachannel - * @param options An object providing configuration options for the data channel. - */ - public createDataChannel(label: string, options: RTCDataChannelInit) { - const dataChannel = this.peerConn.createDataChannel(label, options); - this.emit(CallEvent.DataChannel, dataChannel); - return dataChannel; - } - - public getOpponentMember(): RoomMember { - return this.opponentMember; - } - - public getOpponentSessionId(): string { - return this.opponentSessionId; - } - - public opponentCanBeTransferred(): boolean { - return Boolean(this.opponentCaps && this.opponentCaps["m.call.transferee"]); - } - - public opponentSupportsDTMF(): boolean { - return Boolean(this.opponentCaps && this.opponentCaps["m.call.dtmf"]); - } - - public getRemoteAssertedIdentity(): AssertedIdentity { - return this.remoteAssertedIdentity; - } - - public get type(): CallType { - return (this.hasLocalUserMediaVideoTrack || this.hasRemoteUserMediaVideoTrack) - ? CallType.Video - : CallType.Voice; - } - - public get hasLocalUserMediaVideoTrack(): boolean { - return this.localUsermediaStream?.getVideoTracks().length > 0; - } - - public get hasRemoteUserMediaVideoTrack(): boolean { - return this.getRemoteFeeds().some((feed) => { - return ( - feed.purpose === SDPStreamMetadataPurpose.Usermedia && - feed.stream.getVideoTracks().length > 0 - ); - }); - } - - public get hasLocalUserMediaAudioTrack(): boolean { - return this.localUsermediaStream?.getAudioTracks().length > 0; - } - - public get hasRemoteUserMediaAudioTrack(): boolean { - return this.getRemoteFeeds().some((feed) => { - return ( - feed.purpose === SDPStreamMetadataPurpose.Usermedia && - feed.stream.getAudioTracks().length > 0 - ); - }); - } - - public get localUsermediaFeed(): CallFeed { - return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); - } - - public get localScreensharingFeed(): CallFeed { - return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); - } - - public get localUsermediaStream(): MediaStream { - return this.localUsermediaFeed?.stream; - } - - public get localScreensharingStream(): MediaStream { - return this.localScreensharingFeed?.stream; - } - - public get remoteUsermediaFeed(): CallFeed { - return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); - } - - public get remoteScreensharingFeed(): CallFeed { - return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); - } - - public get remoteUsermediaStream(): MediaStream { - return this.remoteUsermediaFeed?.stream; - } - - public get remoteScreensharingStream(): MediaStream { - return this.remoteScreensharingFeed?.stream; - } - - private getFeedByStreamId(streamId: string): CallFeed { - return this.getFeeds().find((feed) => feed.stream.id === streamId); - } - - /** - * Returns an array of all CallFeeds - * @returns {Array} CallFeeds - */ - public getFeeds(): Array { - return this.feeds; - } - - /** - * Returns an array of all local CallFeeds - * @returns {Array} local CallFeeds - */ - public getLocalFeeds(): Array { - return this.feeds.filter((feed) => feed.isLocal()); - } - - /** - * Returns an array of all remote CallFeeds - * @returns {Array} remote CallFeeds - */ - public getRemoteFeeds(): Array { - return this.feeds.filter((feed) => !feed.isLocal()); - } - - /** - * Generates and returns localSDPStreamMetadata - * @returns {SDPStreamMetadata} localSDPStreamMetadata - */ - private getLocalSDPStreamMetadata(updateStreamIds = false): SDPStreamMetadata { - const metadata: SDPStreamMetadata = {}; - for (const localFeed of this.getLocalFeeds()) { - if (updateStreamIds) { - localFeed.sdpMetadataStreamId = localFeed.stream.id; - } - - metadata[localFeed.sdpMetadataStreamId] = { - purpose: localFeed.purpose, - audio_muted: localFeed.isAudioMuted(), - video_muted: localFeed.isVideoMuted(), - }; - } - return metadata; - } - - /** - * Returns true if there are no incoming feeds, - * otherwise returns false - * @returns {boolean} no incoming feeds - */ - public noIncomingFeeds(): boolean { - return !this.feeds.some((feed) => !feed.isLocal()); - } - - private pushRemoteFeed(stream: MediaStream): void { - // Fallback to old behavior if the other side doesn't support SDPStreamMetadata - if (!this.opponentSupportsSDPStreamMetadata()) { - this.pushRemoteFeedWithoutMetadata(stream); - return; - } - - const userId = this.getOpponentMember().userId; - const purpose = this.remoteSDPStreamMetadata[stream.id].purpose; - const audioMuted = this.remoteSDPStreamMetadata[stream.id].audio_muted; - const videoMuted = this.remoteSDPStreamMetadata[stream.id].video_muted; - - if (!purpose) { - logger.warn(`Ignoring stream with id ${stream.id} because we didn't get any metadata about it`); - return; - } - - // Try to find a feed with the same purpose as the new stream, - // if we find it replace the old stream with the new one - const existingFeed = this.getRemoteFeeds().find((feed) => feed.purpose === purpose); - if (existingFeed) { - existingFeed.setNewStream(stream); - } else { - this.feeds.push(new CallFeed({ - client: this.client, - roomId: this.roomId, - userId, - stream, - purpose, - audioMuted, - videoMuted, - })); - this.emit(CallEvent.FeedsChanged, this.feeds); - } - - logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}", purpose=${purpose})`); - } - - /** - * This method is used ONLY if the other client doesn't support sending SDPStreamMetadata - */ - private pushRemoteFeedWithoutMetadata(stream: MediaStream): void { - const userId = this.getOpponentMember().userId; - // We can guess the purpose here since the other client can only send one stream - const purpose = SDPStreamMetadataPurpose.Usermedia; - const oldRemoteStream = this.feeds.find((feed) => !feed.isLocal())?.stream; - - // Note that we check by ID and always set the remote stream: Chrome appears - // to make new stream objects when transceiver directionality is changed and the 'active' - // status of streams change - Dave - // If we already have a stream, check this stream has the same id - if (oldRemoteStream && stream.id !== oldRemoteStream.id) { - logger.warn(`Ignoring new stream ID ${stream.id}: we already have stream ID ${oldRemoteStream.id}`); - return; - } - - // Try to find a feed with the same stream id as the new stream, - // if we find it replace the old stream with the new one - const feed = this.getFeedByStreamId(stream.id); - if (feed) { - feed.setNewStream(stream); - } else { - this.feeds.push(new CallFeed({ - client: this.client, - roomId: this.roomId, - audioMuted: false, - videoMuted: false, - userId, - stream, - purpose, - })); - this.emit(CallEvent.FeedsChanged, this.feeds); - } - - logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}")`); - } - - private pushNewLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose, addToPeerConnection = true): void { - const userId = this.client.getUserId(); - - // TODO: Find out what is going on here - // why do we enable audio (and only audio) tracks here? -- matthew - setTracksEnabled(stream.getAudioTracks(), true); - - // We try to replace an existing feed if there already is one with the same purpose - const existingFeed = this.getLocalFeeds().find((feed) => feed.purpose === purpose); - if (existingFeed) { - existingFeed.setNewStream(stream); - } else { - this.pushLocalFeed( - new CallFeed({ - client: this.client, - roomId: this.roomId, - audioMuted: stream.getAudioTracks().length === 0, - videoMuted: stream.getVideoTracks().length === 0, - userId, - stream, - purpose, - }), - addToPeerConnection, - ); - this.emit(CallEvent.FeedsChanged, this.feeds); - } - } - - /** - * Pushes supplied feed to the call - * @param {CallFeed} callFeed to push - * @param {boolean} addToPeerConnection whether to add the tracks to the peer connection - */ - public pushLocalFeed(callFeed: CallFeed, addToPeerConnection = true): void { - if (this.feeds.some((feed) => callFeed.stream.id === feed.stream.id)) { - logger.info(`Ignoring duplicate local stream ${callFeed.stream.id} in call ${this.callId}`); - return; - } - - this.feeds.push(callFeed); - - if (addToPeerConnection) { - const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia ? - this.usermediaSenders : this.screensharingSenders; - // Empty the array - senderArray.splice(0, senderArray.length); - - for (const track of callFeed.stream.getTracks()) { - logger.info( - `Adding track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${callFeed.stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - senderArray.push(this.peerConn.addTrack(track, callFeed.stream)); - } - } - - logger.info( - `Pushed local stream `+ - `(id="${callFeed.stream.id}", `+ - `active="${callFeed.stream.active}", `+ - `purpose="${callFeed.purpose}")`, - ); - - this.emit(CallEvent.FeedsChanged, this.feeds); - } - - /** - * Removes local call feed from the call and its tracks from the peer - * connection - * @param callFeed to remove - */ - public removeLocalFeed(callFeed: CallFeed): void { - const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia - ? this.usermediaSenders - : this.screensharingSenders; - - for (const sender of senderArray) { - this.peerConn.removeTrack(sender); - } - - if (callFeed.purpose === SDPStreamMetadataPurpose.Screenshare) { - this.client.getMediaHandler().stopScreensharingStream(callFeed.stream); - } - - // Empty the array - senderArray.splice(0, senderArray.length); - this.deleteFeed(callFeed); - } - - private deleteAllFeeds(): void { - for (const feed of this.feeds) { - if (!feed.isLocal() || !this.groupCallId) { - feed.dispose(); - } - } - - this.feeds = []; - this.emit(CallEvent.FeedsChanged, this.feeds); - } - - private deleteFeedByStream(stream: MediaStream): void { - const feed = this.getFeedByStreamId(stream.id); - if (!feed) { - logger.warn(`Didn't find the feed with stream id ${stream.id} to delete`); - return; - } - this.deleteFeed(feed); - } - - private deleteFeed(feed: CallFeed): void { - feed.dispose(); - this.feeds.splice(this.feeds.indexOf(feed), 1); - this.emit(CallEvent.FeedsChanged, this.feeds); - } - - // The typescript definitions have this type as 'any' :( - public async getCurrentCallStats(): Promise { - if (this.callHasEnded()) { - return this.callStatsAtEnd; - } - - return this.collectCallStats(); - } - - private async collectCallStats(): Promise { - // This happens when the call fails before it starts. - // For example when we fail to get capture sources - if (!this.peerConn) return; - - const statsReport = await this.peerConn.getStats(); - const stats = []; - for (const item of statsReport) { - stats.push(item[1]); - } - - return stats; - } - - /** - * Configure this call from an invite event. Used by MatrixClient. - * @param {MatrixEvent} event The m.call.invite event - */ - public async initWithInvite(event: MatrixEvent): Promise { - const invite = event.getContent(); - this.direction = CallDirection.Inbound; - - // make sure we have valid turn creds. Unless something's gone wrong, it should - // poll and keep the credentials valid so this should be instant. - const haveTurnCreds = await this.client.checkTurnServers(); - if (!haveTurnCreds) { - logger.warn("Failed to get TURN credentials! Proceeding with call anyway..."); - } - - const sdpStreamMetadata = invite[SDPStreamMetadataKey]; - if (sdpStreamMetadata) { - this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); - } else { - logger.debug("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); - } - - this.peerConn = this.createPeerConnection(); - // we must set the party ID before await-ing on anything: the call event - // handler will start giving us more call events (eg. candidates) so if - // we haven't set the party ID, we'll ignore them. - this.chooseOpponent(event); - try { - await this.peerConn.setRemoteDescription(invite.offer); - await this.addBufferedIceCandidates(); - } catch (e) { - logger.debug("Failed to set remote description", e); - this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); - return; - } - - const remoteStream = this.feeds.find((feed) => !feed.isLocal())?.stream; - - // According to previous comments in this file, firefox at some point did not - // add streams until media started arriving on them. Testing latest firefox - // (81 at time of writing), this is no longer a problem, so let's do it the correct way. - if (!remoteStream || remoteStream.getTracks().length === 0) { - logger.error("No remote stream or no tracks after setting remote description!"); - this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); - return; - } - - this.setState(CallState.Ringing); - - if (event.getLocalAge()) { - setTimeout(() => { - if (this.state == CallState.Ringing) { - logger.debug("Call invite has expired. Hanging up."); - this.hangupParty = CallParty.Remote; // effectively - this.setState(CallState.Ended); - this.stopAllMedia(); - if (this.peerConn.signalingState != 'closed') { - this.peerConn.close(); - } - this.emit(CallEvent.Hangup); - } - }, invite.lifetime - event.getLocalAge()); - } - } - - /** - * Configure this call from a hangup or reject event. Used by MatrixClient. - * @param {MatrixEvent} event The m.call.hangup event - */ - public initWithHangup(event: MatrixEvent): void { - // perverse as it may seem, sometimes we want to instantiate a call with a - // hangup message (because when getting the state of the room on load, events - // come in reverse order and we want to remember that a call has been hung up) - this.setState(CallState.Ended); - } - - private shouldAnswerWithMediaType( - wantedValue: boolean | undefined, valueOfTheOtherSide: boolean | undefined, type: "audio" | "video", - ): boolean { - if (wantedValue && !valueOfTheOtherSide) { - // TODO: Figure out how to do this - logger.warn(`Unable to answer with ${type} because the other side isn't sending it either.`); - return false; - } else if ( - !utils.isNullOrUndefined(wantedValue) && - wantedValue !== valueOfTheOtherSide && - !this.opponentSupportsSDPStreamMetadata() - ) { - logger.warn( - `Unable to answer with ${type}=${wantedValue} because the other side doesn't support it. ` + - `Answering with ${type}=${valueOfTheOtherSide}.`, - ); - return valueOfTheOtherSide; - } - return wantedValue ?? valueOfTheOtherSide; - } - - /** - * Answer a call. - */ - public async answer(audio?: boolean, video?: boolean): Promise { - if (this.inviteOrAnswerSent) return; - // TODO: Figure out how to do this - if (audio === false && video === false) throw new Error("You CANNOT answer a call without media"); - - if (!this.localUsermediaStream && !this.waitForLocalAVStream) { - const prevState = this.state; - const answerWithAudio = this.shouldAnswerWithMediaType(audio, this.hasRemoteUserMediaAudioTrack, "audio"); - const answerWithVideo = this.shouldAnswerWithMediaType(video, this.hasRemoteUserMediaVideoTrack, "video"); - - this.setState(CallState.WaitLocalMedia); - this.waitForLocalAVStream = true; - - try { - const stream = await this.client.getMediaHandler().getUserMediaStream( - answerWithAudio, answerWithVideo, - ); - this.waitForLocalAVStream = false; - const usermediaFeed = new CallFeed({ - client: this.client, - roomId: this.roomId, - userId: this.client.getUserId(), - stream, - purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: stream.getAudioTracks().length === 0, - videoMuted: stream.getVideoTracks().length === 0, - }); - - const feeds = [usermediaFeed]; - - if (this.localScreensharingFeed) { - feeds.push(this.localScreensharingFeed); - } - - this.answerWithCallFeeds(feeds); - } catch (e) { - if (answerWithVideo) { - // Try to answer without video - logger.warn("Failed to getUserMedia(), trying to getUserMedia() without video"); - this.setState(prevState); - this.waitForLocalAVStream = false; - await this.answer(answerWithAudio, false); - } else { - this.getUserMediaFailed(e); - return; - } - } - } else if (this.waitForLocalAVStream) { - this.setState(CallState.WaitLocalMedia); - } - } - - public answerWithCallFeeds(callFeeds: CallFeed[]): void { - if (this.inviteOrAnswerSent) return; - - logger.debug(`Answering call ${this.callId}`); - - this.queueGotCallFeedsForAnswer(callFeeds); - } - - /** - * Replace this call with a new call, e.g. for glare resolution. Used by - * MatrixClient. - * @param {MatrixCall} newCall The new call. - */ - public replacedBy(newCall: MatrixCall): void { - if (this.state === CallState.WaitLocalMedia) { - logger.debug("Telling new call to wait for local media"); - newCall.waitForLocalAVStream = true; - } else if ([CallState.CreateOffer, CallState.InviteSent].includes(this.state)) { - if (newCall.direction === CallDirection.Outbound) { - newCall.queueGotCallFeedsForAnswer([]); - } else { - logger.debug("Handing local stream to new call"); - newCall.queueGotCallFeedsForAnswer(this.getLocalFeeds().map(feed => feed.clone())); - } - } - this.successor = newCall; - this.emit(CallEvent.Replaced, newCall); - this.hangup(CallErrorCode.Replaced, true); - } - - /** - * Hangup a call. - * @param {string} reason The reason why the call is being hung up. - * @param {boolean} suppressEvent True to suppress emitting an event. - */ - public hangup(reason: CallErrorCode, suppressEvent: boolean): void { - if (this.callHasEnded()) return; - - logger.debug("Ending call " + this.callId); - this.terminate(CallParty.Local, reason, !suppressEvent); - // We don't want to send hangup here if we didn't even get to sending an invite - if (this.state === CallState.WaitLocalMedia) return; - const content = {}; - // Don't send UserHangup reason to older clients - if ((this.opponentVersion && this.opponentVersion >= 1) || reason !== CallErrorCode.UserHangup) { - content["reason"] = reason; - } - this.sendVoipEvent(EventType.CallHangup, content); - } - - /** - * Reject a call - * This used to be done by calling hangup, but is a separate method and protocol - * event as of MSC2746. - */ - public reject(): void { - if (this.state !== CallState.Ringing) { - throw Error("Call must be in 'ringing' state to reject!"); - } - - if (this.opponentVersion < 1) { - logger.info( - `Opponent version is less than 1 (${this.opponentVersion}): sending hangup instead of reject`, - ); - this.hangup(CallErrorCode.UserHangup, true); - return; - } - - logger.debug("Rejecting call: " + this.callId); - this.terminate(CallParty.Local, CallErrorCode.UserHangup, true); - this.sendVoipEvent(EventType.CallReject, {}); - } - - /** - * Adds an audio and/or video track - upgrades the call - * @param {boolean} audio should add an audio track - * @param {boolean} video should add an video track - */ - private async upgradeCall( - audio: boolean, video: boolean, - ): Promise { - // We don't do call downgrades - if (!audio && !video) return; - if (!this.opponentSupportsSDPStreamMetadata()) return; - - try { - const upgradeAudio = audio && !this.hasLocalUserMediaAudioTrack; - const upgradeVideo = video && !this.hasLocalUserMediaVideoTrack; - logger.debug(`Upgrading call: audio?=${upgradeAudio} video?=${upgradeVideo}`); - - const stream = await this.client.getMediaHandler().getUserMediaStream(upgradeAudio, upgradeVideo); - if (upgradeAudio && upgradeVideo) { - if (this.hasLocalUserMediaAudioTrack) return; - if (this.hasLocalUserMediaVideoTrack) return; - - this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); - } else if (upgradeAudio) { - if (this.hasLocalUserMediaAudioTrack) return; - - const audioTrack = stream.getAudioTracks()[0]; - this.localUsermediaStream.addTrack(audioTrack); - this.peerConn.addTrack(audioTrack, this.localUsermediaStream); - } else if (upgradeVideo) { - if (this.hasLocalUserMediaVideoTrack) return; - - const videoTrack = stream.getVideoTracks()[0]; - this.localUsermediaStream.addTrack(videoTrack); - this.peerConn.addTrack(videoTrack, this.localUsermediaStream); - } - } catch (error) { - logger.error("Failed to upgrade the call", error); - this.emit(CallEvent.Error, - new CallError(CallErrorCode.NoUserMedia, "Failed to get camera access: ", error), - ); - } - } - - /** - * Returns true if this.remoteSDPStreamMetadata is defined, otherwise returns false - * @returns {boolean} can screenshare - */ - public opponentSupportsSDPStreamMetadata(): boolean { - return Boolean(this.remoteSDPStreamMetadata); - } - - /** - * If there is a screensharing stream returns true, otherwise returns false - * @returns {boolean} is screensharing - */ - public isScreensharing(): boolean { - return Boolean(this.localScreensharingStream); - } - - /** - * Starts/stops screensharing - * @param enabled the desired screensharing state - * @param {string} desktopCapturerSourceId optional id of the desktop capturer source to use - * @returns {boolean} new screensharing state - */ - public async setScreensharingEnabled( - enabled: boolean, desktopCapturerSourceId?: string, - ): Promise { - // Skip if there is nothing to do - if (enabled && this.isScreensharing()) { - logger.warn(`There is already a screensharing stream - there is nothing to do!`); - return true; - } else if (!enabled && !this.isScreensharing()) { - logger.warn(`There already isn't a screensharing stream - there is nothing to do!`); - return false; - } - - // Fallback to replaceTrack() - if (!this.opponentSupportsSDPStreamMetadata()) { - return await this.setScreensharingEnabledWithoutMetadataSupport(enabled, desktopCapturerSourceId); - } - - logger.debug(`Set screensharing enabled? ${enabled}`); - if (enabled) { - try { - const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId); - if (!stream) return false; - this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare); - return true; - } catch (err) { - logger.error("Failed to get screen-sharing stream:", err); - return false; - } - } else { - for (const sender of this.screensharingSenders) { - this.peerConn.removeTrack(sender); - } - this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream); - this.deleteFeedByStream(this.localScreensharingStream); - return false; - } - } - - /** - * Starts/stops screensharing - * Should be used ONLY if the opponent doesn't support SDPStreamMetadata - * @param enabled the desired screensharing state - * @param {string} desktopCapturerSourceId optional id of the desktop capturer source to use - * @returns {boolean} new screensharing state - */ - private async setScreensharingEnabledWithoutMetadataSupport( - enabled: boolean, desktopCapturerSourceId?: string, - ): Promise { - logger.debug(`Set screensharing enabled? ${enabled} using replaceTrack()`); - if (enabled) { - try { - const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId); - if (!stream) return false; - - const track = stream.getTracks().find((track) => { - return track.kind === "video"; - }); - const sender = this.usermediaSenders.find((sender) => { - return sender.track?.kind === "video"; - }); - sender.replaceTrack(track); - - this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare, false); - - return true; - } catch (err) { - logger.error("Failed to get screen-sharing stream:", err); - return false; - } - } else { - const track = this.localUsermediaStream.getTracks().find((track) => { - return track.kind === "video"; - }); - const sender = this.usermediaSenders.find((sender) => { - return sender.track?.kind === "video"; - }); - sender.replaceTrack(track); - - this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream); - this.deleteFeedByStream(this.localScreensharingStream); - - return false; - } - } - - /** - * Request a new local usermedia stream with the current device id. - */ - public async updateLocalUsermediaStream(stream: MediaStream) { - const callFeed = this.localUsermediaFeed; - callFeed.setNewStream(stream); - const micShouldBeMuted = callFeed.isAudioMuted() || this.remoteOnHold; - const vidShouldBeMuted = callFeed.isVideoMuted() || this.remoteOnHold; - setTracksEnabled(stream.getAudioTracks(), !micShouldBeMuted); - setTracksEnabled(stream.getVideoTracks(), !vidShouldBeMuted); - - const newSenders = []; - - for (const track of stream.getTracks()) { - const oldSender = this.usermediaSenders.find((sender) => { - return sender.track?.kind === track.kind; - }); - - let newSender: RTCRtpSender; - - try { - logger.info( - `Replacing track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - await oldSender.replaceTrack(track); - newSender = oldSender; - } catch (error) { - logger.info( - `Adding track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - newSender = this.peerConn.addTrack(track, stream); - } - - newSenders.push(newSender); - } - - this.usermediaSenders = newSenders; - } - - /** - * Set whether our outbound video should be muted or not. - * @param {boolean} muted True to mute the outbound video. - * @returns the new mute state - */ - public async setLocalVideoMuted(muted: boolean): Promise { - if (!await this.client.getMediaHandler().hasVideoDevice()) { - return this.isLocalVideoMuted(); - } - - if (!this.hasLocalUserMediaVideoTrack && !muted) { - await this.upgradeCall(false, true); - return this.isLocalVideoMuted(); - } - this.localUsermediaFeed?.setVideoMuted(muted); - this.updateMuteStatus(); - return this.isLocalVideoMuted(); - } - - /** - * Check if local video is muted. - * - * If there are multiple video tracks, all of the tracks need to be muted - * for this to return true. This means if there are no video tracks, this will - * return true. - * @return {Boolean} True if the local preview video is muted, else false - * (including if the call is not set up yet). - */ - public isLocalVideoMuted(): boolean { - return this.localUsermediaFeed?.isVideoMuted(); - } - - /** - * Set whether the microphone should be muted or not. - * @param {boolean} muted True to mute the mic. - * @returns the new mute state - */ - public async setMicrophoneMuted(muted: boolean): Promise { - if (!await this.client.getMediaHandler().hasAudioDevice()) { - return this.isMicrophoneMuted(); - } - - if (!this.hasLocalUserMediaAudioTrack && !muted) { - await this.upgradeCall(true, false); - return this.isMicrophoneMuted(); - } - this.localUsermediaFeed?.setAudioMuted(muted); - this.updateMuteStatus(); - return this.isMicrophoneMuted(); - } - - /** - * Check if the microphone is muted. - * - * If there are multiple audio tracks, all of the tracks need to be muted - * for this to return true. This means if there are no audio tracks, this will - * return true. - * @return {Boolean} True if the mic is muted, else false (including if the call - * is not set up yet). - */ - public isMicrophoneMuted(): boolean { - return this.localUsermediaFeed?.isAudioMuted(); - } - - /** - * @returns true if we have put the party on the other side of the call on hold - * (that is, we are signalling to them that we are not listening) - */ - public isRemoteOnHold(): boolean { - return this.remoteOnHold; - } - - public setRemoteOnHold(onHold: boolean): void { - if (this.isRemoteOnHold() === onHold) return; - this.remoteOnHold = onHold; - - for (const transceiver of this.peerConn.getTransceivers()) { - // We don't send hold music or anything so we're not actually - // sending anything, but sendrecv is fairly standard for hold and - // it makes it a lot easier to figure out who's put who on hold. - transceiver.direction = onHold ? 'sendonly' : 'sendrecv'; - } - this.updateMuteStatus(); - - this.emit(CallEvent.RemoteHoldUnhold, this.remoteOnHold); - } - - /** - * Indicates whether we are 'on hold' to the remote party (ie. if true, - * they cannot hear us). - * @returns true if the other party has put us on hold - */ - public isLocalOnHold(): boolean { - if (this.state !== CallState.Connected) return false; - - let callOnHold = true; - - // We consider a call to be on hold only if *all* the tracks are on hold - // (is this the right thing to do?) - for (const transceiver of this.peerConn.getTransceivers()) { - const trackOnHold = ['inactive', 'recvonly'].includes(transceiver.currentDirection); - - if (!trackOnHold) callOnHold = false; - } - - return callOnHold; - } - - /** - * Sends a DTMF digit to the other party - * @param digit The digit (nb. string - '#' and '*' are dtmf too) - */ - public sendDtmfDigit(digit: string): void { - for (const sender of this.peerConn.getSenders()) { - if (sender.track.kind === 'audio' && sender.dtmf) { - sender.dtmf.insertDTMF(digit); - return; - } - } - - throw new Error("Unable to find a track to send DTMF on"); - } - - private updateMuteStatus(): void { - this.sendVoipEvent(EventType.CallSDPStreamMetadataChangedPrefix, { - [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(), - }); - - const micShouldBeMuted = this.localUsermediaFeed?.isAudioMuted() || this.remoteOnHold; - const vidShouldBeMuted = this.localUsermediaFeed?.isVideoMuted() || this.remoteOnHold; - - setTracksEnabled(this.localUsermediaStream.getAudioTracks(), !micShouldBeMuted); - setTracksEnabled(this.localUsermediaStream.getVideoTracks(), !vidShouldBeMuted); - } - - private gotCallFeedsForInvite(callFeeds: CallFeed[], requestScreenshareFeed = false): void { - if (this.successor) { - this.successor.queueGotCallFeedsForAnswer(callFeeds); - return; - } - if (this.callHasEnded()) { - this.stopAllMedia(); - return; - } - - for (const feed of callFeeds) { - this.pushLocalFeed(feed); - } - - if (requestScreenshareFeed) { - this.peerConn.addTransceiver("video", { - direction: "recvonly", - }); - } - - this.setState(CallState.CreateOffer); - - logger.debug("gotUserMediaForInvite"); - // Now we wait for the negotiationneeded event - } - - private async sendAnswer(): Promise { - const answerContent = { - answer: { - sdp: this.peerConn.localDescription.sdp, - // type is now deprecated as of Matrix VoIP v1, but - // required to still be sent for backwards compat - type: this.peerConn.localDescription.type, - }, - [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(true), - } as MCallAnswer; - - answerContent.capabilities = { - 'm.call.transferee': this.client.supportsCallTransfer, - 'm.call.dtmf': false, - }; - - // We have just taken the local description from the peerConn which will - // contain all the local candidates added so far, so we can discard any candidates - // we had queued up because they'll be in the answer. - logger.info(`Discarding ${this.candidateSendQueue.length} candidates that will be sent in answer`); - this.candidateSendQueue = []; - - try { - await this.sendVoipEvent(EventType.CallAnswer, answerContent); - // If this isn't the first time we've tried to send the answer, - // we may have candidates queued up, so send them now. - this.inviteOrAnswerSent = true; - } catch (error) { - // We've failed to answer: back to the ringing state - this.setState(CallState.Ringing); - this.client.cancelPendingEvent(error.event); - - let code = CallErrorCode.SendAnswer; - let message = "Failed to send answer"; - if (error.name == 'UnknownDeviceError') { - code = CallErrorCode.UnknownDevices; - message = "Unknown devices present in the room"; - } - this.emit(CallEvent.Error, new CallError(code, message, error)); - throw error; - } - - // error handler re-throws so this won't happen on error, but - // we don't want the same error handling on the candidate queue - this.sendCandidateQueue(); - } - - private queueGotCallFeedsForAnswer(callFeeds: CallFeed[]): void { - // Ensure only one negotiate/answer event is being processed at a time. - if (this.responsePromiseChain) { - this.responsePromiseChain = - this.responsePromiseChain.then(() => this.gotCallFeedsForAnswer(callFeeds)); - } else { - this.responsePromiseChain = this.gotCallFeedsForAnswer(callFeeds); - } - } - - private async gotCallFeedsForAnswer(callFeeds: CallFeed[]): Promise { - if (this.callHasEnded()) return; - - this.waitForLocalAVStream = false; - - for (const feed of callFeeds) { - this.pushLocalFeed(feed); - } - - this.setState(CallState.CreateAnswer); - - let myAnswer; - try { - this.getRidOfRTXCodecs(); - myAnswer = await this.peerConn.createAnswer(); - } catch (err) { - logger.debug("Failed to create answer: ", err); - this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true); - return; - } - - try { - await this.peerConn.setLocalDescription(myAnswer); - this.setState(CallState.Connecting); - - // Allow a short time for initial candidates to be gathered - await new Promise(resolve => { - setTimeout(resolve, 200); - }); - - this.sendAnswer(); - } catch (err) { - logger.debug("Error setting local description!", err); - this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); - return; - } - - // HACK: Safari doesn't like it when we reuse MediaStreams. In most cases - // we can get around this by calling MediaStream.clone(), however inbound - // calls seem to still be broken unless we getUserMedia again and replace - // all MediaStreams using sender.replaceTrack - if (isSafari) { - await new Promise(resolve => { - setTimeout(resolve, 200); - }); - - if (this.state === CallState.Ended) { - return; - } - - const callFeed = this.localUsermediaFeed; - const stream = callFeed.stream; - - if (!stream.active) { - throw new Error(`Call ${this.callId} has an inactive stream ${ - stream.id} and its tracks cannot be replaced`); - } - - const newSenders = []; - - for (const track of this.localUsermediaStream.getTracks()) { - const oldSender = this.usermediaSenders.find((sender) => { - return sender.track?.kind === track.kind; - }); - - if (track.readyState === "ended") { - throw new Error(`Call ${this.callId} tried to replace track ${track.id} in the ended state`); - } - - let newSender: RTCRtpSender; - - try { - logger.info( - `Replacing track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - await oldSender.replaceTrack(track); - newSender = oldSender; - } catch (error) { - logger.info( - `Adding track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - newSender = this.peerConn.addTrack(track, stream); - } - - newSenders.push(newSender); - } - - this.usermediaSenders = newSenders; - } - } - - /** - * Internal - * @param {Object} event - */ - private gotLocalIceCandidate = (event: RTCPeerConnectionIceEvent): Promise => { - if (event.candidate) { - logger.debug( - "Call " + this.callId + " got local ICE " + event.candidate.sdpMid + " candidate: " + - event.candidate.candidate, - ); - - if (this.callHasEnded()) return; - - // As with the offer, note we need to make a copy of this object, not - // pass the original: that broke in Chrome ~m43. - if (event.candidate.candidate !== '' || !this.sentEndOfCandidates) { - this.queueCandidate(event.candidate); - - if (event.candidate.candidate === '') this.sentEndOfCandidates = true; - } - } - }; - - private onIceGatheringStateChange = (event: Event): void => { - logger.debug("ice gathering state changed to " + this.peerConn.iceGatheringState); - if (this.peerConn.iceGatheringState === 'complete' && !this.sentEndOfCandidates) { - // If we didn't get an empty-string candidate to signal the end of candidates, - // create one ourselves now gathering has finished. - // We cast because the interface lists all the properties as required but we - // only want to send 'candidate' - // XXX: We probably want to send either sdpMid or sdpMLineIndex, as it's not strictly - // correct to have a candidate that lacks both of these. We'd have to figure out what - // previous candidates had been sent with and copy them. - const c = { - candidate: '', - } as RTCIceCandidate; - this.queueCandidate(c); - this.sentEndOfCandidates = true; - } - }; - - public async onRemoteIceCandidatesReceived(ev: MatrixEvent): Promise { - if (this.callHasEnded()) { - //debuglog("Ignoring remote ICE candidate because call has ended"); - return; - } - - const content = ev.getContent(); - const candidates = content.candidates; - if (!candidates) { - logger.info("Ignoring candidates event with no candidates!"); - return; - } - - const fromPartyId = content.version === 0 ? null : content.party_id || null; - - if (this.opponentPartyId === undefined) { - // we haven't picked an opponent yet so save the candidates - logger.info(`Buffering ${candidates.length} candidates until we pick an opponent`); - const bufferedCandidates = this.remoteCandidateBuffer.get(fromPartyId) || []; - bufferedCandidates.push(...candidates); - this.remoteCandidateBuffer.set(fromPartyId, bufferedCandidates); - return; - } - - if (!this.partyIdMatches(content)) { - logger.info( - `Ignoring candidates from party ID ${content.party_id}: ` + - `we have chosen party ID ${this.opponentPartyId}`, - ); - - return; - } - - await this.addIceCandidates(candidates); - } - - /** - * Used by MatrixClient. - * @param {Object} msg - */ - public async onAnswerReceived(event: MatrixEvent): Promise { - const content = event.getContent(); - logger.debug(`Got answer for call ID ${this.callId} from party ID ${content.party_id}`); - - if (this.callHasEnded()) { - logger.debug(`Ignoring answer because call ID ${this.callId} has ended`); - return; - } - - if (this.opponentPartyId !== undefined) { - logger.info( - `Ignoring answer from party ID ${content.party_id}: ` + - `we already have an answer/reject from ${this.opponentPartyId}`, - ); - return; - } - - this.chooseOpponent(event); - await this.addBufferedIceCandidates(); - - this.setState(CallState.Connecting); - - const sdpStreamMetadata = content[SDPStreamMetadataKey]; - if (sdpStreamMetadata) { - this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); - } else { - logger.warn("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); - } - - try { - await this.peerConn.setRemoteDescription(content.answer); - } catch (e) { - logger.debug("Failed to set remote description", e); - this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); - return; - } - - // If the answer we selected has a party_id, send a select_answer event - // We do this after setting the remote description since otherwise we'd block - // call setup on it - if (this.opponentPartyId !== null) { - try { - await this.sendVoipEvent(EventType.CallSelectAnswer, { - selected_party_id: this.opponentPartyId, - }); - } catch (err) { - // This isn't fatal, and will just mean that if another party has raced to answer - // the call, they won't know they got rejected, so we carry on & don't retry. - logger.warn("Failed to send select_answer event", err); - } - } - } - - public async onSelectAnswerReceived(event: MatrixEvent): Promise { - if (this.direction !== CallDirection.Inbound) { - logger.warn("Got select_answer for an outbound call: ignoring"); - return; - } - - const selectedPartyId = event.getContent().selected_party_id; - - if (selectedPartyId === undefined || selectedPartyId === null) { - logger.warn("Got nonsensical select_answer with null/undefined selected_party_id: ignoring"); - return; - } - - if (selectedPartyId !== this.ourPartyId) { - logger.info(`Got select_answer for party ID ${selectedPartyId}: we are party ID ${this.ourPartyId}.`); - // The other party has picked somebody else's answer - this.terminate(CallParty.Remote, CallErrorCode.AnsweredElsewhere, true); - } - } - - public async onNegotiateReceived(event: MatrixEvent): Promise { - const content = event.getContent(); - const description = content.description; - if (!description || !description.sdp || !description.type) { - logger.info("Ignoring invalid m.call.negotiate event"); - return; - } - // Politeness always follows the direction of the call: in a glare situation, - // we pick either the inbound or outbound call, so one side will always be - // inbound and one outbound - const polite = this.direction === CallDirection.Inbound; - - // Here we follow the perfect negotiation logic from - // https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation - const offerCollision = ( - (description.type === 'offer') && - (this.makingOffer || this.peerConn.signalingState !== 'stable') - ); - - this.ignoreOffer = !polite && offerCollision; - if (this.ignoreOffer) { - logger.info("Ignoring colliding negotiate event because we're impolite"); - return; - } - - const prevLocalOnHold = this.isLocalOnHold(); - - const sdpStreamMetadata = content[SDPStreamMetadataKey]; - if (sdpStreamMetadata) { - this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); - } else { - logger.warn("Received negotiation event without SDPStreamMetadata!"); - } - - try { - await this.peerConn.setRemoteDescription(description); - - if (description.type === 'offer') { - this.getRidOfRTXCodecs(); - await this.peerConn.setLocalDescription(); - - this.sendVoipEvent(EventType.CallNegotiate, { - description: this.peerConn.localDescription, - [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(true), - }); - } - } catch (err) { - logger.warn("Failed to complete negotiation", err); - } - - const newLocalOnHold = this.isLocalOnHold(); - if (prevLocalOnHold !== newLocalOnHold) { - this.emit(CallEvent.LocalHoldUnhold, newLocalOnHold); - // also this one for backwards compat - this.emit(CallEvent.HoldUnhold, newLocalOnHold); - } - } - - private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { - this.remoteSDPStreamMetadata = utils.recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); - for (const feed of this.getRemoteFeeds()) { - const streamId = feed.stream.id; - feed.setAudioMuted(this.remoteSDPStreamMetadata[streamId]?.audio_muted); - feed.setVideoMuted(this.remoteSDPStreamMetadata[streamId]?.video_muted); - feed.purpose = this.remoteSDPStreamMetadata[streamId]?.purpose; - } - } - - public onSDPStreamMetadataChangedReceived(event: MatrixEvent): void { - const content = event.getContent(); - const metadata = content[SDPStreamMetadataKey]; - this.updateRemoteSDPStreamMetadata(metadata); - } - - public async onAssertedIdentityReceived(event: MatrixEvent): Promise { - const content = event.getContent(); - if (!content.asserted_identity) return; - - this.remoteAssertedIdentity = { - id: content.asserted_identity.id, - displayName: content.asserted_identity.display_name, - }; - this.emit(CallEvent.AssertedIdentityChanged); - } - - private callHasEnded(): boolean { - // This exists as workaround to typescript trying to be clever and erroring - // when putting if (this.state === CallState.Ended) return; twice in the same - // function, even though that function is async. - return this.state === CallState.Ended; - } - - private queueGotLocalOffer(): void { - // Ensure only one negotiate/answer event is being processed at a time. - if (this.responsePromiseChain) { - this.responsePromiseChain = - this.responsePromiseChain.then(() => this.wrappedGotLocalOffer()); - } else { - this.responsePromiseChain = this.wrappedGotLocalOffer(); - } - } - - private async wrappedGotLocalOffer(): Promise { - this.makingOffer = true; - try { - this.getRidOfRTXCodecs(); - await this.gotLocalOffer(); - } catch (e) { - this.getLocalOfferFailed(e); - return; - } finally { - this.makingOffer = false; - } - } - - private async gotLocalOffer(): Promise { - logger.debug("Setting local description"); - - if (this.callHasEnded()) { - logger.debug("Ignoring newly created offer on call ID " + this.callId + - " because the call has ended"); - return; - } - - try { - await this.peerConn.setLocalDescription(); - } catch (err) { - logger.debug("Error setting local description!", err); - this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); - return; - } - - if (this.peerConn.iceGatheringState === 'gathering') { - // Allow a short time for initial candidates to be gathered - await new Promise(resolve => { - setTimeout(resolve, 200); - }); - } - - if (this.callHasEnded()) return; - - const eventType = this.state === CallState.CreateOffer ? EventType.CallInvite : EventType.CallNegotiate; - - const content = { - lifetime: CALL_TIMEOUT_MS, - } as MCallInviteNegotiate; - - if (eventType === EventType.CallInvite && this.invitee) { - content.invitee = this.invitee; - } - - // clunky because TypeScript can't follow the types through if we use an expression as the key - if (this.state === CallState.CreateOffer) { - content.offer = this.peerConn.localDescription; - } else { - content.description = this.peerConn.localDescription; - } - - content.capabilities = { - 'm.call.transferee': this.client.supportsCallTransfer, - 'm.call.dtmf': false, - }; - - content[SDPStreamMetadataKey] = this.getLocalSDPStreamMetadata(true); - - // Get rid of any candidates waiting to be sent: they'll be included in the local - // description we just got and will send in the offer. - logger.info(`Discarding ${this.candidateSendQueue.length} candidates that will be sent in offer`); - this.candidateSendQueue = []; - - try { - await this.sendVoipEvent(eventType, content); - } catch (error) { - logger.error("Failed to send invite", error); - if (error.event) this.client.cancelPendingEvent(error.event); - - let code = CallErrorCode.SignallingFailed; - let message = "Signalling failed"; - if (this.state === CallState.CreateOffer) { - code = CallErrorCode.SendInvite; - message = "Failed to send invite"; - } - if (error.name == 'UnknownDeviceError') { - code = CallErrorCode.UnknownDevices; - message = "Unknown devices present in the room"; - } - - this.emit(CallEvent.Error, new CallError(code, message, error)); - this.terminate(CallParty.Local, code, false); - - // no need to carry on & send the candidate queue, but we also - // don't want to rethrow the error - return; - } - - this.sendCandidateQueue(); - if (this.state === CallState.CreateOffer) { - this.inviteOrAnswerSent = true; - this.setState(CallState.InviteSent); - this.inviteTimeout = setTimeout(() => { - this.inviteTimeout = null; - if (this.state === CallState.InviteSent) { - this.hangup(CallErrorCode.InviteTimeout, false); - } - }, CALL_TIMEOUT_MS); - } - } - - private getLocalOfferFailed = (err: Error): void => { - logger.error("Failed to get local offer", err); - - this.emit( - CallEvent.Error, - new CallError( - CallErrorCode.LocalOfferFailed, - "Failed to get local offer!", err, - ), - ); - this.terminate(CallParty.Local, CallErrorCode.LocalOfferFailed, false); - }; - - private getUserMediaFailed = (err: Error): void => { - if (this.successor) { - this.successor.getUserMediaFailed(err); - return; - } - - logger.warn("Failed to get user media - ending call", err); - - this.emit( - CallEvent.Error, - new CallError( - CallErrorCode.NoUserMedia, - "Couldn't start capturing media! Is your microphone set up and " + - "does this app have permission?", err, - ), - ); - this.terminate(CallParty.Local, CallErrorCode.NoUserMedia, false); - }; - - private onIceConnectionStateChanged = (): void => { - if (this.callHasEnded()) { - return; // because ICE can still complete as we're ending the call - } - logger.debug( - "Call ID " + this.callId + ": ICE connection state changed to: " + this.peerConn.iceConnectionState, - ); - // ideally we'd consider the call to be connected when we get media but - // chrome doesn't implement any of the 'onstarted' events yet - if (this.peerConn.iceConnectionState == 'connected') { - clearTimeout(this.iceDisconnectedTimeout); - this.setState(CallState.Connected); - - if (!this.callLengthInterval) { - this.callLengthInterval = setInterval(() => { - this.callLength++; - this.emit(CallEvent.LengthChanged, this.callLength); - }, 1000); - } - } else if (this.peerConn.iceConnectionState == 'failed') { - // Firefox for Android does not yet have support for restartIce() - if (this.peerConn.restartIce) { - this.peerConn.restartIce(); - } else { - this.hangup(CallErrorCode.IceFailed, false); - } - } else if (this.peerConn.iceConnectionState == 'disconnected') { - this.iceDisconnectedTimeout = setTimeout(() => { - this.hangup(CallErrorCode.IceFailed, false); - }, 30 * 1000); - } - }; - - private onSignallingStateChanged = (): void => { - logger.debug( - "call " + this.callId + ": Signalling state changed to: " + - this.peerConn.signalingState, - ); - }; - - private onTrack = (ev: RTCTrackEvent): void => { - if (ev.streams.length === 0) { - logger.warn(`Streamless ${ev.track.kind} found: ignoring.`); - return; - } - - const stream = ev.streams[0]; - this.pushRemoteFeed(stream); - stream.addEventListener("removetrack", () => { - logger.log(`Removing track streamId: ${stream.id}`); - this.deleteFeedByStream(stream); - }); - }; - - private onDataChannel = (ev: RTCDataChannelEvent): void => { - this.emit(CallEvent.DataChannel, ev.channel); - }; - - /** - * This method removes all video/rtx codecs from screensharing video - * transceivers. This is necessary since they can cause problems. Without - * this the following steps should produce an error: - * Chromium calls Firefox - * Firefox answers - * Firefox starts screen-sharing - * Chromium starts screen-sharing - * Call crashes for Chromium with: - * [96685:23:0518/162603.933321:ERROR:webrtc_video_engine.cc(3296)] RTX codec (PT=97) mapped to PT=96 which is not in the codec list. - * [96685:23:0518/162603.933377:ERROR:webrtc_video_engine.cc(1171)] GetChangedRecvParameters called without any video codecs. - * [96685:23:0518/162603.933430:ERROR:sdp_offer_answer.cc(4302)] Failed to set local video description recv parameters for m-section with mid='2'. (INVALID_PARAMETER) - */ - private getRidOfRTXCodecs(): void { - // RTCRtpReceiver.getCapabilities and RTCRtpSender.getCapabilities don't seem to be supported on FF - if (!RTCRtpReceiver.getCapabilities || !RTCRtpSender.getCapabilities) return; - - const recvCodecs = RTCRtpReceiver.getCapabilities("video").codecs; - const sendCodecs = RTCRtpSender.getCapabilities("video").codecs; - const codecs = [...sendCodecs, ...recvCodecs]; - - for (const codec of codecs) { - if (codec.mimeType === "video/rtx") { - const rtxCodecIndex = codecs.indexOf(codec); - codecs.splice(rtxCodecIndex, 1); - } - } - - for (const trans of this.peerConn.getTransceivers()) { - if ( - this.screensharingSenders.includes(trans.sender) && - ( - trans.sender.track?.kind === "video" || - trans.receiver.track?.kind === "video" - ) - ) { - trans.setCodecPreferences(codecs); - } - } - } - - private onNegotiationNeeded = async (): Promise => { - logger.info("Negotiation is needed!"); - - if (this.state !== CallState.CreateOffer && this.opponentVersion === 0) { - logger.info("Opponent does not support renegotiation: ignoring negotiationneeded event"); - return; - } - - this.queueGotLocalOffer(); - }; - - public onHangupReceived = (msg: MCallHangupReject): void => { - logger.debug("Hangup received for call ID " + this.callId); - - // party ID must match (our chosen partner hanging up the call) or be undefined (we haven't chosen - // a partner yet but we're treating the hangup as a reject as per VoIP v0) - if (this.partyIdMatches(msg) || this.state === CallState.Ringing) { - // default reason is user_hangup - this.terminate(CallParty.Remote, msg.reason || CallErrorCode.UserHangup, true); - } else { - logger.info(`Ignoring message from party ID ${msg.party_id}: our partner is ${this.opponentPartyId}`); - } - }; - - public onRejectReceived = (msg: MCallHangupReject): void => { - logger.debug("Reject received for call ID " + this.callId); - - // No need to check party_id for reject because if we'd received either - // an answer or reject, we wouldn't be in state InviteSent - - const shouldTerminate = ( - // reject events also end the call if it's ringing: it's another of - // our devices rejecting the call. - ([CallState.InviteSent, CallState.Ringing].includes(this.state)) || - // also if we're in the init state and it's an inbound call, since - // this means we just haven't entered the ringing state yet - this.state === CallState.Fledgling && this.direction === CallDirection.Inbound - ); - - if (shouldTerminate) { - this.terminate(CallParty.Remote, msg.reason || CallErrorCode.UserHangup, true); - } else { - logger.debug(`Call is in state: ${this.state}: ignoring reject`); - } - }; - - public onAnsweredElsewhere = (msg: MCallAnswer): void => { - logger.debug("Call ID " + this.callId + " answered elsewhere"); - this.terminate(CallParty.Remote, CallErrorCode.AnsweredElsewhere, true); - }; - - private setState(state: CallState): void { - const oldState = this.state; - this.state = state; - this.emit(CallEvent.State, state, oldState); - } - - /** - * Internal - * @param {string} eventType - * @param {Object} content - * @return {Promise} - */ - private sendVoipEvent(eventType: string, content: object): Promise { - const realContent = Object.assign({}, content, { - version: VOIP_PROTO_VERSION, - call_id: this.callId, - party_id: this.ourPartyId, - conf_id: this.groupCallId, - }); - - if (this.opponentDeviceId) { - this.emit(CallEvent.SendVoipEvent, { - type: "toDevice", - eventType, - userId: this.invitee || this.getOpponentMember().userId, - opponentDeviceId: this.opponentDeviceId, - content: { - ...realContent, - device_id: this.client.deviceId, - sender_session_id: this.client.getSessionId(), - dest_session_id: this.opponentSessionId, - }, - }); - - return this.client.sendToDevice(eventType, { - [this.invitee || this.getOpponentMember().userId]: { - [this.opponentDeviceId]: { - ...realContent, - device_id: this.client.deviceId, - sender_session_id: this.client.getSessionId(), - dest_session_id: this.opponentSessionId, - }, - }, - }); - } else { - this.emit(CallEvent.SendVoipEvent, { - type: "sendEvent", - eventType, - roomId: this.roomId, - content: realContent, - userId: this.invitee || this.getOpponentMember().userId, - }); - - return this.client.sendEvent(this.roomId, eventType, realContent); - } - } - - private queueCandidate(content: RTCIceCandidate): void { - // We partially de-trickle candidates by waiting for `delay` before sending them - // amalgamated, in order to avoid sending too many m.call.candidates events and hitting - // rate limits in Matrix. - // In practice, it'd be better to remove rate limits for m.call.* - - // N.B. this deliberately lets you queue and send blank candidates, which MSC2746 - // currently proposes as the way to indicate that candidate gathering is complete. - // This will hopefully be changed to an explicit rather than implicit notification - // shortly. - this.candidateSendQueue.push(content); - - // Don't send the ICE candidates yet if the call is in the ringing state: this - // means we tried to pick (ie. started generating candidates) and then failed to - // send the answer and went back to the ringing state. Queue up the candidates - // to send if we successfully send the answer. - // Equally don't send if we haven't yet sent the answer because we can send the - // first batch of candidates along with the answer - if (this.state === CallState.Ringing || !this.inviteOrAnswerSent) return; - - // MSC2746 recommends these values (can be quite long when calling because the - // callee will need a while to answer the call) - const delay = this.direction === CallDirection.Inbound ? 500 : 2000; - - if (this.candidateSendTries === 0) { - setTimeout(() => { - this.sendCandidateQueue(); - }, delay); - } - } - - /* - * Transfers this call to another user - */ - public async transfer(targetUserId: string): Promise { - // Fetch the target user's global profile info: their room avatar / displayname - // could be different in whatever room we share with them. - const profileInfo = await this.client.getProfileInfo(targetUserId); - - const replacementId = genCallID(); - - const body = { - replacement_id: genCallID(), - target_user: { - id: targetUserId, - display_name: profileInfo.displayname, - avatar_url: profileInfo.avatar_url, - }, - create_call: replacementId, - } as MCallReplacesEvent; - - await this.sendVoipEvent(EventType.CallReplaces, body); - - await this.terminate(CallParty.Local, CallErrorCode.Transfered, true); - } - - /* - * Transfers this call to the target call, effectively 'joining' the - * two calls (so the remote parties on each call are connected together). - */ - public async transferToCall(transferTargetCall?: MatrixCall): Promise { - const targetProfileInfo = await this.client.getProfileInfo(transferTargetCall.getOpponentMember().userId); - const transfereeProfileInfo = await this.client.getProfileInfo(this.getOpponentMember().userId); - - const newCallId = genCallID(); - - const bodyToTransferTarget = { - // the replacements on each side have their own ID, and it's distinct from the - // ID of the new call (but we can use the same function to generate it) - replacement_id: genCallID(), - target_user: { - id: this.getOpponentMember().userId, - display_name: transfereeProfileInfo.displayname, - avatar_url: transfereeProfileInfo.avatar_url, - }, - await_call: newCallId, - } as MCallReplacesEvent; - - await transferTargetCall.sendVoipEvent(EventType.CallReplaces, bodyToTransferTarget); - - const bodyToTransferee = { - replacement_id: genCallID(), - target_user: { - id: transferTargetCall.getOpponentMember().userId, - display_name: targetProfileInfo.displayname, - avatar_url: targetProfileInfo.avatar_url, - }, - create_call: newCallId, - } as MCallReplacesEvent; - - await this.sendVoipEvent(EventType.CallReplaces, bodyToTransferee); - - await this.terminate(CallParty.Local, CallErrorCode.Replaced, true); - await transferTargetCall.terminate(CallParty.Local, CallErrorCode.Transfered, true); - } - - private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean): Promise { - if (this.callHasEnded()) return; - - this.hangupParty = hangupParty; - this.hangupReason = hangupReason; - this.setState(CallState.Ended); - - if (this.inviteTimeout) { - clearTimeout(this.inviteTimeout); - this.inviteTimeout = null; - } - if (this.callLengthInterval) { - clearInterval(this.callLengthInterval); - this.callLengthInterval = null; - } - - this.callStatsAtEnd = await this.collectCallStats(); - - // Order is important here: first we stopAllMedia() and only then we can deleteAllFeeds() - this.stopAllMedia(); - this.deleteAllFeeds(); - - if (this.peerConn && this.peerConn.signalingState !== 'closed') { - this.peerConn.close(); - } - if (shouldEmit) { - this.emit(CallEvent.Hangup, this); - } - - this.client.callEventHandler.calls.delete(this.callId); - } - - private stopAllMedia(): void { - logger.debug(!this.groupCallId ? "Stopping all media" : "Stopping all media except local feeds" ); - for (const feed of this.feeds) { - if ( - feed.isLocal() && - feed.purpose === SDPStreamMetadataPurpose.Usermedia && - !this.groupCallId - ) { - this.client.getMediaHandler().stopUserMediaStream(feed.stream); - } else if ( - feed.isLocal() && - feed.purpose === SDPStreamMetadataPurpose.Screenshare && - !this.groupCallId - ) { - this.client.getMediaHandler().stopScreensharingStream(feed.stream); - } else if (!feed.isLocal() || !this.groupCallId) { - for (const track of feed.stream.getTracks()) { - track.stop(); - } - } - } - } - - private checkForErrorListener(): void { - if (this.listeners("error").length === 0) { - throw new Error( - "You MUST attach an error listener using call.on('error', function() {})", - ); - } - } - - private async sendCandidateQueue(): Promise { - if (this.candidateSendQueue.length === 0 || this.callHasEnded()) { - return; - } - - const candidates = this.candidateSendQueue; - this.candidateSendQueue = []; - ++this.candidateSendTries; - const content = { - candidates: candidates, - }; - logger.debug("Attempting to send " + candidates.length + " candidates"); - try { - await this.sendVoipEvent(EventType.CallCandidates, content); - // reset our retry count if we have successfully sent our candidates - // otherwise queueCandidate() will refuse to try to flush the queue - this.candidateSendTries = 0; - - // Try to send candidates again just in case we received more candidates while sending. - this.sendCandidateQueue(); - } catch (error) { - // don't retry this event: we'll send another one later as we might - // have more candidates by then. - if (error.event) this.client.cancelPendingEvent(error.event); - - // put all the candidates we failed to send back in the queue - this.candidateSendQueue.push(...candidates); - - if (this.candidateSendTries > 5) { - logger.debug( - "Failed to send candidates on attempt " + this.candidateSendTries + - ". Giving up on this call.", error, - ); - - const code = CallErrorCode.SignallingFailed; - const message = "Signalling failed"; - - this.emit(CallEvent.Error, new CallError(code, message, error)); - this.hangup(code, false); - - return; - } - - const delayMs = 500 * Math.pow(2, this.candidateSendTries); - ++this.candidateSendTries; - logger.debug("Failed to send candidates. Retrying in " + delayMs + "ms", error); - setTimeout(() => { - this.sendCandidateQueue(); - }, delayMs); - } - } - - /** - * Place a call to this room. - * @throws if you have not specified a listener for 'error' events. - * @throws if have passed audio=false. - */ - public async placeCall(audio: boolean, video: boolean): Promise { - if (!audio) { - throw new Error("You CANNOT start a call without audio"); - } - this.setState(CallState.WaitLocalMedia); - - try { - const stream = await this.client.getMediaHandler().getUserMediaStream(audio, video); - const callFeed = new CallFeed({ - client: this.client, - roomId: this.roomId, - userId: this.client.getUserId(), - stream, - purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: stream.getAudioTracks().length === 0, - videoMuted: stream.getVideoTracks().length === 0, - }); - await this.placeCallWithCallFeeds([callFeed]); - } catch (e) { - this.getUserMediaFailed(e); - return; - } - } - - /** - * Place a call to this room with call feed. - * @param {CallFeed[]} callFeeds to use - * @throws if you have not specified a listener for 'error' events. - * @throws if have passed audio=false. - */ - public async placeCallWithCallFeeds(callFeeds: CallFeed[], requestScreenshareFeed = false): Promise { - this.checkForErrorListener(); - this.direction = CallDirection.Outbound; - - // XXX Find a better way to do this - this.client.callEventHandler.calls.set(this.callId, this); - - // make sure we have valid turn creds. Unless something's gone wrong, it should - // poll and keep the credentials valid so this should be instant. - const haveTurnCreds = await this.client.checkTurnServers(); - if (!haveTurnCreds) { - logger.warn("Failed to get TURN credentials! Proceeding with call anyway..."); - } - - // create the peer connection now so it can be gathering candidates while we get user - // media (assuming a candidate pool size is configured) - this.peerConn = this.createPeerConnection(); - this.gotCallFeedsForInvite(callFeeds, requestScreenshareFeed); - } - - private createPeerConnection(): RTCPeerConnection { - const pc = new window.RTCPeerConnection({ - iceTransportPolicy: this.forceTURN ? 'relay' : undefined, - iceServers: this.turnServers, - iceCandidatePoolSize: this.client.iceCandidatePoolSize, - }); - - // 'connectionstatechange' would be better, but firefox doesn't implement that. - pc.addEventListener('iceconnectionstatechange', this.onIceConnectionStateChanged); - pc.addEventListener('signalingstatechange', this.onSignallingStateChanged); - pc.addEventListener('icecandidate', this.gotLocalIceCandidate); - pc.addEventListener('icegatheringstatechange', this.onIceGatheringStateChange); - pc.addEventListener('track', this.onTrack); - pc.addEventListener('negotiationneeded', this.onNegotiationNeeded); - pc.addEventListener('datachannel', this.onDataChannel); - - return pc; - } - - private partyIdMatches(msg: MCallBase): boolean { - // They must either match or both be absent (in which case opponentPartyId will be null) - // Also we ignore party IDs on the invite/offer if the version is 0, so we must do the same - // here and use null if the version is 0 (woe betide any opponent sending messages in the - // same call with different versions) - const msgPartyId = msg.version === 0 ? null : msg.party_id || null; - return msgPartyId === this.opponentPartyId; - } - - // Commits to an opponent for the call - // ev: An invite or answer event - private chooseOpponent(ev: MatrixEvent): void { - // I choo-choo-choose you - const msg = ev.getContent(); - - logger.debug(`Choosing party ID ${msg.party_id} for call ID ${this.callId}`); - - this.opponentVersion = msg.version; - if (this.opponentVersion === 0) { - // set to null to indicate that we've chosen an opponent, but because - // they're v0 they have no party ID (even if they sent one, we're ignoring it) - this.opponentPartyId = null; - } else { - // set to their party ID, or if they're naughty and didn't send one despite - // not being v0, set it to null to indicate we picked an opponent with no - // party ID - this.opponentPartyId = msg.party_id || null; - } - this.opponentCaps = msg.capabilities || {} as CallCapabilities; - this.opponentMember = this.client.getRoom(this.roomId).getMember(ev.getSender()); - } - - private async addBufferedIceCandidates(): Promise { - const bufferedCandidates = this.remoteCandidateBuffer.get(this.opponentPartyId); - if (bufferedCandidates) { - logger.info(`Adding ${bufferedCandidates.length} buffered candidates for opponent ${this.opponentPartyId}`); - await this.addIceCandidates(bufferedCandidates); - } - this.remoteCandidateBuffer = null; - } - - private async addIceCandidates(candidates: RTCIceCandidate[]): Promise { - for (const candidate of candidates) { - if ( - (candidate.sdpMid === null || candidate.sdpMid === undefined) && - (candidate.sdpMLineIndex === null || candidate.sdpMLineIndex === undefined) - ) { - logger.debug("Ignoring remote ICE candidate with no sdpMid or sdpMLineIndex"); - continue; - } - logger.debug( - "Call " + this.callId + " got remote ICE " + candidate.sdpMid + " candidate: " + candidate.candidate, - ); - try { - await this.peerConn.addIceCandidate(candidate); - } catch (err) { - if (!this.ignoreOffer) { - logger.info("Failed to add remote ICE candidate", err); - } - } - } - } - - public get hasPeerConnection(): boolean { - return Boolean(this.peerConn); - } -} - -export function setTracksEnabled(tracks: Array, enabled: boolean): void { - for (let i = 0; i < tracks.length; i++) { - tracks[i].enabled = enabled; - } -} - -/** - * DEPRECATED - * Use client.createCall() - * - * Create a new Matrix call for the browser. - * @param {MatrixClient} client The client instance to use. - * @param {string} roomId The room the call is in. - * @param {Object?} options DEPRECATED optional options map. - * @param {boolean} options.forceTURN DEPRECATED whether relay through TURN should be - * forced. This option is deprecated - use opts.forceTURN when creating the matrix client - * since it's only possible to set this option on outbound calls. - * @return {MatrixCall} the call or null if the browser doesn't support calling. - */ -export function createNewMatrixCall(client: any, roomId: string, options?: CallOpts): MatrixCall { - // typeof prevents Node from erroring on an undefined reference - if (typeof(window) === 'undefined' || typeof(document) === 'undefined') { - // NB. We don't log here as apps try to create a call object as a test for - // whether calls are supported, so we shouldn't fill the logs up. - return null; - } - - // Firefox throws on so little as accessing the RTCPeerConnection when operating in - // a secure mode. There's some information at https://bugzilla.mozilla.org/show_bug.cgi?id=1542616 - // though the concern is that the browser throwing a SecurityError will brick the - // client creation process. - try { - const supported = Boolean( - window.RTCPeerConnection || window.RTCSessionDescription || - window.RTCIceCandidate || navigator.mediaDevices, - ); - if (!supported) { - // Adds a lot of noise to test runs, so disable logging there. - if (process.env.NODE_ENV !== "test") { - logger.error("WebRTC is not supported in this browser / environment"); - } - return null; - } - } catch (e) { - logger.error("Exception thrown when trying to access WebRTC", e); - return null; - } - - const optionsForceTURN = options ? options.forceTURN : false; - - const opts: CallOpts = { - client: client, - roomId: roomId, - invitee: options?.invitee, - turnServers: client.getTurnServers(), - // call level options - forceTURN: client.forceTURN || optionsForceTURN, - opponentDeviceId: options?.opponentDeviceId, - opponentSessionId: options?.opponentSessionId, - groupCallId: options?.groupCallId, - }; - const call = new MatrixCall(opts); - - client.reEmitter.reEmit(call, Object.values(CallEvent)); - - return call; -} - diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index 33b408f6..a220984e 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -1,3 +1,26 @@ +## TODO + - PeerCall + - send invite + - find out if we need to do something different when renegotation is triggered (a subsequent onnegotiationneeded event) whether + we sent the invite/offer or answer. e.g. do we always do createOffer/setLocalDescription and then send it over a matrix negotiation event? even if we before called createAnswer. + - handle receiving offer and send anwser + - handle sending ice candidates + - handle ice candidates finished (iceGatheringState === 'complete') + - handle receiving ice candidates + - handle sending renegotiation + - handle receiving renegotiation + - reject call + - hangup call + - handle muting tracks + - handle remote track being muted + - handle adding/removing tracks to an ongoing call + - handle sdp metadata + - Participant + - handle glare + - encrypt to_device message with olm + - batch outgoing to_device messages in one request to homeserver for operations that will send out an event to all participants (e.g. mute) + - find out if we should start muted or not? + ## Store ongoing calls Add store with all ongoing calls so when we quit and start again, we don't have to go through all the past calls to know which ones might still be ongoing. diff --git a/src/matrix/calls/group/Participant.ts b/src/matrix/calls/group/Participant.ts index 5abdaf63..26747e56 100644 --- a/src/matrix/calls/group/Participant.ts +++ b/src/matrix/calls/group/Participant.ts @@ -14,6 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */ +import {EventType} from "../PeerCall"; +import type {PeerCallHandler} from "../PeerCall"; + class Participant implements PeerCallHandler { private peerCall?: PeerCall; @@ -27,19 +30,18 @@ class Participant implements PeerCallHandler { sendInvite() { this.peerCall = new PeerCall(this, this.webRTC); - this.peerCall.setLocalMedia(this.localMedia); - this.peerCall.sendOffer(); + this.peerCall.call(this.localMedia); } /** From PeerCallHandler * @internal */ - override emitUpdate() { + emitUpdate(params: any) { } /** From PeerCallHandler * @internal */ - override onSendSignallingMessage() { + onSendSignallingMessage(type: EventType, content: Record) { // TODO: this needs to be encrypted with olm first this.hsApi.sendToDevice(type, {[this.userId]: {[this.deviceId ?? "*"]: content}}); } diff --git a/src/platform/types/WebRTC.ts b/src/platform/types/WebRTC.ts index 9a10ce5e..aff74f62 100644 --- a/src/platform/types/WebRTC.ts +++ b/src/platform/types/WebRTC.ts @@ -33,7 +33,7 @@ export interface PeerConnectionHandler { onDataChannelChanged(dataChannel: DataChannel | undefined); onNegotiationNeeded(); // request the type of incoming stream - getPurposeForStreamId(trackId: string): StreamPurpose; + getPurposeForStreamId(streamId: string): StreamPurpose; } // does it make sense to wrap this? export interface DataChannel { @@ -46,11 +46,12 @@ export interface PeerConnection { get dataChannel(): DataChannel | undefined; createOffer(): Promise; createAnswer(): Promise; - setLocalDescription(description: RTCSessionDescriptionInit); + setLocalDescription(description?: RTCSessionDescriptionInit): Promise; setRemoteDescription(description: RTCSessionDescriptionInit): Promise; addIceCandidate(candidate: RTCIceCandidate): Promise; addTrack(track: Track): void; removeTrack(track: Track): boolean; replaceTrack(oldTrack: Track, newTrack: Track): Promise; createDataChannel(): DataChannel; + dispose(): void; } diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 36d5eb75..917f42af 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -49,12 +49,12 @@ class DOMPeerConnection implements PeerConnection { return this.peerConnection.createAnswer(); } - setLocalDescription(description: RTCSessionDescriptionInit) { - this.peerConnection.setLocalDescription(description); + setLocalDescription(description?: RTCSessionDescriptionInit): Promise { + return this.peerConnection.setLocalDescription(description); } - setRemoteDescription(description: RTCSessionDescriptionInit) { - this.peerConnection.setRemoteDescription(description); + setRemoteDescription(description: RTCSessionDescriptionInit): Promise { + return this.peerConnection.setRemoteDescription(description); } addIceCandidate(candidate: RTCIceCandidate): Promise { @@ -66,6 +66,9 @@ class DOMPeerConnection implements PeerConnection { throw new Error("Not a TrackWrapper"); } this.peerConnection.addTrack(track.track, track.stream); + if (track.type === TrackType.ScreenShare) { + this.getRidOfRTXCodecs(track); + } } removeTrack(track: Track): boolean { @@ -87,6 +90,9 @@ class DOMPeerConnection implements PeerConnection { const sender = this.peerConnection.getSenders().find(s => s.track === oldTrack.track); if (sender) { await sender.replaceTrack(newTrack.track); + if (newTrack.type === TrackType.ScreenShare) { + this.getRidOfRTXCodecs(newTrack); + } return true; } return false; @@ -106,6 +112,17 @@ class DOMPeerConnection implements PeerConnection { pc.addEventListener('datachannel', this); } + private deregisterHandler() { + const pc = this.peerConnection; + pc.removeEventListener('negotiationneeded', this); + pc.removeEventListener('icecandidate', this); + pc.removeEventListener('iceconnectionstatechange', this); + pc.removeEventListener('icegatheringstatechange', this); + pc.removeEventListener('signalingstatechange', this); + pc.removeEventListener('track', this); + pc.removeEventListener('datachannel', this); + } + /** @internal */ handleEvent(evt: Event) { switch (evt.type) { @@ -129,6 +146,10 @@ class DOMPeerConnection implements PeerConnection { } } + dispose(): void { + this.deregisterHandler(); + } + private handleLocalIceCandidate(event: RTCPeerConnectionIceEvent) { if (event.candidate) { this.handler.onLocalIceCandidate(event.candidate); @@ -145,10 +166,15 @@ class DOMPeerConnection implements PeerConnection { } private handleRemoteTrack(evt: RTCTrackEvent) { + // the tracks on the new stream (with their stream) const updatedTracks = evt.streams.flatMap(stream => stream.getTracks().map(track => {return {stream, track};})); - const withoutRemovedTracks = this._remoteTracks.filter(t => !updatedTracks.some(ut => t.track == ut.track)); - const addedTracks = updatedTracks.filter(ut => !this._remoteTracks.some(t => t.track === ut.track)); + // of the tracks we already know about, filter the ones that aren't in the new stream + const withoutRemovedTracks = this._remoteTracks.filter(t => !updatedTracks.some(ut => t.track.id === ut.track.id)); + // of the new tracks, filter the ones that we didn't already knew about + const addedTracks = updatedTracks.filter(ut => !this._remoteTracks.some(t => t.track.id === ut.track.id)); + // wrap them const wrappedAddedTracks = addedTracks.map(t => this.wrapRemoteTrack(t.track, t.stream)); + // and concat the tracks for other streams with the added tracks this._remoteTracks = withoutRemovedTracks.concat(...wrappedAddedTracks); this.handler.onRemoteTracksChanged(this.remoteTracks); } @@ -162,4 +188,44 @@ class DOMPeerConnection implements PeerConnection { } return wrapTrack(track, stream, type); } + + /** + * This method removes all video/rtx codecs from screensharing video + * transceivers. This is necessary since they can cause problems. Without + * this the following steps should produce an error: + * Chromium calls Firefox + * Firefox answers + * Firefox starts screen-sharing + * Chromium starts screen-sharing + * Call crashes for Chromium with: + * [96685:23:0518/162603.933321:ERROR:webrtc_video_engine.cc(3296)] RTX codec (PT=97) mapped to PT=96 which is not in the codec list. + * [96685:23:0518/162603.933377:ERROR:webrtc_video_engine.cc(1171)] GetChangedRecvParameters called without any video codecs. + * [96685:23:0518/162603.933430:ERROR:sdp_offer_answer.cc(4302)] Failed to set local video description recv parameters for m-section with mid='2'. (INVALID_PARAMETER) + */ + private getRidOfRTXCodecs(screensharingTrack: TrackWrapper): void { + // RTCRtpReceiver.getCapabilities and RTCRtpSender.getCapabilities don't seem to be supported on FF + if (!RTCRtpReceiver.getCapabilities || !RTCRtpSender.getCapabilities) return; + + const recvCodecs = RTCRtpReceiver.getCapabilities("video")?.codecs ?? []; + const sendCodecs = RTCRtpSender.getCapabilities("video")?.codecs ?? []; + const codecs = [...sendCodecs, ...recvCodecs]; + + for (const codec of codecs) { + if (codec.mimeType === "video/rtx") { + const rtxCodecIndex = codecs.indexOf(codec); + codecs.splice(rtxCodecIndex, 1); + } + } + + for (const trans of this.peerConnection.getTransceivers()) { + if (trans.sender.track === screensharingTrack.track && + ( + trans.sender.track?.kind === "video" || + trans.receiver.track?.kind === "video" + ) + ) { + trans.setCodecPreferences(codecs); + } + } + } } diff --git a/src/utils/AsyncQueue.ts b/src/utils/AsyncQueue.ts new file mode 100644 index 00000000..0686314c --- /dev/null +++ b/src/utils/AsyncQueue.ts @@ -0,0 +1,52 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +export class AsyncQueue { + private isRunning = false; + private queue: T[] = []; + private error?: Error; + + constructor( + private readonly reducer: (v: V, t: T) => Promise, + private value: V, + private readonly contains: (t: T, queue: T[]) => boolean = (t, queue) => queue.includes(t) + ) {} + + push(t: T) { + if (this.contains(t, this.queue)) { + return; + } + this.queue.push(t); + this.runLoopIfNeeded(); + } + + private async runLoopIfNeeded() { + if (this.isRunning || this.error) { + return; + } + this.isRunning = true; + try { + let item: T | undefined; + while (item = this.queue.shift()) { + this.value = await this.reducer(this.value, item); + } + } catch (err) { + this.error = err; + } finally { + this.isRunning = false; + } + } +} From 179c7e74b56511fafaf9972568d2bb7928be4889 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Fri, 25 Feb 2022 16:54:00 +0100 Subject: [PATCH 006/323] WIP6 --- src/matrix/calls/CallHandler.ts | 2 + src/matrix/calls/LocalMedia.ts | 57 ++ src/matrix/calls/PeerCall.ts | 822 ++++++++++----------------- src/matrix/calls/TODO.md | 72 ++- src/platform/types/WebRTC.ts | 13 +- src/platform/web/dom/MediaDevices.ts | 7 +- src/platform/web/dom/WebRTC.ts | 32 +- src/utils/recursivelyAssign.ts | 39 ++ 8 files changed, 487 insertions(+), 557 deletions(-) create mode 100644 src/matrix/calls/LocalMedia.ts create mode 100644 src/utils/recursivelyAssign.ts diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index a31ff666..4a2027f3 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -44,6 +44,8 @@ export class GroupCallHandler { } + // TODO: check and poll turn server credentials here + handleRoomState(room: Room, events: StateEvent[], log: ILogItem) { // first update call events for (const event of events) { diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts new file mode 100644 index 00000000..2429ff2c --- /dev/null +++ b/src/matrix/calls/LocalMedia.ts @@ -0,0 +1,57 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {StreamPurpose} from "../../platform/types/WebRTC"; +import {Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; +import {SDPStreamMetadata} from "./callEventTypes"; + +export class LocalMedia { + constructor( + public readonly cameraTrack?: Track, + public readonly screenShareTrack?: Track, + public readonly microphoneTrack?: AudioTrack + ) {} + + withTracks(tracks: Track[]) { + const cameraTrack = tracks.find(t => t.type === TrackType.Camera) ?? this.cameraTrack; + const screenShareTrack = tracks.find(t => t.type === TrackType.ScreenShare) ?? this.screenShareTrack; + const microphoneTrack = tracks.find(t => t.type === TrackType.Microphone) ?? this.microphoneTrack; + if (cameraTrack && microphoneTrack && cameraTrack.streamId !== microphoneTrack.streamId) { + throw new Error("The camera and audio track should have the same stream id"); + } + return new LocalMedia(cameraTrack, screenShareTrack, microphoneTrack as AudioTrack); + } + + get tracks(): Track[] { return []; } + + getSDPMetadata(): SDPStreamMetadata { + const metadata = {}; + const userMediaTrack = this.microphoneTrack ?? this.cameraTrack; + if (userMediaTrack) { + metadata[userMediaTrack.streamId] = { + purpose: StreamPurpose.UserMedia, + audio_muted: this.microphoneTrack?.muted ?? false, + video_muted: this.cameraTrack?.muted ?? false, + }; + } + if (this.screenShareTrack) { + metadata[this.screenShareTrack.streamId] = { + purpose: StreamPurpose.ScreenShare + }; + } + return metadata; + } +} diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 47dfaa3e..17e49afd 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -15,7 +15,7 @@ limitations under the License. */ import {ObservableMap} from "../../observable/map/ObservableMap"; - +import {recursivelyAssign} from "../../utils/recursivelyAssign"; import {AsyncQueue} from "../../utils/AsyncQueue"; import type {Room} from "../room/Room"; import type {StateEvent} from "../storage/types"; @@ -23,8 +23,284 @@ import type {ILogItem} from "../../logging/types"; import {WebRTC, PeerConnection, PeerConnectionHandler, StreamPurpose} from "../../platform/types/WebRTC"; import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; +import type {LocalMedia} from "./LocalMedia"; -import { randomString } from '../randomstring'; +// when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already +// do for sharing keys will be best as that already deals with room tracking. +/** + * Does WebRTC signalling for a single PeerConnection, and deals with WebRTC wrappers from platform + * */ +/** Implements a call between two peers with the signalling state keeping, while still delegating the signalling message sending. Used by GroupCall.*/ +class PeerCall { + private readonly peerConnection: PeerConnection; + private state = CallState.Fledgling; + private direction: CallDirection; + // A queue for candidates waiting to go out. + // We try to amalgamate candidates into a single candidate message where + // possible + private candidateSendQueue: Array = []; + // If candidates arrive before we've picked an opponent (which, in particular, + // will happen if the opponent sends candidates eagerly before the user answers + // the call) we buffer them up here so we can then add the ones from the party we pick + private remoteCandidateBuffer? = new Map(); + + private logger: any; + private remoteSDPStreamMetadata?: SDPStreamMetadata; + private responsePromiseChain?: Promise; + private opponentPartyId?: PartyId; + + constructor( + private readonly handler: PeerCallHandler, + private localMedia: LocalMedia, + webRTC: WebRTC + ) { + const outer = this; + this.peerConnection = webRTC.createPeerConnection({ + onIceConnectionStateChange(state: RTCIceConnectionState) {}, + onLocalIceCandidate(candidate: RTCIceCandidate) {}, + onIceGatheringStateChange(state: RTCIceGatheringState) {}, + onRemoteTracksChanged(tracks: Track[]) {}, + onDataChannelChanged(dataChannel: DataChannel | undefined) {}, + onNegotiationNeeded() { + const promiseCreator = () => outer.handleNegotiation(); + outer.responsePromiseChain = outer.responsePromiseChain?.then(promiseCreator) ?? promiseCreator(); + }, + getPurposeForStreamId(streamId: string): SDPStreamMetadataPurpose { + return outer.remoteSDPStreamMetadata?.[streamId]?.purpose ?? SDPStreamMetadataPurpose.Usermedia; + } + }); + this.logger = { + debug(...args) { console.log.apply(console, ["WebRTC debug:", ...args])}, + log(...args) { console.log.apply(console, ["WebRTC log:", ...args])}, + warn(...args) { console.log.apply(console, ["WebRTC warn:", ...args])}, + error(...args) { console.error.apply(console, ["WebRTC error:", ...args])}, + } + } + + handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId) { + switch (message.type) { + case EventType.Invite: + this.handleInvite(message.content); + break; + case EventType.Answer: + this.handleAnswer(message.content); + break; + case EventType.Candidates: + this.handleRemoteIceCandidates(message.content); + break; + case EventType.Hangup: + } + } + + async call(localMediaPromise: Promise): Promise { + if (this.state !== CallState.Fledgling) { + return; + } + this.direction = CallDirection.Outbound; + this.setState(CallState.WaitLocalMedia); + try { + this.localMedia = await localMediaPromise; + } catch (err) { + this.setState(CallState.Ended); + return; + } + this.setState(CallState.CreateOffer); + // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called + for (const t of this.localMedia.tracks) { + this.peerConnection.addTrack(t); + } + await this.waitForState(CallState.InviteSent); + } + + async answer() { + + } + + async hangup() { + + } + + async updateLocalMedia(localMediaPromise: Promise) { + const oldMedia = this.localMedia; + this.localMedia = await localMediaPromise; + + const applyTrack = (selectTrack: (media: LocalMedia) => Track | undefined) => { + const oldTrack = selectTrack(oldMedia); + const newTrack = selectTrack(this.localMedia); + if (oldTrack && newTrack) { + this.peerConnection.replaceTrack(oldTrack, newTrack); + } else if (oldTrack) { + this.peerConnection.removeTrack(oldTrack); + } else if (newTrack) { + this.peerConnection.addTrack(newTrack); + } + }; + + // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called + applyTrack(m => m.microphoneTrack); + applyTrack(m => m.cameraTrack); + applyTrack(m => m.screenShareTrack); + } + + // calls are serialized and deduplicated by negotiationQueue + private handleNegotiation = async (): Promise => { + try { + await this.peerConnection.setLocalDescription(); + } catch (err) { + this.logger.debug(`Call ${this.callId} Error setting local description!`, err); + this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); + return; + } + + if (this.peerConnection.iceGatheringState === 'gathering') { + // Allow a short time for initial candidates to be gathered + await new Promise(resolve => setTimeout(resolve, 200)); + } + + if (this.state === CallState.Ended) { + return; + } + + const offer = this.peerConnection.localDescription!; + // Get rid of any candidates waiting to be sent: they'll be included in the local + // description we just got and will send in the offer. + this.logger.info(`Call ${this.callId} Discarding ${ + this.candidateSendQueue.length} candidates that will be sent in offer`); + this.candidateSendQueue = []; + + // need to queue this + const content = { + offer, + [SDPStreamMetadataKey]: this.localMedia.getSDPMetadata(), + version: 1, + lifetime: CALL_TIMEOUT_MS + }; + if (this.state === CallState.CreateOffer) { + await this.handler.sendSignallingMessage({type: EventType.Invite, content}); + this.setState(CallState.InviteSent); + } + }; + + private async handleInvite(content: InviteContent, partyId: PartyId): Promise { + if (this.state !== CallState.Fledgling || this.opponentPartyId !== undefined) { + // TODO: hangup or ignore? + return; + } + + // we must set the party ID before await-ing on anything: the call event + // handler will start giving us more call events (eg. candidates) so if + // we haven't set the party ID, we'll ignore them. + this.opponentPartyId = partyId; + this.direction = CallDirection.Inbound; + + const sdpStreamMetadata = content[SDPStreamMetadataKey]; + if (sdpStreamMetadata) { + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); + } else { + this.logger.debug(`Call ${ + this.callId} did not get any SDPStreamMetadata! Can not send/receive multiple streams`); + } + + try { + await this.peerConnection.setRemoteDescription(content.offer); + await this.addBufferedIceCandidates(); + } catch (e) { + this.logger.debug(`Call ${this.callId} failed to set remote description`, e); + this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); + return; + } + + // According to previous comments in this file, firefox at some point did not + // add streams until media started arriving on them. Testing latest firefox + // (81 at time of writing), this is no longer a problem, so let's do it the correct way. + if (this.peerConnection.remoteTracks.length === 0) { + this.logger.error(`Call ${this.callId} no remote stream or no tracks after setting remote description!`); + this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); + return; + } + + this.setState(CallState.Ringing); + + setTimeout(() => { + if (this.state == CallState.Ringing) { + this.logger.debug(`Call ${this.callId} invite has expired. Hanging up.`); + this.hangupParty = CallParty.Remote; // effectively + this.setState(CallState.Ended); + this.stopAllMedia(); + if (this.peerConnection.signalingState != 'closed') { + this.peerConnection.close(); + } + this.emit(CallEvent.Hangup); + } + }, content.lifetime ?? CALL_TIMEOUT_MS /* - event.getLocalAge() */ ); + } + + private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { + this.remoteSDPStreamMetadata = recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); + // will rerequest stream purpose for all tracks and set track.type accordingly + this.peerConnection.notifyStreamPurposeChanged(); + for (const track of this.peerConnection.remoteTracks) { + const streamMetaData = this.remoteSDPStreamMetadata?.[track.streamId]; + if (streamMetaData) { + if (track.type === TrackType.Microphone) { + track.setMuted(streamMetaData.audio_muted); + } else { // Camera or ScreenShare + track.setMuted(streamMetaData.video_muted); + } + } + } + } + + + private async addBufferedIceCandidates(): Promise { + const bufferedCandidates = this.remoteCandidateBuffer!.get(this.opponentPartyId!); + if (bufferedCandidates) { + this.logger.info(`Call ${this.callId} Adding ${ + bufferedCandidates.length} buffered candidates for opponent ${this.opponentPartyId}`); + await this.addIceCandidates(bufferedCandidates); + } + this.remoteCandidateBuffer = undefined; + } + + private async addIceCandidates(candidates: RTCIceCandidate[]): Promise { + for (const candidate of candidates) { + if ( + (candidate.sdpMid === null || candidate.sdpMid === undefined) && + (candidate.sdpMLineIndex === null || candidate.sdpMLineIndex === undefined) + ) { + this.logger.debug(`Call ${this.callId} ignoring remote ICE candidate with no sdpMid or sdpMLineIndex`); + continue; + } + this.logger.debug(`Call ${this.callId} got remote ICE ${candidate.sdpMid} candidate: ${candidate.candidate}`); + try { + await this.peerConnection.addIceCandidate(candidate); + } catch (err) { + if (!this.ignoreOffer) { + this.logger.info(`Call ${this.callId} failed to add remote ICE candidate`, err); + } + } + } + } + + + private setState(state: CallState): void { + const oldState = this.state; + this.state = state; + this.handler.emitUpdate(); + } + + private waitForState(state: CallState): Promise { + + } + + private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean): Promise { + + } +} + + + +//import { randomString } from '../randomstring'; import { MCallReplacesEvent, MCallAnswer, @@ -41,41 +317,13 @@ import { MCallHangupReject, } from './callEventTypes'; - -const GROUP_CALL_TYPE = "m.call"; -const GROUP_CALL_MEMBER_TYPE = "m.call.member"; - - -/** - * Fires whenever an error occurs when call.js encounters an issue with setting up the call. - *

- * The error given will have a code equal to either `MatrixCall.ERR_LOCAL_OFFER_FAILED` or - * `MatrixCall.ERR_NO_USER_MEDIA`. `ERR_LOCAL_OFFER_FAILED` is emitted when the local client - * fails to create an offer. `ERR_NO_USER_MEDIA` is emitted when the user has denied access - * to their audio/video hardware. - * - * @event module:webrtc/call~MatrixCall#"error" - * @param {Error} err The error raised by MatrixCall. - * @example - * matrixCall.on("error", function(err){ - * console.error(err.code, err); - * }); - */ - // null is used as a special value meaning that the we're in a legacy 1:1 call // without MSC2746 that doesn't provide an id which device sent the message. type PartyId = string | null; -interface TurnServer { - urls: Array; - username?: string; - password?: string; - ttl?: number; -} - -interface AssertedIdentity { - id: string; - displayName: string; +export enum CallParty { + Local = 'local', + Remote = 'remote', } export enum CallState { @@ -90,21 +338,11 @@ export enum CallState { Ended = 'ended', } -export enum CallType { - Voice = 'voice', - Video = 'video', -} - export enum CallDirection { Inbound = 'inbound', Outbound = 'outbound', } -export enum CallParty { - Local = 'local', - Remote = 'remote', -} - export enum EventType { Invite = "m.call.invite", Candidates = "m.call.candidates", @@ -120,30 +358,6 @@ export enum EventType { AssertedIdentityPrefix = "org.matrix.call.asserted_identity", } -export enum CallEvent { - Hangup = 'hangup', - State = 'state', - Error = 'error', - Replaced = 'replaced', - - // The value of isLocalOnHold() has changed - LocalHoldUnhold = 'local_hold_unhold', - // The value of isRemoteOnHold() has changed - RemoteHoldUnhold = 'remote_hold_unhold', - // backwards compat alias for LocalHoldUnhold: remove in a major version bump - HoldUnhold = 'hold_unhold', - // Feeds have changed - FeedsChanged = 'feeds_changed', - - AssertedIdentityChanged = 'asserted_identity_changed', - - LengthChanged = 'length_changed', - - DataChannel = 'datachannel', - - SendVoipEvent = "send_voip_event", -} - export enum CallErrorCode { /** The user chose to end the call */ UserHangup = 'user_hangup', @@ -254,477 +468,21 @@ export class CallError extends Error { } } -export function genCallID(): string { - return Date.now().toString() + randomString(16); -} - -enum CallSetupMessageType { - Invite = "m.call.invite", - Answer = "m.call.answer", - Candidates = "m.call.candidates", - Hangup = "m.call.hangup", -} - -const CALL_ID = "m.call_id"; -const CALL_TERMINATED = "m.terminated"; - -class LocalMedia { - constructor( - public readonly cameraTrack?: Track, - public readonly screenShareTrack?: Track, - public readonly microphoneTrack?: AudioTrack - ) {} - - withTracks(tracks: Track[]) { - const cameraTrack = tracks.find(t => t.type === TrackType.Camera) ?? this.cameraTrack; - const screenShareTrack = tracks.find(t => t.type === TrackType.ScreenShare) ?? this.screenShareTrack; - const microphoneTrack = tracks.find(t => t.type === TrackType.Microphone) ?? this.microphoneTrack; - if (cameraTrack && microphoneTrack && cameraTrack.streamId !== microphoneTrack.streamId) { - throw new Error("The camera and audio track should have the same stream id"); - } - return new LocalMedia(cameraTrack, screenShareTrack, microphoneTrack as AudioTrack); - } - - get tracks(): Track[] { return []; } - - getSDPMetadata(): any { - const metadata = {}; - const userMediaTrack = this.microphoneTrack ?? this.cameraTrack; - if (userMediaTrack) { - metadata[userMediaTrack.streamId] = { - purpose: StreamPurpose.UserMedia, - audio_muted: this.microphoneTrack?.muted ?? false, - video_muted: this.cameraTrack?.muted ?? false, - }; - } - if (this.screenShareTrack) { - metadata[this.screenShareTrack.streamId] = { - purpose: StreamPurpose.ScreenShare - }; - } - return metadata; - } +type InviteContent = { + offer: RTCSessionDescriptionInit, + [SDPStreamMetadataKey]: SDPStreamMetadata, + version?: number, + lifetime?: number } export type InviteMessage = { type: EventType.Invite, - content: { - version: number - } + content: InviteContent } +export type SignallingMessage = InviteMessage; + export interface PeerCallHandler { emitUpdate(peerCall: PeerCall, params: any); - sendSignallingMessage(type: EventType, content: Record); -} - -// when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already -// do for sharing keys will be best as that already deals with room tracking. -/** - * Does WebRTC signalling for a single PeerConnection, and deals with WebRTC wrappers from platform - * */ -/** Implements a call between two peers with the signalling state keeping, while still delegating the signalling message sending. Used by GroupCall.*/ -class PeerCall implements PeerConnectionHandler { - private readonly peerConnection: PeerConnection; - - - public state = CallState.Fledgling; - public hangupParty: CallParty; - public hangupReason: string; - public direction: CallDirection; - public peerConn?: RTCPeerConnection; - - // A queue for candidates waiting to go out. - // We try to amalgamate candidates into a single candidate message where - // possible - private candidateSendQueue: Array = []; - private candidateSendTries = 0; - private sentEndOfCandidates = false; - - private inviteOrAnswerSent = false; - private waitForLocalAVStream: boolean; - private opponentVersion: number | string; - // The party ID of the other side: undefined if we haven't chosen a partner - // yet, null if we have but they didn't send a party ID. - private opponentPartyId: PartyId; - private opponentCaps: CallCapabilities; - private inviteTimeout: number; - private iceDisconnectedTimeout: number; - - // The logic of when & if a call is on hold is nontrivial and explained in is*OnHold - // This flag represents whether we want the other party to be on hold - private remoteOnHold = false; - - // the stats for the call at the point it ended. We can't get these after we - // tear the call down, so we just grab a snapshot before we stop the call. - // The typescript definitions have this type as 'any' :( - private callStatsAtEnd: any[]; - - // Perfect negotiation state: https://www.w3.org/TR/webrtc/#perfect-negotiation-example - private makingOffer = false; - private ignoreOffer: boolean; - - // If candidates arrive before we've picked an opponent (which, in particular, - // will happen if the opponent sends candidates eagerly before the user answers - // the call) we buffer them up here so we can then add the ones from the party we pick - private remoteCandidateBuffer: Map; - - private remoteAssertedIdentity: AssertedIdentity; - - private remoteSDPStreamMetadata?: SDPStreamMetadata; - - private negotiationQueue: AsyncQueue; - - constructor( - private readonly handler: PeerCallHandler, - private localMedia: LocalMedia, - webRTC: WebRTC - ) { - this.peerConnection = webRTC.createPeerConnection(this); - // TODO: should we use this to serialize all state changes? - this.negotiationQueue = new AsyncQueue(this.handleNegotiation, void); - } - - // PeerConnectionHandler method - onIceConnectionStateChange(state: RTCIceConnectionState) {} - // PeerConnectionHandler method - onLocalIceCandidate(candidate: RTCIceCandidate) {} - // PeerConnectionHandler method - onIceGatheringStateChange(state: RTCIceGatheringState) {} - // PeerConnectionHandler method - onRemoteTracksChanged(tracks: Track[]) {} - // PeerConnectionHandler method - onDataChannelChanged(dataChannel: DataChannel | undefined) {} - // PeerConnectionHandler method - onNegotiationNeeded() { - // trigger handleNegotiation - this.negotiationQueue.push(void); - } - - // calls are serialized and deduplicated by negotiationQueue - private handleNegotiation = async (): Promise => { - const offer = await this.peerConnection.createOffer(); - this.peerConnection.setLocalDescription(offer); - // need to queue this - const message = { - offer, - sdp_stream_metadata: this.localMedia.getSDPMetadata(), - version: 1 - } - if (this.state === CallState.Fledgling) { - const sendPromise = this.handler.sendSignallingMessage(EventType.Invite, message); - this.setState(CallState.InviteSent); - } else { - await this.handler.sendSignallingMessage(EventType.Negotiate, message); - } - }; - - async sendInvite(localMediaPromise: Promise): Promise { - if (this.state !== CallState.Fledgling) { - return; - } - this.setState(CallState.WaitLocalMedia); - this.localMedia = await localMediaPromise; - // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called - for (const t of this.localMedia.tracks) { - this.peerConnection.addTrack(t); - } - await this.waitForState(CallState.Ended, CallState.InviteSent); - } - - async sendAnswer(localMediaPromise: Promise): Promise { - if (this.callHasEnded()) return; - - if (this.state !== CallState.Ringing) { - return; - } - - this.setState(CallState.WaitLocalMedia); - this.waitForLocalAVStream = true; - this.localMedia = await localMediaPromise; - this.waitForLocalAVStream = false; - - // enqueue the following - - // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called - for (const t of this.localMedia.tracks) { - this.peerConnection.addTrack(t); - } - - this.setState(CallState.CreateAnswer); - - let myAnswer; - try { - myAnswer = await this.peerConn.createAnswer(); - } catch (err) { - logger.debug("Failed to create answer: ", err); - this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true); - return; - } - - try { - await this.peerConn.setLocalDescription(myAnswer); - this.setState(CallState.Connecting); - - // Allow a short time for initial candidates to be gathered - await new Promise(resolve => { - setTimeout(resolve, 200); - }); - // inlined sendAnswer - const answerContent = { - answer: { - sdp: this.peerConn.localDescription.sdp, - // type is now deprecated as of Matrix VoIP v1, but - // required to still be sent for backwards compat - type: this.peerConn.localDescription.type, - }, - [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(true), - } as MCallAnswer; - - answerContent.capabilities = { - 'm.call.transferee': this.client.supportsCallTransfer, - 'm.call.dtmf': false, - }; - - // We have just taken the local description from the peerConn which will - // contain all the local candidates added so far, so we can discard any candidates - // we had queued up because they'll be in the answer. - logger.info(`Discarding ${this.candidateSendQueue.length} candidates that will be sent in answer`); - this.candidateSendQueue = []; - - try { - await this.sendVoipEvent(EventType.CallAnswer, answerContent); - // If this isn't the first time we've tried to send the answer, - // we may have candidates queued up, so send them now. - this.inviteOrAnswerSent = true; - } catch (error) { - // We've failed to answer: back to the ringing state - this.setState(CallState.Ringing); - this.client.cancelPendingEvent(error.event); - - let code = CallErrorCode.SendAnswer; - let message = "Failed to send answer"; - if (error.name == 'UnknownDeviceError') { - code = CallErrorCode.UnknownDevices; - message = "Unknown devices present in the room"; - } - this.emit(CallEvent.Error, new CallError(code, message, error)); - throw error; - } - - // error handler re-throws so this won't happen on error, but - // we don't want the same error handling on the candidate queue - this.sendCandidateQueue(); - } catch (err) { - logger.debug("Error setting local description!", err); - this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); - return; - } - } - - async updateLocalMedia(localMediaPromise: Promise) { - const oldMedia = this.localMedia; - this.localMedia = await localMediaPromise; - - const applyTrack = (selectTrack: (media: LocalMedia) => Track | undefined) => { - const oldTrack = selectTrack(oldMedia); - const newTrack = selectTrack(this.localMedia); - if (oldTrack && newTrack) { - this.peerConnection.replaceTrack(oldTrack, newTrack); - } else if (oldTrack) { - this.peerConnection.removeTrack(oldTrack); - } else if (newTrack) { - this.peerConnection.addTrack(newTrack); - } - }; - - // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called - applyTrack(m => m.microphoneTrack); - applyTrack(m => m.cameraTrack); - applyTrack(m => m.screenShareTrack); - } - - - /** - * Replace this call with a new call, e.g. for glare resolution. Used by - * MatrixClient. - * @param {MatrixCall} newCall The new call. - */ - public replacedBy(newCall: MatrixCall): void { - if (this.state === CallState.WaitLocalMedia) { - logger.debug("Telling new call to wait for local media"); - newCall.waitForLocalAVStream = true; - } else if ([CallState.CreateOffer, CallState.InviteSent].includes(this.state)) { - if (newCall.direction === CallDirection.Outbound) { - newCall.queueGotCallFeedsForAnswer([]); - } else { - logger.debug("Handing local stream to new call"); - newCall.queueGotCallFeedsForAnswer(this.getLocalFeeds().map(feed => feed.clone())); - } - } - this.successor = newCall; - this.emit(CallEvent.Replaced, newCall); - this.hangup(CallErrorCode.Replaced, true); - } - - /** - * Hangup a call. - * @param {string} reason The reason why the call is being hung up. - * @param {boolean} suppressEvent True to suppress emitting an event. - */ - public hangup(reason: CallErrorCode, suppressEvent: boolean): void { - if (this.callHasEnded()) return; - - logger.debug("Ending call " + this.callId); - this.terminate(CallParty.Local, reason, !suppressEvent); - // We don't want to send hangup here if we didn't even get to sending an invite - if (this.state === CallState.WaitLocalMedia) return; - const content = {}; - // Don't send UserHangup reason to older clients - if ((this.opponentVersion && this.opponentVersion >= 1) || reason !== CallErrorCode.UserHangup) { - content["reason"] = reason; - } - this.sendVoipEvent(EventType.CallHangup, content); - } - - /** - * Reject a call - * This used to be done by calling hangup, but is a separate method and protocol - * event as of MSC2746. - */ - public reject(): void { - if (this.state !== CallState.Ringing) { - throw Error("Call must be in 'ringing' state to reject!"); - } - - if (this.opponentVersion < 1) { - logger.info( - `Opponent version is less than 1 (${this.opponentVersion}): sending hangup instead of reject`, - ); - this.hangup(CallErrorCode.UserHangup, true); - return; - } - - logger.debug("Rejecting call: " + this.callId); - this.terminate(CallParty.Local, CallErrorCode.UserHangup, true); - this.sendVoipEvent(EventType.CallReject, {}); - } - - // request the type of incoming track - getPurposeForStreamId(streamId: string): StreamPurpose { - // TODO: should we return a promise here for the case where the metadata hasn't arrived yet? - const metaData = this.remoteSDPStreamMetadata[streamId]; - return metadata?.purpose as StreamPurpose ?? StreamPurpose.UserMedia; - } - - private setState(state: CallState): void { - const oldState = this.state; - this.state = state; - this.handler.emitUpdate(); - if (this.inviteDeferred) { - if (this.state === CallState.InviteSent) { - this.inviteDeferred.resolve(); - } - } - } - - handleIncomingSignallingMessage(type: CallSetupMessageType, content: Record, partyId: PartyId) { - switch (type) { - case CallSetupMessageType.Invite: - case CallSetupMessageType.Answer: - this.handleAnswer(content); - break; - case CallSetupMessageType.Candidates: - this.handleRemoteIceCandidates(content); - break; - case CallSetupMessageType.Hangup: - } - } - - private async handleAnswer(content: MCallAnswer, partyId: PartyId) { - // add buffered ice candidates to peerConnection - if (this.opponentPartyId !== undefined) { - return; - } - this.opponentPartyId = partyId; - const bufferedCandidates = this.remoteCandidateBuffer?.get(partyId); - if (bufferedCandidates) { - this.addIceCandidates(bufferedCandidates); - } - this.remoteCandidateBuffer = undefined; - - this.setState(CallState.Connecting); - - const sdpStreamMetadata = content[SDPStreamMetadataKey]; - if (sdpStreamMetadata) { - this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); - } else { - logger.warn("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); - } - - try { - await this.peerConnection.setRemoteDescription(content.answer); - } catch (e) { - logger.debug("Failed to set remote description", e); - this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); - return; - } - - // If the answer we selected has a party_id, send a select_answer event - // We do this after setting the remote description since otherwise we'd block - // call setup on it - if (this.opponentPartyId !== null) { - try { - await this.sendVoipEvent(EventType.CallSelectAnswer, { - selected_party_id: this.opponentPartyId, - }); - } catch (err) { - // This isn't fatal, and will just mean that if another party has raced to answer - // the call, they won't know they got rejected, so we carry on & don't retry. - logger.warn("Failed to send select_answer event", err); - } - } - } - - private handleRemoteIceCandidates(content: Record) { - if (this.state === CallState.Ended) { - return; - } - const candidates = content.candidates; - if (!candidates) { - return; - } - if (this.opponentPartyId === undefined) { - if (!this.remoteCandidateBuffer) { - this.remoteCandidateBuffer = new Map(); - } - const bufferedCandidates = this.remoteCandidateBuffer.get(fromPartyId) || []; - bufferedCandidates.push(...candidates); - this.remoteCandidateBuffer.set(fromPartyId, bufferedCandidates); - } else { - this.addIceCandidates(candidates); - } - } - - private async addIceCandidates(candidates: RTCIceCandidate[]): Promise { - for (const candidate of candidates) { - if ( - (candidate.sdpMid === null || candidate.sdpMid === undefined) && - (candidate.sdpMLineIndex === null || candidate.sdpMLineIndex === undefined) - ) { - logger.debug("Ignoring remote ICE candidate with no sdpMid or sdpMLineIndex"); - continue; - } - logger.debug( - "Call " + this.callId + " got remote ICE " + candidate.sdpMid + " candidate: " + candidate.candidate, - ); - try { - await this.peerConnection.addIceCandidate(candidate); - } catch (err) { - if (!this.ignoreOffer) { - logger.info("Failed to add remote ICE candidate", err); - } - } - } - } + sendSignallingMessage(message: InviteMessage); } diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index a220984e..397c9d38 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -1,8 +1,19 @@ + - relevant MSCs next to spec: + - https://github.com/matrix-org/matrix-doc/pull/2746 Improved Signalling for 1:1 VoIP + - https://github.com/matrix-org/matrix-doc/pull/2747 Transferring VoIP Calls + - https://github.com/matrix-org/matrix-doc/pull/3077 Support for multi-stream VoIP + - https://github.com/matrix-org/matrix-doc/pull/3086 Asserted identity on VoIP calls + - https://github.com/matrix-org/matrix-doc/pull/3291 Muting in VoIP calls + - https://github.com/matrix-org/matrix-doc/pull/3401 Native Group VoIP Signalling + ## TODO - PeerCall - send invite - - find out if we need to do something different when renegotation is triggered (a subsequent onnegotiationneeded event) whether - we sent the invite/offer or answer. e.g. do we always do createOffer/setLocalDescription and then send it over a matrix negotiation event? even if we before called createAnswer. + - implement terminate + - implement waitForState + + - find out if we need to do something different when renegotation is triggered (a subsequent onnegotiationneeded event) whether + we sent the invite/offer or answer. e.g. do we always do createOffer/setLocalDescription and then send it over a matrix negotiation event? even if we before called createAnswer. - handle receiving offer and send anwser - handle sending ice candidates - handle ice candidates finished (iceGatheringState === 'complete') @@ -36,12 +47,29 @@ we wait for other participants to add their user and device (in the sources) for each (userid, deviceid) - if userId < ourUserId - - we setup a peer connection + - get local media + - we setup a peer connection + - add local tracks - we wait for negotation event to get sdp + - peerConn.createOffer + - peerConn.setLocalDescription - we send an m.call.invite - else - wait for invite from other side +on local ice candidate: + - if we haven't ... sent invite yet? or received answer? buffer candidate + - otherwise send candidate (without buffering?) + +on incoming call: + - ring, offer to answer + +answering incoming call + - get local media + - peerConn.setRemoteDescription + - add local tracks to peerConn + - peerConn.createAnswer() + - peerConn.setLocalDescription in some cases, we will actually send the invite to all devices (e.g. SFU), so we probably still need to handle multiple anwsers? @@ -50,9 +78,6 @@ so we would send an invite to multiple devices and pick the one for which we received the anwser first. between invite and anwser, we could already receive ice candidates that we need to buffer. -should a PeerCall only exist after we've received an answer? -Before that, we could have a PeerCallInvite - updating the metadata: @@ -64,3 +89,38 @@ if just muting: use m.call.sdp_stream_metadata_changed party identification - for 1:1 calls, we identify with a party_id - for group calls, we identify with a device_id + + + + +## TODO + +Build basic version of PeerCall +Build basic version of GroupCall +Make it possible to olm encrypt the messages +Do work needed for state events + - receiving (almost done?) + - sending +Expose call objects +Write view model +write view + +## Calls questions\ + - how do we handle glare between group calls (e.g. different state events with different call ids?) + - Split up DOM part into platform code? What abstractions to choose? + Does it make sense to come up with our own API very similar to DOM api? + - what code do we copy over vs what do we implement ourselves? + - MatrixCall: perhaps we can copy it over and modify it to our needs? Seems to have a lot of edge cases implemented. + - what is partyId about? + - CallFeed: I need better understand where it is used. It's basically a wrapper around a MediaStream with volume detection. Could it make sense to put this in platform for example? + + - which parts of MSC2746 are still relevant for group calls? + - which parts of MSC2747 are still relevant for group calls? it seems mostly orthogonal? + - SOLVED: how does switching channels work? This was only enabled by MSC 2746 + - you do getUserMedia()/getDisplayMedia() to get the stream(s) + - you call removeTrack/addTrack on the peerConnection + - you receive a negotiationneeded event + - you call createOffer + - you send m.call.negotiate + - SOLVED: wrt to MSC2746, is the screen share track and the audio track (and video track) part of the same stream? or do screen share tracks need to go in a different stream? it sounds incompatible with the MSC2746 requirement. + - SOLVED: how does muting work? MediaStreamTrack.enabled diff --git a/src/platform/types/WebRTC.ts b/src/platform/types/WebRTC.ts index aff74f62..8b224dfe 100644 --- a/src/platform/types/WebRTC.ts +++ b/src/platform/types/WebRTC.ts @@ -15,11 +15,7 @@ limitations under the License. */ import {Track, TrackType} from "./MediaDevices"; - -export enum StreamPurpose { - UserMedia = "m.usermedia", - ScreenShare = "m.screenshare" -} +import {SDPStreamMetadataPurpose} from "../../matrix/calls/callEventTypes"; export interface WebRTC { createPeerConnection(handler: PeerConnectionHandler): PeerConnection; @@ -33,7 +29,7 @@ export interface PeerConnectionHandler { onDataChannelChanged(dataChannel: DataChannel | undefined); onNegotiationNeeded(); // request the type of incoming stream - getPurposeForStreamId(streamId: string): StreamPurpose; + getPurposeForStreamId(streamId: string): SDPStreamMetadataPurpose; } // does it make sense to wrap this? export interface DataChannel { @@ -42,8 +38,11 @@ export interface DataChannel { } export interface PeerConnection { - get remoteTracks(): Track[] | undefined; + notifyStreamPurposeChanged(): void; + get remoteTracks(): Track[]; get dataChannel(): DataChannel | undefined; + get iceGatheringState(): RTCIceGatheringState; + get localDescription(): RTCSessionDescription | undefined; createOffer(): Promise; createAnswer(): Promise; setLocalDescription(description?: RTCSessionDescriptionInit): Promise; diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index 7d5c8f4f..7af15168 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -90,9 +90,10 @@ export class TrackWrapper implements Track { constructor( public readonly track: MediaStreamTrack, public readonly stream: MediaStream, - public readonly type: TrackType + private _type: TrackType, ) {} + get type(): TrackType { return this._type; } get label(): string { return this.track.label; } get id(): string { return this.track.id; } get streamId(): string { return this.stream.id; } @@ -102,6 +103,10 @@ export class TrackWrapper implements Track { setMuted(muted: boolean): void { this.track.enabled = !muted; } + + setType(type: TrackType): void { + this._type = type; + } } export class AudioTrackWrapper extends TrackWrapper { diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 917f42af..98c77ff9 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -16,7 +16,8 @@ limitations under the License. import {TrackWrapper, wrapTrack} from "./MediaDevices"; import {Track, TrackType} from "../../types/MediaDevices"; -import {WebRTC, PeerConnectionHandler, DataChannel, PeerConnection, StreamPurpose} from "../../types/WebRTC"; +import {WebRTC, PeerConnectionHandler, DataChannel, PeerConnection} from "../../types/WebRTC"; +import {SDPStreamMetadataPurpose} from "../../../matrix/calls/callEventTypes"; const POLLING_INTERVAL = 200; // ms export const SPEAKING_THRESHOLD = -60; // dB @@ -25,8 +26,8 @@ const SPEAKING_SAMPLE_COUNT = 8; // samples class DOMPeerConnection implements PeerConnection { private readonly peerConnection: RTCPeerConnection; private readonly handler: PeerConnectionHandler; - private dataChannelWrapper?: DOMDataChannel; - private _remoteTracks: TrackWrapper[]; + //private dataChannelWrapper?: DOMDataChannel; + private _remoteTracks: TrackWrapper[] = []; constructor(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize = 0) { this.handler = handler; @@ -39,7 +40,9 @@ class DOMPeerConnection implements PeerConnection { } get remoteTracks(): Track[] { return this._remoteTracks; } - get dataChannel(): DataChannel | undefined { return this.dataChannelWrapper; } + get dataChannel(): DataChannel | undefined { return undefined; } + get iceGatheringState(): RTCIceGatheringState { return this.peerConnection.iceGatheringState; } + get localDescription(): RTCSessionDescription | undefined { return this.peerConnection.localDescription ?? undefined; } createOffer(): Promise { return this.peerConnection.createOffer(); @@ -97,6 +100,14 @@ class DOMPeerConnection implements PeerConnection { } return false; } + + notifyStreamPurposeChanged(): void { + for (const track of this.remoteTracks) { + const wrapper = track as TrackWrapper; + wrapper.setType(this.getRemoteTrackType(wrapper.track, wrapper.streamId)); + } + } + createDataChannel(): DataChannel { return new DataChannel(this.peerConnection.createDataChannel()); } @@ -173,20 +184,19 @@ class DOMPeerConnection implements PeerConnection { // of the new tracks, filter the ones that we didn't already knew about const addedTracks = updatedTracks.filter(ut => !this._remoteTracks.some(t => t.track.id === ut.track.id)); // wrap them - const wrappedAddedTracks = addedTracks.map(t => this.wrapRemoteTrack(t.track, t.stream)); + const wrappedAddedTracks = addedTracks.map(t => wrapTrack(t.track, t.stream, this.getRemoteTrackType(t.track, t.stream.id))); // and concat the tracks for other streams with the added tracks this._remoteTracks = withoutRemovedTracks.concat(...wrappedAddedTracks); this.handler.onRemoteTracksChanged(this.remoteTracks); } - private wrapRemoteTrack(track: MediaStreamTrack, stream: MediaStream): TrackWrapper { - let type: TrackType; + + private getRemoteTrackType(track: MediaStreamTrack, streamId: string): TrackType { if (track.kind === "video") { - const purpose = this.handler.getPurposeForStreamId(stream.id); - type = purpose === StreamPurpose.UserMedia ? TrackType.Camera : TrackType.ScreenShare; + const purpose = this.handler.getPurposeForStreamId(streamId); + return purpose === SDPStreamMetadataPurpose.Usermedia ? TrackType.Camera : TrackType.ScreenShare; } else { - type = TrackType.Microphone; + return TrackType.Microphone; } - return wrapTrack(track, stream, type); } /** diff --git a/src/utils/recursivelyAssign.ts b/src/utils/recursivelyAssign.ts new file mode 100644 index 00000000..adf5f2ef --- /dev/null +++ b/src/utils/recursivelyAssign.ts @@ -0,0 +1,39 @@ +/* +Copyright 2015, 2016 OpenMarket Ltd +Copyright 2019 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + + +/** + * This function is similar to Object.assign() but it assigns recursively and + * allows you to ignore nullish values from the source + * + * @param {Object} target + * @param {Object} source + * @returns the target object + */ +export function recursivelyAssign(target: Object, source: Object, ignoreNullish = false): any { + for (const [sourceKey, sourceValue] of Object.entries(source)) { + if (target[sourceKey] instanceof Object && sourceValue) { + recursivelyAssign(target[sourceKey], sourceValue); + continue; + } + if ((sourceValue !== null && sourceValue !== undefined) || !ignoreNullish) { + target[sourceKey] = sourceValue; + continue; + } + } + return target; +} \ No newline at end of file From 98b77fc7615ac3bed972e0ae3dc34e0eb18fddba Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Tue, 1 Mar 2022 15:36:31 +0100 Subject: [PATCH 007/323] WIP7 --- src/matrix/calls/PeerCall.ts | 174 ++++++++++++++++++++++++++- src/platform/types/MediaDevices.ts | 1 + src/platform/web/dom/MediaDevices.ts | 4 + 3 files changed, 173 insertions(+), 6 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 17e49afd..60d89e45 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -21,7 +21,7 @@ import type {Room} from "../room/Room"; import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; -import {WebRTC, PeerConnection, PeerConnectionHandler, StreamPurpose} from "../../platform/types/WebRTC"; +import {WebRTC, PeerConnection, PeerConnectionHandler} from "../../platform/types/WebRTC"; import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; import type {LocalMedia} from "./LocalMedia"; @@ -48,6 +48,7 @@ class PeerCall { private remoteSDPStreamMetadata?: SDPStreamMetadata; private responsePromiseChain?: Promise; private opponentPartyId?: PartyId; + private hangupParty: CallParty; constructor( private readonly handler: PeerCallHandler, @@ -80,7 +81,7 @@ class PeerCall { handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId) { switch (message.type) { case EventType.Invite: - this.handleInvite(message.content); + this.handleInvite(message.content, partyId); break; case EventType.Answer: this.handleAnswer(message.content); @@ -112,15 +113,52 @@ class PeerCall { await this.waitForState(CallState.InviteSent); } - async answer() { + async answer(localMediaPromise: Promise): Promise { + if (this.state !== CallState.Ringing) { + return; + } + this.setState(CallState.WaitLocalMedia); + try { + this.localMedia = await localMediaPromise; + } catch (err) { + this.setState(CallState.Ended); + return; + } + this.setState(CallState.CreateAnswer); + for (const t of this.localMedia.tracks) { + this.peerConnection.addTrack(t); + } + let myAnswer: RTCSessionDescriptionInit; + try { + myAnswer = await this.peerConnection.createAnswer(); + } catch (err) { + this.logger.debug(`Call ${this.callId} Failed to create answer: `, err); + this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true); + return; + } + + try { + await this.peerConnection.setLocalDescription(myAnswer); + this.setState(CallState.Connecting); + + } catch (err) { + this.logger.debug(`Call ${this.callId} Error setting local description!`, err); + this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); + return; + } + // Allow a short time for initial candidates to be gathered + await new Promise(resolve => { + setTimeout(resolve, 200); + }); + this.sendAnswer(); } async hangup() { } - async updateLocalMedia(localMediaPromise: Promise) { + async setMedia(localMediaPromise: Promise) { const oldMedia = this.localMedia; this.localMedia = await localMediaPromise; @@ -179,6 +217,16 @@ class PeerCall { await this.handler.sendSignallingMessage({type: EventType.Invite, content}); this.setState(CallState.InviteSent); } + this.sendCandidateQueue(); + + if (this.state === CallState.CreateOffer) { + this.inviteTimeout = setTimeout(() => { + this.inviteTimeout = null; + if (this.state === CallState.InviteSent) { + this.hangup(CallErrorCode.InviteTimeout); + } + }, CALL_TIMEOUT_MS); + } }; private async handleInvite(content: InviteContent, partyId: PartyId): Promise { @@ -202,6 +250,7 @@ class PeerCall { } try { + // Q: Why do we set the remote description before accepting the call? To start creating ICE candidates? await this.peerConnection.setRemoteDescription(content.offer); await this.addBufferedIceCandidates(); } catch (e) { @@ -235,6 +284,88 @@ class PeerCall { }, content.lifetime ?? CALL_TIMEOUT_MS /* - event.getLocalAge() */ ); } + private async sendAnswer(): Promise { + const answerMessage: AnswerMessage = { + type: EventType.Answer, + content: { + answer: { + sdp: this.peerConnection.localDescription!.sdp, + type: this.peerConnection.localDescription!.type, + }, + [SDPStreamMetadataKey]: this.localMedia.getSDPMetadata(), + } + }; + + // We have just taken the local description from the peerConn which will + // contain all the local candidates added so far, so we can discard any candidates + // we had queued up because they'll be in the answer. + this.logger.info(`Call ${this.callId} Discarding ${ + this.candidateSendQueue.length} candidates that will be sent in answer`); + this.candidateSendQueue = []; + + try { + await this.handler.sendSignallingMessage(answerMessage); + } catch (error) { + this.terminate(CallParty.Local, CallErrorCode.SendAnswer, false); + throw error; + } + + // error handler re-throws so this won't happen on error, but + // we don't want the same error handling on the candidate queue + this.sendCandidateQueue(); + } + + private queueCandidate(content: RTCIceCandidate): void { + // We partially de-trickle candidates by waiting for `delay` before sending them + // amalgamated, in order to avoid sending too many m.call.candidates events and hitting + // rate limits in Matrix. + // In practice, it'd be better to remove rate limits for m.call.* + + // N.B. this deliberately lets you queue and send blank candidates, which MSC2746 + // currently proposes as the way to indicate that candidate gathering is complete. + // This will hopefully be changed to an explicit rather than implicit notification + // shortly. + this.candidateSendQueue.push(content); + + // Don't send the ICE candidates yet if the call is in the ringing state + if (this.state === CallState.Ringing) return; + + // MSC2746 recommends these values (can be quite long when calling because the + // callee will need a while to answer the call) + const delay = this.direction === CallDirection.Inbound ? 500 : 2000; + + setTimeout(() => { + this.sendCandidateQueue(); + }, delay); + } + + + private async sendCandidateQueue(): Promise { + if (this.candidateSendQueue.length === 0 || this.state === CallState.Ended) { + return; + } + + const candidates = this.candidateSendQueue; + this.candidateSendQueue = []; + const candidatesMessage: CandidatesMessage = { + type: EventType.Candidates, + content: { + candidates: candidates, + } + }; + this.logger.debug(`Call ${this.callId} attempting to send ${candidates.length} candidates`); + try { + await this.handler.sendSignallingMessage(candidatesMessage); + // Try to send candidates again just in case we received more candidates while sending. + this.sendCandidateQueue(); + } catch (error) { + // don't retry this event: we'll send another one later as we might + // have more candidates by then. + // put all the candidates we failed to send back in the queue + this.terminate(CallParty.Local, CallErrorCode.SignallingFailed, false); + } + } + private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { this.remoteSDPStreamMetadata = recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); // will rerequest stream purpose for all tracks and set track.type accordingly @@ -296,6 +427,12 @@ class PeerCall { private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean): Promise { } + + private stopAllMedia(): void { + for (const track of this.localMedia.tracks) { + track.stop(); + } + } } @@ -480,9 +617,34 @@ export type InviteMessage = { content: InviteContent } -export type SignallingMessage = InviteMessage; +type AnwserContent = { + answer: { + sdp: string, + // type is now deprecated as of Matrix VoIP v1, but + // required to still be sent for backwards compat + type: RTCSdpType, + }, + [SDPStreamMetadataKey]: SDPStreamMetadata, +} + +export type AnswerMessage = { + type: EventType.Answer, + content: AnwserContent +} + +type CandidatesContent = { + candidates: RTCIceCandidate[] +} + +export type CandidatesMessage = { + type: EventType.Candidates, + content: CandidatesContent +} + + +export type SignallingMessage = InviteMessage | AnswerMessage | CandidatesMessage; export interface PeerCallHandler { emitUpdate(peerCall: PeerCall, params: any); - sendSignallingMessage(message: InviteMessage); + sendSignallingMessage(message: SignallingMessage); } diff --git a/src/platform/types/MediaDevices.ts b/src/platform/types/MediaDevices.ts index 3dda3059..8bf608ce 100644 --- a/src/platform/types/MediaDevices.ts +++ b/src/platform/types/MediaDevices.ts @@ -35,6 +35,7 @@ export interface Track { get settings(): MediaTrackSettings; get muted(): boolean; setMuted(muted: boolean): void; + stop(): void; } export interface AudioTrack extends Track { diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index 7af15168..e1991a1c 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -107,6 +107,10 @@ export class TrackWrapper implements Track { setType(type: TrackType): void { this._type = type; } + + stop() { + this.track.stop(); + } } export class AudioTrackWrapper extends TrackWrapper { From 25b01480737e6935ecef8d59509c9d40e88394e1 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Tue, 1 Mar 2022 17:01:47 +0100 Subject: [PATCH 008/323] WIP8 --- src/matrix/calls/PeerCall.ts | 30 +++++++++++++++--------------- src/platform/types/types.ts | 8 ++++++++ 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 60d89e45..75b9cc2d 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -21,6 +21,7 @@ import type {Room} from "../room/Room"; import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; +import type {TimeoutCreator, Timeout} from "../../platform/types/types"; import {WebRTC, PeerConnection, PeerConnectionHandler} from "../../platform/types/WebRTC"; import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; import type {LocalMedia} from "./LocalMedia"; @@ -49,10 +50,12 @@ class PeerCall { private responsePromiseChain?: Promise; private opponentPartyId?: PartyId; private hangupParty: CallParty; + private hangupTimeout?: Timeout; constructor( private readonly handler: PeerCallHandler, private localMedia: LocalMedia, + private readonly createTimeout: TimeoutCreator, webRTC: WebRTC ) { const outer = this; @@ -84,7 +87,7 @@ class PeerCall { this.handleInvite(message.content, partyId); break; case EventType.Answer: - this.handleAnswer(message.content); + this.handleAnswer(message.content, partyId); break; case EventType.Candidates: this.handleRemoteIceCandidates(message.content); @@ -148,9 +151,7 @@ class PeerCall { return; } // Allow a short time for initial candidates to be gathered - await new Promise(resolve => { - setTimeout(resolve, 200); - }); + await this.createTimeout(200).elapsed(); this.sendAnswer(); } @@ -180,7 +181,7 @@ class PeerCall { applyTrack(m => m.screenShareTrack); } - // calls are serialized and deduplicated by negotiationQueue + // calls are serialized and deduplicated by responsePromiseChain private handleNegotiation = async (): Promise => { try { await this.peerConnection.setLocalDescription(); @@ -192,7 +193,7 @@ class PeerCall { if (this.peerConnection.iceGatheringState === 'gathering') { // Allow a short time for initial candidates to be gathered - await new Promise(resolve => setTimeout(resolve, 200)); + await this.createTimeout(200).elapsed(); } if (this.state === CallState.Ended) { @@ -220,12 +221,12 @@ class PeerCall { this.sendCandidateQueue(); if (this.state === CallState.CreateOffer) { - this.inviteTimeout = setTimeout(() => { - this.inviteTimeout = null; - if (this.state === CallState.InviteSent) { - this.hangup(CallErrorCode.InviteTimeout); - } - }, CALL_TIMEOUT_MS); + this.hangupTimeout = this.createTimeout(CALL_TIMEOUT_MS); + await this.hangupTimeout.elapsed(); + // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between + if (this.state === CallState.InviteSent) { + this.hangup(CallErrorCode.InviteTimeout); + } } }; @@ -333,10 +334,9 @@ class PeerCall { // MSC2746 recommends these values (can be quite long when calling because the // callee will need a while to answer the call) const delay = this.direction === CallDirection.Inbound ? 500 : 2000; - - setTimeout(() => { + this.createTimeout(delay).elapsed().then(() => { this.sendCandidateQueue(); - }, delay); + }); } diff --git a/src/platform/types/types.ts b/src/platform/types/types.ts index da8ec8e7..c59278b9 100644 --- a/src/platform/types/types.ts +++ b/src/platform/types/types.ts @@ -43,3 +43,11 @@ export type File = { readonly name: string; readonly blob: IBlobHandle; } + +export interface Timeout { + elapsed(): Promise; + abort(): void; + dispose(): void; +}; + +export type TimeoutCreator = (timeout: number) => Timeout; From ecf7eab3ee3ebfb5ae31b0fca85dc2990c2bb349 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 2 Mar 2022 13:53:22 +0100 Subject: [PATCH 009/323] WIP8 - implement PeerCall.handleAnswer and other things --- src/matrix/calls/PeerCall.ts | 118 +++++++++++++++++++++++++-------- src/matrix/calls/TODO.md | 2 +- src/platform/types/WebRTC.ts | 2 + src/platform/web/dom/WebRTC.ts | 5 ++ 4 files changed, 99 insertions(+), 28 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 75b9cc2d..a08263f6 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -17,6 +17,7 @@ limitations under the License. import {ObservableMap} from "../../observable/map/ObservableMap"; import {recursivelyAssign} from "../../utils/recursivelyAssign"; import {AsyncQueue} from "../../utils/AsyncQueue"; +import {Disposables, Disposable} from "../../utils/Disposables"; import type {Room} from "../room/Room"; import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; @@ -50,7 +51,8 @@ class PeerCall { private responsePromiseChain?: Promise; private opponentPartyId?: PartyId; private hangupParty: CallParty; - private hangupTimeout?: Timeout; + private disposables = new Disposables(); + private statePromiseMap = new Map void, promise: Promise}>(); constructor( private readonly handler: PeerCallHandler, @@ -144,14 +146,13 @@ class PeerCall { try { await this.peerConnection.setLocalDescription(myAnswer); this.setState(CallState.Connecting); - } catch (err) { this.logger.debug(`Call ${this.callId} Error setting local description!`, err); this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); return; } // Allow a short time for initial candidates to be gathered - await this.createTimeout(200).elapsed(); + await this.delay(200); this.sendAnswer(); } @@ -193,7 +194,7 @@ class PeerCall { if (this.peerConnection.iceGatheringState === 'gathering') { // Allow a short time for initial candidates to be gathered - await this.createTimeout(200).elapsed(); + await this.delay(200); } if (this.state === CallState.Ended) { @@ -221,8 +222,7 @@ class PeerCall { this.sendCandidateQueue(); if (this.state === CallState.CreateOffer) { - this.hangupTimeout = this.createTimeout(CALL_TIMEOUT_MS); - await this.hangupTimeout.elapsed(); + await this.delay(CALL_TIMEOUT_MS); // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between if (this.state === CallState.InviteSent) { this.hangup(CallErrorCode.InviteTimeout); @@ -271,18 +271,55 @@ class PeerCall { this.setState(CallState.Ringing); - setTimeout(() => { - if (this.state == CallState.Ringing) { - this.logger.debug(`Call ${this.callId} invite has expired. Hanging up.`); - this.hangupParty = CallParty.Remote; // effectively - this.setState(CallState.Ended); - this.stopAllMedia(); - if (this.peerConnection.signalingState != 'closed') { - this.peerConnection.close(); - } - this.emit(CallEvent.Hangup); + await this.delay(content.lifetime ?? CALL_TIMEOUT_MS); + // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between + if (this.state === CallState.Ringing) { + this.logger.debug(`Call ${this.callId} invite has expired. Hanging up.`); + this.hangupParty = CallParty.Remote; // effectively + this.setState(CallState.Ended); + this.stopAllMedia(); + if (this.peerConnection.signalingState != 'closed') { + this.peerConnection.close(); } - }, content.lifetime ?? CALL_TIMEOUT_MS /* - event.getLocalAge() */ ); + } + } + + private async handleAnswer(content: AnwserContent, partyId: PartyId): Promise { + this.logger.debug(`Got answer for call ID ${this.callId} from party ID ${content.party_id}`); + + if (this.state === CallState.Ended) { + this.logger.debug(`Ignoring answer because call ID ${this.callId} has ended`); + return; + } + + if (this.opponentPartyId !== undefined) { + this.logger.info( + `Call ${this.callId} ` + + `Ignoring answer from party ID ${content.party_id}: ` + + `we already have an answer/reject from ${this.opponentPartyId}`, + ); + return; + } + + this.opponentPartyId = partyId; + await this.addBufferedIceCandidates(); + + this.setState(CallState.Connecting); + + const sdpStreamMetadata = content[SDPStreamMetadataKey]; + if (sdpStreamMetadata) { + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); + } else { + this.logger.warn(`Call ${this.callId} Did not get any SDPStreamMetadata! Can not send/receive multiple streams`); + } + + try { + await this.peerConnection.setRemoteDescription(content.answer); + } catch (e) { + this.logger.debug(`Call ${this.callId} Failed to set remote description`, e); + this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); + return; + } } private async sendAnswer(): Promise { @@ -333,8 +370,7 @@ class PeerCall { // MSC2746 recommends these values (can be quite long when calling because the // callee will need a while to answer the call) - const delay = this.direction === CallDirection.Inbound ? 500 : 2000; - this.createTimeout(delay).elapsed().then(() => { + this.delay(this.direction === CallDirection.Inbound ? 500 : 2000).then(() => { this.sendCandidateQueue(); }); } @@ -384,13 +420,15 @@ class PeerCall { private async addBufferedIceCandidates(): Promise { - const bufferedCandidates = this.remoteCandidateBuffer!.get(this.opponentPartyId!); - if (bufferedCandidates) { - this.logger.info(`Call ${this.callId} Adding ${ - bufferedCandidates.length} buffered candidates for opponent ${this.opponentPartyId}`); - await this.addIceCandidates(bufferedCandidates); + if (this.remoteCandidateBuffer && this.opponentPartyId) { + const bufferedCandidates = this.remoteCandidateBuffer.get(this.opponentPartyId); + if (bufferedCandidates) { + this.logger.info(`Call ${this.callId} Adding ${ + bufferedCandidates.length} buffered candidates for opponent ${this.opponentPartyId}`); + await this.addIceCandidates(bufferedCandidates); + } + this.remoteCandidateBuffer = undefined; } - this.remoteCandidateBuffer = undefined; } private async addIceCandidates(candidates: RTCIceCandidate[]): Promise { @@ -417,11 +455,25 @@ class PeerCall { private setState(state: CallState): void { const oldState = this.state; this.state = state; - this.handler.emitUpdate(); + let deferred = this.statePromiseMap.get(state); + if (deferred) { + deferred.resolve(); + this.statePromiseMap.delete(state); + } + this.handler.emitUpdate(this, undefined); } private waitForState(state: CallState): Promise { - + let deferred = this.statePromiseMap.get(state); + if (!deferred) { + let resolve; + const promise = new Promise(r => { + resolve = r; + }); + deferred = {resolve, promise}; + this.statePromiseMap.set(state, deferred); + } + return deferred.promise; } private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean): Promise { @@ -433,6 +485,18 @@ class PeerCall { track.stop(); } } + + private async delay(timeoutMs: number): Promise { + // Allow a short time for initial candidates to be gathered + const timeout = this.disposables.track(this.createTimeout(timeoutMs)); + await timeout.elapsed(); + this.disposables.untrack(timeout); + } + + public dispose(): void { + this.disposables.dispose(); + // TODO: dispose peerConnection? + } } diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index 397c9d38..69268b1d 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -105,7 +105,7 @@ Expose call objects Write view model write view -## Calls questions\ +## Calls questions - how do we handle glare between group calls (e.g. different state events with different call ids?) - Split up DOM part into platform code? What abstractions to choose? Does it make sense to come up with our own API very similar to DOM api? diff --git a/src/platform/types/WebRTC.ts b/src/platform/types/WebRTC.ts index 8b224dfe..df8133ee 100644 --- a/src/platform/types/WebRTC.ts +++ b/src/platform/types/WebRTC.ts @@ -42,6 +42,7 @@ export interface PeerConnection { get remoteTracks(): Track[]; get dataChannel(): DataChannel | undefined; get iceGatheringState(): RTCIceGatheringState; + get signalingState(): RTCSignalingState; get localDescription(): RTCSessionDescription | undefined; createOffer(): Promise; createAnswer(): Promise; @@ -53,4 +54,5 @@ export interface PeerConnection { replaceTrack(oldTrack: Track, newTrack: Track): Promise; createDataChannel(): DataChannel; dispose(): void; + close(): void; } diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 98c77ff9..08c0d96d 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -43,6 +43,7 @@ class DOMPeerConnection implements PeerConnection { get dataChannel(): DataChannel | undefined { return undefined; } get iceGatheringState(): RTCIceGatheringState { return this.peerConnection.iceGatheringState; } get localDescription(): RTCSessionDescription | undefined { return this.peerConnection.localDescription ?? undefined; } + get signalingState(): RTCSignalingState { return this.peerConnection.signalingState; } createOffer(): Promise { return this.peerConnection.createOffer(); @@ -64,6 +65,10 @@ class DOMPeerConnection implements PeerConnection { return this.peerConnection.addIceCandidate(candidate); } + close(): void { + return this.peerConnection.close(); + } + addTrack(track: Track): void { if (!(track instanceof TrackWrapper)) { throw new Error("Not a TrackWrapper"); From 6fe90e60db68007359b88ed5673ba085b455feb4 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Mon, 7 Mar 2022 10:15:54 +0100 Subject: [PATCH 010/323] WIP9 --- src/matrix/DeviceMessageHandler.js | 6 ++- src/matrix/calls/CallHandler.ts | 5 +-- src/matrix/calls/PeerCall.ts | 22 ++++++++--- src/matrix/calls/TODO.md | 61 ++++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 9 deletions(-) diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index c6bce31e..20f4abd3 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -53,10 +53,14 @@ export class DeviceMessageHandler { } const newRoomKeys = this._megolmDecryption.roomKeysFromDeviceMessages(olmDecryptChanges.results, log); const callMessages = olmDecryptChanges.results.filter(dr => this._callHandler.handlesDeviceMessageEventType(dr.event?.type)); + // load devices by sender key await Promise.all(callMessages.map(async dr => { dr.setDevice(await this._getDevice(dr.senderCurve25519Key, txn)); - this._callHandler.handleDeviceMessage(dr.device.userId, dr.device.deviceId, dr.event.type, dr.event.content, log); })); + // TODO: pass this in the prep and run it in afterSync or afterSyncComplete (as callHandler can send events as well)? + for (const dr of callMessages) { + this._callHandler.handleDeviceMessage(dr.device.userId, dr.device.deviceId, dr.event.type, dr.event.content, log); + } // TODO: somehow include rooms that received a call to_device message in the sync state? // or have updates flow through event emitter? // well, we don't really need to update the room other then when a call starts or stops diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 4a2027f3..1dad9ce8 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -109,8 +109,7 @@ class GroupParticipant implements PeerCallHandler { sendInvite() { this.peerCall = new PeerCall(this, this.webRTC); - this.peerCall.setLocalMedia(this.localMedia); - this.peerCall.sendOffer(); + this.peerCall.call(this.localMedia); } /** From PeerCallHandler @@ -140,7 +139,7 @@ class GroupCall { async participate(tracks: Track[]) { this.localMedia = LocalMedia.fromTracks(tracks); for (const [,participant] of this.participants) { - participant.setLocalMedia(this.localMedia.clone()); + participant.setMedia(this.localMedia.clone()); } // send m.call.member state event diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index a08263f6..a02de99f 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -86,13 +86,23 @@ class PeerCall { handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId) { switch (message.type) { case EventType.Invite: - this.handleInvite(message.content, partyId); + // determining whether or not an incoming invite glares + // with an instance of PeerCall is different for group calls + // and 1:1 calls, so done outside of this class. + // If you pass an event for another call id in here it will assume it glares. + + //const newCallId = message.content.call_id; + //if (this.id && newCallId !== this.id) { + // this.handleInviteGlare(message.content); + //} else { + this.handleInvite(message.content, partyId); + //} break; case EventType.Answer: this.handleAnswer(message.content, partyId); break; case EventType.Candidates: - this.handleRemoteIceCandidates(message.content); + this.handleRemoteIceCandidates(message.content, partyId); break; case EventType.Hangup: } @@ -184,6 +194,8 @@ class PeerCall { // calls are serialized and deduplicated by responsePromiseChain private handleNegotiation = async (): Promise => { + // TODO: does this make sense to have this state if we're already connected? + this.setState(CallState.MakingOffer) try { await this.peerConnection.setLocalDescription(); } catch (err) { @@ -221,7 +233,7 @@ class PeerCall { } this.sendCandidateQueue(); - if (this.state === CallState.CreateOffer) { + if (this.state === CallState.InviteSent) { await this.delay(CALL_TIMEOUT_MS); // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between if (this.state === CallState.InviteSent) { @@ -495,7 +507,7 @@ class PeerCall { public dispose(): void { this.disposables.dispose(); - // TODO: dispose peerConnection? + this.peerConnection.dispose(); } } @@ -529,9 +541,9 @@ export enum CallParty { export enum CallState { Fledgling = 'fledgling', - InviteSent = 'invite_sent', WaitLocalMedia = 'wait_local_media', CreateOffer = 'create_offer', + InviteSent = 'invite_sent', CreateAnswer = 'create_answer', Connecting = 'connecting', Connected = 'connected', diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index 69268b1d..da2b1ad6 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -124,3 +124,64 @@ write view - you send m.call.negotiate - SOLVED: wrt to MSC2746, is the screen share track and the audio track (and video track) part of the same stream? or do screen share tracks need to go in a different stream? it sounds incompatible with the MSC2746 requirement. - SOLVED: how does muting work? MediaStreamTrack.enabled + - SOLVED: so, what's the difference between the call_id and the conf_id in group call events? + - call_id is the specific 1:1 call, conf_id is the thing in the m.call state event key + - so a group call has a conf_id with MxN peer calls, each having their call_id. + +I think we need to synchronize the negotiation needed because we don't use a CallState to guard it... + + + +## Thursday 3-3 notes + +we probably best keep the perfect negotiation flags, as they are needed for both starting the call AND renegotiation? if only for the former, it would make sense as it is a step in setting up the call, but if the call is ongoing, does it make sense to have a MakingOffer state? it actually looks like they are only needed for renegotiation! for call setup we compare the call_ids. What does that mean for these flags? + + +List state transitions + +FROM CALLER FROM CALLEE + +Fledgling Fledgling + V calling `call()` V handleInvite +WaitLocalMedia Ringing + V media promise resolves V answer() +CreateOffer WaitLocalMedia + V add tracks V media promise resolves + V wait for negotionneeded events CreateAnswer + V setLocalDescription() V + V send invite events +InviteSent + V receive anwser, setRemoteDescription() | + \__________________________________________________/ + V + Connecting + V receive ice candidates and + iceConnectionState becomes 'connected' + Connected + V hangup for some reason + Ended + +## From callee + +Fledgling +Ringing +WaitLocalMedia +CreateAnswer +Connecting +Connected +Ended + +Fledgling +WaitLocalMedia +CreateOffer +InviteSent +CreateAnswer +Connecting +Connected +Ringing +Ended + +so if we don't want to bother with having two call objects, we can make the existing call hangup his old call_id? That way we keep the old peerConnection. + + +when glare, won't we drop both calls? No: https://github.com/matrix-org/matrix-spec-proposals/pull/2746#discussion_r819388754 From 60da85d64102146cf912fc30356e3d0203f31d51 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 9 Mar 2022 11:29:39 +0100 Subject: [PATCH 011/323] WIP10 --- package.json | 2 +- src/matrix/DeviceMessageHandler.js | 2 +- src/matrix/calls/Call.ts | 2493 ----------------- src/matrix/calls/CallFeed.ts | 274 -- src/matrix/calls/CallHandler.ts | 113 +- src/matrix/calls/PeerCall.ts | 415 +-- src/matrix/calls/TODO.md | 49 +- src/matrix/calls/callEventTypes.ts | 41 +- .../calls/group/{Call.ts => GroupCall.ts} | 21 +- src/matrix/calls/group/Participant.ts | 49 +- src/platform/web/dom/MediaDevices.ts | 6 +- src/platform/web/dom/WebRTC.ts | 2 +- yarn.lock | 6 +- 13 files changed, 347 insertions(+), 3126 deletions(-) delete mode 100644 src/matrix/calls/Call.ts delete mode 100644 src/matrix/calls/CallFeed.ts rename src/matrix/calls/group/{Call.ts => GroupCall.ts} (83%) diff --git a/package.json b/package.json index 12c73994..e356347b 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,7 @@ "postcss-flexbugs-fixes": "^5.0.2", "regenerator-runtime": "^0.13.7", "text-encoding": "^0.7.0", - "typescript": "^4.3.5", + "typescript": "^4.4", "vite": "^2.6.14", "xxhashjs": "^0.2.2" }, diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index 20f4abd3..470559a9 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -59,7 +59,7 @@ export class DeviceMessageHandler { })); // TODO: pass this in the prep and run it in afterSync or afterSyncComplete (as callHandler can send events as well)? for (const dr of callMessages) { - this._callHandler.handleDeviceMessage(dr.device.userId, dr.device.deviceId, dr.event.type, dr.event.content, log); + this._callHandler.handleDeviceMessage(dr.device.userId, dr.device.deviceId, dr.event, log); } // TODO: somehow include rooms that received a call to_device message in the sync state? // or have updates flow through event emitter? diff --git a/src/matrix/calls/Call.ts b/src/matrix/calls/Call.ts deleted file mode 100644 index d6ac4612..00000000 --- a/src/matrix/calls/Call.ts +++ /dev/null @@ -1,2493 +0,0 @@ -/* -Copyright 2015, 2016 OpenMarket Ltd -Copyright 2017 New Vector Ltd -Copyright 2019, 2020 The Matrix.org Foundation C.I.C. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { - MCallReplacesEvent, - MCallAnswer, - MCallInviteNegotiate, - CallCapabilities, - SDPStreamMetadataPurpose, - SDPStreamMetadata, - SDPStreamMetadataKey, - MCallSDPStreamMetadataChanged, - MCallSelectAnswer, - MCAllAssertedIdentity, - MCallCandidates, - MCallBase, - MCallHangupReject, -} from './callEventTypes'; -import { CallFeed } from './CallFeed'; - -// events: hangup, error(err), replaced(call), state(state, oldState) - -/** - * Fires whenever an error occurs when call.js encounters an issue with setting up the call. - *

- * The error given will have a code equal to either `MatrixCall.ERR_LOCAL_OFFER_FAILED` or - * `MatrixCall.ERR_NO_USER_MEDIA`. `ERR_LOCAL_OFFER_FAILED` is emitted when the local client - * fails to create an offer. `ERR_NO_USER_MEDIA` is emitted when the user has denied access - * to their audio/video hardware. - * - * @event module:webrtc/call~MatrixCall#"error" - * @param {Error} err The error raised by MatrixCall. - * @example - * matrixCall.on("error", function(err){ - * console.error(err.code, err); - * }); - */ - -interface CallOpts { - roomId?: string; - invitee?: string; - client?: any; // Fix when client is TSified - forceTURN?: boolean; - turnServers?: Array; - opponentDeviceId?: string; - opponentSessionId?: string; - groupCallId?: string; -} - -interface TurnServer { - urls: Array; - username?: string; - password?: string; - ttl?: number; -} - -interface AssertedIdentity { - id: string; - displayName: string; -} - -export enum CallState { - Fledgling = 'fledgling', - InviteSent = 'invite_sent', - WaitLocalMedia = 'wait_local_media', - CreateOffer = 'create_offer', - CreateAnswer = 'create_answer', - Connecting = 'connecting', - Connected = 'connected', - Ringing = 'ringing', - Ended = 'ended', -} - -export enum CallType { - Voice = 'voice', - Video = 'video', -} - -export enum CallDirection { - Inbound = 'inbound', - Outbound = 'outbound', -} - -export enum CallParty { - Local = 'local', - Remote = 'remote', -} - -export enum CallEvent { - Hangup = 'hangup', - State = 'state', - Error = 'error', - Replaced = 'replaced', - - // The value of isLocalOnHold() has changed - LocalHoldUnhold = 'local_hold_unhold', - // The value of isRemoteOnHold() has changed - RemoteHoldUnhold = 'remote_hold_unhold', - // backwards compat alias for LocalHoldUnhold: remove in a major version bump - HoldUnhold = 'hold_unhold', - // Feeds have changed - FeedsChanged = 'feeds_changed', - - AssertedIdentityChanged = 'asserted_identity_changed', - - LengthChanged = 'length_changed', - - DataChannel = 'datachannel', - - SendVoipEvent = "send_voip_event", -} - -export enum CallErrorCode { - /** The user chose to end the call */ - UserHangup = 'user_hangup', - - /** An error code when the local client failed to create an offer. */ - LocalOfferFailed = 'local_offer_failed', - /** - * An error code when there is no local mic/camera to use. This may be because - * the hardware isn't plugged in, or the user has explicitly denied access. - */ - NoUserMedia = 'no_user_media', - - /** - * Error code used when a call event failed to send - * because unknown devices were present in the room - */ - UnknownDevices = 'unknown_devices', - - /** - * Error code used when we fail to send the invite - * for some reason other than there being unknown devices - */ - SendInvite = 'send_invite', - - /** - * An answer could not be created - */ - CreateAnswer = 'create_answer', - - /** - * Error code used when we fail to send the answer - * for some reason other than there being unknown devices - */ - SendAnswer = 'send_answer', - - /** - * The session description from the other side could not be set - */ - SetRemoteDescription = 'set_remote_description', - - /** - * The session description from this side could not be set - */ - SetLocalDescription = 'set_local_description', - - /** - * A different device answered the call - */ - AnsweredElsewhere = 'answered_elsewhere', - - /** - * No media connection could be established to the other party - */ - IceFailed = 'ice_failed', - - /** - * The invite timed out whilst waiting for an answer - */ - InviteTimeout = 'invite_timeout', - - /** - * The call was replaced by another call - */ - Replaced = 'replaced', - - /** - * Signalling for the call could not be sent (other than the initial invite) - */ - SignallingFailed = 'signalling_timeout', - - /** - * The remote party is busy - */ - UserBusy = 'user_busy', - - /** - * We transferred the call off to somewhere else - */ - Transfered = 'transferred', - - /** - * A call from the same user was found with a new session id - */ - NewSession = 'new_session', -} - -/** - * The version field that we set in m.call.* events - */ -const VOIP_PROTO_VERSION = 1; - -/** The fallback ICE server to use for STUN or TURN protocols. */ -const FALLBACK_ICE_SERVER = 'stun:turn.matrix.org'; - -/** The length of time a call can be ringing for. */ -const CALL_TIMEOUT_MS = 60000; - -const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); - -export class CallError extends Error { - code: string; - - constructor(code: CallErrorCode, msg: string, err: Error) { - // Still don't think there's any way to have proper nested errors - super(msg + ": " + err); - - this.code = code; - } -} - -export function genCallID(): string { - return Date.now().toString() + randomString(16); -} - -/** - * Construct a new Matrix Call. - * @constructor - * @param {Object} opts Config options. - * @param {string} opts.roomId The room ID for this call. - * @param {Object} opts.webRtc The WebRTC globals from the browser. - * @param {boolean} opts.forceTURN whether relay through TURN should be forced. - * @param {Object} opts.URL The URL global. - * @param {Array} opts.turnServers Optional. A list of TURN servers. - * @param {MatrixClient} opts.client The Matrix Client instance to send events to. - */ -export class MatrixCall extends EventEmitter { - public roomId: string; - public callId: string; - public invitee?: string; - public state = CallState.Fledgling; - public hangupParty: CallParty; - public hangupReason: string; - public direction: CallDirection; - public ourPartyId: string; - public peerConn?: RTCPeerConnection; - - private client: MatrixClient; - private forceTURN: boolean; - private turnServers: Array; - // A queue for candidates waiting to go out. - // We try to amalgamate candidates into a single candidate message where - // possible - private candidateSendQueue: Array = []; - private candidateSendTries = 0; - private sentEndOfCandidates = false; - private feeds: Array = []; - private usermediaSenders: Array = []; - private screensharingSenders: Array = []; - private inviteOrAnswerSent = false; - private waitForLocalAVStream: boolean; - private successor: MatrixCall; - private opponentMember: RoomMember; - private opponentVersion: number | string; - // The party ID of the other side: undefined if we haven't chosen a partner - // yet, null if we have but they didn't send a party ID. - private opponentPartyId: string; - private opponentCaps: CallCapabilities; - private inviteTimeout: number; - private iceDisconnectedTimeout: number; - - // The logic of when & if a call is on hold is nontrivial and explained in is*OnHold - // This flag represents whether we want the other party to be on hold - private remoteOnHold = false; - - // the stats for the call at the point it ended. We can't get these after we - // tear the call down, so we just grab a snapshot before we stop the call. - // The typescript definitions have this type as 'any' :( - private callStatsAtEnd: any[]; - - // Perfect negotiation state: https://www.w3.org/TR/webrtc/#perfect-negotiation-example - private makingOffer = false; - private ignoreOffer: boolean; - - private responsePromiseChain?: Promise; - - // If candidates arrive before we've picked an opponent (which, in particular, - // will happen if the opponent sends candidates eagerly before the user answers - // the call) we buffer them up here so we can then add the ones from the party we pick - private remoteCandidateBuffer = new Map(); - - private remoteAssertedIdentity: AssertedIdentity; - - private remoteSDPStreamMetadata: SDPStreamMetadata; - - private callLengthInterval: number; - private callLength = 0; - - private opponentDeviceId: string; - private opponentSessionId: string; - public groupCallId: string; - - constructor(opts: CallOpts) { - super(); - this.roomId = opts.roomId; - this.invitee = opts.invitee; - this.client = opts.client; - this.forceTURN = opts.forceTURN; - this.ourPartyId = this.client.deviceId; - this.opponentDeviceId = opts.opponentDeviceId; - this.opponentSessionId = opts.opponentSessionId; - this.groupCallId = opts.groupCallId; - // Array of Objects with urls, username, credential keys - this.turnServers = opts.turnServers || []; - if (this.turnServers.length === 0 && this.client.isFallbackICEServerAllowed()) { - this.turnServers.push({ - urls: [FALLBACK_ICE_SERVER], - }); - } - for (const server of this.turnServers) { - utils.checkObjectHasKeys(server, ["urls"]); - } - this.callId = genCallID(); - } - - /** - * Place a voice call to this room. - * @throws If you have not specified a listener for 'error' events. - */ - public async placeVoiceCall(): Promise { - await this.placeCall(true, false); - } - - /** - * Place a video call to this room. - * @throws If you have not specified a listener for 'error' events. - */ - public async placeVideoCall(): Promise { - await this.placeCall(true, true); - } - - /** - * Create a datachannel using this call's peer connection. - * @param label A human readable label for this datachannel - * @param options An object providing configuration options for the data channel. - */ - public createDataChannel(label: string, options: RTCDataChannelInit) { - const dataChannel = this.peerConn.createDataChannel(label, options); - this.emit(CallEvent.DataChannel, dataChannel); - return dataChannel; - } - - public getOpponentMember(): RoomMember { - return this.opponentMember; - } - - public getOpponentSessionId(): string { - return this.opponentSessionId; - } - - public opponentCanBeTransferred(): boolean { - return Boolean(this.opponentCaps && this.opponentCaps["m.call.transferee"]); - } - - public opponentSupportsDTMF(): boolean { - return Boolean(this.opponentCaps && this.opponentCaps["m.call.dtmf"]); - } - - public getRemoteAssertedIdentity(): AssertedIdentity { - return this.remoteAssertedIdentity; - } - - public get type(): CallType { - return (this.hasLocalUserMediaVideoTrack || this.hasRemoteUserMediaVideoTrack) - ? CallType.Video - : CallType.Voice; - } - - public get hasLocalUserMediaVideoTrack(): boolean { - return this.localUsermediaStream?.getVideoTracks().length > 0; - } - - public get hasRemoteUserMediaVideoTrack(): boolean { - return this.getRemoteFeeds().some((feed) => { - return ( - feed.purpose === SDPStreamMetadataPurpose.Usermedia && - feed.stream.getVideoTracks().length > 0 - ); - }); - } - - public get hasLocalUserMediaAudioTrack(): boolean { - return this.localUsermediaStream?.getAudioTracks().length > 0; - } - - public get hasRemoteUserMediaAudioTrack(): boolean { - return this.getRemoteFeeds().some((feed) => { - return ( - feed.purpose === SDPStreamMetadataPurpose.Usermedia && - feed.stream.getAudioTracks().length > 0 - ); - }); - } - - public get localUsermediaFeed(): CallFeed { - return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); - } - - public get localScreensharingFeed(): CallFeed { - return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); - } - - public get localUsermediaStream(): MediaStream { - return this.localUsermediaFeed?.stream; - } - - public get localScreensharingStream(): MediaStream { - return this.localScreensharingFeed?.stream; - } - - public get remoteUsermediaFeed(): CallFeed { - return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); - } - - public get remoteScreensharingFeed(): CallFeed { - return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); - } - - public get remoteUsermediaStream(): MediaStream { - return this.remoteUsermediaFeed?.stream; - } - - public get remoteScreensharingStream(): MediaStream { - return this.remoteScreensharingFeed?.stream; - } - - private getFeedByStreamId(streamId: string): CallFeed { - return this.getFeeds().find((feed) => feed.stream.id === streamId); - } - - /** - * Returns an array of all CallFeeds - * @returns {Array} CallFeeds - */ - public getFeeds(): Array { - return this.feeds; - } - - /** - * Returns an array of all local CallFeeds - * @returns {Array} local CallFeeds - */ - public getLocalFeeds(): Array { - return this.feeds.filter((feed) => feed.isLocal()); - } - - /** - * Returns an array of all remote CallFeeds - * @returns {Array} remote CallFeeds - */ - public getRemoteFeeds(): Array { - return this.feeds.filter((feed) => !feed.isLocal()); - } - - /** - * Generates and returns localSDPStreamMetadata - * @returns {SDPStreamMetadata} localSDPStreamMetadata - */ - private getLocalSDPStreamMetadata(updateStreamIds = false): SDPStreamMetadata { - const metadata: SDPStreamMetadata = {}; - for (const localFeed of this.getLocalFeeds()) { - if (updateStreamIds) { - localFeed.sdpMetadataStreamId = localFeed.stream.id; - } - - metadata[localFeed.sdpMetadataStreamId] = { - purpose: localFeed.purpose, - audio_muted: localFeed.isAudioMuted(), - video_muted: localFeed.isVideoMuted(), - }; - } - return metadata; - } - - /** - * Returns true if there are no incoming feeds, - * otherwise returns false - * @returns {boolean} no incoming feeds - */ - public noIncomingFeeds(): boolean { - return !this.feeds.some((feed) => !feed.isLocal()); - } - - private pushRemoteFeed(stream: MediaStream): void { - // Fallback to old behavior if the other side doesn't support SDPStreamMetadata - if (!this.opponentSupportsSDPStreamMetadata()) { - this.pushRemoteFeedWithoutMetadata(stream); - return; - } - - const userId = this.getOpponentMember().userId; - const purpose = this.remoteSDPStreamMetadata[stream.id].purpose; - const audioMuted = this.remoteSDPStreamMetadata[stream.id].audio_muted; - const videoMuted = this.remoteSDPStreamMetadata[stream.id].video_muted; - - if (!purpose) { - logger.warn(`Ignoring stream with id ${stream.id} because we didn't get any metadata about it`); - return; - } - - // Try to find a feed with the same purpose as the new stream, - // if we find it replace the old stream with the new one - const existingFeed = this.getRemoteFeeds().find((feed) => feed.purpose === purpose); - if (existingFeed) { - existingFeed.setNewStream(stream); - } else { - this.feeds.push(new CallFeed({ - client: this.client, - roomId: this.roomId, - userId, - stream, - purpose, - audioMuted, - videoMuted, - })); - this.emit(CallEvent.FeedsChanged, this.feeds); - } - - logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}", purpose=${purpose})`); - } - - /** - * This method is used ONLY if the other client doesn't support sending SDPStreamMetadata - */ - private pushRemoteFeedWithoutMetadata(stream: MediaStream): void { - const userId = this.getOpponentMember().userId; - // We can guess the purpose here since the other client can only send one stream - const purpose = SDPStreamMetadataPurpose.Usermedia; - const oldRemoteStream = this.feeds.find((feed) => !feed.isLocal())?.stream; - - // Note that we check by ID and always set the remote stream: Chrome appears - // to make new stream objects when transceiver directionality is changed and the 'active' - // status of streams change - Dave - // If we already have a stream, check this stream has the same id - if (oldRemoteStream && stream.id !== oldRemoteStream.id) { - logger.warn(`Ignoring new stream ID ${stream.id}: we already have stream ID ${oldRemoteStream.id}`); - return; - } - - // Try to find a feed with the same stream id as the new stream, - // if we find it replace the old stream with the new one - const feed = this.getFeedByStreamId(stream.id); - if (feed) { - feed.setNewStream(stream); - } else { - this.feeds.push(new CallFeed({ - client: this.client, - roomId: this.roomId, - audioMuted: false, - videoMuted: false, - userId, - stream, - purpose, - })); - this.emit(CallEvent.FeedsChanged, this.feeds); - } - - logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}")`); - } - - private pushNewLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose, addToPeerConnection = true): void { - const userId = this.client.getUserId(); - - // TODO: Find out what is going on here - // why do we enable audio (and only audio) tracks here? -- matthew - setTracksEnabled(stream.getAudioTracks(), true); - - // We try to replace an existing feed if there already is one with the same purpose - const existingFeed = this.getLocalFeeds().find((feed) => feed.purpose === purpose); - if (existingFeed) { - existingFeed.setNewStream(stream); - } else { - this.pushLocalFeed( - new CallFeed({ - client: this.client, - roomId: this.roomId, - audioMuted: stream.getAudioTracks().length === 0, - videoMuted: stream.getVideoTracks().length === 0, - userId, - stream, - purpose, - }), - addToPeerConnection, - ); - this.emit(CallEvent.FeedsChanged, this.feeds); - } - } - - /** - * Pushes supplied feed to the call - * @param {CallFeed} callFeed to push - * @param {boolean} addToPeerConnection whether to add the tracks to the peer connection - */ - public pushLocalFeed(callFeed: CallFeed, addToPeerConnection = true): void { - if (this.feeds.some((feed) => callFeed.stream.id === feed.stream.id)) { - logger.info(`Ignoring duplicate local stream ${callFeed.stream.id} in call ${this.callId}`); - return; - } - - this.feeds.push(callFeed); - - if (addToPeerConnection) { - const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia ? - this.usermediaSenders : this.screensharingSenders; - // Empty the array - senderArray.splice(0, senderArray.length); - - for (const track of callFeed.stream.getTracks()) { - logger.info( - `Adding track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${callFeed.stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - senderArray.push(this.peerConn.addTrack(track, callFeed.stream)); - } - } - - logger.info( - `Pushed local stream `+ - `(id="${callFeed.stream.id}", `+ - `active="${callFeed.stream.active}", `+ - `purpose="${callFeed.purpose}")`, - ); - - this.emit(CallEvent.FeedsChanged, this.feeds); - } - - /** - * Removes local call feed from the call and its tracks from the peer - * connection - * @param callFeed to remove - */ - public removeLocalFeed(callFeed: CallFeed): void { - const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia - ? this.usermediaSenders - : this.screensharingSenders; - - for (const sender of senderArray) { - this.peerConn.removeTrack(sender); - } - - if (callFeed.purpose === SDPStreamMetadataPurpose.Screenshare) { - this.client.getMediaHandler().stopScreensharingStream(callFeed.stream); - } - - // Empty the array - senderArray.splice(0, senderArray.length); - this.deleteFeed(callFeed); - } - - private deleteAllFeeds(): void { - for (const feed of this.feeds) { - if (!feed.isLocal() || !this.groupCallId) { - feed.dispose(); - } - } - - this.feeds = []; - this.emit(CallEvent.FeedsChanged, this.feeds); - } - - private deleteFeedByStream(stream: MediaStream): void { - const feed = this.getFeedByStreamId(stream.id); - if (!feed) { - logger.warn(`Didn't find the feed with stream id ${stream.id} to delete`); - return; - } - this.deleteFeed(feed); - } - - private deleteFeed(feed: CallFeed): void { - feed.dispose(); - this.feeds.splice(this.feeds.indexOf(feed), 1); - this.emit(CallEvent.FeedsChanged, this.feeds); - } - - // The typescript definitions have this type as 'any' :( - public async getCurrentCallStats(): Promise { - if (this.callHasEnded()) { - return this.callStatsAtEnd; - } - - return this.collectCallStats(); - } - - private async collectCallStats(): Promise { - // This happens when the call fails before it starts. - // For example when we fail to get capture sources - if (!this.peerConn) return; - - const statsReport = await this.peerConn.getStats(); - const stats = []; - for (const item of statsReport) { - stats.push(item[1]); - } - - return stats; - } - - /** - * Configure this call from an invite event. Used by MatrixClient. - * @param {MatrixEvent} event The m.call.invite event - */ - public async initWithInvite(event: MatrixEvent): Promise { - const invite = event.getContent(); - this.direction = CallDirection.Inbound; - - // make sure we have valid turn creds. Unless something's gone wrong, it should - // poll and keep the credentials valid so this should be instant. - const haveTurnCreds = await this.client.checkTurnServers(); - if (!haveTurnCreds) { - logger.warn("Failed to get TURN credentials! Proceeding with call anyway..."); - } - - const sdpStreamMetadata = invite[SDPStreamMetadataKey]; - if (sdpStreamMetadata) { - this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); - } else { - logger.debug("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); - } - - this.peerConn = this.createPeerConnection(); - // we must set the party ID before await-ing on anything: the call event - // handler will start giving us more call events (eg. candidates) so if - // we haven't set the party ID, we'll ignore them. - this.chooseOpponent(event); - try { - await this.peerConn.setRemoteDescription(invite.offer); - await this.addBufferedIceCandidates(); - } catch (e) { - logger.debug("Failed to set remote description", e); - this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); - return; - } - - const remoteStream = this.feeds.find((feed) => !feed.isLocal())?.stream; - - // According to previous comments in this file, firefox at some point did not - // add streams until media started arriving on them. Testing latest firefox - // (81 at time of writing), this is no longer a problem, so let's do it the correct way. - if (!remoteStream || remoteStream.getTracks().length === 0) { - logger.error("No remote stream or no tracks after setting remote description!"); - this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); - return; - } - - this.setState(CallState.Ringing); - - if (event.getLocalAge()) { - setTimeout(() => { - if (this.state == CallState.Ringing) { - logger.debug("Call invite has expired. Hanging up."); - this.hangupParty = CallParty.Remote; // effectively - this.setState(CallState.Ended); - this.stopAllMedia(); - if (this.peerConn.signalingState != 'closed') { - this.peerConn.close(); - } - this.emit(CallEvent.Hangup); - } - }, invite.lifetime - event.getLocalAge()); - } - } - - /** - * Configure this call from a hangup or reject event. Used by MatrixClient. - * @param {MatrixEvent} event The m.call.hangup event - */ - public initWithHangup(event: MatrixEvent): void { - // perverse as it may seem, sometimes we want to instantiate a call with a - // hangup message (because when getting the state of the room on load, events - // come in reverse order and we want to remember that a call has been hung up) - this.setState(CallState.Ended); - } - - private shouldAnswerWithMediaType( - wantedValue: boolean | undefined, valueOfTheOtherSide: boolean | undefined, type: "audio" | "video", - ): boolean { - if (wantedValue && !valueOfTheOtherSide) { - // TODO: Figure out how to do this - logger.warn(`Unable to answer with ${type} because the other side isn't sending it either.`); - return false; - } else if ( - !utils.isNullOrUndefined(wantedValue) && - wantedValue !== valueOfTheOtherSide && - !this.opponentSupportsSDPStreamMetadata() - ) { - logger.warn( - `Unable to answer with ${type}=${wantedValue} because the other side doesn't support it. ` + - `Answering with ${type}=${valueOfTheOtherSide}.`, - ); - return valueOfTheOtherSide; - } - return wantedValue ?? valueOfTheOtherSide; - } - - /** - * Answer a call. - */ - public async answer(audio?: boolean, video?: boolean): Promise { - if (this.inviteOrAnswerSent) return; - // TODO: Figure out how to do this - if (audio === false && video === false) throw new Error("You CANNOT answer a call without media"); - - if (!this.localUsermediaStream && !this.waitForLocalAVStream) { - const prevState = this.state; - const answerWithAudio = this.shouldAnswerWithMediaType(audio, this.hasRemoteUserMediaAudioTrack, "audio"); - const answerWithVideo = this.shouldAnswerWithMediaType(video, this.hasRemoteUserMediaVideoTrack, "video"); - - this.setState(CallState.WaitLocalMedia); - this.waitForLocalAVStream = true; - - try { - const stream = await this.client.getMediaHandler().getUserMediaStream( - answerWithAudio, answerWithVideo, - ); - this.waitForLocalAVStream = false; - const usermediaFeed = new CallFeed({ - client: this.client, - roomId: this.roomId, - userId: this.client.getUserId(), - stream, - purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: stream.getAudioTracks().length === 0, - videoMuted: stream.getVideoTracks().length === 0, - }); - - const feeds = [usermediaFeed]; - - if (this.localScreensharingFeed) { - feeds.push(this.localScreensharingFeed); - } - - this.answerWithCallFeeds(feeds); - } catch (e) { - if (answerWithVideo) { - // Try to answer without video - logger.warn("Failed to getUserMedia(), trying to getUserMedia() without video"); - this.setState(prevState); - this.waitForLocalAVStream = false; - await this.answer(answerWithAudio, false); - } else { - this.getUserMediaFailed(e); - return; - } - } - } else if (this.waitForLocalAVStream) { - this.setState(CallState.WaitLocalMedia); - } - } - - public answerWithCallFeeds(callFeeds: CallFeed[]): void { - if (this.inviteOrAnswerSent) return; - - logger.debug(`Answering call ${this.callId}`); - - this.queueGotCallFeedsForAnswer(callFeeds); - } - - /** - * Replace this call with a new call, e.g. for glare resolution. Used by - * MatrixClient. - * @param {MatrixCall} newCall The new call. - */ - public replacedBy(newCall: MatrixCall): void { - if (this.state === CallState.WaitLocalMedia) { - logger.debug("Telling new call to wait for local media"); - newCall.waitForLocalAVStream = true; - } else if ([CallState.CreateOffer, CallState.InviteSent].includes(this.state)) { - if (newCall.direction === CallDirection.Outbound) { - newCall.queueGotCallFeedsForAnswer([]); - } else { - logger.debug("Handing local stream to new call"); - newCall.queueGotCallFeedsForAnswer(this.getLocalFeeds().map(feed => feed.clone())); - } - } - this.successor = newCall; - this.emit(CallEvent.Replaced, newCall); - this.hangup(CallErrorCode.Replaced, true); - } - - /** - * Hangup a call. - * @param {string} reason The reason why the call is being hung up. - * @param {boolean} suppressEvent True to suppress emitting an event. - */ - public hangup(reason: CallErrorCode, suppressEvent: boolean): void { - if (this.callHasEnded()) return; - - logger.debug("Ending call " + this.callId); - this.terminate(CallParty.Local, reason, !suppressEvent); - // We don't want to send hangup here if we didn't even get to sending an invite - if (this.state === CallState.WaitLocalMedia) return; - const content = {}; - // Don't send UserHangup reason to older clients - if ((this.opponentVersion && this.opponentVersion >= 1) || reason !== CallErrorCode.UserHangup) { - content["reason"] = reason; - } - this.sendVoipEvent(EventType.CallHangup, content); - } - - /** - * Reject a call - * This used to be done by calling hangup, but is a separate method and protocol - * event as of MSC2746. - */ - public reject(): void { - if (this.state !== CallState.Ringing) { - throw Error("Call must be in 'ringing' state to reject!"); - } - - if (this.opponentVersion < 1) { - logger.info( - `Opponent version is less than 1 (${this.opponentVersion}): sending hangup instead of reject`, - ); - this.hangup(CallErrorCode.UserHangup, true); - return; - } - - logger.debug("Rejecting call: " + this.callId); - this.terminate(CallParty.Local, CallErrorCode.UserHangup, true); - this.sendVoipEvent(EventType.CallReject, {}); - } - - /** - * Adds an audio and/or video track - upgrades the call - * @param {boolean} audio should add an audio track - * @param {boolean} video should add an video track - */ - private async upgradeCall( - audio: boolean, video: boolean, - ): Promise { - // We don't do call downgrades - if (!audio && !video) return; - if (!this.opponentSupportsSDPStreamMetadata()) return; - - try { - const upgradeAudio = audio && !this.hasLocalUserMediaAudioTrack; - const upgradeVideo = video && !this.hasLocalUserMediaVideoTrack; - logger.debug(`Upgrading call: audio?=${upgradeAudio} video?=${upgradeVideo}`); - - const stream = await this.client.getMediaHandler().getUserMediaStream(upgradeAudio, upgradeVideo); - if (upgradeAudio && upgradeVideo) { - if (this.hasLocalUserMediaAudioTrack) return; - if (this.hasLocalUserMediaVideoTrack) return; - - this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); - } else if (upgradeAudio) { - if (this.hasLocalUserMediaAudioTrack) return; - - const audioTrack = stream.getAudioTracks()[0]; - this.localUsermediaStream.addTrack(audioTrack); - this.peerConn.addTrack(audioTrack, this.localUsermediaStream); - } else if (upgradeVideo) { - if (this.hasLocalUserMediaVideoTrack) return; - - const videoTrack = stream.getVideoTracks()[0]; - this.localUsermediaStream.addTrack(videoTrack); - this.peerConn.addTrack(videoTrack, this.localUsermediaStream); - } - } catch (error) { - logger.error("Failed to upgrade the call", error); - this.emit(CallEvent.Error, - new CallError(CallErrorCode.NoUserMedia, "Failed to get camera access: ", error), - ); - } - } - - /** - * Returns true if this.remoteSDPStreamMetadata is defined, otherwise returns false - * @returns {boolean} can screenshare - */ - public opponentSupportsSDPStreamMetadata(): boolean { - return Boolean(this.remoteSDPStreamMetadata); - } - - /** - * If there is a screensharing stream returns true, otherwise returns false - * @returns {boolean} is screensharing - */ - public isScreensharing(): boolean { - return Boolean(this.localScreensharingStream); - } - - /** - * Starts/stops screensharing - * @param enabled the desired screensharing state - * @param {string} desktopCapturerSourceId optional id of the desktop capturer source to use - * @returns {boolean} new screensharing state - */ - public async setScreensharingEnabled( - enabled: boolean, desktopCapturerSourceId?: string, - ): Promise { - // Skip if there is nothing to do - if (enabled && this.isScreensharing()) { - logger.warn(`There is already a screensharing stream - there is nothing to do!`); - return true; - } else if (!enabled && !this.isScreensharing()) { - logger.warn(`There already isn't a screensharing stream - there is nothing to do!`); - return false; - } - - // Fallback to replaceTrack() - if (!this.opponentSupportsSDPStreamMetadata()) { - return await this.setScreensharingEnabledWithoutMetadataSupport(enabled, desktopCapturerSourceId); - } - - logger.debug(`Set screensharing enabled? ${enabled}`); - if (enabled) { - try { - const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId); - if (!stream) return false; - this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare); - return true; - } catch (err) { - logger.error("Failed to get screen-sharing stream:", err); - return false; - } - } else { - for (const sender of this.screensharingSenders) { - this.peerConn.removeTrack(sender); - } - this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream); - this.deleteFeedByStream(this.localScreensharingStream); - return false; - } - } - - /** - * Starts/stops screensharing - * Should be used ONLY if the opponent doesn't support SDPStreamMetadata - * @param enabled the desired screensharing state - * @param {string} desktopCapturerSourceId optional id of the desktop capturer source to use - * @returns {boolean} new screensharing state - */ - private async setScreensharingEnabledWithoutMetadataSupport( - enabled: boolean, desktopCapturerSourceId?: string, - ): Promise { - logger.debug(`Set screensharing enabled? ${enabled} using replaceTrack()`); - if (enabled) { - try { - const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId); - if (!stream) return false; - - const track = stream.getTracks().find((track) => { - return track.kind === "video"; - }); - const sender = this.usermediaSenders.find((sender) => { - return sender.track?.kind === "video"; - }); - sender.replaceTrack(track); - - this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare, false); - - return true; - } catch (err) { - logger.error("Failed to get screen-sharing stream:", err); - return false; - } - } else { - const track = this.localUsermediaStream.getTracks().find((track) => { - return track.kind === "video"; - }); - const sender = this.usermediaSenders.find((sender) => { - return sender.track?.kind === "video"; - }); - sender.replaceTrack(track); - - this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream); - this.deleteFeedByStream(this.localScreensharingStream); - - return false; - } - } - - /** - * Request a new local usermedia stream with the current device id. - */ - public async updateLocalUsermediaStream(stream: MediaStream) { - const callFeed = this.localUsermediaFeed; - callFeed.setNewStream(stream); - const micShouldBeMuted = callFeed.isAudioMuted() || this.remoteOnHold; - const vidShouldBeMuted = callFeed.isVideoMuted() || this.remoteOnHold; - setTracksEnabled(stream.getAudioTracks(), !micShouldBeMuted); - setTracksEnabled(stream.getVideoTracks(), !vidShouldBeMuted); - - const newSenders = []; - - for (const track of stream.getTracks()) { - const oldSender = this.usermediaSenders.find((sender) => { - return sender.track?.kind === track.kind; - }); - - let newSender: RTCRtpSender; - - try { - logger.info( - `Replacing track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - await oldSender.replaceTrack(track); - newSender = oldSender; - } catch (error) { - logger.info( - `Adding track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - newSender = this.peerConn.addTrack(track, stream); - } - - newSenders.push(newSender); - } - - this.usermediaSenders = newSenders; - } - - /** - * Set whether our outbound video should be muted or not. - * @param {boolean} muted True to mute the outbound video. - * @returns the new mute state - */ - public async setLocalVideoMuted(muted: boolean): Promise { - if (!await this.client.getMediaHandler().hasVideoDevice()) { - return this.isLocalVideoMuted(); - } - - if (!this.hasLocalUserMediaVideoTrack && !muted) { - await this.upgradeCall(false, true); - return this.isLocalVideoMuted(); - } - this.localUsermediaFeed?.setVideoMuted(muted); - this.updateMuteStatus(); - return this.isLocalVideoMuted(); - } - - /** - * Check if local video is muted. - * - * If there are multiple video tracks, all of the tracks need to be muted - * for this to return true. This means if there are no video tracks, this will - * return true. - * @return {Boolean} True if the local preview video is muted, else false - * (including if the call is not set up yet). - */ - public isLocalVideoMuted(): boolean { - return this.localUsermediaFeed?.isVideoMuted(); - } - - /** - * Set whether the microphone should be muted or not. - * @param {boolean} muted True to mute the mic. - * @returns the new mute state - */ - public async setMicrophoneMuted(muted: boolean): Promise { - if (!await this.client.getMediaHandler().hasAudioDevice()) { - return this.isMicrophoneMuted(); - } - - if (!this.hasLocalUserMediaAudioTrack && !muted) { - await this.upgradeCall(true, false); - return this.isMicrophoneMuted(); - } - this.localUsermediaFeed?.setAudioMuted(muted); - this.updateMuteStatus(); - return this.isMicrophoneMuted(); - } - - /** - * Check if the microphone is muted. - * - * If there are multiple audio tracks, all of the tracks need to be muted - * for this to return true. This means if there are no audio tracks, this will - * return true. - * @return {Boolean} True if the mic is muted, else false (including if the call - * is not set up yet). - */ - public isMicrophoneMuted(): boolean { - return this.localUsermediaFeed?.isAudioMuted(); - } - - /** - * @returns true if we have put the party on the other side of the call on hold - * (that is, we are signalling to them that we are not listening) - */ - public isRemoteOnHold(): boolean { - return this.remoteOnHold; - } - - public setRemoteOnHold(onHold: boolean): void { - if (this.isRemoteOnHold() === onHold) return; - this.remoteOnHold = onHold; - - for (const transceiver of this.peerConn.getTransceivers()) { - // We don't send hold music or anything so we're not actually - // sending anything, but sendrecv is fairly standard for hold and - // it makes it a lot easier to figure out who's put who on hold. - transceiver.direction = onHold ? 'sendonly' : 'sendrecv'; - } - this.updateMuteStatus(); - - this.emit(CallEvent.RemoteHoldUnhold, this.remoteOnHold); - } - - /** - * Indicates whether we are 'on hold' to the remote party (ie. if true, - * they cannot hear us). - * @returns true if the other party has put us on hold - */ - public isLocalOnHold(): boolean { - if (this.state !== CallState.Connected) return false; - - let callOnHold = true; - - // We consider a call to be on hold only if *all* the tracks are on hold - // (is this the right thing to do?) - for (const transceiver of this.peerConn.getTransceivers()) { - const trackOnHold = ['inactive', 'recvonly'].includes(transceiver.currentDirection); - - if (!trackOnHold) callOnHold = false; - } - - return callOnHold; - } - - /** - * Sends a DTMF digit to the other party - * @param digit The digit (nb. string - '#' and '*' are dtmf too) - */ - public sendDtmfDigit(digit: string): void { - for (const sender of this.peerConn.getSenders()) { - if (sender.track.kind === 'audio' && sender.dtmf) { - sender.dtmf.insertDTMF(digit); - return; - } - } - - throw new Error("Unable to find a track to send DTMF on"); - } - - private updateMuteStatus(): void { - this.sendVoipEvent(EventType.CallSDPStreamMetadataChangedPrefix, { - [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(), - }); - - const micShouldBeMuted = this.localUsermediaFeed?.isAudioMuted() || this.remoteOnHold; - const vidShouldBeMuted = this.localUsermediaFeed?.isVideoMuted() || this.remoteOnHold; - - setTracksEnabled(this.localUsermediaStream.getAudioTracks(), !micShouldBeMuted); - setTracksEnabled(this.localUsermediaStream.getVideoTracks(), !vidShouldBeMuted); - } - - private gotCallFeedsForInvite(callFeeds: CallFeed[], requestScreenshareFeed = false): void { - if (this.successor) { - this.successor.queueGotCallFeedsForAnswer(callFeeds); - return; - } - if (this.callHasEnded()) { - this.stopAllMedia(); - return; - } - - for (const feed of callFeeds) { - this.pushLocalFeed(feed); - } - - if (requestScreenshareFeed) { - this.peerConn.addTransceiver("video", { - direction: "recvonly", - }); - } - - this.setState(CallState.CreateOffer); - - logger.debug("gotUserMediaForInvite"); - // Now we wait for the negotiationneeded event - } - - private async sendAnswer(): Promise { - const answerContent = { - answer: { - sdp: this.peerConn.localDescription.sdp, - // type is now deprecated as of Matrix VoIP v1, but - // required to still be sent for backwards compat - type: this.peerConn.localDescription.type, - }, - [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(true), - } as MCallAnswer; - - answerContent.capabilities = { - 'm.call.transferee': this.client.supportsCallTransfer, - 'm.call.dtmf': false, - }; - - // We have just taken the local description from the peerConn which will - // contain all the local candidates added so far, so we can discard any candidates - // we had queued up because they'll be in the answer. - logger.info(`Discarding ${this.candidateSendQueue.length} candidates that will be sent in answer`); - this.candidateSendQueue = []; - - try { - await this.sendVoipEvent(EventType.CallAnswer, answerContent); - // If this isn't the first time we've tried to send the answer, - // we may have candidates queued up, so send them now. - this.inviteOrAnswerSent = true; - } catch (error) { - // We've failed to answer: back to the ringing state - this.setState(CallState.Ringing); - this.client.cancelPendingEvent(error.event); - - let code = CallErrorCode.SendAnswer; - let message = "Failed to send answer"; - if (error.name == 'UnknownDeviceError') { - code = CallErrorCode.UnknownDevices; - message = "Unknown devices present in the room"; - } - this.emit(CallEvent.Error, new CallError(code, message, error)); - throw error; - } - - // error handler re-throws so this won't happen on error, but - // we don't want the same error handling on the candidate queue - this.sendCandidateQueue(); - } - - private queueGotCallFeedsForAnswer(callFeeds: CallFeed[]): void { - // Ensure only one negotiate/answer event is being processed at a time. - if (this.responsePromiseChain) { - this.responsePromiseChain = - this.responsePromiseChain.then(() => this.gotCallFeedsForAnswer(callFeeds)); - } else { - this.responsePromiseChain = this.gotCallFeedsForAnswer(callFeeds); - } - } - - private async gotCallFeedsForAnswer(callFeeds: CallFeed[]): Promise { - if (this.callHasEnded()) return; - - this.waitForLocalAVStream = false; - - for (const feed of callFeeds) { - this.pushLocalFeed(feed); - } - - this.setState(CallState.CreateAnswer); - - let myAnswer; - try { - this.getRidOfRTXCodecs(); - myAnswer = await this.peerConn.createAnswer(); - } catch (err) { - logger.debug("Failed to create answer: ", err); - this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true); - return; - } - - try { - await this.peerConn.setLocalDescription(myAnswer); - this.setState(CallState.Connecting); - - // Allow a short time for initial candidates to be gathered - await new Promise(resolve => { - setTimeout(resolve, 200); - }); - - this.sendAnswer(); - } catch (err) { - logger.debug("Error setting local description!", err); - this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); - return; - } - - // HACK: Safari doesn't like it when we reuse MediaStreams. In most cases - // we can get around this by calling MediaStream.clone(), however inbound - // calls seem to still be broken unless we getUserMedia again and replace - // all MediaStreams using sender.replaceTrack - if (isSafari) { - await new Promise(resolve => { - setTimeout(resolve, 200); - }); - - if (this.state === CallState.Ended) { - return; - } - - const callFeed = this.localUsermediaFeed; - const stream = callFeed.stream; - - if (!stream.active) { - throw new Error(`Call ${this.callId} has an inactive stream ${ - stream.id} and its tracks cannot be replaced`); - } - - const newSenders = []; - - for (const track of this.localUsermediaStream.getTracks()) { - const oldSender = this.usermediaSenders.find((sender) => { - return sender.track?.kind === track.kind; - }); - - if (track.readyState === "ended") { - throw new Error(`Call ${this.callId} tried to replace track ${track.id} in the ended state`); - } - - let newSender: RTCRtpSender; - - try { - logger.info( - `Replacing track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - await oldSender.replaceTrack(track); - newSender = oldSender; - } catch (error) { - logger.info( - `Adding track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - newSender = this.peerConn.addTrack(track, stream); - } - - newSenders.push(newSender); - } - - this.usermediaSenders = newSenders; - } - } - - /** - * Internal - * @param {Object} event - */ - private gotLocalIceCandidate = (event: RTCPeerConnectionIceEvent): Promise => { - if (event.candidate) { - logger.debug( - "Call " + this.callId + " got local ICE " + event.candidate.sdpMid + " candidate: " + - event.candidate.candidate, - ); - - if (this.callHasEnded()) return; - - // As with the offer, note we need to make a copy of this object, not - // pass the original: that broke in Chrome ~m43. - if (event.candidate.candidate !== '' || !this.sentEndOfCandidates) { - this.queueCandidate(event.candidate); - - if (event.candidate.candidate === '') this.sentEndOfCandidates = true; - } - } - }; - - private onIceGatheringStateChange = (event: Event): void => { - logger.debug("ice gathering state changed to " + this.peerConn.iceGatheringState); - if (this.peerConn.iceGatheringState === 'complete' && !this.sentEndOfCandidates) { - // If we didn't get an empty-string candidate to signal the end of candidates, - // create one ourselves now gathering has finished. - // We cast because the interface lists all the properties as required but we - // only want to send 'candidate' - // XXX: We probably want to send either sdpMid or sdpMLineIndex, as it's not strictly - // correct to have a candidate that lacks both of these. We'd have to figure out what - // previous candidates had been sent with and copy them. - const c = { - candidate: '', - } as RTCIceCandidate; - this.queueCandidate(c); - this.sentEndOfCandidates = true; - } - }; - - public async onRemoteIceCandidatesReceived(ev: MatrixEvent): Promise { - if (this.callHasEnded()) { - //debuglog("Ignoring remote ICE candidate because call has ended"); - return; - } - - const content = ev.getContent(); - const candidates = content.candidates; - if (!candidates) { - logger.info("Ignoring candidates event with no candidates!"); - return; - } - - const fromPartyId = content.version === 0 ? null : content.party_id || null; - - if (this.opponentPartyId === undefined) { - // we haven't picked an opponent yet so save the candidates - logger.info(`Buffering ${candidates.length} candidates until we pick an opponent`); - const bufferedCandidates = this.remoteCandidateBuffer.get(fromPartyId) || []; - bufferedCandidates.push(...candidates); - this.remoteCandidateBuffer.set(fromPartyId, bufferedCandidates); - return; - } - - if (!this.partyIdMatches(content)) { - logger.info( - `Ignoring candidates from party ID ${content.party_id}: ` + - `we have chosen party ID ${this.opponentPartyId}`, - ); - - return; - } - - await this.addIceCandidates(candidates); - } - - /** - * Used by MatrixClient. - * @param {Object} msg - */ - public async onAnswerReceived(event: MatrixEvent): Promise { - const content = event.getContent(); - logger.debug(`Got answer for call ID ${this.callId} from party ID ${content.party_id}`); - - if (this.callHasEnded()) { - logger.debug(`Ignoring answer because call ID ${this.callId} has ended`); - return; - } - - if (this.opponentPartyId !== undefined) { - logger.info( - `Ignoring answer from party ID ${content.party_id}: ` + - `we already have an answer/reject from ${this.opponentPartyId}`, - ); - return; - } - - this.chooseOpponent(event); - await this.addBufferedIceCandidates(); - - this.setState(CallState.Connecting); - - const sdpStreamMetadata = content[SDPStreamMetadataKey]; - if (sdpStreamMetadata) { - this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); - } else { - logger.warn("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); - } - - try { - await this.peerConn.setRemoteDescription(content.answer); - } catch (e) { - logger.debug("Failed to set remote description", e); - this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); - return; - } - - // If the answer we selected has a party_id, send a select_answer event - // We do this after setting the remote description since otherwise we'd block - // call setup on it - if (this.opponentPartyId !== null) { - try { - await this.sendVoipEvent(EventType.CallSelectAnswer, { - selected_party_id: this.opponentPartyId, - }); - } catch (err) { - // This isn't fatal, and will just mean that if another party has raced to answer - // the call, they won't know they got rejected, so we carry on & don't retry. - logger.warn("Failed to send select_answer event", err); - } - } - } - - public async onSelectAnswerReceived(event: MatrixEvent): Promise { - if (this.direction !== CallDirection.Inbound) { - logger.warn("Got select_answer for an outbound call: ignoring"); - return; - } - - const selectedPartyId = event.getContent().selected_party_id; - - if (selectedPartyId === undefined || selectedPartyId === null) { - logger.warn("Got nonsensical select_answer with null/undefined selected_party_id: ignoring"); - return; - } - - if (selectedPartyId !== this.ourPartyId) { - logger.info(`Got select_answer for party ID ${selectedPartyId}: we are party ID ${this.ourPartyId}.`); - // The other party has picked somebody else's answer - this.terminate(CallParty.Remote, CallErrorCode.AnsweredElsewhere, true); - } - } - - public async onNegotiateReceived(event: MatrixEvent): Promise { - const content = event.getContent(); - const description = content.description; - if (!description || !description.sdp || !description.type) { - logger.info("Ignoring invalid m.call.negotiate event"); - return; - } - // Politeness always follows the direction of the call: in a glare situation, - // we pick either the inbound or outbound call, so one side will always be - // inbound and one outbound - const polite = this.direction === CallDirection.Inbound; - - // Here we follow the perfect negotiation logic from - // https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation - const offerCollision = ( - (description.type === 'offer') && - (this.makingOffer || this.peerConn.signalingState !== 'stable') - ); - - this.ignoreOffer = !polite && offerCollision; - if (this.ignoreOffer) { - logger.info("Ignoring colliding negotiate event because we're impolite"); - return; - } - - const prevLocalOnHold = this.isLocalOnHold(); - - const sdpStreamMetadata = content[SDPStreamMetadataKey]; - if (sdpStreamMetadata) { - this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); - } else { - logger.warn("Received negotiation event without SDPStreamMetadata!"); - } - - try { - await this.peerConn.setRemoteDescription(description); - - if (description.type === 'offer') { - this.getRidOfRTXCodecs(); - await this.peerConn.setLocalDescription(); - - this.sendVoipEvent(EventType.CallNegotiate, { - description: this.peerConn.localDescription, - [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(true), - }); - } - } catch (err) { - logger.warn("Failed to complete negotiation", err); - } - - const newLocalOnHold = this.isLocalOnHold(); - if (prevLocalOnHold !== newLocalOnHold) { - this.emit(CallEvent.LocalHoldUnhold, newLocalOnHold); - // also this one for backwards compat - this.emit(CallEvent.HoldUnhold, newLocalOnHold); - } - } - - private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { - this.remoteSDPStreamMetadata = utils.recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); - for (const feed of this.getRemoteFeeds()) { - const streamId = feed.stream.id; - feed.setAudioMuted(this.remoteSDPStreamMetadata[streamId]?.audio_muted); - feed.setVideoMuted(this.remoteSDPStreamMetadata[streamId]?.video_muted); - feed.purpose = this.remoteSDPStreamMetadata[streamId]?.purpose; - } - } - - public onSDPStreamMetadataChangedReceived(event: MatrixEvent): void { - const content = event.getContent(); - const metadata = content[SDPStreamMetadataKey]; - this.updateRemoteSDPStreamMetadata(metadata); - } - - public async onAssertedIdentityReceived(event: MatrixEvent): Promise { - const content = event.getContent(); - if (!content.asserted_identity) return; - - this.remoteAssertedIdentity = { - id: content.asserted_identity.id, - displayName: content.asserted_identity.display_name, - }; - this.emit(CallEvent.AssertedIdentityChanged); - } - - private callHasEnded(): boolean { - // This exists as workaround to typescript trying to be clever and erroring - // when putting if (this.state === CallState.Ended) return; twice in the same - // function, even though that function is async. - return this.state === CallState.Ended; - } - - private queueGotLocalOffer(): void { - // Ensure only one negotiate/answer event is being processed at a time. - if (this.responsePromiseChain) { - this.responsePromiseChain = - this.responsePromiseChain.then(() => this.wrappedGotLocalOffer()); - } else { - this.responsePromiseChain = this.wrappedGotLocalOffer(); - } - } - - private async wrappedGotLocalOffer(): Promise { - this.makingOffer = true; - try { - this.getRidOfRTXCodecs(); - await this.gotLocalOffer(); - } catch (e) { - this.getLocalOfferFailed(e); - return; - } finally { - this.makingOffer = false; - } - } - - private async gotLocalOffer(): Promise { - logger.debug("Setting local description"); - - if (this.callHasEnded()) { - logger.debug("Ignoring newly created offer on call ID " + this.callId + - " because the call has ended"); - return; - } - - try { - await this.peerConn.setLocalDescription(); - } catch (err) { - logger.debug("Error setting local description!", err); - this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); - return; - } - - if (this.peerConn.iceGatheringState === 'gathering') { - // Allow a short time for initial candidates to be gathered - await new Promise(resolve => { - setTimeout(resolve, 200); - }); - } - - if (this.callHasEnded()) return; - - const eventType = this.state === CallState.CreateOffer ? EventType.CallInvite : EventType.CallNegotiate; - - const content = { - lifetime: CALL_TIMEOUT_MS, - } as MCallInviteNegotiate; - - if (eventType === EventType.CallInvite && this.invitee) { - content.invitee = this.invitee; - } - - // clunky because TypeScript can't follow the types through if we use an expression as the key - if (this.state === CallState.CreateOffer) { - content.offer = this.peerConn.localDescription; - } else { - content.description = this.peerConn.localDescription; - } - - content.capabilities = { - 'm.call.transferee': this.client.supportsCallTransfer, - 'm.call.dtmf': false, - }; - - content[SDPStreamMetadataKey] = this.getLocalSDPStreamMetadata(true); - - // Get rid of any candidates waiting to be sent: they'll be included in the local - // description we just got and will send in the offer. - logger.info(`Discarding ${this.candidateSendQueue.length} candidates that will be sent in offer`); - this.candidateSendQueue = []; - - try { - await this.sendVoipEvent(eventType, content); - } catch (error) { - logger.error("Failed to send invite", error); - if (error.event) this.client.cancelPendingEvent(error.event); - - let code = CallErrorCode.SignallingFailed; - let message = "Signalling failed"; - if (this.state === CallState.CreateOffer) { - code = CallErrorCode.SendInvite; - message = "Failed to send invite"; - } - if (error.name == 'UnknownDeviceError') { - code = CallErrorCode.UnknownDevices; - message = "Unknown devices present in the room"; - } - - this.emit(CallEvent.Error, new CallError(code, message, error)); - this.terminate(CallParty.Local, code, false); - - // no need to carry on & send the candidate queue, but we also - // don't want to rethrow the error - return; - } - - this.sendCandidateQueue(); - if (this.state === CallState.CreateOffer) { - this.inviteOrAnswerSent = true; - this.setState(CallState.InviteSent); - this.inviteTimeout = setTimeout(() => { - this.inviteTimeout = null; - if (this.state === CallState.InviteSent) { - this.hangup(CallErrorCode.InviteTimeout, false); - } - }, CALL_TIMEOUT_MS); - } - } - - private getLocalOfferFailed = (err: Error): void => { - logger.error("Failed to get local offer", err); - - this.emit( - CallEvent.Error, - new CallError( - CallErrorCode.LocalOfferFailed, - "Failed to get local offer!", err, - ), - ); - this.terminate(CallParty.Local, CallErrorCode.LocalOfferFailed, false); - }; - - private getUserMediaFailed = (err: Error): void => { - if (this.successor) { - this.successor.getUserMediaFailed(err); - return; - } - - logger.warn("Failed to get user media - ending call", err); - - this.emit( - CallEvent.Error, - new CallError( - CallErrorCode.NoUserMedia, - "Couldn't start capturing media! Is your microphone set up and " + - "does this app have permission?", err, - ), - ); - this.terminate(CallParty.Local, CallErrorCode.NoUserMedia, false); - }; - - private onIceConnectionStateChanged = (): void => { - if (this.callHasEnded()) { - return; // because ICE can still complete as we're ending the call - } - logger.debug( - "Call ID " + this.callId + ": ICE connection state changed to: " + this.peerConn.iceConnectionState, - ); - // ideally we'd consider the call to be connected when we get media but - // chrome doesn't implement any of the 'onstarted' events yet - if (this.peerConn.iceConnectionState == 'connected') { - clearTimeout(this.iceDisconnectedTimeout); - this.setState(CallState.Connected); - - if (!this.callLengthInterval) { - this.callLengthInterval = setInterval(() => { - this.callLength++; - this.emit(CallEvent.LengthChanged, this.callLength); - }, 1000); - } - } else if (this.peerConn.iceConnectionState == 'failed') { - // Firefox for Android does not yet have support for restartIce() - if (this.peerConn.restartIce) { - this.peerConn.restartIce(); - } else { - this.hangup(CallErrorCode.IceFailed, false); - } - } else if (this.peerConn.iceConnectionState == 'disconnected') { - this.iceDisconnectedTimeout = setTimeout(() => { - this.hangup(CallErrorCode.IceFailed, false); - }, 30 * 1000); - } - }; - - private onSignallingStateChanged = (): void => { - logger.debug( - "call " + this.callId + ": Signalling state changed to: " + - this.peerConn.signalingState, - ); - }; - - private onTrack = (ev: RTCTrackEvent): void => { - if (ev.streams.length === 0) { - logger.warn(`Streamless ${ev.track.kind} found: ignoring.`); - return; - } - - const stream = ev.streams[0]; - this.pushRemoteFeed(stream); - stream.addEventListener("removetrack", () => { - logger.log(`Removing track streamId: ${stream.id}`); - this.deleteFeedByStream(stream); - }); - }; - - private onDataChannel = (ev: RTCDataChannelEvent): void => { - this.emit(CallEvent.DataChannel, ev.channel); - }; - - /** - * This method removes all video/rtx codecs from screensharing video - * transceivers. This is necessary since they can cause problems. Without - * this the following steps should produce an error: - * Chromium calls Firefox - * Firefox answers - * Firefox starts screen-sharing - * Chromium starts screen-sharing - * Call crashes for Chromium with: - * [96685:23:0518/162603.933321:ERROR:webrtc_video_engine.cc(3296)] RTX codec (PT=97) mapped to PT=96 which is not in the codec list. - * [96685:23:0518/162603.933377:ERROR:webrtc_video_engine.cc(1171)] GetChangedRecvParameters called without any video codecs. - * [96685:23:0518/162603.933430:ERROR:sdp_offer_answer.cc(4302)] Failed to set local video description recv parameters for m-section with mid='2'. (INVALID_PARAMETER) - */ - private getRidOfRTXCodecs(): void { - // RTCRtpReceiver.getCapabilities and RTCRtpSender.getCapabilities don't seem to be supported on FF - if (!RTCRtpReceiver.getCapabilities || !RTCRtpSender.getCapabilities) return; - - const recvCodecs = RTCRtpReceiver.getCapabilities("video").codecs; - const sendCodecs = RTCRtpSender.getCapabilities("video").codecs; - const codecs = [...sendCodecs, ...recvCodecs]; - - for (const codec of codecs) { - if (codec.mimeType === "video/rtx") { - const rtxCodecIndex = codecs.indexOf(codec); - codecs.splice(rtxCodecIndex, 1); - } - } - - for (const trans of this.peerConn.getTransceivers()) { - if ( - this.screensharingSenders.includes(trans.sender) && - ( - trans.sender.track?.kind === "video" || - trans.receiver.track?.kind === "video" - ) - ) { - trans.setCodecPreferences(codecs); - } - } - } - - private onNegotiationNeeded = async (): Promise => { - logger.info("Negotiation is needed!"); - - if (this.state !== CallState.CreateOffer && this.opponentVersion === 0) { - logger.info("Opponent does not support renegotiation: ignoring negotiationneeded event"); - return; - } - - this.queueGotLocalOffer(); - }; - - public onHangupReceived = (msg: MCallHangupReject): void => { - logger.debug("Hangup received for call ID " + this.callId); - - // party ID must match (our chosen partner hanging up the call) or be undefined (we haven't chosen - // a partner yet but we're treating the hangup as a reject as per VoIP v0) - if (this.partyIdMatches(msg) || this.state === CallState.Ringing) { - // default reason is user_hangup - this.terminate(CallParty.Remote, msg.reason || CallErrorCode.UserHangup, true); - } else { - logger.info(`Ignoring message from party ID ${msg.party_id}: our partner is ${this.opponentPartyId}`); - } - }; - - public onRejectReceived = (msg: MCallHangupReject): void => { - logger.debug("Reject received for call ID " + this.callId); - - // No need to check party_id for reject because if we'd received either - // an answer or reject, we wouldn't be in state InviteSent - - const shouldTerminate = ( - // reject events also end the call if it's ringing: it's another of - // our devices rejecting the call. - ([CallState.InviteSent, CallState.Ringing].includes(this.state)) || - // also if we're in the init state and it's an inbound call, since - // this means we just haven't entered the ringing state yet - this.state === CallState.Fledgling && this.direction === CallDirection.Inbound - ); - - if (shouldTerminate) { - this.terminate(CallParty.Remote, msg.reason || CallErrorCode.UserHangup, true); - } else { - logger.debug(`Call is in state: ${this.state}: ignoring reject`); - } - }; - - public onAnsweredElsewhere = (msg: MCallAnswer): void => { - logger.debug("Call ID " + this.callId + " answered elsewhere"); - this.terminate(CallParty.Remote, CallErrorCode.AnsweredElsewhere, true); - }; - - private setState(state: CallState): void { - const oldState = this.state; - this.state = state; - this.emit(CallEvent.State, state, oldState); - } - - /** - * Internal - * @param {string} eventType - * @param {Object} content - * @return {Promise} - */ - private sendVoipEvent(eventType: string, content: object): Promise { - const realContent = Object.assign({}, content, { - version: VOIP_PROTO_VERSION, - call_id: this.callId, - party_id: this.ourPartyId, - conf_id: this.groupCallId, - }); - - if (this.opponentDeviceId) { - this.emit(CallEvent.SendVoipEvent, { - type: "toDevice", - eventType, - userId: this.invitee || this.getOpponentMember().userId, - opponentDeviceId: this.opponentDeviceId, - content: { - ...realContent, - device_id: this.client.deviceId, - sender_session_id: this.client.getSessionId(), - dest_session_id: this.opponentSessionId, - }, - }); - - return this.client.sendToDevice(eventType, { - [this.invitee || this.getOpponentMember().userId]: { - [this.opponentDeviceId]: { - ...realContent, - device_id: this.client.deviceId, - sender_session_id: this.client.getSessionId(), - dest_session_id: this.opponentSessionId, - }, - }, - }); - } else { - this.emit(CallEvent.SendVoipEvent, { - type: "sendEvent", - eventType, - roomId: this.roomId, - content: realContent, - userId: this.invitee || this.getOpponentMember().userId, - }); - - return this.client.sendEvent(this.roomId, eventType, realContent); - } - } - - private queueCandidate(content: RTCIceCandidate): void { - // We partially de-trickle candidates by waiting for `delay` before sending them - // amalgamated, in order to avoid sending too many m.call.candidates events and hitting - // rate limits in Matrix. - // In practice, it'd be better to remove rate limits for m.call.* - - // N.B. this deliberately lets you queue and send blank candidates, which MSC2746 - // currently proposes as the way to indicate that candidate gathering is complete. - // This will hopefully be changed to an explicit rather than implicit notification - // shortly. - this.candidateSendQueue.push(content); - - // Don't send the ICE candidates yet if the call is in the ringing state: this - // means we tried to pick (ie. started generating candidates) and then failed to - // send the answer and went back to the ringing state. Queue up the candidates - // to send if we successfully send the answer. - // Equally don't send if we haven't yet sent the answer because we can send the - // first batch of candidates along with the answer - if (this.state === CallState.Ringing || !this.inviteOrAnswerSent) return; - - // MSC2746 recommends these values (can be quite long when calling because the - // callee will need a while to answer the call) - const delay = this.direction === CallDirection.Inbound ? 500 : 2000; - - if (this.candidateSendTries === 0) { - setTimeout(() => { - this.sendCandidateQueue(); - }, delay); - } - } - - /* - * Transfers this call to another user - */ - public async transfer(targetUserId: string): Promise { - // Fetch the target user's global profile info: their room avatar / displayname - // could be different in whatever room we share with them. - const profileInfo = await this.client.getProfileInfo(targetUserId); - - const replacementId = genCallID(); - - const body = { - replacement_id: genCallID(), - target_user: { - id: targetUserId, - display_name: profileInfo.displayname, - avatar_url: profileInfo.avatar_url, - }, - create_call: replacementId, - } as MCallReplacesEvent; - - await this.sendVoipEvent(EventType.CallReplaces, body); - - await this.terminate(CallParty.Local, CallErrorCode.Transfered, true); - } - - /* - * Transfers this call to the target call, effectively 'joining' the - * two calls (so the remote parties on each call are connected together). - */ - public async transferToCall(transferTargetCall?: MatrixCall): Promise { - const targetProfileInfo = await this.client.getProfileInfo(transferTargetCall.getOpponentMember().userId); - const transfereeProfileInfo = await this.client.getProfileInfo(this.getOpponentMember().userId); - - const newCallId = genCallID(); - - const bodyToTransferTarget = { - // the replacements on each side have their own ID, and it's distinct from the - // ID of the new call (but we can use the same function to generate it) - replacement_id: genCallID(), - target_user: { - id: this.getOpponentMember().userId, - display_name: transfereeProfileInfo.displayname, - avatar_url: transfereeProfileInfo.avatar_url, - }, - await_call: newCallId, - } as MCallReplacesEvent; - - await transferTargetCall.sendVoipEvent(EventType.CallReplaces, bodyToTransferTarget); - - const bodyToTransferee = { - replacement_id: genCallID(), - target_user: { - id: transferTargetCall.getOpponentMember().userId, - display_name: targetProfileInfo.displayname, - avatar_url: targetProfileInfo.avatar_url, - }, - create_call: newCallId, - } as MCallReplacesEvent; - - await this.sendVoipEvent(EventType.CallReplaces, bodyToTransferee); - - await this.terminate(CallParty.Local, CallErrorCode.Replaced, true); - await transferTargetCall.terminate(CallParty.Local, CallErrorCode.Transfered, true); - } - - private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean): Promise { - if (this.callHasEnded()) return; - - this.hangupParty = hangupParty; - this.hangupReason = hangupReason; - this.setState(CallState.Ended); - - if (this.inviteTimeout) { - clearTimeout(this.inviteTimeout); - this.inviteTimeout = null; - } - if (this.callLengthInterval) { - clearInterval(this.callLengthInterval); - this.callLengthInterval = null; - } - - this.callStatsAtEnd = await this.collectCallStats(); - - // Order is important here: first we stopAllMedia() and only then we can deleteAllFeeds() - this.stopAllMedia(); - this.deleteAllFeeds(); - - if (this.peerConn && this.peerConn.signalingState !== 'closed') { - this.peerConn.close(); - } - if (shouldEmit) { - this.emit(CallEvent.Hangup, this); - } - - this.client.callEventHandler.calls.delete(this.callId); - } - - private stopAllMedia(): void { - logger.debug(!this.groupCallId ? "Stopping all media" : "Stopping all media except local feeds" ); - for (const feed of this.feeds) { - if ( - feed.isLocal() && - feed.purpose === SDPStreamMetadataPurpose.Usermedia && - !this.groupCallId - ) { - this.client.getMediaHandler().stopUserMediaStream(feed.stream); - } else if ( - feed.isLocal() && - feed.purpose === SDPStreamMetadataPurpose.Screenshare && - !this.groupCallId - ) { - this.client.getMediaHandler().stopScreensharingStream(feed.stream); - } else if (!feed.isLocal() || !this.groupCallId) { - for (const track of feed.stream.getTracks()) { - track.stop(); - } - } - } - } - - private checkForErrorListener(): void { - if (this.listeners("error").length === 0) { - throw new Error( - "You MUST attach an error listener using call.on('error', function() {})", - ); - } - } - - private async sendCandidateQueue(): Promise { - if (this.candidateSendQueue.length === 0 || this.callHasEnded()) { - return; - } - - const candidates = this.candidateSendQueue; - this.candidateSendQueue = []; - ++this.candidateSendTries; - const content = { - candidates: candidates, - }; - logger.debug("Attempting to send " + candidates.length + " candidates"); - try { - await this.sendVoipEvent(EventType.CallCandidates, content); - // reset our retry count if we have successfully sent our candidates - // otherwise queueCandidate() will refuse to try to flush the queue - this.candidateSendTries = 0; - - // Try to send candidates again just in case we received more candidates while sending. - this.sendCandidateQueue(); - } catch (error) { - // don't retry this event: we'll send another one later as we might - // have more candidates by then. - if (error.event) this.client.cancelPendingEvent(error.event); - - // put all the candidates we failed to send back in the queue - this.candidateSendQueue.push(...candidates); - - if (this.candidateSendTries > 5) { - logger.debug( - "Failed to send candidates on attempt " + this.candidateSendTries + - ". Giving up on this call.", error, - ); - - const code = CallErrorCode.SignallingFailed; - const message = "Signalling failed"; - - this.emit(CallEvent.Error, new CallError(code, message, error)); - this.hangup(code, false); - - return; - } - - const delayMs = 500 * Math.pow(2, this.candidateSendTries); - ++this.candidateSendTries; - logger.debug("Failed to send candidates. Retrying in " + delayMs + "ms", error); - setTimeout(() => { - this.sendCandidateQueue(); - }, delayMs); - } - } - - /** - * Place a call to this room. - * @throws if you have not specified a listener for 'error' events. - * @throws if have passed audio=false. - */ - public async placeCall(audio: boolean, video: boolean): Promise { - if (!audio) { - throw new Error("You CANNOT start a call without audio"); - } - this.setState(CallState.WaitLocalMedia); - - try { - const stream = await this.client.getMediaHandler().getUserMediaStream(audio, video); - const callFeed = new CallFeed({ - client: this.client, - roomId: this.roomId, - userId: this.client.getUserId(), - stream, - purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: stream.getAudioTracks().length === 0, - videoMuted: stream.getVideoTracks().length === 0, - }); - await this.placeCallWithCallFeeds([callFeed]); - } catch (e) { - this.getUserMediaFailed(e); - return; - } - } - - /** - * Place a call to this room with call feed. - * @param {CallFeed[]} callFeeds to use - * @throws if you have not specified a listener for 'error' events. - * @throws if have passed audio=false. - */ - public async placeCallWithCallFeeds(callFeeds: CallFeed[], requestScreenshareFeed = false): Promise { - this.checkForErrorListener(); - this.direction = CallDirection.Outbound; - - // XXX Find a better way to do this - this.client.callEventHandler.calls.set(this.callId, this); - - // make sure we have valid turn creds. Unless something's gone wrong, it should - // poll and keep the credentials valid so this should be instant. - const haveTurnCreds = await this.client.checkTurnServers(); - if (!haveTurnCreds) { - logger.warn("Failed to get TURN credentials! Proceeding with call anyway..."); - } - - // create the peer connection now so it can be gathering candidates while we get user - // media (assuming a candidate pool size is configured) - this.peerConn = this.createPeerConnection(); - this.gotCallFeedsForInvite(callFeeds, requestScreenshareFeed); - } - - private createPeerConnection(): RTCPeerConnection { - const pc = new window.RTCPeerConnection({ - iceTransportPolicy: this.forceTURN ? 'relay' : undefined, - iceServers: this.turnServers, - iceCandidatePoolSize: this.client.iceCandidatePoolSize, - }); - - // 'connectionstatechange' would be better, but firefox doesn't implement that. - pc.addEventListener('iceconnectionstatechange', this.onIceConnectionStateChanged); - pc.addEventListener('signalingstatechange', this.onSignallingStateChanged); - pc.addEventListener('icecandidate', this.gotLocalIceCandidate); - pc.addEventListener('icegatheringstatechange', this.onIceGatheringStateChange); - pc.addEventListener('track', this.onTrack); - pc.addEventListener('negotiationneeded', this.onNegotiationNeeded); - pc.addEventListener('datachannel', this.onDataChannel); - - return pc; - } - - private partyIdMatches(msg: MCallBase): boolean { - // They must either match or both be absent (in which case opponentPartyId will be null) - // Also we ignore party IDs on the invite/offer if the version is 0, so we must do the same - // here and use null if the version is 0 (woe betide any opponent sending messages in the - // same call with different versions) - const msgPartyId = msg.version === 0 ? null : msg.party_id || null; - return msgPartyId === this.opponentPartyId; - } - - // Commits to an opponent for the call - // ev: An invite or answer event - private chooseOpponent(ev: MatrixEvent): void { - // I choo-choo-choose you - const msg = ev.getContent(); - - logger.debug(`Choosing party ID ${msg.party_id} for call ID ${this.callId}`); - - this.opponentVersion = msg.version; - if (this.opponentVersion === 0) { - // set to null to indicate that we've chosen an opponent, but because - // they're v0 they have no party ID (even if they sent one, we're ignoring it) - this.opponentPartyId = null; - } else { - // set to their party ID, or if they're naughty and didn't send one despite - // not being v0, set it to null to indicate we picked an opponent with no - // party ID - this.opponentPartyId = msg.party_id || null; - } - this.opponentCaps = msg.capabilities || {} as CallCapabilities; - this.opponentMember = this.client.getRoom(this.roomId).getMember(ev.getSender()); - } - - private async addBufferedIceCandidates(): Promise { - const bufferedCandidates = this.remoteCandidateBuffer.get(this.opponentPartyId); - if (bufferedCandidates) { - logger.info(`Adding ${bufferedCandidates.length} buffered candidates for opponent ${this.opponentPartyId}`); - await this.addIceCandidates(bufferedCandidates); - } - this.remoteCandidateBuffer = null; - } - - private async addIceCandidates(candidates: RTCIceCandidate[]): Promise { - for (const candidate of candidates) { - if ( - (candidate.sdpMid === null || candidate.sdpMid === undefined) && - (candidate.sdpMLineIndex === null || candidate.sdpMLineIndex === undefined) - ) { - logger.debug("Ignoring remote ICE candidate with no sdpMid or sdpMLineIndex"); - continue; - } - logger.debug( - "Call " + this.callId + " got remote ICE " + candidate.sdpMid + " candidate: " + candidate.candidate, - ); - try { - await this.peerConn.addIceCandidate(candidate); - } catch (err) { - if (!this.ignoreOffer) { - logger.info("Failed to add remote ICE candidate", err); - } - } - } - } - - public get hasPeerConnection(): boolean { - return Boolean(this.peerConn); - } -} - -export function setTracksEnabled(tracks: Array, enabled: boolean): void { - for (let i = 0; i < tracks.length; i++) { - tracks[i].enabled = enabled; - } -} - -/** - * DEPRECATED - * Use client.createCall() - * - * Create a new Matrix call for the browser. - * @param {MatrixClient} client The client instance to use. - * @param {string} roomId The room the call is in. - * @param {Object?} options DEPRECATED optional options map. - * @param {boolean} options.forceTURN DEPRECATED whether relay through TURN should be - * forced. This option is deprecated - use opts.forceTURN when creating the matrix client - * since it's only possible to set this option on outbound calls. - * @return {MatrixCall} the call or null if the browser doesn't support calling. - */ -export function createNewMatrixCall(client: any, roomId: string, options?: CallOpts): MatrixCall { - // typeof prevents Node from erroring on an undefined reference - if (typeof(window) === 'undefined' || typeof(document) === 'undefined') { - // NB. We don't log here as apps try to create a call object as a test for - // whether calls are supported, so we shouldn't fill the logs up. - return null; - } - - // Firefox throws on so little as accessing the RTCPeerConnection when operating in - // a secure mode. There's some information at https://bugzilla.mozilla.org/show_bug.cgi?id=1542616 - // though the concern is that the browser throwing a SecurityError will brick the - // client creation process. - try { - const supported = Boolean( - window.RTCPeerConnection || window.RTCSessionDescription || - window.RTCIceCandidate || navigator.mediaDevices, - ); - if (!supported) { - // Adds a lot of noise to test runs, so disable logging there. - if (process.env.NODE_ENV !== "test") { - logger.error("WebRTC is not supported in this browser / environment"); - } - return null; - } - } catch (e) { - logger.error("Exception thrown when trying to access WebRTC", e); - return null; - } - - const optionsForceTURN = options ? options.forceTURN : false; - - const opts: CallOpts = { - client: client, - roomId: roomId, - invitee: options?.invitee, - turnServers: client.getTurnServers(), - // call level options - forceTURN: client.forceTURN || optionsForceTURN, - opponentDeviceId: options?.opponentDeviceId, - opponentSessionId: options?.opponentSessionId, - groupCallId: options?.groupCallId, - }; - const call = new MatrixCall(opts); - - client.reEmitter.reEmit(call, Object.values(CallEvent)); - - return call; -} diff --git a/src/matrix/calls/CallFeed.ts b/src/matrix/calls/CallFeed.ts deleted file mode 100644 index c8cc8662..00000000 --- a/src/matrix/calls/CallFeed.ts +++ /dev/null @@ -1,274 +0,0 @@ -/* -Copyright 2021 Å imon Brandner - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { SDPStreamMetadataPurpose } from "./callEventTypes"; - -const POLLING_INTERVAL = 200; // ms -export const SPEAKING_THRESHOLD = -60; // dB -const SPEAKING_SAMPLE_COUNT = 8; // samples - -export interface ICallFeedOpts { - client: MatrixClient; - roomId: string; - userId: string; - stream: MediaStream; - purpose: SDPStreamMetadataPurpose; - audioMuted: boolean; - videoMuted: boolean; -} - -export enum CallFeedEvent { - NewStream = "new_stream", - MuteStateChanged = "mute_state_changed", - VolumeChanged = "volume_changed", - Speaking = "speaking", -} - -export class CallFeed extends EventEmitter { - public stream: MediaStream; - public sdpMetadataStreamId: string; - public userId: string; - public purpose: SDPStreamMetadataPurpose; - public speakingVolumeSamples: number[]; - - private client: MatrixClient; - private roomId: string; - private audioMuted: boolean; - private videoMuted: boolean; - private measuringVolumeActivity = false; - private audioContext: AudioContext; - private analyser: AnalyserNode; - private frequencyBinCount: Float32Array; - private speakingThreshold = SPEAKING_THRESHOLD; - private speaking = false; - private volumeLooperTimeout: number; - - constructor(opts: ICallFeedOpts) { - super(); - - this.client = opts.client; - this.roomId = opts.roomId; - this.userId = opts.userId; - this.purpose = opts.purpose; - this.audioMuted = opts.audioMuted; - this.videoMuted = opts.videoMuted; - this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); - this.sdpMetadataStreamId = opts.stream.id; - - this.updateStream(null, opts.stream); - - if (this.hasAudioTrack) { - this.initVolumeMeasuring(); - } - } - - private get hasAudioTrack(): boolean { - return this.stream.getAudioTracks().length > 0; - } - - private updateStream(oldStream: MediaStream, newStream: MediaStream): void { - if (newStream === oldStream) return; - - if (oldStream) { - oldStream.removeEventListener("addtrack", this.onAddTrack); - this.measureVolumeActivity(false); - } - if (newStream) { - this.stream = newStream; - newStream.addEventListener("addtrack", this.onAddTrack); - - if (this.hasAudioTrack) { - this.initVolumeMeasuring(); - } else { - this.measureVolumeActivity(false); - } - } - - this.emit(CallFeedEvent.NewStream, this.stream); - } - - private initVolumeMeasuring(): void { - const AudioContext = window.AudioContext || window.webkitAudioContext; - if (!this.hasAudioTrack || !AudioContext) return; - - this.audioContext = new AudioContext(); - - this.analyser = this.audioContext.createAnalyser(); - this.analyser.fftSize = 512; - this.analyser.smoothingTimeConstant = 0.1; - - const mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.stream); - mediaStreamAudioSourceNode.connect(this.analyser); - - this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount); - } - - private onAddTrack = (): void => { - this.emit(CallFeedEvent.NewStream, this.stream); - }; - - /** - * Returns callRoom member - * @returns member of the callRoom - */ - public getMember(): RoomMember { - const callRoom = this.client.getRoom(this.roomId); - return callRoom.getMember(this.userId); - } - - /** - * Returns true if CallFeed is local, otherwise returns false - * @returns {boolean} is local? - */ - public isLocal(): boolean { - return this.userId === this.client.getUserId(); - } - - /** - * Returns true if audio is muted or if there are no audio - * tracks, otherwise returns false - * @returns {boolean} is audio muted? - */ - public isAudioMuted(): boolean { - return this.stream.getAudioTracks().length === 0 || this.audioMuted; - } - - /** - * Returns true video is muted or if there are no video - * tracks, otherwise returns false - * @returns {boolean} is video muted? - */ - public isVideoMuted(): boolean { - // We assume only one video track - return this.stream.getVideoTracks().length === 0 || this.videoMuted; - } - - public isSpeaking(): boolean { - return this.speaking; - } - - /** - * Replaces the current MediaStream with a new one. - * This method should be only used by MatrixCall. - * @param newStream new stream with which to replace the current one - */ - public setNewStream(newStream: MediaStream): void { - this.updateStream(this.stream, newStream); - } - - /** - * Set feed's internal audio mute state - * @param muted is the feed's audio muted? - */ - public setAudioMuted(muted: boolean): void { - this.audioMuted = muted; - this.speakingVolumeSamples.fill(-Infinity); - this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); - } - - /** - * Set feed's internal video mute state - * @param muted is the feed's video muted? - */ - public setVideoMuted(muted: boolean): void { - this.videoMuted = muted; - this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); - } - - /** - * Starts emitting volume_changed events where the emitter value is in decibels - * @param enabled emit volume changes - */ - public measureVolumeActivity(enabled: boolean): void { - if (enabled) { - if (!this.audioContext || !this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return; - - this.measuringVolumeActivity = true; - this.volumeLooper(); - } else { - this.measuringVolumeActivity = false; - this.speakingVolumeSamples.fill(-Infinity); - this.emit(CallFeedEvent.VolumeChanged, -Infinity); - } - } - - public setSpeakingThreshold(threshold: number) { - this.speakingThreshold = threshold; - } - - private volumeLooper = () => { - if (!this.analyser) return; - - if (!this.measuringVolumeActivity) return; - - this.analyser.getFloatFrequencyData(this.frequencyBinCount); - - let maxVolume = -Infinity; - for (let i = 0; i < this.frequencyBinCount.length; i++) { - if (this.frequencyBinCount[i] > maxVolume) { - maxVolume = this.frequencyBinCount[i]; - } - } - - this.speakingVolumeSamples.shift(); - this.speakingVolumeSamples.push(maxVolume); - - this.emit(CallFeedEvent.VolumeChanged, maxVolume); - - let newSpeaking = false; - - for (let i = 0; i < this.speakingVolumeSamples.length; i++) { - const volume = this.speakingVolumeSamples[i]; - - if (volume > this.speakingThreshold) { - newSpeaking = true; - break; - } - } - - if (this.speaking !== newSpeaking) { - this.speaking = newSpeaking; - this.emit(CallFeedEvent.Speaking, this.speaking); - } - - this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL); - }; - - public clone(): CallFeed { - const mediaHandler = this.client.getMediaHandler(); - const stream = this.stream.clone(); - - if (this.purpose === SDPStreamMetadataPurpose.Usermedia) { - mediaHandler.userMediaStreams.push(stream); - } else { - mediaHandler.screensharingStreams.push(stream); - } - - return new CallFeed({ - client: this.client, - roomId: this.roomId, - userId: this.userId, - stream, - purpose: this.purpose, - audioMuted: this.audioMuted, - videoMuted: this.videoMuted, - }); - } - - public dispose(): void { - clearTimeout(this.volumeLooperTimeout); - } -} diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 1dad9ce8..d7d673ed 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -22,6 +22,8 @@ import type {ILogItem} from "../../logging/types"; import {WebRTC, PeerConnection, PeerConnectionHandler, StreamPurpose} from "../../platform/types/WebRTC"; import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; +import type {SignallingMessage} from "./PeerCall"; +import type {MGroupCallBase} from "./callEventTypes"; const GROUP_CALL_TYPE = "m.call"; const GROUP_CALL_MEMBER_TYPE = "m.call.member"; @@ -33,7 +35,7 @@ enum CallSetupMessageType { Hangup = "m.call.hangup", } -const CALL_ID = "m.call_id"; +const CONF_ID = "conf_id"; const CALL_TERMINATED = "m.terminated"; export class GroupCallHandler { @@ -69,7 +71,7 @@ export class GroupCallHandler { const participant = event.state_key; const sources = event.content["m.sources"]; for (const source of sources) { - const call = this.calls.get(source[CALL_ID]); + const call = this.calls.get(source[CONF_ID]); if (call && !call.isTerminated) { call.addParticipant(participant, source); } @@ -85,110 +87,9 @@ export class GroupCallHandler { eventType === CallSetupMessageType.Hangup; } - handleDeviceMessage(senderUserId: string, senderDeviceId: string, eventType: string, content: Record, log: ILogItem) { - const callId = content[CALL_ID]; - const call = this.calls.get(callId); - call?.handleDeviceMessage(senderUserId, senderDeviceId, eventType, content, log); + handleDeviceMessage(senderUserId: string, senderDeviceId: string, event: SignallingMessage, log: ILogItem) { + const call = this.calls.get(event.content.conf_id); + call?.handleDeviceMessage(senderUserId, senderDeviceId, event, log); } } -function participantId(senderUserId: string, senderDeviceId: string | null) { - return JSON.stringify(senderUserId) + JSON.stringify(senderDeviceId); -} - -class GroupParticipant implements PeerCallHandler { - private peerCall?: PeerCall; - - constructor( - private readonly userId: string, - private readonly deviceId: string, - private localMedia: LocalMedia | undefined, - private readonly webRTC: WebRTC, - private readonly hsApi: HomeServerApi - ) {} - - sendInvite() { - this.peerCall = new PeerCall(this, this.webRTC); - this.peerCall.call(this.localMedia); - } - - /** From PeerCallHandler - * @internal */ - override emitUpdate() { - - } - - /** From PeerCallHandler - * @internal */ - override onSendSignallingMessage() { - // TODO: this needs to be encrypted with olm first - this.hsApi.sendToDevice(type, {[this.userId]: {[this.deviceId ?? "*"]: content}}); - } -} - -class GroupCall { - private readonly participants: ObservableMap = new ObservableMap(); - private localMedia?: LocalMedia; - - constructor(private readonly ownUserId: string, private callEvent: StateEvent, private readonly room: Room, private readonly webRTC: WebRTC) { - - } - - get id(): string { return this.callEvent.state_key; } - - async participate(tracks: Track[]) { - this.localMedia = LocalMedia.fromTracks(tracks); - for (const [,participant] of this.participants) { - participant.setMedia(this.localMedia.clone()); - } - // send m.call.member state event - - // send invite to all participants that are < my userId - for (const [,participant] of this.participants) { - if (participant.userId < this.ownUserId) { - participant.sendInvite(); - } - } - } - - updateCallEvent(callEvent: StateEvent) { - this.callEvent = callEvent; - } - - addParticipant(userId, source) { - const participantId = getParticipantId(userId, source.device_id); - const participant = this.participants.get(participantId); - if (participant) { - participant.updateSource(source); - } else { - participant.add(participantId, new GroupParticipant(userId, source.device_id, this.localMedia?.clone(), this.webRTC)); - } - } - - handleDeviceMessage(senderUserId: string, senderDeviceId: string, eventType: string, content: Record, log: ILogItem) { - const participantId = getParticipantId(senderUserId, senderDeviceId); - let peerCall = this.participants.get(participantId); - let hasDeviceInKey = true; - if (!peerCall) { - hasDeviceInKey = false; - peerCall = this.participants.get(getParticipantId(senderUserId, null)) - } - if (peerCall) { - peerCall.handleIncomingSignallingMessage(eventType, content, senderDeviceId); - if (!hasDeviceInKey && peerCall.opponentPartyId) { - this.participants.delete(getParticipantId(senderUserId, null)); - this.participants.add(getParticipantId(senderUserId, peerCall.opponentPartyId)); - } - } else { - // create peerCall - } - } - - get id(): string { - return this.callEvent.state_key; - } - - get isTerminated(): boolean { - return !!this.callEvent.content[CALL_TERMINATED]; - } -} diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index a02de99f..5040c805 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -17,26 +17,41 @@ limitations under the License. import {ObservableMap} from "../../observable/map/ObservableMap"; import {recursivelyAssign} from "../../utils/recursivelyAssign"; import {AsyncQueue} from "../../utils/AsyncQueue"; -import {Disposables, Disposable} from "../../utils/Disposables"; +import {Disposables, IDisposable} from "../../utils/Disposables"; import type {Room} from "../room/Room"; import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; import type {TimeoutCreator, Timeout} from "../../platform/types/types"; -import {WebRTC, PeerConnection, PeerConnectionHandler} from "../../platform/types/WebRTC"; +import {WebRTC, PeerConnection, PeerConnectionHandler, DataChannel} from "../../platform/types/WebRTC"; import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; import type {LocalMedia} from "./LocalMedia"; +import { + SDPStreamMetadataKey, + SDPStreamMetadataPurpose +} from "./callEventTypes"; +import type { + MCallBase, + MCallInvite, + MCallAnswer, + MCallSDPStreamMetadataChanged, + MCallCandidates, + MCallHangupReject, + SDPStreamMetadata, +} from "./callEventTypes"; + // when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already // do for sharing keys will be best as that already deals with room tracking. /** * Does WebRTC signalling for a single PeerConnection, and deals with WebRTC wrappers from platform * */ /** Implements a call between two peers with the signalling state keeping, while still delegating the signalling message sending. Used by GroupCall.*/ -class PeerCall { +export class PeerCall implements IDisposable { private readonly peerConnection: PeerConnection; private state = CallState.Fledgling; private direction: CallDirection; + private localMedia?: LocalMedia; // A queue for candidates waiting to go out. // We try to amalgamate candidates into a single candidate message where // possible @@ -54,9 +69,13 @@ class PeerCall { private disposables = new Disposables(); private statePromiseMap = new Map void, promise: Promise}>(); + // perfect negotiation flags + private makingOffer: boolean = false; + private ignoreOffer: boolean = false; + constructor( + private callId: string, // generated or from invite private readonly handler: PeerCallHandler, - private localMedia: LocalMedia, private readonly createTimeout: TimeoutCreator, webRTC: WebRTC ) { @@ -83,29 +102,8 @@ class PeerCall { } } - handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId) { - switch (message.type) { - case EventType.Invite: - // determining whether or not an incoming invite glares - // with an instance of PeerCall is different for group calls - // and 1:1 calls, so done outside of this class. - // If you pass an event for another call id in here it will assume it glares. - - //const newCallId = message.content.call_id; - //if (this.id && newCallId !== this.id) { - // this.handleInviteGlare(message.content); - //} else { - this.handleInvite(message.content, partyId); - //} - break; - case EventType.Answer: - this.handleAnswer(message.content, partyId); - break; - case EventType.Candidates: - this.handleRemoteIceCandidates(message.content, partyId); - break; - case EventType.Hangup: - } + get remoteTracks(): Track[] { + return this.peerConnection.remoteTracks; } async call(localMediaPromise: Promise): Promise { @@ -125,7 +123,9 @@ class PeerCall { for (const t of this.localMedia.tracks) { this.peerConnection.addTrack(t); } - await this.waitForState(CallState.InviteSent); + // TODO: in case of glare, we would not go to InviteSent if we haven't started sending yet + // but we would go straight to CreateAnswer, so also need to wait for that state + await this.waitForState([CallState.InviteSent, CallState.CreateAnswer]); } async answer(localMediaPromise: Promise): Promise { @@ -166,15 +166,11 @@ class PeerCall { this.sendAnswer(); } - async hangup() { - - } - async setMedia(localMediaPromise: Promise) { const oldMedia = this.localMedia; this.localMedia = await localMediaPromise; - const applyTrack = (selectTrack: (media: LocalMedia) => Track | undefined) => { + const applyTrack = (selectTrack: (media: LocalMedia | undefined) => Track | undefined) => { const oldTrack = selectTrack(oldMedia); const newTrack = selectTrack(this.localMedia); if (oldTrack && newTrack) { @@ -187,50 +183,101 @@ class PeerCall { }; // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called - applyTrack(m => m.microphoneTrack); - applyTrack(m => m.cameraTrack); - applyTrack(m => m.screenShareTrack); + applyTrack(m => m?.microphoneTrack); + applyTrack(m => m?.cameraTrack); + applyTrack(m => m?.screenShareTrack); + } + + async reject() { + + } + + async hangup(errorCode: CallErrorCode) { + } + + async handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId): Promise { + switch (message.type) { + case EventType.Invite: + if (this.callId !== message.content.call_id) { + await this.handleInviteGlare(message.content, partyId); + } else { + await this.handleFirstInvite(message.content, partyId); + } + break; + case EventType.Answer: + await this.handleAnswer(message.content, partyId); + break; + //case EventType.Candidates: + // await this.handleRemoteIceCandidates(message.content, partyId); + // break; + case EventType.Hangup: + default: + throw new Error(`Unknown event type for call: ${message.type}`); + } + } + + private sendHangupWithCallId(callId: string, reason?: CallErrorCode): Promise { + const content = { + call_id: callId, + version: 1, + }; + if (reason) { + content["reason"] = reason; + } + return this.handler.sendSignallingMessage({ + type: EventType.Hangup, + content + }); } // calls are serialized and deduplicated by responsePromiseChain private handleNegotiation = async (): Promise => { - // TODO: does this make sense to have this state if we're already connected? - this.setState(CallState.MakingOffer) + this.makingOffer = true; try { - await this.peerConnection.setLocalDescription(); - } catch (err) { - this.logger.debug(`Call ${this.callId} Error setting local description!`, err); - this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); - return; + try { + await this.peerConnection.setLocalDescription(); + } catch (err) { + this.logger.debug(`Call ${this.callId} Error setting local description!`, err); + this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); + return; + } + + if (this.peerConnection.iceGatheringState === 'gathering') { + // Allow a short time for initial candidates to be gathered + await this.delay(200); + } + + if (this.state === CallState.Ended) { + return; + } + + const offer = this.peerConnection.localDescription!; + // Get rid of any candidates waiting to be sent: they'll be included in the local + // description we just got and will send in the offer. + this.logger.info(`Call ${this.callId} Discarding ${ + this.candidateSendQueue.length} candidates that will be sent in offer`); + this.candidateSendQueue = []; + + // need to queue this + const content = { + call_id: this.callId, + offer, + [SDPStreamMetadataKey]: this.localMedia!.getSDPMetadata(), + version: 1, + lifetime: CALL_TIMEOUT_MS + }; + if (this.state === CallState.CreateOffer) { + await this.handler.sendSignallingMessage({type: EventType.Invite, content}); + this.setState(CallState.InviteSent); + } else if (this.state === CallState.Connected || this.state === CallState.Connecting) { + // send Negotiate message + //await this.handler.sendSignallingMessage({type: EventType.Invite, content}); + //this.setState(CallState.InviteSent); + } + } finally { + this.makingOffer = false; } - if (this.peerConnection.iceGatheringState === 'gathering') { - // Allow a short time for initial candidates to be gathered - await this.delay(200); - } - - if (this.state === CallState.Ended) { - return; - } - - const offer = this.peerConnection.localDescription!; - // Get rid of any candidates waiting to be sent: they'll be included in the local - // description we just got and will send in the offer. - this.logger.info(`Call ${this.callId} Discarding ${ - this.candidateSendQueue.length} candidates that will be sent in offer`); - this.candidateSendQueue = []; - - // need to queue this - const content = { - offer, - [SDPStreamMetadataKey]: this.localMedia.getSDPMetadata(), - version: 1, - lifetime: CALL_TIMEOUT_MS - }; - if (this.state === CallState.CreateOffer) { - await this.handler.sendSignallingMessage({type: EventType.Invite, content}); - this.setState(CallState.InviteSent); - } this.sendCandidateQueue(); if (this.state === CallState.InviteSent) { @@ -242,11 +289,47 @@ class PeerCall { } }; - private async handleInvite(content: InviteContent, partyId: PartyId): Promise { + private async handleInviteGlare(content: MCallInvite, partyId: PartyId): Promise { + // this is only called when the ids are different + const newCallId = content.call_id; + if (this.callId! > newCallId) { + this.logger.log( + "Glare detected: answering incoming call " + newCallId + + " and canceling outgoing call " + this.callId, + ); + + /* + first, we should set CallDirection + we should anser the call + */ + + // TODO: review states to be unambigous, WaitLocalMedia for sending offer or answer? + // How do we interrupt `call()`? well, perhaps we need to not just await InviteSent but also CreateAnswer? + if (this.state === CallState.Fledgling || this.state === CallState.CreateOffer || this.state === CallState.WaitLocalMedia) { + + } else { + await this.sendHangupWithCallId(this.callId, CallErrorCode.Replaced); + } + await this.handleInvite(content, partyId); + await this.answer(Promise.resolve(this.localMedia!)); + } else { + this.logger.log( + "Glare detected: rejecting incoming call " + newCallId + + " and keeping outgoing call " + this.callId, + ); + await this.sendHangupWithCallId(newCallId, CallErrorCode.Replaced); + } + } + + private async handleFirstInvite(content: MCallInvite, partyId: PartyId): Promise { if (this.state !== CallState.Fledgling || this.opponentPartyId !== undefined) { // TODO: hangup or ignore? return; } + await this.handleInvite(content, partyId); + } + + private async handleInvite(content: MCallInvite, partyId: PartyId): Promise { // we must set the party ID before await-ing on anything: the call event // handler will start giving us more call events (eg. candidates) so if @@ -296,8 +379,8 @@ class PeerCall { } } - private async handleAnswer(content: AnwserContent, partyId: PartyId): Promise { - this.logger.debug(`Got answer for call ID ${this.callId} from party ID ${content.party_id}`); + private async handleAnswer(content: MCallAnswer, partyId: PartyId): Promise { + this.logger.debug(`Got answer for call ID ${this.callId} from party ID ${partyId}`); if (this.state === CallState.Ended) { this.logger.debug(`Ignoring answer because call ID ${this.callId} has ended`); @@ -307,7 +390,7 @@ class PeerCall { if (this.opponentPartyId !== undefined) { this.logger.info( `Call ${this.callId} ` + - `Ignoring answer from party ID ${content.party_id}: ` + + `Ignoring answer from party ID ${partyId}: ` + `we already have an answer/reject from ${this.opponentPartyId}`, ); return; @@ -334,16 +417,66 @@ class PeerCall { } } + // private async onNegotiateReceived(event: MatrixEvent): Promise { + // const content = event.getContent(); + // const description = content.description; + // if (!description || !description.sdp || !description.type) { + // this.logger.info(`Call ${this.callId} Ignoring invalid m.call.negotiate event`); + // return; + // } + // // Politeness always follows the direction of the call: in a glare situation, + // // we pick either the inbound or outbound call, so one side will always be + // // inbound and one outbound + // const polite = this.direction === CallDirection.Inbound; + + // // Here we follow the perfect negotiation logic from + // // https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation + // const offerCollision = ( + // (description.type === 'offer') && + // (this.makingOffer || this.peerConnection.signalingState !== 'stable') + // ); + + // this.ignoreOffer = !polite && offerCollision; + // if (this.ignoreOffer) { + // this.logger.info(`Call ${this.callId} Ignoring colliding negotiate event because we're impolite`); + // return; + // } + + // const sdpStreamMetadata = content[SDPStreamMetadataKey]; + // if (sdpStreamMetadata) { + // this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); + // } else { + // this.logger.warn(`Call ${this.callId} Received negotiation event without SDPStreamMetadata!`); + // } + + // try { + // await this.peerConnection.setRemoteDescription(description); + + // if (description.type === 'offer') { + // await this.peerConnection.setLocalDescription(); + // await this.handler.sendSignallingMessage({ + // type: EventType.CallNegotiate, + // content: { + // description: this.peerConnection.localDescription!, + // [SDPStreamMetadataKey]: this.localMedia.getSDPMetadata(), + // } + // }); + // } + // } catch (err) { + // this.logger.warn(`Call ${this.callId} Failed to complete negotiation`, err); + // } + // } + private async sendAnswer(): Promise { - const answerMessage: AnswerMessage = { - type: EventType.Answer, - content: { - answer: { - sdp: this.peerConnection.localDescription!.sdp, - type: this.peerConnection.localDescription!.type, - }, - [SDPStreamMetadataKey]: this.localMedia.getSDPMetadata(), - } + const localDescription = this.peerConnection.localDescription!; + const answerContent: MCallAnswer = { + call_id: this.callId, + version: 1, + answer: { + sdp: localDescription.sdp, + type: localDescription.type, + }, + [SDPStreamMetadataKey]: this.localMedia!.getSDPMetadata(), }; // We have just taken the local description from the peerConn which will @@ -354,7 +487,7 @@ class PeerCall { this.candidateSendQueue = []; try { - await this.handler.sendSignallingMessage(answerMessage); + await this.handler.sendSignallingMessage({type: EventType.Answer, content: answerContent}); } catch (error) { this.terminate(CallParty.Local, CallErrorCode.SendAnswer, false); throw error; @@ -387,7 +520,6 @@ class PeerCall { }); } - private async sendCandidateQueue(): Promise { if (this.candidateSendQueue.length === 0 || this.state === CallState.Ended) { return; @@ -395,15 +527,16 @@ class PeerCall { const candidates = this.candidateSendQueue; this.candidateSendQueue = []; - const candidatesMessage: CandidatesMessage = { - type: EventType.Candidates, - content: { - candidates: candidates, - } - }; this.logger.debug(`Call ${this.callId} attempting to send ${candidates.length} candidates`); try { - await this.handler.sendSignallingMessage(candidatesMessage); + await this.handler.sendSignallingMessage({ + type: EventType.Candidates, + content: { + call_id: this.callId, + version: 1, + candidates + } + }); // Try to send candidates again just in case we received more candidates while sending. this.sendCandidateQueue(); } catch (error) { @@ -430,7 +563,6 @@ class PeerCall { } } - private async addBufferedIceCandidates(): Promise { if (this.remoteCandidateBuffer && this.opponentPartyId) { const bufferedCandidates = this.remoteCandidateBuffer.get(this.opponentPartyId); @@ -463,7 +595,6 @@ class PeerCall { } } - private setState(state: CallState): void { const oldState = this.state; this.state = state; @@ -475,17 +606,20 @@ class PeerCall { this.handler.emitUpdate(this, undefined); } - private waitForState(state: CallState): Promise { - let deferred = this.statePromiseMap.get(state); - if (!deferred) { - let resolve; - const promise = new Promise(r => { - resolve = r; - }); - deferred = {resolve, promise}; - this.statePromiseMap.set(state, deferred); - } - return deferred.promise; + private waitForState(states: CallState[]): Promise { + // TODO: rework this, do we need to clean up the promises? + return Promise.race(states.map(state => { + let deferred = this.statePromiseMap.get(state); + if (!deferred) { + let resolve; + const promise = new Promise(r => { + resolve = r; + }); + deferred = {resolve, promise}; + this.statePromiseMap.set(state, deferred); + } + return deferred.promise; + })); } private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean): Promise { @@ -493,8 +627,10 @@ class PeerCall { } private stopAllMedia(): void { - for (const track of this.localMedia.tracks) { - track.stop(); + if (this.localMedia) { + for (const track of this.localMedia.tracks) { + track.stop(); + } } } @@ -514,21 +650,6 @@ class PeerCall { //import { randomString } from '../randomstring'; -import { - MCallReplacesEvent, - MCallAnswer, - MCallInviteNegotiate, - CallCapabilities, - SDPStreamMetadataPurpose, - SDPStreamMetadata, - SDPStreamMetadataKey, - MCallSDPStreamMetadataChanged, - MCallSelectAnswer, - MCAllAssertedIdentity, - MCallCandidates, - MCallBase, - MCallHangupReject, -} from './callEventTypes'; // null is used as a special value meaning that the we're in a legacy 1:1 call // without MSC2746 that doesn't provide an id which device sent the message. @@ -681,46 +802,18 @@ export class CallError extends Error { } } -type InviteContent = { - offer: RTCSessionDescriptionInit, - [SDPStreamMetadataKey]: SDPStreamMetadata, - version?: number, - lifetime?: number -} - -export type InviteMessage = { - type: EventType.Invite, - content: InviteContent -} - -type AnwserContent = { - answer: { - sdp: string, - // type is now deprecated as of Matrix VoIP v1, but - // required to still be sent for backwards compat - type: RTCSdpType, - }, - [SDPStreamMetadataKey]: SDPStreamMetadata, -} - -export type AnswerMessage = { - type: EventType.Answer, - content: AnwserContent -} - -type CandidatesContent = { - candidates: RTCIceCandidate[] -} - -export type CandidatesMessage = { - type: EventType.Candidates, - content: CandidatesContent -} - - -export type SignallingMessage = InviteMessage | AnswerMessage | CandidatesMessage; +export type SignallingMessage = + {type: EventType.Invite, content: MCallInvite} | + {type: EventType.Answer, content: MCallAnswer} | + {type: EventType.SDPStreamMetadataChanged | EventType.SDPStreamMetadataChangedPrefix, content: MCallSDPStreamMetadataChanged} | + {type: EventType.Candidates, content: MCallCandidates} | + {type: EventType.Hangup | EventType.Reject, content: MCallHangupReject}; export interface PeerCallHandler { emitUpdate(peerCall: PeerCall, params: any); - sendSignallingMessage(message: SignallingMessage); + sendSignallingMessage(message: SignallingMessage); +} + +export function tests() { + } diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index da2b1ad6..a8fbeafd 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -130,57 +130,34 @@ write view I think we need to synchronize the negotiation needed because we don't use a CallState to guard it... - - ## Thursday 3-3 notes we probably best keep the perfect negotiation flags, as they are needed for both starting the call AND renegotiation? if only for the former, it would make sense as it is a step in setting up the call, but if the call is ongoing, does it make sense to have a MakingOffer state? it actually looks like they are only needed for renegotiation! for call setup we compare the call_ids. What does that mean for these flags? -List state transitions +## Peer call state transitions FROM CALLER FROM CALLEE Fledgling Fledgling - V calling `call()` V handleInvite + V `call()` V `handleInvite()`: setRemoteDescription(event.offer), add buffered candidates WaitLocalMedia Ringing - V media promise resolves V answer() -CreateOffer WaitLocalMedia - V add tracks V media promise resolves - V wait for negotionneeded events CreateAnswer - V setLocalDescription() V - V send invite events -InviteSent - V receive anwser, setRemoteDescription() | - \__________________________________________________/ + V media promise resolves V `answer()` + V add local tracks WaitLocalMedia +CreateOffer V media promise resolves + V wait for negotionneeded events V add local tracks + V setLocalDescription() CreateAnswer + V send invite event V setLocalDescription(createAnswer()) +InviteSent | + V receive anwser, setRemoteDescription() | + \___________________________________________________/ V Connecting - V receive ice candidates and - iceConnectionState becomes 'connected' + V receive ice candidates and iceConnectionState becomes 'connected' Connected - V hangup for some reason + V `hangup()` or some terminate condition Ended -## From callee - -Fledgling -Ringing -WaitLocalMedia -CreateAnswer -Connecting -Connected -Ended - -Fledgling -WaitLocalMedia -CreateOffer -InviteSent -CreateAnswer -Connecting -Connected -Ringing -Ended - so if we don't want to bother with having two call objects, we can make the existing call hangup his old call_id? That way we keep the old peerConnection. diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index cd0fbb9f..aa1bc079 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -1,11 +1,14 @@ // allow non-camelcase as these are events type that go onto the wire /* eslint-disable camelcase */ -import { CallErrorCode } from "./Call"; - // TODO: Change to "sdp_stream_metadata" when MSC3077 is merged export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata"; +export interface SessionDescription { + sdp?: string; + type: RTCSdpType +} + export enum SDPStreamMetadataPurpose { Usermedia = "m.usermedia", Screenshare = "m.screenshare", @@ -32,40 +35,36 @@ export interface CallReplacesTarget { avatar_url: string; } -export interface MCallBase { +export type MCallBase = { call_id: string; version: string | number; - party_id?: string; - sender_session_id?: string; - dest_session_id?: string; } -export interface MCallAnswer extends MCallBase { - answer: RTCSessionDescription; +export type MGroupCallBase = MCallBase & { + conf_id: string; +} + +export type MCallAnswer = Base & { + answer: SessionDescription; capabilities?: CallCapabilities; [SDPStreamMetadataKey]: SDPStreamMetadata; } -export interface MCallSelectAnswer extends MCallBase { +export type MCallSelectAnswer = Base & { selected_party_id: string; } -export interface MCallInviteNegotiate extends MCallBase { - offer: RTCSessionDescription; - description: RTCSessionDescription; +export type MCallInvite = Base & { + offer: SessionDescription; lifetime: number; - capabilities?: CallCapabilities; - invitee?: string; - sender_session_id?: string; - dest_session_id?: string; [SDPStreamMetadataKey]: SDPStreamMetadata; } -export interface MCallSDPStreamMetadataChanged extends MCallBase { +export type MCallSDPStreamMetadataChanged = Base & { [SDPStreamMetadataKey]: SDPStreamMetadata; } -export interface MCallReplacesEvent extends MCallBase { +export type MCallReplacesEvent = Base & { replacement_id: string; target_user: CallReplacesTarget; create_call: string; @@ -73,7 +72,7 @@ export interface MCallReplacesEvent extends MCallBase { target_room: string; } -export interface MCAllAssertedIdentity extends MCallBase { +export type MCAllAssertedIdentity = Base & { asserted_identity: { id: string; display_name: string; @@ -81,11 +80,11 @@ export interface MCAllAssertedIdentity extends MCallBase { }; } -export interface MCallCandidates extends MCallBase { +export type MCallCandidates = Base & { candidates: RTCIceCandidate[]; } -export interface MCallHangupReject extends MCallBase { +export type MCallHangupReject = Base & { reason?: CallErrorCode; } diff --git a/src/matrix/calls/group/Call.ts b/src/matrix/calls/group/GroupCall.ts similarity index 83% rename from src/matrix/calls/group/Call.ts rename to src/matrix/calls/group/GroupCall.ts index 9abef197..e05f572d 100644 --- a/src/matrix/calls/group/Call.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -15,14 +15,17 @@ limitations under the License. */ import {ObservableMap} from "../../../observable/map/ObservableMap"; +import {Participant} from "./Participant"; +import {LocalMedia} from "../LocalMedia"; +import type {Track} from "../../../platform/types/MediaDevices"; -function participantId(senderUserId: string, senderDeviceId: string | null) { +function getParticipantId(senderUserId: string, senderDeviceId: string | null) { return JSON.stringify(senderUserId) + JSON.stringify(senderDeviceId); } -class Call { +export class GroupCall { private readonly participants: ObservableMap = new ObservableMap(); - private localMedia?: LocalMedia; + private localMedia?: Promise; constructor(private readonly ownUserId: string, private callEvent: StateEvent, private readonly room: Room, private readonly webRTC: WebRTC) { @@ -30,17 +33,17 @@ class Call { get id(): string { return this.callEvent.state_key; } - async participate(tracks: Track[]) { - this.localMedia = LocalMedia.fromTracks(tracks); + async participate(tracks: Promise) { + this.localMedia = tracks.then(tracks => LocalMedia.fromTracks(tracks)); for (const [,participant] of this.participants) { - participant.setLocalMedia(this.localMedia.clone()); + participant.setLocalMedia(this.localMedia.then(localMedia => localMedia.clone())); } // send m.call.member state event // send invite to all participants that are < my userId for (const [,participant] of this.participants) { if (participant.userId < this.ownUserId) { - participant.sendInvite(); + participant.call(); } } } @@ -78,10 +81,6 @@ class Call { } } - get id(): string { - return this.callEvent.state_key; - } - get isTerminated(): boolean { return !!this.callEvent.content[CALL_TERMINATED]; } diff --git a/src/matrix/calls/group/Participant.ts b/src/matrix/calls/group/Participant.ts index 26747e56..2b873aa0 100644 --- a/src/matrix/calls/group/Participant.ts +++ b/src/matrix/calls/group/Participant.ts @@ -14,35 +14,54 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {EventType} from "../PeerCall"; +import {EventType, PeerCall, SignallingMessage} from "../PeerCall"; +import {makeTxnId} from "../../common"; + import type {PeerCallHandler} from "../PeerCall"; +import type {LocalMedia} from "../LocalMedia"; +import type {HomeServerApi} from "../../net/HomeServerApi"; +import type {Track} from "../../../platform/types/MediaDevices"; +import type {MCallBase, MGroupCallBase} from "../callEventTypes"; +import type {GroupCall} from "./GroupCall"; +import type {RoomMember} from "../../room/members/RoomMember"; -class Participant implements PeerCallHandler { - private peerCall?: PeerCall; - +export class Participant implements PeerCallHandler { constructor( - private readonly userId: string, - private readonly deviceId: string, - private localMedia: LocalMedia | undefined, - private readonly webRTC: WebRTC, - private readonly hsApi: HomeServerApi + public readonly member: RoomMember, + private readonly deviceId: string | undefined, + private readonly peerCall: PeerCall, + private readonly hsApi: HomeServerApi, + private readonly groupCall: GroupCall ) {} - sendInvite() { - this.peerCall = new PeerCall(this, this.webRTC); - this.peerCall.call(this.localMedia); + /* @internal */ + call(localMedia: Promise) { + this.peerCall.call(localMedia); + } + + get remoteTracks(): Track[] { + return this.peerCall.remoteTracks; } /** From PeerCallHandler * @internal */ emitUpdate(params: any) { - + this.groupCall.emitParticipantUpdate(this, params); } /** From PeerCallHandler * @internal */ - onSendSignallingMessage(type: EventType, content: Record) { + async sendSignallingMessage(message: SignallingMessage) { + const groupMessage = message as SignallingMessage; + groupMessage.content.conf_id = this.groupCall.id; // TODO: this needs to be encrypted with olm first - this.hsApi.sendToDevice(type, {[this.userId]: {[this.deviceId ?? "*"]: content}}); + + const request = this.hsApi.sendToDevice( + groupMessage.type, + {[this.member.userId]: { + [this.deviceId ?? "*"]: groupMessage.content + } + }, makeTxnId()); + await request.response(); } } diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index e1991a1c..445ff22d 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -144,7 +144,7 @@ export class AudioTrackWrapper extends TrackWrapper { } else { this.measuringVolumeActivity = false; this.speakingVolumeSamples.fill(-Infinity); - this.emit(CallFeedEvent.VolumeChanged, -Infinity); + // this.emit(CallFeedEvent.VolumeChanged, -Infinity); } } @@ -186,7 +186,7 @@ export class AudioTrackWrapper extends TrackWrapper { this.speakingVolumeSamples.shift(); this.speakingVolumeSamples.push(maxVolume); - this.emit(CallFeedEvent.VolumeChanged, maxVolume); + // this.emit(CallFeedEvent.VolumeChanged, maxVolume); let newSpeaking = false; @@ -201,7 +201,7 @@ export class AudioTrackWrapper extends TrackWrapper { if (this.speaking !== newSpeaking) { this.speaking = newSpeaking; - this.emit(CallFeedEvent.Speaking, this.speaking); + // this.emit(CallFeedEvent.Speaking, this.speaking); } this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL) as unknown as number; diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 08c0d96d..0025dfec 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -114,7 +114,7 @@ class DOMPeerConnection implements PeerConnection { } createDataChannel(): DataChannel { - return new DataChannel(this.peerConnection.createDataChannel()); + return undefined as any;// new DataChannel(this.peerConnection.createDataChannel()); } private registerHandler() { diff --git a/yarn.lock b/yarn.lock index 7bcefdd4..0eb74b87 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1486,9 +1486,9 @@ type-fest@^0.20.2: integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ== typescript@^4.3.5: - version "4.3.5" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.3.5.tgz#4d1c37cc16e893973c45a06886b7113234f119f4" - integrity sha512-DqQgihaQ9cUrskJo9kIyW/+g0Vxsk8cDtZ52a3NGh0YNTfpUSArXSohyUGnvbPazEPLu398C0UxmKSOrPumUzA== + version "4.6.2" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.6.2.tgz#fe12d2727b708f4eef40f51598b3398baa9611d4" + integrity sha512-HM/hFigTBHZhLXshn9sN37H085+hQGeJHJ/X7LpBWLID/fbc2acUMfU+lGD98X81sKP+pFa9f0DZmCwB9GnbAg== typeson-registry@^1.0.0-alpha.20: version "1.0.0-alpha.39" From 4bedd4737b59e4455642e83c21f553c828933cc5 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 9 Mar 2022 18:53:51 +0100 Subject: [PATCH 012/323] WIP11 --- src/matrix/calls/CallHandler.ts | 105 ++++++++++++++++---------- src/matrix/calls/PeerCall.ts | 33 +++------ src/matrix/calls/callEventTypes.ts | 111 +++++++++++++++++++++++++++- src/matrix/calls/group/GroupCall.ts | 50 ++++++------- src/matrix/room/Room.js | 3 + 5 files changed, 213 insertions(+), 89 deletions(-) diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index d7d673ed..d84be9e3 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -17,33 +17,34 @@ limitations under the License. import {ObservableMap} from "../../observable/map/ObservableMap"; import type {Room} from "../room/Room"; +import type {MemberChange} from "../room/members/RoomMember"; import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; +import type {Platform} from "../../platform/web/Platform"; import {WebRTC, PeerConnection, PeerConnectionHandler, StreamPurpose} from "../../platform/types/WebRTC"; import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; -import type {SignallingMessage} from "./PeerCall"; -import type {MGroupCallBase} from "./callEventTypes"; +import {handlesEventType, PeerCall, PeerCallHandler} from "./PeerCall"; +import {EventType} from "./callEventTypes"; +import type {SignallingMessage, MGroupCallBase} from "./callEventTypes"; +import type {GroupCall} from "./group/GroupCall"; const GROUP_CALL_TYPE = "m.call"; const GROUP_CALL_MEMBER_TYPE = "m.call.member"; - -enum CallSetupMessageType { - Invite = "m.call.invite", - Answer = "m.call.answer", - Candidates = "m.call.candidates", - Hangup = "m.call.hangup", -} - -const CONF_ID = "conf_id"; const CALL_TERMINATED = "m.terminated"; export class GroupCallHandler { + + private createPeerCall: (callId: string, handler: PeerCallHandler) => PeerCall; // group calls by call id public readonly calls: ObservableMap = new ObservableMap(); + // map of userId to set of conf_id's they are in + private memberToCallIds: Map> = new Map(); - constructor() { - + constructor(hsApi: HomeServerApi, platform: Platform, ownUserId: string, ownDeviceId: string) { + this.createPeerCall = (callId: string, handler: PeerCallHandler) => { + return new PeerCall(callId, handler, platform.createTimeout, platform.webRTC); + } } // TODO: check and poll turn server credentials here @@ -51,43 +52,69 @@ export class GroupCallHandler { handleRoomState(room: Room, events: StateEvent[], log: ILogItem) { // first update call events for (const event of events) { - if (event.type === GROUP_CALL_TYPE) { - const callId = event.state_key; - let call = this.calls.get(callId); - if (call) { - call.updateCallEvent(event); - if (call.isTerminated) { - this.calls.remove(call.id); - } - } else { - call = new GroupCall(event, room); - this.calls.set(call.id, call); - } + if (event.type === EventType.GroupCall) { + this.handleCallEvent(event); } } // then update participants for (const event of events) { - if (event.type === GROUP_CALL_MEMBER_TYPE) { - const participant = event.state_key; - const sources = event.content["m.sources"]; - for (const source of sources) { - const call = this.calls.get(source[CONF_ID]); - if (call && !call.isTerminated) { - call.addParticipant(participant, source); - } - } + if (event.type === EventType.GroupCallMember) { + this.handleCallMemberEvent(event); } } } - handlesDeviceMessageEventType(eventType: string | undefined): boolean { - return eventType === CallSetupMessageType.Invite || - eventType === CallSetupMessageType.Candidates || - eventType === CallSetupMessageType.Answer || - eventType === CallSetupMessageType.Hangup; + updateRoomMembers(room: Room, memberChanges: Map) { + + } + + private handleCallEvent(event: StateEvent) { + const callId = event.state_key; + let call = this.calls.get(callId); + if (call) { + call.updateCallEvent(event); + if (call.isTerminated) { + this.calls.remove(call.id); + } + } else { + call = new GroupCall(event, room, this.createPeerCall); + this.calls.set(call.id, call); + } + } + + private handleCallMemberEvent(event: StateEvent) { + const participant = event.state_key; + const calls = event.content["m.calls"] ?? []; + const newCallIdsMemberOf = new Set(calls.map(call => { + const callId = call["m.call_id"]; + const groupCall = this.calls.get(callId); + // TODO: also check the participant when receiving the m.call event + groupCall?.addParticipant(participant, call); + return callId; + })); + let previousCallIdsMemberOf = this.memberToCallIds.get(participant); + // remove user as participant of any calls not present anymore + if (previousCallIdsMemberOf) { + for (const previousCallId of previousCallIdsMemberOf) { + if (!newCallIdsMemberOf.has(previousCallId)) { + const groupCall = this.calls.get(previousCallId); + groupCall?.removeParticipant(participant); + } + } + } + if (newCallIdsMemberOf.size === 0) { + this.memberToCallIds.delete(participant); + } else { + this.memberToCallIds.set(participant, newCallIdsMemberOf); + } + } + + handlesDeviceMessageEventType(eventType: string): boolean { + return handlesEventType(eventType); } handleDeviceMessage(senderUserId: string, senderDeviceId: string, event: SignallingMessage, log: ILogItem) { + // TODO: buffer messages for calls we haven't received the state event for yet? const call = this.calls.get(event.content.conf_id); call?.handleDeviceMessage(senderUserId, senderDeviceId, event, log); } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 5040c805..f06b291d 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -29,7 +29,8 @@ import type {LocalMedia} from "./LocalMedia"; import { SDPStreamMetadataKey, - SDPStreamMetadataPurpose + SDPStreamMetadataPurpose, + EventType, } from "./callEventTypes"; import type { MCallBase, @@ -39,6 +40,7 @@ import type { MCallCandidates, MCallHangupReject, SDPStreamMetadata, + SignallingMessage } from "./callEventTypes"; // when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already @@ -677,21 +679,6 @@ export enum CallDirection { Outbound = 'outbound', } -export enum EventType { - Invite = "m.call.invite", - Candidates = "m.call.candidates", - Answer = "m.call.answer", - Hangup = "m.call.hangup", - Reject = "m.call.reject", - SelectAnswer = "m.call.select_answer", - Negotiate = "m.call.negotiate", - SDPStreamMetadataChanged = "m.call.sdp_stream_metadata_changed", - SDPStreamMetadataChangedPrefix = "org.matrix.call.sdp_stream_metadata_changed", - Replaces = "m.call.replaces", - AssertedIdentity = "m.call.asserted_identity", - AssertedIdentityPrefix = "org.matrix.call.asserted_identity", -} - export enum CallErrorCode { /** The user chose to end the call */ UserHangup = 'user_hangup', @@ -802,18 +789,18 @@ export class CallError extends Error { } } -export type SignallingMessage = - {type: EventType.Invite, content: MCallInvite} | - {type: EventType.Answer, content: MCallAnswer} | - {type: EventType.SDPStreamMetadataChanged | EventType.SDPStreamMetadataChangedPrefix, content: MCallSDPStreamMetadataChanged} | - {type: EventType.Candidates, content: MCallCandidates} | - {type: EventType.Hangup | EventType.Reject, content: MCallHangupReject}; - export interface PeerCallHandler { emitUpdate(peerCall: PeerCall, params: any); sendSignallingMessage(message: SignallingMessage); } +export function handlesEventType(eventType: string): boolean { + return eventType === EventType.Invite || + eventType === EventType.Candidates || + eventType === EventType.Answer || + eventType === EventType.Hangup; +} + export function tests() { } diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index aa1bc079..0e9eb8f8 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -1,6 +1,24 @@ // allow non-camelcase as these are events type that go onto the wire /* eslint-disable camelcase */ + +export enum EventType { + GroupCall = "m.call", + GroupCallMember = "m.call.member", + Invite = "m.call.invite", + Candidates = "m.call.candidates", + Answer = "m.call.answer", + Hangup = "m.call.hangup", + Reject = "m.call.reject", + SelectAnswer = "m.call.select_answer", + Negotiate = "m.call.negotiate", + SDPStreamMetadataChanged = "m.call.sdp_stream_metadata_changed", + SDPStreamMetadataChangedPrefix = "org.matrix.call.sdp_stream_metadata_changed", + Replaces = "m.call.replaces", + AssertedIdentity = "m.call.asserted_identity", + AssertedIdentityPrefix = "org.matrix.call.asserted_identity", +} + // TODO: Change to "sdp_stream_metadata" when MSC3077 is merged export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata"; @@ -88,4 +106,95 @@ export type MCallHangupReject = Base & { reason?: CallErrorCode; } -/* eslint-enable camelcase */ +export enum CallErrorCode { + /** The user chose to end the call */ + UserHangup = 'user_hangup', + + /** An error code when the local client failed to create an offer. */ + LocalOfferFailed = 'local_offer_failed', + /** + * An error code when there is no local mic/camera to use. This may be because + * the hardware isn't plugged in, or the user has explicitly denied access. + */ + NoUserMedia = 'no_user_media', + + /** + * Error code used when a call event failed to send + * because unknown devices were present in the room + */ + UnknownDevices = 'unknown_devices', + + /** + * Error code used when we fail to send the invite + * for some reason other than there being unknown devices + */ + SendInvite = 'send_invite', + + /** + * An answer could not be created + */ + CreateAnswer = 'create_answer', + + /** + * Error code used when we fail to send the answer + * for some reason other than there being unknown devices + */ + SendAnswer = 'send_answer', + + /** + * The session description from the other side could not be set + */ + SetRemoteDescription = 'set_remote_description', + + /** + * The session description from this side could not be set + */ + SetLocalDescription = 'set_local_description', + + /** + * A different device answered the call + */ + AnsweredElsewhere = 'answered_elsewhere', + + /** + * No media connection could be established to the other party + */ + IceFailed = 'ice_failed', + + /** + * The invite timed out whilst waiting for an answer + */ + InviteTimeout = 'invite_timeout', + + /** + * The call was replaced by another call + */ + Replaced = 'replaced', + + /** + * Signalling for the call could not be sent (other than the initial invite) + */ + SignallingFailed = 'signalling_timeout', + + /** + * The remote party is busy + */ + UserBusy = 'user_busy', + + /** + * We transferred the call off to somewhere else + */ + Transfered = 'transferred', + + /** + * A call from the same user was found with a new session id + */ + NewSession = 'new_session', +} + +export type SignallingMessage = + {type: EventType.Invite, content: MCallInvite} | + {type: EventType.Answer, content: MCallAnswer} | + {type: EventType.SDPStreamMetadataChanged | EventType.SDPStreamMetadataChangedPrefix, content: MCallSDPStreamMetadataChanged} | + {type: EventType.Candidates, content: MCallCandidates} | + {type: EventType.Hangup | EventType.Reject, content: MCallHangupReject}; diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index e05f572d..dce50846 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -18,16 +18,21 @@ import {ObservableMap} from "../../../observable/map/ObservableMap"; import {Participant} from "./Participant"; import {LocalMedia} from "../LocalMedia"; import type {Track} from "../../../platform/types/MediaDevices"; - -function getParticipantId(senderUserId: string, senderDeviceId: string | null) { - return JSON.stringify(senderUserId) + JSON.stringify(senderDeviceId); -} +import type {SignallingMessage, MGroupCallBase} from "../callEventTypes"; +import type {Room} from "../../room/Room"; +import type {StateEvent} from "../../storage/types"; +import type {Platform} from "../../../platform/web/Platform"; export class GroupCall { private readonly participants: ObservableMap = new ObservableMap(); private localMedia?: Promise; - constructor(private readonly ownUserId: string, private callEvent: StateEvent, private readonly room: Room, private readonly webRTC: WebRTC) { + constructor( + private readonly ownUserId: string, + private callEvent: StateEvent, + private readonly room: Room, + private readonly platform: Platform + ) { } @@ -52,32 +57,25 @@ export class GroupCall { this.callEvent = callEvent; } - addParticipant(userId, source) { - const participantId = getParticipantId(userId, source.device_id); - const participant = this.participants.get(participantId); + addParticipant(userId, memberCallInfo) { + let participant = this.participants.get(userId); if (participant) { - participant.updateSource(source); + participant.updateCallInfo(memberCallInfo); } else { - participant.add(participantId, new Participant(userId, source.device_id, this.localMedia?.clone(), this.webRTC)); + participant = new Participant(userId, source.device_id, this.localMedia?.clone(), this.webRTC); + participant.updateCallInfo(memberCallInfo); + this.participants.add(userId, participant); } } - handleDeviceMessage(senderUserId: string, senderDeviceId: string, eventType: string, content: Record, log: ILogItem) { - const participantId = getParticipantId(senderUserId, senderDeviceId); - let peerCall = this.participants.get(participantId); - let hasDeviceInKey = true; - if (!peerCall) { - hasDeviceInKey = false; - peerCall = this.participants.get(getParticipantId(senderUserId, null)) - } - if (peerCall) { - peerCall.handleIncomingSignallingMessage(eventType, content, senderDeviceId); - if (!hasDeviceInKey && peerCall.opponentPartyId) { - this.participants.delete(getParticipantId(senderUserId, null)); - this.participants.add(getParticipantId(senderUserId, peerCall.opponentPartyId)); - } - } else { - // create peerCall + removeParticipant(userId) { + + } + + handleDeviceMessage(userId: string, senderDeviceId: string, message: SignallingMessage, log: ILogItem) { + let participant = this.participants.get(userId); + if (participant) { + participant.handleIncomingSignallingMessage(message, senderDeviceId); } } diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js index b9ec82a3..d0fcbc84 100644 --- a/src/matrix/room/Room.js +++ b/src/matrix/room/Room.js @@ -218,6 +218,9 @@ export class Room extends BaseRoom { if (this._memberList) { this._memberList.afterSync(memberChanges); } + if (this._callHandler) { + this._callHandler.updateRoomMembers(this, memberChanges); + } if (this._observedMembers) { this._updateObservedMembers(memberChanges); } From 6da4a4209c95598da1daf451a82f6aa0649d8f62 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 10 Mar 2022 14:53:31 +0100 Subject: [PATCH 013/323] WIP: work on group calling code --- src/matrix/DeviceMessageHandler.js | 2 +- src/matrix/Session.js | 14 ++- src/matrix/calls/CallHandler.ts | 126 ++++++++++++++------------ src/matrix/calls/PeerCall.ts | 79 ++++++++-------- src/matrix/calls/group/GroupCall.ts | 90 +++++++++++------- src/matrix/calls/group/Member.ts | 112 +++++++++++++++++++++++ src/matrix/calls/group/Participant.ts | 67 -------------- src/matrix/common.js | 6 +- src/matrix/e2ee/olm/Encryption.ts | 2 +- src/matrix/net/HomeServerApi.ts | 4 + src/platform/types/MediaDevices.ts | 1 + 11 files changed, 301 insertions(+), 202 deletions(-) create mode 100644 src/matrix/calls/group/Member.ts delete mode 100644 src/matrix/calls/group/Participant.ts diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index 470559a9..91ef82f6 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -59,7 +59,7 @@ export class DeviceMessageHandler { })); // TODO: pass this in the prep and run it in afterSync or afterSyncComplete (as callHandler can send events as well)? for (const dr of callMessages) { - this._callHandler.handleDeviceMessage(dr.device.userId, dr.device.deviceId, dr.event, log); + this._callHandler.handleDeviceMessage(dr.event, dr.device.userId, dr.device.deviceId, log); } // TODO: somehow include rooms that received a call to_device message in the sync state? // or have updates flow through event emitter? diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 3d9b13c8..94fb5dee 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -73,6 +73,19 @@ export class Session { }; this._roomsBeingCreated = new ObservableMap(); this._user = new User(sessionInfo.userId); + this._callHandler = new CallHandler({ + createTimeout: this._platform.clock.createTimeout, + hsApi: this._hsApi, + encryptDeviceMessage: async (roomId, message, log) => { + if (!this._deviceTracker || !this._olmEncryption) { + throw new Error("encryption is not enabled"); + } + await this._deviceTracker.trackRoom(roomId, log); + const devices = await this._deviceTracker.devicesForTrackedRoom(roomId, this._hsApi, log); + const encryptedMessage = await this._olmEncryption.encrypt(message.type, message.content, devices, this._hsApi, log); + return encryptedMessage; + } + }); this._deviceMessageHandler = new DeviceMessageHandler({storage, callHandler: this._callHandler}); this._olm = olm; this._olmUtil = null; @@ -100,7 +113,6 @@ export class Session { this._createRoomEncryption = this._createRoomEncryption.bind(this); this._forgetArchivedRoom = this._forgetArchivedRoom.bind(this); this.needsKeyBackup = new ObservableValue(false); - this._callHandler = new CallHandler(this._platform, this._hsApi); } get fingerprintKey() { diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index d84be9e3..0e70fb5a 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -15,48 +15,53 @@ limitations under the License. */ import {ObservableMap} from "../../observable/map/ObservableMap"; +import {WebRTC, PeerConnection, PeerConnectionHandler} from "../../platform/types/WebRTC"; +import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; +import {handlesEventType} from "./PeerCall"; +import {EventType} from "./callEventTypes"; +import {GroupCall} from "./group/GroupCall"; import type {Room} from "../room/Room"; import type {MemberChange} from "../room/members/RoomMember"; import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; import type {Platform} from "../../platform/web/Platform"; - -import {WebRTC, PeerConnection, PeerConnectionHandler, StreamPurpose} from "../../platform/types/WebRTC"; -import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; -import {handlesEventType, PeerCall, PeerCallHandler} from "./PeerCall"; -import {EventType} from "./callEventTypes"; +import type {BaseObservableMap} from "../../observable/map/BaseObservableMap"; import type {SignallingMessage, MGroupCallBase} from "./callEventTypes"; -import type {GroupCall} from "./group/GroupCall"; +import type {Options as GroupCallOptions} from "./group/GroupCall"; const GROUP_CALL_TYPE = "m.call"; const GROUP_CALL_MEMBER_TYPE = "m.call.member"; const CALL_TERMINATED = "m.terminated"; -export class GroupCallHandler { +export type Options = Omit; - private createPeerCall: (callId: string, handler: PeerCallHandler) => PeerCall; +export class GroupCallHandler { // group calls by call id - public readonly calls: ObservableMap = new ObservableMap(); + private readonly _calls: ObservableMap = new ObservableMap(); // map of userId to set of conf_id's they are in private memberToCallIds: Map> = new Map(); + private groupCallOptions: GroupCallOptions; - constructor(hsApi: HomeServerApi, platform: Platform, ownUserId: string, ownDeviceId: string) { - this.createPeerCall = (callId: string, handler: PeerCallHandler) => { - return new PeerCall(callId, handler, platform.createTimeout, platform.webRTC); - } + constructor(private readonly options: Options) { + this.groupCallOptions = Object.assign({}, this.options, { + emitUpdate: (groupCall, params) => this._calls.update(groupCall.id, params) + }); } + get calls(): BaseObservableMap { return this._calls; } + // TODO: check and poll turn server credentials here + /** @internal */ handleRoomState(room: Room, events: StateEvent[], log: ILogItem) { // first update call events for (const event of events) { if (event.type === EventType.GroupCall) { - this.handleCallEvent(event); + this.handleCallEvent(event, room); } } - // then update participants + // then update members for (const event of events) { if (event.type === EventType.GroupCallMember) { this.handleCallMemberEvent(event); @@ -64,59 +69,62 @@ export class GroupCallHandler { } } + /** @internal */ updateRoomMembers(room: Room, memberChanges: Map) { } - private handleCallEvent(event: StateEvent) { - const callId = event.state_key; - let call = this.calls.get(callId); - if (call) { - call.updateCallEvent(event); - if (call.isTerminated) { - this.calls.remove(call.id); - } - } else { - call = new GroupCall(event, room, this.createPeerCall); - this.calls.set(call.id, call); - } - } - - private handleCallMemberEvent(event: StateEvent) { - const participant = event.state_key; - const calls = event.content["m.calls"] ?? []; - const newCallIdsMemberOf = new Set(calls.map(call => { - const callId = call["m.call_id"]; - const groupCall = this.calls.get(callId); - // TODO: also check the participant when receiving the m.call event - groupCall?.addParticipant(participant, call); - return callId; - })); - let previousCallIdsMemberOf = this.memberToCallIds.get(participant); - // remove user as participant of any calls not present anymore - if (previousCallIdsMemberOf) { - for (const previousCallId of previousCallIdsMemberOf) { - if (!newCallIdsMemberOf.has(previousCallId)) { - const groupCall = this.calls.get(previousCallId); - groupCall?.removeParticipant(participant); - } - } - } - if (newCallIdsMemberOf.size === 0) { - this.memberToCallIds.delete(participant); - } else { - this.memberToCallIds.set(participant, newCallIdsMemberOf); - } - } - + /** @internal */ handlesDeviceMessageEventType(eventType: string): boolean { return handlesEventType(eventType); } - handleDeviceMessage(senderUserId: string, senderDeviceId: string, event: SignallingMessage, log: ILogItem) { + /** @internal */ + handleDeviceMessage(message: SignallingMessage, userId: string, deviceId: string, log: ILogItem) { // TODO: buffer messages for calls we haven't received the state event for yet? - const call = this.calls.get(event.content.conf_id); - call?.handleDeviceMessage(senderUserId, senderDeviceId, event, log); + const call = this._calls.get(message.content.conf_id); + call?.handleDeviceMessage(message, userId, deviceId, log); + } + + private handleCallEvent(event: StateEvent, room: Room) { + const callId = event.state_key; + let call = this._calls.get(callId); + if (call) { + call.updateCallEvent(event); + if (call.isTerminated) { + this._calls.remove(call.id); + } + } else { + call = new GroupCall(event, room, this.groupCallOptions); + this._calls.set(call.id, call); + } + } + + private handleCallMemberEvent(event: StateEvent) { + const userId = event.state_key; + const calls = event.content["m.calls"] ?? []; + const newCallIdsMemberOf = new Set(calls.map(call => { + const callId = call["m.call_id"]; + const groupCall = this._calls.get(callId); + // TODO: also check the member when receiving the m.call event + groupCall?.addMember(userId, call); + return callId; + })); + let previousCallIdsMemberOf = this.memberToCallIds.get(userId); + // remove user as member of any calls not present anymore + if (previousCallIdsMemberOf) { + for (const previousCallId of previousCallIdsMemberOf) { + if (!newCallIdsMemberOf.has(previousCallId)) { + const groupCall = this._calls.get(previousCallId); + groupCall?.removeMember(userId); + } + } + } + if (newCallIdsMemberOf.size === 0) { + this.memberToCallIds.delete(userId); + } else { + this.memberToCallIds.set(userId, newCallIdsMemberOf); + } } } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index f06b291d..ed8351ac 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -43,6 +43,13 @@ import type { SignallingMessage } from "./callEventTypes"; +export type Options = { + webRTC: WebRTC, + createTimeout: TimeoutCreator, + emitUpdate: (peerCall: PeerCall, params: any) => void; + sendSignallingMessage: (message: SignallingMessage, log: ILogItem) => Promise; +}; + // when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already // do for sharing keys will be best as that already deals with room tracking. /** @@ -51,7 +58,7 @@ import type { /** Implements a call between two peers with the signalling state keeping, while still delegating the signalling message sending. Used by GroupCall.*/ export class PeerCall implements IDisposable { private readonly peerConnection: PeerConnection; - private state = CallState.Fledgling; + private _state = CallState.Fledgling; private direction: CallDirection; private localMedia?: LocalMedia; // A queue for candidates waiting to go out. @@ -74,15 +81,12 @@ export class PeerCall implements IDisposable { // perfect negotiation flags private makingOffer: boolean = false; private ignoreOffer: boolean = false; - constructor( private callId: string, // generated or from invite - private readonly handler: PeerCallHandler, - private readonly createTimeout: TimeoutCreator, - webRTC: WebRTC + private readonly options: Options ) { const outer = this; - this.peerConnection = webRTC.createPeerConnection({ + this.peerConnection = options.webRTC.createPeerConnection({ onIceConnectionStateChange(state: RTCIceConnectionState) {}, onLocalIceCandidate(candidate: RTCIceCandidate) {}, onIceGatheringStateChange(state: RTCIceGatheringState) {}, @@ -104,12 +108,14 @@ export class PeerCall implements IDisposable { } } + get state(): CallState { return this._state; } + get remoteTracks(): Track[] { return this.peerConnection.remoteTracks; } async call(localMediaPromise: Promise): Promise { - if (this.state !== CallState.Fledgling) { + if (this._state !== CallState.Fledgling) { return; } this.direction = CallDirection.Outbound; @@ -131,7 +137,7 @@ export class PeerCall implements IDisposable { } async answer(localMediaPromise: Promise): Promise { - if (this.state !== CallState.Ringing) { + if (this._state !== CallState.Ringing) { return; } this.setState(CallState.WaitLocalMedia); @@ -197,7 +203,7 @@ export class PeerCall implements IDisposable { async hangup(errorCode: CallErrorCode) { } - async handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId): Promise { + async handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId, log: ILogItem): Promise { switch (message.type) { case EventType.Invite: if (this.callId !== message.content.call_id) { @@ -226,10 +232,10 @@ export class PeerCall implements IDisposable { if (reason) { content["reason"] = reason; } - return this.handler.sendSignallingMessage({ + return this.options.sendSignallingMessage({ type: EventType.Hangup, content - }); + }, undefined); } // calls are serialized and deduplicated by responsePromiseChain @@ -249,7 +255,7 @@ export class PeerCall implements IDisposable { await this.delay(200); } - if (this.state === CallState.Ended) { + if (this._state === CallState.Ended) { return; } @@ -268,12 +274,12 @@ export class PeerCall implements IDisposable { version: 1, lifetime: CALL_TIMEOUT_MS }; - if (this.state === CallState.CreateOffer) { - await this.handler.sendSignallingMessage({type: EventType.Invite, content}); + if (this._state === CallState.CreateOffer) { + await this.options.sendSignallingMessage({type: EventType.Invite, content}); this.setState(CallState.InviteSent); - } else if (this.state === CallState.Connected || this.state === CallState.Connecting) { + } else if (this._state === CallState.Connected || this._state === CallState.Connecting) { // send Negotiate message - //await this.handler.sendSignallingMessage({type: EventType.Invite, content}); + //await this.options.sendSignallingMessage({type: EventType.Invite, content}); //this.setState(CallState.InviteSent); } } finally { @@ -282,10 +288,10 @@ export class PeerCall implements IDisposable { this.sendCandidateQueue(); - if (this.state === CallState.InviteSent) { + if (this._state === CallState.InviteSent) { await this.delay(CALL_TIMEOUT_MS); // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between - if (this.state === CallState.InviteSent) { + if (this._state === CallState.InviteSent) { this.hangup(CallErrorCode.InviteTimeout); } } @@ -307,7 +313,7 @@ export class PeerCall implements IDisposable { // TODO: review states to be unambigous, WaitLocalMedia for sending offer or answer? // How do we interrupt `call()`? well, perhaps we need to not just await InviteSent but also CreateAnswer? - if (this.state === CallState.Fledgling || this.state === CallState.CreateOffer || this.state === CallState.WaitLocalMedia) { + if (this._state === CallState.Fledgling || this._state === CallState.CreateOffer || this._state === CallState.WaitLocalMedia) { } else { await this.sendHangupWithCallId(this.callId, CallErrorCode.Replaced); @@ -324,7 +330,7 @@ export class PeerCall implements IDisposable { } private async handleFirstInvite(content: MCallInvite, partyId: PartyId): Promise { - if (this.state !== CallState.Fledgling || this.opponentPartyId !== undefined) { + if (this._state !== CallState.Fledgling || this.opponentPartyId !== undefined) { // TODO: hangup or ignore? return; } @@ -370,7 +376,7 @@ export class PeerCall implements IDisposable { await this.delay(content.lifetime ?? CALL_TIMEOUT_MS); // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between - if (this.state === CallState.Ringing) { + if (this._state === CallState.Ringing) { this.logger.debug(`Call ${this.callId} invite has expired. Hanging up.`); this.hangupParty = CallParty.Remote; // effectively this.setState(CallState.Ended); @@ -384,7 +390,7 @@ export class PeerCall implements IDisposable { private async handleAnswer(content: MCallAnswer, partyId: PartyId): Promise { this.logger.debug(`Got answer for call ID ${this.callId} from party ID ${partyId}`); - if (this.state === CallState.Ended) { + if (this._state === CallState.Ended) { this.logger.debug(`Ignoring answer because call ID ${this.callId} has ended`); return; } @@ -456,7 +462,7 @@ export class PeerCall implements IDisposable { // if (description.type === 'offer') { // await this.peerConnection.setLocalDescription(); - // await this.handler.sendSignallingMessage({ + // await this.options.sendSignallingMessage({ // type: EventType.CallNegotiate, // content: { // description: this.peerConnection.localDescription!, @@ -471,7 +477,7 @@ export class PeerCall implements IDisposable { private async sendAnswer(): Promise { const localDescription = this.peerConnection.localDescription!; - const answerContent: MCallAnswer = { + const answerContent: MCallAnswer = { call_id: this.callId, version: 1, answer: { @@ -489,7 +495,7 @@ export class PeerCall implements IDisposable { this.candidateSendQueue = []; try { - await this.handler.sendSignallingMessage({type: EventType.Answer, content: answerContent}); + await this.options.sendSignallingMessage({type: EventType.Answer, content: answerContent}, undefined); } catch (error) { this.terminate(CallParty.Local, CallErrorCode.SendAnswer, false); throw error; @@ -513,7 +519,7 @@ export class PeerCall implements IDisposable { this.candidateSendQueue.push(content); // Don't send the ICE candidates yet if the call is in the ringing state - if (this.state === CallState.Ringing) return; + if (this._state === CallState.Ringing) return; // MSC2746 recommends these values (can be quite long when calling because the // callee will need a while to answer the call) @@ -523,7 +529,7 @@ export class PeerCall implements IDisposable { } private async sendCandidateQueue(): Promise { - if (this.candidateSendQueue.length === 0 || this.state === CallState.Ended) { + if (this.candidateSendQueue.length === 0 || this._state === CallState.Ended) { return; } @@ -531,14 +537,14 @@ export class PeerCall implements IDisposable { this.candidateSendQueue = []; this.logger.debug(`Call ${this.callId} attempting to send ${candidates.length} candidates`); try { - await this.handler.sendSignallingMessage({ + await this.options.sendSignallingMessage({ type: EventType.Candidates, content: { call_id: this.callId, version: 1, candidates - } - }); + }, + }, undefined); // Try to send candidates again just in case we received more candidates while sending. this.sendCandidateQueue(); } catch (error) { @@ -598,14 +604,14 @@ export class PeerCall implements IDisposable { } private setState(state: CallState): void { - const oldState = this.state; - this.state = state; + const oldState = this._state; + this._state = state; let deferred = this.statePromiseMap.get(state); if (deferred) { deferred.resolve(); this.statePromiseMap.delete(state); } - this.handler.emitUpdate(this, undefined); + this.options.emitUpdate(this, undefined); } private waitForState(states: CallState[]): Promise { @@ -638,7 +644,7 @@ export class PeerCall implements IDisposable { private async delay(timeoutMs: number): Promise { // Allow a short time for initial candidates to be gathered - const timeout = this.disposables.track(this.createTimeout(timeoutMs)); + const timeout = this.disposables.track(this.options.createTimeout(timeoutMs)); await timeout.elapsed(); this.disposables.untrack(timeout); } @@ -789,11 +795,6 @@ export class CallError extends Error { } } -export interface PeerCallHandler { - emitUpdate(peerCall: PeerCall, params: any); - sendSignallingMessage(message: SignallingMessage); -} - export function handlesEventType(eventType: string): boolean { return eventType === EventType.Invite || eventType === EventType.Candidates || diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index dce50846..7266645d 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -15,71 +15,95 @@ limitations under the License. */ import {ObservableMap} from "../../../observable/map/ObservableMap"; -import {Participant} from "./Participant"; +import {Member} from "./Member"; import {LocalMedia} from "../LocalMedia"; +import {RoomMember} from "../../room/members/RoomMember"; +import type {Options as MemberOptions} from "./Member"; +import type {BaseObservableMap} from "../../../observable/map/BaseObservableMap"; import type {Track} from "../../../platform/types/MediaDevices"; import type {SignallingMessage, MGroupCallBase} from "../callEventTypes"; import type {Room} from "../../room/Room"; import type {StateEvent} from "../../storage/types"; import type {Platform} from "../../../platform/web/Platform"; +import type {EncryptedMessage} from "../../e2ee/olm/Encryption"; +import type {ILogItem} from "../../../logging/types"; + +export type Options = Omit & { + emitUpdate: (call: GroupCall, params?: any) => void; + encryptDeviceMessage: (roomId: string, message: SignallingMessage, log: ILogItem) => Promise, +}; export class GroupCall { - private readonly participants: ObservableMap = new ObservableMap(); + private readonly _members: ObservableMap = new ObservableMap(); private localMedia?: Promise; + private _memberOptions: MemberOptions; constructor( - private readonly ownUserId: string, private callEvent: StateEvent, private readonly room: Room, - private readonly platform: Platform + private readonly options: Options ) { - + this._memberOptions = Object.assign({ + confId: this.id, + emitUpdate: member => this._members.update(member.member.userId, member), + encryptDeviceMessage: (message: SignallingMessage, log) => { + return this.options.encryptDeviceMessage(this.room.id, message, log); + } + }, options); } + get members(): BaseObservableMap { return this._members; } + get id(): string { return this.callEvent.state_key; } - async participate(tracks: Promise) { - this.localMedia = tracks.then(tracks => LocalMedia.fromTracks(tracks)); - for (const [,participant] of this.participants) { - participant.setLocalMedia(this.localMedia.then(localMedia => localMedia.clone())); - } - // send m.call.member state event + get isTerminated(): boolean { + return this.callEvent.content["m.terminated"] === true; + } - // send invite to all participants that are < my userId - for (const [,participant] of this.participants) { - if (participant.userId < this.ownUserId) { - participant.call(); - } + async join(tracks: Promise) { + this.localMedia = tracks.then(tracks => LocalMedia.fromTracks(tracks)); + // send m.call.member state event + const request = this.options.hsApi.sendState(this.room.id, "m.call.member", this.options.ownUserId, { + + }); + await request.response(); + // send invite to all members that are < my userId + for (const [,member] of this._members) { + member.connect(this.localMedia); } } + /** @internal */ updateCallEvent(callEvent: StateEvent) { this.callEvent = callEvent; + // TODO: emit update } - addParticipant(userId, memberCallInfo) { - let participant = this.participants.get(userId); - if (participant) { - participant.updateCallInfo(memberCallInfo); + /** @internal */ + addMember(userId, memberCallInfo) { + let member = this._members.get(userId); + if (member) { + member.updateCallInfo(memberCallInfo); } else { - participant = new Participant(userId, source.device_id, this.localMedia?.clone(), this.webRTC); - participant.updateCallInfo(memberCallInfo); - this.participants.add(userId, participant); + member = new Member(RoomMember.fromUserId(this.room.id, userId, "join"), this._memberOptions); + member.updateCallInfo(memberCallInfo); + this._members.add(userId, member); } } - removeParticipant(userId) { - + /** @internal */ + removeMember(userId) { + this._members.remove(userId); } - handleDeviceMessage(userId: string, senderDeviceId: string, message: SignallingMessage, log: ILogItem) { - let participant = this.participants.get(userId); - if (participant) { - participant.handleIncomingSignallingMessage(message, senderDeviceId); + /** @internal */ + handleDeviceMessage(message: SignallingMessage, userId: string, deviceId: string, log: ILogItem) { + // TODO: return if we are not membering to the call + let member = this._members.get(userId); + if (member) { + member.handleDeviceMessage(message, deviceId, log); + } else { + // we haven't received the m.call.member yet for this caller. buffer the device messages or create the member/call anyway? } } - - get isTerminated(): boolean { - return !!this.callEvent.content[CALL_TERMINATED]; - } } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts new file mode 100644 index 00000000..c6791568 --- /dev/null +++ b/src/matrix/calls/group/Member.ts @@ -0,0 +1,112 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {PeerCall, CallState} from "../PeerCall"; +import {makeTxnId, makeId} from "../../common"; +import {EventType} from "../callEventTypes"; + +import type {Options as PeerCallOptions} from "../PeerCall"; +import type {LocalMedia} from "../LocalMedia"; +import type {HomeServerApi} from "../../net/HomeServerApi"; +import type {Track} from "../../../platform/types/MediaDevices"; +import type {MCallBase, MGroupCallBase, SignallingMessage} from "../callEventTypes"; +import type {GroupCall} from "./GroupCall"; +import type {RoomMember} from "../../room/members/RoomMember"; +import type {EncryptedMessage} from "../../e2ee/olm/Encryption"; +import type {ILogItem} from "../../../logging/types"; + +export type Options = Omit & { + confId: string, + ownUserId: string, + hsApi: HomeServerApi, + encryptDeviceMessage: (message: SignallingMessage, log: ILogItem) => Promise, + emitUpdate: (participant: Member, params?: any) => void, +} + +export class Member { + private peerCall?: PeerCall; + private localMedia?: Promise; + + constructor( + public readonly member: RoomMember, + private readonly options: Options + ) {} + + get remoteTracks(): Track[] { + return this.peerCall?.remoteTracks ?? []; + } + + get isConnected(): boolean { + return this.peerCall?.state === CallState.Connected; + } + + /* @internal */ + connect(localMedia: Promise) { + this.localMedia = localMedia; + // otherwise wait for it to connect + if (this.member.userId < this.options.ownUserId) { + this.peerCall = this._createPeerCall(makeId("c")); + this.peerCall.call(localMedia); + } + } + + /** @internal */ + updateCallInfo(memberCallInfo) { + + } + + /** @internal */ + emitUpdate = (peerCall: PeerCall, params: any) => { + if (peerCall.state === CallState.Ringing) { + peerCall.answer(this.localMedia!); + } + this.options.emitUpdate(this, params); + } + + /** From PeerCallHandler + * @internal */ + sendSignallingMessage = async (message: SignallingMessage, log: ILogItem) => { + const groupMessage = message as SignallingMessage; + groupMessage.content.conf_id = this.options.confId; + const encryptedMessage = await this.options.encryptDeviceMessage(groupMessage, log); + const request = this.options.hsApi.sendToDevice( + "m.room.encrypted", + {[this.member.userId]: { + ["*"]: encryptedMessage.content + } + }, makeTxnId(), {log}); + await request.response(); + } + + /** @internal */ + handleDeviceMessage(message: SignallingMessage, deviceId: string, log: ILogItem) { + if (message.type === EventType.Invite && !this.peerCall) { + this.peerCall = this._createPeerCall(message.content.call_id); + } + if (this.peerCall) { + this.peerCall.handleIncomingSignallingMessage(message, deviceId, log); + } else { + // TODO: need to buffer events until invite comes? + } + } + + private _createPeerCall(callId: string): PeerCall { + return new PeerCall(callId, Object.assign({}, this.options, { + emitUpdate: this.emitUpdate, + sendSignallingMessage: this.sendSignallingMessage + })); + } +} diff --git a/src/matrix/calls/group/Participant.ts b/src/matrix/calls/group/Participant.ts deleted file mode 100644 index 2b873aa0..00000000 --- a/src/matrix/calls/group/Participant.ts +++ /dev/null @@ -1,67 +0,0 @@ -/* -Copyright 2022 The Matrix.org Foundation C.I.C. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import {EventType, PeerCall, SignallingMessage} from "../PeerCall"; -import {makeTxnId} from "../../common"; - -import type {PeerCallHandler} from "../PeerCall"; -import type {LocalMedia} from "../LocalMedia"; -import type {HomeServerApi} from "../../net/HomeServerApi"; -import type {Track} from "../../../platform/types/MediaDevices"; -import type {MCallBase, MGroupCallBase} from "../callEventTypes"; -import type {GroupCall} from "./GroupCall"; -import type {RoomMember} from "../../room/members/RoomMember"; - -export class Participant implements PeerCallHandler { - constructor( - public readonly member: RoomMember, - private readonly deviceId: string | undefined, - private readonly peerCall: PeerCall, - private readonly hsApi: HomeServerApi, - private readonly groupCall: GroupCall - ) {} - - /* @internal */ - call(localMedia: Promise) { - this.peerCall.call(localMedia); - } - - get remoteTracks(): Track[] { - return this.peerCall.remoteTracks; - } - - /** From PeerCallHandler - * @internal */ - emitUpdate(params: any) { - this.groupCall.emitParticipantUpdate(this, params); - } - - /** From PeerCallHandler - * @internal */ - async sendSignallingMessage(message: SignallingMessage) { - const groupMessage = message as SignallingMessage; - groupMessage.content.conf_id = this.groupCall.id; - // TODO: this needs to be encrypted with olm first - - const request = this.hsApi.sendToDevice( - groupMessage.type, - {[this.member.userId]: { - [this.deviceId ?? "*"]: groupMessage.content - } - }, makeTxnId()); - await request.response(); - } -} diff --git a/src/matrix/common.js b/src/matrix/common.js index ba7876ed..5919ad9c 100644 --- a/src/matrix/common.js +++ b/src/matrix/common.js @@ -16,9 +16,13 @@ limitations under the License. */ export function makeTxnId() { + return makeId("t"); +} + +export function makeId(prefix) { const n = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER); const str = n.toString(16); - return "t" + "0".repeat(14 - str.length) + str; + return prefix + "0".repeat(14 - str.length) + str; } export function isTxnId(txnId) { diff --git a/src/matrix/e2ee/olm/Encryption.ts b/src/matrix/e2ee/olm/Encryption.ts index 9b754272..dcc9f0b1 100644 --- a/src/matrix/e2ee/olm/Encryption.ts +++ b/src/matrix/e2ee/olm/Encryption.ts @@ -311,7 +311,7 @@ class EncryptionTarget { } } -class EncryptedMessage { +export class EncryptedMessage { constructor( public readonly content: OlmEncryptedMessageContent, public readonly device: DeviceIdentity diff --git a/src/matrix/net/HomeServerApi.ts b/src/matrix/net/HomeServerApi.ts index e9902ef8..30406c34 100644 --- a/src/matrix/net/HomeServerApi.ts +++ b/src/matrix/net/HomeServerApi.ts @@ -159,6 +159,10 @@ export class HomeServerApi { state(roomId: string, eventType: string, stateKey: string, options?: BaseRequestOptions): IHomeServerRequest { return this._get(`/rooms/${encodeURIComponent(roomId)}/state/${encodeURIComponent(eventType)}/${encodeURIComponent(stateKey)}`, {}, undefined, options); } + + sendState(roomId: string, eventType: string, stateKey: string, content: Record, options?: BaseRequestOptions): IHomeServerRequest { + return this._put(`/rooms/${encodeURIComponent(roomId)}/state/${encodeURIComponent(eventType)}/${encodeURIComponent(stateKey)}`, {}, content, options); + } getLoginFlows(): IHomeServerRequest { return this._unauthedRequest("GET", this._url("/login")); diff --git a/src/platform/types/MediaDevices.ts b/src/platform/types/MediaDevices.ts index 8bf608ce..ed9015bf 100644 --- a/src/platform/types/MediaDevices.ts +++ b/src/platform/types/MediaDevices.ts @@ -17,6 +17,7 @@ limitations under the License. export interface MediaDevices { // filter out audiooutput enumerate(): Promise; + // to assign to a video element, we downcast to WrappedTrack and use the stream property. getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise; getScreenShareTrack(): Promise; } From b2ac4bc291b869935468e724e35de22445e0c670 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 10 Mar 2022 17:39:29 +0100 Subject: [PATCH 014/323] WIP13 --- src/matrix/calls/TODO.md | 5 ++- src/matrix/calls/group/GroupCall.ts | 56 +++++++++++++++++++++++++---- src/matrix/calls/group/Member.ts | 2 +- 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index a8fbeafd..ad8fe185 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -96,12 +96,15 @@ party identification ## TODO Build basic version of PeerCall + - add candidates code Build basic version of GroupCall -Make it possible to olm encrypt the messages + - add state, block invalid actions +DONE: Make it possible to olm encrypt the messages Do work needed for state events - receiving (almost done?) - sending Expose call objects +expose volume events from audiotrack to group call Write view model write view diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 7266645d..9e80b0ff 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -27,17 +27,31 @@ import type {StateEvent} from "../../storage/types"; import type {Platform} from "../../../platform/web/Platform"; import type {EncryptedMessage} from "../../e2ee/olm/Encryption"; import type {ILogItem} from "../../../logging/types"; +import type {Storage} from "../../storage/idb/Storage"; + +export enum GroupCallState { + LocalCallFeedUninitialized = "local_call_feed_uninitialized", + InitializingLocalCallFeed = "initializing_local_call_feed", + LocalCallFeedInitialized = "local_call_feed_initialized", + Joining = "entering", + Joined = "entered", + Ended = "ended", +} export type Options = Omit & { emitUpdate: (call: GroupCall, params?: any) => void; encryptDeviceMessage: (roomId: string, message: SignallingMessage, log: ILogItem) => Promise, + storage: Storage, + ownDeviceId: string }; export class GroupCall { private readonly _members: ObservableMap = new ObservableMap(); private localMedia?: Promise; private _memberOptions: MemberOptions; - + private _state: GroupCallState = GroupCallState.LocalCallFeedInitialized; + + // TODO: keep connected state and deal constructor( private callEvent: StateEvent, private readonly room: Room, @@ -52,6 +66,10 @@ export class GroupCall { }, options); } + static async create(roomId: string, options: Options): Promise { + + } + get members(): BaseObservableMap { return this._members; } get id(): string { return this.callEvent.state_key; } @@ -60,12 +78,11 @@ export class GroupCall { return this.callEvent.content["m.terminated"] === true; } - async join(tracks: Promise) { - this.localMedia = tracks.then(tracks => LocalMedia.fromTracks(tracks)); + async join(localMedia: Promise) { + this.localMedia = localMedia; + const memberContent = await this._createOrUpdateOwnMemberStateContent(); // send m.call.member state event - const request = this.options.hsApi.sendState(this.room.id, "m.call.member", this.options.ownUserId, { - - }); + const request = this.options.hsApi.sendState(this.room.id, "m.call.member", this.options.ownUserId, memberContent); await request.response(); // send invite to all members that are < my userId for (const [,member] of this._members) { @@ -106,4 +123,31 @@ export class GroupCall { // we haven't received the m.call.member yet for this caller. buffer the device messages or create the member/call anyway? } } + + private async _createOrUpdateOwnMemberStateContent() { + const {storage} = this.options; + const txn = await storage.readTxn([storage.storeNames.roomState]); + const stateEvent = await txn.roomState.get(this.room.id, "m.call.member", this.options.ownUserId); + const stateContent = stateEvent?.event?.content ?? { + ["m.calls"]: [] + }; + const callsInfo = stateContent["m.calls"]; + let callInfo = callsInfo.find(c => c["m.call_id"] === this.id); + if (!callInfo) { + callInfo = { + ["m.call_id"]: this.id, + ["m.devices"]: [] + }; + callsInfo.push(callInfo); + } + const devicesInfo = callInfo["m.devices"]; + let deviceInfo = devicesInfo.find(d => d["device_id"] === this.options.ownDeviceId); + if (!deviceInfo) { + deviceInfo = { + ["device_id"]: this.options.ownDeviceId + }; + devicesInfo.push(deviceInfo); + } + return stateContent; + } } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index c6791568..cf020f37 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -65,7 +65,7 @@ export class Member { /** @internal */ updateCallInfo(memberCallInfo) { - + // m.calls object from the m.call.member event } /** @internal */ From b213a45c5c759ad90d3fc7adde489dcbf2ae5c09 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 11 Mar 2022 14:40:37 +0100 Subject: [PATCH 015/323] WIP: work on group call state transitions --- src/matrix/calls/CallHandler.ts | 26 ++++++- src/matrix/calls/PeerCall.ts | 5 +- src/matrix/calls/TODO.md | 8 +- src/matrix/calls/group/GroupCall.ts | 116 ++++++++++++++++++++-------- src/matrix/calls/group/Member.ts | 14 ++-- 5 files changed, 119 insertions(+), 50 deletions(-) diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 0e70fb5a..a0cd8473 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -21,6 +21,7 @@ import {handlesEventType} from "./PeerCall"; import {EventType} from "./callEventTypes"; import {GroupCall} from "./group/GroupCall"; +import type {LocalMedia} from "./LocalMedia"; import type {Room} from "../room/Room"; import type {MemberChange} from "../room/members/RoomMember"; import type {StateEvent} from "../storage/types"; @@ -49,6 +50,22 @@ export class GroupCallHandler { }); } + async createCall(roomId: string, localMedia: LocalMedia, name: string): Promise { + const call = new GroupCall(undefined, undefined, roomId, this.groupCallOptions); + this._calls.set(call.id, call); + try { + await call.create(localMedia, name); + } catch (err) { + if (err.name === "ConnectionError") { + // if we're offline, give up and remove the call again + this._calls.remove(call.id); + } + throw err; + } + await call.join(localMedia); + return call; + } + get calls(): BaseObservableMap { return this._calls; } // TODO: check and poll turn server credentials here @@ -58,7 +75,7 @@ export class GroupCallHandler { // first update call events for (const event of events) { if (event.type === EventType.GroupCall) { - this.handleCallEvent(event, room); + this.handleCallEvent(event, room.id); } } // then update members @@ -71,7 +88,8 @@ export class GroupCallHandler { /** @internal */ updateRoomMembers(room: Room, memberChanges: Map) { - + // TODO: also have map for roomId to calls, so we can easily update members + // we will also need this to get the call for a room } /** @internal */ @@ -86,7 +104,7 @@ export class GroupCallHandler { call?.handleDeviceMessage(message, userId, deviceId, log); } - private handleCallEvent(event: StateEvent, room: Room) { + private handleCallEvent(event: StateEvent, roomId: string) { const callId = event.state_key; let call = this._calls.get(callId); if (call) { @@ -95,7 +113,7 @@ export class GroupCallHandler { this._calls.remove(call.id); } } else { - call = new GroupCall(event, room, this.groupCallOptions); + call = new GroupCall(event.state_key, event.content, roomId, this.groupCallOptions); this._calls.set(call.id, call); } } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index ed8351ac..270e0fa4 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -127,12 +127,11 @@ export class PeerCall implements IDisposable { return; } this.setState(CallState.CreateOffer); - // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called for (const t of this.localMedia.tracks) { this.peerConnection.addTrack(t); } - // TODO: in case of glare, we would not go to InviteSent if we haven't started sending yet - // but we would go straight to CreateAnswer, so also need to wait for that state + // after adding the local tracks, and wait for handleNegotiation to be called, + // or invite glare where we give up our invite and answer instead await this.waitForState([CallState.InviteSent, CallState.CreateAnswer]); } diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index ad8fe185..83d706f6 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -97,12 +97,12 @@ party identification Build basic version of PeerCall - add candidates code -Build basic version of GroupCall - - add state, block invalid actions +DONE: Build basic version of GroupCall + - DONE: add state, block invalid actions DONE: Make it possible to olm encrypt the messages Do work needed for state events - - receiving (almost done?) - - sending + - DONEish: receiving (almost done?) + - DONEish: sending Expose call objects expose volume events from audiotrack to group call Write view model diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 9e80b0ff..6c26c995 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -18,6 +18,8 @@ import {ObservableMap} from "../../../observable/map/ObservableMap"; import {Member} from "./Member"; import {LocalMedia} from "../LocalMedia"; import {RoomMember} from "../../room/members/RoomMember"; +import {makeId} from "../../common"; + import type {Options as MemberOptions} from "./Member"; import type {BaseObservableMap} from "../../../observable/map/BaseObservableMap"; import type {Track} from "../../../platform/types/MediaDevices"; @@ -30,12 +32,11 @@ import type {ILogItem} from "../../../logging/types"; import type {Storage} from "../../storage/idb/Storage"; export enum GroupCallState { - LocalCallFeedUninitialized = "local_call_feed_uninitialized", - InitializingLocalCallFeed = "initializing_local_call_feed", - LocalCallFeedInitialized = "local_call_feed_initialized", - Joining = "entering", - Joined = "entered", - Ended = "ended", + Fledgling = "fledgling", + Creating = "creating", + Created = "created", + Joining = "joining", + Joined = "joined", } export type Options = Omit & { @@ -46,70 +47,112 @@ export type Options = Omit = new ObservableMap(); - private localMedia?: Promise; + private _localMedia?: LocalMedia; private _memberOptions: MemberOptions; - private _state: GroupCallState = GroupCallState.LocalCallFeedInitialized; - - // TODO: keep connected state and deal + private _state: GroupCallState; + constructor( - private callEvent: StateEvent, - private readonly room: Room, + id: string | undefined, + private callContent: Record | undefined, + private readonly roomId: string, private readonly options: Options ) { + this.id = id ?? makeId("conf-"); + this._state = id ? GroupCallState.Created : GroupCallState.Fledgling; this._memberOptions = Object.assign({ confId: this.id, emitUpdate: member => this._members.update(member.member.userId, member), encryptDeviceMessage: (message: SignallingMessage, log) => { - return this.options.encryptDeviceMessage(this.room.id, message, log); + return this.options.encryptDeviceMessage(this.roomId, message, log); } }, options); } - static async create(roomId: string, options: Options): Promise { - - } - + get localMedia(): LocalMedia | undefined { return this._localMedia; } get members(): BaseObservableMap { return this._members; } - get id(): string { return this.callEvent.state_key; } - get isTerminated(): boolean { - return this.callEvent.content["m.terminated"] === true; + return this.callContent?.["m.terminated"] === true; } - async join(localMedia: Promise) { - this.localMedia = localMedia; - const memberContent = await this._createOrUpdateOwnMemberStateContent(); + async join(localMedia: LocalMedia) { + if (this._state !== GroupCallState.Created) { + return; + } + this._state = GroupCallState.Joining; + this._localMedia = localMedia; + const memberContent = await this._joinCallMemberContent(); // send m.call.member state event - const request = this.options.hsApi.sendState(this.room.id, "m.call.member", this.options.ownUserId, memberContent); + const request = this.options.hsApi.sendState(this.roomId, "m.call.member", this.options.ownUserId, memberContent); await request.response(); // send invite to all members that are < my userId for (const [,member] of this._members) { - member.connect(this.localMedia); + member.connect(this._localMedia); + } + } + + async leave() { + const memberContent = await this._leaveCallMemberContent(); + // send m.call.member state event + if (memberContent) { + const request = this.options.hsApi.sendState(this.roomId, "m.call.member", this.options.ownUserId, memberContent); + await request.response(); } } /** @internal */ - updateCallEvent(callEvent: StateEvent) { - this.callEvent = callEvent; - // TODO: emit update + async create(localMedia: LocalMedia, name: string) { + if (this._state !== GroupCallState.Fledgling) { + return; + } + this._state = GroupCallState.Creating; + this.callContent = { + "m.type": localMedia.cameraTrack ? "m.video" : "m.voice", + "m.name": name, + "m.intent": "m.ring" + }; + const request = this.options.hsApi.sendState(this.roomId, "m.call", this.id, this.callContent); + await request.response(); + } + + /** @internal */ + updateCallEvent(callContent: Record) { + this.callContent = callContent; + if (this._state === GroupCallState.Creating) { + this._state = GroupCallState.Created; + } } /** @internal */ addMember(userId, memberCallInfo) { + if (userId === this.options.ownUserId) { + if (this._state === GroupCallState.Joining) { + this._state = GroupCallState.Joined; + } + return; + } let member = this._members.get(userId); if (member) { member.updateCallInfo(memberCallInfo); } else { - member = new Member(RoomMember.fromUserId(this.room.id, userId, "join"), this._memberOptions); - member.updateCallInfo(memberCallInfo); + member = new Member(RoomMember.fromUserId(this.roomId, userId, "join"), memberCallInfo, this._memberOptions); this._members.add(userId, member); + if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { + member.connect(this._localMedia!); + } } } /** @internal */ removeMember(userId) { + if (userId === this.options.ownUserId) { + if (this._state === GroupCallState.Joined) { + this._state = GroupCallState.Created; + } + return; + } this._members.remove(userId); } @@ -124,10 +167,10 @@ export class GroupCall { } } - private async _createOrUpdateOwnMemberStateContent() { + private async _joinCallMemberContent() { const {storage} = this.options; const txn = await storage.readTxn([storage.storeNames.roomState]); - const stateEvent = await txn.roomState.get(this.room.id, "m.call.member", this.options.ownUserId); + const stateEvent = await txn.roomState.get(this.roomId, "m.call.member", this.options.ownUserId); const stateContent = stateEvent?.event?.content ?? { ["m.calls"]: [] }; @@ -150,4 +193,13 @@ export class GroupCall { } return stateContent; } + + private async _leaveCallMemberContent(): Promise | undefined> { + const {storage} = this.options; + const txn = await storage.readTxn([storage.storeNames.roomState]); + const stateEvent = await txn.roomState.get(this.roomId, "m.call.member", this.options.ownUserId); + const callsInfo = stateEvent?.event?.content?.["m.calls"]; + callsInfo?.filter(c => c["m.call_id"] === this.id); + return stateEvent?.event.content; + } } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index cf020f37..0c574bb0 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -38,10 +38,11 @@ export type Options = Omit; + private localMedia?: LocalMedia; constructor( public readonly member: RoomMember, + private memberCallInfo: Record, private readonly options: Options ) {} @@ -53,13 +54,13 @@ export class Member { return this.peerCall?.state === CallState.Connected; } - /* @internal */ - connect(localMedia: Promise) { + /** @internal */ + connect(localMedia: LocalMedia) { this.localMedia = localMedia; // otherwise wait for it to connect if (this.member.userId < this.options.ownUserId) { this.peerCall = this._createPeerCall(makeId("c")); - this.peerCall.call(localMedia); + this.peerCall.call(Promise.resolve(localMedia.clone())); } } @@ -71,13 +72,12 @@ export class Member { /** @internal */ emitUpdate = (peerCall: PeerCall, params: any) => { if (peerCall.state === CallState.Ringing) { - peerCall.answer(this.localMedia!); + peerCall.answer(Promise.resolve(this.localMedia!)); } this.options.emitUpdate(this, params); } - /** From PeerCallHandler - * @internal */ + /** @internal */ sendSignallingMessage = async (message: SignallingMessage, log: ILogItem) => { const groupMessage = message as SignallingMessage; groupMessage.content.conf_id = this.options.confId; From 3c160c8a37c2c429f913ae2dc0cf3999b527c53a Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 11 Mar 2022 16:35:32 +0100 Subject: [PATCH 016/323] handle remote ice candidates --- src/matrix/calls/PeerCall.ts | 48 +++++++++++++++++++++++++++++++----- src/matrix/calls/TODO.md | 1 + 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 270e0fa4..fa56601e 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -105,7 +105,7 @@ export class PeerCall implements IDisposable { log(...args) { console.log.apply(console, ["WebRTC log:", ...args])}, warn(...args) { console.log.apply(console, ["WebRTC warn:", ...args])}, error(...args) { console.error.apply(console, ["WebRTC error:", ...args])}, - } + }; } get state(): CallState { return this._state; } @@ -214,9 +214,9 @@ export class PeerCall implements IDisposable { case EventType.Answer: await this.handleAnswer(message.content, partyId); break; - //case EventType.Candidates: - // await this.handleRemoteIceCandidates(message.content, partyId); - // break; + case EventType.Candidates: + await this.handleRemoteIceCandidates(message.content, partyId); + break; case EventType.Hangup: default: throw new Error(`Unknown event type for call: ${message.type}`); @@ -296,7 +296,7 @@ export class PeerCall implements IDisposable { } }; - private async handleInviteGlare(content: MCallInvite, partyId: PartyId): Promise { + private async handleInviteGlare(content: MCallInvite, partyId: PartyId): Promise { // this is only called when the ids are different const newCallId = content.call_id; if (this.callId! > newCallId) { @@ -386,7 +386,7 @@ export class PeerCall implements IDisposable { } } - private async handleAnswer(content: MCallAnswer, partyId: PartyId): Promise { + private async handleAnswer(content: MCallAnswer, partyId: PartyId): Promise { this.logger.debug(`Got answer for call ID ${this.callId} from party ID ${partyId}`); if (this._state === CallState.Ended) { @@ -424,6 +424,42 @@ export class PeerCall implements IDisposable { } } + async handleRemoteIceCandidates(content: MCallCandidates, partyId) { + if (this.state === CallState.Ended) { + //debuglog("Ignoring remote ICE candidate because call has ended"); + return; + } + + const candidates = content.candidates; + if (!candidates) { + this.logger.info(`Call ${this.callId} Ignoring candidates event with no candidates!`); + return; + } + + const fromPartyId = content.version === 0 ? null : partyId || null; + + if (this.opponentPartyId === undefined) { + // we haven't picked an opponent yet so save the candidates + this.logger.info(`Call ${this.callId} Buffering ${candidates.length} candidates until we pick an opponent`); + const bufferedCandidates = this.remoteCandidateBuffer!.get(fromPartyId) || []; + bufferedCandidates.push(...candidates); + this.remoteCandidateBuffer!.set(fromPartyId, bufferedCandidates); + return; + } + + if (this.opponentPartyId !== partyId) { + this.logger.info( + `Call ${this.callId} `+ + `Ignoring candidates from party ID ${partyId}: ` + + `we have chosen party ID ${this.opponentPartyId}`, + ); + + return; + } + + await this.addIceCandidates(candidates); + } + // private async onNegotiateReceived(event: MatrixEvent): Promise { // const content = event.getContent(); // const description = content.description; diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index 83d706f6..23306c56 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -103,6 +103,7 @@ DONE: Make it possible to olm encrypt the messages Do work needed for state events - DONEish: receiving (almost done?) - DONEish: sending +logging Expose call objects expose volume events from audiotrack to group call Write view model From f6744926850f7e483f07b7e275a675aa2bb1915d Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 11 Mar 2022 16:56:21 +0100 Subject: [PATCH 017/323] remove local media promises (handle them outside of call code) + glare --- src/matrix/calls/PeerCall.ts | 41 +++++++++----------------------- src/matrix/calls/TODO.md | 8 +++---- src/matrix/calls/group/Member.ts | 4 ++-- 3 files changed, 17 insertions(+), 36 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index fa56601e..8725756f 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -114,18 +114,12 @@ export class PeerCall implements IDisposable { return this.peerConnection.remoteTracks; } - async call(localMediaPromise: Promise): Promise { + async call(localMedia: LocalMedia): Promise { if (this._state !== CallState.Fledgling) { return; } + this.localMedia = localMedia; this.direction = CallDirection.Outbound; - this.setState(CallState.WaitLocalMedia); - try { - this.localMedia = await localMediaPromise; - } catch (err) { - this.setState(CallState.Ended); - return; - } this.setState(CallState.CreateOffer); for (const t of this.localMedia.tracks) { this.peerConnection.addTrack(t); @@ -135,17 +129,11 @@ export class PeerCall implements IDisposable { await this.waitForState([CallState.InviteSent, CallState.CreateAnswer]); } - async answer(localMediaPromise: Promise): Promise { + async answer(localMedia: LocalMedia): Promise { if (this._state !== CallState.Ringing) { return; } - this.setState(CallState.WaitLocalMedia); - try { - this.localMedia = await localMediaPromise; - } catch (err) { - this.setState(CallState.Ended); - return; - } + this.localMedia = localMedia; this.setState(CallState.CreateAnswer); for (const t of this.localMedia.tracks) { this.peerConnection.addTrack(t); @@ -274,7 +262,7 @@ export class PeerCall implements IDisposable { lifetime: CALL_TIMEOUT_MS }; if (this._state === CallState.CreateOffer) { - await this.options.sendSignallingMessage({type: EventType.Invite, content}); + await this.options.sendSignallingMessage({type: EventType.Invite, content}, undefined); this.setState(CallState.InviteSent); } else if (this._state === CallState.Connected || this._state === CallState.Connecting) { // send Negotiate message @@ -304,21 +292,15 @@ export class PeerCall implements IDisposable { "Glare detected: answering incoming call " + newCallId + " and canceling outgoing call " + this.callId, ); - - /* - first, we should set CallDirection - we should anser the call - */ - - // TODO: review states to be unambigous, WaitLocalMedia for sending offer or answer? // How do we interrupt `call()`? well, perhaps we need to not just await InviteSent but also CreateAnswer? - if (this._state === CallState.Fledgling || this._state === CallState.CreateOffer || this._state === CallState.WaitLocalMedia) { - + if (this._state === CallState.Fledgling || this._state === CallState.CreateOffer) { + // TODO: don't send invite! } else { await this.sendHangupWithCallId(this.callId, CallErrorCode.Replaced); } await this.handleInvite(content, partyId); - await this.answer(Promise.resolve(this.localMedia!)); + // TODO: need to skip state check + await this.answer(this.localMedia!); } else { this.logger.log( "Glare detected: rejecting incoming call " + newCallId + @@ -328,7 +310,7 @@ export class PeerCall implements IDisposable { } } - private async handleFirstInvite(content: MCallInvite, partyId: PartyId): Promise { + private async handleFirstInvite(content: MCallInvite, partyId: PartyId): Promise { if (this._state !== CallState.Fledgling || this.opponentPartyId !== undefined) { // TODO: hangup or ignore? return; @@ -336,7 +318,7 @@ export class PeerCall implements IDisposable { await this.handleInvite(content, partyId); } - private async handleInvite(content: MCallInvite, partyId: PartyId): Promise { + private async handleInvite(content: MCallInvite, partyId: PartyId): Promise { // we must set the party ID before await-ing on anything: the call event // handler will start giving us more call events (eg. candidates) so if @@ -705,7 +687,6 @@ export enum CallParty { export enum CallState { Fledgling = 'fledgling', - WaitLocalMedia = 'wait_local_media', CreateOffer = 'create_offer', InviteSent = 'invite_sent', CreateAnswer = 'create_answer', diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index 23306c56..a07da60e 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -145,10 +145,10 @@ FROM CALLER FROM CALLEE Fledgling Fledgling V `call()` V `handleInvite()`: setRemoteDescription(event.offer), add buffered candidates -WaitLocalMedia Ringing - V media promise resolves V `answer()` - V add local tracks WaitLocalMedia -CreateOffer V media promise resolves + V Ringing + V V `answer()` +CreateOffer V + V add local tracks V V wait for negotionneeded events V add local tracks V setLocalDescription() CreateAnswer V send invite event V setLocalDescription(createAnswer()) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 0c574bb0..bd1613cd 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -60,7 +60,7 @@ export class Member { // otherwise wait for it to connect if (this.member.userId < this.options.ownUserId) { this.peerCall = this._createPeerCall(makeId("c")); - this.peerCall.call(Promise.resolve(localMedia.clone())); + this.peerCall.call(localMedia); } } @@ -72,7 +72,7 @@ export class Member { /** @internal */ emitUpdate = (peerCall: PeerCall, params: any) => { if (peerCall.state === CallState.Ringing) { - peerCall.answer(Promise.resolve(this.localMedia!)); + peerCall.answer(this.localMedia!); } this.options.emitUpdate(this, params); } From 1bccbbfa082ea2e7a0f3157fd54d773612ef31c7 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 16 Mar 2022 19:10:51 +0100 Subject: [PATCH 018/323] fix typescript errors --- src/matrix/calls/LocalMedia.ts | 6 +- src/matrix/calls/PeerCall.ts | 101 +++++++++++++++++++++++++++++---- 2 files changed, 92 insertions(+), 15 deletions(-) diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 2429ff2c..4f5d3b7a 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {StreamPurpose} from "../../platform/types/WebRTC"; +import {SDPStreamMetadataPurpose} from "./callEventTypes"; import {Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; import {SDPStreamMetadata} from "./callEventTypes"; @@ -42,14 +42,14 @@ export class LocalMedia { const userMediaTrack = this.microphoneTrack ?? this.cameraTrack; if (userMediaTrack) { metadata[userMediaTrack.streamId] = { - purpose: StreamPurpose.UserMedia, + purpose: SDPStreamMetadataPurpose.Usermedia, audio_muted: this.microphoneTrack?.muted ?? false, video_muted: this.cameraTrack?.muted ?? false, }; } if (this.screenShareTrack) { metadata[this.screenShareTrack.streamId] = { - purpose: StreamPurpose.ScreenShare + purpose: SDPStreamMetadataPurpose.Screenshare }; } return metadata; diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 8725756f..a4ea8cca 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -21,6 +21,7 @@ import {Disposables, IDisposable} from "../../utils/Disposables"; import type {Room} from "../room/Room"; import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; +import {Instance as logger} from "../../logging/NullLogger"; import type {TimeoutCreator, Timeout} from "../../platform/types/types"; import {WebRTC, PeerConnection, PeerConnectionHandler, DataChannel} from "../../platform/types/WebRTC"; @@ -81,16 +82,28 @@ export class PeerCall implements IDisposable { // perfect negotiation flags private makingOffer: boolean = false; private ignoreOffer: boolean = false; + + private sentEndOfCandidates: boolean = false; + private iceDisconnectedTimeout?: Timeout; + constructor( - private callId: string, // generated or from invite + private callId: string, private readonly options: Options ) { const outer = this; this.peerConnection = options.webRTC.createPeerConnection({ - onIceConnectionStateChange(state: RTCIceConnectionState) {}, - onLocalIceCandidate(candidate: RTCIceCandidate) {}, - onIceGatheringStateChange(state: RTCIceGatheringState) {}, - onRemoteTracksChanged(tracks: Track[]) {}, + onIceConnectionStateChange(state: RTCIceConnectionState) { + outer.onIceConnectionStateChange(state); + }, + onLocalIceCandidate(candidate: RTCIceCandidate) { + outer.handleLocalIceCandidate(candidate); + }, + onIceGatheringStateChange(state: RTCIceGatheringState) { + outer.handleIceGatheringState(state); + }, + onRemoteTracksChanged(tracks: Track[]) { + outer.options.emitUpdate(outer, undefined); + }, onDataChannelChanged(dataChannel: DataChannel | undefined) {}, onNegotiationNeeded() { const promiseCreator = () => outer.handleNegotiation(); @@ -158,7 +171,7 @@ export class PeerCall implements IDisposable { } // Allow a short time for initial candidates to be gathered await this.delay(200); - this.sendAnswer(); + await this.sendAnswer(); } async setMedia(localMediaPromise: Promise) { @@ -187,7 +200,11 @@ export class PeerCall implements IDisposable { } - async hangup(errorCode: CallErrorCode) { + async hangup(errorCode: CallErrorCode): Promise { + if (this._state !== CallState.Ended) { + this._state = CallState.Ended; + await this.sendHangupWithCallId(this.callId, errorCode); + } } async handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId, log: ILogItem): Promise { @@ -222,7 +239,7 @@ export class PeerCall implements IDisposable { return this.options.sendSignallingMessage({ type: EventType.Hangup, content - }, undefined); + }, logger.item); } // calls are serialized and deduplicated by responsePromiseChain @@ -262,7 +279,7 @@ export class PeerCall implements IDisposable { lifetime: CALL_TIMEOUT_MS }; if (this._state === CallState.CreateOffer) { - await this.options.sendSignallingMessage({type: EventType.Invite, content}, undefined); + await this.options.sendSignallingMessage({type: EventType.Invite, content}, logger.item); this.setState(CallState.InviteSent); } else if (this._state === CallState.Connected || this._state === CallState.Connecting) { // send Negotiate message @@ -406,7 +423,42 @@ export class PeerCall implements IDisposable { } } - async handleRemoteIceCandidates(content: MCallCandidates, partyId) { + private handleIceGatheringState(state: RTCIceGatheringState) { + this.logger.debug(`Call ${this.callId} ice gathering state changed to ${state}`); + if (state === 'complete' && !this.sentEndOfCandidates) { + // If we didn't get an empty-string candidate to signal the end of candidates, + // create one ourselves now gathering has finished. + // We cast because the interface lists all the properties as required but we + // only want to send 'candidate' + // XXX: We probably want to send either sdpMid or sdpMLineIndex, as it's not strictly + // correct to have a candidate that lacks both of these. We'd have to figure out what + // previous candidates had been sent with and copy them. + const c = { + candidate: '', + } as RTCIceCandidate; + this.queueCandidate(c); + this.sentEndOfCandidates = true; + } + } + + private handleLocalIceCandidate(candidate: RTCIceCandidate) { + this.logger.debug( + "Call " + this.callId + " got local ICE " + candidate.sdpMid + " candidate: " + + candidate.candidate, + ); + + if (this._state === CallState.Ended) return; + + // As with the offer, note we need to make a copy of this object, not + // pass the original: that broke in Chrome ~m43. + if (candidate.candidate !== '' || !this.sentEndOfCandidates) { + this.queueCandidate(candidate); + + if (candidate.candidate === '') this.sentEndOfCandidates = true; + } + } + + private async handleRemoteIceCandidates(content: MCallCandidates, partyId) { if (this.state === CallState.Ended) { //debuglog("Ignoring remote ICE candidate because call has ended"); return; @@ -512,7 +564,7 @@ export class PeerCall implements IDisposable { this.candidateSendQueue = []; try { - await this.options.sendSignallingMessage({type: EventType.Answer, content: answerContent}, undefined); + await this.options.sendSignallingMessage({type: EventType.Answer, content: answerContent}, logger.item); } catch (error) { this.terminate(CallParty.Local, CallErrorCode.SendAnswer, false); throw error; @@ -561,7 +613,7 @@ export class PeerCall implements IDisposable { version: 1, candidates }, - }, undefined); + }, logger.item); // Try to send candidates again just in case we received more candidates while sending. this.sendCandidateQueue(); } catch (error) { @@ -620,6 +672,31 @@ export class PeerCall implements IDisposable { } } + private onIceConnectionStateChange = (state: RTCIceConnectionState): void => { + if (this._state === CallState.Ended) { + return; // because ICE can still complete as we're ending the call + } + this.logger.debug( + "Call ID " + this.callId + ": ICE connection state changed to: " + state, + ); + // ideally we'd consider the call to be connected when we get media but + // chrome doesn't implement any of the 'onstarted' events yet + if (state == 'connected') { + this.iceDisconnectedTimeout?.abort(); + this.iceDisconnectedTimeout = undefined; + this.setState(CallState.Connected); + } else if (state == 'failed') { + this.iceDisconnectedTimeout?.abort(); + this.iceDisconnectedTimeout = undefined; + this.hangup(CallErrorCode.IceFailed); + } else if (state == 'disconnected') { + this.iceDisconnectedTimeout = this.options.createTimeout(30 * 1000); + this.iceDisconnectedTimeout.elapsed().then(() => { + this.hangup(CallErrorCode.IceFailed); + }, () => { /* ignore AbortError */ }); + } + }; + private setState(state: CallState): void { const oldState = this._state; this._state = state; From 07bc0a2376d2e910521071f9448d20f11f4849fe Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 17 Mar 2022 11:30:40 +0100 Subject: [PATCH 019/323] move observable values each in their own file --- src/domain/navigation/Navigation.js | 3 +- src/domain/session/RoomGridViewModel.js | 2 +- src/domain/session/RoomViewModelObservable.js | 2 +- .../room/timeline/ReactionsViewModel.js | 2 +- .../session/settings/KeyBackupViewModel.js | 5 +- src/lib.ts | 8 +- src/matrix/Client.js | 2 +- src/matrix/Session.js | 16 +- src/matrix/Sync.js | 2 +- src/matrix/e2ee/megolm/keybackup/KeyBackup.ts | 2 +- src/matrix/net/Reconnector.ts | 2 +- src/matrix/room/BaseRoom.js | 2 +- src/matrix/room/ObservedEventMap.js | 2 +- src/mocks/Clock.js | 2 +- src/observable/ObservableValue.ts | 248 ------------------ src/observable/value/BaseObservableValue.ts | 83 ++++++ .../value/FlatMapObservableValue.ts | 109 ++++++++ src/observable/value/ObservableValue.ts | 82 ++++++ src/observable/value/PickMapObservable.ts | 89 +++++++ .../value/RetainedObservableValue.ts | 31 +++ src/platform/web/dom/History.js | 2 +- src/platform/web/dom/OnlineStatus.js | 2 +- src/utils/AbortableOperation.ts | 3 +- 23 files changed, 429 insertions(+), 272 deletions(-) delete mode 100644 src/observable/ObservableValue.ts create mode 100644 src/observable/value/BaseObservableValue.ts create mode 100644 src/observable/value/FlatMapObservableValue.ts create mode 100644 src/observable/value/ObservableValue.ts create mode 100644 src/observable/value/PickMapObservable.ts create mode 100644 src/observable/value/RetainedObservableValue.ts diff --git a/src/domain/navigation/Navigation.js b/src/domain/navigation/Navigation.js index 340ae0d5..c2b2b54c 100644 --- a/src/domain/navigation/Navigation.js +++ b/src/domain/navigation/Navigation.js @@ -14,7 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {BaseObservableValue, ObservableValue} from "../../observable/ObservableValue"; +import {ObservableValue} from "../../observable/value/ObservableValue"; +import {BaseObservableValue} from "../../observable/value/BaseObservableValue"; export class Navigation { constructor(allowsChild) { diff --git a/src/domain/session/RoomGridViewModel.js b/src/domain/session/RoomGridViewModel.js index a7d19054..5d42f0f6 100644 --- a/src/domain/session/RoomGridViewModel.js +++ b/src/domain/session/RoomGridViewModel.js @@ -186,7 +186,7 @@ export class RoomGridViewModel extends ViewModel { } import {createNavigation} from "../navigation/index.js"; -import {ObservableValue} from "../../observable/ObservableValue"; +import {ObservableValue} from "../../observable/value/ObservableValue"; export function tests() { class RoomVMMock { diff --git a/src/domain/session/RoomViewModelObservable.js b/src/domain/session/RoomViewModelObservable.js index 52833332..86e2bb5d 100644 --- a/src/domain/session/RoomViewModelObservable.js +++ b/src/domain/session/RoomViewModelObservable.js @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {ObservableValue} from "../../observable/ObservableValue"; +import {ObservableValue} from "../../observable/value/ObservableValue"; import {RoomStatus} from "../../matrix/room/common"; /** diff --git a/src/domain/session/room/timeline/ReactionsViewModel.js b/src/domain/session/room/timeline/ReactionsViewModel.js index 4f366af0..214ec17f 100644 --- a/src/domain/session/room/timeline/ReactionsViewModel.js +++ b/src/domain/session/room/timeline/ReactionsViewModel.js @@ -189,7 +189,7 @@ import {HomeServer as MockHomeServer} from "../../../../mocks/HomeServer.js"; // other imports import {BaseMessageTile} from "./tiles/BaseMessageTile.js"; import {MappedList} from "../../../../observable/list/MappedList"; -import {ObservableValue} from "../../../../observable/ObservableValue"; +import {ObservableValue} from "../../../../observable/value/ObservableValue"; import {PowerLevels} from "../../../../matrix/room/PowerLevels.js"; export function tests() { diff --git a/src/domain/session/settings/KeyBackupViewModel.js b/src/domain/session/settings/KeyBackupViewModel.js index 243b0d7c..b4ee9a0e 100644 --- a/src/domain/session/settings/KeyBackupViewModel.js +++ b/src/domain/session/settings/KeyBackupViewModel.js @@ -17,6 +17,7 @@ limitations under the License. import {ViewModel} from "../../ViewModel"; import {KeyType} from "../../../matrix/ssss/index"; import {createEnum} from "../../../utils/enum"; +import {FlatMapObservableValue} from "../../../observable/value/FlatMapObservableValue"; export const Status = createEnum("Enabled", "SetupKey", "SetupPhrase", "Pending", "NewVersionAvailable"); export const BackupWriteStatus = createEnum("Writing", "Stopped", "Done", "Pending"); @@ -29,8 +30,8 @@ export class KeyBackupViewModel extends ViewModel { this._isBusy = false; this._dehydratedDeviceId = undefined; this._status = undefined; - this._backupOperation = this._session.keyBackup.flatMap(keyBackup => keyBackup.operationInProgress); - this._progress = this._backupOperation.flatMap(op => op.progress); + this._backupOperation = new FlatMapObservableValue(this._session.keyBackup, keyBackup => keyBackup.operationInProgress); + this._progress = new FlatMapObservableValue(this._backupOperation, op => op.progress); this.track(this._backupOperation.subscribe(() => { // see if needsNewKey might be set this._reevaluateStatus(); diff --git a/src/lib.ts b/src/lib.ts index a0ada84f..cb939949 100644 --- a/src/lib.ts +++ b/src/lib.ts @@ -46,8 +46,6 @@ export { ConcatList, ObservableMap } from "./observable/index"; -export { - BaseObservableValue, - ObservableValue, - RetainedObservableValue -} from "./observable/ObservableValue"; +export {BaseObservableValue} from "./observable/value/BaseObservableValue"; +export {ObservableValue} from "./observable/value/ObservableValue"; +export {RetainedObservableValue} from "./observable/value/RetainedObservableValue"; diff --git a/src/matrix/Client.js b/src/matrix/Client.js index b24c1ec9..83861cb7 100644 --- a/src/matrix/Client.js +++ b/src/matrix/Client.js @@ -18,7 +18,7 @@ limitations under the License. import {createEnum} from "../utils/enum"; import {lookupHomeserver} from "./well-known.js"; import {AbortableOperation} from "../utils/AbortableOperation"; -import {ObservableValue} from "../observable/ObservableValue"; +import {ObservableValue} from "../observable/value/ObservableValue"; import {HomeServerApi} from "./net/HomeServerApi"; import {Reconnector, ConnectionStatus} from "./net/Reconnector"; import {ExponentialRetryDelay} from "./net/ExponentialRetryDelay"; diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 94fb5dee..69cd9ee0 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -45,7 +45,8 @@ import { keyFromDehydratedDeviceKey as createSSSSKeyFromDehydratedDeviceKey } from "./ssss/index"; import {SecretStorage} from "./ssss/SecretStorage"; -import {ObservableValue, RetainedObservableValue} from "../observable/ObservableValue"; +import {ObservableValue} from "../observable/value/ObservableValue"; +import {RetainedObservableValue} from "../observable/value/RetainedObservableValue"; const PICKLE_KEY = "DEFAULT_KEY"; const PUSHER_KEY = "pusher"; @@ -997,9 +998,18 @@ export function tests() { return { "session data is not modified until after sync": async (assert) => { - const session = new Session({storage: createStorageMock({ + const storage = createStorageMock({ sync: {token: "a", filterId: 5} - }), sessionInfo: {userId: ""}}); + }); + const session = new Session({ + storage, + sessionInfo: {userId: ""}, + platform: { + clock: { + createTimeout: () => undefined + } + } + }); await session.load(); let syncSet = false; const syncTxn = { diff --git a/src/matrix/Sync.js b/src/matrix/Sync.js index 3574213e..8e880def 100644 --- a/src/matrix/Sync.js +++ b/src/matrix/Sync.js @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {ObservableValue} from "../observable/ObservableValue"; +import {ObservableValue} from "../observable/value/ObservableValue"; import {createEnum} from "../utils/enum"; const INCREMENTAL_TIMEOUT = 30000; diff --git a/src/matrix/e2ee/megolm/keybackup/KeyBackup.ts b/src/matrix/e2ee/megolm/keybackup/KeyBackup.ts index 43631552..3da1c704 100644 --- a/src/matrix/e2ee/megolm/keybackup/KeyBackup.ts +++ b/src/matrix/e2ee/megolm/keybackup/KeyBackup.ts @@ -19,7 +19,7 @@ import {StoredRoomKey, keyFromBackup} from "../decryption/RoomKey"; import {MEGOLM_ALGORITHM} from "../../common"; import * as Curve25519 from "./Curve25519"; import {AbortableOperation} from "../../../../utils/AbortableOperation"; -import {ObservableValue} from "../../../../observable/ObservableValue"; +import {ObservableValue} from "../../../../observable/value/ObservableValue"; import {SetAbortableFn} from "../../../../utils/AbortableOperation"; import type {BackupInfo, SessionData, SessionKeyInfo, SessionInfo, KeyBackupPayload} from "./types"; diff --git a/src/matrix/net/Reconnector.ts b/src/matrix/net/Reconnector.ts index bc54ab73..a3739425 100644 --- a/src/matrix/net/Reconnector.ts +++ b/src/matrix/net/Reconnector.ts @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {ObservableValue} from "../../observable/ObservableValue"; +import {ObservableValue} from "../../observable/value/ObservableValue"; import type {ExponentialRetryDelay} from "./ExponentialRetryDelay"; import type {TimeMeasure} from "../../platform/web/dom/Clock.js"; import type {OnlineStatus} from "../../platform/web/dom/OnlineStatus.js"; diff --git a/src/matrix/room/BaseRoom.js b/src/matrix/room/BaseRoom.js index dda3e2e5..cc65a320 100644 --- a/src/matrix/room/BaseRoom.js +++ b/src/matrix/room/BaseRoom.js @@ -29,7 +29,7 @@ import {ObservedEventMap} from "./ObservedEventMap.js"; import {DecryptionSource} from "../e2ee/common.js"; import {ensureLogItem} from "../../logging/utils"; import {PowerLevels} from "./PowerLevels.js"; -import {RetainedObservableValue} from "../../observable/ObservableValue"; +import {RetainedObservableValue} from "../../observable/value/RetainedObservableValue"; import {TimelineReader} from "./timeline/persistence/TimelineReader"; const EVENT_ENCRYPTED_TYPE = "m.room.encrypted"; diff --git a/src/matrix/room/ObservedEventMap.js b/src/matrix/room/ObservedEventMap.js index 6b20f85e..8ee1bca8 100644 --- a/src/matrix/room/ObservedEventMap.js +++ b/src/matrix/room/ObservedEventMap.js @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {BaseObservableValue} from "../../observable/ObservableValue"; +import {BaseObservableValue} from "../../observable/value/BaseObservableValue"; export class ObservedEventMap { constructor(notifyEmpty) { diff --git a/src/mocks/Clock.js b/src/mocks/Clock.js index 440c4cb4..b9d5457d 100644 --- a/src/mocks/Clock.js +++ b/src/mocks/Clock.js @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {ObservableValue} from "../observable/ObservableValue"; +import {ObservableValue} from "../observable/value/ObservableValue"; class Timeout { constructor(elapsed, ms) { diff --git a/src/observable/ObservableValue.ts b/src/observable/ObservableValue.ts deleted file mode 100644 index ad0a226d..00000000 --- a/src/observable/ObservableValue.ts +++ /dev/null @@ -1,248 +0,0 @@ -/* -Copyright 2020 Bruno Windels - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import {AbortError} from "../utils/error"; -import {BaseObservable} from "./BaseObservable"; -import type {SubscriptionHandle} from "./BaseObservable"; - -// like an EventEmitter, but doesn't have an event type -export abstract class BaseObservableValue extends BaseObservable<(value: T) => void> { - emit(argument: T) { - for (const h of this._handlers) { - h(argument); - } - } - - abstract get(): T; - - waitFor(predicate: (value: T) => boolean): IWaitHandle { - if (predicate(this.get())) { - return new ResolvedWaitForHandle(Promise.resolve(this.get())); - } else { - return new WaitForHandle(this, predicate); - } - } - - flatMap(mapper: (value: T) => (BaseObservableValue | undefined)): BaseObservableValue { - return new FlatMapObservableValue(this, mapper); - } -} - -interface IWaitHandle { - promise: Promise; - dispose(): void; -} - -class WaitForHandle implements IWaitHandle { - private _promise: Promise - private _reject: ((reason?: any) => void) | null; - private _subscription: (() => void) | null; - - constructor(observable: BaseObservableValue, predicate: (value: T) => boolean) { - this._promise = new Promise((resolve, reject) => { - this._reject = reject; - this._subscription = observable.subscribe(v => { - if (predicate(v)) { - this._reject = null; - resolve(v); - this.dispose(); - } - }); - }); - } - - get promise(): Promise { - return this._promise; - } - - dispose() { - if (this._subscription) { - this._subscription(); - this._subscription = null; - } - if (this._reject) { - this._reject(new AbortError()); - this._reject = null; - } - } -} - -class ResolvedWaitForHandle implements IWaitHandle { - constructor(public promise: Promise) {} - dispose() {} -} - -export class ObservableValue extends BaseObservableValue { - private _value: T; - - constructor(initialValue: T) { - super(); - this._value = initialValue; - } - - get(): T { - return this._value; - } - - set(value: T): void { - if (value !== this._value) { - this._value = value; - this.emit(this._value); - } - } -} - -export class RetainedObservableValue extends ObservableValue { - private _freeCallback: () => void; - - constructor(initialValue: T, freeCallback: () => void) { - super(initialValue); - this._freeCallback = freeCallback; - } - - onUnsubscribeLast() { - super.onUnsubscribeLast(); - this._freeCallback(); - } -} - -export class FlatMapObservableValue extends BaseObservableValue { - private sourceSubscription?: SubscriptionHandle; - private targetSubscription?: SubscriptionHandle; - - constructor( - private readonly source: BaseObservableValue

, - private readonly mapper: (value: P) => (BaseObservableValue | undefined) - ) { - super(); - } - - onUnsubscribeLast() { - super.onUnsubscribeLast(); - this.sourceSubscription = this.sourceSubscription!(); - if (this.targetSubscription) { - this.targetSubscription = this.targetSubscription(); - } - } - - onSubscribeFirst() { - super.onSubscribeFirst(); - this.sourceSubscription = this.source.subscribe(() => { - this.updateTargetSubscription(); - this.emit(this.get()); - }); - this.updateTargetSubscription(); - } - - private updateTargetSubscription() { - const sourceValue = this.source.get(); - if (sourceValue) { - const target = this.mapper(sourceValue); - if (target) { - if (!this.targetSubscription) { - this.targetSubscription = target.subscribe(() => this.emit(this.get())); - } - return; - } - } - // if no sourceValue or target - if (this.targetSubscription) { - this.targetSubscription = this.targetSubscription(); - } - } - - get(): C | undefined { - const sourceValue = this.source.get(); - if (!sourceValue) { - return undefined; - } - const mapped = this.mapper(sourceValue); - return mapped?.get(); - } -} - -export function tests() { - return { - "set emits an update": assert => { - const a = new ObservableValue(0); - let fired = false; - const subscription = a.subscribe(v => { - fired = true; - assert.strictEqual(v, 5); - }); - a.set(5); - assert(fired); - subscription(); - }, - "set doesn't emit if value hasn't changed": assert => { - const a = new ObservableValue(5); - let fired = false; - const subscription = a.subscribe(() => { - fired = true; - }); - a.set(5); - a.set(5); - assert(!fired); - subscription(); - }, - "waitFor promise resolves on matching update": async assert => { - const a = new ObservableValue(5); - const handle = a.waitFor(v => v === 6); - Promise.resolve().then(() => { - a.set(6); - }); - await handle.promise; - assert.strictEqual(a.get(), 6); - }, - "waitFor promise rejects when disposed": async assert => { - const a = new ObservableValue(0); - const handle = a.waitFor(() => false); - Promise.resolve().then(() => { - handle.dispose(); - }); - await assert.rejects(handle.promise, AbortError); - }, - "flatMap.get": assert => { - const a = new ObservableValue}>(undefined); - const countProxy = a.flatMap(a => a!.count); - assert.strictEqual(countProxy.get(), undefined); - const count = new ObservableValue(0); - a.set({count}); - assert.strictEqual(countProxy.get(), 0); - }, - "flatMap update from source": assert => { - const a = new ObservableValue}>(undefined); - const updates: (number | undefined)[] = []; - a.flatMap(a => a!.count).subscribe(count => { - updates.push(count); - }); - const count = new ObservableValue(0); - a.set({count}); - assert.deepEqual(updates, [0]); - }, - "flatMap update from target": assert => { - const a = new ObservableValue}>(undefined); - const updates: (number | undefined)[] = []; - a.flatMap(a => a!.count).subscribe(count => { - updates.push(count); - }); - const count = new ObservableValue(0); - a.set({count}); - count.set(5); - assert.deepEqual(updates, [0, 5]); - } - } -} diff --git a/src/observable/value/BaseObservableValue.ts b/src/observable/value/BaseObservableValue.ts new file mode 100644 index 00000000..85437262 --- /dev/null +++ b/src/observable/value/BaseObservableValue.ts @@ -0,0 +1,83 @@ +/* +Copyright 2020 Bruno Windels + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {AbortError} from "../../utils/error"; +import {BaseObservable} from "../BaseObservable"; +import type {SubscriptionHandle} from "../BaseObservable"; +import {FlatMapObservableValue} from "./FlatMapObservableValue"; + +// like an EventEmitter, but doesn't have an event type +export abstract class BaseObservableValue extends BaseObservable<(value: T) => void> { + emit(argument: T) { + for (const h of this._handlers) { + h(argument); + } + } + + abstract get(): T; + + waitFor(predicate: (value: T) => boolean): IWaitHandle { + if (predicate(this.get())) { + return new ResolvedWaitForHandle(Promise.resolve(this.get())); + } else { + return new WaitForHandle(this, predicate); + } + } +} + +interface IWaitHandle { + promise: Promise; + dispose(): void; +} + +class WaitForHandle implements IWaitHandle { + private _promise: Promise + private _reject: ((reason?: any) => void) | null; + private _subscription: (() => void) | null; + + constructor(observable: BaseObservableValue, predicate: (value: T) => boolean) { + this._promise = new Promise((resolve, reject) => { + this._reject = reject; + this._subscription = observable.subscribe(v => { + if (predicate(v)) { + this._reject = null; + resolve(v); + this.dispose(); + } + }); + }); + } + + get promise(): Promise { + return this._promise; + } + + dispose() { + if (this._subscription) { + this._subscription(); + this._subscription = null; + } + if (this._reject) { + this._reject(new AbortError()); + this._reject = null; + } + } +} + +class ResolvedWaitForHandle implements IWaitHandle { + constructor(public promise: Promise) {} + dispose() {} +} diff --git a/src/observable/value/FlatMapObservableValue.ts b/src/observable/value/FlatMapObservableValue.ts new file mode 100644 index 00000000..9dff07a6 --- /dev/null +++ b/src/observable/value/FlatMapObservableValue.ts @@ -0,0 +1,109 @@ +/* +Copyright 2020 Bruno Windels + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {BaseObservableValue} from "./BaseObservableValue"; +import {SubscriptionHandle} from "../BaseObservable"; + +export class FlatMapObservableValue extends BaseObservableValue { + private sourceSubscription?: SubscriptionHandle; + private targetSubscription?: SubscriptionHandle; + + constructor( + private readonly source: BaseObservableValue

, + private readonly mapper: (value: P) => (BaseObservableValue | undefined) + ) { + super(); + } + + onUnsubscribeLast() { + super.onUnsubscribeLast(); + this.sourceSubscription = this.sourceSubscription!(); + if (this.targetSubscription) { + this.targetSubscription = this.targetSubscription(); + } + } + + onSubscribeFirst() { + super.onSubscribeFirst(); + this.sourceSubscription = this.source.subscribe(() => { + this.updateTargetSubscription(); + this.emit(this.get()); + }); + this.updateTargetSubscription(); + } + + private updateTargetSubscription() { + const sourceValue = this.source.get(); + if (sourceValue) { + const target = this.mapper(sourceValue); + if (target) { + if (!this.targetSubscription) { + this.targetSubscription = target.subscribe(() => this.emit(this.get())); + } + return; + } + } + // if no sourceValue or target + if (this.targetSubscription) { + this.targetSubscription = this.targetSubscription(); + } + } + + get(): C | undefined { + const sourceValue = this.source.get(); + if (!sourceValue) { + return undefined; + } + const mapped = this.mapper(sourceValue); + return mapped?.get(); + } +} + +import {ObservableValue} from "./ObservableValue"; + +export function tests() { + return { + "flatMap.get": assert => { + const a = new ObservableValue}>(undefined); + const countProxy = new FlatMapObservableValue(a, a => a!.count); + assert.strictEqual(countProxy.get(), undefined); + const count = new ObservableValue(0); + a.set({count}); + assert.strictEqual(countProxy.get(), 0); + }, + "flatMap update from source": assert => { + const a = new ObservableValue}>(undefined); + const updates: (number | undefined)[] = []; + new FlatMapObservableValue(a, a => a!.count).subscribe(count => { + updates.push(count); + }); + const count = new ObservableValue(0); + a.set({count}); + assert.deepEqual(updates, [0]); + }, + "flatMap update from target": assert => { + const a = new ObservableValue}>(undefined); + const updates: (number | undefined)[] = []; + new FlatMapObservableValue(a, a => a!.count).subscribe(count => { + updates.push(count); + }); + const count = new ObservableValue(0); + a.set({count}); + count.set(5); + assert.deepEqual(updates, [0, 5]); + } + } +} diff --git a/src/observable/value/ObservableValue.ts b/src/observable/value/ObservableValue.ts new file mode 100644 index 00000000..d75a0d76 --- /dev/null +++ b/src/observable/value/ObservableValue.ts @@ -0,0 +1,82 @@ +/* +Copyright 2020 Bruno Windels + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {AbortError} from "../../utils/error"; +import {BaseObservableValue} from "./BaseObservableValue"; + +export class ObservableValue extends BaseObservableValue { + private _value: T; + + constructor(initialValue: T) { + super(); + this._value = initialValue; + } + + get(): T { + return this._value; + } + + set(value: T): void { + if (value !== this._value) { + this._value = value; + this.emit(this._value); + } + } +} + +export function tests() { + return { + "set emits an update": assert => { + const a = new ObservableValue(0); + let fired = false; + const subscription = a.subscribe(v => { + fired = true; + assert.strictEqual(v, 5); + }); + a.set(5); + assert(fired); + subscription(); + }, + "set doesn't emit if value hasn't changed": assert => { + const a = new ObservableValue(5); + let fired = false; + const subscription = a.subscribe(() => { + fired = true; + }); + a.set(5); + a.set(5); + assert(!fired); + subscription(); + }, + "waitFor promise resolves on matching update": async assert => { + const a = new ObservableValue(5); + const handle = a.waitFor(v => v === 6); + Promise.resolve().then(() => { + a.set(6); + }); + await handle.promise; + assert.strictEqual(a.get(), 6); + }, + "waitFor promise rejects when disposed": async assert => { + const a = new ObservableValue(0); + const handle = a.waitFor(() => false); + Promise.resolve().then(() => { + handle.dispose(); + }); + await assert.rejects(handle.promise, AbortError); + } + } +} diff --git a/src/observable/value/PickMapObservable.ts b/src/observable/value/PickMapObservable.ts new file mode 100644 index 00000000..835cdf90 --- /dev/null +++ b/src/observable/value/PickMapObservable.ts @@ -0,0 +1,89 @@ +/* +Copyright 2020 Bruno Windels + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {BaseObservableValue} from "./BaseObservableValue"; +import {BaseObservableMap, IMapObserver} from "../map/BaseObservableMap"; +import {SubscriptionHandle} from "../BaseObservable"; + +function pickLowestKey(currentKey: K, newKey: K): boolean { + return newKey < currentKey; +} + +export class PickMapObservable implements IMapObserver extends BaseObservableValue { + + private key?: K; + private mapSubscription?: SubscriptionHandle; + + constructor( + private readonly map: BaseObservableMap, + private readonly pickKey: (currentKey: K, newKey: K) => boolean = pickLowestKey + ) { + super(); + } + + private trySetKey(newKey: K): boolean { + if (this.key === undefined || this.pickKey(this.key, newKey)) { + this.key = newKey; + return true; + } + return false; + } + + onReset(): void { + this.key = undefined; + this.emit(this.get()); + } + + onAdd(key: K, value:V): void { + if (this.trySetKey(key)) { + this.emit(this.get()); + } + } + + onUpdate(key: K, value: V, params: any): void {} + + onRemove(key: K, value: V): void { + if (key === this.key) { + this.key = undefined; + let changed = false; + for (const [key] of this.map) { + changed = this.trySetKey(key) || changed; + } + if (changed) { + this.emit(this.get()); + } + } + } + + onSubscribeFirst(): void { + this.mapSubscription = this.map.subscribe(this); + for (const [key] of this.map) { + this.trySetKey(key); + } + } + + onUnsubscribeLast(): void { + this.mapSubscription(); + this.key = undefined; + } + + get(): V | undefined { + if (this.key !== undefined) { + return this.map.get(this.key); + } + return undefined; + } +} diff --git a/src/observable/value/RetainedObservableValue.ts b/src/observable/value/RetainedObservableValue.ts new file mode 100644 index 00000000..edfb6c15 --- /dev/null +++ b/src/observable/value/RetainedObservableValue.ts @@ -0,0 +1,31 @@ +/* +Copyright 2020 Bruno Windels + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {ObservableValue} from "./ObservableValue"; + +export class RetainedObservableValue extends ObservableValue { + private _freeCallback: () => void; + + constructor(initialValue: T, freeCallback: () => void) { + super(initialValue); + this._freeCallback = freeCallback; + } + + onUnsubscribeLast() { + super.onUnsubscribeLast(); + this._freeCallback(); + } +} diff --git a/src/platform/web/dom/History.js b/src/platform/web/dom/History.js index d51974bb..96576626 100644 --- a/src/platform/web/dom/History.js +++ b/src/platform/web/dom/History.js @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {BaseObservableValue} from "../../../observable/ObservableValue"; +import {BaseObservableValue} from "../../../observable/value/BaseObservableValue"; export class History extends BaseObservableValue { handleEvent(event) { diff --git a/src/platform/web/dom/OnlineStatus.js b/src/platform/web/dom/OnlineStatus.js index 48e4e912..fd114603 100644 --- a/src/platform/web/dom/OnlineStatus.js +++ b/src/platform/web/dom/OnlineStatus.js @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {BaseObservableValue} from "../../../observable/ObservableValue"; +import {BaseObservableValue} from "../../../observable/value/BaseObservableValue"; export class OnlineStatus extends BaseObservableValue { constructor() { diff --git a/src/utils/AbortableOperation.ts b/src/utils/AbortableOperation.ts index fba71a8c..b3f663bd 100644 --- a/src/utils/AbortableOperation.ts +++ b/src/utils/AbortableOperation.ts @@ -14,7 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {BaseObservableValue, ObservableValue} from "../observable/ObservableValue"; +import {BaseObservableValue} from "../observable/value/BaseObservableValue"; +import {ObservableValue} from "../observable/value/ObservableValue"; export interface IAbortable { abort(); From 6daae797e5e38589d8cc215e7d3f3abc8313c033 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 17 Mar 2022 11:31:12 +0100 Subject: [PATCH 020/323] fix some ts/lint errors --- src/matrix/DeviceMessageHandler.js | 2 +- src/matrix/Session.js | 7 +++++++ src/matrix/calls/CallHandler.ts | 2 +- src/matrix/calls/PeerCall.ts | 2 +- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index 91ef82f6..80fd1592 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -16,7 +16,7 @@ limitations under the License. import {OLM_ALGORITHM} from "./e2ee/common.js"; import {countBy, groupBy} from "../utils/groupBy"; -import {LRUCache} from "../../utils/LRUCache"; +import {LRUCache} from "../utils/LRUCache"; export class DeviceMessageHandler { constructor({storage, callHandler}) { diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 69cd9ee0..dbc0b17e 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -47,6 +47,7 @@ import { import {SecretStorage} from "./ssss/SecretStorage"; import {ObservableValue} from "../observable/value/ObservableValue"; import {RetainedObservableValue} from "../observable/value/RetainedObservableValue"; +import {CallHandler} from "./calls/CallHandler"; const PICKLE_KEY = "DEFAULT_KEY"; const PUSHER_KEY = "pusher"; @@ -81,6 +82,8 @@ export class Session { if (!this._deviceTracker || !this._olmEncryption) { throw new Error("encryption is not enabled"); } + // TODO: just get the devices we're sending the message to, not all the room devices + // although we probably already fetched all devices to send messages in the likely e2ee room await this._deviceTracker.trackRoom(roomId, log); const devices = await this._deviceTracker.devicesForTrackedRoom(roomId, this._hsApi, log); const encryptedMessage = await this._olmEncryption.encrypt(message.type, message.content, devices, this._hsApi, log); @@ -132,6 +135,10 @@ export class Session { return this._sessionInfo.userId; } + get callHandler() { + return this._callHandler; + } + // called once this._e2eeAccount is assigned _setupEncryption() { // TODO: this should all go in a wrapper in e2ee/ that is bootstrapped by passing in the account diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index a0cd8473..a36b17f6 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -37,7 +37,7 @@ const CALL_TERMINATED = "m.terminated"; export type Options = Omit; -export class GroupCallHandler { +export class CallHandler { // group calls by call id private readonly _calls: ObservableMap = new ObservableMap(); // map of userId to set of conf_id's they are in diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index a4ea8cca..9a702ebd 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -875,7 +875,7 @@ const FALLBACK_ICE_SERVER = 'stun:turn.matrix.org'; /** The length of time a call can be ringing for. */ const CALL_TIMEOUT_MS = 60000; -const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); +//const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); export class CallError extends Error { code: string; From e482e3aeef0af29171418592b30d0b2df2150b40 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 17 Mar 2022 11:31:36 +0100 Subject: [PATCH 021/323] expose mediaDevices and webRTC from platform --- src/platform/web/Platform.js | 4 ++++ src/platform/web/dom/MediaDevices.ts | 2 +- src/platform/web/dom/WebRTC.ts | 6 ++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/platform/web/Platform.js b/src/platform/web/Platform.js index 984bc45c..7426b138 100644 --- a/src/platform/web/Platform.js +++ b/src/platform/web/Platform.js @@ -38,6 +38,8 @@ import {downloadInIframe} from "./dom/download.js"; import {Disposables} from "../../utils/Disposables"; import {parseHTML} from "./parsehtml.js"; import {handleAvatarError} from "./ui/avatar"; +import {MediaDevicesWrapper} from "./dom/MediaDevices"; +import {DOMWebRTC} from "./dom/WebRTC"; function addScript(src) { return new Promise(function (resolve, reject) { @@ -163,6 +165,8 @@ export class Platform { this._disposables = new Disposables(); this._olmPromise = undefined; this._workerPromise = undefined; + this.mediaDevices = new MediaDevicesWrapper(navigator.mediaDevices); + this.webRTC = new DOMWebRTC(); } _createLogger(isDevelopment) { diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index 445ff22d..22f3d634 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -21,7 +21,7 @@ const POLLING_INTERVAL = 200; // ms export const SPEAKING_THRESHOLD = -60; // dB const SPEAKING_SAMPLE_COUNT = 8; // samples -class MediaDevicesWrapper implements IMediaDevices { +export class MediaDevicesWrapper implements IMediaDevices { constructor(private readonly mediaDevices: MediaDevices) {} enumerate(): Promise { diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 0025dfec..39a6f9d2 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -23,6 +23,12 @@ const POLLING_INTERVAL = 200; // ms export const SPEAKING_THRESHOLD = -60; // dB const SPEAKING_SAMPLE_COUNT = 8; // samples +export class DOMWebRTC implements WebRTC { + createPeerConnection(handler: PeerConnectionHandler): PeerConnection { + return new DOMPeerConnection(handler, false, []); + } +} + class DOMPeerConnection implements PeerConnection { private readonly peerConnection: RTCPeerConnection; private readonly handler: PeerConnectionHandler; From e760b8e5567425d016e88fe64ebf4b04aa66871a Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 17 Mar 2022 13:04:14 +0100 Subject: [PATCH 022/323] basic view model setup --- src/domain/session/SessionViewModel.js | 2 +- src/domain/session/room/CallViewModel.ts | 34 +++++++++++++++++++ src/domain/session/room/RoomViewModel.js | 29 +++++++++++++++- src/matrix/calls/group/GroupCall.ts | 6 +++- ...bservable.ts => PickMapObservableValue.ts} | 12 +++---- src/platform/web/ui/session/room/RoomView.js | 4 +++ 6 files changed, 78 insertions(+), 9 deletions(-) create mode 100644 src/domain/session/room/CallViewModel.ts rename src/observable/value/{PickMapObservable.ts => PickMapObservableValue.ts} (87%) diff --git a/src/domain/session/SessionViewModel.js b/src/domain/session/SessionViewModel.js index a67df3a7..0c9dddaa 100644 --- a/src/domain/session/SessionViewModel.js +++ b/src/domain/session/SessionViewModel.js @@ -174,7 +174,7 @@ export class SessionViewModel extends ViewModel { _createRoomViewModelInstance(roomId) { const room = this._client.session.rooms.get(roomId); if (room) { - const roomVM = new RoomViewModel(this.childOptions({room})); + const roomVM = new RoomViewModel(this.childOptions({room, session: this._client.session})); roomVM.load(); return roomVM; } diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts new file mode 100644 index 00000000..52f04d32 --- /dev/null +++ b/src/domain/session/room/CallViewModel.ts @@ -0,0 +1,34 @@ +/* +Copyright 2020 Bruno Windels +Copyright 2020 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {ViewModel} from "../../ViewModel"; +import type {GroupCall} from "../../../matrix/calls/group/GroupCall"; + +export class CallViewModel extends ViewModel { + + private call: GroupCall; + + constructor(options) { + super(options); + const {call} = options; + this.call = call; + } + + get name(): string { + return this.call.name; + } +} diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 71060728..4540fdb1 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -17,15 +17,18 @@ limitations under the License. import {TimelineViewModel} from "./timeline/TimelineViewModel.js"; import {ComposerViewModel} from "./ComposerViewModel.js" +import {CallViewModel} from "./CallViewModel" +import {PickMapObservableValue} from "../../../observable/value/PickMapObservableValue"; import {avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../avatar"; import {tilesCreator} from "./timeline/tilesCreator.js"; import {ViewModel} from "../../ViewModel"; import {imageToInfo} from "../common.js"; +import {LocalMedia} from "../../../matrix/calls/LocalMedia"; export class RoomViewModel extends ViewModel { constructor(options) { super(options); - const {room} = options; + const {room, session} = options; this._room = room; this._timelineVM = null; this._tilesCreator = null; @@ -40,6 +43,19 @@ export class RoomViewModel extends ViewModel { } this._clearUnreadTimout = null; this._closeUrl = this.urlCreator.urlUntilSegment("session"); + // pick call for this room with lowest key + this._callObservable = new PickMapObservableValue(session.callHandler.calls.filterValues(c => c.roomId === this.roomId)); + this._callViewModel = undefined; + this.track(this._callObservable.subscribe(call => { + this._callViewModel = this.disposeTracked(this._callViewModel); + if (call) { + this._callViewModel = new CallViewModel(this.childOptions({call})); + } + this.emitChange("callViewModel"); + })); + if (this._callObservable.get()) { + this._callViewModel = new CallViewModel(this.childOptions({call: this._callObservable.get()})); + } } async load() { @@ -308,6 +324,10 @@ export class RoomViewModel extends ViewModel { return this._composerVM; } + get callViewModel() { + return this._callViewModel; + } + openDetailsPanel() { let path = this.navigation.path.until("room"); path = path.with(this.navigation.segment("right-panel", true)); @@ -320,6 +340,13 @@ export class RoomViewModel extends ViewModel { this._composerVM.setReplyingTo(entry); } } + + async startCall() { + const mediaTracks = await this.platform.mediaDevices.getMediaTracks(true, true); + const localMedia = LocalMedia.fromTracks(mediaTracks); + // this will set the callViewModel above as a call will be added to callHandler.calls + await this.session.callHandler.createCall(this.roomId, localMedia, "A call " + Math.round(this.platform.random() * 100)); + } } function videoToInfo(video) { diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 6c26c995..8ea4528a 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -56,7 +56,7 @@ export class GroupCall { constructor( id: string | undefined, private callContent: Record | undefined, - private readonly roomId: string, + public readonly roomId: string, private readonly options: Options ) { this.id = id ?? makeId("conf-"); @@ -77,6 +77,10 @@ export class GroupCall { return this.callContent?.["m.terminated"] === true; } + get name(): string { + return this.callContent?.["m.name"]; + } + async join(localMedia: LocalMedia) { if (this._state !== GroupCallState.Created) { return; diff --git a/src/observable/value/PickMapObservable.ts b/src/observable/value/PickMapObservableValue.ts similarity index 87% rename from src/observable/value/PickMapObservable.ts rename to src/observable/value/PickMapObservableValue.ts index 835cdf90..a20ec04e 100644 --- a/src/observable/value/PickMapObservable.ts +++ b/src/observable/value/PickMapObservableValue.ts @@ -22,7 +22,7 @@ function pickLowestKey(currentKey: K, newKey: K): boolean { return newKey < currentKey; } -export class PickMapObservable implements IMapObserver extends BaseObservableValue { +export class PickMapObservableValue extends BaseObservableValue implements IMapObserver{ private key?: K; private mapSubscription?: SubscriptionHandle; @@ -34,7 +34,7 @@ export class PickMapObservable implements IMapObserver extends BaseO super(); } - private trySetKey(newKey: K): boolean { + private updateKey(newKey: K): boolean { if (this.key === undefined || this.pickKey(this.key, newKey)) { this.key = newKey; return true; @@ -48,7 +48,7 @@ export class PickMapObservable implements IMapObserver extends BaseO } onAdd(key: K, value:V): void { - if (this.trySetKey(key)) { + if (this.updateKey(key)) { this.emit(this.get()); } } @@ -60,7 +60,7 @@ export class PickMapObservable implements IMapObserver extends BaseO this.key = undefined; let changed = false; for (const [key] of this.map) { - changed = this.trySetKey(key) || changed; + changed = this.updateKey(key) || changed; } if (changed) { this.emit(this.get()); @@ -71,12 +71,12 @@ export class PickMapObservable implements IMapObserver extends BaseO onSubscribeFirst(): void { this.mapSubscription = this.map.subscribe(this); for (const [key] of this.map) { - this.trySetKey(key); + this.updateKey(key); } } onUnsubscribeLast(): void { - this.mapSubscription(); + this.mapSubscription!(); this.key = undefined; } diff --git a/src/platform/web/ui/session/room/RoomView.js b/src/platform/web/ui/session/room/RoomView.js index c172766a..541cc4d8 100644 --- a/src/platform/web/ui/session/room/RoomView.js +++ b/src/platform/web/ui/session/room/RoomView.js @@ -52,6 +52,9 @@ export class RoomView extends TemplateView { ]), t.div({className: "RoomView_body"}, [ t.div({className: "RoomView_error"}, vm => vm.error), + t.map(vm => vm.callViewModel, (callViewModel, t) => { + return t.p(["A call is in progress", callViewModel => callViewModel.name]) + }), t.mapView(vm => vm.timelineViewModel, timelineViewModel => { return timelineViewModel ? new TimelineView(timelineViewModel) : @@ -69,6 +72,7 @@ export class RoomView extends TemplateView { const vm = this.value; const options = []; options.push(Menu.option(vm.i18n`Room details`, () => vm.openDetailsPanel())) + options.push(Menu.option(vm.i18n`Start call`, () => vm.startCall())) if (vm.canLeave) { options.push(Menu.option(vm.i18n`Leave room`, () => this._confirmToLeaveRoom()).setDestructive()); } From 4be82cd4722db5ee2e5bfd5b675732adc5df288b Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 17 Mar 2022 13:07:55 +0100 Subject: [PATCH 023/323] WIP on UI --- src/domain/session/room/CallViewModel.ts | 37 +++++++++++++---- src/domain/session/room/RoomViewModel.js | 9 +++- src/platform/web/ui/session/room/CallView.ts | 43 ++++++++++++++++++++ src/platform/web/ui/session/room/RoomView.js | 5 ++- 4 files changed, 81 insertions(+), 13 deletions(-) create mode 100644 src/platform/web/ui/session/room/CallView.ts diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 52f04d32..0c5f28d2 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -1,6 +1,5 @@ /* -Copyright 2020 Bruno Windels -Copyright 2020 The Matrix.org Foundation C.I.C. +Copyright 2022 The Matrix.org Foundation C.I.C. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -15,20 +14,40 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {ViewModel} from "../../ViewModel"; +import {ViewModel, Options as BaseOptions} from "../../ViewModel"; import type {GroupCall} from "../../../matrix/calls/group/GroupCall"; +import type {Member} from "../../../matrix/calls/group/Member"; +import type {BaseObservableList} from "../../../observable/list/BaseObservableList"; +import type {Track} from "../../../platform/types/MediaDevices"; -export class CallViewModel extends ViewModel { +type Options = BaseOptions & {call: GroupCall}; - private call: GroupCall; +export class CallViewModel extends ViewModel { + + public readonly memberViewModels: BaseObservableList; - constructor(options) { + constructor(options: Options) { super(options); - const {call} = options; - this.call = call; + this.memberViewModels = this.getOption("call").members + .mapValues(member => new CallMemberViewModel(this.childOptions({member}))) + .sortValues((a, b) => { + + }); } get name(): string { - return this.call.name; + return this.getOption("call").name; + } + + get localTracks(): Track[] { + return this.getOption("call").localMedia?.tracks ?? []; + } +} + +type MemberOptions = BaseOptions & {member: Member}; + +class CallMemberViewModel extends ViewModel { + get tracks(): Track[] { + return this.getOption("member").remoteTracks; } } diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 4540fdb1..f79be3c0 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -43,6 +43,10 @@ export class RoomViewModel extends ViewModel { } this._clearUnreadTimout = null; this._closeUrl = this.urlCreator.urlUntilSegment("session"); + this._setupCallViewModel(); + } + + _setupCallViewModel() { // pick call for this room with lowest key this._callObservable = new PickMapObservableValue(session.callHandler.calls.filterValues(c => c.roomId === this.roomId)); this._callViewModel = undefined; @@ -53,8 +57,9 @@ export class RoomViewModel extends ViewModel { } this.emitChange("callViewModel"); })); - if (this._callObservable.get()) { - this._callViewModel = new CallViewModel(this.childOptions({call: this._callObservable.get()})); + const call = this._callObservable.get(); + if (call) { + this._callViewModel = new CallViewModel(this.childOptions({call})); } } diff --git a/src/platform/web/ui/session/room/CallView.ts b/src/platform/web/ui/session/room/CallView.ts new file mode 100644 index 00000000..2c10f82f --- /dev/null +++ b/src/platform/web/ui/session/room/CallView.ts @@ -0,0 +1,43 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {TemplateView, TemplateBuilder} from "../../general/TemplateView"; +import {Track, TrackType} from "../../../../types/MediaDevices"; +import type {TrackWrapper} from "../../../dom/MediaDevices"; +import type {CallViewModel} from "../../../../../domain/session/room/CallViewModel"; + +function bindVideoTracks(t: TemplateBuilder, video: HTMLVideoElement, propSelector: (vm: T) => Track[]) { + t.mapSideEffect(propSelector, tracks => { + if (tracks.length) { + video.srcObject = (tracks[0] as TrackWrapper).stream; + } + }); +} + +export class CallView extends TemplateView { + render(t: TemplateBuilder, vm: CallViewModel): HTMLElement { + return t.div({class: "CallView"}, [ + t.div({class: "CallView_me"}, bindVideoTracks(t, t.video(), vm => vm.localTracks)), + t.view(new ListView(vm.memberViewModels, vm => new MemberView(vm))) + ]); + } +} + +class MemberView extends TemplateView { + render(t, vm) { + return bindVideoTracks(t, t.video(), vm => vm.tracks); + } +} diff --git a/src/platform/web/ui/session/room/RoomView.js b/src/platform/web/ui/session/room/RoomView.js index 541cc4d8..4d010a7c 100644 --- a/src/platform/web/ui/session/room/RoomView.js +++ b/src/platform/web/ui/session/room/RoomView.js @@ -23,6 +23,7 @@ import {TimelineLoadingView} from "./TimelineLoadingView.js"; import {MessageComposer} from "./MessageComposer.js"; import {RoomArchivedView} from "./RoomArchivedView.js"; import {AvatarView} from "../../AvatarView.js"; +import {CallView} from "./CallView"; export class RoomView extends TemplateView { constructor(options) { @@ -52,8 +53,8 @@ export class RoomView extends TemplateView { ]), t.div({className: "RoomView_body"}, [ t.div({className: "RoomView_error"}, vm => vm.error), - t.map(vm => vm.callViewModel, (callViewModel, t) => { - return t.p(["A call is in progress", callViewModel => callViewModel.name]) + t.mapView(vm => vm.callViewModel, callViewModel => { + return new CallView(callViewModel); }), t.mapView(vm => vm.timelineViewModel, timelineViewModel => { return timelineViewModel ? From cad2aa760d73556d862429e87492fdc23fd79593 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 21 Mar 2022 16:30:13 +0100 Subject: [PATCH 024/323] some fixes --- src/domain/session/room/CallViewModel.ts | 20 ++++++++++++++++---- src/domain/session/room/RoomViewModel.js | 8 +++++--- src/matrix/calls/CallHandler.ts | 1 + src/matrix/calls/LocalMedia.ts | 8 +++++++- src/matrix/room/Room.js | 15 ++++++--------- src/platform/web/ui/session/room/CallView.ts | 10 +++++++--- src/platform/web/ui/session/room/RoomView.js | 4 +--- 7 files changed, 43 insertions(+), 23 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 0c5f28d2..95d9b188 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -30,24 +30,36 @@ export class CallViewModel extends ViewModel { super(options); this.memberViewModels = this.getOption("call").members .mapValues(member => new CallMemberViewModel(this.childOptions({member}))) - .sortValues((a, b) => { - - }); + .sortValues((a, b) => a.compare(b)); } get name(): string { return this.getOption("call").name; } + get id(): string { + return this.getOption("call").id; + } + get localTracks(): Track[] { + console.log("localTracks", this.getOption("call").localMedia); return this.getOption("call").localMedia?.tracks ?? []; } } type MemberOptions = BaseOptions & {member: Member}; -class CallMemberViewModel extends ViewModel { +export class CallMemberViewModel extends ViewModel { get tracks(): Track[] { return this.getOption("member").remoteTracks; } + + compare(other: CallMemberViewModel): number { + const myUserId = this.getOption("member").member.userId; + const otherUserId = other.getOption("member").member.userId; + if(myUserId === otherUserId) { + return 0; + } + return myUserId < otherUserId ? -1 : 1; + } } diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index f79be3c0..1d4bca42 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -48,7 +48,7 @@ export class RoomViewModel extends ViewModel { _setupCallViewModel() { // pick call for this room with lowest key - this._callObservable = new PickMapObservableValue(session.callHandler.calls.filterValues(c => c.roomId === this.roomId)); + this._callObservable = new PickMapObservableValue(this.getOption("session").callHandler.calls.filterValues(c => c.roomId === this._room.id)); this._callViewModel = undefined; this.track(this._callObservable.subscribe(call => { this._callViewModel = this.disposeTracked(this._callViewModel); @@ -347,10 +347,12 @@ export class RoomViewModel extends ViewModel { } async startCall() { + const session = this.getOption("session"); const mediaTracks = await this.platform.mediaDevices.getMediaTracks(true, true); - const localMedia = LocalMedia.fromTracks(mediaTracks); + const localMedia = new LocalMedia().withTracks(mediaTracks); + console.log("localMedia", localMedia.tracks); // this will set the callViewModel above as a call will be added to callHandler.calls - await this.session.callHandler.createCall(this.roomId, localMedia, "A call " + Math.round(this.platform.random() * 100)); + await session.callHandler.createCall(this._room.id, localMedia, "A call " + Math.round(this.platform.random() * 100)); } } diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index a36b17f6..a9b59f4f 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -52,6 +52,7 @@ export class CallHandler { async createCall(roomId: string, localMedia: LocalMedia, name: string): Promise { const call = new GroupCall(undefined, undefined, roomId, this.groupCallOptions); + console.log("created call with id", call.id); this._calls.set(call.id, call); try { await call.create(localMedia, name); diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 4f5d3b7a..1fd4aad6 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -35,7 +35,13 @@ export class LocalMedia { return new LocalMedia(cameraTrack, screenShareTrack, microphoneTrack as AudioTrack); } - get tracks(): Track[] { return []; } + get tracks(): Track[] { + const tracks: Track[] = []; + if (this.cameraTrack) { tracks.push(this.cameraTrack); } + if (this.screenShareTrack) { tracks.push(this.screenShareTrack); } + if (this.microphoneTrack) { tracks.push(this.microphoneTrack); } + return tracks; + } getSDPMetadata(): SDPStreamMetadata { const metadata = {}; diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js index d0fcbc84..ca11cdd8 100644 --- a/src/matrix/room/Room.js +++ b/src/matrix/room/Room.js @@ -93,7 +93,7 @@ export class Room extends BaseRoom { } } - this._updateCallHandler(roomResponse); + this._updateCallHandler(roomResponse, log); return { roomEncryption, @@ -448,20 +448,17 @@ export class Room extends BaseRoom { return this._sendQueue.pendingEvents; } - _updateCallHandler(roomResponse) { + _updateCallHandler(roomResponse, log) { if (this._callHandler) { const stateEvents = roomResponse.state?.events; if (stateEvents) { - for (const e of stateEvents) { - this._callHandler.handleRoomState(this, e); - } + this._callHandler.handleRoomState(this, stateEvents, log); } let timelineEvents = roomResponse.timeline?.events; if (timelineEvents) { - for (const e of timelineEvents) { - if (typeof e.state_key === "string") { - this._callHandler.handleRoomState(this, e); - } + const timelineStateEvents = timelineEvents.filter(e => typeof e.state_key === "string"); + if (timelineEvents.length !== 0) { + this._callHandler.handleRoomState(this, timelineStateEvents, log); } } } diff --git a/src/platform/web/ui/session/room/CallView.ts b/src/platform/web/ui/session/room/CallView.ts index 2c10f82f..29d4eee9 100644 --- a/src/platform/web/ui/session/room/CallView.ts +++ b/src/platform/web/ui/session/room/CallView.ts @@ -15,29 +15,33 @@ limitations under the License. */ import {TemplateView, TemplateBuilder} from "../../general/TemplateView"; +import {ListView} from "../../general/ListView"; import {Track, TrackType} from "../../../../types/MediaDevices"; import type {TrackWrapper} from "../../../dom/MediaDevices"; -import type {CallViewModel} from "../../../../../domain/session/room/CallViewModel"; +import type {CallViewModel, CallMemberViewModel} from "../../../../../domain/session/room/CallViewModel"; function bindVideoTracks(t: TemplateBuilder, video: HTMLVideoElement, propSelector: (vm: T) => Track[]) { t.mapSideEffect(propSelector, tracks => { + console.log("tracks", tracks); if (tracks.length) { video.srcObject = (tracks[0] as TrackWrapper).stream; } }); + return video; } export class CallView extends TemplateView { render(t: TemplateBuilder, vm: CallViewModel): HTMLElement { return t.div({class: "CallView"}, [ + t.p(`Call ${vm.name} (${vm.id})`), t.div({class: "CallView_me"}, bindVideoTracks(t, t.video(), vm => vm.localTracks)), - t.view(new ListView(vm.memberViewModels, vm => new MemberView(vm))) + t.view(new ListView({list: vm.memberViewModels}, vm => new MemberView(vm))) ]); } } class MemberView extends TemplateView { - render(t, vm) { + render(t: TemplateBuilder, vm: CallMemberViewModel) { return bindVideoTracks(t, t.video(), vm => vm.tracks); } } diff --git a/src/platform/web/ui/session/room/RoomView.js b/src/platform/web/ui/session/room/RoomView.js index 4d010a7c..2936adf3 100644 --- a/src/platform/web/ui/session/room/RoomView.js +++ b/src/platform/web/ui/session/room/RoomView.js @@ -53,9 +53,7 @@ export class RoomView extends TemplateView { ]), t.div({className: "RoomView_body"}, [ t.div({className: "RoomView_error"}, vm => vm.error), - t.mapView(vm => vm.callViewModel, callViewModel => { - return new CallView(callViewModel); - }), + t.mapView(vm => vm.callViewModel, callViewModel => callViewModel ? new CallView(callViewModel) : null), t.mapView(vm => vm.timelineViewModel, timelineViewModel => { return timelineViewModel ? new TimelineView(timelineViewModel) : From 9efd191f4eccb5d00b06ee2c4804294c6d5168d2 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 22 Mar 2022 20:29:31 +0100 Subject: [PATCH 025/323] some more fixes --- src/domain/session/room/CallViewModel.ts | 1 - src/domain/session/room/RoomViewModel.js | 17 +++++++++++------ src/matrix/Session.js | 6 +++++- src/matrix/calls/CallHandler.ts | 3 +++ src/matrix/calls/group/GroupCall.ts | 5 ++++- src/observable/value/PickMapObservableValue.ts | 4 +++- src/platform/web/ui/session/room/CallView.ts | 6 +++--- 7 files changed, 29 insertions(+), 13 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 95d9b188..7fcabf02 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -42,7 +42,6 @@ export class CallViewModel extends ViewModel { } get localTracks(): Track[] { - console.log("localTracks", this.getOption("call").localMedia); return this.getOption("call").localMedia?.tracks ?? []; } } diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 1d4bca42..81ca58d2 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -347,12 +347,17 @@ export class RoomViewModel extends ViewModel { } async startCall() { - const session = this.getOption("session"); - const mediaTracks = await this.platform.mediaDevices.getMediaTracks(true, true); - const localMedia = new LocalMedia().withTracks(mediaTracks); - console.log("localMedia", localMedia.tracks); - // this will set the callViewModel above as a call will be added to callHandler.calls - await session.callHandler.createCall(this._room.id, localMedia, "A call " + Math.round(this.platform.random() * 100)); + try { + const session = this.getOption("session"); + const mediaTracks = await this.platform.mediaDevices.getMediaTracks(true, true); + const localMedia = new LocalMedia().withTracks(mediaTracks); + console.log("localMedia", localMedia.tracks); + // this will set the callViewModel above as a call will be added to callHandler.calls + await session.callHandler.createCall(this._room.id, localMedia, "A call " + Math.round(this.platform.random() * 100)); + } catch (err) { + console.error(err.stack); + alert(err.message); + } } } diff --git a/src/matrix/Session.js b/src/matrix/Session.js index dbc0b17e..e604e068 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -88,7 +88,11 @@ export class Session { const devices = await this._deviceTracker.devicesForTrackedRoom(roomId, this._hsApi, log); const encryptedMessage = await this._olmEncryption.encrypt(message.type, message.content, devices, this._hsApi, log); return encryptedMessage; - } + }, + storage: this._storage, + webRTC: this._platform.webRTC, + ownDeviceId: sessionInfo.deviceId, + ownUserId: sessionInfo.userId, }); this._deviceMessageHandler = new DeviceMessageHandler({storage, callHandler: this._callHandler}); this._olm = olm; diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index a9b59f4f..269f6020 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -63,7 +63,9 @@ export class CallHandler { } throw err; } + console.log("joining call I just created"); await call.join(localMedia); + console.log("joined!"); return call; } @@ -73,6 +75,7 @@ export class CallHandler { /** @internal */ handleRoomState(room: Room, events: StateEvent[], log: ILogItem) { + console.log("handling room state"); // first update call events for (const event of events) { if (event.type === EventType.GroupCall) { diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 8ea4528a..d22135d1 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -49,7 +49,7 @@ export type Options = Omit = new ObservableMap(); - private _localMedia?: LocalMedia; + private _localMedia?: LocalMedia = undefined; private _memberOptions: MemberOptions; private _state: GroupCallState; @@ -87,10 +87,12 @@ export class GroupCall { } this._state = GroupCallState.Joining; this._localMedia = localMedia; + this.options.emitUpdate(this); const memberContent = await this._joinCallMemberContent(); // send m.call.member state event const request = this.options.hsApi.sendState(this.roomId, "m.call.member", this.options.ownUserId, memberContent); await request.response(); + this.options.emitUpdate(this); // send invite to all members that are < my userId for (const [,member] of this._members) { member.connect(this._localMedia); @@ -119,6 +121,7 @@ export class GroupCall { }; const request = this.options.hsApi.sendState(this.roomId, "m.call", this.id, this.callContent); await request.response(); + this._state = GroupCallState.Created; } /** @internal */ diff --git a/src/observable/value/PickMapObservableValue.ts b/src/observable/value/PickMapObservableValue.ts index a20ec04e..67f9d562 100644 --- a/src/observable/value/PickMapObservableValue.ts +++ b/src/observable/value/PickMapObservableValue.ts @@ -53,7 +53,9 @@ export class PickMapObservableValue extends BaseObservableValue(t: TemplateBuilder, video: HTMLVideoElement, prop export class CallView extends TemplateView { render(t: TemplateBuilder, vm: CallViewModel): HTMLElement { return t.div({class: "CallView"}, [ - t.p(`Call ${vm.name} (${vm.id})`), - t.div({class: "CallView_me"}, bindVideoTracks(t, t.video(), vm => vm.localTracks)), + t.p(["Call ", vm => vm.name, vm => ` (${vm.id})`]), + t.div({class: "CallView_me"}, bindVideoTracks(t, t.video({autoplay: true}), vm => vm.localTracks)), t.view(new ListView({list: vm.memberViewModels}, vm => new MemberView(vm))) ]); } @@ -42,6 +42,6 @@ export class CallView extends TemplateView { class MemberView extends TemplateView { render(t: TemplateBuilder, vm: CallMemberViewModel) { - return bindVideoTracks(t, t.video(), vm => vm.tracks); + return bindVideoTracks(t, t.video({autoplay: true}), vm => vm.tracks); } } From 0a37fd561e3c713faf4639d3413918d5bd111d06 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 23 Mar 2022 12:23:10 +0100 Subject: [PATCH 026/323] just enough view code to join a call --- src/domain/session/room/RoomViewModel.js | 7 +-- .../session/room/timeline/tiles/CallTile.js | 48 +++++++++++++++++++ .../session/room/timeline/tilesCreator.js | 3 ++ src/matrix/DeviceMessageHandler.js | 1 + src/matrix/Session.js | 7 +-- src/matrix/Sync.js | 1 + src/matrix/calls/PeerCall.ts | 1 + src/matrix/calls/group/GroupCall.ts | 15 ++++-- src/matrix/calls/group/Member.ts | 14 +++--- src/matrix/common.js | 17 +++++++ src/matrix/e2ee/RoomEncryption.js | 14 ++---- src/platform/web/ui/session/room/common.ts | 3 ++ .../ui/session/room/timeline/CallTileView.ts | 38 +++++++++++++++ 13 files changed, 141 insertions(+), 28 deletions(-) create mode 100644 src/domain/session/room/timeline/tiles/CallTile.js create mode 100644 src/platform/web/ui/session/room/timeline/CallTileView.ts diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 81ca58d2..2b47673f 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -48,7 +48,8 @@ export class RoomViewModel extends ViewModel { _setupCallViewModel() { // pick call for this room with lowest key - this._callObservable = new PickMapObservableValue(this.getOption("session").callHandler.calls.filterValues(c => c.roomId === this._room.id)); + const calls = this.getOption("session").callHandler.calls; + this._callObservable = new PickMapObservableValue(calls.filterValues(c => c.roomId === this._room.id && c.hasJoined)); this._callViewModel = undefined; this.track(this._callObservable.subscribe(call => { this._callViewModel = this.disposeTracked(this._callViewModel); @@ -68,6 +69,7 @@ export class RoomViewModel extends ViewModel { try { const timeline = await this._room.openTimeline(); this._tilesCreator = tilesCreator(this.childOptions({ + session: this.getOption("session"), roomVM: this, timeline, })); @@ -349,9 +351,8 @@ export class RoomViewModel extends ViewModel { async startCall() { try { const session = this.getOption("session"); - const mediaTracks = await this.platform.mediaDevices.getMediaTracks(true, true); + const mediaTracks = await this.platform.mediaDevices.getMediaTracks(false, true); const localMedia = new LocalMedia().withTracks(mediaTracks); - console.log("localMedia", localMedia.tracks); // this will set the callViewModel above as a call will be added to callHandler.calls await session.callHandler.createCall(this._room.id, localMedia, "A call " + Math.round(this.platform.random() * 100)); } catch (err) { diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js new file mode 100644 index 00000000..7129bb75 --- /dev/null +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -0,0 +1,48 @@ +/* +Copyright 2020 Bruno Windels + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {SimpleTile} from "./SimpleTile.js"; +import {LocalMedia} from "../../../../../matrix/calls/LocalMedia"; + +// TODO: timeline entries for state events with the same state key and type +// should also update previous entries in the timeline, so we can update the name of the call, whether it is terminated, etc ... + +// alternatively, we could just subscribe to the GroupCall and spontanously emit an update when it updates + +export class CallTile extends SimpleTile { + + get shape() { + return "call"; + } + + get name() { + return this._entry.content["m.name"]; + } + + get _call() { + const calls = this.getOption("session").callHandler.calls; + return calls.get(this._entry.stateKey); + } + + async join() { + const call = this._call; + if (call) { + const mediaTracks = await this.platform.mediaDevices.getMediaTracks(false, true); + const localMedia = new LocalMedia().withTracks(mediaTracks); + await call.join(localMedia); + } + } +} diff --git a/src/domain/session/room/timeline/tilesCreator.js b/src/domain/session/room/timeline/tilesCreator.js index dc9a850e..659a5e76 100644 --- a/src/domain/session/room/timeline/tilesCreator.js +++ b/src/domain/session/room/timeline/tilesCreator.js @@ -26,6 +26,7 @@ import {RoomMemberTile} from "./tiles/RoomMemberTile.js"; import {EncryptedEventTile} from "./tiles/EncryptedEventTile.js"; import {EncryptionEnabledTile} from "./tiles/EncryptionEnabledTile.js"; import {MissingAttachmentTile} from "./tiles/MissingAttachmentTile.js"; +import {CallTile} from "./tiles/CallTile.js"; export function tilesCreator(baseOptions) { const tilesCreator = function tilesCreator(entry, emitUpdate) { @@ -71,6 +72,8 @@ export function tilesCreator(baseOptions) { return new EncryptedEventTile(options); case "m.room.encryption": return new EncryptionEnabledTile(options); + case "m.call": + return entry.stateKey ? new CallTile(options) : null; default: // unknown type not rendered return null; diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index 80fd1592..11c10750 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -86,6 +86,7 @@ export class DeviceMessageHandler { this._senderDeviceCache.set(device); } } + console.log("incoming device message", senderKey, device, this._senderDeviceCache); return device; } } diff --git a/src/matrix/Session.js b/src/matrix/Session.js index e604e068..27689013 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -78,14 +78,15 @@ export class Session { this._callHandler = new CallHandler({ createTimeout: this._platform.clock.createTimeout, hsApi: this._hsApi, - encryptDeviceMessage: async (roomId, message, log) => { + encryptDeviceMessage: async (roomId, userId, message, log) => { if (!this._deviceTracker || !this._olmEncryption) { throw new Error("encryption is not enabled"); } // TODO: just get the devices we're sending the message to, not all the room devices // although we probably already fetched all devices to send messages in the likely e2ee room - await this._deviceTracker.trackRoom(roomId, log); - const devices = await this._deviceTracker.devicesForTrackedRoom(roomId, this._hsApi, log); + await this._deviceTracker.trackRoom(this.rooms.get(roomId), log); + const devices = await this._deviceTracker.devicesForRoomMembers(roomId, [userId], this._hsApi, log); + console.log("devices", devices); const encryptedMessage = await this._olmEncryption.encrypt(message.type, message.content, devices, this._hsApi, log); return encryptedMessage; }, diff --git a/src/matrix/Sync.js b/src/matrix/Sync.js index 8e880def..b4ea702f 100644 --- a/src/matrix/Sync.js +++ b/src/matrix/Sync.js @@ -224,6 +224,7 @@ export class Sync { _openPrepareSyncTxn() { const storeNames = this._storage.storeNames; return this._storage.readTxn([ + storeNames.deviceIdentities, // to read device from olm messages storeNames.olmSessions, storeNames.inboundGroupSessions, // to read fragments when loading sync writer when rejoining archived room diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 9a702ebd..319fde3a 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -114,6 +114,7 @@ export class PeerCall implements IDisposable { } }); this.logger = { + info(...args) { console.info.apply(console, ["WebRTC debug:", ...args])}, debug(...args) { console.log.apply(console, ["WebRTC debug:", ...args])}, log(...args) { console.log.apply(console, ["WebRTC log:", ...args])}, warn(...args) { console.log.apply(console, ["WebRTC warn:", ...args])}, diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index d22135d1..901eb3b3 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -41,7 +41,7 @@ export enum GroupCallState { export type Options = Omit & { emitUpdate: (call: GroupCall, params?: any) => void; - encryptDeviceMessage: (roomId: string, message: SignallingMessage, log: ILogItem) => Promise, + encryptDeviceMessage: (roomId: string, userId: string, message: SignallingMessage, log: ILogItem) => Promise, storage: Storage, ownDeviceId: string }; @@ -61,13 +61,13 @@ export class GroupCall { ) { this.id = id ?? makeId("conf-"); this._state = id ? GroupCallState.Created : GroupCallState.Fledgling; - this._memberOptions = Object.assign({ + this._memberOptions = Object.assign({}, options, { confId: this.id, emitUpdate: member => this._members.update(member.member.userId, member), - encryptDeviceMessage: (message: SignallingMessage, log) => { - return this.options.encryptDeviceMessage(this.roomId, message, log); + encryptDeviceMessage: (userId: string, message: SignallingMessage, log) => { + return this.options.encryptDeviceMessage(this.roomId, userId, message, log); } - }, options); + }); } get localMedia(): LocalMedia | undefined { return this._localMedia; } @@ -99,6 +99,10 @@ export class GroupCall { } } + get hasJoined() { + return this._state === GroupCallState.Joining || this._state === GroupCallState.Joined; + } + async leave() { const memberContent = await this._leaveCallMemberContent(); // send m.call.member state event @@ -165,6 +169,7 @@ export class GroupCall { /** @internal */ handleDeviceMessage(message: SignallingMessage, userId: string, deviceId: string, log: ILogItem) { + console.log("incoming to_device call signalling message from", userId, deviceId, message); // TODO: return if we are not membering to the call let member = this._members.get(userId); if (member) { diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index bd1613cd..0a80bbef 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -17,6 +17,7 @@ limitations under the License. import {PeerCall, CallState} from "../PeerCall"; import {makeTxnId, makeId} from "../../common"; import {EventType} from "../callEventTypes"; +import {formatToDeviceMessagesPayload} from "../../common"; import type {Options as PeerCallOptions} from "../PeerCall"; import type {LocalMedia} from "../LocalMedia"; @@ -32,7 +33,7 @@ export type Options = Omit, log: ILogItem) => Promise, + encryptDeviceMessage: (userId: string, message: SignallingMessage, log: ILogItem) => Promise, emitUpdate: (participant: Member, params?: any) => void, } @@ -81,13 +82,14 @@ export class Member { sendSignallingMessage = async (message: SignallingMessage, log: ILogItem) => { const groupMessage = message as SignallingMessage; groupMessage.content.conf_id = this.options.confId; - const encryptedMessage = await this.options.encryptDeviceMessage(groupMessage, log); + const encryptedMessages = await this.options.encryptDeviceMessage(this.member.userId, groupMessage, log); + const payload = formatToDeviceMessagesPayload(encryptedMessages); const request = this.options.hsApi.sendToDevice( "m.room.encrypted", - {[this.member.userId]: { - ["*"]: encryptedMessage.content - } - }, makeTxnId(), {log}); + payload, + makeTxnId(), + {log} + ); await request.response(); } diff --git a/src/matrix/common.js b/src/matrix/common.js index 5919ad9c..7cd72ae1 100644 --- a/src/matrix/common.js +++ b/src/matrix/common.js @@ -15,6 +15,9 @@ See the License for the specific language governing permissions and limitations under the License. */ +import {groupBy} from "../utils/groupBy"; + + export function makeTxnId() { return makeId("t"); } @@ -29,6 +32,20 @@ export function isTxnId(txnId) { return txnId.startsWith("t") && txnId.length === 15; } +export function formatToDeviceMessagesPayload(messages) { + const messagesByUser = groupBy(messages, message => message.device.userId); + const payload = { + messages: Array.from(messagesByUser.entries()).reduce((userMap, [userId, messages]) => { + userMap[userId] = messages.reduce((deviceMap, message) => { + deviceMap[message.device.deviceId] = message.content; + return deviceMap; + }, {}); + return userMap; + }, {}) + }; + return payload; +} + export function tests() { return { "isTxnId succeeds on result of makeTxnId": assert => { diff --git a/src/matrix/e2ee/RoomEncryption.js b/src/matrix/e2ee/RoomEncryption.js index 80f57507..cb0dd333 100644 --- a/src/matrix/e2ee/RoomEncryption.js +++ b/src/matrix/e2ee/RoomEncryption.js @@ -18,7 +18,7 @@ import {MEGOLM_ALGORITHM, DecryptionSource} from "./common.js"; import {groupEventsBySession} from "./megolm/decryption/utils"; import {mergeMap} from "../../utils/mergeMap"; import {groupBy} from "../../utils/groupBy"; -import {makeTxnId} from "../common.js"; +import {makeTxnId, formatToDeviceMessagesPayload} from "../common.js"; const ENCRYPTED_TYPE = "m.room.encrypted"; // how often ensureMessageKeyIsShared can check if it needs to @@ -386,6 +386,7 @@ export class RoomEncryption { await writeTxn.complete(); } + // TODO: make this use _sendMessagesToDevices async _sendSharedMessageToDevices(type, message, devices, hsApi, log) { const devicesByUser = groupBy(devices, device => device.userId); const payload = { @@ -403,16 +404,7 @@ export class RoomEncryption { async _sendMessagesToDevices(type, messages, hsApi, log) { log.set("messages", messages.length); - const messagesByUser = groupBy(messages, message => message.device.userId); - const payload = { - messages: Array.from(messagesByUser.entries()).reduce((userMap, [userId, messages]) => { - userMap[userId] = messages.reduce((deviceMap, message) => { - deviceMap[message.device.deviceId] = message.content; - return deviceMap; - }, {}); - return userMap; - }, {}) - }; + const payload = formatToDeviceMessagesPayload(messages); const txnId = makeTxnId(); await hsApi.sendToDevice(type, payload, txnId, {log}).response(); } diff --git a/src/platform/web/ui/session/room/common.ts b/src/platform/web/ui/session/room/common.ts index 5048211a..a2732ff4 100644 --- a/src/platform/web/ui/session/room/common.ts +++ b/src/platform/web/ui/session/room/common.ts @@ -24,6 +24,7 @@ import {AnnouncementView} from "./timeline/AnnouncementView.js"; import {RedactedView} from "./timeline/RedactedView.js"; import {SimpleTile} from "../../../../../domain/session/room/timeline/tiles/SimpleTile.js"; import {GapView} from "./timeline/GapView.js"; +import {CallTileView} from "./timeline/CallTileView"; export type TileView = GapView | AnnouncementView | TextMessageView | ImageView | VideoView | FileView | LocationView | MissingAttachmentView | RedactedView; @@ -51,5 +52,7 @@ export function viewClassForEntry(entry: SimpleTile): TileViewConstructor | unde return MissingAttachmentView; case "redacted": return RedactedView; + case "call": + return CallTileView; } } diff --git a/src/platform/web/ui/session/room/timeline/CallTileView.ts b/src/platform/web/ui/session/room/timeline/CallTileView.ts new file mode 100644 index 00000000..dfb04228 --- /dev/null +++ b/src/platform/web/ui/session/room/timeline/CallTileView.ts @@ -0,0 +1,38 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {TemplateView} from "../../../general/TemplateView"; +import type {CallTile} from "../../../../../../domain/session/room/timeline/tiles/CallTile"; + +export class CallTileView extends TemplateView { + render(t, vm) { + return t.li( + {className: "AnnouncementView"}, + t.div([ + "Call ", + vm => vm.name, + t.button({className: "CallTileView_join"}, "Join") + ]) + ); + } + + /* This is called by the parent ListView, which just has 1 listener for the whole list */ + onClick(evt) { + if (evt.target.className === "CallTileView_join") { + this.value.join(); + } + } +} From a0a07355d47c336883eab7b00b06a9033701259b Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 24 Mar 2022 13:52:19 +0100 Subject: [PATCH 027/323] more improvements, make hangup work --- src/domain/session/room/CallViewModel.ts | 26 +++++-- src/domain/session/room/RoomViewModel.js | 9 ++- .../room/timeline/tiles/BaseMessageTile.js | 8 --- .../session/room/timeline/tiles/CallTile.js | 59 +++++++++++++-- .../session/room/timeline/tiles/SimpleTile.js | 8 +++ .../session/room/timeline/tilesCreator.js | 4 +- src/matrix/Session.js | 1 - src/matrix/calls/CallHandler.ts | 8 +-- src/matrix/calls/LocalMedia.ts | 6 ++ src/matrix/calls/PeerCall.ts | 4 ++ src/matrix/calls/group/GroupCall.ts | 72 +++++++++++++++---- src/matrix/calls/group/Member.ts | 8 +++ src/matrix/room/Room.js | 2 +- .../value/PickMapObservableValue.ts | 8 +-- src/platform/web/ui/session/room/CallView.ts | 7 +- .../ui/session/room/timeline/CallTileView.ts | 8 ++- 16 files changed, 182 insertions(+), 56 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 7fcabf02..30b18bc1 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -33,16 +33,26 @@ export class CallViewModel extends ViewModel { .sortValues((a, b) => a.compare(b)); } + private get call(): GroupCall { + return this.getOption("call"); + } + get name(): string { - return this.getOption("call").name; + return this.call.name; } get id(): string { - return this.getOption("call").id; + return this.call.id; } get localTracks(): Track[] { - return this.getOption("call").localMedia?.tracks ?? []; + return this.call.localMedia?.tracks ?? []; + } + + leave() { + if (this.call.hasJoined) { + this.call.leave(); + } } } @@ -50,12 +60,16 @@ type MemberOptions = BaseOptions & {member: Member}; export class CallMemberViewModel extends ViewModel { get tracks(): Track[] { - return this.getOption("member").remoteTracks; + return this.member.remoteTracks; + } + + private get member(): Member { + return this.getOption("member"); } compare(other: CallMemberViewModel): number { - const myUserId = this.getOption("member").member.userId; - const otherUserId = other.getOption("member").member.userId; + const myUserId = this.member.member.userId; + const otherUserId = other.member.member.userId; if(myUserId === otherUserId) { return 0; } diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 2b47673f..6dd8c8bd 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -49,12 +49,17 @@ export class RoomViewModel extends ViewModel { _setupCallViewModel() { // pick call for this room with lowest key const calls = this.getOption("session").callHandler.calls; - this._callObservable = new PickMapObservableValue(calls.filterValues(c => c.roomId === this._room.id && c.hasJoined)); + this._callObservable = new PickMapObservableValue(calls.filterValues(c => { + return c.roomId === this._room.id && c.hasJoined; + })); this._callViewModel = undefined; this.track(this._callObservable.subscribe(call => { + if (call && this._callViewModel && call.id === this._callViewModel.id) { + return; + } this._callViewModel = this.disposeTracked(this._callViewModel); if (call) { - this._callViewModel = new CallViewModel(this.childOptions({call})); + this._callViewModel = this.track(new CallViewModel(this.childOptions({call}))); } this.emitChange("callViewModel"); })); diff --git a/src/domain/session/room/timeline/tiles/BaseMessageTile.js b/src/domain/session/room/timeline/tiles/BaseMessageTile.js index 3385a587..c2041a2c 100644 --- a/src/domain/session/room/timeline/tiles/BaseMessageTile.js +++ b/src/domain/session/room/timeline/tiles/BaseMessageTile.js @@ -49,14 +49,6 @@ export class BaseMessageTile extends SimpleTile { return `https://matrix.to/#/${encodeURIComponent(this.sender)}`; } - get displayName() { - return this._entry.displayName || this.sender; - } - - get sender() { - return this._entry.sender; - } - get memberPanelLink() { return `${this.urlCreator.urlUntilSegment("room")}/member/${this.sender}`; } diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index 7129bb75..3e3918b6 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -23,6 +23,27 @@ import {LocalMedia} from "../../../../../matrix/calls/LocalMedia"; // alternatively, we could just subscribe to the GroupCall and spontanously emit an update when it updates export class CallTile extends SimpleTile { + + constructor(options) { + super(options); + const calls = this.getOption("session").callHandler.calls; + this._call = calls.get(this._entry.stateKey); + this._callSubscription = undefined; + if (this._call) { + this._callSubscription = this._call.disposableOn("change", () => { + // unsubscribe when terminated + if (this._call.isTerminated) { + this._callSubscription = this._callSubscription(); + this._call = undefined; + } + this.emitChange(); + }); + } + } + + get confId() { + return this._entry.stateKey; + } get shape() { return "call"; @@ -32,17 +53,43 @@ export class CallTile extends SimpleTile { return this._entry.content["m.name"]; } - get _call() { - const calls = this.getOption("session").callHandler.calls; - return calls.get(this._entry.stateKey); + get canJoin() { + return this._call && !this._call.hasJoined; + } + + get canLeave() { + return this._call && this._call.hasJoined; + } + + get label() { + if (this._call) { + if (this._call.hasJoined) { + return `Ongoing call (${this.name}, ${this.confId})`; + } else { + return `${this.displayName} started a call (${this.name}, ${this.confId})`; + } + } else { + return `Call finished, started by ${this.displayName} (${this.name}, ${this.confId})`; + } } async join() { - const call = this._call; - if (call) { + if (this.canJoin) { const mediaTracks = await this.platform.mediaDevices.getMediaTracks(false, true); const localMedia = new LocalMedia().withTracks(mediaTracks); - await call.join(localMedia); + await this._call.join(localMedia); + } + } + + async leave() { + if (this.canLeave) { + this._call.leave(); + } + } + + dispose() { + if (this._callSubscription) { + this._callSubscription = this._callSubscription(); } } } diff --git a/src/domain/session/room/timeline/tiles/SimpleTile.js b/src/domain/session/room/timeline/tiles/SimpleTile.js index af2b0e12..d70a0a37 100644 --- a/src/domain/session/room/timeline/tiles/SimpleTile.js +++ b/src/domain/session/room/timeline/tiles/SimpleTile.js @@ -154,4 +154,12 @@ export class SimpleTile extends ViewModel { get _ownMember() { return this._options.timeline.me; } + + get displayName() { + return this._entry.displayName || this.sender; + } + + get sender() { + return this._entry.sender; + } } diff --git a/src/domain/session/room/timeline/tilesCreator.js b/src/domain/session/room/timeline/tilesCreator.js index 659a5e76..f35f6536 100644 --- a/src/domain/session/room/timeline/tilesCreator.js +++ b/src/domain/session/room/timeline/tilesCreator.js @@ -73,7 +73,9 @@ export function tilesCreator(baseOptions) { case "m.room.encryption": return new EncryptionEnabledTile(options); case "m.call": - return entry.stateKey ? new CallTile(options) : null; + // if prevContent is present, it's an update to a call event, which we don't render + // as the original event is updated through the call object which receive state event updates + return entry.stateKey && !entry.prevContent ? new CallTile(options) : null; default: // unknown type not rendered return null; diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 27689013..56822c70 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -86,7 +86,6 @@ export class Session { // although we probably already fetched all devices to send messages in the likely e2ee room await this._deviceTracker.trackRoom(this.rooms.get(roomId), log); const devices = await this._deviceTracker.devicesForRoomMembers(roomId, [userId], this._hsApi, log); - console.log("devices", devices); const encryptedMessage = await this._olmEncryption.encrypt(message.type, message.content, devices, this._hsApi, log); return encryptedMessage; }, diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 269f6020..7b8a7f16 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -112,7 +112,7 @@ export class CallHandler { const callId = event.state_key; let call = this._calls.get(callId); if (call) { - call.updateCallEvent(event); + call.updateCallEvent(event.content); if (call.isTerminated) { this._calls.remove(call.id); } @@ -125,13 +125,13 @@ export class CallHandler { private handleCallMemberEvent(event: StateEvent) { const userId = event.state_key; const calls = event.content["m.calls"] ?? []; - const newCallIdsMemberOf = new Set(calls.map(call => { + for (const call of calls) { const callId = call["m.call_id"]; const groupCall = this._calls.get(callId); // TODO: also check the member when receiving the m.call event groupCall?.addMember(userId, call); - return callId; - })); + }; + const newCallIdsMemberOf = new Set(calls.map(call => call["m.call_id"])); let previousCallIdsMemberOf = this.memberToCallIds.get(userId); // remove user as member of any calls not present anymore if (previousCallIdsMemberOf) { diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 1fd4aad6..b148dd92 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -60,4 +60,10 @@ export class LocalMedia { } return metadata; } + + dispose() { + this.cameraTrack?.stop(); + this.microphoneTrack?.stop(); + this.screenShareTrack?.stop(); + } } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 319fde3a..c6c6fca3 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -748,6 +748,10 @@ export class PeerCall implements IDisposable { this.disposables.dispose(); this.peerConnection.dispose(); } + + public close(): void { + this.peerConnection.close(); + } } diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 901eb3b3..a3148518 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -19,6 +19,7 @@ import {Member} from "./Member"; import {LocalMedia} from "../LocalMedia"; import {RoomMember} from "../../room/members/RoomMember"; import {makeId} from "../../common"; +import {EventEmitter} from "../../../utils/EventEmitter"; import type {Options as MemberOptions} from "./Member"; import type {BaseObservableMap} from "../../../observable/map/BaseObservableMap"; @@ -31,6 +32,9 @@ import type {EncryptedMessage} from "../../e2ee/olm/Encryption"; import type {ILogItem} from "../../../logging/types"; import type {Storage} from "../../storage/idb/Storage"; +const CALL_TYPE = "m.call"; +const CALL_MEMBER_TYPE = "m.call.member"; + export enum GroupCallState { Fledgling = "fledgling", Creating = "creating", @@ -46,7 +50,7 @@ export type Options = Omit { public readonly id: string; private readonly _members: ObservableMap = new ObservableMap(); private _localMedia?: LocalMedia = undefined; @@ -59,6 +63,7 @@ export class GroupCall { public readonly roomId: string, private readonly options: Options ) { + super(); this.id = id ?? makeId("conf-"); this._state = id ? GroupCallState.Created : GroupCallState.Fledgling; this._memberOptions = Object.assign({}, options, { @@ -87,12 +92,12 @@ export class GroupCall { } this._state = GroupCallState.Joining; this._localMedia = localMedia; - this.options.emitUpdate(this); - const memberContent = await this._joinCallMemberContent(); + this.emitChange(); + const memberContent = await this._createJoinPayload(); // send m.call.member state event - const request = this.options.hsApi.sendState(this.roomId, "m.call.member", this.options.ownUserId, memberContent); + const request = this.options.hsApi.sendState(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId, memberContent); await request.response(); - this.options.emitUpdate(this); + this.emitChange(); // send invite to all members that are < my userId for (const [,member] of this._members) { member.connect(this._localMedia); @@ -107,25 +112,41 @@ export class GroupCall { const memberContent = await this._leaveCallMemberContent(); // send m.call.member state event if (memberContent) { - const request = this.options.hsApi.sendState(this.roomId, "m.call.member", this.options.ownUserId, memberContent); + const request = this.options.hsApi.sendState(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId, memberContent); await request.response(); + // our own user isn't included in members, so not in the count + if (this._members.size === 0) { + this.terminate(); + } } } + async terminate() { + if (this._state === GroupCallState.Fledgling) { + return; + } + const request = this.options.hsApi.sendState(this.roomId, CALL_TYPE, this.id, Object.assign({}, this.callContent, { + "m.terminated": true + })); + await request.response(); + } + /** @internal */ async create(localMedia: LocalMedia, name: string) { if (this._state !== GroupCallState.Fledgling) { return; } this._state = GroupCallState.Creating; + this.emitChange(); this.callContent = { "m.type": localMedia.cameraTrack ? "m.video" : "m.voice", "m.name": name, "m.intent": "m.ring" }; - const request = this.options.hsApi.sendState(this.roomId, "m.call", this.id, this.callContent); + const request = this.options.hsApi.sendState(this.roomId, CALL_TYPE, this.id, this.callContent); await request.response(); this._state = GroupCallState.Created; + this.emitChange(); } /** @internal */ @@ -134,6 +155,7 @@ export class GroupCall { if (this._state === GroupCallState.Creating) { this._state = GroupCallState.Created; } + this.emitChange(); } /** @internal */ @@ -141,6 +163,7 @@ export class GroupCall { if (userId === this.options.ownUserId) { if (this._state === GroupCallState.Joining) { this._state = GroupCallState.Joined; + this.emitChange(); } return; } @@ -160,11 +183,21 @@ export class GroupCall { removeMember(userId) { if (userId === this.options.ownUserId) { if (this._state === GroupCallState.Joined) { + this._localMedia?.dispose(); + this._localMedia = undefined; + for (const [,member] of this._members) { + member.disconnect(); + } this._state = GroupCallState.Created; } - return; + } else { + const member = this._members.get(userId); + if (member) { + this._members.remove(userId); + member.disconnect(); + } } - this._members.remove(userId); + this.emitChange(); } /** @internal */ @@ -179,10 +212,10 @@ export class GroupCall { } } - private async _joinCallMemberContent() { + private async _createJoinPayload() { const {storage} = this.options; const txn = await storage.readTxn([storage.storeNames.roomState]); - const stateEvent = await txn.roomState.get(this.roomId, "m.call.member", this.options.ownUserId); + const stateEvent = await txn.roomState.get(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId); const stateContent = stateEvent?.event?.content ?? { ["m.calls"]: [] }; @@ -209,9 +242,18 @@ export class GroupCall { private async _leaveCallMemberContent(): Promise | undefined> { const {storage} = this.options; const txn = await storage.readTxn([storage.storeNames.roomState]); - const stateEvent = await txn.roomState.get(this.roomId, "m.call.member", this.options.ownUserId); - const callsInfo = stateEvent?.event?.content?.["m.calls"]; - callsInfo?.filter(c => c["m.call_id"] === this.id); - return stateEvent?.event.content; + const stateEvent = await txn.roomState.get(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId); + if (stateEvent) { + const content = stateEvent.event.content; + const callsInfo = content["m.calls"]; + content["m.calls"] = callsInfo?.filter(c => c["m.call_id"] !== this.id); + return content; + + } + } + + protected emitChange() { + this.emit("change"); + this.options.emitUpdate(this); } } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 0a80bbef..dedde429 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -65,6 +65,14 @@ export class Member { } } + /** @internal */ + disconnect() { + this.peerCall?.close(); + this.peerCall?.dispose(); + this.peerCall = undefined; + this.localMedia = undefined; + } + /** @internal */ updateCallInfo(memberCallInfo) { // m.calls object from the m.call.member event diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js index ca11cdd8..ff1926b4 100644 --- a/src/matrix/room/Room.js +++ b/src/matrix/room/Room.js @@ -451,7 +451,7 @@ export class Room extends BaseRoom { _updateCallHandler(roomResponse, log) { if (this._callHandler) { const stateEvents = roomResponse.state?.events; - if (stateEvents) { + if (stateEvents?.length) { this._callHandler.handleRoomState(this, stateEvents, log); } let timelineEvents = roomResponse.timeline?.events; diff --git a/src/observable/value/PickMapObservableValue.ts b/src/observable/value/PickMapObservableValue.ts index 67f9d562..b493d841 100644 --- a/src/observable/value/PickMapObservableValue.ts +++ b/src/observable/value/PickMapObservableValue.ts @@ -60,13 +60,11 @@ export class PickMapObservableValue extends BaseObservableValue(t: TemplateBuilder, video: HTMLVideoElement, propSelector: (vm: T) => Track[]) { t.mapSideEffect(propSelector, tracks => { - console.log("tracks", tracks); if (tracks.length) { video.srcObject = (tracks[0] as TrackWrapper).stream; } @@ -33,8 +32,8 @@ function bindVideoTracks(t: TemplateBuilder, video: HTMLVideoElement, prop export class CallView extends TemplateView { render(t: TemplateBuilder, vm: CallViewModel): HTMLElement { return t.div({class: "CallView"}, [ - t.p(["Call ", vm => vm.name, vm => ` (${vm.id})`]), - t.div({class: "CallView_me"}, bindVideoTracks(t, t.video({autoplay: true}), vm => vm.localTracks)), + t.p(vm => `Call ${vm.name} (${vm.id})`), + t.div({class: "CallView_me"}, bindVideoTracks(t, t.video({autoplay: true, width: 240}), vm => vm.localTracks)), t.view(new ListView({list: vm.memberViewModels}, vm => new MemberView(vm))) ]); } @@ -42,6 +41,6 @@ export class CallView extends TemplateView { class MemberView extends TemplateView { render(t: TemplateBuilder, vm: CallMemberViewModel) { - return bindVideoTracks(t, t.video({autoplay: true}), vm => vm.tracks); + return bindVideoTracks(t, t.video({autoplay: true, width: 360}), vm => vm.tracks); } } diff --git a/src/platform/web/ui/session/room/timeline/CallTileView.ts b/src/platform/web/ui/session/room/timeline/CallTileView.ts index dfb04228..e0ca00bd 100644 --- a/src/platform/web/ui/session/room/timeline/CallTileView.ts +++ b/src/platform/web/ui/session/room/timeline/CallTileView.ts @@ -22,9 +22,9 @@ export class CallTileView extends TemplateView { return t.li( {className: "AnnouncementView"}, t.div([ - "Call ", - vm => vm.name, - t.button({className: "CallTileView_join"}, "Join") + vm => vm.label, + t.button({className: "CallTileView_join", hidden: vm => !vm.canJoin}, "Join"), + t.button({className: "CallTileView_leave", hidden: vm => !vm.canLeave}, "Leave") ]) ); } @@ -33,6 +33,8 @@ export class CallTileView extends TemplateView { onClick(evt) { if (evt.target.className === "CallTileView_join") { this.value.join(); + } else if (evt.target.className === "CallTileView_leave") { + this.value.leave(); } } } From eaf92b382bfe5155f19f256bd7b9f437d92b8d20 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 25 Mar 2022 14:43:02 +0100 Subject: [PATCH 028/323] add structured logging to call code --- src/logging/BaseLogger.ts | 23 +- src/logging/LogItem.ts | 9 +- src/logging/NullLogger.ts | 9 +- src/logging/types.ts | 13 + src/matrix/DeviceMessageHandler.js | 1 - src/matrix/Session.js | 7 +- src/matrix/calls/CallHandler.ts | 28 +- src/matrix/calls/PeerCall.ts | 479 +++++++++++++++------------- src/matrix/calls/group/GroupCall.ts | 242 +++++++------- src/matrix/calls/group/Member.ts | 18 +- 10 files changed, 478 insertions(+), 351 deletions(-) diff --git a/src/logging/BaseLogger.ts b/src/logging/BaseLogger.ts index e32b9f0f..21643c48 100644 --- a/src/logging/BaseLogger.ts +++ b/src/logging/BaseLogger.ts @@ -36,6 +36,15 @@ export abstract class BaseLogger implements ILogger { this._persistItem(item, undefined, false); } + /** Prefer `run()` or `log()` above this method; only use it if you have a long-running operation + * *without* a single call stack that should be logged into one sub-tree. + * You need to call `finish()` on the returned item or it will stay open until the app unloads. */ + child(labelOrValues: LabelOrValues, logLevel: LogLevel = LogLevel.Info, filterCreator?: FilterCreator): ILogItem { + const item = new DeferredPersistRootLogItem(labelOrValues, logLevel, this, filterCreator); + this._openItems.add(item); + return item; + } + /** if item is a log item, wrap the callback in a child of it, otherwise start a new root log item. */ wrapOrRun(item: ILogItem | undefined, labelOrValues: LabelOrValues, callback: LogCallback, logLevel?: LogLevel, filterCreator?: FilterCreator): T { if (item) { @@ -127,7 +136,7 @@ export abstract class BaseLogger implements ILogger { _finishOpenItems() { for (const openItem of this._openItems) { - openItem.finish(); + openItem.forceFinish(); try { // for now, serialize with an all-permitting filter // as the createFilter function would get a distorted image anyway @@ -158,3 +167,15 @@ export abstract class BaseLogger implements ILogger { return Math.round(this._platform.random() * Number.MAX_SAFE_INTEGER); } } + +class DeferredPersistRootLogItem extends LogItem { + finish() { + super.finish(); + (this._logger as BaseLogger)._persistItem(this, undefined, false); + } + + forceFinish() { + super.finish(); + /// no need to persist when force-finishing as _finishOpenItems above will do it + } +} diff --git a/src/logging/LogItem.ts b/src/logging/LogItem.ts index b47b69c1..216cc6bb 100644 --- a/src/logging/LogItem.ts +++ b/src/logging/LogItem.ts @@ -25,7 +25,7 @@ export class LogItem implements ILogItem { public error?: Error; public end?: number; private _values: LogItemValues; - private _logger: BaseLogger; + protected _logger: BaseLogger; private _filterCreator?: FilterCreator; private _children?: Array; @@ -221,6 +221,11 @@ export class LogItem implements ILogItem { } } + /** @internal */ + forceFinish(): void { + this.finish(); + } + // expose log level without needing import everywhere get level(): typeof LogLevel { return LogLevel; @@ -235,7 +240,7 @@ export class LogItem implements ILogItem { child(labelOrValues: LabelOrValues, logLevel?: LogLevel, filterCreator?: FilterCreator): LogItem { if (this.end) { - console.trace("log item is finished, additional logs will likely not be recorded"); + console.trace(`log item ${this.values.l} finished, additional log ${JSON.stringify(labelOrValues)} will likely not be recorded`); } if (!logLevel) { logLevel = this.logLevel || LogLevel.Info; diff --git a/src/logging/NullLogger.ts b/src/logging/NullLogger.ts index 21c3d349..adc2b843 100644 --- a/src/logging/NullLogger.ts +++ b/src/logging/NullLogger.ts @@ -23,6 +23,10 @@ export class NullLogger implements ILogger { log(): void {} + child(): ILogItem { + return this.item; + } + run(_, callback: LogCallback): T { return callback(this.item); } @@ -50,13 +54,13 @@ export class NullLogger implements ILogger { } export class NullLogItem implements ILogItem { - public readonly logger: NullLogger; + public readonly logger: ILogger; public readonly logLevel: LogLevel; public children?: Array; public values: LogItemValues; public error?: Error; - constructor(logger: NullLogger) { + constructor(logger: ILogger) { this.logger = logger; } @@ -99,6 +103,7 @@ export class NullLogItem implements ILogItem { } finish(): void {} + forceFinish(): void {} serialize(): undefined { return undefined; diff --git a/src/logging/types.ts b/src/logging/types.ts index bf9861a5..8e35dbf3 100644 --- a/src/logging/types.ts +++ b/src/logging/types.ts @@ -51,11 +51,24 @@ export interface ILogItem { catch(err: Error): Error; serialize(filter: LogFilter, parentStartTime: number | undefined, forced: boolean): ISerializedItem | undefined; finish(): void; + forceFinish(): void; child(labelOrValues: LabelOrValues, logLevel?: LogLevel, filterCreator?: FilterCreator): ILogItem; } +/* +extend both ILogger and ILogItem from this interface, but need to rename ILogger.run => wrap then. Or both to `span`? + +export interface ILogItemCreator { + child(labelOrValues: LabelOrValues, logLevel?: LogLevel, filterCreator?: FilterCreator): ILogItem; + refDetached(logItem: ILogItem, logLevel?: LogLevel): void; + log(labelOrValues: LabelOrValues, logLevel?: LogLevel): ILogItem; + wrap(labelOrValues: LabelOrValues, callback: LogCallback, logLevel?: LogLevel, filterCreator?: FilterCreator): T; + get level(): typeof LogLevel; +} +*/ export interface ILogger { log(labelOrValues: LabelOrValues, logLevel?: LogLevel): void; + child(labelOrValues: LabelOrValues, logLevel?: LogLevel, filterCreator?: FilterCreator): ILogItem; wrapOrRun(item: ILogItem | undefined, labelOrValues: LabelOrValues, callback: LogCallback, logLevel?: LogLevel, filterCreator?: FilterCreator): T; runDetached(labelOrValues: LabelOrValues, callback: LogCallback, logLevel?: LogLevel, filterCreator?: FilterCreator): ILogItem; run(labelOrValues: LabelOrValues, callback: LogCallback, logLevel?: LogLevel, filterCreator?: FilterCreator): T; diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index 11c10750..80fd1592 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -86,7 +86,6 @@ export class DeviceMessageHandler { this._senderDeviceCache.set(device); } } - console.log("incoming device message", senderKey, device, this._senderDeviceCache); return device; } } diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 56822c70..a1e1cc28 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -84,8 +84,10 @@ export class Session { } // TODO: just get the devices we're sending the message to, not all the room devices // although we probably already fetched all devices to send messages in the likely e2ee room - await this._deviceTracker.trackRoom(this.rooms.get(roomId), log); - const devices = await this._deviceTracker.devicesForRoomMembers(roomId, [userId], this._hsApi, log); + const devices = await log.wrap("get device keys", async log => { + await this._deviceTracker.trackRoom(this.rooms.get(roomId), log); + return this._deviceTracker.devicesForRoomMembers(roomId, [userId], this._hsApi, log); + }); const encryptedMessage = await this._olmEncryption.encrypt(message.type, message.content, devices, this._hsApi, log); return encryptedMessage; }, @@ -93,6 +95,7 @@ export class Session { webRTC: this._platform.webRTC, ownDeviceId: sessionInfo.deviceId, ownUserId: sessionInfo.userId, + logger: this._platform.logger, }); this._deviceMessageHandler = new DeviceMessageHandler({storage, callHandler: this._callHandler}); this._olm = olm; diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 7b8a7f16..ef617cd2 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -25,7 +25,7 @@ import type {LocalMedia} from "./LocalMedia"; import type {Room} from "../room/Room"; import type {MemberChange} from "../room/members/RoomMember"; import type {StateEvent} from "../storage/types"; -import type {ILogItem} from "../../logging/types"; +import type {ILogItem, ILogger} from "../../logging/types"; import type {Platform} from "../../platform/web/Platform"; import type {BaseObservableMap} from "../../observable/map/BaseObservableMap"; import type {SignallingMessage, MGroupCallBase} from "./callEventTypes"; @@ -35,7 +35,9 @@ const GROUP_CALL_TYPE = "m.call"; const GROUP_CALL_MEMBER_TYPE = "m.call.member"; const CALL_TERMINATED = "m.terminated"; -export type Options = Omit; +export type Options = Omit & { + logger: ILogger +}; export class CallHandler { // group calls by call id @@ -51,7 +53,8 @@ export class CallHandler { } async createCall(roomId: string, localMedia: LocalMedia, name: string): Promise { - const call = new GroupCall(undefined, undefined, roomId, this.groupCallOptions); + const logItem = this.options.logger.child({l: "call", incoming: false}); + const call = new GroupCall(undefined, undefined, roomId, this.groupCallOptions, logItem); console.log("created call with id", call.id); this._calls.set(call.id, call); try { @@ -59,6 +62,7 @@ export class CallHandler { } catch (err) { if (err.name === "ConnectionError") { // if we're offline, give up and remove the call again + call.dispose(); this._calls.remove(call.id); } throw err; @@ -79,13 +83,13 @@ export class CallHandler { // first update call events for (const event of events) { if (event.type === EventType.GroupCall) { - this.handleCallEvent(event, room.id); + this.handleCallEvent(event, room.id, log); } } // then update members for (const event of events) { if (event.type === EventType.GroupCallMember) { - this.handleCallMemberEvent(event); + this.handleCallMemberEvent(event, log); } } } @@ -108,28 +112,30 @@ export class CallHandler { call?.handleDeviceMessage(message, userId, deviceId, log); } - private handleCallEvent(event: StateEvent, roomId: string) { + private handleCallEvent(event: StateEvent, roomId: string, log: ILogItem) { const callId = event.state_key; let call = this._calls.get(callId); if (call) { - call.updateCallEvent(event.content); + call.updateCallEvent(event.content, log); if (call.isTerminated) { + call.dispose(); this._calls.remove(call.id); } } else { - call = new GroupCall(event.state_key, event.content, roomId, this.groupCallOptions); + const logItem = this.options.logger.child({l: "call", incoming: true}); + call = new GroupCall(event.state_key, event.content, roomId, this.groupCallOptions, logItem); this._calls.set(call.id, call); } } - private handleCallMemberEvent(event: StateEvent) { + private handleCallMemberEvent(event: StateEvent, log: ILogItem) { const userId = event.state_key; const calls = event.content["m.calls"] ?? []; for (const call of calls) { const callId = call["m.call_id"]; const groupCall = this._calls.get(callId); // TODO: also check the member when receiving the m.call event - groupCall?.addMember(userId, call); + groupCall?.addMember(userId, call, log); }; const newCallIdsMemberOf = new Set(calls.map(call => call["m.call_id"])); let previousCallIdsMemberOf = this.memberToCallIds.get(userId); @@ -138,7 +144,7 @@ export class CallHandler { for (const previousCallId of previousCallIdsMemberOf) { if (!newCallIdsMemberOf.has(previousCallId)) { const groupCall = this._calls.get(previousCallId); - groupCall?.removeMember(userId); + groupCall?.removeMember(userId, log); } } } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index c6c6fca3..c4355e63 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -21,7 +21,6 @@ import {Disposables, IDisposable} from "../../utils/Disposables"; import type {Room} from "../room/Room"; import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; -import {Instance as logger} from "../../logging/NullLogger"; import type {TimeoutCreator, Timeout} from "../../platform/types/types"; import {WebRTC, PeerConnection, PeerConnectionHandler, DataChannel} from "../../platform/types/WebRTC"; @@ -69,9 +68,8 @@ export class PeerCall implements IDisposable { // If candidates arrive before we've picked an opponent (which, in particular, // will happen if the opponent sends candidates eagerly before the user answers // the call) we buffer them up here so we can then add the ones from the party we pick - private remoteCandidateBuffer? = new Map(); + private remoteCandidateBuffer? = new Map(); - private logger: any; private remoteSDPStreamMetadata?: SDPStreamMetadata; private responsePromiseChain?: Promise; private opponentPartyId?: PartyId; @@ -88,38 +86,44 @@ export class PeerCall implements IDisposable { constructor( private callId: string, - private readonly options: Options + private readonly options: Options, + private readonly logItem: ILogItem, ) { const outer = this; this.peerConnection = options.webRTC.createPeerConnection({ onIceConnectionStateChange(state: RTCIceConnectionState) { - outer.onIceConnectionStateChange(state); + outer.logItem.wrap({l: "onIceConnectionStateChange", status: state}, log => { + outer.onIceConnectionStateChange(state, log); + }); }, onLocalIceCandidate(candidate: RTCIceCandidate) { - outer.handleLocalIceCandidate(candidate); + outer.logItem.wrap("onLocalIceCandidate", log => { + outer.handleLocalIceCandidate(candidate, log); + }); }, onIceGatheringStateChange(state: RTCIceGatheringState) { - outer.handleIceGatheringState(state); + outer.logItem.wrap({l: "onIceGatheringStateChange", status: state}, log => { + outer.handleIceGatheringState(state, log); + }); }, onRemoteTracksChanged(tracks: Track[]) { - outer.options.emitUpdate(outer, undefined); + outer.logItem.wrap("onRemoteTracksChanged", log => { + outer.options.emitUpdate(outer, undefined); + }); }, onDataChannelChanged(dataChannel: DataChannel | undefined) {}, onNegotiationNeeded() { - const promiseCreator = () => outer.handleNegotiation(); + const log = outer.logItem.child("onNegotiationNeeded"); + const promiseCreator = async () => { + await outer.handleNegotiation(log); + log.finish(); + }; outer.responsePromiseChain = outer.responsePromiseChain?.then(promiseCreator) ?? promiseCreator(); }, getPurposeForStreamId(streamId: string): SDPStreamMetadataPurpose { return outer.remoteSDPStreamMetadata?.[streamId]?.purpose ?? SDPStreamMetadataPurpose.Usermedia; } }); - this.logger = { - info(...args) { console.info.apply(console, ["WebRTC debug:", ...args])}, - debug(...args) { console.log.apply(console, ["WebRTC debug:", ...args])}, - log(...args) { console.log.apply(console, ["WebRTC log:", ...args])}, - warn(...args) { console.log.apply(console, ["WebRTC warn:", ...args])}, - error(...args) { console.error.apply(console, ["WebRTC error:", ...args])}, - }; } get state(): CallState { return this._state; } @@ -128,108 +132,127 @@ export class PeerCall implements IDisposable { return this.peerConnection.remoteTracks; } - async call(localMedia: LocalMedia): Promise { - if (this._state !== CallState.Fledgling) { - return; - } - this.localMedia = localMedia; - this.direction = CallDirection.Outbound; - this.setState(CallState.CreateOffer); - for (const t of this.localMedia.tracks) { - this.peerConnection.addTrack(t); - } - // after adding the local tracks, and wait for handleNegotiation to be called, - // or invite glare where we give up our invite and answer instead - await this.waitForState([CallState.InviteSent, CallState.CreateAnswer]); - } - - async answer(localMedia: LocalMedia): Promise { - if (this._state !== CallState.Ringing) { - return; - } - this.localMedia = localMedia; - this.setState(CallState.CreateAnswer); - for (const t of this.localMedia.tracks) { - this.peerConnection.addTrack(t); - } - - let myAnswer: RTCSessionDescriptionInit; - try { - myAnswer = await this.peerConnection.createAnswer(); - } catch (err) { - this.logger.debug(`Call ${this.callId} Failed to create answer: `, err); - this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true); - return; - } - - try { - await this.peerConnection.setLocalDescription(myAnswer); - this.setState(CallState.Connecting); - } catch (err) { - this.logger.debug(`Call ${this.callId} Error setting local description!`, err); - this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); - return; - } - // Allow a short time for initial candidates to be gathered - await this.delay(200); - await this.sendAnswer(); - } - - async setMedia(localMediaPromise: Promise) { - const oldMedia = this.localMedia; - this.localMedia = await localMediaPromise; - - const applyTrack = (selectTrack: (media: LocalMedia | undefined) => Track | undefined) => { - const oldTrack = selectTrack(oldMedia); - const newTrack = selectTrack(this.localMedia); - if (oldTrack && newTrack) { - this.peerConnection.replaceTrack(oldTrack, newTrack); - } else if (oldTrack) { - this.peerConnection.removeTrack(oldTrack); - } else if (newTrack) { - this.peerConnection.addTrack(newTrack); + call(localMedia: LocalMedia): Promise { + return this.logItem.wrap("call", async log => { + if (this._state !== CallState.Fledgling) { + return; } - }; + this.localMedia = localMedia; + this.direction = CallDirection.Outbound; + this.setState(CallState.CreateOffer); + for (const t of this.localMedia.tracks) { + this.peerConnection.addTrack(t); + } + // after adding the local tracks, and wait for handleNegotiation to be called, + // or invite glare where we give up our invite and answer instead + await this.waitForState([CallState.InviteSent, CallState.CreateAnswer]); + }); + } - // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called - applyTrack(m => m?.microphoneTrack); - applyTrack(m => m?.cameraTrack); - applyTrack(m => m?.screenShareTrack); + answer(localMedia: LocalMedia): Promise { + return this.logItem.wrap("answer", async log => { + if (this._state !== CallState.Ringing) { + return; + } + this.localMedia = localMedia; + this.setState(CallState.CreateAnswer); + for (const t of this.localMedia.tracks) { + this.peerConnection.addTrack(t); + } + + let myAnswer: RTCSessionDescriptionInit; + try { + myAnswer = await this.peerConnection.createAnswer(); + } catch (err) { + await log.wrap(`Failed to create answer`, log => { + log.catch(err); + this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true, log); + }); + return; + } + + try { + await this.peerConnection.setLocalDescription(myAnswer); + this.setState(CallState.Connecting); + } catch (err) { + await log.wrap(`Error setting local description!`, log => { + log.catch(err); + this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true, log); + }); + return; + } + // Allow a short time for initial candidates to be gathered + try { await this.delay(200); } + catch (err) { return; } + await this.sendAnswer(log); + }); + } + + setMedia(localMediaPromise: Promise): Promise { + return this.logItem.wrap("setMedia", async log => { + const oldMedia = this.localMedia; + this.localMedia = await localMediaPromise; + + const applyTrack = (selectTrack: (media: LocalMedia | undefined) => Track | undefined) => { + const oldTrack = selectTrack(oldMedia); + const newTrack = selectTrack(this.localMedia); + if (oldTrack && newTrack) { + this.peerConnection.replaceTrack(oldTrack, newTrack); + } else if (oldTrack) { + this.peerConnection.removeTrack(oldTrack); + } else if (newTrack) { + this.peerConnection.addTrack(newTrack); + } + }; + + // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called + applyTrack(m => m?.microphoneTrack); + applyTrack(m => m?.cameraTrack); + applyTrack(m => m?.screenShareTrack); + }); } async reject() { } - async hangup(errorCode: CallErrorCode): Promise { + hangup(errorCode: CallErrorCode): Promise { + return this.logItem.wrap("hangup", log => { + return this._hangup(errorCode, log); + }); + } + + private async _hangup(errorCode: CallErrorCode, log: ILogItem): Promise { if (this._state !== CallState.Ended) { this._state = CallState.Ended; - await this.sendHangupWithCallId(this.callId, errorCode); + await this.sendHangupWithCallId(this.callId, errorCode, log); } } - async handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId, log: ILogItem): Promise { - switch (message.type) { - case EventType.Invite: - if (this.callId !== message.content.call_id) { - await this.handleInviteGlare(message.content, partyId); - } else { - await this.handleFirstInvite(message.content, partyId); - } - break; - case EventType.Answer: - await this.handleAnswer(message.content, partyId); - break; - case EventType.Candidates: - await this.handleRemoteIceCandidates(message.content, partyId); - break; - case EventType.Hangup: - default: - throw new Error(`Unknown event type for call: ${message.type}`); - } + handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId): Promise { + return this.logItem.wrap({l: "receive", id: message.type, partyId}, async log => { + switch (message.type) { + case EventType.Invite: + if (this.callId !== message.content.call_id) { + await this.handleInviteGlare(message.content, partyId, log); + } else { + await this.handleFirstInvite(message.content, partyId, log); + } + break; + case EventType.Answer: + await this.handleAnswer(message.content, partyId, log); + break; + case EventType.Candidates: + await this.handleRemoteIceCandidates(message.content, partyId, log); + break; + case EventType.Hangup: + default: + throw new Error(`Unknown event type for call: ${message.type}`); + } + }); } - private sendHangupWithCallId(callId: string, reason?: CallErrorCode): Promise { + private sendHangupWithCallId(callId: string, reason: CallErrorCode | undefined, log: ILogItem): Promise { const content = { call_id: callId, version: 1, @@ -237,27 +260,28 @@ export class PeerCall implements IDisposable { if (reason) { content["reason"] = reason; } - return this.options.sendSignallingMessage({ + return this.sendSignallingMessage({ type: EventType.Hangup, content - }, logger.item); + }, log); } // calls are serialized and deduplicated by responsePromiseChain - private handleNegotiation = async (): Promise => { + private handleNegotiation = async (log: ILogItem): Promise => { this.makingOffer = true; try { try { await this.peerConnection.setLocalDescription(); } catch (err) { - this.logger.debug(`Call ${this.callId} Error setting local description!`, err); - this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); + log.log(`Error setting local description!`).catch(err); + this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true, log); return; } if (this.peerConnection.iceGatheringState === 'gathering') { // Allow a short time for initial candidates to be gathered - await this.delay(200); + try { await this.delay(200); } + catch (err) { return; } } if (this._state === CallState.Ended) { @@ -267,7 +291,7 @@ export class PeerCall implements IDisposable { const offer = this.peerConnection.localDescription!; // Get rid of any candidates waiting to be sent: they'll be included in the local // description we just got and will send in the offer. - this.logger.info(`Call ${this.callId} Discarding ${ + log.log(`Discarding ${ this.candidateSendQueue.length} candidates that will be sent in offer`); this.candidateSendQueue = []; @@ -280,63 +304,64 @@ export class PeerCall implements IDisposable { lifetime: CALL_TIMEOUT_MS }; if (this._state === CallState.CreateOffer) { - await this.options.sendSignallingMessage({type: EventType.Invite, content}, logger.item); + await this.sendSignallingMessage({type: EventType.Invite, content}, log); this.setState(CallState.InviteSent); } else if (this._state === CallState.Connected || this._state === CallState.Connecting) { // send Negotiate message - //await this.options.sendSignallingMessage({type: EventType.Invite, content}); + //await this.sendSignallingMessage({type: EventType.Invite, content}); //this.setState(CallState.InviteSent); } } finally { this.makingOffer = false; } - this.sendCandidateQueue(); + this.sendCandidateQueue(log); if (this._state === CallState.InviteSent) { - await this.delay(CALL_TIMEOUT_MS); + try { await this.delay(CALL_TIMEOUT_MS); } + catch (err) { return; } // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between if (this._state === CallState.InviteSent) { - this.hangup(CallErrorCode.InviteTimeout); + this._hangup(CallErrorCode.InviteTimeout, log); } } }; - private async handleInviteGlare(content: MCallInvite, partyId: PartyId): Promise { + private async handleInviteGlare(content: MCallInvite, partyId: PartyId, log: ILogItem): Promise { // this is only called when the ids are different const newCallId = content.call_id; if (this.callId! > newCallId) { - this.logger.log( + log.log( "Glare detected: answering incoming call " + newCallId + - " and canceling outgoing call " + this.callId, + " and canceling outgoing call ", ); // How do we interrupt `call()`? well, perhaps we need to not just await InviteSent but also CreateAnswer? if (this._state === CallState.Fledgling || this._state === CallState.CreateOffer) { // TODO: don't send invite! } else { - await this.sendHangupWithCallId(this.callId, CallErrorCode.Replaced); + await this.sendHangupWithCallId(this.callId, CallErrorCode.Replaced, log); } - await this.handleInvite(content, partyId); + await this.handleInvite(content, partyId, log); // TODO: need to skip state check await this.answer(this.localMedia!); } else { - this.logger.log( + log.log( "Glare detected: rejecting incoming call " + newCallId + - " and keeping outgoing call " + this.callId, + " and keeping outgoing call ", ); - await this.sendHangupWithCallId(newCallId, CallErrorCode.Replaced); + await this.sendHangupWithCallId(newCallId, CallErrorCode.Replaced, log); } } - private async handleFirstInvite(content: MCallInvite, partyId: PartyId): Promise { + private async handleFirstInvite(content: MCallInvite, partyId: PartyId, log: ILogItem): Promise { if (this._state !== CallState.Fledgling || this.opponentPartyId !== undefined) { // TODO: hangup or ignore? return; } - await this.handleInvite(content, partyId); + await this.handleInvite(content, partyId, log); } - private async handleInvite(content: MCallInvite, partyId: PartyId): Promise { + private async handleInvite(content: MCallInvite, partyId: PartyId, log: ILogItem): Promise { // we must set the party ID before await-ing on anything: the call event // handler will start giving us more call events (eg. candidates) so if @@ -348,17 +373,18 @@ export class PeerCall implements IDisposable { if (sdpStreamMetadata) { this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); } else { - this.logger.debug(`Call ${ - this.callId} did not get any SDPStreamMetadata! Can not send/receive multiple streams`); + log.log(`Call did not get any SDPStreamMetadata! Can not send/receive multiple streams`); } try { // Q: Why do we set the remote description before accepting the call? To start creating ICE candidates? await this.peerConnection.setRemoteDescription(content.offer); - await this.addBufferedIceCandidates(); + await this.addBufferedIceCandidates(log); } catch (e) { - this.logger.debug(`Call ${this.callId} failed to set remote description`, e); - this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); + await log.wrap(`Call failed to set remote description`, async log => { + log.catch(e); + return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false, log); + }); return; } @@ -366,17 +392,19 @@ export class PeerCall implements IDisposable { // add streams until media started arriving on them. Testing latest firefox // (81 at time of writing), this is no longer a problem, so let's do it the correct way. if (this.peerConnection.remoteTracks.length === 0) { - this.logger.error(`Call ${this.callId} no remote stream or no tracks after setting remote description!`); - this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); + await log.wrap(`Call no remote stream or no tracks after setting remote description!`, async log => { + return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false, log); + }); return; } this.setState(CallState.Ringing); - await this.delay(content.lifetime ?? CALL_TIMEOUT_MS); + try { await this.delay(content.lifetime ?? CALL_TIMEOUT_MS); } + catch (err) { return; } // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between if (this._state === CallState.Ringing) { - this.logger.debug(`Call ${this.callId} invite has expired. Hanging up.`); + log.log(`Invite has expired. Hanging up.`); this.hangupParty = CallParty.Remote; // effectively this.setState(CallState.Ended); this.stopAllMedia(); @@ -386,25 +414,19 @@ export class PeerCall implements IDisposable { } } - private async handleAnswer(content: MCallAnswer, partyId: PartyId): Promise { - this.logger.debug(`Got answer for call ID ${this.callId} from party ID ${partyId}`); - + private async handleAnswer(content: MCallAnswer, partyId: PartyId, log: ILogItem): Promise { if (this._state === CallState.Ended) { - this.logger.debug(`Ignoring answer because call ID ${this.callId} has ended`); + log.log(`Ignoring answer because call has ended`); return; } if (this.opponentPartyId !== undefined) { - this.logger.info( - `Call ${this.callId} ` + - `Ignoring answer from party ID ${partyId}: ` + - `we already have an answer/reject from ${this.opponentPartyId}`, - ); + log.log(`Ignoring answer: we already have an answer/reject from ${this.opponentPartyId}`); return; } this.opponentPartyId = partyId; - await this.addBufferedIceCandidates(); + await this.addBufferedIceCandidates(log); this.setState(CallState.Connecting); @@ -412,20 +434,22 @@ export class PeerCall implements IDisposable { if (sdpStreamMetadata) { this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); } else { - this.logger.warn(`Call ${this.callId} Did not get any SDPStreamMetadata! Can not send/receive multiple streams`); + log.log(`Did not get any SDPStreamMetadata! Can not send/receive multiple streams`); } try { await this.peerConnection.setRemoteDescription(content.answer); } catch (e) { - this.logger.debug(`Call ${this.callId} Failed to set remote description`, e); - this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); + await log.wrap(`Failed to set remote description`, log => { + log.catch(e); + this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false, log); + }); return; } } - private handleIceGatheringState(state: RTCIceGatheringState) { - this.logger.debug(`Call ${this.callId} ice gathering state changed to ${state}`); + private handleIceGatheringState(state: RTCIceGatheringState, log: ILogItem) { + log.set("state", state); if (state === 'complete' && !this.sentEndOfCandidates) { // If we didn't get an empty-string candidate to signal the end of candidates, // create one ourselves now gathering has finished. @@ -437,37 +461,37 @@ export class PeerCall implements IDisposable { const c = { candidate: '', } as RTCIceCandidate; - this.queueCandidate(c); + this.queueCandidate(c, log); this.sentEndOfCandidates = true; } } - private handleLocalIceCandidate(candidate: RTCIceCandidate) { - this.logger.debug( - "Call " + this.callId + " got local ICE " + candidate.sdpMid + " candidate: " + - candidate.candidate, - ); - - if (this._state === CallState.Ended) return; + private handleLocalIceCandidate(candidate: RTCIceCandidate, log: ILogItem) { + log.set("sdpMid", candidate.sdpMid); + log.set("candidate", candidate.candidate); + if (this._state === CallState.Ended) { + return; + } // As with the offer, note we need to make a copy of this object, not // pass the original: that broke in Chrome ~m43. if (candidate.candidate !== '' || !this.sentEndOfCandidates) { - this.queueCandidate(candidate); - - if (candidate.candidate === '') this.sentEndOfCandidates = true; + this.queueCandidate(candidate, log); + if (candidate.candidate === '') { + this.sentEndOfCandidates = true; + } } } - private async handleRemoteIceCandidates(content: MCallCandidates, partyId) { + private async handleRemoteIceCandidates(content: MCallCandidates, partyId: PartyId, log: ILogItem) { if (this.state === CallState.Ended) { - //debuglog("Ignoring remote ICE candidate because call has ended"); + log.log("Ignoring remote ICE candidate because call has ended"); return; } const candidates = content.candidates; if (!candidates) { - this.logger.info(`Call ${this.callId} Ignoring candidates event with no candidates!`); + log.log(`Ignoring candidates event with no candidates!`); return; } @@ -475,7 +499,7 @@ export class PeerCall implements IDisposable { if (this.opponentPartyId === undefined) { // we haven't picked an opponent yet so save the candidates - this.logger.info(`Call ${this.callId} Buffering ${candidates.length} candidates until we pick an opponent`); + log.log(`Buffering ${candidates.length} candidates until we pick an opponent`); const bufferedCandidates = this.remoteCandidateBuffer!.get(fromPartyId) || []; bufferedCandidates.push(...candidates); this.remoteCandidateBuffer!.set(fromPartyId, bufferedCandidates); @@ -483,8 +507,7 @@ export class PeerCall implements IDisposable { } if (this.opponentPartyId !== partyId) { - this.logger.info( - `Call ${this.callId} `+ + log.log( `Ignoring candidates from party ID ${partyId}: ` + `we have chosen party ID ${this.opponentPartyId}`, ); @@ -492,14 +515,14 @@ export class PeerCall implements IDisposable { return; } - await this.addIceCandidates(candidates); + await this.addIceCandidates(candidates, log); } // private async onNegotiateReceived(event: MatrixEvent): Promise { // const content = event.getContent(); // const description = content.description; // if (!description || !description.sdp || !description.type) { - // this.logger.info(`Call ${this.callId} Ignoring invalid m.call.negotiate event`); + // this.logger.info(`Ignoring invalid m.call.negotiate event`); // return; // } // // Politeness always follows the direction of the call: in a glare situation, @@ -516,7 +539,7 @@ export class PeerCall implements IDisposable { // this.ignoreOffer = !polite && offerCollision; // if (this.ignoreOffer) { - // this.logger.info(`Call ${this.callId} Ignoring colliding negotiate event because we're impolite`); + // this.logger.info(`Ignoring colliding negotiate event because we're impolite`); // return; // } @@ -524,7 +547,7 @@ export class PeerCall implements IDisposable { // if (sdpStreamMetadata) { // this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); // } else { - // this.logger.warn(`Call ${this.callId} Received negotiation event without SDPStreamMetadata!`); + // this.logger.warn(`Received negotiation event without SDPStreamMetadata!`); // } // try { @@ -532,7 +555,7 @@ export class PeerCall implements IDisposable { // if (description.type === 'offer') { // await this.peerConnection.setLocalDescription(); - // await this.options.sendSignallingMessage({ + // await this.sendSignallingMessage({ // type: EventType.CallNegotiate, // content: { // description: this.peerConnection.localDescription!, @@ -541,11 +564,11 @@ export class PeerCall implements IDisposable { // }); // } // } catch (err) { - // this.logger.warn(`Call ${this.callId} Failed to complete negotiation`, err); + // this.logger.warn(`Failed to complete negotiation`, err); // } // } - private async sendAnswer(): Promise { + private async sendAnswer(log: ILogItem): Promise { const localDescription = this.peerConnection.localDescription!; const answerContent: MCallAnswer = { call_id: this.callId, @@ -560,23 +583,23 @@ export class PeerCall implements IDisposable { // We have just taken the local description from the peerConn which will // contain all the local candidates added so far, so we can discard any candidates // we had queued up because they'll be in the answer. - this.logger.info(`Call ${this.callId} Discarding ${ + log.log(`Discarding ${ this.candidateSendQueue.length} candidates that will be sent in answer`); this.candidateSendQueue = []; try { - await this.options.sendSignallingMessage({type: EventType.Answer, content: answerContent}, logger.item); + await this.sendSignallingMessage({type: EventType.Answer, content: answerContent}, log); } catch (error) { - this.terminate(CallParty.Local, CallErrorCode.SendAnswer, false); + this.terminate(CallParty.Local, CallErrorCode.SendAnswer, false, log); throw error; } // error handler re-throws so this won't happen on error, but // we don't want the same error handling on the candidate queue - this.sendCandidateQueue(); + this.sendCandidateQueue(log); } - private queueCandidate(content: RTCIceCandidate): void { + private queueCandidate(content: RTCIceCandidate, log: ILogItem): void { // We partially de-trickle candidates by waiting for `delay` before sending them // amalgamated, in order to avoid sending too many m.call.candidates events and hitting // rate limits in Matrix. @@ -593,36 +616,48 @@ export class PeerCall implements IDisposable { // MSC2746 recommends these values (can be quite long when calling because the // callee will need a while to answer the call) - this.delay(this.direction === CallDirection.Inbound ? 500 : 2000).then(() => { - this.sendCandidateQueue(); - }); + const sendLogItem = this.logItem.child("wait to send candidates"); + log.refDetached(sendLogItem); + this.delay(this.direction === CallDirection.Inbound ? 500 : 2000) + .then(() => { + return this.sendCandidateQueue(sendLogItem); + }, err => {}) // swallow delay AbortError + .finally(() => { + sendLogItem.finish(); + }); } - private async sendCandidateQueue(): Promise { - if (this.candidateSendQueue.length === 0 || this._state === CallState.Ended) { - return; - } + private async sendCandidateQueue(log: ILogItem): Promise { + return log.wrap("send candidates queue", async log => { + log.set("queueLength", this.candidateSendQueue.length); - const candidates = this.candidateSendQueue; - this.candidateSendQueue = []; - this.logger.debug(`Call ${this.callId} attempting to send ${candidates.length} candidates`); - try { - await this.options.sendSignallingMessage({ - type: EventType.Candidates, - content: { - call_id: this.callId, - version: 1, - candidates - }, - }, logger.item); - // Try to send candidates again just in case we received more candidates while sending. - this.sendCandidateQueue(); - } catch (error) { - // don't retry this event: we'll send another one later as we might - // have more candidates by then. - // put all the candidates we failed to send back in the queue - this.terminate(CallParty.Local, CallErrorCode.SignallingFailed, false); - } + if (this.candidateSendQueue.length === 0 || this._state === CallState.Ended) { + return; + } + + const candidates = this.candidateSendQueue; + this.candidateSendQueue = []; + try { + await this.sendSignallingMessage({ + type: EventType.Candidates, + content: { + call_id: this.callId, + version: 1, + candidates + }, + }, log); + // Try to send candidates again just in case we received more candidates while sending. + this.sendCandidateQueue(log); + } catch (error) { + log.catch(error); + // don't retry this event: we'll send another one later as we might + // have more candidates by then. + // put all the candidates we failed to send back in the queue + + // TODO: terminate doesn't seem to vibe with the comment above? + this.terminate(CallParty.Local, CallErrorCode.SignallingFailed, false, log); + } + }); } private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { @@ -641,44 +676,44 @@ export class PeerCall implements IDisposable { } } - private async addBufferedIceCandidates(): Promise { + private async addBufferedIceCandidates(log: ILogItem): Promise { if (this.remoteCandidateBuffer && this.opponentPartyId) { const bufferedCandidates = this.remoteCandidateBuffer.get(this.opponentPartyId); if (bufferedCandidates) { - this.logger.info(`Call ${this.callId} Adding ${ + log.log(`Adding ${ bufferedCandidates.length} buffered candidates for opponent ${this.opponentPartyId}`); - await this.addIceCandidates(bufferedCandidates); + await this.addIceCandidates(bufferedCandidates, log); } this.remoteCandidateBuffer = undefined; } } - private async addIceCandidates(candidates: RTCIceCandidate[]): Promise { + private async addIceCandidates(candidates: RTCIceCandidate[], log: ILogItem): Promise { for (const candidate of candidates) { if ( (candidate.sdpMid === null || candidate.sdpMid === undefined) && (candidate.sdpMLineIndex === null || candidate.sdpMLineIndex === undefined) ) { - this.logger.debug(`Call ${this.callId} ignoring remote ICE candidate with no sdpMid or sdpMLineIndex`); + log.log(`Ignoring remote ICE candidate with no sdpMid or sdpMLineIndex`); continue; } - this.logger.debug(`Call ${this.callId} got remote ICE ${candidate.sdpMid} candidate: ${candidate.candidate}`); + log.log(`Got remote ICE ${candidate.sdpMid} candidate: ${candidate.candidate}`); try { await this.peerConnection.addIceCandidate(candidate); } catch (err) { if (!this.ignoreOffer) { - this.logger.info(`Call ${this.callId} failed to add remote ICE candidate`, err); + log.log(`Failed to add remote ICE candidate`, err); } } } } - private onIceConnectionStateChange = (state: RTCIceConnectionState): void => { + private onIceConnectionStateChange = (state: RTCIceConnectionState, log: ILogItem): void => { if (this._state === CallState.Ended) { return; // because ICE can still complete as we're ending the call } - this.logger.debug( - "Call ID " + this.callId + ": ICE connection state changed to: " + state, + log.log( + "ICE connection state changed to: " + state, ); // ideally we'd consider the call to be connected when we get media but // chrome doesn't implement any of the 'onstarted' events yet @@ -689,11 +724,11 @@ export class PeerCall implements IDisposable { } else if (state == 'failed') { this.iceDisconnectedTimeout?.abort(); this.iceDisconnectedTimeout = undefined; - this.hangup(CallErrorCode.IceFailed); + this._hangup(CallErrorCode.IceFailed, log); } else if (state == 'disconnected') { this.iceDisconnectedTimeout = this.options.createTimeout(30 * 1000); this.iceDisconnectedTimeout.elapsed().then(() => { - this.hangup(CallErrorCode.IceFailed); + this._hangup(CallErrorCode.IceFailed, log); }, () => { /* ignore AbortError */ }); } }; @@ -725,7 +760,7 @@ export class PeerCall implements IDisposable { })); } - private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean): Promise { + private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean, log: ILogItem): Promise { } @@ -744,6 +779,12 @@ export class PeerCall implements IDisposable { this.disposables.untrack(timeout); } + private sendSignallingMessage(message: SignallingMessage, log: ILogItem) { + return log.wrap({l: "send", id: message.type}, async log => { + return this.options.sendSignallingMessage(message, log); + }); + } + public dispose(): void { this.disposables.dispose(); this.peerConnection.dispose(); diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index a3148518..86c93ba3 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -61,10 +61,12 @@ export class GroupCall extends EventEmitter<{change: never}> { id: string | undefined, private callContent: Record | undefined, public readonly roomId: string, - private readonly options: Options + private readonly options: Options, + private readonly logItem: ILogItem, ) { super(); this.id = id ?? makeId("conf-"); + logItem.set("id", this.id); this._state = id ? GroupCallState.Created : GroupCallState.Fledgling; this._memberOptions = Object.assign({}, options, { confId: this.id, @@ -86,132 +88,158 @@ export class GroupCall extends EventEmitter<{change: never}> { return this.callContent?.["m.name"]; } - async join(localMedia: LocalMedia) { - if (this._state !== GroupCallState.Created) { - return; - } - this._state = GroupCallState.Joining; - this._localMedia = localMedia; - this.emitChange(); - const memberContent = await this._createJoinPayload(); - // send m.call.member state event - const request = this.options.hsApi.sendState(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId, memberContent); - await request.response(); - this.emitChange(); - // send invite to all members that are < my userId - for (const [,member] of this._members) { - member.connect(this._localMedia); - } + join(localMedia: LocalMedia): Promise { + return this.logItem.wrap("join", async log => { + if (this._state !== GroupCallState.Created) { + return; + } + this._state = GroupCallState.Joining; + this._localMedia = localMedia; + this.emitChange(); + const memberContent = await this._createJoinPayload(); + // send m.call.member state event + const request = this.options.hsApi.sendState(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId, memberContent, {log}); + await request.response(); + this.emitChange(); + // send invite to all members that are < my userId + for (const [,member] of this._members) { + member.connect(this._localMedia); + } + }); } get hasJoined() { return this._state === GroupCallState.Joining || this._state === GroupCallState.Joined; } - async leave() { - const memberContent = await this._leaveCallMemberContent(); - // send m.call.member state event - if (memberContent) { - const request = this.options.hsApi.sendState(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId, memberContent); - await request.response(); - // our own user isn't included in members, so not in the count - if (this._members.size === 0) { - this.terminate(); - } - } - } - - async terminate() { - if (this._state === GroupCallState.Fledgling) { - return; - } - const request = this.options.hsApi.sendState(this.roomId, CALL_TYPE, this.id, Object.assign({}, this.callContent, { - "m.terminated": true - })); - await request.response(); - } - - /** @internal */ - async create(localMedia: LocalMedia, name: string) { - if (this._state !== GroupCallState.Fledgling) { - return; - } - this._state = GroupCallState.Creating; - this.emitChange(); - this.callContent = { - "m.type": localMedia.cameraTrack ? "m.video" : "m.voice", - "m.name": name, - "m.intent": "m.ring" - }; - const request = this.options.hsApi.sendState(this.roomId, CALL_TYPE, this.id, this.callContent); - await request.response(); - this._state = GroupCallState.Created; - this.emitChange(); - } - - /** @internal */ - updateCallEvent(callContent: Record) { - this.callContent = callContent; - if (this._state === GroupCallState.Creating) { - this._state = GroupCallState.Created; - } - this.emitChange(); - } - - /** @internal */ - addMember(userId, memberCallInfo) { - if (userId === this.options.ownUserId) { - if (this._state === GroupCallState.Joining) { - this._state = GroupCallState.Joined; - this.emitChange(); - } - return; - } - let member = this._members.get(userId); - if (member) { - member.updateCallInfo(memberCallInfo); - } else { - member = new Member(RoomMember.fromUserId(this.roomId, userId, "join"), memberCallInfo, this._memberOptions); - this._members.add(userId, member); - if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { - member.connect(this._localMedia!); - } - } - } - - /** @internal */ - removeMember(userId) { - if (userId === this.options.ownUserId) { - if (this._state === GroupCallState.Joined) { - this._localMedia?.dispose(); - this._localMedia = undefined; - for (const [,member] of this._members) { - member.disconnect(); + leave(): Promise { + return this.logItem.wrap("leave", async log => { + const memberContent = await this._leaveCallMemberContent(); + // send m.call.member state event + if (memberContent) { + const request = this.options.hsApi.sendState(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId, memberContent, {log}); + await request.response(); + // our own user isn't included in members, so not in the count + if (this._members.size === 0) { + await this.terminate(); } + } + }); + } + + terminate(): Promise { + return this.logItem.wrap("terminate", async log => { + if (this._state === GroupCallState.Fledgling) { + return; + } + const request = this.options.hsApi.sendState(this.roomId, CALL_TYPE, this.id, Object.assign({}, this.callContent, { + "m.terminated": true + }), {log}); + await request.response(); + }); + } + + /** @internal */ + create(localMedia: LocalMedia, name: string): Promise { + return this.logItem.wrap("create", async log => { + if (this._state !== GroupCallState.Fledgling) { + return; + } + this._state = GroupCallState.Creating; + this.emitChange(); + this.callContent = { + "m.type": localMedia.cameraTrack ? "m.video" : "m.voice", + "m.name": name, + "m.intent": "m.ring" + }; + const request = this.options.hsApi.sendState(this.roomId, CALL_TYPE, this.id, this.callContent, {log}); + await request.response(); + this._state = GroupCallState.Created; + this.emitChange(); + }); + } + + /** @internal */ + updateCallEvent(callContent: Record, syncLog: ILogItem) { + this.logItem.wrap("updateCallEvent", log => { + syncLog.refDetached(log); + this.callContent = callContent; + if (this._state === GroupCallState.Creating) { this._state = GroupCallState.Created; } - } else { - const member = this._members.get(userId); - if (member) { - this._members.remove(userId); - member.disconnect(); - } - } - this.emitChange(); + log.set("status", this._state); + this.emitChange(); + }); } /** @internal */ - handleDeviceMessage(message: SignallingMessage, userId: string, deviceId: string, log: ILogItem) { - console.log("incoming to_device call signalling message from", userId, deviceId, message); + addMember(userId: string, memberCallInfo, syncLog: ILogItem) { + this.logItem.wrap({l: "addMember", id: userId}, log => { + syncLog.refDetached(log); + + if (userId === this.options.ownUserId) { + if (this._state === GroupCallState.Joining) { + this._state = GroupCallState.Joined; + this.emitChange(); + } + return; + } + let member = this._members.get(userId); + if (member) { + member.updateCallInfo(memberCallInfo); + } else { + const logItem = this.logItem.child("member"); + member = new Member(RoomMember.fromUserId(this.roomId, userId, "join"), memberCallInfo, this._memberOptions, logItem); + this._members.add(userId, member); + if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { + member.connect(this._localMedia!); + } + } + }); + } + + /** @internal */ + removeMember(userId: string, syncLog: ILogItem) { + this.logItem.wrap({l: "removeMember", id: userId}, log => { + syncLog.refDetached(log); + if (userId === this.options.ownUserId) { + if (this._state === GroupCallState.Joined) { + this._localMedia?.dispose(); + this._localMedia = undefined; + for (const [,member] of this._members) { + member.disconnect(); + } + this._state = GroupCallState.Created; + } + } else { + const member = this._members.get(userId); + if (member) { + this._members.remove(userId); + member.disconnect(); + } + } + this.emitChange(); + }); + } + + /** @internal */ + handleDeviceMessage(message: SignallingMessage, userId: string, deviceId: string, syncLog: ILogItem) { // TODO: return if we are not membering to the call let member = this._members.get(userId); if (member) { - member.handleDeviceMessage(message, deviceId, log); + member.handleDeviceMessage(message, deviceId, syncLog); } else { + const item = this.logItem.log({l: "could not find member for signalling message", userId, deviceId}); + syncLog.refDetached(item); // we haven't received the m.call.member yet for this caller. buffer the device messages or create the member/call anyway? } } + /** @internal */ + dispose() { + this.logItem.finish(); + } + private async _createJoinPayload() { const {storage} = this.options; const txn = await storage.readTxn([storage.storeNames.roomState]); diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index dedde429..a2dc43de 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -44,8 +44,11 @@ export class Member { constructor( public readonly member: RoomMember, private memberCallInfo: Record, - private readonly options: Options - ) {} + private readonly options: Options, + private readonly logItem: ILogItem, + ) { + logItem.set("id", member.userId); + } get remoteTracks(): Track[] { return this.peerCall?.remoteTracks ?? []; @@ -57,6 +60,7 @@ export class Member { /** @internal */ connect(localMedia: LocalMedia) { + this.logItem.log("connect"); this.localMedia = localMedia; // otherwise wait for it to connect if (this.member.userId < this.options.ownUserId) { @@ -71,6 +75,7 @@ export class Member { this.peerCall?.dispose(); this.peerCall = undefined; this.localMedia = undefined; + this.logItem.log("disconnect"); } /** @internal */ @@ -87,7 +92,7 @@ export class Member { } /** @internal */ - sendSignallingMessage = async (message: SignallingMessage, log: ILogItem) => { + sendSignallingMessage = async (message: SignallingMessage, log: ILogItem): Promise => { const groupMessage = message as SignallingMessage; groupMessage.content.conf_id = this.options.confId; const encryptedMessages = await this.options.encryptDeviceMessage(this.member.userId, groupMessage, log); @@ -102,12 +107,13 @@ export class Member { } /** @internal */ - handleDeviceMessage(message: SignallingMessage, deviceId: string, log: ILogItem) { + handleDeviceMessage(message: SignallingMessage, deviceId: string, syncLog: ILogItem) { + syncLog.refDetached(this.logItem); if (message.type === EventType.Invite && !this.peerCall) { this.peerCall = this._createPeerCall(message.content.call_id); } if (this.peerCall) { - this.peerCall.handleIncomingSignallingMessage(message, deviceId, log); + this.peerCall.handleIncomingSignallingMessage(message, deviceId); } else { // TODO: need to buffer events until invite comes? } @@ -117,6 +123,6 @@ export class Member { return new PeerCall(callId, Object.assign({}, this.options, { emitUpdate: this.emitUpdate, sendSignallingMessage: this.sendSignallingMessage - })); + }), this.logItem); } } From 4bf171def9ce111421f8ab5900a9aa0b1d58c6a3 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 25 Mar 2022 14:43:22 +0100 Subject: [PATCH 029/323] small fixes --- scripts/logviewer/main.js | 2 +- src/domain/session/SessionViewModel.js | 2 +- src/observable/value/PickMapObservableValue.ts | 2 +- src/platform/web/ui/session/room/common.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/logviewer/main.js b/scripts/logviewer/main.js index b6883667..dce82fb8 100644 --- a/scripts/logviewer/main.js +++ b/scripts/logviewer/main.js @@ -395,4 +395,4 @@ document.getElementById("showAll").addEventListener("click", () => { for (const node of document.querySelectorAll(".hidden")) { node.classList.remove("hidden"); } -}); \ No newline at end of file +}); diff --git a/src/domain/session/SessionViewModel.js b/src/domain/session/SessionViewModel.js index 0c9dddaa..4e5930b1 100644 --- a/src/domain/session/SessionViewModel.js +++ b/src/domain/session/SessionViewModel.js @@ -191,7 +191,7 @@ export class SessionViewModel extends ViewModel { async _createArchivedRoomViewModel(roomId) { const room = await this._client.session.loadArchivedRoom(roomId); if (room) { - const roomVM = new RoomViewModel(this.childOptions({room})); + const roomVM = new RoomViewModel(this.childOptions({room, session: this._client.session})); roomVM.load(); return roomVM; } diff --git a/src/observable/value/PickMapObservableValue.ts b/src/observable/value/PickMapObservableValue.ts index b493d841..0b30fcfd 100644 --- a/src/observable/value/PickMapObservableValue.ts +++ b/src/observable/value/PickMapObservableValue.ts @@ -62,7 +62,7 @@ export class PickMapObservableValue extends BaseObservableValue Date: Fri, 25 Mar 2022 16:53:12 +0100 Subject: [PATCH 030/323] log state changes in PeerCall --- src/matrix/calls/PeerCall.ts | 61 ++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index c4355e63..975b1dff 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -113,10 +113,10 @@ export class PeerCall implements IDisposable { }, onDataChannelChanged(dataChannel: DataChannel | undefined) {}, onNegotiationNeeded() { - const log = outer.logItem.child("onNegotiationNeeded"); - const promiseCreator = async () => { - await outer.handleNegotiation(log); - log.finish(); + const promiseCreator = () => { + return outer.logItem.wrap("onNegotiationNeeded", log => { + return outer.handleNegotiation(log); + }); }; outer.responsePromiseChain = outer.responsePromiseChain?.then(promiseCreator) ?? promiseCreator(); }, @@ -139,7 +139,7 @@ export class PeerCall implements IDisposable { } this.localMedia = localMedia; this.direction = CallDirection.Outbound; - this.setState(CallState.CreateOffer); + this.setState(CallState.CreateOffer, log); for (const t of this.localMedia.tracks) { this.peerConnection.addTrack(t); } @@ -155,7 +155,7 @@ export class PeerCall implements IDisposable { return; } this.localMedia = localMedia; - this.setState(CallState.CreateAnswer); + this.setState(CallState.CreateAnswer, log); for (const t of this.localMedia.tracks) { this.peerConnection.addTrack(t); } @@ -173,7 +173,7 @@ export class PeerCall implements IDisposable { try { await this.peerConnection.setLocalDescription(myAnswer); - this.setState(CallState.Connecting); + this.setState(CallState.Connecting, log); } catch (err) { await log.wrap(`Error setting local description!`, log => { log.catch(err); @@ -305,7 +305,7 @@ export class PeerCall implements IDisposable { }; if (this._state === CallState.CreateOffer) { await this.sendSignallingMessage({type: EventType.Invite, content}, log); - this.setState(CallState.InviteSent); + this.setState(CallState.InviteSent, log); } else if (this._state === CallState.Connected || this._state === CallState.Connecting) { // send Negotiate message //await this.sendSignallingMessage({type: EventType.Invite, content}); @@ -317,14 +317,16 @@ export class PeerCall implements IDisposable { this.sendCandidateQueue(log); - if (this._state === CallState.InviteSent) { - try { await this.delay(CALL_TIMEOUT_MS); } - catch (err) { return; } - // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between + await log.wrap("invite timeout", async log => { if (this._state === CallState.InviteSent) { - this._hangup(CallErrorCode.InviteTimeout, log); + try { await this.delay(CALL_TIMEOUT_MS); } + catch (err) { return; } + // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between + if (this._state === CallState.InviteSent) { + this._hangup(CallErrorCode.InviteTimeout, log); + } } - } + }); }; private async handleInviteGlare(content: MCallInvite, partyId: PartyId, log: ILogItem): Promise { @@ -398,7 +400,7 @@ export class PeerCall implements IDisposable { return; } - this.setState(CallState.Ringing); + this.setState(CallState.Ringing, log); try { await this.delay(content.lifetime ?? CALL_TIMEOUT_MS); } catch (err) { return; } @@ -406,7 +408,7 @@ export class PeerCall implements IDisposable { if (this._state === CallState.Ringing) { log.log(`Invite has expired. Hanging up.`); this.hangupParty = CallParty.Remote; // effectively - this.setState(CallState.Ended); + this.setState(CallState.Ended, log); this.stopAllMedia(); if (this.peerConnection.signalingState != 'closed') { this.peerConnection.close(); @@ -428,7 +430,7 @@ export class PeerCall implements IDisposable { this.opponentPartyId = partyId; await this.addBufferedIceCandidates(log); - this.setState(CallState.Connecting); + this.setState(CallState.Connecting, log); const sdpStreamMetadata = content[SDPStreamMetadataKey]; if (sdpStreamMetadata) { @@ -449,7 +451,6 @@ export class PeerCall implements IDisposable { } private handleIceGatheringState(state: RTCIceGatheringState, log: ILogItem) { - log.set("state", state); if (state === 'complete' && !this.sentEndOfCandidates) { // If we didn't get an empty-string candidate to signal the end of candidates, // create one ourselves now gathering has finished. @@ -712,15 +713,12 @@ export class PeerCall implements IDisposable { if (this._state === CallState.Ended) { return; // because ICE can still complete as we're ending the call } - log.log( - "ICE connection state changed to: " + state, - ); // ideally we'd consider the call to be connected when we get media but // chrome doesn't implement any of the 'onstarted' events yet if (state == 'connected') { this.iceDisconnectedTimeout?.abort(); this.iceDisconnectedTimeout = undefined; - this.setState(CallState.Connected); + this.setState(CallState.Connected, log); } else if (state == 'failed') { this.iceDisconnectedTimeout?.abort(); this.iceDisconnectedTimeout = undefined; @@ -733,15 +731,18 @@ export class PeerCall implements IDisposable { } }; - private setState(state: CallState): void { - const oldState = this._state; - this._state = state; - let deferred = this.statePromiseMap.get(state); - if (deferred) { - deferred.resolve(); - this.statePromiseMap.delete(state); + private setState(state: CallState, log: ILogItem): void { + if (state !== this._state) { + log.log({l: "change state", status: state, oldState: this._state}); + const oldState = this._state; + this._state = state; + let deferred = this.statePromiseMap.get(state); + if (deferred) { + deferred.resolve(); + this.statePromiseMap.delete(state); + } + this.options.emitUpdate(this, undefined); } - this.options.emitUpdate(this, undefined); } private waitForState(states: CallState[]): Promise { From ba45178e04d625e3b909aad7b50d747f84fe08bb Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 29 Mar 2022 12:00:54 +0200 Subject: [PATCH 031/323] implement terminate and hangup (currently unused) --- src/matrix/calls/PeerCall.ts | 45 ++++++++++++++++++++++---------- src/matrix/calls/group/Member.ts | 11 ++++---- 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 975b1dff..3e23b3a6 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -166,7 +166,7 @@ export class PeerCall implements IDisposable { } catch (err) { await log.wrap(`Failed to create answer`, log => { log.catch(err); - this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true, log); + this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, log); }); return; } @@ -177,7 +177,7 @@ export class PeerCall implements IDisposable { } catch (err) { await log.wrap(`Error setting local description!`, log => { log.catch(err); - this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true, log); + this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, log); }); return; } @@ -212,6 +212,7 @@ export class PeerCall implements IDisposable { }); } + /** group calls would handle reject at the group call level, not at the peer call level */ async reject() { } @@ -223,10 +224,11 @@ export class PeerCall implements IDisposable { } private async _hangup(errorCode: CallErrorCode, log: ILogItem): Promise { - if (this._state !== CallState.Ended) { - this._state = CallState.Ended; - await this.sendHangupWithCallId(this.callId, errorCode, log); + if (this._state === CallState.Ended) { + return; } + this.terminate(CallParty.Local, errorCode, log); + await this.sendHangupWithCallId(this.callId, errorCode, log); } handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId): Promise { @@ -257,6 +259,7 @@ export class PeerCall implements IDisposable { call_id: callId, version: 1, }; + // TODO: Don't send UserHangup reason to older clients if (reason) { content["reason"] = reason; } @@ -274,7 +277,7 @@ export class PeerCall implements IDisposable { await this.peerConnection.setLocalDescription(); } catch (err) { log.log(`Error setting local description!`).catch(err); - this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true, log); + this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, log); return; } @@ -385,7 +388,7 @@ export class PeerCall implements IDisposable { } catch (e) { await log.wrap(`Call failed to set remote description`, async log => { log.catch(e); - return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false, log); + return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); }); return; } @@ -395,7 +398,7 @@ export class PeerCall implements IDisposable { // (81 at time of writing), this is no longer a problem, so let's do it the correct way. if (this.peerConnection.remoteTracks.length === 0) { await log.wrap(`Call no remote stream or no tracks after setting remote description!`, async log => { - return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false, log); + return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); }); return; } @@ -444,7 +447,7 @@ export class PeerCall implements IDisposable { } catch (e) { await log.wrap(`Failed to set remote description`, log => { log.catch(e); - this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false, log); + this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); }); return; } @@ -591,7 +594,7 @@ export class PeerCall implements IDisposable { try { await this.sendSignallingMessage({type: EventType.Answer, content: answerContent}, log); } catch (error) { - this.terminate(CallParty.Local, CallErrorCode.SendAnswer, false, log); + this.terminate(CallParty.Local, CallErrorCode.SendAnswer, log); throw error; } @@ -656,7 +659,7 @@ export class PeerCall implements IDisposable { // put all the candidates we failed to send back in the queue // TODO: terminate doesn't seem to vibe with the comment above? - this.terminate(CallParty.Local, CallErrorCode.SignallingFailed, false, log); + this.terminate(CallParty.Local, CallErrorCode.SignallingFailed, log); } }); } @@ -761,8 +764,19 @@ export class PeerCall implements IDisposable { })); } - private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean, log: ILogItem): Promise { + private terminate(hangupParty: CallParty, hangupReason: CallErrorCode, log: ILogItem): void { + if (this._state === CallState.Ended) { + return; + } + this.hangupParty = hangupParty; + // this.hangupReason = hangupReason; + this.setState(CallState.Ended, log); + this.stopAllMedia(); + + if (this.peerConnection && this.peerConnection.signalingState !== 'closed') { + this.peerConnection.close(); + } } private stopAllMedia(): void { @@ -791,8 +805,11 @@ export class PeerCall implements IDisposable { this.peerConnection.dispose(); } - public close(): void { - this.peerConnection.close(); + public close(reason: CallErrorCode | undefined, log: ILogItem): void { + if (reason === undefined) { + reason = CallErrorCode.UserHangup; + } + this.terminate(CallParty.Local, reason, log); } } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index a2dc43de..89e0db4c 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -71,11 +71,12 @@ export class Member { /** @internal */ disconnect() { - this.peerCall?.close(); - this.peerCall?.dispose(); - this.peerCall = undefined; - this.localMedia = undefined; - this.logItem.log("disconnect"); + this.logItem.wrap("disconnect", log => { + this.peerCall?.close(undefined, log); + this.peerCall?.dispose(); + this.peerCall = undefined; + this.localMedia = undefined; + }); } /** @internal */ From c54ffd4fc3ef67c1fe14d7ac0b72583248933022 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 29 Mar 2022 17:13:33 +0200 Subject: [PATCH 032/323] support multiple devices in call per user --- src/matrix/calls/CallHandler.ts | 6 +- src/matrix/calls/LocalMedia.ts | 5 ++ src/matrix/calls/TODO.md | 1 + src/matrix/calls/callEventTypes.ts | 13 ++++ src/matrix/calls/group/GroupCall.ts | 92 +++++++++++++++++++++-------- src/matrix/calls/group/Member.ts | 38 ++++++++---- src/observable/map/ObservableMap.ts | 6 +- 7 files changed, 117 insertions(+), 44 deletions(-) diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index ef617cd2..678f4f44 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -55,7 +55,6 @@ export class CallHandler { async createCall(roomId: string, localMedia: LocalMedia, name: string): Promise { const logItem = this.options.logger.child({l: "call", incoming: false}); const call = new GroupCall(undefined, undefined, roomId, this.groupCallOptions, logItem); - console.log("created call with id", call.id); this._calls.set(call.id, call); try { await call.create(localMedia, name); @@ -67,9 +66,7 @@ export class CallHandler { } throw err; } - console.log("joining call I just created"); await call.join(localMedia); - console.log("joined!"); return call; } @@ -79,7 +76,6 @@ export class CallHandler { /** @internal */ handleRoomState(room: Room, events: StateEvent[], log: ILogItem) { - console.log("handling room state"); // first update call events for (const event of events) { if (event.type === EventType.GroupCall) { @@ -135,7 +131,7 @@ export class CallHandler { const callId = call["m.call_id"]; const groupCall = this._calls.get(callId); // TODO: also check the member when receiving the m.call event - groupCall?.addMember(userId, call, log); + groupCall?.updateMember(userId, call, log); }; const newCallIdsMemberOf = new Set(calls.map(call => call["m.call_id"])); let previousCallIdsMemberOf = this.memberToCallIds.get(userId); diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index b148dd92..b64bdee5 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -61,6 +61,11 @@ export class LocalMedia { return metadata; } + clone() { + // TODO: implement + return this; + } + dispose() { this.cameraTrack?.stop(); this.microphoneTrack?.stop(); diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index a07da60e..4faf4f4e 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -108,6 +108,7 @@ Expose call objects expose volume events from audiotrack to group call Write view model write view + - handle glare edge-cases (not yet sent): https://spec.matrix.org/latest/client-server-api/#glare ## Calls questions - how do we handle glare between group calls (e.g. different state events with different call ids?) diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index 0e9eb8f8..4416087b 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -22,6 +22,19 @@ export enum EventType { // TODO: Change to "sdp_stream_metadata" when MSC3077 is merged export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata"; +export interface CallDeviceMembership { + device_id: string +} + +export interface CallMembership { + ["m.call_id"]: string, + ["m.devices"]: CallDeviceMembership[] +} + +export interface CallMemberContent { + ["m.calls"]: CallMembership[]; +} + export interface SessionDescription { sdp?: string; type: RTCSdpType diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 86c93ba3..3ef6a5ff 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -24,7 +24,7 @@ import {EventEmitter} from "../../../utils/EventEmitter"; import type {Options as MemberOptions} from "./Member"; import type {BaseObservableMap} from "../../../observable/map/BaseObservableMap"; import type {Track} from "../../../platform/types/MediaDevices"; -import type {SignallingMessage, MGroupCallBase} from "../callEventTypes"; +import type {SignallingMessage, MGroupCallBase, CallMembership} from "../callEventTypes"; import type {Room} from "../../room/Room"; import type {StateEvent} from "../../storage/types"; import type {Platform} from "../../../platform/web/Platform"; @@ -43,11 +43,22 @@ export enum GroupCallState { Joined = "joined", } +function getMemberKey(userId: string, deviceId: string) { + return JSON.stringify(userId)+`,`+JSON.stringify(deviceId); +} + +function memberKeyIsForUser(key: string, userId: string) { + return key.startsWith(JSON.stringify(userId)+`,`); +} + +function getDeviceFromMemberKey(key: string): string { + return JSON.parse(`[${key}]`)[1]; +} + export type Options = Omit & { emitUpdate: (call: GroupCall, params?: any) => void; encryptDeviceMessage: (roomId: string, userId: string, message: SignallingMessage, log: ILogItem) => Promise, storage: Storage, - ownDeviceId: string }; export class GroupCall extends EventEmitter<{change: never}> { @@ -70,7 +81,7 @@ export class GroupCall extends EventEmitter<{change: never}> { this._state = id ? GroupCallState.Created : GroupCallState.Fledgling; this._memberOptions = Object.assign({}, options, { confId: this.id, - emitUpdate: member => this._members.update(member.member.userId, member), + emitUpdate: member => this._members.update(getMemberKey(member.userId, member.deviceId), member), encryptDeviceMessage: (userId: string, message: SignallingMessage, log) => { return this.options.encryptDeviceMessage(this.roomId, userId, message, log); } @@ -173,26 +184,42 @@ export class GroupCall extends EventEmitter<{change: never}> { } /** @internal */ - addMember(userId: string, memberCallInfo, syncLog: ILogItem) { - this.logItem.wrap({l: "addMember", id: userId}, log => { + updateMember(userId: string, callMembership: CallMembership, syncLog: ILogItem) { + this.logItem.wrap({l: "updateMember", id: userId}, log => { syncLog.refDetached(log); - - if (userId === this.options.ownUserId) { - if (this._state === GroupCallState.Joining) { - this._state = GroupCallState.Joined; - this.emitChange(); + const devices = callMembership["m.devices"]; + const previousDeviceIds = this.getDeviceIdsForUserId(userId); + for (const device of devices) { + const deviceId = device.device_id; + const memberKey = getMemberKey(userId, deviceId); + if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) { + if (this._state === GroupCallState.Joining) { + this._state = GroupCallState.Joined; + this.emitChange(); + } + return; + } + let member = this._members.get(memberKey); + if (member) { + member.updateCallInfo(device); + } else { + const logItem = this.logItem.child("member"); + member = new Member( + RoomMember.fromUserId(this.roomId, userId, "join"), + device, this._memberOptions, logItem + ); + this._members.add(memberKey, member); + if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { + member.connect(this._localMedia!.clone()); + } } - return; } - let member = this._members.get(userId); - if (member) { - member.updateCallInfo(memberCallInfo); - } else { - const logItem = this.logItem.child("member"); - member = new Member(RoomMember.fromUserId(this.roomId, userId, "join"), memberCallInfo, this._memberOptions, logItem); - this._members.add(userId, member); - if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { - member.connect(this._localMedia!); + + const newDeviceIds = new Set(devices.map(call => call.device_id)); + // remove user as member of any calls not present anymore + for (const previousDeviceId of previousDeviceIds) { + if (!newDeviceIds.has(previousDeviceId)) { + this.removeMemberDevice(userId, previousDeviceId, syncLog); } } }); @@ -200,9 +227,24 @@ export class GroupCall extends EventEmitter<{change: never}> { /** @internal */ removeMember(userId: string, syncLog: ILogItem) { - this.logItem.wrap({l: "removeMember", id: userId}, log => { + const deviceIds = this.getDeviceIdsForUserId(userId); + for (const deviceId of deviceIds) { + this.removeMemberDevice(userId, deviceId, syncLog); + } + } + + private getDeviceIdsForUserId(userId: string): string[] { + return Array.from(this._members.keys()) + .filter(key => memberKeyIsForUser(key, userId)) + .map(key => getDeviceFromMemberKey(key)); + } + + /** @internal */ + private removeMemberDevice(userId: string, deviceId: string, syncLog: ILogItem) { + const memberKey = getMemberKey(userId, deviceId); + this.logItem.wrap({l: "removeMemberDevice", id: memberKey}, log => { syncLog.refDetached(log); - if (userId === this.options.ownUserId) { + if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) { if (this._state === GroupCallState.Joined) { this._localMedia?.dispose(); this._localMedia = undefined; @@ -212,9 +254,9 @@ export class GroupCall extends EventEmitter<{change: never}> { this._state = GroupCallState.Created; } } else { - const member = this._members.get(userId); + const member = this._members.get(memberKey); if (member) { - this._members.remove(userId); + this._members.remove(memberKey); member.disconnect(); } } @@ -225,7 +267,7 @@ export class GroupCall extends EventEmitter<{change: never}> { /** @internal */ handleDeviceMessage(message: SignallingMessage, userId: string, deviceId: string, syncLog: ILogItem) { // TODO: return if we are not membering to the call - let member = this._members.get(userId); + let member = this._members.get(getMemberKey(userId, deviceId)); if (member) { member.handleDeviceMessage(message, deviceId, syncLog); } else { diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 89e0db4c..d31837f4 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -23,7 +23,7 @@ import type {Options as PeerCallOptions} from "../PeerCall"; import type {LocalMedia} from "../LocalMedia"; import type {HomeServerApi} from "../../net/HomeServerApi"; import type {Track} from "../../../platform/types/MediaDevices"; -import type {MCallBase, MGroupCallBase, SignallingMessage} from "../callEventTypes"; +import type {MCallBase, MGroupCallBase, SignallingMessage, CallDeviceMembership} from "../callEventTypes"; import type {GroupCall} from "./GroupCall"; import type {RoomMember} from "../../room/members/RoomMember"; import type {EncryptedMessage} from "../../e2ee/olm/Encryption"; @@ -32,6 +32,7 @@ import type {ILogItem} from "../../../logging/types"; export type Options = Omit & { confId: string, ownUserId: string, + ownDeviceId: string, hsApi: HomeServerApi, encryptDeviceMessage: (userId: string, message: SignallingMessage, log: ILogItem) => Promise, emitUpdate: (participant: Member, params?: any) => void, @@ -43,7 +44,7 @@ export class Member { constructor( public readonly member: RoomMember, - private memberCallInfo: Record, + private callDeviceMembership: CallDeviceMembership, private readonly options: Options, private readonly logItem: ILogItem, ) { @@ -58,15 +59,30 @@ export class Member { return this.peerCall?.state === CallState.Connected; } + get userId(): string { + return this.member.userId; + } + + get deviceId(): string { + return this.callDeviceMembership.device_id; + } + /** @internal */ connect(localMedia: LocalMedia) { - this.logItem.log("connect"); - this.localMedia = localMedia; - // otherwise wait for it to connect - if (this.member.userId < this.options.ownUserId) { - this.peerCall = this._createPeerCall(makeId("c")); - this.peerCall.call(localMedia); - } + this.logItem.wrap("connect", () => { + this.localMedia = localMedia; + // otherwise wait for it to connect + let shouldInitiateCall; + if (this.member.userId === this.options.ownUserId) { + shouldInitiateCall = this.deviceId < this.options.ownDeviceId; + } else { + shouldInitiateCall = this.member.userId < this.options.ownUserId; + } + if (shouldInitiateCall) { + this.peerCall = this._createPeerCall(makeId("c")); + this.peerCall.call(localMedia); + } + }); } /** @internal */ @@ -80,8 +96,8 @@ export class Member { } /** @internal */ - updateCallInfo(memberCallInfo) { - // m.calls object from the m.call.member event + updateCallInfo(callDeviceMembership: CallDeviceMembership) { + this.callDeviceMembership = callDeviceMembership; } /** @internal */ diff --git a/src/observable/map/ObservableMap.ts b/src/observable/map/ObservableMap.ts index d604ab0a..79662e29 100644 --- a/src/observable/map/ObservableMap.ts +++ b/src/observable/map/ObservableMap.ts @@ -80,15 +80,15 @@ export class ObservableMap extends BaseObservableMap { return this._values.size; } - [Symbol.iterator](): Iterator<[K, V]> { + [Symbol.iterator](): IterableIterator<[K, V]> { return this._values.entries(); } - values(): Iterator { + values(): IterableIterator { return this._values.values(); } - keys(): Iterator { + keys(): IterableIterator { return this._values.keys(); } } From d7360e774103feae4225c7be1afb67760e7a1489 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 30 Mar 2022 15:18:46 +0200 Subject: [PATCH 033/323] fix multiple device support --- src/matrix/DeviceMessageHandler.js | 6 +- src/matrix/calls/CallHandler.ts | 4 +- src/matrix/calls/PeerCall.ts | 14 +-- src/matrix/calls/group/GroupCall.ts | 133 +++++++++++++++++---------- src/matrix/calls/group/Member.ts | 4 +- src/platform/types/MediaDevices.ts | 1 + src/platform/web/dom/MediaDevices.ts | 4 + 7 files changed, 103 insertions(+), 63 deletions(-) diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index 80fd1592..ee10716e 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -59,7 +59,11 @@ export class DeviceMessageHandler { })); // TODO: pass this in the prep and run it in afterSync or afterSyncComplete (as callHandler can send events as well)? for (const dr of callMessages) { - this._callHandler.handleDeviceMessage(dr.event, dr.device.userId, dr.device.deviceId, log); + if (dr.device) { + this._callHandler.handleDeviceMessage(dr.event, dr.device.userId, dr.device.deviceId, log); + } else { + console.error("could not deliver message because don't have device for sender key", dr.event); + } } // TODO: somehow include rooms that received a call to_device message in the sync state? // or have updates flow through event emitter? diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 678f4f44..89730391 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -131,7 +131,7 @@ export class CallHandler { const callId = call["m.call_id"]; const groupCall = this._calls.get(callId); // TODO: also check the member when receiving the m.call event - groupCall?.updateMember(userId, call, log); + groupCall?.updateMembership(userId, call, log); }; const newCallIdsMemberOf = new Set(calls.map(call => call["m.call_id"])); let previousCallIdsMemberOf = this.memberToCallIds.get(userId); @@ -140,7 +140,7 @@ export class CallHandler { for (const previousCallId of previousCallIdsMemberOf) { if (!newCallIdsMemberOf.has(previousCallId)) { const groupCall = this._calls.get(previousCallId); - groupCall?.removeMember(userId, log); + groupCall?.removeMembership(userId, log); } } } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 3e23b3a6..b85cee61 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -412,7 +412,8 @@ export class PeerCall implements IDisposable { log.log(`Invite has expired. Hanging up.`); this.hangupParty = CallParty.Remote; // effectively this.setState(CallState.Ended, log); - this.stopAllMedia(); + //this.localMedia?.dispose(); + //this.localMedia = undefined; if (this.peerConnection.signalingState != 'closed') { this.peerConnection.close(); } @@ -772,21 +773,14 @@ export class PeerCall implements IDisposable { this.hangupParty = hangupParty; // this.hangupReason = hangupReason; this.setState(CallState.Ended, log); - this.stopAllMedia(); + //this.localMedia?.dispose(); + //this.localMedia = undefined; if (this.peerConnection && this.peerConnection.signalingState !== 'closed') { this.peerConnection.close(); } } - private stopAllMedia(): void { - if (this.localMedia) { - for (const track of this.localMedia.tracks) { - track.stop(); - } - } - } - private async delay(timeoutMs: number): Promise { // Allow a short time for initial candidates to be gathered const timeout = this.disposables.track(this.options.createTimeout(timeoutMs)); diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 3ef6a5ff..98e3381d 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -95,10 +95,18 @@ export class GroupCall extends EventEmitter<{change: never}> { return this.callContent?.["m.terminated"] === true; } + get isRinging(): boolean { + return this._state === GroupCallState.Created && this.intent === "m.ring" && !this.isMember(this.options.ownUserId); + } + get name(): string { return this.callContent?.["m.name"]; } + get intent(): string { + return this.callContent?.["m.intent"]; + } + join(localMedia: LocalMedia): Promise { return this.logItem.wrap("join", async log => { if (this._state !== GroupCallState.Created) { @@ -134,6 +142,8 @@ export class GroupCall extends EventEmitter<{change: never}> { if (this._members.size === 0) { await this.terminate(); } + } else { + log.set("already_left", true); } }); } @@ -184,7 +194,7 @@ export class GroupCall extends EventEmitter<{change: never}> { } /** @internal */ - updateMember(userId: string, callMembership: CallMembership, syncLog: ILogItem) { + updateMembership(userId: string, callMembership: CallMembership, syncLog: ILogItem) { this.logItem.wrap({l: "updateMember", id: userId}, log => { syncLog.refDetached(log); const devices = callMembership["m.devices"]; @@ -192,45 +202,62 @@ export class GroupCall extends EventEmitter<{change: never}> { for (const device of devices) { const deviceId = device.device_id; const memberKey = getMemberKey(userId, deviceId); - if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) { - if (this._state === GroupCallState.Joining) { - this._state = GroupCallState.Joined; - this.emitChange(); + log.wrap({l: "update device member", id: memberKey}, log => { + if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) { + if (this._state === GroupCallState.Joining) { + log.set("update_own", true); + this._state = GroupCallState.Joined; + this.emitChange(); + } + } else { + let member = this._members.get(memberKey); + if (member) { + log.set("update", true); + member!.updateCallInfo(device); + } else { + const logItem = this.logItem.child({l: "member", id: memberKey}); + log.set("add", true); + log.refDetached(logItem); + member = new Member( + RoomMember.fromUserId(this.roomId, userId, "join"), + device, this._memberOptions, logItem + ); + this._members.add(memberKey, member); + if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { + member.connect(this._localMedia!.clone()); + } + } } - return; - } - let member = this._members.get(memberKey); - if (member) { - member.updateCallInfo(device); - } else { - const logItem = this.logItem.child("member"); - member = new Member( - RoomMember.fromUserId(this.roomId, userId, "join"), - device, this._memberOptions, logItem - ); - this._members.add(memberKey, member); - if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { - member.connect(this._localMedia!.clone()); - } - } + }); } const newDeviceIds = new Set(devices.map(call => call.device_id)); // remove user as member of any calls not present anymore for (const previousDeviceId of previousDeviceIds) { if (!newDeviceIds.has(previousDeviceId)) { - this.removeMemberDevice(userId, previousDeviceId, syncLog); + log.wrap({l: "remove device member", id: getMemberKey(userId, previousDeviceId)}, log => { + this.removeMemberDevice(userId, previousDeviceId, log); + }); } } + if (userId === this.options.ownUserId && !newDeviceIds.has(this.options.ownDeviceId)) { + this.removeOwnDevice(log); + } }); } /** @internal */ - removeMember(userId: string, syncLog: ILogItem) { + removeMembership(userId: string, syncLog: ILogItem) { const deviceIds = this.getDeviceIdsForUserId(userId); - for (const deviceId of deviceIds) { - this.removeMemberDevice(userId, deviceId, syncLog); - } + this.logItem.wrap("removeMember", log => { + syncLog.refDetached(log); + for (const deviceId of deviceIds) { + this.removeMemberDevice(userId, deviceId, log); + } + if (userId === this.options.ownUserId) { + this.removeOwnDevice(log); + } + }); } private getDeviceIdsForUserId(userId: string): string[] { @@ -239,26 +266,32 @@ export class GroupCall extends EventEmitter<{change: never}> { .map(key => getDeviceFromMemberKey(key)); } + private isMember(userId: string): boolean { + return Array.from(this._members.keys()).some(key => memberKeyIsForUser(key, userId)); + } + + private removeOwnDevice(log: ILogItem) { + if (this._state === GroupCallState.Joined) { + log.set("leave_own", true); + this._localMedia?.dispose(); + this._localMedia = undefined; + for (const [,member] of this._members) { + member.disconnect(); + } + this._state = GroupCallState.Created; + this.emitChange(); + } + } + /** @internal */ - private removeMemberDevice(userId: string, deviceId: string, syncLog: ILogItem) { + private removeMemberDevice(userId: string, deviceId: string, log: ILogItem) { const memberKey = getMemberKey(userId, deviceId); - this.logItem.wrap({l: "removeMemberDevice", id: memberKey}, log => { - syncLog.refDetached(log); - if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) { - if (this._state === GroupCallState.Joined) { - this._localMedia?.dispose(); - this._localMedia = undefined; - for (const [,member] of this._members) { - member.disconnect(); - } - this._state = GroupCallState.Created; - } - } else { - const member = this._members.get(memberKey); - if (member) { - this._members.remove(memberKey); - member.disconnect(); - } + log.wrap({l: "removeMemberDevice", id: memberKey}, log => { + const member = this._members.get(memberKey); + if (member) { + log.set("leave", true); + this._members.remove(memberKey); + member.disconnect(); } this.emitChange(); }); @@ -315,9 +348,15 @@ export class GroupCall extends EventEmitter<{change: never}> { const stateEvent = await txn.roomState.get(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId); if (stateEvent) { const content = stateEvent.event.content; - const callsInfo = content["m.calls"]; - content["m.calls"] = callsInfo?.filter(c => c["m.call_id"] !== this.id); - return content; + const callInfo = content["m.calls"]?.find(c => c["m.call_id"] === this.id); + if (callInfo) { + const devicesInfo = callInfo["m.devices"]; + const deviceIndex = devicesInfo.findIndex(d => d["device_id"] === this.options.ownDeviceId); + if (deviceIndex !== -1) { + devicesInfo.splice(deviceIndex, 1); + return content; + } + } } } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index d31837f4..cd26fba2 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -47,9 +47,7 @@ export class Member { private callDeviceMembership: CallDeviceMembership, private readonly options: Options, private readonly logItem: ILogItem, - ) { - logItem.set("id", member.userId); - } + ) {} get remoteTracks(): Track[] { return this.peerCall?.remoteTracks ?? []; diff --git a/src/platform/types/MediaDevices.ts b/src/platform/types/MediaDevices.ts index ed9015bf..db267871 100644 --- a/src/platform/types/MediaDevices.ts +++ b/src/platform/types/MediaDevices.ts @@ -37,6 +37,7 @@ export interface Track { get muted(): boolean; setMuted(muted: boolean): void; stop(): void; + clone(): Track; } export interface AudioTrack extends Track { diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index 22f3d634..49fd7fa5 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -111,6 +111,10 @@ export class TrackWrapper implements Track { stop() { this.track.stop(); } + + clone() { + return this.track.clone(); + } } export class AudioTrackWrapper extends TrackWrapper { From 42b470b06b649d0223d19b97e27650405effaf08 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 30 Mar 2022 15:19:07 +0200 Subject: [PATCH 034/323] helper to print open items with console logger --- src/logging/ConsoleLogger.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/logging/ConsoleLogger.ts b/src/logging/ConsoleLogger.ts index f48c72b2..1643e48a 100644 --- a/src/logging/ConsoleLogger.ts +++ b/src/logging/ConsoleLogger.ts @@ -25,6 +25,12 @@ export class ConsoleLogger extends BaseLogger { async export(): Promise { return undefined; } + + printOpenItems(): void { + for (const item of this._openItems) { + this._persistItem(item); + } + } } const excludedKeysFromTable = ["l", "id"]; From 1ad5db73a933314c18db0ab519f69a1123cfb323 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 6 Apr 2022 18:11:06 +0200 Subject: [PATCH 035/323] some logviewer improvement to help debug call signalling --- scripts/logviewer/main.js | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/scripts/logviewer/main.js b/scripts/logviewer/main.js index dce82fb8..4c6b3792 100644 --- a/scripts/logviewer/main.js +++ b/scripts/logviewer/main.js @@ -22,6 +22,7 @@ const main = document.querySelector("main"); let selectedItemNode; let rootItem; let itemByRef; +let itemsRefFrom; const logLevels = [undefined, "All", "Debug", "Detail", "Info", "Warn", "Error", "Fatal", "Off"]; @@ -49,6 +50,7 @@ window.addEventListener("hashchange", () => { const id = window.location.hash.substr(1); const itemNode = document.getElementById(id); if (itemNode && itemNode.closest("main")) { + ensureParentsExpanded(itemNode); selectNode(itemNode); itemNode.scrollIntoView({behavior: "smooth", block: "nearest"}); } @@ -70,6 +72,14 @@ function selectNode(itemNode) { showItemDetails(item, parent, selectedItemNode); } +function ensureParentsExpanded(itemNode) { + let li = itemNode.parentElement.parentElement; + while (li.tagName === "LI") { + li.classList.add("expanded"); + li = li.parentElement.parentElement; + } +} + function stringifyItemValue(value) { if (typeof value === "object" && value !== null) { return JSON.stringify(value, undefined, 2); @@ -102,6 +112,11 @@ function showItemDetails(item, parent, itemNode) { } else { valueNode = `unknown ref ${value}`; } + } else if (key === "refId") { + const refSources = itemsRefFrom.get(value) ?? []; + valueNode = t.div([t.p([`${value}`, t.br(),`Found these references:`]),t.ul(refSources.map(item => { + return t.li(t.a({href: `#${item.id}`}, itemCaption(item))); + }))]); } else { valueNode = stringifyItemValue(value); } @@ -153,7 +168,8 @@ async function loadFile() { logs.items.sort((a, b) => itemStart(a) - itemStart(b)); rootItem = {c: logs.items}; itemByRef = new Map(); - preprocessRecursively(rootItem, null, itemByRef, []); + itemsRefFrom = new Map(); + preprocessRecursively(rootItem, null, itemByRef, itemsRefFrom, []); const fragment = logs.items.reduce((fragment, item, i, items) => { const prevItem = i === 0 ? null : items[i - 1]; @@ -167,18 +183,26 @@ async function loadFile() { } // TODO: make this use processRecursively -function preprocessRecursively(item, parentElement, refsMap, path) { +function preprocessRecursively(item, parentElement, refsMap, refsFromMap, path) { item.s = (parentElement?.s || 0) + item.s; if (itemRefSource(item)) { refsMap.set(itemRefSource(item), item); } + if (itemRef(item)) { + let refs = refsFromMap.get(itemRef(item)); + if (!refs) { + refs = []; + refsFromMap.set(itemRef(item), refs); + } + refs.push(item); + } if (itemChildren(item)) { for (let i = 0; i < itemChildren(item).length; i += 1) { // do it in advance for a child as we don't want to do it for the rootItem const child = itemChildren(item)[i]; const childPath = path.concat(i); child.id = childPath.join("/"); - preprocessRecursively(child, item, refsMap, childPath); + preprocessRecursively(child, item, refsMap, refsFromMap, childPath); } } } From 2852834ce34cdbda85a1a13d22ad4ff03daa81a1 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 7 Apr 2022 10:32:23 +0200 Subject: [PATCH 036/323] persist calls so they can be quickly loaded after a restart also use event prefixes compatible with Element Call/MSC --- src/domain/session/SessionViewModel.js | 2 + .../session/room/timeline/tilesCreator.js | 2 +- src/matrix/Session.js | 2 +- src/matrix/Sync.js | 1 + src/matrix/calls/CallHandler.ts | 99 ++++++++++++++++--- src/matrix/calls/callEventTypes.ts | 12 ++- src/matrix/calls/group/GroupCall.ts | 41 ++++---- src/matrix/room/Room.js | 9 +- src/matrix/storage/common.ts | 1 + src/matrix/storage/idb/Transaction.ts | 5 + src/matrix/storage/idb/schema.ts | 8 +- src/matrix/storage/idb/stores/CallStore.ts | 83 ++++++++++++++++ .../storage/idb/stores/RoomStateStore.ts | 12 ++- 13 files changed, 229 insertions(+), 48 deletions(-) create mode 100644 src/matrix/storage/idb/stores/CallStore.ts diff --git a/src/domain/session/SessionViewModel.js b/src/domain/session/SessionViewModel.js index 4e5930b1..1242edf4 100644 --- a/src/domain/session/SessionViewModel.js +++ b/src/domain/session/SessionViewModel.js @@ -99,6 +99,8 @@ export class SessionViewModel extends ViewModel { start() { this._sessionStatusViewModel.start(); + //this._client.session.callHandler.loadCalls("m.prompt"); + this._client.session.callHandler.loadCalls("m.ring"); } get activeMiddleViewModel() { diff --git a/src/domain/session/room/timeline/tilesCreator.js b/src/domain/session/room/timeline/tilesCreator.js index f35f6536..ad562df0 100644 --- a/src/domain/session/room/timeline/tilesCreator.js +++ b/src/domain/session/room/timeline/tilesCreator.js @@ -72,7 +72,7 @@ export function tilesCreator(baseOptions) { return new EncryptedEventTile(options); case "m.room.encryption": return new EncryptionEnabledTile(options); - case "m.call": + case "org.matrix.msc3401.call": // if prevContent is present, it's an update to a call event, which we don't render // as the original event is updated through the call object which receive state event updates return entry.stateKey && !entry.prevContent ? new CallTile(options) : null; diff --git a/src/matrix/Session.js b/src/matrix/Session.js index a1e1cc28..aa7dbf7b 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -76,7 +76,7 @@ export class Session { this._roomsBeingCreated = new ObservableMap(); this._user = new User(sessionInfo.userId); this._callHandler = new CallHandler({ - createTimeout: this._platform.clock.createTimeout, + clock: this._platform.clock, hsApi: this._hsApi, encryptDeviceMessage: async (roomId, userId, message, log) => { if (!this._deviceTracker || !this._olmEncryption) { diff --git a/src/matrix/Sync.js b/src/matrix/Sync.js index b4ea702f..4f907563 100644 --- a/src/matrix/Sync.js +++ b/src/matrix/Sync.js @@ -344,6 +344,7 @@ export class Sync { // to decrypt and store new room keys storeNames.olmSessions, storeNames.inboundGroupSessions, + storeNames.calls, ]); } diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 89730391..7cd60208 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -18,8 +18,9 @@ import {ObservableMap} from "../../observable/map/ObservableMap"; import {WebRTC, PeerConnection, PeerConnectionHandler} from "../../platform/types/WebRTC"; import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; import {handlesEventType} from "./PeerCall"; -import {EventType} from "./callEventTypes"; +import {EventType, CallIntent} from "./callEventTypes"; import {GroupCall} from "./group/GroupCall"; +import {makeId} from "../common"; import type {LocalMedia} from "./LocalMedia"; import type {Room} from "../room/Room"; @@ -30,13 +31,17 @@ import type {Platform} from "../../platform/web/Platform"; import type {BaseObservableMap} from "../../observable/map/BaseObservableMap"; import type {SignallingMessage, MGroupCallBase} from "./callEventTypes"; import type {Options as GroupCallOptions} from "./group/GroupCall"; +import type {Transaction} from "../storage/idb/Transaction"; +import type {CallEntry} from "../storage/idb/stores/CallStore"; +import type {Clock} from "../../platform/web/dom/Clock"; const GROUP_CALL_TYPE = "m.call"; const GROUP_CALL_MEMBER_TYPE = "m.call.member"; const CALL_TERMINATED = "m.terminated"; -export type Options = Omit & { - logger: ILogger +export type Options = Omit & { + logger: ILogger, + clock: Clock }; export class CallHandler { @@ -48,25 +53,86 @@ export class CallHandler { constructor(private readonly options: Options) { this.groupCallOptions = Object.assign({}, this.options, { - emitUpdate: (groupCall, params) => this._calls.update(groupCall.id, params) + emitUpdate: (groupCall, params) => this._calls.update(groupCall.id, params), + createTimeout: this.options.clock.createTimeout, + }); + } + + async loadCalls(intent: CallIntent = CallIntent.Ring) { + const txn = await this._getLoadTxn(); + const callEntries = await txn.calls.getByIntent(intent); + this._loadCallEntries(callEntries, txn); + } + + async loadCallsForRoom(intent: CallIntent, roomId: string) { + const txn = await this._getLoadTxn(); + const callEntries = await txn.calls.getByIntentAndRoom(intent, roomId); + this._loadCallEntries(callEntries, txn); + } + + private async _getLoadTxn(): Promise { + const names = this.options.storage.storeNames; + const txn = await this.options.storage.readTxn([ + names.calls, + names.roomState + ]); + return txn; + } + + private async _loadCallEntries(callEntries: CallEntry[], txn: Transaction): Promise { + return this.options.logger.run("loading calls", async log => { + log.set("entries", callEntries.length); + await Promise.all(callEntries.map(async callEntry => { + if (this._calls.get(callEntry.callId)) { + return; + } + const event = await txn.roomState.get(callEntry.roomId, EventType.GroupCall, callEntry.callId); + if (event) { + const logItem = this.options.logger.child({l: "call", loaded: true}); + const call = new GroupCall(event.event.state_key, false, event.event.content, event.roomId, this.groupCallOptions, logItem); + this._calls.set(call.id, call); + } + })); + const roomIds = Array.from(new Set(callEntries.map(e => e.roomId))); + await Promise.all(roomIds.map(async roomId => { + const ownCallsMemberEvent = await txn.roomState.get(roomId, EventType.GroupCallMember, this.options.ownUserId); + if (ownCallsMemberEvent) { + this.handleCallMemberEvent(ownCallsMemberEvent.event, log); + } + // TODO: we should be loading the other members as well at some point + })); + log.set("newSize", this._calls.size); }); } async createCall(roomId: string, localMedia: LocalMedia, name: string): Promise { const logItem = this.options.logger.child({l: "call", incoming: false}); - const call = new GroupCall(undefined, undefined, roomId, this.groupCallOptions, logItem); + const call = new GroupCall(makeId("conf-"), true, { + "m.name": name, + "m.intent": CallIntent.Ring + }, roomId, this.groupCallOptions, logItem); this._calls.set(call.id, call); + try { - await call.create(localMedia, name); + await call.create(localMedia); + await call.join(localMedia); + // store call info so it will ring again when reopening the app + const txn = await this.options.storage.readWriteTxn([this.options.storage.storeNames.calls]); + txn.calls.add({ + intent: call.intent, + callId: call.id, + timestamp: this.options.clock.now(), + roomId: roomId + }); + await txn.complete(); } catch (err) { - if (err.name === "ConnectionError") { + //if (err.name === "ConnectionError") { // if we're offline, give up and remove the call again call.dispose(); this._calls.remove(call.id); - } + //} throw err; } - await call.join(localMedia); return call; } @@ -75,11 +141,11 @@ export class CallHandler { // TODO: check and poll turn server credentials here /** @internal */ - handleRoomState(room: Room, events: StateEvent[], log: ILogItem) { + handleRoomState(room: Room, events: StateEvent[], txn: Transaction, log: ILogItem) { // first update call events for (const event of events) { if (event.type === EventType.GroupCall) { - this.handleCallEvent(event, room.id, log); + this.handleCallEvent(event, room.id, txn, log); } } // then update members @@ -108,7 +174,7 @@ export class CallHandler { call?.handleDeviceMessage(message, userId, deviceId, log); } - private handleCallEvent(event: StateEvent, roomId: string, log: ILogItem) { + private handleCallEvent(event: StateEvent, roomId: string, txn: Transaction, log: ILogItem) { const callId = event.state_key; let call = this._calls.get(callId); if (call) { @@ -116,11 +182,18 @@ export class CallHandler { if (call.isTerminated) { call.dispose(); this._calls.remove(call.id); + txn.calls.remove(call.intent, roomId, call.id); } } else { const logItem = this.options.logger.child({l: "call", incoming: true}); - call = new GroupCall(event.state_key, event.content, roomId, this.groupCallOptions, logItem); + call = new GroupCall(event.state_key, false, event.content, roomId, this.groupCallOptions, logItem); this._calls.set(call.id, call); + txn.calls.add({ + intent: call.intent, + callId: call.id, + timestamp: event.origin_server_ts, + roomId: roomId + }); } } diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index 4416087b..a32a7739 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -1,10 +1,10 @@ // allow non-camelcase as these are events type that go onto the wire /* eslint-disable camelcase */ - +import type {StateEvent} from "../storage/types"; export enum EventType { - GroupCall = "m.call", - GroupCallMember = "m.call.member", + GroupCall = "org.matrix.msc3401.call", + GroupCallMember = "org.matrix.msc3401.call.member", Invite = "m.call.invite", Candidates = "m.call.candidates", Answer = "m.call.answer", @@ -211,3 +211,9 @@ export type SignallingMessage = {type: EventType.SDPStreamMetadataChanged | EventType.SDPStreamMetadataChangedPrefix, content: MCallSDPStreamMetadataChanged} | {type: EventType.Candidates, content: MCallCandidates} | {type: EventType.Hangup | EventType.Reject, content: MCallHangupReject}; + +export enum CallIntent { + Ring = "m.ring", + Prompt = "m.prompt", + Room = "m.room", +}; diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 98e3381d..3a28c9a7 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -18,8 +18,8 @@ import {ObservableMap} from "../../../observable/map/ObservableMap"; import {Member} from "./Member"; import {LocalMedia} from "../LocalMedia"; import {RoomMember} from "../../room/members/RoomMember"; -import {makeId} from "../../common"; import {EventEmitter} from "../../../utils/EventEmitter"; +import {EventType, CallIntent} from "../callEventTypes"; import type {Options as MemberOptions} from "./Member"; import type {BaseObservableMap} from "../../../observable/map/BaseObservableMap"; @@ -32,9 +32,6 @@ import type {EncryptedMessage} from "../../e2ee/olm/Encryption"; import type {ILogItem} from "../../../logging/types"; import type {Storage} from "../../storage/idb/Storage"; -const CALL_TYPE = "m.call"; -const CALL_MEMBER_TYPE = "m.call.member"; - export enum GroupCallState { Fledgling = "fledgling", Creating = "creating", @@ -62,23 +59,22 @@ export type Options = Omit { - public readonly id: string; private readonly _members: ObservableMap = new ObservableMap(); private _localMedia?: LocalMedia = undefined; private _memberOptions: MemberOptions; private _state: GroupCallState; constructor( - id: string | undefined, - private callContent: Record | undefined, + public readonly id: string, + newCall: boolean, + private callContent: Record, public readonly roomId: string, private readonly options: Options, private readonly logItem: ILogItem, ) { super(); - this.id = id ?? makeId("conf-"); logItem.set("id", this.id); - this._state = id ? GroupCallState.Created : GroupCallState.Fledgling; + this._state = newCall ? GroupCallState.Fledgling : GroupCallState.Created; this._memberOptions = Object.assign({}, options, { confId: this.id, emitUpdate: member => this._members.update(getMemberKey(member.userId, member.deviceId), member), @@ -103,7 +99,7 @@ export class GroupCall extends EventEmitter<{change: never}> { return this.callContent?.["m.name"]; } - get intent(): string { + get intent(): CallIntent { return this.callContent?.["m.intent"]; } @@ -117,7 +113,7 @@ export class GroupCall extends EventEmitter<{change: never}> { this.emitChange(); const memberContent = await this._createJoinPayload(); // send m.call.member state event - const request = this.options.hsApi.sendState(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId, memberContent, {log}); + const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCallMember, this.options.ownUserId, memberContent, {log}); await request.response(); this.emitChange(); // send invite to all members that are < my userId @@ -136,10 +132,10 @@ export class GroupCall extends EventEmitter<{change: never}> { const memberContent = await this._leaveCallMemberContent(); // send m.call.member state event if (memberContent) { - const request = this.options.hsApi.sendState(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId, memberContent, {log}); + const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCallMember, this.options.ownUserId, memberContent, {log}); await request.response(); // our own user isn't included in members, so not in the count - if (this._members.size === 0) { + if (this.intent === CallIntent.Ring && this._members.size === 0) { await this.terminate(); } } else { @@ -153,7 +149,7 @@ export class GroupCall extends EventEmitter<{change: never}> { if (this._state === GroupCallState.Fledgling) { return; } - const request = this.options.hsApi.sendState(this.roomId, CALL_TYPE, this.id, Object.assign({}, this.callContent, { + const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCall, this.id, Object.assign({}, this.callContent, { "m.terminated": true }), {log}); await request.response(); @@ -161,19 +157,17 @@ export class GroupCall extends EventEmitter<{change: never}> { } /** @internal */ - create(localMedia: LocalMedia, name: string): Promise { + create(localMedia: LocalMedia): Promise { return this.logItem.wrap("create", async log => { if (this._state !== GroupCallState.Fledgling) { return; } this._state = GroupCallState.Creating; this.emitChange(); - this.callContent = { + this.callContent = Object.assign({ "m.type": localMedia.cameraTrack ? "m.video" : "m.voice", - "m.name": name, - "m.intent": "m.ring" - }; - const request = this.options.hsApi.sendState(this.roomId, CALL_TYPE, this.id, this.callContent, {log}); + }, this.callContent); + const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCall, this.id, this.callContent!, {log}); await request.response(); this._state = GroupCallState.Created; this.emitChange(); @@ -318,7 +312,7 @@ export class GroupCall extends EventEmitter<{change: never}> { private async _createJoinPayload() { const {storage} = this.options; const txn = await storage.readTxn([storage.storeNames.roomState]); - const stateEvent = await txn.roomState.get(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId); + const stateEvent = await txn.roomState.get(this.roomId, EventType.GroupCallMember, this.options.ownUserId); const stateContent = stateEvent?.event?.content ?? { ["m.calls"]: [] }; @@ -335,7 +329,8 @@ export class GroupCall extends EventEmitter<{change: never}> { let deviceInfo = devicesInfo.find(d => d["device_id"] === this.options.ownDeviceId); if (!deviceInfo) { deviceInfo = { - ["device_id"]: this.options.ownDeviceId + ["device_id"]: this.options.ownDeviceId, + feeds: [{purpose: "m.usermedia"}] }; devicesInfo.push(deviceInfo); } @@ -345,7 +340,7 @@ export class GroupCall extends EventEmitter<{change: never}> { private async _leaveCallMemberContent(): Promise | undefined> { const {storage} = this.options; const txn = await storage.readTxn([storage.storeNames.roomState]); - const stateEvent = await txn.roomState.get(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId); + const stateEvent = await txn.roomState.get(this.roomId, EventType.GroupCallMember, this.options.ownUserId); if (stateEvent) { const content = stateEvent.event.content; const callInfo = content["m.calls"]?.find(c => c["m.call_id"] === this.id); diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js index ff1926b4..34f35af8 100644 --- a/src/matrix/room/Room.js +++ b/src/matrix/room/Room.js @@ -93,8 +93,6 @@ export class Room extends BaseRoom { } } - this._updateCallHandler(roomResponse, log); - return { roomEncryption, summaryChanges, @@ -181,6 +179,7 @@ export class Room extends BaseRoom { removedPendingEvents = await this._sendQueue.removeRemoteEchos(roomResponse.timeline.events, txn, log); } const powerLevelsEvent = this._getPowerLevelsEvent(roomResponse); + this._updateCallHandler(roomResponse, txn, log); return { summaryChanges, roomEncryption, @@ -448,17 +447,17 @@ export class Room extends BaseRoom { return this._sendQueue.pendingEvents; } - _updateCallHandler(roomResponse, log) { + _updateCallHandler(roomResponse, txn, log) { if (this._callHandler) { const stateEvents = roomResponse.state?.events; if (stateEvents?.length) { - this._callHandler.handleRoomState(this, stateEvents, log); + this._callHandler.handleRoomState(this, stateEvents, txn, log); } let timelineEvents = roomResponse.timeline?.events; if (timelineEvents) { const timelineStateEvents = timelineEvents.filter(e => typeof e.state_key === "string"); if (timelineEvents.length !== 0) { - this._callHandler.handleRoomState(this, timelineStateEvents, log); + this._callHandler.handleRoomState(this, timelineStateEvents, txn, log); } } } diff --git a/src/matrix/storage/common.ts b/src/matrix/storage/common.ts index 23bb0d31..e1e34917 100644 --- a/src/matrix/storage/common.ts +++ b/src/matrix/storage/common.ts @@ -33,6 +33,7 @@ export enum StoreNames { groupSessionDecryptions = "groupSessionDecryptions", operations = "operations", accountData = "accountData", + calls = "calls" } export const STORE_NAMES: Readonly = Object.values(StoreNames); diff --git a/src/matrix/storage/idb/Transaction.ts b/src/matrix/storage/idb/Transaction.ts index 80894105..7a8de420 100644 --- a/src/matrix/storage/idb/Transaction.ts +++ b/src/matrix/storage/idb/Transaction.ts @@ -36,6 +36,7 @@ import {OutboundGroupSessionStore} from "./stores/OutboundGroupSessionStore"; import {GroupSessionDecryptionStore} from "./stores/GroupSessionDecryptionStore"; import {OperationStore} from "./stores/OperationStore"; import {AccountDataStore} from "./stores/AccountDataStore"; +import {CallStore} from "./stores/CallStore"; import type {ILogger, ILogItem} from "../../../logging/types"; export type IDBKey = IDBValidKey | IDBKeyRange; @@ -167,6 +168,10 @@ export class Transaction { get accountData(): AccountDataStore { return this._store(StoreNames.accountData, idbStore => new AccountDataStore(idbStore)); } + + get calls(): CallStore { + return this._store(StoreNames.calls, idbStore => new CallStore(idbStore)); + } async complete(log?: ILogItem): Promise { try { diff --git a/src/matrix/storage/idb/schema.ts b/src/matrix/storage/idb/schema.ts index 7819130e..4461ae15 100644 --- a/src/matrix/storage/idb/schema.ts +++ b/src/matrix/storage/idb/schema.ts @@ -34,7 +34,8 @@ export const schema: MigrationFunc[] = [ backupAndRestoreE2EEAccountToLocalStorage, clearAllStores, addInboundSessionBackupIndex, - migrateBackupStatus + migrateBackupStatus, + createCallStore ]; // TODO: how to deal with git merge conflicts of this array? @@ -309,3 +310,8 @@ async function migrateBackupStatus(db: IDBDatabase, txn: IDBTransaction, localSt log.set("countWithoutSession", countWithoutSession); log.set("countWithSession", countWithSession); } + +//v17 create calls store +function createCallStore(db: IDBDatabase) : void { + db.createObjectStore("calls", {keyPath: "key"}); +} diff --git a/src/matrix/storage/idb/stores/CallStore.ts b/src/matrix/storage/idb/stores/CallStore.ts new file mode 100644 index 00000000..566bcc40 --- /dev/null +++ b/src/matrix/storage/idb/stores/CallStore.ts @@ -0,0 +1,83 @@ +/* +Copyright 2020 Bruno Windels +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {Store} from "../Store"; +import {StateEvent} from "../../types"; +import {MIN_UNICODE, MAX_UNICODE} from "./common"; + +function encodeKey(intent: string, roomId: string, callId: string) { + return `${intent}|${roomId}|${callId}`; +} + +function decodeStorageEntry(storageEntry: CallStorageEntry): CallEntry { + const [intent, roomId, callId] = storageEntry.key.split("|"); + return {intent, roomId, callId, timestamp: storageEntry.timestamp}; +} + +export interface CallEntry { + intent: string; + roomId: string; + callId: string; + timestamp: number; +} + +type CallStorageEntry = { + key: string; + timestamp: number; +} + +export class CallStore { + private _callStore: Store; + + constructor(idbStore: Store) { + this._callStore = idbStore; + } + + async getByIntent(intent: string): Promise { + const range = this._callStore.IDBKeyRange.bound( + encodeKey(intent, MIN_UNICODE, MIN_UNICODE), + encodeKey(intent, MAX_UNICODE, MAX_UNICODE), + true, + true + ); + const storageEntries = await this._callStore.selectAll(range); + return storageEntries.map(e => decodeStorageEntry(e)); + } + + async getByIntentAndRoom(intent: string, roomId: string): Promise { + const range = this._callStore.IDBKeyRange.bound( + encodeKey(intent, roomId, MIN_UNICODE), + encodeKey(intent, roomId, MAX_UNICODE), + true, + true + ); + const storageEntries = await this._callStore.selectAll(range); + return storageEntries.map(e => decodeStorageEntry(e)); + } + + add(entry: CallEntry) { + const storageEntry: CallStorageEntry = { + key: encodeKey(entry.intent, entry.roomId, entry.callId), + timestamp: entry.timestamp + }; + this._callStore.add(storageEntry); + } + + remove(intent: string, roomId: string, callId: string): void { + this._callStore.delete(encodeKey(intent, roomId, callId)); + } +} diff --git a/src/matrix/storage/idb/stores/RoomStateStore.ts b/src/matrix/storage/idb/stores/RoomStateStore.ts index d2bf811d..99315e9e 100644 --- a/src/matrix/storage/idb/stores/RoomStateStore.ts +++ b/src/matrix/storage/idb/stores/RoomStateStore.ts @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {MAX_UNICODE} from "./common"; +import {MIN_UNICODE, MAX_UNICODE} from "./common"; import {Store} from "../Store"; import {StateEvent} from "../../types"; @@ -41,6 +41,16 @@ export class RoomStateStore { return this._roomStateStore.get(key); } + getAllForType(roomId: string, type: string): Promise { + const range = this._roomStateStore.IDBKeyRange.bound( + encodeKey(roomId, type, MIN_UNICODE), + encodeKey(roomId, type, MAX_UNICODE), + true, + true + ); + return this._roomStateStore.selectAll(range); + } + set(roomId: string, event: StateEvent): void { const key = encodeKey(roomId, event.type, event.state_key); const entry = {roomId, event, key}; From ad1cceac86be15dfe8b442ecc1fd6c3670d5cf44 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 7 Apr 2022 10:33:12 +0200 Subject: [PATCH 037/323] fix error thrown during request when response code is not used --- src/matrix/net/RequestScheduler.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/matrix/net/RequestScheduler.ts b/src/matrix/net/RequestScheduler.ts index dc5c501b..c6e546a1 100644 --- a/src/matrix/net/RequestScheduler.ts +++ b/src/matrix/net/RequestScheduler.ts @@ -27,8 +27,8 @@ class Request implements IHomeServerRequest { public readonly args: any[]; private responseResolve: (result: any) => void; public responseReject: (error: Error) => void; - private responseCodeResolve: (result: any) => void; - private responseCodeReject: (result: any) => void; + private responseCodeResolve?: (result: any) => void; + private responseCodeReject?: (result: any) => void; private _requestResult?: IHomeServerRequest; private readonly _responsePromise: Promise; private _responseCodePromise: Promise; @@ -73,7 +73,7 @@ class Request implements IHomeServerRequest { const response = await this._requestResult?.response(); this.responseResolve(response); const responseCode = await this._requestResult?.responseCode(); - this.responseCodeResolve(responseCode); + this.responseCodeResolve?.(responseCode); } get requestResult() { From fe6e7b09b568fa608a397a963b24b2e73b19d044 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 7 Apr 2022 16:50:16 +0200 Subject: [PATCH 038/323] don't encrypt to_device messages for now --- src/matrix/DeviceMessageHandler.js | 38 ++++++++++++++++++++---------- src/matrix/calls/group/Member.ts | 18 +++++++++++--- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index ee10716e..c33236aa 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -38,6 +38,7 @@ export class DeviceMessageHandler { async prepareSync(toDeviceEvents, lock, txn, log) { log.set("messageTypes", countBy(toDeviceEvents, e => e.type)); + this._handleUnencryptedCallEvents(toDeviceEvents, log); const encryptedEvents = toDeviceEvents.filter(e => e.type === "m.room.encrypted"); if (!this._olmDecryption) { log.log("can't decrypt, encryption not enabled", log.level.Warn); @@ -52,19 +53,21 @@ export class DeviceMessageHandler { log.child("decrypt_error").catch(err); } const newRoomKeys = this._megolmDecryption.roomKeysFromDeviceMessages(olmDecryptChanges.results, log); - const callMessages = olmDecryptChanges.results.filter(dr => this._callHandler.handlesDeviceMessageEventType(dr.event?.type)); - // load devices by sender key - await Promise.all(callMessages.map(async dr => { - dr.setDevice(await this._getDevice(dr.senderCurve25519Key, txn)); - })); - // TODO: pass this in the prep and run it in afterSync or afterSyncComplete (as callHandler can send events as well)? - for (const dr of callMessages) { - if (dr.device) { - this._callHandler.handleDeviceMessage(dr.event, dr.device.userId, dr.device.deviceId, log); - } else { - console.error("could not deliver message because don't have device for sender key", dr.event); - } - } + + // const callMessages = olmDecryptChanges.results.filter(dr => this._callHandler.handlesDeviceMessageEventType(dr.event?.type)); + // // load devices by sender key + // await Promise.all(callMessages.map(async dr => { + // dr.setDevice(await this._getDevice(dr.senderCurve25519Key, txn)); + // })); + // // TODO: pass this in the prep and run it in afterSync or afterSyncComplete (as callHandler can send events as well)? + // for (const dr of callMessages) { + // if (dr.device) { + // this._callHandler.handleDeviceMessage(dr.event, dr.device.userId, dr.device.deviceId, log); + // } else { + // console.error("could not deliver message because don't have device for sender key", dr.event); + // } + // } + // TODO: somehow include rooms that received a call to_device message in the sync state? // or have updates flow through event emitter? // well, we don't really need to update the room other then when a call starts or stops @@ -73,6 +76,15 @@ export class DeviceMessageHandler { } } + _handleUnencryptedCallEvents(toDeviceEvents, log) { + const callMessages = toDeviceEvents.filter(e => this._callHandler.handlesDeviceMessageEventType(e.type)); + for (const event of callMessages) { + const userId = event.sender; + const deviceId = event.content.device_id; + this._callHandler.handleDeviceMessage(event, userId, deviceId, log); + } + } + /** check that prep is not undefined before calling this */ async writeSync(prep, txn) { // write olm changes diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index cd26fba2..fe2de0af 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -110,10 +110,22 @@ export class Member { sendSignallingMessage = async (message: SignallingMessage, log: ILogItem): Promise => { const groupMessage = message as SignallingMessage; groupMessage.content.conf_id = this.options.confId; - const encryptedMessages = await this.options.encryptDeviceMessage(this.member.userId, groupMessage, log); - const payload = formatToDeviceMessagesPayload(encryptedMessages); + groupMessage.content.device_id = this.options.ownDeviceId; + groupMessage.content.party_id = this.options.ownDeviceId; + groupMessage.content.sender_session_id = this.options.sessionId; + groupMessage.content.dest_session_id = this.destSessionId!; + // const encryptedMessages = await this.options.encryptDeviceMessage(this.member.userId, groupMessage, log); + // const payload = formatToDeviceMessagesPayload(encryptedMessages); + const payload = { + messages: { + [this.member.userId]: { + ['*']: groupMessage.content + } + } + }; const request = this.options.hsApi.sendToDevice( - "m.room.encrypted", + message.type, + //"m.room.encrypted", payload, makeTxnId(), {log} From 64728003870e948ccf6e2d8ea11a70a1acc0d7d0 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 7 Apr 2022 16:53:37 +0200 Subject: [PATCH 039/323] impl session id so EC does not ignore our messages --- src/matrix/calls/CallHandler.ts | 2 ++ src/matrix/calls/PeerCall.ts | 5 +++++ src/matrix/calls/callEventTypes.ts | 5 +++++ src/matrix/calls/group/Member.ts | 8 ++++++++ 4 files changed, 20 insertions(+) diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 7cd60208..f60c1fe6 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -50,11 +50,13 @@ export class CallHandler { // map of userId to set of conf_id's they are in private memberToCallIds: Map> = new Map(); private groupCallOptions: GroupCallOptions; + private sessionId = makeId("s"); constructor(private readonly options: Options) { this.groupCallOptions = Object.assign({}, this.options, { emitUpdate: (groupCall, params) => this._calls.update(groupCall.id, params), createTimeout: this.options.clock.createTimeout, + sessionId: this.sessionId }); } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index b85cee61..1b869f47 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -61,6 +61,7 @@ export class PeerCall implements IDisposable { private _state = CallState.Fledgling; private direction: CallDirection; private localMedia?: LocalMedia; + private seq: number = 0; // A queue for candidates waiting to go out. // We try to amalgamate candidates into a single candidate message where // possible @@ -258,6 +259,7 @@ export class PeerCall implements IDisposable { const content = { call_id: callId, version: 1, + seq: this.seq++, }; // TODO: Don't send UserHangup reason to older clients if (reason) { @@ -304,6 +306,7 @@ export class PeerCall implements IDisposable { offer, [SDPStreamMetadataKey]: this.localMedia!.getSDPMetadata(), version: 1, + seq: this.seq++, lifetime: CALL_TIMEOUT_MS }; if (this._state === CallState.CreateOffer) { @@ -578,6 +581,7 @@ export class PeerCall implements IDisposable { const answerContent: MCallAnswer = { call_id: this.callId, version: 1, + seq: this.seq++, answer: { sdp: localDescription.sdp, type: localDescription.type, @@ -648,6 +652,7 @@ export class PeerCall implements IDisposable { content: { call_id: this.callId, version: 1, + seq: this.seq++, candidates }, }, log); diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index a32a7739..9189911f 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -69,10 +69,15 @@ export interface CallReplacesTarget { export type MCallBase = { call_id: string; version: string | number; + seq: number; } export type MGroupCallBase = MCallBase & { conf_id: string; + device_id: string; + sender_session_id: string; + dest_session_id: string; + party_id: string; // Should not need this? } export type MCallAnswer = Base & { diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index fe2de0af..7404d647 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -33,6 +33,7 @@ export type Options = Omit, log: ILogItem) => Promise, emitUpdate: (participant: Member, params?: any) => void, @@ -41,6 +42,7 @@ export type Options = Omit Date: Thu, 7 Apr 2022 16:53:57 +0200 Subject: [PATCH 040/323] fix who initiates call, needs to be lower, not higher --- src/matrix/calls/group/Member.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 7404d647..e3b1545b 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -73,10 +73,11 @@ export class Member { this.localMedia = localMedia; // otherwise wait for it to connect let shouldInitiateCall; + // the lexicographically lower side initiates the call if (this.member.userId === this.options.ownUserId) { - shouldInitiateCall = this.deviceId < this.options.ownDeviceId; + shouldInitiateCall = this.deviceId > this.options.ownDeviceId; } else { - shouldInitiateCall = this.member.userId < this.options.ownUserId; + shouldInitiateCall = this.member.userId > this.options.ownUserId; } if (shouldInitiateCall) { this.peerCall = this._createPeerCall(makeId("c")); From 1dc46127c3f45ba38369ac0ec40118a90ce4be0c Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 7 Apr 2022 16:54:24 +0200 Subject: [PATCH 041/323] no need to throw here --- src/matrix/calls/PeerCall.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 1b869f47..232aa612 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -250,7 +250,8 @@ export class PeerCall implements IDisposable { break; case EventType.Hangup: default: - throw new Error(`Unknown event type for call: ${message.type}`); + log.log(`Unknown event type for call: ${message.type}`); + break; } }); } From bade40acc61ada815ed2a5fd63e0cc9df2f97bae Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 7 Apr 2022 16:54:36 +0200 Subject: [PATCH 042/323] log track length --- src/matrix/calls/PeerCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 232aa612..3dbb74fc 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -108,7 +108,7 @@ export class PeerCall implements IDisposable { }); }, onRemoteTracksChanged(tracks: Track[]) { - outer.logItem.wrap("onRemoteTracksChanged", log => { + outer.logItem.wrap({l: "onRemoteTracksChanged", length: tracks.length}, log => { outer.options.emitUpdate(outer, undefined); }); }, From b133f58f7a3b7fa401a7294a0928a8800a75fa40 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 7 Apr 2022 16:54:47 +0200 Subject: [PATCH 043/323] don't throw here for now, although it is probably a sign of why the tracks disappear --- src/matrix/calls/PeerCall.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 3dbb74fc..7338cf2c 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -400,12 +400,12 @@ export class PeerCall implements IDisposable { // According to previous comments in this file, firefox at some point did not // add streams until media started arriving on them. Testing latest firefox // (81 at time of writing), this is no longer a problem, so let's do it the correct way. - if (this.peerConnection.remoteTracks.length === 0) { - await log.wrap(`Call no remote stream or no tracks after setting remote description!`, async log => { - return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); - }); - return; - } + // if (this.peerConnection.remoteTracks.length === 0) { + // await log.wrap(`Call no remote stream or no tracks after setting remote description!`, async log => { + // return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); + // }); + // return; + // } this.setState(CallState.Ringing, log); From a78ae52a54c52b3c73cb4016180b5544861bac51 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 7 Apr 2022 16:55:10 +0200 Subject: [PATCH 044/323] to test with EC, also load prompt calls at startup --- src/domain/session/SessionViewModel.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/domain/session/SessionViewModel.js b/src/domain/session/SessionViewModel.js index 1242edf4..715e51c5 100644 --- a/src/domain/session/SessionViewModel.js +++ b/src/domain/session/SessionViewModel.js @@ -99,8 +99,9 @@ export class SessionViewModel extends ViewModel { start() { this._sessionStatusViewModel.start(); - //this._client.session.callHandler.loadCalls("m.prompt"); this._client.session.callHandler.loadCalls("m.ring"); + // TODO: only do this when opening the room + this._client.session.callHandler.loadCalls("m.prompt"); } get activeMiddleViewModel() { From ad140d5af1f1809274990b9649cb5c87bef0ea5d Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 7 Apr 2022 16:55:26 +0200 Subject: [PATCH 045/323] only show video feed when connected --- src/domain/session/room/CallViewModel.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 30b18bc1..72353e4e 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -29,6 +29,7 @@ export class CallViewModel extends ViewModel { constructor(options: Options) { super(options); this.memberViewModels = this.getOption("call").members + .filterValues(member => member.isConnected) .mapValues(member => new CallMemberViewModel(this.childOptions({member}))) .sortValues((a, b) => a.compare(b)); } From 8a06663023019003b23c839bbb80ef5f64fe329a Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 7 Apr 2022 16:55:41 +0200 Subject: [PATCH 046/323] load all call members for now at startup later on we can be smarter and load then once you interact with the call --- src/matrix/calls/CallHandler.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index f60c1fe6..43fbcdb7 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -97,9 +97,13 @@ export class CallHandler { })); const roomIds = Array.from(new Set(callEntries.map(e => e.roomId))); await Promise.all(roomIds.map(async roomId => { - const ownCallsMemberEvent = await txn.roomState.get(roomId, EventType.GroupCallMember, this.options.ownUserId); - if (ownCallsMemberEvent) { - this.handleCallMemberEvent(ownCallsMemberEvent.event, log); + // const ownCallsMemberEvent = await txn.roomState.get(roomId, EventType.GroupCallMember, this.options.ownUserId); + // if (ownCallsMemberEvent) { + // this.handleCallMemberEvent(ownCallsMemberEvent.event, log); + // } + const callsMemberEvents = await txn.roomState.getAllForType(roomId, EventType.GroupCallMember); + for (const entry of callsMemberEvents) { + this.handleCallMemberEvent(entry.event, log); } // TODO: we should be loading the other members as well at some point })); From 156f5b78bf132e4bb3c2197e5e7ee034c239c646 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 11 Apr 2022 12:36:02 +0200 Subject: [PATCH 047/323] use session_id from member event to set dest_session_id so our invite event isn't ignored by EC --- src/matrix/calls/callEventTypes.ts | 3 ++- src/matrix/calls/group/Member.ts | 8 +------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index 9189911f..4726175c 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -23,7 +23,8 @@ export enum EventType { export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata"; export interface CallDeviceMembership { - device_id: string + device_id: string, + session_id: string } export interface CallMembership { diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index e3b1545b..2929e26c 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -42,7 +42,6 @@ export type Options = Omit Date: Mon, 11 Apr 2022 12:37:05 +0200 Subject: [PATCH 048/323] remove unused constants --- src/matrix/calls/CallHandler.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 43fbcdb7..d658121e 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -35,10 +35,6 @@ import type {Transaction} from "../storage/idb/Transaction"; import type {CallEntry} from "../storage/idb/stores/CallStore"; import type {Clock} from "../../platform/web/dom/Clock"; -const GROUP_CALL_TYPE = "m.call"; -const GROUP_CALL_MEMBER_TYPE = "m.call.member"; -const CALL_TERMINATED = "m.terminated"; - export type Options = Omit & { logger: ILogger, clock: Clock From 302d4bc02d426f2cebc8e3091e5ac7a1354163b9 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 11 Apr 2022 13:39:18 +0200 Subject: [PATCH 049/323] use session id from member event, and also send it for other party --- src/matrix/calls/group/GroupCall.ts | 1 + src/matrix/calls/group/Member.ts | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 3a28c9a7..886cb53a 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -330,6 +330,7 @@ export class GroupCall extends EventEmitter<{change: never}> { if (!deviceInfo) { deviceInfo = { ["device_id"]: this.options.ownDeviceId, + ["session_id"]: this.options.sessionId, feeds: [{purpose: "m.usermedia"}] }; devicesInfo.push(deviceInfo); diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 2929e26c..4086d060 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -138,6 +138,11 @@ export class Member { /** @internal */ handleDeviceMessage(message: SignallingMessage, deviceId: string, syncLog: ILogItem) { syncLog.refDetached(this.logItem); + const destSessionId = message.content.dest_session_id; + if (destSessionId !== this.options.sessionId) { + this.logItem.log({l: "ignoring to_device event with wrong session_id", destSessionId, type: message.type}); + return; + } if (message.type === EventType.Invite && !this.peerCall) { this.peerCall = this._createPeerCall(message.content.call_id); } From 81530608313fd39b5cd95f475322bdbc325e2fbd Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 11 Apr 2022 13:39:40 +0200 Subject: [PATCH 050/323] only send to target device, not all user devices --- src/matrix/calls/group/Member.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 4086d060..05d64b92 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -121,7 +121,7 @@ export class Member { const payload = { messages: { [this.member.userId]: { - ['*']: groupMessage.content + [this.callDeviceMembership.device_id]: groupMessage.content } } }; From 8e82aad86be656250b732458fa3a560659332fd9 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 11 Apr 2022 14:54:53 +0200 Subject: [PATCH 051/323] fix logic error that made tracks disappear on the second track event --- src/platform/web/dom/WebRTC.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 39a6f9d2..8c840ddc 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -188,10 +188,11 @@ class DOMPeerConnection implements PeerConnection { } private handleRemoteTrack(evt: RTCTrackEvent) { + // TODO: unit test this code somehow // the tracks on the new stream (with their stream) const updatedTracks = evt.streams.flatMap(stream => stream.getTracks().map(track => {return {stream, track};})); // of the tracks we already know about, filter the ones that aren't in the new stream - const withoutRemovedTracks = this._remoteTracks.filter(t => !updatedTracks.some(ut => t.track.id === ut.track.id)); + const withoutRemovedTracks = this._remoteTracks.filter(t => updatedTracks.some(ut => t.track.id === ut.track.id)); // of the new tracks, filter the ones that we didn't already knew about const addedTracks = updatedTracks.filter(ut => !this._remoteTracks.some(t => t.track.id === ut.track.id)); // wrap them From c02e1de00194aaff3f9efdd9e592ebc54dc2d681 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 11 Apr 2022 14:55:14 +0200 Subject: [PATCH 052/323] log when renegotiation would be triggered --- src/matrix/calls/PeerCall.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 7338cf2c..ed10eebb 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -314,6 +314,7 @@ export class PeerCall implements IDisposable { await this.sendSignallingMessage({type: EventType.Invite, content}, log); this.setState(CallState.InviteSent, log); } else if (this._state === CallState.Connected || this._state === CallState.Connecting) { + log.log("would send renegotiation now but not implemented"); // send Negotiate message //await this.sendSignallingMessage({type: EventType.Invite, content}); //this.setState(CallState.InviteSent); From b84c90891ca90038c05e0e65186cb2bd4369ed78 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 11 Apr 2022 15:53:34 +0200 Subject: [PATCH 053/323] add very early datachannel support --- src/matrix/calls/LocalMedia.ts | 9 +++++++-- src/matrix/calls/PeerCall.ts | 18 ++++++++++++++++-- src/matrix/calls/group/Member.ts | 4 ++++ src/platform/types/WebRTC.ts | 10 ++-------- src/platform/web/dom/WebRTC.ts | 6 +++--- 5 files changed, 32 insertions(+), 15 deletions(-) diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index b64bdee5..fd91c61c 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -22,7 +22,8 @@ export class LocalMedia { constructor( public readonly cameraTrack?: Track, public readonly screenShareTrack?: Track, - public readonly microphoneTrack?: AudioTrack + public readonly microphoneTrack?: AudioTrack, + public readonly dataChannelOptions?: RTCDataChannelInit, ) {} withTracks(tracks: Track[]) { @@ -32,7 +33,11 @@ export class LocalMedia { if (cameraTrack && microphoneTrack && cameraTrack.streamId !== microphoneTrack.streamId) { throw new Error("The camera and audio track should have the same stream id"); } - return new LocalMedia(cameraTrack, screenShareTrack, microphoneTrack as AudioTrack); + return new LocalMedia(cameraTrack, screenShareTrack, microphoneTrack as AudioTrack, this.dataChannelOptions); + } + + withDataChannel(options: RTCDataChannelInit): LocalMedia { + return new LocalMedia(this.cameraTrack, this.screenShareTrack, this.microphoneTrack as AudioTrack, options); } get tracks(): Track[] { diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index ed10eebb..7e4dbb4a 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -85,6 +85,8 @@ export class PeerCall implements IDisposable { private sentEndOfCandidates: boolean = false; private iceDisconnectedTimeout?: Timeout; + private _dataChannel?: any; + constructor( private callId: string, private readonly options: Options, @@ -112,7 +114,12 @@ export class PeerCall implements IDisposable { outer.options.emitUpdate(outer, undefined); }); }, - onDataChannelChanged(dataChannel: DataChannel | undefined) {}, + onRemoteDataChannel(dataChannel: any | undefined) { + outer.logItem.wrap("onRemoteDataChannel", log => { + outer._dataChannel = dataChannel; + outer.options.emitUpdate(outer, undefined); + }); + }, onNegotiationNeeded() { const promiseCreator = () => { return outer.logItem.wrap("onNegotiationNeeded", log => { @@ -127,6 +134,8 @@ export class PeerCall implements IDisposable { }); } + get dataChannel(): any | undefined { return this._dataChannel; } + get state(): CallState { return this._state; } get remoteTracks(): Track[] { @@ -144,6 +153,9 @@ export class PeerCall implements IDisposable { for (const t of this.localMedia.tracks) { this.peerConnection.addTrack(t); } + if (this.localMedia.dataChannelOptions) { + this._dataChannel = this.peerConnection.createDataChannel(this.localMedia.dataChannelOptions); + } // after adding the local tracks, and wait for handleNegotiation to be called, // or invite glare where we give up our invite and answer instead await this.waitForState([CallState.InviteSent, CallState.CreateAnswer]); @@ -160,7 +172,9 @@ export class PeerCall implements IDisposable { for (const t of this.localMedia.tracks) { this.peerConnection.addTrack(t); } - + if (this.localMedia.dataChannelOptions) { + this._dataChannel = this.peerConnection.createDataChannel(this.localMedia.dataChannelOptions); + } let myAnswer: RTCSessionDescriptionInit; try { myAnswer = await this.peerConnection.createAnswer(); diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 05d64b92..7d849929 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -66,6 +66,10 @@ export class Member { return this.callDeviceMembership.device_id; } + get dataChannel(): any | undefined { + return this.peerCall?.dataChannel; + } + /** @internal */ connect(localMedia: LocalMedia) { this.logItem.wrap("connect", () => { diff --git a/src/platform/types/WebRTC.ts b/src/platform/types/WebRTC.ts index df8133ee..130cb7cb 100644 --- a/src/platform/types/WebRTC.ts +++ b/src/platform/types/WebRTC.ts @@ -26,21 +26,15 @@ export interface PeerConnectionHandler { onLocalIceCandidate(candidate: RTCIceCandidate); onIceGatheringStateChange(state: RTCIceGatheringState); onRemoteTracksChanged(tracks: Track[]); - onDataChannelChanged(dataChannel: DataChannel | undefined); + onRemoteDataChannel(dataChannel: any | undefined); onNegotiationNeeded(); // request the type of incoming stream getPurposeForStreamId(streamId: string): SDPStreamMetadataPurpose; } -// does it make sense to wrap this? -export interface DataChannel { - close(); - send(); -} export interface PeerConnection { notifyStreamPurposeChanged(): void; get remoteTracks(): Track[]; - get dataChannel(): DataChannel | undefined; get iceGatheringState(): RTCIceGatheringState; get signalingState(): RTCSignalingState; get localDescription(): RTCSessionDescription | undefined; @@ -52,7 +46,7 @@ export interface PeerConnection { addTrack(track: Track): void; removeTrack(track: Track): boolean; replaceTrack(oldTrack: Track, newTrack: Track): Promise; - createDataChannel(): DataChannel; + createDataChannel(options: RTCDataChannelInit): any; dispose(): void; close(): void; } diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 8c840ddc..78977eb8 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -46,7 +46,6 @@ class DOMPeerConnection implements PeerConnection { } get remoteTracks(): Track[] { return this._remoteTracks; } - get dataChannel(): DataChannel | undefined { return undefined; } get iceGatheringState(): RTCIceGatheringState { return this.peerConnection.iceGatheringState; } get localDescription(): RTCSessionDescription | undefined { return this.peerConnection.localDescription ?? undefined; } get signalingState(): RTCSignalingState { return this.peerConnection.signalingState; } @@ -119,8 +118,8 @@ class DOMPeerConnection implements PeerConnection { } } - createDataChannel(): DataChannel { - return undefined as any;// new DataChannel(this.peerConnection.createDataChannel()); + createDataChannel(options: RTCDataChannelInit): any { + return this.peerConnection.createDataChannel("channel", options); } private registerHandler() { @@ -164,6 +163,7 @@ class DOMPeerConnection implements PeerConnection { this.handler.onNegotiationNeeded(); break; case "datachannel": + this.handler.onRemoteDataChannel((evt as RTCDataChannelEvent).channel); break; } } From 9be64730b6ecfd34bc62454e4c3e53f21d18e676 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 11 Apr 2022 15:54:06 +0200 Subject: [PATCH 054/323] don't automatically join a call we create --- src/domain/session/room/RoomViewModel.js | 3 ++- src/matrix/calls/CallHandler.ts | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 6dd8c8bd..87b4aa9d 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -359,7 +359,8 @@ export class RoomViewModel extends ViewModel { const mediaTracks = await this.platform.mediaDevices.getMediaTracks(false, true); const localMedia = new LocalMedia().withTracks(mediaTracks); // this will set the callViewModel above as a call will be added to callHandler.calls - await session.callHandler.createCall(this._room.id, localMedia, "A call " + Math.round(this.platform.random() * 100)); + const call = await session.callHandler.createCall(this._room.id, localMedia, "A call " + Math.round(this.platform.random() * 100)); + await call.join(localMedia); } catch (err) { console.error(err.stack); alert(err.message); diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index d658121e..31829396 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -117,7 +117,6 @@ export class CallHandler { try { await call.create(localMedia); - await call.join(localMedia); // store call info so it will ring again when reopening the app const txn = await this.options.storage.readWriteTxn([this.options.storage.storeNames.calls]); txn.calls.add({ From 387bad73b01686aed3280719db20a856f16bcb61 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 11 Apr 2022 15:54:20 +0200 Subject: [PATCH 055/323] remove debug alert --- src/domain/session/room/RoomViewModel.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 87b4aa9d..0683a0f2 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -363,7 +363,6 @@ export class RoomViewModel extends ViewModel { await call.join(localMedia); } catch (err) { console.error(err.stack); - alert(err.message); } } } From e0efbaeb4e79ae014ae16b24f89fe92dfcc61687 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 11 Apr 2022 15:54:31 +0200 Subject: [PATCH 056/323] show start time in console logger --- src/logging/ConsoleLogger.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/logging/ConsoleLogger.ts b/src/logging/ConsoleLogger.ts index 1643e48a..1d8232f9 100644 --- a/src/logging/ConsoleLogger.ts +++ b/src/logging/ConsoleLogger.ts @@ -45,7 +45,7 @@ function filterValues(values: LogItemValues): LogItemValues | null { } function printToConsole(item: LogItem): void { - const label = `${itemCaption(item)} (${item.duration}ms)`; + const label = `${itemCaption(item)} (@${item.start}ms, duration: ${item.duration}ms)`; const filteredValues = filterValues(item.values); const shouldGroup = item.children || filteredValues; if (shouldGroup) { From c99fc2ad706a0e481546078363430c51f0abca40 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 11 Apr 2022 15:54:41 +0200 Subject: [PATCH 057/323] use deviceId getter in Member --- src/matrix/calls/group/Member.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 7d849929..3eb5f17f 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -125,7 +125,7 @@ export class Member { const payload = { messages: { [this.member.userId]: { - [this.callDeviceMembership.device_id]: groupMessage.content + [this.deviceId]: groupMessage.content } } }; From 5cacdcfee0d3c9a6d3664cd5e86dead7d8cef96f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 11 Apr 2022 15:55:02 +0200 Subject: [PATCH 058/323] Add leave button to call view --- src/platform/web/ui/session/room/CallView.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/platform/web/ui/session/room/CallView.ts b/src/platform/web/ui/session/room/CallView.ts index ae7b4138..f8386116 100644 --- a/src/platform/web/ui/session/room/CallView.ts +++ b/src/platform/web/ui/session/room/CallView.ts @@ -34,7 +34,10 @@ export class CallView extends TemplateView { return t.div({class: "CallView"}, [ t.p(vm => `Call ${vm.name} (${vm.id})`), t.div({class: "CallView_me"}, bindVideoTracks(t, t.video({autoplay: true, width: 240}), vm => vm.localTracks)), - t.view(new ListView({list: vm.memberViewModels}, vm => new MemberView(vm))) + t.view(new ListView({list: vm.memberViewModels}, vm => new MemberView(vm))), + t.div({class: "buttons"}, [ + t.button({onClick: () => vm.leave()}, "Leave") + ]) ]); } } From 517e796e902a27d5b5ae81367480589f0c066ff5 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 11 Apr 2022 15:56:31 +0200 Subject: [PATCH 059/323] remove obsolete import --- src/matrix/calls/PeerCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 7e4dbb4a..87b96139 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -23,7 +23,7 @@ import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; import type {TimeoutCreator, Timeout} from "../../platform/types/types"; -import {WebRTC, PeerConnection, PeerConnectionHandler, DataChannel} from "../../platform/types/WebRTC"; +import {WebRTC, PeerConnection, PeerConnectionHandler} from "../../platform/types/WebRTC"; import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; import type {LocalMedia} from "./LocalMedia"; From a710f394eb2c923f3752f42ea613f652f44ab9eb Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 11 Apr 2022 15:57:23 +0200 Subject: [PATCH 060/323] fix lint warning --- src/domain/session/room/RoomViewModel.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 0683a0f2..c12ed3b8 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -28,7 +28,7 @@ import {LocalMedia} from "../../../matrix/calls/LocalMedia"; export class RoomViewModel extends ViewModel { constructor(options) { super(options); - const {room, session} = options; + const {room} = options; this._room = room; this._timelineVM = null; this._tilesCreator = null; From fd5b2aa7bb0ff31ff84e614ca07c823b9862dd59 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 11 Apr 2022 16:29:46 +0200 Subject: [PATCH 061/323] only create datachannel on side that sends invite --- src/matrix/calls/PeerCall.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 87b96139..fc37807d 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -172,9 +172,6 @@ export class PeerCall implements IDisposable { for (const t of this.localMedia.tracks) { this.peerConnection.addTrack(t); } - if (this.localMedia.dataChannelOptions) { - this._dataChannel = this.peerConnection.createDataChannel(this.localMedia.dataChannelOptions); - } let myAnswer: RTCSessionDescriptionInit; try { myAnswer = await this.peerConnection.createAnswer(); From 797cb23cc7adb06111d4a3e3e1c8bdd7628aaa09 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 12 Apr 2022 14:02:13 +0200 Subject: [PATCH 062/323] implement receiving hangup, and retry on connection failure --- src/matrix/calls/PeerCall.ts | 16 +++++++++++++++- src/matrix/calls/group/Member.ts | 24 ++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index fc37807d..fa1063ec 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -86,6 +86,7 @@ export class PeerCall implements IDisposable { private iceDisconnectedTimeout?: Timeout; private _dataChannel?: any; + private _hangupReason?: CallErrorCode; constructor( private callId: string, @@ -138,6 +139,8 @@ export class PeerCall implements IDisposable { get state(): CallState { return this._state; } + get hangupReason(): CallErrorCode | undefined { return this._hangupReason; } + get remoteTracks(): Track[] { return this.peerConnection.remoteTracks; } @@ -374,6 +377,17 @@ export class PeerCall implements IDisposable { } } + private handleHangupReceived(content: MCallHangupReject, log: ILogItem) { + // party ID must match (our chosen partner hanging up the call) or be undefined (we haven't chosen + // a partner yet but we're treating the hangup as a reject as per VoIP v0) + // if (this.state === CallState.Ringing) { + // default reason is user_hangup + this.terminate(CallParty.Remote, content.reason || CallErrorCode.UserHangup, log); + // } else { + // log.set("ignored", true); + // } + }; + private async handleFirstInvite(content: MCallInvite, partyId: PartyId, log: ILogItem): Promise { if (this._state !== CallState.Fledgling || this.opponentPartyId !== undefined) { // TODO: hangup or ignore? @@ -789,7 +803,7 @@ export class PeerCall implements IDisposable { } this.hangupParty = hangupParty; - // this.hangupReason = hangupReason; + this._hangupReason = hangupReason; this.setState(CallState.Ended, log); //this.localMedia?.dispose(); //this.localMedia = undefined; diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 3eb5f17f..0ac7df99 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -16,7 +16,7 @@ limitations under the License. import {PeerCall, CallState} from "../PeerCall"; import {makeTxnId, makeId} from "../../common"; -import {EventType} from "../callEventTypes"; +import {EventType, CallErrorCode} from "../callEventTypes"; import {formatToDeviceMessagesPayload} from "../../common"; import type {Options as PeerCallOptions} from "../PeerCall"; @@ -39,9 +39,19 @@ export type Options = Omit void, } +const errorCodesWithoutRetry = [ + CallErrorCode.UserHangup, + CallErrorCode.AnsweredElsewhere, + CallErrorCode.Replaced, + CallErrorCode.UserBusy, + CallErrorCode.Transfered, + CallErrorCode.NewSession +]; + export class Member { private peerCall?: PeerCall; private localMedia?: LocalMedia; + private retryCount: number = 0; constructor( public readonly member: RoomMember, @@ -109,6 +119,17 @@ export class Member { if (peerCall.state === CallState.Ringing) { peerCall.answer(this.localMedia!); } + else if (peerCall.state === CallState.Ended) { + const hangupReason = peerCall.hangupReason; + peerCall.dispose(); + this.peerCall = undefined; + if (hangupReason && !errorCodesWithoutRetry.includes(hangupReason)) { + this.retryCount += 1; + if (this.retryCount <= 3) { + this.connect(this.localMedia!); + } + } + } this.options.emitUpdate(this, params); } @@ -151,7 +172,6 @@ export class Member { this.peerCall = this._createPeerCall(message.content.call_id); } if (this.peerCall) { - const prevState = this.peerCall.state; this.peerCall.handleIncomingSignallingMessage(message, deviceId); } else { // TODO: need to buffer events until invite comes? From 2635adb2325009c7a51af1e53b42e0cdff3c52ee Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 12 Apr 2022 14:02:38 +0200 Subject: [PATCH 063/323] hardcode turn server for now --- src/matrix/Session.js | 4 ++++ src/matrix/calls/PeerCall.ts | 4 +++- src/platform/types/WebRTC.ts | 2 +- src/platform/web/dom/WebRTC.ts | 6 +++--- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index aa7dbf7b..bd361434 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -96,6 +96,10 @@ export class Session { ownDeviceId: sessionInfo.deviceId, ownUserId: sessionInfo.userId, logger: this._platform.logger, + turnServers: [{ + urls: ["stun:turn.matrix.org"], + }], + forceTURN: false, }); this._deviceMessageHandler = new DeviceMessageHandler({storage, callHandler: this._callHandler}); this._olm = olm; diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index fa1063ec..8559982c 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -45,6 +45,8 @@ import type { export type Options = { webRTC: WebRTC, + forceTURN: boolean, + turnServers: RTCIceServer[], createTimeout: TimeoutCreator, emitUpdate: (peerCall: PeerCall, params: any) => void; sendSignallingMessage: (message: SignallingMessage, log: ILogItem) => Promise; @@ -132,7 +134,7 @@ export class PeerCall implements IDisposable { getPurposeForStreamId(streamId: string): SDPStreamMetadataPurpose { return outer.remoteSDPStreamMetadata?.[streamId]?.purpose ?? SDPStreamMetadataPurpose.Usermedia; } - }); + }, this.options.forceTURN, this.options.turnServers, 0); } get dataChannel(): any | undefined { return this._dataChannel; } diff --git a/src/platform/types/WebRTC.ts b/src/platform/types/WebRTC.ts index 130cb7cb..a1a7b7cd 100644 --- a/src/platform/types/WebRTC.ts +++ b/src/platform/types/WebRTC.ts @@ -18,7 +18,7 @@ import {Track, TrackType} from "./MediaDevices"; import {SDPStreamMetadataPurpose} from "../../matrix/calls/callEventTypes"; export interface WebRTC { - createPeerConnection(handler: PeerConnectionHandler): PeerConnection; + createPeerConnection(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize): PeerConnection; } export interface PeerConnectionHandler { diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 78977eb8..861a91f1 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -24,8 +24,8 @@ export const SPEAKING_THRESHOLD = -60; // dB const SPEAKING_SAMPLE_COUNT = 8; // samples export class DOMWebRTC implements WebRTC { - createPeerConnection(handler: PeerConnectionHandler): PeerConnection { - return new DOMPeerConnection(handler, false, []); + createPeerConnection(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize): PeerConnection { + return new DOMPeerConnection(handler, forceTURN, turnServers, iceCandidatePoolSize); } } @@ -35,7 +35,7 @@ class DOMPeerConnection implements PeerConnection { //private dataChannelWrapper?: DOMDataChannel; private _remoteTracks: TrackWrapper[] = []; - constructor(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize = 0) { + constructor(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize) { this.handler = handler; this.peerConnection = new RTCPeerConnection({ iceTransportPolicy: forceTURN ? 'relay' : undefined, From 0e9307608b3ad04bc2687fa4c1766a1eda12865e Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 12 Apr 2022 14:02:57 +0200 Subject: [PATCH 064/323] update TODO --- src/matrix/calls/TODO.md | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index 4faf4f4e..ae35447b 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -6,7 +6,25 @@ - https://github.com/matrix-org/matrix-doc/pull/3291 Muting in VoIP calls - https://github.com/matrix-org/matrix-doc/pull/3401 Native Group VoIP Signalling + ## TODO + - DONE: implement receiving hangup + - making logging better + - implement renegotiation + - implement to_device messages arriving before m.call(.member) state event + - implement muting tracks with m.call.sdp_stream_metadata_changed + - implement cloning the localMedia so it works in safari? + - DONE: implement 3 retries per peer + - reeable crypto & implement fetching olm keys before sending encrypted signalling message + - local echo for join/leave buttons? + - make UI pretsy + - figure out video layout + - figure out nav structure + - batch outgoing to_device messages in one request to homeserver for operations that will send out an event to all participants (e.g. mute) + - don't load all members when loading calls to know whether they are ringing and joined by ourself + - only load our own member once, then have a way to load additional members on a call. + +## TODO (old) - PeerCall - send invite - implement terminate @@ -34,7 +52,7 @@ ## Store ongoing calls -Add store with all ongoing calls so when we quit and start again, we don't have to go through all the past calls to know which ones might still be ongoing. +DONE: Add store with all ongoing calls so when we quit and start again, we don't have to go through all the past calls to know which ones might still be ongoing. ## Notes @@ -104,10 +122,10 @@ Do work needed for state events - DONEish: receiving (almost done?) - DONEish: sending logging -Expose call objects -expose volume events from audiotrack to group call -Write view model -write view +DONE: Expose call objects + expose volume events from audiotrack to group call +DONE: Write view model +DONE: write view - handle glare edge-cases (not yet sent): https://spec.matrix.org/latest/client-server-api/#glare ## Calls questions From 36dc463d2325c857493072c2a69d681589e22684 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 12 Apr 2022 21:20:15 +0200 Subject: [PATCH 065/323] update TODO --- src/matrix/calls/TODO.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index ae35447b..8c041fcc 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -11,6 +11,8 @@ - DONE: implement receiving hangup - making logging better - implement renegotiation + - finish session id support + - call peers are essentially identified by (userid, deviceid, sessionid). If see a new session id, we first disconnect from the current member so we're ready to connect with a clean slate again (in a member event, also in to_device? no harm I suppose, given olm encryption ensures you can't spoof the deviceid). - implement to_device messages arriving before m.call(.member) state event - implement muting tracks with m.call.sdp_stream_metadata_changed - implement cloning the localMedia so it works in safari? @@ -23,7 +25,7 @@ - batch outgoing to_device messages in one request to homeserver for operations that will send out an event to all participants (e.g. mute) - don't load all members when loading calls to know whether they are ringing and joined by ourself - only load our own member once, then have a way to load additional members on a call. - + - see if we remove partyId entirely, it is only used for detecting remote echo which is not an issue for group calls? see https://github.com/matrix-org/matrix-spec-proposals/blob/dbkr/msc2746/proposals/2746-reliable-voip.md#add-party_id-to-all-voip-events ## TODO (old) - PeerCall - send invite From 2d4301fe5a5c451100f68e275f7a6fe90b0f7ba2 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 12 Apr 2022 21:20:24 +0200 Subject: [PATCH 066/323] WIP: expose streams, senders and receivers --- .../session/room/timeline/tiles/CallTile.js | 1 + src/matrix/calls/LocalMedia.ts | 44 +-- src/matrix/calls/PeerCall.ts | 4 +- src/platform/types/MediaDevices.ts | 31 +- src/platform/types/WebRTC.ts | 42 ++- src/platform/web/dom/MediaDevices.ts | 79 +++-- src/platform/web/dom/WebRTC.ts | 299 ++++++++++++------ 7 files changed, 308 insertions(+), 192 deletions(-) diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index 32807958..2d2d9dab 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -75,6 +75,7 @@ export class CallTile extends SimpleTile { async join() { if (this.canJoin) { const mediaTracks = await this.platform.mediaDevices.getMediaTracks(false, true); + // const screenShareTrack = await this.platform.mediaDevices.getScreenShareTrack(); const localMedia = new LocalMedia().withTracks(mediaTracks); await this._call.join(localMedia); } diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index fd91c61c..6eb7a225 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -15,37 +15,26 @@ limitations under the License. */ import {SDPStreamMetadataPurpose} from "./callEventTypes"; -import {Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; +import {Stream} from "../../platform/types/MediaDevices"; import {SDPStreamMetadata} from "./callEventTypes"; export class LocalMedia { constructor( - public readonly cameraTrack?: Track, - public readonly screenShareTrack?: Track, - public readonly microphoneTrack?: AudioTrack, + public readonly userMedia?: Stream, + public readonly screenShare?: Stream, public readonly dataChannelOptions?: RTCDataChannelInit, ) {} - withTracks(tracks: Track[]) { - const cameraTrack = tracks.find(t => t.type === TrackType.Camera) ?? this.cameraTrack; - const screenShareTrack = tracks.find(t => t.type === TrackType.ScreenShare) ?? this.screenShareTrack; - const microphoneTrack = tracks.find(t => t.type === TrackType.Microphone) ?? this.microphoneTrack; - if (cameraTrack && microphoneTrack && cameraTrack.streamId !== microphoneTrack.streamId) { - throw new Error("The camera and audio track should have the same stream id"); - } - return new LocalMedia(cameraTrack, screenShareTrack, microphoneTrack as AudioTrack, this.dataChannelOptions); + withUserMedia(stream: Stream) { + return new LocalMedia(stream, this.screenShare, this.dataChannelOptions); + } + + withScreenShare(stream: Stream) { + return new LocalMedia(this.userMedia, stream, this.dataChannelOptions); } withDataChannel(options: RTCDataChannelInit): LocalMedia { - return new LocalMedia(this.cameraTrack, this.screenShareTrack, this.microphoneTrack as AudioTrack, options); - } - - get tracks(): Track[] { - const tracks: Track[] = []; - if (this.cameraTrack) { tracks.push(this.cameraTrack); } - if (this.screenShareTrack) { tracks.push(this.screenShareTrack); } - if (this.microphoneTrack) { tracks.push(this.microphoneTrack); } - return tracks; + return new LocalMedia(this.userMedia, this.screenShare, options); } getSDPMetadata(): SDPStreamMetadata { @@ -54,8 +43,8 @@ export class LocalMedia { if (userMediaTrack) { metadata[userMediaTrack.streamId] = { purpose: SDPStreamMetadataPurpose.Usermedia, - audio_muted: this.microphoneTrack?.muted ?? false, - video_muted: this.cameraTrack?.muted ?? false, + audio_muted: this.microphoneTrack?.muted ?? true, + video_muted: this.cameraTrack?.muted ?? true, }; } if (this.screenShareTrack) { @@ -67,13 +56,12 @@ export class LocalMedia { } clone() { - // TODO: implement - return this; + return new LocalMedia(this.userMedia?.clone(), this.screenShare?.clone(), this.dataChannelOptions); } dispose() { - this.cameraTrack?.stop(); - this.microphoneTrack?.stop(); - this.screenShareTrack?.stop(); + this.userMedia?.audioTrack?.stop(); + this.userMedia?.videoTrack?.stop(); + this.screenShare?.videoTrack?.stop(); } } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 8559982c..644ed160 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -701,8 +701,6 @@ export class PeerCall implements IDisposable { private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { this.remoteSDPStreamMetadata = recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); - // will rerequest stream purpose for all tracks and set track.type accordingly - this.peerConnection.notifyStreamPurposeChanged(); for (const track of this.peerConnection.remoteTracks) { const streamMetaData = this.remoteSDPStreamMetadata?.[track.streamId]; if (streamMetaData) { @@ -757,6 +755,8 @@ export class PeerCall implements IDisposable { this.iceDisconnectedTimeout?.abort(); this.iceDisconnectedTimeout = undefined; this.setState(CallState.Connected, log); + const transceivers = this.peerConnection.peerConnection.getTransceivers(); + console.log(transceivers); } else if (state == 'failed') { this.iceDisconnectedTimeout?.abort(); this.iceDisconnectedTimeout = undefined; diff --git a/src/platform/types/MediaDevices.ts b/src/platform/types/MediaDevices.ts index db267871..d0edbbea 100644 --- a/src/platform/types/MediaDevices.ts +++ b/src/platform/types/MediaDevices.ts @@ -18,29 +18,32 @@ export interface MediaDevices { // filter out audiooutput enumerate(): Promise; // to assign to a video element, we downcast to WrappedTrack and use the stream property. - getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise; - getScreenShareTrack(): Promise; + getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise; + getScreenShareTrack(): Promise; } -export enum TrackType { - ScreenShare, - Camera, - Microphone, +export interface Stream { + readonly audioTrack: AudioTrack | undefined; + readonly videoTrack: Track | undefined; + readonly id: string; + clone(): Stream; +} + +export enum TrackKind { + Video = "video", + Audio = "audio" } export interface Track { - get type(): TrackType; - get label(): string; - get id(): string; - get streamId(): string; - get settings(): MediaTrackSettings; - get muted(): boolean; - setMuted(muted: boolean): void; + readonly kind: TrackKind; + readonly label: string; + readonly id: string; + readonly settings: MediaTrackSettings; stop(): void; - clone(): Track; } export interface AudioTrack extends Track { + // TODO: how to emit updates on this? get isSpeaking(): boolean; } diff --git a/src/platform/types/WebRTC.ts b/src/platform/types/WebRTC.ts index a1a7b7cd..a0512163 100644 --- a/src/platform/types/WebRTC.ts +++ b/src/platform/types/WebRTC.ts @@ -14,38 +14,62 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {Track, TrackType} from "./MediaDevices"; +import {Track, Stream} from "./MediaDevices"; import {SDPStreamMetadataPurpose} from "../../matrix/calls/callEventTypes"; export interface WebRTC { createPeerConnection(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize): PeerConnection; } +export interface StreamSender { + get stream(): Stream; + get audioSender(): TrackSender | undefined; + get videoSender(): TrackSender | undefined; +} + +export interface StreamReceiver { + get stream(): Stream; + get audioReceiver(): TrackReceiver | undefined; + get videoReceiver(): TrackReceiver | undefined; +} + +export interface TrackReceiver { + get track(): Track; + get enabled(): boolean; + enable(enabled: boolean); // this modifies the transceiver direction +} + +export interface TrackSender extends TrackReceiver { + /** replaces the track if possible without renegotiation. Can throw. */ + replaceTrack(track: Track): Promise; + /** make any needed adjustments to the sender or transceiver settings + * depending on the purpose, after adding the track to the connection */ + prepareForPurpose(purpose: SDPStreamMetadataPurpose): void; +} + export interface PeerConnectionHandler { onIceConnectionStateChange(state: RTCIceConnectionState); onLocalIceCandidate(candidate: RTCIceCandidate); onIceGatheringStateChange(state: RTCIceGatheringState); - onRemoteTracksChanged(tracks: Track[]); + onRemoteStreamRemoved(stream: Stream); + onRemoteTracksAdded(receiver: TrackReceiver); onRemoteDataChannel(dataChannel: any | undefined); onNegotiationNeeded(); - // request the type of incoming stream - getPurposeForStreamId(streamId: string): SDPStreamMetadataPurpose; } export interface PeerConnection { - notifyStreamPurposeChanged(): void; - get remoteTracks(): Track[]; get iceGatheringState(): RTCIceGatheringState; get signalingState(): RTCSignalingState; get localDescription(): RTCSessionDescription | undefined; + get localStreams(): ReadonlyArray; + get remoteStreams(): ReadonlyArray; createOffer(): Promise; createAnswer(): Promise; setLocalDescription(description?: RTCSessionDescriptionInit): Promise; setRemoteDescription(description: RTCSessionDescriptionInit): Promise; addIceCandidate(candidate: RTCIceCandidate): Promise; - addTrack(track: Track): void; - removeTrack(track: Track): boolean; - replaceTrack(oldTrack: Track, newTrack: Track): Promise; + addTrack(track: Track): TrackSender | undefined; + removeTrack(track: TrackSender): void; createDataChannel(options: RTCDataChannelInit): any; dispose(): void; close(): void; diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index 49fd7fa5..52ca132b 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {MediaDevices as IMediaDevices, TrackType, Track, AudioTrack} from "../../types/MediaDevices"; +import {MediaDevices as IMediaDevices, Stream, Track, TrackKind, AudioTrack} from "../../types/MediaDevices"; const POLLING_INTERVAL = 200; // ms export const SPEAKING_THRESHOLD = -60; // dB @@ -28,22 +28,14 @@ export class MediaDevicesWrapper implements IMediaDevices { return this.mediaDevices.enumerateDevices(); } - async getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise { + async getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise { const stream = await this.mediaDevices.getUserMedia(this.getUserMediaContraints(audio, video)); - const tracks = stream.getTracks().map(t => { - const type = t.kind === "audio" ? TrackType.Microphone : TrackType.Camera; - return wrapTrack(t, stream, type); - }); - return tracks; + return new StreamWrapper(stream); } - async getScreenShareTrack(): Promise { + async getScreenShareTrack(): Promise { const stream = await this.mediaDevices.getDisplayMedia(this.getScreenshareContraints()); - const videoTrack = stream.getTracks().find(t => t.kind === "video"); - if (videoTrack) { - return wrapTrack(videoTrack, stream, TrackType.ScreenShare); - } - return; + return new StreamWrapper(stream); } private getUserMediaContraints(audio: boolean | MediaDeviceInfo, video: boolean | MediaDeviceInfo): MediaStreamConstraints { @@ -78,43 +70,50 @@ export class MediaDevicesWrapper implements IMediaDevices { } } -export function wrapTrack(track: MediaStreamTrack, stream: MediaStream, type: TrackType) { - if (track.kind === "audio") { - return new AudioTrackWrapper(track, stream, type); - } else { - return new TrackWrapper(track, stream, type); +export class StreamWrapper implements Stream { + + public audioTrack: AudioTrackWrapper | undefined; + public videoTrack: TrackWrapper | undefined; + + constructor(public readonly stream: MediaStream) { + for (const track of stream.getTracks()) { + this.update(track); + } + } + + get id(): string { return this.stream.id; } + + clone(): Stream { + return new StreamWrapper(this.stream.clone()); + } + + update(track: MediaStreamTrack): TrackWrapper | undefined { + if (track.kind === "video") { + if (!this.videoTrack || track.id !== this.videoTrack.track.id) { + this.videoTrack = new TrackWrapper(track, this.stream); + return this.videoTrack; + } + } else if (track.kind === "audio") { + if (!this.audioTrack || track.id !== this.audioTrack.track.id) { + this.audioTrack = new AudioTrackWrapper(track, this.stream); + return this.audioTrack; + } + } } } export class TrackWrapper implements Track { constructor( public readonly track: MediaStreamTrack, - public readonly stream: MediaStream, - private _type: TrackType, + public readonly stream: MediaStream ) {} - get type(): TrackType { return this._type; } + get kind(): TrackKind { return this.track.kind as TrackKind; } get label(): string { return this.track.label; } get id(): string { return this.track.id; } - get streamId(): string { return this.stream.id; } - get muted(): boolean { return this.track.muted; } get settings(): MediaTrackSettings { return this.track.getSettings(); } - setMuted(muted: boolean): void { - this.track.enabled = !muted; - } - - setType(type: TrackType): void { - this._type = type; - } - - stop() { - this.track.stop(); - } - - clone() { - return this.track.clone(); - } + stop() { this.track.stop(); } } export class AudioTrackWrapper extends TrackWrapper { @@ -127,8 +126,8 @@ export class AudioTrackWrapper extends TrackWrapper { private volumeLooperTimeout: number; private speakingVolumeSamples: number[]; - constructor(track: MediaStreamTrack, stream: MediaStream, type: TrackType) { - super(track, stream, type); + constructor(track: MediaStreamTrack, stream: MediaStream) { + super(track, stream); this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); this.initVolumeMeasuring(); this.measureVolumeActivity(true); diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 861a91f1..6760182d 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -14,9 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {TrackWrapper, wrapTrack} from "./MediaDevices"; -import {Track, TrackType} from "../../types/MediaDevices"; -import {WebRTC, PeerConnectionHandler, DataChannel, PeerConnection} from "../../types/WebRTC"; +import {StreamWrapper, TrackWrapper, AudioTrackWrapper} from "./MediaDevices"; +import {Stream, Track, AudioTrack, TrackKind} from "../../types/MediaDevices"; +import {WebRTC, PeerConnectionHandler, StreamSender, TrackSender, StreamReceiver, TrackReceiver, PeerConnection} from "../../types/WebRTC"; import {SDPStreamMetadataPurpose} from "../../../matrix/calls/callEventTypes"; const POLLING_INTERVAL = 200; // ms @@ -29,11 +29,171 @@ export class DOMWebRTC implements WebRTC { } } +export class RemoteStreamWrapper extends StreamWrapper { + constructor(stream: MediaStream, private readonly emptyCallback: (stream: RemoteStreamWrapper) => void) { + super(stream); + this.stream.addEventListener("removetrack", this.onTrackRemoved); + } + + onTrackRemoved = (evt: MediaStreamTrackEvent) => { + if (evt.track.id === this.audioTrack?.track.id) { + this.audioTrack = undefined; + } else if (evt.track.id === this.videoTrack?.track.id) { + this.videoTrack = undefined; + } + if (!this.audioTrack && !this.videoTrack) { + this.emptyCallback(this); + } + }; + + dispose() { + this.stream.removeEventListener("removetrack", this.onTrackRemoved); + } +} + +export class DOMStreamSender implements StreamSender { + public audioSender: DOMTrackSender | undefined; + public videoSender: DOMTrackSender | undefined; + + constructor(public readonly stream: StreamWrapper) {} + + update(transceivers: ReadonlyArray, sender: RTCRtpSender): DOMTrackSender | undefined { + const transceiver = transceivers.find(t => t.sender === sender); + if (transceiver && sender.track) { + const trackWrapper = this.stream.update(sender.track); + if (trackWrapper) { + if (trackWrapper.kind === TrackKind.Video) { + this.videoSender = new DOMTrackSender(trackWrapper, transceiver); + return this.videoSender; + } else { + this.audioSender = new DOMTrackSender(trackWrapper, transceiver); + return this.audioSender; + } + } + } + } +} + +export class DOMStreamReceiver implements StreamReceiver { + public audioReceiver: DOMTrackReceiver | undefined; + public videoReceiver: DOMTrackReceiver | undefined; + + constructor(public readonly stream: RemoteStreamWrapper) {} + + update(event: RTCTrackEvent): DOMTrackReceiver | undefined { + const {receiver} = event; + const {track} = receiver; + const trackWrapper = this.stream.update(track); + if (trackWrapper) { + if (trackWrapper.kind === TrackKind.Video) { + this.videoReceiver = new DOMTrackReceiver(trackWrapper, event.transceiver); + return this.videoReceiver; + } else { + this.audioReceiver = new DOMTrackReceiver(trackWrapper, event.transceiver); + return this.audioReceiver; + } + } + } +} + +export class DOMTrackSenderOrReceiver implements TrackReceiver { + constructor( + public readonly track: TrackWrapper, + public readonly transceiver: RTCRtpTransceiver, + private readonly exclusiveValue: RTCRtpTransceiverDirection, + private readonly excludedValue: RTCRtpTransceiverDirection + ) {} + + get enabled(): boolean { + return this.transceiver.currentDirection === "sendrecv" || + this.transceiver.currentDirection === this.exclusiveValue; + } + + enable(enabled: boolean) { + if (enabled !== this.enabled) { + if (enabled) { + if (this.transceiver.currentDirection === "inactive") { + this.transceiver.direction = this.exclusiveValue; + } else { + this.transceiver.direction = "sendrecv"; + } + } else { + if (this.transceiver.currentDirection === "sendrecv") { + this.transceiver.direction = this.excludedValue; + } else { + this.transceiver.direction = "inactive"; + } + } + } + } +} + +export class DOMTrackReceiver extends DOMTrackSenderOrReceiver { + constructor( + track: TrackWrapper, + transceiver: RTCRtpTransceiver, + ) { + super(track, transceiver, "recvonly", "sendonly"); + } +} + +export class DOMTrackSender extends DOMTrackSenderOrReceiver { + constructor( + track: TrackWrapper, + transceiver: RTCRtpTransceiver, + ) { + super(track, transceiver, "sendonly", "recvonly"); + } + /** replaces the track if possible without renegotiation. Can throw. */ + replaceTrack(track: Track): Promise { + return this.transceiver.sender.replaceTrack(track ? (track as TrackWrapper).track : null); + } + + prepareForPurpose(purpose: SDPStreamMetadataPurpose): void { + if (purpose === SDPStreamMetadataPurpose.Screenshare) { + this.getRidOfRTXCodecs(); + } + } + + /** + * This method removes all video/rtx codecs from screensharing video + * transceivers. This is necessary since they can cause problems. Without + * this the following steps should produce an error: + * Chromium calls Firefox + * Firefox answers + * Firefox starts screen-sharing + * Chromium starts screen-sharing + * Call crashes for Chromium with: + * [96685:23:0518/162603.933321:ERROR:webrtc_video_engine.cc(3296)] RTX codec (PT=97) mapped to PT=96 which is not in the codec list. + * [96685:23:0518/162603.933377:ERROR:webrtc_video_engine.cc(1171)] GetChangedRecvParameters called without any video codecs. + * [96685:23:0518/162603.933430:ERROR:sdp_offer_answer.cc(4302)] Failed to set local video description recv parameters for m-section with mid='2'. (INVALID_PARAMETER) + */ + private getRidOfRTXCodecs(): void { + // RTCRtpReceiver.getCapabilities and RTCRtpSender.getCapabilities don't seem to be supported on FF + if (!RTCRtpReceiver.getCapabilities || !RTCRtpSender.getCapabilities) return; + + const recvCodecs = RTCRtpReceiver.getCapabilities("video")?.codecs ?? []; + const sendCodecs = RTCRtpSender.getCapabilities("video")?.codecs ?? []; + const codecs = [...sendCodecs, ...recvCodecs]; + + for (const codec of codecs) { + if (codec.mimeType === "video/rtx") { + const rtxCodecIndex = codecs.indexOf(codec); + codecs.splice(rtxCodecIndex, 1); + } + } + if (this.transceiver.sender.track?.kind === "video" || + this.transceiver.receiver.track?.kind === "video") { + this.transceiver.setCodecPreferences(codecs); + } + } +} + class DOMPeerConnection implements PeerConnection { private readonly peerConnection: RTCPeerConnection; private readonly handler: PeerConnectionHandler; - //private dataChannelWrapper?: DOMDataChannel; - private _remoteTracks: TrackWrapper[] = []; + public readonly localStreams: DOMStreamSender[]; + public readonly remoteStreams: DOMStreamReceiver[]; constructor(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize) { this.handler = handler; @@ -45,7 +205,6 @@ class DOMPeerConnection implements PeerConnection { this.registerHandler(); } - get remoteTracks(): Track[] { return this._remoteTracks; } get iceGatheringState(): RTCIceGatheringState { return this.peerConnection.iceGatheringState; } get localDescription(): RTCSessionDescription | undefined { return this.peerConnection.localDescription ?? undefined; } get signalingState(): RTCSignalingState { return this.peerConnection.signalingState; } @@ -74,48 +233,26 @@ class DOMPeerConnection implements PeerConnection { return this.peerConnection.close(); } - addTrack(track: Track): void { + addTrack(track: Track): DOMTrackSender | undefined { if (!(track instanceof TrackWrapper)) { throw new Error("Not a TrackWrapper"); } - this.peerConnection.addTrack(track.track, track.stream); - if (track.type === TrackType.ScreenShare) { - this.getRidOfRTXCodecs(track); + const sender = this.peerConnection.addTrack(track.track, track.stream); + let streamSender: DOMStreamSender | undefined = this.localStreams.find(s => s.stream.id === track.stream.id); + if (!streamSender) { + streamSender = new DOMStreamSender(new StreamWrapper(track.stream)); + this.localStreams.push(streamSender); } + const trackSender = streamSender.update(this.peerConnection.getTransceivers(), sender); + return trackSender; } - removeTrack(track: Track): boolean { - if (!(track instanceof TrackWrapper)) { - throw new Error("Not a TrackWrapper"); - } - const sender = this.peerConnection.getSenders().find(s => s.track === track.track); - if (sender) { - this.peerConnection.removeTrack(sender); - return true; - } - return false; - } - - async replaceTrack(oldTrack: Track, newTrack: Track): Promise { - if (!(oldTrack instanceof TrackWrapper) || !(newTrack instanceof TrackWrapper)) { - throw new Error("Not a TrackWrapper"); - } - const sender = this.peerConnection.getSenders().find(s => s.track === oldTrack.track); - if (sender) { - await sender.replaceTrack(newTrack.track); - if (newTrack.type === TrackType.ScreenShare) { - this.getRidOfRTXCodecs(newTrack); - } - return true; - } - return false; - } - - notifyStreamPurposeChanged(): void { - for (const track of this.remoteTracks) { - const wrapper = track as TrackWrapper; - wrapper.setType(this.getRemoteTrackType(wrapper.track, wrapper.streamId)); + removeTrack(sender: TrackSender): void { + if (!(sender instanceof DOMTrackSender)) { + throw new Error("Not a DOMTrackSender"); } + this.peerConnection.removeTrack((sender as DOMTrackSender).transceiver.sender); + // TODO: update localStreams } createDataChannel(options: RTCDataChannelInit): any { @@ -170,6 +307,9 @@ class DOMPeerConnection implements PeerConnection { dispose(): void { this.deregisterHandler(); + for (const r of this.remoteStreams) { + r.stream.dispose(); + } } private handleLocalIceCandidate(event: RTCPeerConnectionIceEvent) { @@ -187,67 +327,28 @@ class DOMPeerConnection implements PeerConnection { } } + onRemoteStreamEmpty = (stream: RemoteStreamWrapper): void => { + const idx = this.remoteStreams.findIndex(r => r.stream === stream); + if (idx !== -1) { + this.remoteStreams.splice(idx, 1); + this.handler.onRemoteStreamRemoved(stream); + } + } + private handleRemoteTrack(evt: RTCTrackEvent) { - // TODO: unit test this code somehow - // the tracks on the new stream (with their stream) - const updatedTracks = evt.streams.flatMap(stream => stream.getTracks().map(track => {return {stream, track};})); - // of the tracks we already know about, filter the ones that aren't in the new stream - const withoutRemovedTracks = this._remoteTracks.filter(t => updatedTracks.some(ut => t.track.id === ut.track.id)); - // of the new tracks, filter the ones that we didn't already knew about - const addedTracks = updatedTracks.filter(ut => !this._remoteTracks.some(t => t.track.id === ut.track.id)); - // wrap them - const wrappedAddedTracks = addedTracks.map(t => wrapTrack(t.track, t.stream, this.getRemoteTrackType(t.track, t.stream.id))); - // and concat the tracks for other streams with the added tracks - this._remoteTracks = withoutRemovedTracks.concat(...wrappedAddedTracks); - this.handler.onRemoteTracksChanged(this.remoteTracks); - } - - private getRemoteTrackType(track: MediaStreamTrack, streamId: string): TrackType { - if (track.kind === "video") { - const purpose = this.handler.getPurposeForStreamId(streamId); - return purpose === SDPStreamMetadataPurpose.Usermedia ? TrackType.Camera : TrackType.ScreenShare; - } else { - return TrackType.Microphone; + if (evt.streams.length !== 0) { + throw new Error("track in multiple streams is not supported"); } - } - - /** - * This method removes all video/rtx codecs from screensharing video - * transceivers. This is necessary since they can cause problems. Without - * this the following steps should produce an error: - * Chromium calls Firefox - * Firefox answers - * Firefox starts screen-sharing - * Chromium starts screen-sharing - * Call crashes for Chromium with: - * [96685:23:0518/162603.933321:ERROR:webrtc_video_engine.cc(3296)] RTX codec (PT=97) mapped to PT=96 which is not in the codec list. - * [96685:23:0518/162603.933377:ERROR:webrtc_video_engine.cc(1171)] GetChangedRecvParameters called without any video codecs. - * [96685:23:0518/162603.933430:ERROR:sdp_offer_answer.cc(4302)] Failed to set local video description recv parameters for m-section with mid='2'. (INVALID_PARAMETER) - */ - private getRidOfRTXCodecs(screensharingTrack: TrackWrapper): void { - // RTCRtpReceiver.getCapabilities and RTCRtpSender.getCapabilities don't seem to be supported on FF - if (!RTCRtpReceiver.getCapabilities || !RTCRtpSender.getCapabilities) return; - - const recvCodecs = RTCRtpReceiver.getCapabilities("video")?.codecs ?? []; - const sendCodecs = RTCRtpSender.getCapabilities("video")?.codecs ?? []; - const codecs = [...sendCodecs, ...recvCodecs]; - - for (const codec of codecs) { - if (codec.mimeType === "video/rtx") { - const rtxCodecIndex = codecs.indexOf(codec); - codecs.splice(rtxCodecIndex, 1); - } + const stream = evt.streams[0]; + const transceivers = this.peerConnection.getTransceivers(); + let streamReceiver: DOMStreamReceiver | undefined = this.remoteStreams.find(r => r.stream.id === stream.id); + if (!streamReceiver) { + streamReceiver = new DOMStreamReceiver(new RemoteStreamWrapper(stream, this.onRemoteStreamEmpty)); + this.remoteStreams.push(streamReceiver); } - - for (const trans of this.peerConnection.getTransceivers()) { - if (trans.sender.track === screensharingTrack.track && - ( - trans.sender.track?.kind === "video" || - trans.receiver.track?.kind === "video" - ) - ) { - trans.setCodecPreferences(codecs); - } + const trackReceiver = streamReceiver.update(evt); + if (trackReceiver) { + this.handler.onRemoteTracksAdded(trackReceiver); } } } From bc118b5c0b87c00e0b96b8219c33a86733769353 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 13 Apr 2022 18:34:01 +0200 Subject: [PATCH 067/323] WIP --- src/domain/session/room/CallViewModel.ts | 10 +- src/domain/session/room/RoomViewModel.js | 5 +- .../session/room/timeline/tiles/CallTile.js | 5 +- src/matrix/calls/CallHandler.ts | 2 +- src/matrix/calls/LocalMedia.ts | 18 -- src/matrix/calls/PeerCall.ts | 158 +++++++++++------- src/matrix/calls/group/GroupCall.ts | 2 +- src/matrix/calls/group/Member.ts | 7 +- src/platform/types/WebRTC.ts | 8 +- src/platform/web/dom/MediaDevices.ts | 8 +- src/platform/web/dom/WebRTC.ts | 35 ++-- src/platform/web/ui/session/room/CallView.ts | 16 +- yarn.lock | 8 +- 13 files changed, 154 insertions(+), 128 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 72353e4e..bc7b54ff 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -18,7 +18,7 @@ import {ViewModel, Options as BaseOptions} from "../../ViewModel"; import type {GroupCall} from "../../../matrix/calls/group/GroupCall"; import type {Member} from "../../../matrix/calls/group/Member"; import type {BaseObservableList} from "../../../observable/list/BaseObservableList"; -import type {Track} from "../../../platform/types/MediaDevices"; +import type {Stream} from "../../../platform/types/MediaDevices"; type Options = BaseOptions & {call: GroupCall}; @@ -46,8 +46,8 @@ export class CallViewModel extends ViewModel { return this.call.id; } - get localTracks(): Track[] { - return this.call.localMedia?.tracks ?? []; + get localStream(): Stream | undefined { + return this.call.localMedia?.userMedia; } leave() { @@ -60,8 +60,8 @@ export class CallViewModel extends ViewModel { type MemberOptions = BaseOptions & {member: Member}; export class CallMemberViewModel extends ViewModel { - get tracks(): Track[] { - return this.member.remoteTracks; + get stream(): Stream | undefined { + return this.member.remoteMedia?.userMedia; } private get member(): Member { diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index e7d7dc9a..868ca189 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -365,8 +365,9 @@ export class RoomViewModel extends ViewModel { async startCall() { try { const session = this.getOption("session"); - const mediaTracks = await this.platform.mediaDevices.getMediaTracks(false, true); - const localMedia = new LocalMedia().withTracks(mediaTracks); + const stream = await this.platform.mediaDevices.getMediaTracks(false, true); + const localMedia = new LocalMedia().withUserMedia(stream); + await this._call.join(localMedia); // this will set the callViewModel above as a call will be added to callHandler.calls const call = await session.callHandler.createCall(this._room.id, localMedia, "A call " + Math.round(this.platform.random() * 100)); await call.join(localMedia); diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index 2d2d9dab..0bc12698 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -74,9 +74,8 @@ export class CallTile extends SimpleTile { async join() { if (this.canJoin) { - const mediaTracks = await this.platform.mediaDevices.getMediaTracks(false, true); - // const screenShareTrack = await this.platform.mediaDevices.getScreenShareTrack(); - const localMedia = new LocalMedia().withTracks(mediaTracks); + const stream = await this.platform.mediaDevices.getMediaTracks(false, true); + const localMedia = new LocalMedia().withUserMedia(stream); await this._call.join(localMedia); } } diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 31829396..5208b72e 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -16,7 +16,7 @@ limitations under the License. import {ObservableMap} from "../../observable/map/ObservableMap"; import {WebRTC, PeerConnection, PeerConnectionHandler} from "../../platform/types/WebRTC"; -import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; +import {MediaDevices, Track, AudioTrack} from "../../platform/types/MediaDevices"; import {handlesEventType} from "./PeerCall"; import {EventType, CallIntent} from "./callEventTypes"; import {GroupCall} from "./group/GroupCall"; diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 6eb7a225..6622f641 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -37,24 +37,6 @@ export class LocalMedia { return new LocalMedia(this.userMedia, this.screenShare, options); } - getSDPMetadata(): SDPStreamMetadata { - const metadata = {}; - const userMediaTrack = this.microphoneTrack ?? this.cameraTrack; - if (userMediaTrack) { - metadata[userMediaTrack.streamId] = { - purpose: SDPStreamMetadataPurpose.Usermedia, - audio_muted: this.microphoneTrack?.muted ?? true, - video_muted: this.cameraTrack?.muted ?? true, - }; - } - if (this.screenShareTrack) { - metadata[this.screenShareTrack.streamId] = { - purpose: SDPStreamMetadataPurpose.Screenshare - }; - } - return metadata; - } - clone() { return new LocalMedia(this.userMedia?.clone(), this.screenShare?.clone(), this.dataChannelOptions); } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 644ed160..78ffc818 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -23,8 +23,8 @@ import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; import type {TimeoutCreator, Timeout} from "../../platform/types/types"; -import {WebRTC, PeerConnection, PeerConnectionHandler} from "../../platform/types/WebRTC"; -import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; +import {WebRTC, PeerConnection, PeerConnectionHandler, TrackSender, TrackReceiver} from "../../platform/types/WebRTC"; +import {MediaDevices, Track, AudioTrack, Stream} from "../../platform/types/MediaDevices"; import type {LocalMedia} from "./LocalMedia"; import { @@ -52,6 +52,10 @@ export type Options = { sendSignallingMessage: (message: SignallingMessage, log: ILogItem) => Promise; }; +export class RemoteMedia { + constructor(public userMedia?: Stream | undefined, public screenShare?: Stream | undefined) {} +} + // when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already // do for sharing keys will be best as that already deals with room tracking. /** @@ -89,6 +93,7 @@ export class PeerCall implements IDisposable { private _dataChannel?: any; private _hangupReason?: CallErrorCode; + private _remoteMedia: RemoteMedia; constructor( private callId: string, @@ -96,6 +101,7 @@ export class PeerCall implements IDisposable { private readonly logItem: ILogItem, ) { const outer = this; + this._remoteMedia = new RemoteMedia(); this.peerConnection = options.webRTC.createPeerConnection({ onIceConnectionStateChange(state: RTCIceConnectionState) { outer.logItem.wrap({l: "onIceConnectionStateChange", status: state}, log => { @@ -112,9 +118,14 @@ export class PeerCall implements IDisposable { outer.handleIceGatheringState(state, log); }); }, - onRemoteTracksChanged(tracks: Track[]) { - outer.logItem.wrap({l: "onRemoteTracksChanged", length: tracks.length}, log => { - outer.options.emitUpdate(outer, undefined); + onRemoteStreamRemoved(stream: Stream) { + outer.logItem.wrap("onRemoteStreamRemoved", log => { + outer.updateRemoteMedia(log); + }); + }, + onRemoteTracksAdded(trackReceiver: TrackReceiver) { + outer.logItem.wrap("onRemoteTracksAdded", log => { + outer.updateRemoteMedia(log); }); }, onRemoteDataChannel(dataChannel: any | undefined) { @@ -130,9 +141,6 @@ export class PeerCall implements IDisposable { }); }; outer.responsePromiseChain = outer.responsePromiseChain?.then(promiseCreator) ?? promiseCreator(); - }, - getPurposeForStreamId(streamId: string): SDPStreamMetadataPurpose { - return outer.remoteSDPStreamMetadata?.[streamId]?.purpose ?? SDPStreamMetadataPurpose.Usermedia; } }, this.options.forceTURN, this.options.turnServers, 0); } @@ -143,8 +151,9 @@ export class PeerCall implements IDisposable { get hangupReason(): CallErrorCode | undefined { return this._hangupReason; } - get remoteTracks(): Track[] { - return this.peerConnection.remoteTracks; + // we should keep an object with streams by purpose ... e.g. RemoteMedia? + get remoteMedia(): Readonly { + return this._remoteMedia; } call(localMedia: LocalMedia): Promise { @@ -152,13 +161,10 @@ export class PeerCall implements IDisposable { if (this._state !== CallState.Fledgling) { return; } - this.localMedia = localMedia; this.direction = CallDirection.Outbound; this.setState(CallState.CreateOffer, log); - for (const t of this.localMedia.tracks) { - this.peerConnection.addTrack(t); - } - if (this.localMedia.dataChannelOptions) { + this.setMedia(localMedia); + if (this.localMedia?.dataChannelOptions) { this._dataChannel = this.peerConnection.createDataChannel(this.localMedia.dataChannelOptions); } // after adding the local tracks, and wait for handleNegotiation to be called, @@ -172,11 +178,8 @@ export class PeerCall implements IDisposable { if (this._state !== CallState.Ringing) { return; } - this.localMedia = localMedia; this.setState(CallState.CreateAnswer, log); - for (const t of this.localMedia.tracks) { - this.peerConnection.addTrack(t); - } + this.setMedia(localMedia, log); let myAnswer: RTCSessionDescriptionInit; try { myAnswer = await this.peerConnection.createAnswer(); @@ -205,27 +208,40 @@ export class PeerCall implements IDisposable { }); } - setMedia(localMediaPromise: Promise): Promise { - return this.logItem.wrap("setMedia", async log => { + setMedia(localMedia: LocalMedia, logItem: ILogItem = this.logItem): Promise { + return logItem.wrap("setMedia", async log => { const oldMedia = this.localMedia; - this.localMedia = await localMediaPromise; + this.localMedia = localMedia; + const applyStream = (oldStream: Stream | undefined, stream: Stream | undefined, logLabel: string) => { + const streamSender = oldMedia ? this.peerConnection.localStreams.get(oldStream!.id) : undefined; - const applyTrack = (selectTrack: (media: LocalMedia | undefined) => Track | undefined) => { - const oldTrack = selectTrack(oldMedia); - const newTrack = selectTrack(this.localMedia); - if (oldTrack && newTrack) { - this.peerConnection.replaceTrack(oldTrack, newTrack); - } else if (oldTrack) { - this.peerConnection.removeTrack(oldTrack); - } else if (newTrack) { - this.peerConnection.addTrack(newTrack); + const applyTrack = (oldTrack: Track | undefined, sender: TrackSender | undefined, track: Track | undefined) => { + if (track) { + if (oldTrack && sender) { + log.wrap(`replacing ${logLabel} ${track.kind} track`, log => { + sender.replaceTrack(track); + }); + } else { + log.wrap(`adding ${logLabel} ${track.kind} track`, log => { + this.peerConnection.addTrack(track); + }); + } + } else { + if (sender) { + log.wrap(`replacing ${logLabel} ${sender.track.kind} track`, log => { + this.peerConnection.removeTrack(sender); + }); + } + } } - }; - // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called - applyTrack(m => m?.microphoneTrack); - applyTrack(m => m?.cameraTrack); - applyTrack(m => m?.screenShareTrack); + applyTrack(oldStream?.audioTrack, streamSender?.audioSender, stream?.audioTrack); + applyTrack(oldStream?.videoTrack, streamSender?.videoSender, stream?.videoTrack); + } + + applyStream(oldMedia?.userMedia, localMedia?.userMedia, "userMedia"); + applyStream(oldMedia?.screenShare, localMedia?.screenShare, "screenShare"); + // TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method }); } @@ -321,7 +337,7 @@ export class PeerCall implements IDisposable { const content = { call_id: this.callId, offer, - [SDPStreamMetadataKey]: this.localMedia!.getSDPMetadata(), + [SDPStreamMetadataKey]: this.getSDPMetadata(), version: 1, seq: this.seq++, lifetime: CALL_TIMEOUT_MS @@ -408,7 +424,7 @@ export class PeerCall implements IDisposable { const sdpStreamMetadata = content[SDPStreamMetadataKey]; if (sdpStreamMetadata) { - this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata, log); } else { log.log(`Call did not get any SDPStreamMetadata! Can not send/receive multiple streams`); } @@ -470,7 +486,7 @@ export class PeerCall implements IDisposable { const sdpStreamMetadata = content[SDPStreamMetadataKey]; if (sdpStreamMetadata) { - this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata, log); } else { log.log(`Did not get any SDPStreamMetadata! Can not send/receive multiple streams`); } @@ -596,7 +612,7 @@ export class PeerCall implements IDisposable { // type: EventType.CallNegotiate, // content: { // description: this.peerConnection.localDescription!, - // [SDPStreamMetadataKey]: this.localMedia.getSDPMetadata(), + // [SDPStreamMetadataKey]: this.getSDPMetadata(), // } // }); // } @@ -615,7 +631,7 @@ export class PeerCall implements IDisposable { sdp: localDescription.sdp, type: localDescription.type, }, - [SDPStreamMetadataKey]: this.localMedia!.getSDPMetadata(), + [SDPStreamMetadataKey]: this.getSDPMetadata(), }; // We have just taken the local description from the peerConn which will @@ -699,18 +715,11 @@ export class PeerCall implements IDisposable { }); } - private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { + private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata, log: ILogItem): void { + // this will accumulate all updates into one object, so we still have the old stream info when we change stream id this.remoteSDPStreamMetadata = recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); - for (const track of this.peerConnection.remoteTracks) { - const streamMetaData = this.remoteSDPStreamMetadata?.[track.streamId]; - if (streamMetaData) { - if (track.type === TrackType.Microphone) { - track.setMuted(streamMetaData.audio_muted); - } else { // Camera or ScreenShare - track.setMuted(streamMetaData.video_muted); - } - } - } + this.updateRemoteMedia(log); + // TODO: apply muting } private async addBufferedIceCandidates(log: ILogItem): Promise { @@ -755,8 +764,6 @@ export class PeerCall implements IDisposable { this.iceDisconnectedTimeout?.abort(); this.iceDisconnectedTimeout = undefined; this.setState(CallState.Connected, log); - const transceivers = this.peerConnection.peerConnection.getTransceivers(); - console.log(transceivers); } else if (state == 'failed') { this.iceDisconnectedTimeout?.abort(); this.iceDisconnectedTimeout = undefined; @@ -807,14 +814,53 @@ export class PeerCall implements IDisposable { this.hangupParty = hangupParty; this._hangupReason = hangupReason; this.setState(CallState.Ended, log); - //this.localMedia?.dispose(); - //this.localMedia = undefined; + this.localMedia?.dispose(); + this.localMedia = undefined; if (this.peerConnection && this.peerConnection.signalingState !== 'closed') { this.peerConnection.close(); } } + private getSDPMetadata(): SDPStreamMetadata { + const metadata = {}; + if (this.localMedia?.userMedia) { + const streamId = this.localMedia.userMedia.id; + const streamSender = this.peerConnection.localStreams.get(streamId); + metadata[streamId] = { + purpose: SDPStreamMetadataPurpose.Usermedia, + audio_muted: !(streamSender?.audioSender?.enabled), + video_muted: !(streamSender?.videoSender?.enabled), + }; + console.log("video_muted", streamSender?.videoSender?.enabled, streamSender?.videoSender?.transceiver?.direction, streamSender?.videoSender?.transceiver?.currentDirection, JSON.stringify(metadata)); + } + if (this.localMedia?.screenShare) { + const streamId = this.localMedia.screenShare.id; + metadata[streamId] = { + purpose: SDPStreamMetadataPurpose.Screenshare + }; + } + return metadata; + } + + private updateRemoteMedia(log: ILogItem) { + this._remoteMedia.userMedia = undefined; + this._remoteMedia.screenShare = undefined; + if (this.remoteSDPStreamMetadata) { + for (const [streamId, streamReceiver] of this.peerConnection.remoteStreams.entries()) { + const metaData = this.remoteSDPStreamMetadata[streamId]; + if (metaData) { + if (metaData.purpose === SDPStreamMetadataPurpose.Usermedia) { + this._remoteMedia.userMedia = streamReceiver.stream; + } else if (metaData.purpose === SDPStreamMetadataPurpose.Screenshare) { + this._remoteMedia.screenShare = streamReceiver.stream; + } + } + } + } + this.options.emitUpdate(this, undefined); + } + private async delay(timeoutMs: number): Promise { // Allow a short time for initial candidates to be gathered const timeout = this.disposables.track(this.options.createTimeout(timeoutMs)); diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 886cb53a..47d87e5e 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -165,7 +165,7 @@ export class GroupCall extends EventEmitter<{change: never}> { this._state = GroupCallState.Creating; this.emitChange(); this.callContent = Object.assign({ - "m.type": localMedia.cameraTrack ? "m.video" : "m.voice", + "m.type": localMedia.userMedia?.videoTrack ? "m.video" : "m.voice", }, this.callContent); const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCall, this.id, this.callContent!, {log}); await request.response(); diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 0ac7df99..110aba6d 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -19,10 +19,9 @@ import {makeTxnId, makeId} from "../../common"; import {EventType, CallErrorCode} from "../callEventTypes"; import {formatToDeviceMessagesPayload} from "../../common"; -import type {Options as PeerCallOptions} from "../PeerCall"; +import type {Options as PeerCallOptions, RemoteMedia} from "../PeerCall"; import type {LocalMedia} from "../LocalMedia"; import type {HomeServerApi} from "../../net/HomeServerApi"; -import type {Track} from "../../../platform/types/MediaDevices"; import type {MCallBase, MGroupCallBase, SignallingMessage, CallDeviceMembership} from "../callEventTypes"; import type {GroupCall} from "./GroupCall"; import type {RoomMember} from "../../room/members/RoomMember"; @@ -60,8 +59,8 @@ export class Member { private readonly logItem: ILogItem, ) {} - get remoteTracks(): Track[] { - return this.peerCall?.remoteTracks ?? []; + get remoteMedia(): RemoteMedia | undefined { + return this.peerCall?.remoteMedia; } get isConnected(): boolean { diff --git a/src/platform/types/WebRTC.ts b/src/platform/types/WebRTC.ts index a0512163..edb26c0a 100644 --- a/src/platform/types/WebRTC.ts +++ b/src/platform/types/WebRTC.ts @@ -18,7 +18,7 @@ import {Track, Stream} from "./MediaDevices"; import {SDPStreamMetadataPurpose} from "../../matrix/calls/callEventTypes"; export interface WebRTC { - createPeerConnection(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize): PeerConnection; + createPeerConnection(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize: number): PeerConnection; } export interface StreamSender { @@ -41,7 +41,7 @@ export interface TrackReceiver { export interface TrackSender extends TrackReceiver { /** replaces the track if possible without renegotiation. Can throw. */ - replaceTrack(track: Track): Promise; + replaceTrack(track: Track | undefined): Promise; /** make any needed adjustments to the sender or transceiver settings * depending on the purpose, after adding the track to the connection */ prepareForPurpose(purpose: SDPStreamMetadataPurpose): void; @@ -61,8 +61,8 @@ export interface PeerConnection { get iceGatheringState(): RTCIceGatheringState; get signalingState(): RTCSignalingState; get localDescription(): RTCSessionDescription | undefined; - get localStreams(): ReadonlyArray; - get remoteStreams(): ReadonlyArray; + get localStreams(): ReadonlyMap; + get remoteStreams(): ReadonlyMap; createOffer(): Promise; createAnswer(): Promise; setLocalDescription(description?: RTCSessionDescriptionInit): Promise; diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index 52ca132b..3723162a 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -72,8 +72,8 @@ export class MediaDevicesWrapper implements IMediaDevices { export class StreamWrapper implements Stream { - public audioTrack: AudioTrackWrapper | undefined; - public videoTrack: TrackWrapper | undefined; + public audioTrack: AudioTrackWrapper | undefined = undefined; + public videoTrack: TrackWrapper | undefined = undefined; constructor(public readonly stream: MediaStream) { for (const track of stream.getTracks()) { @@ -91,13 +91,13 @@ export class StreamWrapper implements Stream { if (track.kind === "video") { if (!this.videoTrack || track.id !== this.videoTrack.track.id) { this.videoTrack = new TrackWrapper(track, this.stream); - return this.videoTrack; } + return this.videoTrack; } else if (track.kind === "audio") { if (!this.audioTrack || track.id !== this.audioTrack.track.id) { this.audioTrack = new AudioTrackWrapper(track, this.stream); - return this.audioTrack; } + return this.audioTrack; } } } diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 6760182d..672c14d4 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -62,10 +62,10 @@ export class DOMStreamSender implements StreamSender { if (transceiver && sender.track) { const trackWrapper = this.stream.update(sender.track); if (trackWrapper) { - if (trackWrapper.kind === TrackKind.Video) { + if (trackWrapper.kind === TrackKind.Video && (!this.videoSender || this.videoSender.track.id !== trackWrapper.id)) { this.videoSender = new DOMTrackSender(trackWrapper, transceiver); return this.videoSender; - } else { + } else if (trackWrapper.kind === TrackKind.Audio && (!this.audioSender || this.audioSender.track.id !== trackWrapper.id)) { this.audioSender = new DOMTrackSender(trackWrapper, transceiver); return this.audioSender; } @@ -105,20 +105,20 @@ export class DOMTrackSenderOrReceiver implements TrackReceiver { ) {} get enabled(): boolean { - return this.transceiver.currentDirection === "sendrecv" || - this.transceiver.currentDirection === this.exclusiveValue; + return this.transceiver.direction === "sendrecv" || + this.transceiver.direction === this.exclusiveValue; } enable(enabled: boolean) { if (enabled !== this.enabled) { if (enabled) { - if (this.transceiver.currentDirection === "inactive") { + if (this.transceiver.direction === "inactive") { this.transceiver.direction = this.exclusiveValue; } else { this.transceiver.direction = "sendrecv"; } } else { - if (this.transceiver.currentDirection === "sendrecv") { + if (this.transceiver.direction === "sendrecv") { this.transceiver.direction = this.excludedValue; } else { this.transceiver.direction = "inactive"; @@ -145,7 +145,7 @@ export class DOMTrackSender extends DOMTrackSenderOrReceiver { super(track, transceiver, "sendonly", "recvonly"); } /** replaces the track if possible without renegotiation. Can throw. */ - replaceTrack(track: Track): Promise { + replaceTrack(track: Track | undefined): Promise { return this.transceiver.sender.replaceTrack(track ? (track as TrackWrapper).track : null); } @@ -192,8 +192,8 @@ export class DOMTrackSender extends DOMTrackSenderOrReceiver { class DOMPeerConnection implements PeerConnection { private readonly peerConnection: RTCPeerConnection; private readonly handler: PeerConnectionHandler; - public readonly localStreams: DOMStreamSender[]; - public readonly remoteStreams: DOMStreamReceiver[]; + public readonly localStreams: Map = new Map(); + public readonly remoteStreams: Map = new Map(); constructor(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize) { this.handler = handler; @@ -238,10 +238,11 @@ class DOMPeerConnection implements PeerConnection { throw new Error("Not a TrackWrapper"); } const sender = this.peerConnection.addTrack(track.track, track.stream); - let streamSender: DOMStreamSender | undefined = this.localStreams.find(s => s.stream.id === track.stream.id); + let streamSender = this.localStreams.get(track.stream.id); if (!streamSender) { + // TODO: reuse existing stream wrapper here? streamSender = new DOMStreamSender(new StreamWrapper(track.stream)); - this.localStreams.push(streamSender); + this.localStreams.set(track.stream.id, streamSender); } const trackSender = streamSender.update(this.peerConnection.getTransceivers(), sender); return trackSender; @@ -307,7 +308,7 @@ class DOMPeerConnection implements PeerConnection { dispose(): void { this.deregisterHandler(); - for (const r of this.remoteStreams) { + for (const r of this.remoteStreams.values()) { r.stream.dispose(); } } @@ -328,23 +329,21 @@ class DOMPeerConnection implements PeerConnection { } onRemoteStreamEmpty = (stream: RemoteStreamWrapper): void => { - const idx = this.remoteStreams.findIndex(r => r.stream === stream); - if (idx !== -1) { - this.remoteStreams.splice(idx, 1); + if (this.remoteStreams.delete(stream.id)) { this.handler.onRemoteStreamRemoved(stream); } } private handleRemoteTrack(evt: RTCTrackEvent) { - if (evt.streams.length !== 0) { + if (evt.streams.length !== 1) { throw new Error("track in multiple streams is not supported"); } const stream = evt.streams[0]; const transceivers = this.peerConnection.getTransceivers(); - let streamReceiver: DOMStreamReceiver | undefined = this.remoteStreams.find(r => r.stream.id === stream.id); + let streamReceiver: DOMStreamReceiver | undefined = this.remoteStreams.get(stream.id); if (!streamReceiver) { streamReceiver = new DOMStreamReceiver(new RemoteStreamWrapper(stream, this.onRemoteStreamEmpty)); - this.remoteStreams.push(streamReceiver); + this.remoteStreams.set(stream.id, streamReceiver); } const trackReceiver = streamReceiver.update(evt); if (trackReceiver) { diff --git a/src/platform/web/ui/session/room/CallView.ts b/src/platform/web/ui/session/room/CallView.ts index f8386116..115e6035 100644 --- a/src/platform/web/ui/session/room/CallView.ts +++ b/src/platform/web/ui/session/room/CallView.ts @@ -16,14 +16,14 @@ limitations under the License. import {TemplateView, TemplateBuilder} from "../../general/TemplateView"; import {ListView} from "../../general/ListView"; -import {Track, TrackType} from "../../../../types/MediaDevices"; -import type {TrackWrapper} from "../../../dom/MediaDevices"; +import {Stream} from "../../../../types/MediaDevices"; +import type {StreamWrapper} from "../../../dom/MediaDevices"; import type {CallViewModel, CallMemberViewModel} from "../../../../../domain/session/room/CallViewModel"; -function bindVideoTracks(t: TemplateBuilder, video: HTMLVideoElement, propSelector: (vm: T) => Track[]) { - t.mapSideEffect(propSelector, tracks => { - if (tracks.length) { - video.srcObject = (tracks[0] as TrackWrapper).stream; +function bindVideoTracks(t: TemplateBuilder, video: HTMLVideoElement, propSelector: (vm: T) => Stream | undefined) { + t.mapSideEffect(propSelector, stream => { + if (stream) { + video.srcObject = (stream as StreamWrapper).stream; } }); return video; @@ -33,7 +33,7 @@ export class CallView extends TemplateView { render(t: TemplateBuilder, vm: CallViewModel): HTMLElement { return t.div({class: "CallView"}, [ t.p(vm => `Call ${vm.name} (${vm.id})`), - t.div({class: "CallView_me"}, bindVideoTracks(t, t.video({autoplay: true, width: 240}), vm => vm.localTracks)), + t.div({class: "CallView_me"}, bindVideoTracks(t, t.video({autoplay: true, width: 240}), vm => vm.localStream)), t.view(new ListView({list: vm.memberViewModels}, vm => new MemberView(vm))), t.div({class: "buttons"}, [ t.button({onClick: () => vm.leave()}, "Leave") @@ -44,6 +44,6 @@ export class CallView extends TemplateView { class MemberView extends TemplateView { render(t: TemplateBuilder, vm: CallMemberViewModel) { - return bindVideoTracks(t, t.video({autoplay: true, width: 360}), vm => vm.tracks); + return bindVideoTracks(t, t.video({autoplay: true, width: 360}), vm => vm.stream); } } diff --git a/yarn.lock b/yarn.lock index 0eb74b87..6405fe4d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1485,10 +1485,10 @@ type-fest@^0.20.2: resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.20.2.tgz#1bf207f4b28f91583666cb5fbd327887301cd5f4" integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ== -typescript@^4.3.5: - version "4.6.2" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.6.2.tgz#fe12d2727b708f4eef40f51598b3398baa9611d4" - integrity sha512-HM/hFigTBHZhLXshn9sN37H085+hQGeJHJ/X7LpBWLID/fbc2acUMfU+lGD98X81sKP+pFa9f0DZmCwB9GnbAg== +typescript@^4.4: + version "4.6.3" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.6.3.tgz#eefeafa6afdd31d725584c67a0eaba80f6fc6c6c" + integrity sha512-yNIatDa5iaofVozS/uQJEl3JRWLKKGJKh6Yaiv0GLGSuhpFJe7P3SbHZ8/yjAHRQwKRoA6YZqlfjXWmVzoVSMw== typeson-registry@^1.0.0-alpha.20: version "1.0.0-alpha.39" From 2d00d10161b52e83e56b9a331c0e31adede3d1ec Mon Sep 17 00:00:00 2001 From: Robert Long Date: Wed, 13 Apr 2022 13:08:33 -0700 Subject: [PATCH 068/323] Export LocalMedia --- src/lib.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib.ts b/src/lib.ts index e2da5e16..a49eacbc 100644 --- a/src/lib.ts +++ b/src/lib.ts @@ -71,6 +71,7 @@ export {AvatarView} from "./platform/web/ui/AvatarView.js"; export {RoomType} from "./matrix/room/common"; export {EventEmitter} from "./utils/EventEmitter"; export {Disposables} from "./utils/Disposables"; +export {LocalMedia} from "./matrix/calls/LocalMedia"; // these should eventually be moved to another library export { ObservableArray, From 55097e41542ad9833ff6058b0aa73b5eb106a2ed Mon Sep 17 00:00:00 2001 From: Robert Long Date: Wed, 13 Apr 2022 13:08:47 -0700 Subject: [PATCH 069/323] Add intent to CallHandler --- src/matrix/calls/CallHandler.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 31829396..77492e62 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -107,11 +107,11 @@ export class CallHandler { }); } - async createCall(roomId: string, localMedia: LocalMedia, name: string): Promise { + async createCall(roomId: string, localMedia: LocalMedia, name: string, intent: CallIntent = CallIntent.Ring): Promise { const logItem = this.options.logger.child({l: "call", incoming: false}); const call = new GroupCall(makeId("conf-"), true, { "m.name": name, - "m.intent": CallIntent.Ring + "m.intent": intent }, roomId, this.groupCallOptions, logItem); this._calls.set(call.id, call); From ff856d843ce7dedd2677e4def1c8347ac1c36568 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 14 Apr 2022 13:44:11 +0200 Subject: [PATCH 070/323] ensure all member streams are cloned so we can stop them without affecting the main one also, only stop them when disconnecting from the member, rather then when the peer call ends, as we might want to retry connecting to the peer with the same stream. --- src/matrix/calls/LocalMedia.ts | 4 ++-- src/matrix/calls/PeerCall.ts | 1 - src/matrix/calls/group/GroupCall.ts | 7 ++++--- src/matrix/calls/group/Member.ts | 1 + 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 6622f641..6564adac 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -37,10 +37,10 @@ export class LocalMedia { return new LocalMedia(this.userMedia, this.screenShare, options); } - clone() { + clone(): LocalMedia { return new LocalMedia(this.userMedia?.clone(), this.screenShare?.clone(), this.dataChannelOptions); } - + dispose() { this.userMedia?.audioTrack?.stop(); this.userMedia?.videoTrack?.stop(); diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 78ffc818..5b6e2ca8 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -814,7 +814,6 @@ export class PeerCall implements IDisposable { this.hangupParty = hangupParty; this._hangupReason = hangupReason; this.setState(CallState.Ended, log); - this.localMedia?.dispose(); this.localMedia = undefined; if (this.peerConnection && this.peerConnection.signalingState !== 'closed') { diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 47d87e5e..d077f27c 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -118,7 +118,7 @@ export class GroupCall extends EventEmitter<{change: never}> { this.emitChange(); // send invite to all members that are < my userId for (const [,member] of this._members) { - member.connect(this._localMedia); + member.connect(this._localMedia!.clone()); } }); } @@ -218,6 +218,7 @@ export class GroupCall extends EventEmitter<{change: never}> { ); this._members.add(memberKey, member); if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { + // Safari can't send a MediaStream to multiple sources, so clone it member.connect(this._localMedia!.clone()); } } @@ -267,11 +268,11 @@ export class GroupCall extends EventEmitter<{change: never}> { private removeOwnDevice(log: ILogItem) { if (this._state === GroupCallState.Joined) { log.set("leave_own", true); - this._localMedia?.dispose(); - this._localMedia = undefined; for (const [,member] of this._members) { member.disconnect(); } + this._localMedia?.dispose(); + this._localMedia = undefined; this._state = GroupCallState.Created; this.emitChange(); } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 110aba6d..1305fcf1 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -104,6 +104,7 @@ export class Member { this.peerCall?.close(undefined, log); this.peerCall?.dispose(); this.peerCall = undefined; + this.localMedia?.dispose(); this.localMedia = undefined; }); } From 021b8cdcdc8a8f41edd0bdce008c6bdc859e28d6 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 14 Apr 2022 13:45:21 +0200 Subject: [PATCH 071/323] send hangup when leaving the call but not when somebody else leaves the call through a member event --- src/matrix/calls/group/GroupCall.ts | 4 ++-- src/matrix/calls/group/Member.ts | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index d077f27c..892ed67c 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -269,7 +269,7 @@ export class GroupCall extends EventEmitter<{change: never}> { if (this._state === GroupCallState.Joined) { log.set("leave_own", true); for (const [,member] of this._members) { - member.disconnect(); + member.disconnect(true); } this._localMedia?.dispose(); this._localMedia = undefined; @@ -286,7 +286,7 @@ export class GroupCall extends EventEmitter<{change: never}> { if (member) { log.set("leave", true); this._members.remove(memberKey); - member.disconnect(); + member.disconnect(false); } this.emitChange(); }); diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 1305fcf1..06693030 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -99,9 +99,13 @@ export class Member { } /** @internal */ - disconnect() { + disconnect(hangup: boolean) { this.logItem.wrap("disconnect", log => { - this.peerCall?.close(undefined, log); + if (hangup) { + this.peerCall?.hangup(CallErrorCode.UserHangup); + } else { + this.peerCall?.close(undefined, log); + } this.peerCall?.dispose(); this.peerCall = undefined; this.localMedia?.dispose(); From 382fba88bdf757887ac978712fc8004788006bfb Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 14 Apr 2022 23:19:44 +0200 Subject: [PATCH 072/323] WIP for muting --- src/domain/session/room/CallViewModel.ts | 6 + src/matrix/calls/LocalMedia.ts | 14 +- src/matrix/calls/PeerCall.ts | 256 +++++++++---------- src/matrix/calls/TODO.md | 2 +- src/matrix/calls/group/GroupCall.ts | 11 + src/matrix/calls/group/Member.ts | 8 + src/platform/types/MediaDevices.ts | 3 + src/platform/web/dom/MediaDevices.ts | 34 ++- src/platform/web/dom/WebRTC.ts | 8 + src/platform/web/index.html | 2 +- src/platform/web/ui/general/TemplateView.ts | 6 +- src/platform/web/ui/session/room/CallView.ts | 23 +- 12 files changed, 210 insertions(+), 163 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index bc7b54ff..05245cd3 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -55,6 +55,12 @@ export class CallViewModel extends ViewModel { this.call.leave(); } } + + async toggleVideo() { + const localMedia = this.call.localMedia!; + const toggledMedia = localMedia.withMuted(localMedia.microphoneMuted, !localMedia.cameraMuted); + await this.call.setMedia(toggledMedia); + } } type MemberOptions = BaseOptions & {member: Member}; diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 6564adac..7dbc9e17 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -21,24 +21,30 @@ import {SDPStreamMetadata} from "./callEventTypes"; export class LocalMedia { constructor( public readonly userMedia?: Stream, + public readonly microphoneMuted: boolean = false, + public readonly cameraMuted: boolean = false, public readonly screenShare?: Stream, public readonly dataChannelOptions?: RTCDataChannelInit, ) {} + withMuted(microphone: boolean, camera: boolean) { + return new LocalMedia(this.userMedia, microphone, camera, this.screenShare, this.dataChannelOptions); + } + withUserMedia(stream: Stream) { - return new LocalMedia(stream, this.screenShare, this.dataChannelOptions); + return new LocalMedia(stream, this.microphoneMuted, this.cameraMuted, this.screenShare, this.dataChannelOptions); } withScreenShare(stream: Stream) { - return new LocalMedia(this.userMedia, stream, this.dataChannelOptions); + return new LocalMedia(this.userMedia, this.microphoneMuted, this.cameraMuted, stream, this.dataChannelOptions); } withDataChannel(options: RTCDataChannelInit): LocalMedia { - return new LocalMedia(this.userMedia, this.screenShare, options); + return new LocalMedia(this.userMedia, this.microphoneMuted, this.cameraMuted, this.screenShare, options); } clone(): LocalMedia { - return new LocalMedia(this.userMedia?.clone(), this.screenShare?.clone(), this.dataChannelOptions); + return new LocalMedia(this.userMedia?.clone(), this.microphoneMuted, this.cameraMuted, this.screenShare?.clone(), this.dataChannelOptions); } dispose() { diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 5b6e2ca8..b450ac3f 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -31,6 +31,7 @@ import { SDPStreamMetadataKey, SDPStreamMetadataPurpose, EventType, + CallErrorCode, } from "./callEventTypes"; import type { MCallBase, @@ -66,6 +67,7 @@ export class PeerCall implements IDisposable { private readonly peerConnection: PeerConnection; private _state = CallState.Fledgling; private direction: CallDirection; + // we don't own localMedia and should hence not call dispose on it from here private localMedia?: LocalMedia; private seq: number = 0; // A queue for candidates waiting to go out. @@ -151,7 +153,6 @@ export class PeerCall implements IDisposable { get hangupReason(): CallErrorCode | undefined { return this._hangupReason; } - // we should keep an object with streams by purpose ... e.g. RemoteMedia? get remoteMedia(): Readonly { return this._remoteMedia; } @@ -163,7 +164,7 @@ export class PeerCall implements IDisposable { } this.direction = CallDirection.Outbound; this.setState(CallState.CreateOffer, log); - this.setMedia(localMedia); + this.updateLocalMedia(localMedia, log); if (this.localMedia?.dataChannelOptions) { this._dataChannel = this.peerConnection.createDataChannel(this.localMedia.dataChannelOptions); } @@ -179,7 +180,7 @@ export class PeerCall implements IDisposable { return; } this.setState(CallState.CreateAnswer, log); - this.setMedia(localMedia, log); + this.updateLocalMedia(localMedia, log); let myAnswer: RTCSessionDescriptionInit; try { myAnswer = await this.peerConnection.createAnswer(); @@ -208,48 +209,32 @@ export class PeerCall implements IDisposable { }); } - setMedia(localMedia: LocalMedia, logItem: ILogItem = this.logItem): Promise { - return logItem.wrap("setMedia", async log => { - const oldMedia = this.localMedia; - this.localMedia = localMedia; - const applyStream = (oldStream: Stream | undefined, stream: Stream | undefined, logLabel: string) => { - const streamSender = oldMedia ? this.peerConnection.localStreams.get(oldStream!.id) : undefined; + setMedia(localMedia: LocalMedia): Promise { + return this.logItem.wrap("setMedia", async log => { + log.set("userMedia_audio", !!localMedia.userMedia?.audioTrack); + log.set("userMedia_audio_muted", localMedia.microphoneMuted); + log.set("userMedia_video", !!localMedia.userMedia?.videoTrack); + log.set("userMedia_video_muted", localMedia.cameraMuted); + log.set("screenShare_video", !!localMedia.screenShare?.videoTrack); + log.set("datachannel", !!localMedia.dataChannelOptions); - const applyTrack = (oldTrack: Track | undefined, sender: TrackSender | undefined, track: Track | undefined) => { - if (track) { - if (oldTrack && sender) { - log.wrap(`replacing ${logLabel} ${track.kind} track`, log => { - sender.replaceTrack(track); - }); - } else { - log.wrap(`adding ${logLabel} ${track.kind} track`, log => { - this.peerConnection.addTrack(track); - }); - } - } else { - if (sender) { - log.wrap(`replacing ${logLabel} ${sender.track.kind} track`, log => { - this.peerConnection.removeTrack(sender); - }); - } - } + const oldMetaData = this.getSDPMetadata(); + const willRenegotiate = await this.updateLocalMedia(localMedia, log); + if (!willRenegotiate) { + const newMetaData = this.getSDPMetadata(); + if (JSON.stringify(oldMetaData) !== JSON.stringify(newMetaData)) { + const content: MCallSDPStreamMetadataChanged = { + call_id: this.callId, + version: 1, + seq: this.seq++, + [SDPStreamMetadataKey]: newMetaData + }; + await this.sendSignallingMessage({type: EventType.SDPStreamMetadataChanged, content}, log); } - - applyTrack(oldStream?.audioTrack, streamSender?.audioSender, stream?.audioTrack); - applyTrack(oldStream?.videoTrack, streamSender?.videoSender, stream?.videoTrack); } - - applyStream(oldMedia?.userMedia, localMedia?.userMedia, "userMedia"); - applyStream(oldMedia?.screenShare, localMedia?.screenShare, "screenShare"); - // TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method }); } - /** group calls would handle reject at the group call level, not at the peer call level */ - async reject() { - - } - hangup(errorCode: CallErrorCode): Promise { return this.logItem.wrap("hangup", log => { return this._hangup(errorCode, log); @@ -280,7 +265,14 @@ export class PeerCall implements IDisposable { case EventType.Candidates: await this.handleRemoteIceCandidates(message.content, partyId, log); break; + case EventType.SDPStreamMetadataChanged: + case EventType.SDPStreamMetadataChangedPrefix: + this.updateRemoteSDPStreamMetadata(message.content[SDPStreamMetadataKey], log); + break; case EventType.Hangup: + // TODO: this is a bit hacky, double check its what we need + this.terminate(CallParty.Remote, message.content.reason ?? CallErrorCode.UserHangup, log); + break; default: log.log(`Unknown event type for call: ${message.type}`); break; @@ -444,12 +436,12 @@ export class PeerCall implements IDisposable { // According to previous comments in this file, firefox at some point did not // add streams until media started arriving on them. Testing latest firefox // (81 at time of writing), this is no longer a problem, so let's do it the correct way. - // if (this.peerConnection.remoteTracks.length === 0) { - // await log.wrap(`Call no remote stream or no tracks after setting remote description!`, async log => { - // return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); - // }); - // return; - // } + if (this.peerConnection.remoteStreams.size === 0) { + await log.wrap(`Call no remote stream or no tracks after setting remote description!`, async log => { + return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); + }); + return; + } this.setState(CallState.Ringing, log); @@ -719,7 +711,7 @@ export class PeerCall implements IDisposable { // this will accumulate all updates into one object, so we still have the old stream info when we change stream id this.remoteSDPStreamMetadata = recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); this.updateRemoteMedia(log); - // TODO: apply muting + } private async addBufferedIceCandidates(log: ILogItem): Promise { @@ -828,10 +820,9 @@ export class PeerCall implements IDisposable { const streamSender = this.peerConnection.localStreams.get(streamId); metadata[streamId] = { purpose: SDPStreamMetadataPurpose.Usermedia, - audio_muted: !(streamSender?.audioSender?.enabled), - video_muted: !(streamSender?.videoSender?.enabled), + audio_muted: !(streamSender?.audioSender?.enabled && streamSender?.audioSender?.track?.enabled), + video_muted: !(streamSender?.videoSender?.enabled && streamSender?.videoSender?.track?.enabled), }; - console.log("video_muted", streamSender?.videoSender?.enabled, streamSender?.videoSender?.transceiver?.direction, streamSender?.videoSender?.transceiver?.currentDirection, JSON.stringify(metadata)); } if (this.localMedia?.screenShare) { const streamId = this.localMedia.screenShare.id; @@ -851,6 +842,8 @@ export class PeerCall implements IDisposable { if (metaData) { if (metaData.purpose === SDPStreamMetadataPurpose.Usermedia) { this._remoteMedia.userMedia = streamReceiver.stream; + streamReceiver.audioReceiver?.enable(!metaData.audio_muted); + streamReceiver.videoReceiver?.enable(!metaData.video_muted); } else if (metaData.purpose === SDPStreamMetadataPurpose.Screenshare) { this._remoteMedia.screenShare = streamReceiver.stream; } @@ -860,6 +853,77 @@ export class PeerCall implements IDisposable { this.options.emitUpdate(this, undefined); } + private updateLocalMedia(localMedia: LocalMedia, logItem: ILogItem): Promise { + return logItem.wrap("updateLocalMedia", async log => { + let willRenegotiate = false; + const oldMedia = this.localMedia; + this.localMedia = localMedia; + const applyStream = async (oldStream: Stream | undefined, stream: Stream | undefined, oldMuteSettings: LocalMedia | undefined, mutedSettings: LocalMedia | undefined, logLabel: string) => { + const streamSender = oldStream ? this.peerConnection.localStreams.get(oldStream.id) : undefined; + + const applyTrack = async (oldTrack: Track | undefined, sender: TrackSender | undefined, track: Track | undefined, wasMuted: boolean | undefined, muted: boolean | undefined) => { + const changed = (!track && oldTrack) || + (track && !oldTrack) || + (track && oldTrack && !track.equals(oldTrack)); + if (changed) { + if (track) { + if (oldTrack && sender && !track.equals(oldTrack)) { + try { + await log.wrap(`replacing ${logLabel} ${track.kind} track`, log => { + return sender.replaceTrack(track); + }); + } catch (err) { + // can't replace the track without renegotiating + log.wrap(`adding and removing ${logLabel} ${track.kind} track`, log => { + this.peerConnection.removeTrack(sender); + this.peerConnection.addTrack(track); + willRenegotiate = true; + }); + } + } else { + log.wrap(`adding ${logLabel} ${track.kind} track`, log => { + this.peerConnection.addTrack(track); + willRenegotiate = true; + }); + } + } else { + if (sender) { + // this will be used for muting, do we really want to trigger renegotiation here? + // we want to disable the sender, but also remove the track as we don't want to keep + // using the webcam if we don't need to + log.wrap(`removing ${logLabel} ${sender.track.kind} track`, log => { + sender.track.enabled = false; + this.peerConnection.removeTrack(sender); + willRenegotiate = true; + }); + } + } + } else if (track) { + console.log({muted, wasMuted, wasCameraMuted: oldMedia?.cameraMuted}); + if (sender && muted !== wasMuted) { + // TODO: why does unmuting not work? wasMuted is false + log.wrap(`${logLabel} ${track.kind} ${muted ? "muting" : "unmuting"}`, log => { + sender.track.enabled = !muted; + sender.enable(!muted); + willRenegotiate = true; + }); + } else { + log.log(`${logLabel} ${track.kind} track hasn't changed`); + } + } + } + + await applyTrack(oldStream?.audioTrack, streamSender?.audioSender, stream?.audioTrack, oldMuteSettings?.microphoneMuted, mutedSettings?.microphoneMuted); + await applyTrack(oldStream?.videoTrack, streamSender?.videoSender, stream?.videoTrack, oldMuteSettings?.cameraMuted, mutedSettings?.cameraMuted); + } + + await applyStream(oldMedia?.userMedia, localMedia?.userMedia, oldMedia, localMedia, "userMedia"); + applyStream(oldMedia?.screenShare, localMedia?.screenShare, undefined, undefined, "screenShare"); + return willRenegotiate; + // TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method + }); + } + private async delay(timeoutMs: number): Promise { // Allow a short time for initial candidates to be gathered const timeout = this.disposables.track(this.options.createTimeout(timeoutMs)); @@ -915,100 +979,11 @@ export enum CallDirection { Outbound = 'outbound', } -export enum CallErrorCode { - /** The user chose to end the call */ - UserHangup = 'user_hangup', - - /** An error code when the local client failed to create an offer. */ - LocalOfferFailed = 'local_offer_failed', - /** - * An error code when there is no local mic/camera to use. This may be because - * the hardware isn't plugged in, or the user has explicitly denied access. - */ - NoUserMedia = 'no_user_media', - - /** - * Error code used when a call event failed to send - * because unknown devices were present in the room - */ - UnknownDevices = 'unknown_devices', - - /** - * Error code used when we fail to send the invite - * for some reason other than there being unknown devices - */ - SendInvite = 'send_invite', - - /** - * An answer could not be created - */ - CreateAnswer = 'create_answer', - - /** - * Error code used when we fail to send the answer - * for some reason other than there being unknown devices - */ - SendAnswer = 'send_answer', - - /** - * The session description from the other side could not be set - */ - SetRemoteDescription = 'set_remote_description', - - /** - * The session description from this side could not be set - */ - SetLocalDescription = 'set_local_description', - - /** - * A different device answered the call - */ - AnsweredElsewhere = 'answered_elsewhere', - - /** - * No media connection could be established to the other party - */ - IceFailed = 'ice_failed', - - /** - * The invite timed out whilst waiting for an answer - */ - InviteTimeout = 'invite_timeout', - - /** - * The call was replaced by another call - */ - Replaced = 'replaced', - - /** - * Signalling for the call could not be sent (other than the initial invite) - */ - SignallingFailed = 'signalling_timeout', - - /** - * The remote party is busy - */ - UserBusy = 'user_busy', - - /** - * We transferred the call off to somewhere else - */ - Transfered = 'transferred', - - /** - * A call from the same user was found with a new session id - */ - NewSession = 'new_session', -} - /** * The version field that we set in m.call.* events */ const VOIP_PROTO_VERSION = 1; -/** The fallback ICE server to use for STUN or TURN protocols. */ -const FALLBACK_ICE_SERVER = 'stun:turn.matrix.org'; - /** The length of time a call can be ringing for. */ const CALL_TIMEOUT_MS = 60000; @@ -1029,7 +1004,10 @@ export function handlesEventType(eventType: string): boolean { return eventType === EventType.Invite || eventType === EventType.Candidates || eventType === EventType.Answer || - eventType === EventType.Hangup; + eventType === EventType.Hangup || + eventType === EventType.SDPStreamMetadataChanged || + eventType === EventType.SDPStreamMetadataChangedPrefix || + eventType === EventType.Negotiate; } export function tests() { diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index 8c041fcc..a0a64752 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -15,7 +15,7 @@ - call peers are essentially identified by (userid, deviceid, sessionid). If see a new session id, we first disconnect from the current member so we're ready to connect with a clean slate again (in a member event, also in to_device? no harm I suppose, given olm encryption ensures you can't spoof the deviceid). - implement to_device messages arriving before m.call(.member) state event - implement muting tracks with m.call.sdp_stream_metadata_changed - - implement cloning the localMedia so it works in safari? + - DONE: implement cloning the localMedia so it works in safari? - DONE: implement 3 retries per peer - reeable crypto & implement fetching olm keys before sending encrypted signalling message - local echo for join/leave buttons? diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 892ed67c..9fc8d1ee 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -123,6 +123,17 @@ export class GroupCall extends EventEmitter<{change: never}> { }); } + async setMedia(localMedia: LocalMedia): Promise { + if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { + const oldMedia = this._localMedia; + this._localMedia = localMedia; + await Promise.all(Array.from(this._members.values()).map(m => { + return m.setMedia(localMedia!.clone()); + })); + oldMedia?.dispose(); + } + } + get hasJoined() { return this._state === GroupCallState.Joining || this._state === GroupCallState.Joined; } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 06693030..e821e3da 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -182,6 +182,14 @@ export class Member { } } + /** @internal */ + async setMedia(localMedia: LocalMedia): Promise { + const oldMedia = this.localMedia; + this.localMedia = localMedia; + await this.peerCall?.setMedia(localMedia); + oldMedia?.dispose(); + } + private _createPeerCall(callId: string): PeerCall { return new PeerCall(callId, Object.assign({}, this.options, { emitUpdate: this.emitUpdate, diff --git a/src/platform/types/MediaDevices.ts b/src/platform/types/MediaDevices.ts index d0edbbea..85f64f95 100644 --- a/src/platform/types/MediaDevices.ts +++ b/src/platform/types/MediaDevices.ts @@ -39,6 +39,9 @@ export interface Track { readonly label: string; readonly id: string; readonly settings: MediaTrackSettings; + get enabled(): boolean; + set enabled(value: boolean); + equals(track: Track): boolean; stop(): void; } diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index 3723162a..a04fca91 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -75,27 +75,37 @@ export class StreamWrapper implements Stream { public audioTrack: AudioTrackWrapper | undefined = undefined; public videoTrack: TrackWrapper | undefined = undefined; - constructor(public readonly stream: MediaStream) { - for (const track of stream.getTracks()) { - this.update(track); + constructor(public readonly stream: MediaStream, clonedTracks?: {audioTrack?: AudioTrackWrapper, videoTrack?: TrackWrapper}) { + if (clonedTracks) { + this.audioTrack = clonedTracks.audioTrack; + this.videoTrack = clonedTracks.videoTrack; + } else { + for (const track of stream.getTracks()) { + this.update(track); + } } } get id(): string { return this.stream.id; } clone(): Stream { - return new StreamWrapper(this.stream.clone()); + const clonedStream = this.stream.clone(); + const clonedTracks = { + audioTrack: this.audioTrack ? new AudioTrackWrapper(clonedStream.getAudioTracks()[0], clonedStream, this.audioTrack.id): undefined, + videoTrack: this.videoTrack ? new TrackWrapper(clonedStream.getVideoTracks()[0], clonedStream, this.videoTrack.id): undefined, + }; + return new StreamWrapper(clonedStream, clonedTracks); } update(track: MediaStreamTrack): TrackWrapper | undefined { if (track.kind === "video") { if (!this.videoTrack || track.id !== this.videoTrack.track.id) { - this.videoTrack = new TrackWrapper(track, this.stream); + this.videoTrack = new TrackWrapper(track, this.stream, track.id); } return this.videoTrack; } else if (track.kind === "audio") { if (!this.audioTrack || track.id !== this.audioTrack.track.id) { - this.audioTrack = new AudioTrackWrapper(track, this.stream); + this.audioTrack = new AudioTrackWrapper(track, this.stream, track.id); } return this.audioTrack; } @@ -105,14 +115,18 @@ export class StreamWrapper implements Stream { export class TrackWrapper implements Track { constructor( public readonly track: MediaStreamTrack, - public readonly stream: MediaStream + public readonly stream: MediaStream, + public readonly originalId: string, ) {} get kind(): TrackKind { return this.track.kind as TrackKind; } get label(): string { return this.track.label; } get id(): string { return this.track.id; } get settings(): MediaTrackSettings { return this.track.getSettings(); } - + get enabled(): boolean { return this.track.enabled; } + set enabled(enabled: boolean) { this.track.enabled = enabled; } + // test equality across clones + equals(track: Track): boolean { return (track as TrackWrapper).originalId === this.originalId; } stop() { this.track.stop(); } } @@ -126,8 +140,8 @@ export class AudioTrackWrapper extends TrackWrapper { private volumeLooperTimeout: number; private speakingVolumeSamples: number[]; - constructor(track: MediaStreamTrack, stream: MediaStream) { - super(track, stream); + constructor(track: MediaStreamTrack, stream: MediaStream, originalId: string) { + super(track, stream, originalId); this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); this.initVolumeMeasuring(); this.measureVolumeActivity(true); diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 672c14d4..22096699 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -109,8 +109,16 @@ export class DOMTrackSenderOrReceiver implements TrackReceiver { this.transceiver.direction === this.exclusiveValue; } + enableWithoutRenegotiation(enabled: boolean) { + this.track.track.enabled = enabled; + } + enable(enabled: boolean) { if (enabled !== this.enabled) { + // do this first, so we stop sending track data immediately. + // this will still consume bandwidth though, so also disable the transceiver, + // which will trigger a renegotiation though. + this.enableWithoutRenegotiation(enabled); if (enabled) { if (this.transceiver.direction === "inactive") { this.transceiver.direction = this.exclusiveValue; diff --git a/src/platform/web/index.html b/src/platform/web/index.html index 5950d89f..064c61a1 100644 --- a/src/platform/web/index.html +++ b/src/platform/web/index.html @@ -11,7 +11,7 @@ - + + + diff --git a/scripts/logviewer/main.js b/scripts/logviewer/main.js index 3ae860b2..e552a094 100644 --- a/scripts/logviewer/main.js +++ b/scripts/logviewer/main.js @@ -164,7 +164,11 @@ function getRootItemHeader(prevItem, item) { async function loadFile() { const file = await openFile(); document.getElementById("filename").innerText = file.name; - const json = await readFileAsText(file); + await loadBlob(file); +} + +export async function loadBlob(blob) { + const json = await readFileAsText(blob); const logs = JSON.parse(json); logs.items.sort((a, b) => itemStart(a) - itemStart(b)); rootItem = {c: logs.items}; @@ -181,6 +185,7 @@ async function loadFile() { return fragment; }, document.createDocumentFragment()); main.replaceChildren(fragment); + main.scrollTop = main.scrollHeight; } // TODO: make this use processRecursively From d85f93fb16aa505499e4d9b0bef0936edb4b2970 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 10 May 2022 11:02:39 +0200 Subject: [PATCH 123/323] allow opening the logs straight in the log viewer from settings --- .../session/settings/SettingsViewModel.js | 7 ++++- .../web/ui/session/settings/SettingsView.js | 29 ++++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/domain/session/settings/SettingsViewModel.js b/src/domain/session/settings/SettingsViewModel.js index e7990844..d0f2e91d 100644 --- a/src/domain/session/settings/SettingsViewModel.js +++ b/src/domain/session/settings/SettingsViewModel.js @@ -136,9 +136,14 @@ export class SettingsViewModel extends ViewModel { } async exportLogs() { + const logs = await this.exportLogsBlob(); + this.platform.saveFileAs(logs, `hydrogen-logs-${this.platform.clock.now()}.json`); + } + + async exportLogsBlob() { const persister = this.logger.reporters.find(r => typeof r.export === "function"); const logExport = await persister.export(); - this.platform.saveFileAs(logExport.asBlob(), `hydrogen-logs-${this.platform.clock.now()}.json`); + return logExport.asBlob(); } async togglePushNotifications() { diff --git a/src/platform/web/ui/session/settings/SettingsView.js b/src/platform/web/ui/session/settings/SettingsView.js index 93e44307..58992d9f 100644 --- a/src/platform/web/ui/session/settings/SettingsView.js +++ b/src/platform/web/ui/session/settings/SettingsView.js @@ -98,13 +98,18 @@ export class SettingsView extends TemplateView { t.h3("Preferences"), row(t, vm.i18n`Scale down images when sending`, this._imageCompressionRange(t, vm)), ); + const logButtons = [t.button({onClick: () => vm.exportLogs()}, "Export")]; + if (import.meta.env.DEV) { + logButtons.push(t.button({onClick: () => openLogs(vm)}, "Open logs")); + } settingNodes.push( t.h3("Application"), row(t, vm.i18n`Version`, version), row(t, vm.i18n`Storage usage`, vm => `${vm.storageUsage} / ${vm.storageQuota}`), - row(t, vm.i18n`Debug logs`, t.button({onClick: () => vm.exportLogs()}, "Export")), + row(t, vm.i18n`Debug logs`, logButtons), t.p(["Debug logs contain application usage data including your username, the IDs or aliases of the rooms or groups you have visited, the usernames of other users and the names of files you send. They do not contain messages. For more information, review our ", t.a({href: "https://element.io/privacy", target: "_blank", rel: "noopener"}, "privacy policy"), "."]), + t.p([]) ); return t.main({className: "Settings middle"}, [ @@ -136,3 +141,25 @@ export class SettingsView extends TemplateView { })]; } } + +async function openLogs(vm) { + const logviewerUrl = (await import("../../../../../../scripts/logviewer/index.html?url")).default; + const win = window.open(logviewerUrl); + await new Promise(async r => { + let receivedPong = false; + const waitForPong = event => { + if (event.data.type === "pong") { + window.removeEventListener("message", waitForPong); + receivedPong = true; + r(); + } + }; + window.addEventListener("message", waitForPong); + while (!receivedPong) { + win.postMessage({type: "ping"}); + await new Promise(rr => setTimeout(rr), 100); + } + }); + const logs = await vm.exportLogsBlob(); + win.postMessage({type: "open", logs: logs.nativeBlob}); +} From c823bb125fe21d90b42bbd3e8dd6f20608590d1f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 10 May 2022 11:20:25 +0200 Subject: [PATCH 124/323] fix lint error --- .../web/ui/session/settings/SettingsView.js | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/platform/web/ui/session/settings/SettingsView.js b/src/platform/web/ui/session/settings/SettingsView.js index 58992d9f..e07aaff5 100644 --- a/src/platform/web/ui/session/settings/SettingsView.js +++ b/src/platform/web/ui/session/settings/SettingsView.js @@ -145,20 +145,23 @@ export class SettingsView extends TemplateView { async function openLogs(vm) { const logviewerUrl = (await import("../../../../../../scripts/logviewer/index.html?url")).default; const win = window.open(logviewerUrl); - await new Promise(async r => { + await new Promise((resolve, reject) => { let receivedPong = false; const waitForPong = event => { if (event.data.type === "pong") { window.removeEventListener("message", waitForPong); receivedPong = true; - r(); + resolve(); + } + }; + const sendPings = async () => { + while (!receivedPong) { + win.postMessage({type: "ping"}); + await new Promise(rr => setTimeout(rr), 100); } }; window.addEventListener("message", waitForPong); - while (!receivedPong) { - win.postMessage({type: "ping"}); - await new Promise(rr => setTimeout(rr), 100); - } + sendPings().catch(reject); }); const logs = await vm.exportLogsBlob(); win.postMessage({type: "open", logs: logs.nativeBlob}); From 1d900b518453ebb838fdba7aca2ca51ed4216856 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 10 May 2022 12:14:09 +0200 Subject: [PATCH 125/323] finish open window and poll code for logviewer --- .../web/ui/session/settings/SettingsView.js | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/platform/web/ui/session/settings/SettingsView.js b/src/platform/web/ui/session/settings/SettingsView.js index e07aaff5..09d3cb5a 100644 --- a/src/platform/web/ui/session/settings/SettingsView.js +++ b/src/platform/web/ui/session/settings/SettingsView.js @@ -146,18 +146,24 @@ async function openLogs(vm) { const logviewerUrl = (await import("../../../../../../scripts/logviewer/index.html?url")).default; const win = window.open(logviewerUrl); await new Promise((resolve, reject) => { - let receivedPong = false; + let shouldSendPings = true; + const cleanup = () => { + shouldSendPings = false; + window.removeEventListener("message", waitForPong); + }; const waitForPong = event => { if (event.data.type === "pong") { - window.removeEventListener("message", waitForPong); - receivedPong = true; + cleanup(); resolve(); } }; const sendPings = async () => { - while (!receivedPong) { + while (shouldSendPings) { win.postMessage({type: "ping"}); - await new Promise(rr => setTimeout(rr), 100); + await new Promise(rr => setTimeout(rr, 50)); + if (win.closed) { + cleanup(); + } } }; window.addEventListener("message", waitForPong); From f6ea7803f2bc567039fac8c4b9bd6db8d801b318 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 10 May 2022 18:03:15 +0200 Subject: [PATCH 126/323] move logviewer to own package --- package.json | 1 + scripts/logviewer/file.js | 51 --- scripts/logviewer/html.js | 110 ----- scripts/logviewer/index.html | 237 ---------- scripts/logviewer/main.js | 430 ------------------ .../web/ui/session/settings/SettingsView.js | 2 +- yarn.lock | 5 + 7 files changed, 7 insertions(+), 829 deletions(-) delete mode 100644 scripts/logviewer/file.js delete mode 100644 scripts/logviewer/html.js delete mode 100644 scripts/logviewer/index.html delete mode 100644 scripts/logviewer/main.js diff --git a/package.json b/package.json index b1704e43..28b726e2 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ }, "homepage": "https://github.com/vector-im/hydrogen-web/#readme", "devDependencies": { + "@matrixdotorg/structured-logviewer": "^0.0.1", "@typescript-eslint/eslint-plugin": "^4.29.2", "@typescript-eslint/parser": "^4.29.2", "acorn": "^8.6.0", diff --git a/scripts/logviewer/file.js b/scripts/logviewer/file.js deleted file mode 100644 index 64a8422b..00000000 --- a/scripts/logviewer/file.js +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright 2020 The Matrix.org Foundation C.I.C. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -export function openFile(mimeType = null) { - const input = document.createElement("input"); - input.setAttribute("type", "file"); - input.className = "hidden"; - if (mimeType) { - input.setAttribute("accept", mimeType); - } - const promise = new Promise((resolve, reject) => { - const checkFile = () => { - input.removeEventListener("change", checkFile, true); - const file = input.files[0]; - document.body.removeChild(input); - if (file) { - resolve(file); - } else { - reject(new Error("no file picked")); - } - } - input.addEventListener("change", checkFile, true); - }); - // IE11 needs the input to be attached to the document - document.body.appendChild(input); - input.click(); - return promise; -} - -export function readFileAsText(file) { - const reader = new FileReader(); - const promise = new Promise((resolve, reject) => { - reader.addEventListener("load", evt => resolve(evt.target.result)); - reader.addEventListener("error", evt => reject(evt.target.error)); - }); - reader.readAsText(file); - return promise; -} diff --git a/scripts/logviewer/html.js b/scripts/logviewer/html.js deleted file mode 100644 index a965a6ee..00000000 --- a/scripts/logviewer/html.js +++ /dev/null @@ -1,110 +0,0 @@ -/* -Copyright 2020 Bruno Windels - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// DOM helper functions - -export function isChildren(children) { - // children should be an not-object (that's the attributes), or a domnode, or an array - return typeof children !== "object" || !!children.nodeType || Array.isArray(children); -} - -export function classNames(obj, value) { - return Object.entries(obj).reduce((cn, [name, enabled]) => { - if (typeof enabled === "function") { - enabled = enabled(value); - } - if (enabled) { - return cn + (cn.length ? " " : "") + name; - } else { - return cn; - } - }, ""); -} - -export function setAttribute(el, name, value) { - if (name === "className") { - name = "class"; - } - if (value === false) { - el.removeAttribute(name); - } else { - if (value === true) { - value = name; - } - el.setAttribute(name, value); - } -} - -export function el(elementName, attributes, children) { - return elNS(HTML_NS, elementName, attributes, children); -} - -export function elNS(ns, elementName, attributes, children) { - if (attributes && isChildren(attributes)) { - children = attributes; - attributes = null; - } - - const e = document.createElementNS(ns, elementName); - - if (attributes) { - for (let [name, value] of Object.entries(attributes)) { - if (name === "className" && typeof value === "object" && value !== null) { - value = classNames(value); - } - setAttribute(e, name, value); - } - } - - if (children) { - if (!Array.isArray(children)) { - children = [children]; - } - for (let c of children) { - if (!c.nodeType) { - c = text(c); - } - e.appendChild(c); - } - } - return e; -} - -export function text(str) { - return document.createTextNode(str); -} - -export const HTML_NS = "http://www.w3.org/1999/xhtml"; -export const SVG_NS = "http://www.w3.org/2000/svg"; - -export const TAG_NAMES = { - [HTML_NS]: [ - "br", "a", "ol", "ul", "li", "div", "h1", "h2", "h3", "h4", "h5", "h6", - "p", "strong", "em", "span", "img", "section", "main", "article", "aside", - "pre", "button", "time", "input", "textarea", "label", "form", "progress", "output"], - [SVG_NS]: ["svg", "circle"] -}; - -export const tag = {}; - - -for (const [ns, tags] of Object.entries(TAG_NAMES)) { - for (const tagName of tags) { - tag[tagName] = function(attributes, children) { - return elNS(ns, tagName, attributes, children); - } - } -} diff --git a/scripts/logviewer/index.html b/scripts/logviewer/index.html deleted file mode 100644 index 109cf8d1..00000000 --- a/scripts/logviewer/index.html +++ /dev/null @@ -1,237 +0,0 @@ - - - - - - - -

-
- - - - - diff --git a/scripts/logviewer/main.js b/scripts/logviewer/main.js deleted file mode 100644 index e552a094..00000000 --- a/scripts/logviewer/main.js +++ /dev/null @@ -1,430 +0,0 @@ -/* -Copyright 2020 The Matrix.org Foundation C.I.C. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import {tag as t} from "./html.js"; -import {openFile, readFileAsText} from "./file.js"; - -const main = document.querySelector("main"); - -let selectedItemNode; -let rootItem; -let itemByRef; -let itemsRefFrom; - -const logLevels = [undefined, "All", "Debug", "Detail", "Info", "Warn", "Error", "Fatal", "Off"]; - -main.addEventListener("click", event => { - if (event.target.classList.contains("toggleExpanded")) { - const li = event.target.parentElement.parentElement; - li.classList.toggle("expanded"); - } else { - // allow clicking any links other than .item in the timeline, like refs - if (event.target.tagName === "A" && !event.target.classList.contains("item")) { - return; - } - const itemNode = event.target.closest(".item"); - if (itemNode) { - // we don't want scroll to jump when clicking - // so prevent default behaviour, and select and push to history manually - event.preventDefault(); - selectNode(itemNode); - history.pushState(null, null, `#${itemNode.id}`); - } - } -}); - -window.addEventListener("hashchange", () => { - const id = window.location.hash.substr(1); - const itemNode = document.getElementById(id); - if (itemNode && itemNode.closest("main")) { - ensureParentsExpanded(itemNode); - selectNode(itemNode); - itemNode.scrollIntoView({behavior: "smooth", block: "nearest"}); - } -}); - -function selectNode(itemNode) { - if (selectedItemNode) { - selectedItemNode.classList.remove("selected"); - } - selectedItemNode = itemNode; - selectedItemNode.classList.add("selected"); - let item = rootItem; - let parent; - const indices = selectedItemNode.id.split("/").map(i => parseInt(i, 10)); - for(const i of indices) { - parent = item; - item = itemChildren(item)[i]; - } - showItemDetails(item, parent, selectedItemNode); -} - -function ensureParentsExpanded(itemNode) { - let li = itemNode.parentElement.parentElement; - while (li.tagName === "LI") { - li.classList.add("expanded"); - li = li.parentElement.parentElement; - } -} - -function stringifyItemValue(value) { - if (typeof value === "object" && value !== null) { - return JSON.stringify(value, undefined, 2); - } else { - return value + ""; - } -} - -function showItemDetails(item, parent, itemNode) { - const parentOffset = itemStart(parent) ? `${itemStart(item) - itemStart(parent)}ms` : "none"; - const expandButton = t.button("Expand recursively"); - expandButton.addEventListener("click", () => expandResursively(itemNode.parentElement.parentElement)); - const start = itemStart(item); - const aside = t.aside([ - t.h3(itemCaption(item)), - t.p([t.strong("Log level: "), logLevels[itemLevel(item)]]), - t.p([t.strong("Error: "), itemError(item) ? `${itemError(item).name} ${itemError(item).stack}` : "none"]), - t.p([t.strong("Parent offset: "), parentOffset]), - t.p([t.strong("Start: "), new Date(start).toString(), ` (${start})`]), - t.p([t.strong("Duration: "), `${itemDuration(item)}ms`]), - t.p([t.strong("Child count: "), itemChildren(item) ? `${itemChildren(item).length}` : "none"]), - t.p([t.strong("Forced finish: "), (itemForcedFinish(item) || false) + ""]), - t.p(t.strong("Values:")), - t.ul({class: "values"}, Object.entries(itemValues(item)).map(([key, value]) => { - let valueNode; - if (key === "ref") { - const refItem = itemByRef.get(value); - if (refItem) { - valueNode = t.a({href: `#${refItem.id}`}, itemCaption(refItem)); - } else { - valueNode = `unknown ref ${value}`; - } - } else if (key === "refId") { - const refSources = itemsRefFrom.get(value) ?? []; - valueNode = t.div([t.p([`${value}`, t.br(),`Found these references:`]),t.ul(refSources.map(item => { - return t.li(t.a({href: `#${item.id}`}, itemCaption(item))); - }))]); - } else { - valueNode = stringifyItemValue(value); - } - return t.li([ - t.span({className: "key"}, normalizeValueKey(key)), - t.span({className: "value"}, valueNode) - ]); - })), - t.p(expandButton) - ]); - document.querySelector("aside").replaceWith(aside); -} - -function expandResursively(li) { - li.classList.add("expanded"); - const ol = li.querySelector("ol"); - if (ol) { - const len = ol.children.length; - for (let i = 0; i < len; i += 1) { - expandResursively(ol.children[i]); - } - } -} - -document.getElementById("openFile").addEventListener("click", loadFile); - -function getRootItemHeader(prevItem, item) { - if (prevItem) { - const diff = itemStart(item) - itemEnd(prevItem); - if (diff >= 0) { - return `+ ${formatTime(diff)}`; - } else { - const overlap = -diff; - if (overlap >= itemDuration(item)) { - return `ran entirely in parallel with`; - } else { - return `ran ${formatTime(-diff)} in parallel with`; - } - } - } else { - return new Date(itemStart(item)).toString(); - } -} - -async function loadFile() { - const file = await openFile(); - document.getElementById("filename").innerText = file.name; - await loadBlob(file); -} - -export async function loadBlob(blob) { - const json = await readFileAsText(blob); - const logs = JSON.parse(json); - logs.items.sort((a, b) => itemStart(a) - itemStart(b)); - rootItem = {c: logs.items}; - itemByRef = new Map(); - itemsRefFrom = new Map(); - preprocessRecursively(rootItem, null, itemByRef, itemsRefFrom, []); - - const fragment = logs.items.reduce((fragment, item, i, items) => { - const prevItem = i === 0 ? null : items[i - 1]; - fragment.appendChild(t.section([ - t.h2(getRootItemHeader(prevItem, item)), - t.div({className: "timeline"}, t.ol(itemToNode(item, [i]))) - ])); - return fragment; - }, document.createDocumentFragment()); - main.replaceChildren(fragment); - main.scrollTop = main.scrollHeight; -} - -// TODO: make this use processRecursively -function preprocessRecursively(item, parentElement, refsMap, refsFromMap, path) { - item.s = (parentElement?.s || 0) + item.s; - if (itemRefSource(item)) { - refsMap.set(itemRefSource(item), item); - } - if (itemRef(item)) { - let refs = refsFromMap.get(itemRef(item)); - if (!refs) { - refs = []; - refsFromMap.set(itemRef(item), refs); - } - refs.push(item); - } - if (itemChildren(item)) { - for (let i = 0; i < itemChildren(item).length; i += 1) { - // do it in advance for a child as we don't want to do it for the rootItem - const child = itemChildren(item)[i]; - const childPath = path.concat(i); - child.id = childPath.join("/"); - preprocessRecursively(child, item, refsMap, refsFromMap, childPath); - } - } -} - -const MS_IN_SEC = 1000; -const MS_IN_MIN = MS_IN_SEC * 60; -const MS_IN_HOUR = MS_IN_MIN * 60; -const MS_IN_DAY = MS_IN_HOUR * 24; -function formatTime(ms) { - let str = ""; - if (ms > MS_IN_DAY) { - const days = Math.floor(ms / MS_IN_DAY); - ms -= days * MS_IN_DAY; - str += `${days}d`; - } - if (ms > MS_IN_HOUR) { - const hours = Math.floor(ms / MS_IN_HOUR); - ms -= hours * MS_IN_HOUR; - str += `${hours}h`; - } - if (ms > MS_IN_MIN) { - const mins = Math.floor(ms / MS_IN_MIN); - ms -= mins * MS_IN_MIN; - str += `${mins}m`; - } - if (ms > MS_IN_SEC) { - const secs = ms / MS_IN_SEC; - str += `${secs.toFixed(2)}s`; - } else if (ms > 0 || !str.length) { - str += `${ms}ms`; - } - return str; -} - -function itemChildren(item) { return item.c; } -function itemStart(item) { return item.s; } -function itemEnd(item) { return item.s + item.d; } -function itemDuration(item) { return item.d; } -function itemValues(item) { return item.v; } -function itemLevel(item) { return item.l; } -function itemLabel(item) { return item.v?.l; } -function itemType(item) { return item.v?.t; } -function itemError(item) { return item.e; } -function itemForcedFinish(item) { return item.f; } -function itemRef(item) { return item.v?.ref; } -function itemRefSource(item) { return item.v?.refId; } -function itemShortErrorMessage(item) { - if (itemError(item)) { - const e = itemError(item); - return e.name || e.stack.substr(0, e.stack.indexOf("\n")); - } -} - -function itemCaption(item) { - if (itemLabel(item) && itemError(item)) { - return `${itemLabel(item)} (${itemShortErrorMessage(item)})`; - } if (itemType(item) === "network") { - return `${itemValues(item)?.method} ${itemValues(item)?.url}`; - } else if (itemLabel(item) && itemValues(item)?.id) { - return `${itemLabel(item)} ${itemValues(item).id}`; - } else if (itemLabel(item) && itemValues(item)?.status) { - return `${itemLabel(item)} (${itemValues(item).status})`; - } else if (itemLabel(item) && itemValues(item)?.type) { - return `${itemLabel(item)} (${itemValues(item)?.type})`; - } else if (itemRef(item)) { - const refItem = itemByRef.get(itemRef(item)); - if (refItem) { - return `ref "${itemCaption(refItem)}"` - } else { - return `unknown ref ${itemRef(item)}` - } - } else { - return itemLabel(item) || itemType(item); - } -} -function normalizeValueKey(key) { - switch (key) { - case "t": return "type"; - case "l": return "label"; - default: return key; - } -} - -// returns the node and the total range (recursively) occupied by the node -function itemToNode(item) { - const hasChildren = !!itemChildren(item)?.length; - const className = { - item: true, - "has-children": hasChildren, - error: itemError(item), - [`type-${itemType(item)}`]: !!itemType(item), - [`level-${itemLevel(item)}`]: true, - }; - - const id = item.id; - let captionNode; - if (itemRef(item)) { - const refItem = itemByRef.get(itemRef(item)); - if (refItem) { - captionNode = ["ref ", t.a({href: `#${refItem.id}`}, itemCaption(refItem))]; - } - } - if (!captionNode) { - captionNode = itemCaption(item); - } - const li = t.li([ - t.div([ - hasChildren ? t.button({className: "toggleExpanded"}) : "", - t.a({className, id, href: `#${id}`}, [ - t.span({class: "caption"}, captionNode), - t.span({class: "duration"}, `(${formatTime(itemDuration(item))})`), - ]) - ]) - ]); - if (itemChildren(item) && itemChildren(item).length) { - li.appendChild(t.ol(itemChildren(item).map(item => { - return itemToNode(item); - }))); - } - return li; -} - -const highlightForm = document.getElementById("highlightForm"); - -highlightForm.addEventListener("submit", evt => { - evt.preventDefault(); - const matchesOutput = document.getElementById("highlightMatches"); - const query = document.getElementById("highlight").value; - if (query) { - matchesOutput.innerText = "Searching…"; - let matches = 0; - processRecursively(rootItem, item => { - let domNode = document.getElementById(item.id); - if (itemMatchesFilter(item, query)) { - matches += 1; - domNode.classList.add("highlighted"); - domNode = domNode.parentElement; - while (domNode.nodeName !== "SECTION") { - if (domNode.nodeName === "LI") { - domNode.classList.add("expanded"); - } - domNode = domNode.parentElement; - } - } else { - domNode.classList.remove("highlighted"); - } - }); - matchesOutput.innerText = `${matches} matches`; - } else { - for (const node of document.querySelectorAll(".highlighted")) { - node.classList.remove("highlighted"); - } - matchesOutput.innerText = ""; - } -}); - -function itemMatchesFilter(item, query) { - if (itemError(item)) { - if (valueMatchesQuery(itemError(item), query)) { - return true; - } - } - return valueMatchesQuery(itemValues(item), query); -} - -function valueMatchesQuery(value, query) { - if (typeof value === "string") { - return value.includes(query); - } else if (typeof value === "object" && value !== null) { - for (const key in value) { - if (value.hasOwnProperty(key) && valueMatchesQuery(value[key], query)) { - return true; - } - } - } else if (typeof value === "number") { - return value.toString().includes(query); - } - return false; -} - -function processRecursively(item, callback, parentItem) { - if (item.id) { - callback(item, parentItem); - } - if (itemChildren(item)) { - for (let i = 0; i < itemChildren(item).length; i += 1) { - // do it in advance for a child as we don't want to do it for the rootItem - const child = itemChildren(item)[i]; - processRecursively(child, callback, item); - } - } -} - -document.getElementById("collapseAll").addEventListener("click", () => { - for (const node of document.querySelectorAll(".expanded")) { - node.classList.remove("expanded"); - } -}); -document.getElementById("hideCollapsed").addEventListener("click", () => { - for (const node of document.querySelectorAll("section > div.timeline > ol > li:not(.expanded)")) { - node.closest("section").classList.add("hidden"); - } -}); -document.getElementById("hideHighlightedSiblings").addEventListener("click", () => { - for (const node of document.querySelectorAll(".highlighted")) { - const list = node.closest("ol"); - const siblings = Array.from(list.querySelectorAll("li > div > a:not(.highlighted)")).map(n => n.closest("li")); - for (const sibling of siblings) { - if (!sibling.classList.contains("expanded")) { - sibling.classList.add("hidden"); - } - } - } -}); -document.getElementById("showAll").addEventListener("click", () => { - for (const node of document.querySelectorAll(".hidden")) { - node.classList.remove("hidden"); - } -}); diff --git a/src/platform/web/ui/session/settings/SettingsView.js b/src/platform/web/ui/session/settings/SettingsView.js index 09d3cb5a..ffc20f25 100644 --- a/src/platform/web/ui/session/settings/SettingsView.js +++ b/src/platform/web/ui/session/settings/SettingsView.js @@ -143,7 +143,7 @@ export class SettingsView extends TemplateView { } async function openLogs(vm) { - const logviewerUrl = (await import("../../../../../../scripts/logviewer/index.html?url")).default; + const logviewerUrl = (await import("@matrixdotorg/structured-logviewer/index.html?url")).default; const win = window.open(logviewerUrl); await new Promise((resolve, reject) => { let shouldSendPings = true; diff --git a/yarn.lock b/yarn.lock index 6405fe4d..cbce8b06 100644 --- a/yarn.lock +++ b/yarn.lock @@ -56,6 +56,11 @@ version "3.2.3" resolved "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.3.tgz#cc332fdd25c08ef0e40f4d33fc3f822a0f98b6f4" +"@matrixdotorg/structured-logviewer@^0.0.1": + version "0.0.1" + resolved "https://registry.yarnpkg.com/@matrixdotorg/structured-logviewer/-/structured-logviewer-0.0.1.tgz#9c29470b552f874afbb1df16c6e8e9e0c55cbf59" + integrity sha512-IdPYxAFDEoEs2G1ImKCkCxFI3xF1DDctP3N9JOtHRvIPbPPdTT9DyNqKTewCb5zwjNB1mGBrnWyURnHDiOOL3w== + "@nodelib/fs.scandir@2.1.5": version "2.1.5" resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5" From e2621015e179c2d00ef7129a6c2384be3902b189 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 10 May 2022 20:08:58 +0200 Subject: [PATCH 127/323] don't include log viewer in production build --- src/platform/web/ui/session/settings/SettingsView.js | 7 +++++-- vite.config.js | 4 +++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/platform/web/ui/session/settings/SettingsView.js b/src/platform/web/ui/session/settings/SettingsView.js index ffc20f25..f375a4be 100644 --- a/src/platform/web/ui/session/settings/SettingsView.js +++ b/src/platform/web/ui/session/settings/SettingsView.js @@ -142,9 +142,12 @@ export class SettingsView extends TemplateView { } } + async function openLogs(vm) { - const logviewerUrl = (await import("@matrixdotorg/structured-logviewer/index.html?url")).default; - const win = window.open(logviewerUrl); + // Use vite-specific url so this asset doesn't get picked up by vite and included in the production build, + // as opening the logs is only available during dev time, and @matrixdotorg/structured-logviewer is a dev dependency + // This url is what import "@matrixdotorg/structured-logviewer/index.html?url" resolves to with vite. + const win = window.open(`/@fs/${DEFINE_PROJECT_DIR}/node_modules/@matrixdotorg/structured-logviewer/index.html`); await new Promise((resolve, reject) => { let shouldSendPings = true; const cleanup = () => { diff --git a/vite.config.js b/vite.config.js index 87e3d063..97fb8885 100644 --- a/vite.config.js +++ b/vite.config.js @@ -37,6 +37,8 @@ export default defineConfig(({mode}) => { "sw": definePlaceholders }), ], - define: definePlaceholders, + define: Object.assign({ + DEFINE_PROJECT_DIR: JSON.stringify(__dirname) + }, definePlaceholders), }); }); From 21065791a86d53fda3c478bf9853646670338dad Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 10 May 2022 16:58:03 -0700 Subject: [PATCH 128/323] Fix removing members in handleCallMemberEvent --- src/matrix/calls/CallHandler.ts | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 7fbe6103..92f7314c 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -40,11 +40,15 @@ export type Options = Omit & { clock: Clock }; +function getRoomMemberKey(roomId: string, userId: string) { + return JSON.stringify(roomId)+`,`+JSON.stringify(userId); +} + export class CallHandler { // group calls by call id private readonly _calls: ObservableMap = new ObservableMap(); - // map of userId to set of conf_id's they are in - private memberToCallIds: Map> = new Map(); + // map of `"roomId","userId"` to set of conf_id's they are in + private roomMemberToCallIds: Map> = new Map(); private groupCallOptions: GroupCallOptions; private sessionId = makeId("s"); @@ -98,7 +102,7 @@ export class CallHandler { // } const callsMemberEvents = await txn.roomState.getAllForType(roomId, EventType.GroupCallMember); for (const entry of callsMemberEvents) { - this.handleCallMemberEvent(entry.event, log); + this.handleCallMemberEvent(entry.event, roomId, log); } // TODO: we should be loading the other members as well at some point })); @@ -149,7 +153,7 @@ export class CallHandler { // then update members for (const event of events) { if (event.type === EventType.GroupCallMember) { - this.handleCallMemberEvent(event, log); + this.handleCallMemberEvent(event, room.id, log); } } } @@ -194,8 +198,9 @@ export class CallHandler { } } - private handleCallMemberEvent(event: StateEvent, log: ILogItem) { + private handleCallMemberEvent(event: StateEvent, roomId: string, log: ILogItem) { const userId = event.state_key; + const roomMemberKey = getRoomMemberKey(roomId, userId) const calls = event.content["m.calls"] ?? []; for (const call of calls) { const callId = call["m.call_id"]; @@ -204,7 +209,8 @@ export class CallHandler { groupCall?.updateMembership(userId, call, log); }; const newCallIdsMemberOf = new Set(calls.map(call => call["m.call_id"])); - let previousCallIdsMemberOf = this.memberToCallIds.get(userId); + let previousCallIdsMemberOf = this.roomMemberToCallIds.get(roomMemberKey); + // remove user as member of any calls not present anymore if (previousCallIdsMemberOf) { for (const previousCallId of previousCallIdsMemberOf) { @@ -215,9 +221,9 @@ export class CallHandler { } } if (newCallIdsMemberOf.size === 0) { - this.memberToCallIds.delete(userId); + this.roomMemberToCallIds.delete(roomMemberKey); } else { - this.memberToCallIds.set(userId, newCallIdsMemberOf); + this.roomMemberToCallIds.set(roomMemberKey, newCallIdsMemberOf); } } } From 5ee4e39bc78eb2ab28b7c7ac26379fd219ec8195 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 11 May 2022 10:06:05 +0200 Subject: [PATCH 129/323] add return type --- src/matrix/calls/CallHandler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 92f7314c..07ed8492 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -40,7 +40,7 @@ export type Options = Omit & { clock: Clock }; -function getRoomMemberKey(roomId: string, userId: string) { +function getRoomMemberKey(roomId: string, userId: string): string { return JSON.stringify(roomId)+`,`+JSON.stringify(userId); } From a923e7e5e1ef22aabdb615cd52d1c9acfbe7b5da Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 11 May 2022 13:14:23 +0200 Subject: [PATCH 130/323] don't pass errors as log levels --- src/matrix/calls/PeerCall.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 963d6c30..e6ce3571 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -438,7 +438,7 @@ export class PeerCall implements IDisposable { if (this.callId! > newCallId) { log.log( "Glare detected: answering incoming call " + newCallId + - " and canceling outgoing call ", + " and canceling outgoing call " ); // How do we interrupt `call()`? well, perhaps we need to not just await InviteSent but also CreateAnswer? if (this._state === CallState.Fledgling || this._state === CallState.CreateOffer) { @@ -452,7 +452,7 @@ export class PeerCall implements IDisposable { } else { log.log( "Glare detected: rejecting incoming call " + newCallId + - " and keeping outgoing call ", + " and keeping outgoing call " ); await this.sendHangupWithCallId(newCallId, CallErrorCode.Replaced, log); } @@ -625,7 +625,7 @@ export class PeerCall implements IDisposable { if (this.opponentPartyId !== partyId) { log.log( `Ignoring candidates from party ID ${partyId}: ` + - `we have chosen party ID ${this.opponentPartyId}`, + `we have chosen party ID ${this.opponentPartyId}` ); return; @@ -680,7 +680,7 @@ export class PeerCall implements IDisposable { await this.sendSignallingMessage({type: EventType.Negotiate, content}, log); } } catch (err) { - log.log(`Failed to complete negotiation`, err); + log.log(`Failed to complete negotiation`).catch(err); } } @@ -801,12 +801,12 @@ export class PeerCall implements IDisposable { log.log(`Ignoring remote ICE candidate with no sdpMid or sdpMLineIndex`); continue; } - log.log(`Got remote ICE ${candidate.sdpMid} candidate: ${candidate.candidate}`); + const logItem = log.log(`Adding remote ICE ${candidate.sdpMid} candidate: ${candidate.candidate}`); try { await this.peerConnection.addIceCandidate(candidate); } catch (err) { if (!this.ignoreOffer) { - log.log(`Failed to add remote ICE candidate`, err); + logItem.catch(err); } } } From ec1568cf1c1a2545d32b3e3bf3342760fdfb75b7 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 12 May 2022 11:53:29 +0200 Subject: [PATCH 131/323] fix lint error --- .eslintrc.js | 1 + 1 file changed, 1 insertion(+) diff --git a/.eslintrc.js b/.eslintrc.js index cb28f4c8..eb23d387 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -17,6 +17,7 @@ module.exports = { "globals": { "DEFINE_VERSION": "readonly", "DEFINE_GLOBAL_HASH": "readonly", + "DEFINE_PROJECT_DIR": "readonly", // only available in sw.js "DEFINE_UNHASHED_PRECACHED_ASSETS": "readonly", "DEFINE_HASHED_PRECACHED_ASSETS": "readonly", From d727dfd843a5167197c783c5d1ba07748bde17a3 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 12 May 2022 11:58:28 +0200 Subject: [PATCH 132/323] add session.observeRoomState to observe state changes in all rooms and use it for calls this won't be called for state already received and stored in storage, that you need to still do yourself --- src/lib.ts | 8 +++++++ src/matrix/RoomStateHandlerSet.ts | 37 +++++++++++++++++++++++++++++++ src/matrix/Session.js | 9 +++++++- src/matrix/calls/CallHandler.ts | 19 ++++++---------- src/matrix/room/Room.js | 30 ++++++++++++------------- src/matrix/room/common.ts | 11 +++++++++ 6 files changed, 86 insertions(+), 28 deletions(-) create mode 100644 src/matrix/RoomStateHandlerSet.ts diff --git a/src/lib.ts b/src/lib.ts index 0fc6f539..8c5c4e8e 100644 --- a/src/lib.ts +++ b/src/lib.ts @@ -15,11 +15,19 @@ limitations under the License. */ export {Logger} from "./logging/Logger"; +export type {ILogItem} from "./logging/types"; export {IDBLogPersister} from "./logging/IDBLogPersister"; export {ConsoleReporter} from "./logging/ConsoleReporter"; export {Platform} from "./platform/web/Platform.js"; export {Client, LoadStatus} from "./matrix/Client.js"; export {RoomStatus} from "./matrix/room/common"; +// export everything needed to observe state events on all rooms using session.observeRoomState +export type {RoomStateHandler} from "./matrix/room/common"; +export type {MemberChange} from "./matrix/room/members/RoomMember"; +export type {Transaction} from "./matrix/storage/idb/Transaction"; +export type {Room} from "./matrix/room/Room"; +export type {StateEvent} from "./matrix/storage/types"; + // export main view & view models export {createNavigation, createRouter} from "./domain/navigation/index.js"; export {RootViewModel} from "./domain/RootViewModel.js"; diff --git a/src/matrix/RoomStateHandlerSet.ts b/src/matrix/RoomStateHandlerSet.ts new file mode 100644 index 00000000..cf202097 --- /dev/null +++ b/src/matrix/RoomStateHandlerSet.ts @@ -0,0 +1,37 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import type {ILogItem} from "../logging/types"; +import type {StateEvent} from "./storage/types"; +import type {Transaction} from "./storage/idb/Transaction"; +import type {Room} from "./room/Room"; +import type {MemberChange} from "./room/members/RoomMember"; +import type {RoomStateHandler} from "./room/common"; +import {BaseObservable} from "../observable/BaseObservable"; + +/** keeps track of all handlers registered with Session.observeRoomState */ +export class RoomStateHandlerSet extends BaseObservable implements RoomStateHandler { + handleRoomState(room: Room, stateEvent: StateEvent, txn: Transaction, log: ILogItem) { + for(let h of this._handlers) { + h.handleRoomState(room, stateEvent, txn, log); + } + } + updateRoomMembers(room: Room, memberChanges: Map) { + for(let h of this._handlers) { + h.updateRoomMembers(room, memberChanges); + } + } +} diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 9d63d335..6211c456 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -48,6 +48,7 @@ import {SecretStorage} from "./ssss/SecretStorage"; import {ObservableValue} from "../observable/value/ObservableValue"; import {RetainedObservableValue} from "../observable/value/RetainedObservableValue"; import {CallHandler} from "./calls/CallHandler"; +import {RoomStateHandlerSet} from "./RoomStateHandlerSet"; const PICKLE_KEY = "DEFAULT_KEY"; const PUSHER_KEY = "pusher"; @@ -101,6 +102,8 @@ export class Session { }], forceTURN: false, }); + this._roomStateHandler = new RoomStateHandlerSet(); + this.observeRoomState(this._callHandler); this._deviceMessageHandler = new DeviceMessageHandler({storage, callHandler: this._callHandler}); this._olm = olm; this._olmUtil = null; @@ -595,7 +598,7 @@ export class Session { user: this._user, createRoomEncryption: this._createRoomEncryption, platform: this._platform, - callHandler: this._callHandler + roomStateHandler: this._roomStateHandler }); } @@ -937,6 +940,10 @@ export class Session { return observable; } + observeRoomState(roomStateHandler) { + return this._roomStateHandler.subscribe(roomStateHandler); + } + /** Creates an empty (summary isn't loaded) the archived room if it isn't loaded already, assuming sync will either remove it (when rejoining) or diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 07ed8492..2386076b 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -35,6 +35,7 @@ import type {Options as GroupCallOptions} from "./group/GroupCall"; import type {Transaction} from "../storage/idb/Transaction"; import type {CallEntry} from "../storage/idb/stores/CallStore"; import type {Clock} from "../../platform/web/dom/Clock"; +import type {RoomStateHandler} from "../room/common"; export type Options = Omit & { clock: Clock @@ -44,7 +45,7 @@ function getRoomMemberKey(roomId: string, userId: string): string { return JSON.stringify(roomId)+`,`+JSON.stringify(userId); } -export class CallHandler { +export class CallHandler implements RoomStateHandler { // group calls by call id private readonly _calls: ObservableMap = new ObservableMap(); // map of `"roomId","userId"` to set of conf_id's they are in @@ -143,18 +144,12 @@ export class CallHandler { // TODO: check and poll turn server credentials here /** @internal */ - handleRoomState(room: Room, events: StateEvent[], txn: Transaction, log: ILogItem) { - // first update call events - for (const event of events) { - if (event.type === EventType.GroupCall) { - this.handleCallEvent(event, room.id, txn, log); - } + handleRoomState(room: Room, event: StateEvent, txn: Transaction, log: ILogItem) { + if (event.type === EventType.GroupCall) { + this.handleCallEvent(event, room.id, txn, log); } - // then update members - for (const event of events) { - if (event.type === EventType.GroupCallMember) { - this.handleCallMemberEvent(event, room.id, log); - } + if (event.type === EventType.GroupCallMember) { + this.handleCallMemberEvent(event, room.id, log); } } diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js index 34f35af8..556cef7a 100644 --- a/src/matrix/room/Room.js +++ b/src/matrix/room/Room.js @@ -30,7 +30,7 @@ const EVENT_ENCRYPTED_TYPE = "m.room.encrypted"; export class Room extends BaseRoom { constructor(options) { super(options); - this._callHandler = options.callHandler; + this._roomStateHandler = options.roomStateHandler; // TODO: pass pendingEvents to start like pendingOperations? const {pendingEvents} = options; const relationWriter = new RelationWriter({ @@ -179,7 +179,7 @@ export class Room extends BaseRoom { removedPendingEvents = await this._sendQueue.removeRemoteEchos(roomResponse.timeline.events, txn, log); } const powerLevelsEvent = this._getPowerLevelsEvent(roomResponse); - this._updateCallHandler(roomResponse, txn, log); + this._updateRoomStateHandler(roomResponse, txn, log); return { summaryChanges, roomEncryption, @@ -217,9 +217,7 @@ export class Room extends BaseRoom { if (this._memberList) { this._memberList.afterSync(memberChanges); } - if (this._callHandler) { - this._callHandler.updateRoomMembers(this, memberChanges); - } + this._roomStateHandler.updateRoomMembers(this, memberChanges); if (this._observedMembers) { this._updateObservedMembers(memberChanges); } @@ -447,17 +445,19 @@ export class Room extends BaseRoom { return this._sendQueue.pendingEvents; } - _updateCallHandler(roomResponse, txn, log) { - if (this._callHandler) { - const stateEvents = roomResponse.state?.events; - if (stateEvents?.length) { - this._callHandler.handleRoomState(this, stateEvents, txn, log); + _updateRoomStateHandler(roomResponse, txn, log) { + const stateEvents = roomResponse.state?.events; + if (stateEvents) { + for (let i = 0; i < stateEvents.length; i++) { + this._roomStateHandler.handleRoomState(this, stateEvents[i], txn, log); } - let timelineEvents = roomResponse.timeline?.events; - if (timelineEvents) { - const timelineStateEvents = timelineEvents.filter(e => typeof e.state_key === "string"); - if (timelineEvents.length !== 0) { - this._callHandler.handleRoomState(this, timelineStateEvents, txn, log); + } + let timelineEvents = roomResponse.timeline?.events; + if (timelineEvents) { + for (let i = 0; i < timelineEvents.length; i++) { + const event = timelineEvents[i]; + if (typeof event.state_key === "string") { + this._roomStateHandler.handleRoomState(this, event, txn, log); } } } diff --git a/src/matrix/room/common.ts b/src/matrix/room/common.ts index 57ab7023..2fdc2483 100644 --- a/src/matrix/room/common.ts +++ b/src/matrix/room/common.ts @@ -14,6 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. */ +import type {Room} from "./Room"; +import type {StateEvent} from "../storage/types"; +import type {Transaction} from "../storage/idb/Transaction"; +import type {ILogItem} from "../../logging/types"; +import type {MemberChange} from "./members/RoomMember"; + export function getPrevContentFromStateEvent(event) { // where to look for prev_content is a bit of a mess, // see https://matrix.to/#/!NasysSDfxKxZBzJJoE:matrix.org/$DvrAbZJiILkOmOIuRsNoHmh2v7UO5CWp_rYhlGk34fQ?via=matrix.org&via=pixie.town&via=amorgan.xyz @@ -40,3 +46,8 @@ export enum RoomType { Private, Public } + +export interface RoomStateHandler { + handleRoomState(room: Room, stateEvent: StateEvent, txn: Transaction, log: ILogItem); + updateRoomMembers(room: Room, memberChanges: Map); +} From db053385962e0272c86f0ca9b998633f8325a3a1 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 12 May 2022 17:26:29 +0200 Subject: [PATCH 133/323] extract function to iterate over room response state events --- src/matrix/room/ArchivedRoom.js | 8 ++-- src/matrix/room/Room.js | 32 +++++++-------- src/matrix/room/RoomSummary.js | 25 ++---------- src/matrix/room/common.ts | 69 ++++++++++++++++++++++++++++++++- 4 files changed, 91 insertions(+), 43 deletions(-) diff --git a/src/matrix/room/ArchivedRoom.js b/src/matrix/room/ArchivedRoom.js index 1a23d25b..86595163 100644 --- a/src/matrix/room/ArchivedRoom.js +++ b/src/matrix/room/ArchivedRoom.js @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {reduceStateEvents} from "./RoomSummary.js"; +import {iterateResponseStateEvents} from "./common"; import {BaseRoom} from "./BaseRoom.js"; import {RoomMember, EVENT_TYPE as MEMBER_EVENT_TYPE} from "./members/RoomMember.js"; @@ -173,15 +173,15 @@ export class ArchivedRoom extends BaseRoom { } function findKickDetails(roomResponse, ownUserId) { - const kickEvent = reduceStateEvents(roomResponse, (kickEvent, event) => { + let kickEvent; + iterateResponseStateEvents(roomResponse, event => { if (event.type === MEMBER_EVENT_TYPE) { // did we get kicked? if (event.state_key === ownUserId && event.sender !== event.state_key) { kickEvent = event; } } - return kickEvent; - }, null); + }); if (kickEvent) { return { // this is different from the room membership in the sync section, which can only be leave diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js index 556cef7a..a2eadfeb 100644 --- a/src/matrix/room/Room.js +++ b/src/matrix/room/Room.js @@ -23,6 +23,7 @@ import {WrappedError} from "../error.js" import {Heroes} from "./members/Heroes.js"; import {AttachmentUpload} from "./AttachmentUpload.js"; import {DecryptionSource} from "../e2ee/common.js"; +import {iterateResponseStateEvents} from "./common.js"; import {PowerLevels, EVENT_TYPE as POWERLEVELS_EVENT_TYPE } from "./PowerLevels.js"; const EVENT_ENCRYPTED_TYPE = "m.room.encrypted"; @@ -179,7 +180,7 @@ export class Room extends BaseRoom { removedPendingEvents = await this._sendQueue.removeRemoteEchos(roomResponse.timeline.events, txn, log); } const powerLevelsEvent = this._getPowerLevelsEvent(roomResponse); - this._updateRoomStateHandler(roomResponse, txn, log); + this._runRoomStateHandlers(roomResponse, txn, log); return { summaryChanges, roomEncryption, @@ -275,8 +276,13 @@ export class Room extends BaseRoom { } _getPowerLevelsEvent(roomResponse) { - const isPowerlevelEvent = event => event.state_key === "" && event.type === POWERLEVELS_EVENT_TYPE; - const powerLevelEvent = roomResponse.timeline?.events.find(isPowerlevelEvent) ?? roomResponse.state?.events.find(isPowerlevelEvent); + let powerLevelEvent; + iterateResponseStateEvents(roomResponse, event => { + if(event.state_key === "" && event.type === POWERLEVELS_EVENT_TYPE) { + powerLevelEvent = event; + } + + }); return powerLevelEvent; } @@ -445,20 +451,12 @@ export class Room extends BaseRoom { return this._sendQueue.pendingEvents; } - _updateRoomStateHandler(roomResponse, txn, log) { - const stateEvents = roomResponse.state?.events; - if (stateEvents) { - for (let i = 0; i < stateEvents.length; i++) { - this._roomStateHandler.handleRoomState(this, stateEvents[i], txn, log); - } - } - let timelineEvents = roomResponse.timeline?.events; - if (timelineEvents) { - for (let i = 0; i < timelineEvents.length; i++) { - const event = timelineEvents[i]; - if (typeof event.state_key === "string") { - this._roomStateHandler.handleRoomState(this, event, txn, log); - } + /** global room state handlers, run during write sync step */ + _runRoomStateHandlers(roomResponse, txn, log) { + iterateResponseStateEvents(roomResponse, event => { + this._roomStateHandler.handleRoomState(this, event, txn, log); + }); + } } } } diff --git a/src/matrix/room/RoomSummary.js b/src/matrix/room/RoomSummary.js index a3dec467..62608683 100644 --- a/src/matrix/room/RoomSummary.js +++ b/src/matrix/room/RoomSummary.js @@ -15,7 +15,7 @@ limitations under the License. */ import {MEGOLM_ALGORITHM} from "../e2ee/common.js"; - +import {iterateResponseStateEvents} from "./common"; function applyTimelineEntries(data, timelineEntries, isInitialSync, canMarkUnread, ownUserId) { if (timelineEntries.length) { @@ -27,25 +27,6 @@ function applyTimelineEntries(data, timelineEntries, isInitialSync, canMarkUnrea return data; } -export function reduceStateEvents(roomResponse, callback, value) { - const stateEvents = roomResponse?.state?.events; - // state comes before timeline - if (Array.isArray(stateEvents)) { - value = stateEvents.reduce(callback, value); - } - const timelineEvents = roomResponse?.timeline?.events; - // and after that state events in the timeline - if (Array.isArray(timelineEvents)) { - value = timelineEvents.reduce((data, event) => { - if (typeof event.state_key === "string") { - value = callback(value, event); - } - return value; - }, value); - } - return value; -} - function applySyncResponse(data, roomResponse, membership, ownUserId) { if (roomResponse.summary) { data = updateSummary(data, roomResponse.summary); @@ -60,7 +41,9 @@ function applySyncResponse(data, roomResponse, membership, ownUserId) { // process state events in state and in timeline. // non-state events are handled by applyTimelineEntries // so decryption is handled properly - data = reduceStateEvents(roomResponse, (data, event) => processStateEvent(data, event, ownUserId), data); + iterateResponseStateEvents(roomResponse, event => { + data = processStateEvent(data, event, ownUserId); + }); const unreadNotifications = roomResponse.unread_notifications; if (unreadNotifications) { data = processNotificationCounts(data, unreadNotifications); diff --git a/src/matrix/room/common.ts b/src/matrix/room/common.ts index 2fdc2483..38070925 100644 --- a/src/matrix/room/common.ts +++ b/src/matrix/room/common.ts @@ -15,7 +15,7 @@ limitations under the License. */ import type {Room} from "./Room"; -import type {StateEvent} from "../storage/types"; +import type {StateEvent, TimelineEvent} from "../storage/types"; import type {Transaction} from "../storage/idb/Transaction"; import type {ILogItem} from "../../logging/types"; import type {MemberChange} from "./members/RoomMember"; @@ -50,4 +50,71 @@ export enum RoomType { export interface RoomStateHandler { handleRoomState(room: Room, stateEvent: StateEvent, txn: Transaction, log: ILogItem); updateRoomMembers(room: Room, memberChanges: Map); +type RoomResponse = { + state?: { + events?: Array + }, + timeline?: { + events?: Array + } +} + +/** iterates over any state events in a sync room response, in the order that they should be applied (from older to younger events) */ +export function iterateResponseStateEvents(roomResponse: RoomResponse, callback: (StateEvent) => void) { + // first iterate over state events, they precede the timeline + const stateEvents = roomResponse.state?.events; + if (stateEvents) { + for (let i = 0; i < stateEvents.length; i++) { + callback(stateEvents[i]); + } + } + // now see if there are any state events within the timeline + let timelineEvents = roomResponse.timeline?.events; + if (timelineEvents) { + for (let i = 0; i < timelineEvents.length; i++) { + const event = timelineEvents[i]; + if (typeof event.state_key === "string") { + callback(event); + } + } + } +} + +export function tests() { + return { + "test iterateResponseStateEvents with both state and timeline sections": assert => { + const roomResponse = { + state: { + events: [ + {type: "m.room.member", state_key: "1"}, + {type: "m.room.member", state_key: "2", content: {a: 1}}, + ] + }, + timeline: { + events: [ + {type: "m.room.message"}, + {type: "m.room.member", state_key: "3"}, + {type: "m.room.message"}, + {type: "m.room.member", state_key: "2", content: {a: 2}}, + ] + } + } as unknown as RoomResponse; + const expectedStateKeys = ["1", "2", "3", "2"]; + const expectedAForMember2 = [1, 2]; + iterateResponseStateEvents(roomResponse, event => { + assert.strictEqual(event.type, "m.room.member"); + assert.strictEqual(expectedStateKeys.shift(), event.state_key); + if (event.state_key === "2") { + assert.strictEqual(expectedAForMember2.shift(), event.content.a); + } + }); + assert.strictEqual(expectedStateKeys.length, 0); + assert.strictEqual(expectedAForMember2.length, 0); + }, + "test iterateResponseStateEvents with empty response": assert => { + iterateResponseStateEvents({}, () => { + assert.fail("no events expected"); + }); + } + } } From a50ea7e77b84dbe955bf4dc062d0a098924e81d4 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 12 May 2022 17:27:03 +0200 Subject: [PATCH 134/323] add support for observing room state for single room + initial state --- src/lib.ts | 2 +- src/matrix/Session.js | 2 +- src/matrix/calls/CallHandler.ts | 2 +- src/matrix/room/BaseRoom.js | 26 +++++++++ src/matrix/room/Room.js | 12 +++- src/matrix/room/common.ts | 3 - .../room/state/ObservedStateKeyValue.ts | 55 +++++++++++++++++++ src/matrix/room/state/ObservedStateTypeMap.ts | 53 ++++++++++++++++++ .../{ => room/state}/RoomStateHandlerSet.ts | 14 ++--- src/matrix/room/state/types.ts | 38 +++++++++++++ 10 files changed, 192 insertions(+), 15 deletions(-) create mode 100644 src/matrix/room/state/ObservedStateKeyValue.ts create mode 100644 src/matrix/room/state/ObservedStateTypeMap.ts rename src/matrix/{ => room/state}/RoomStateHandlerSet.ts (75%) create mode 100644 src/matrix/room/state/types.ts diff --git a/src/lib.ts b/src/lib.ts index 8c5c4e8e..839fbb15 100644 --- a/src/lib.ts +++ b/src/lib.ts @@ -22,7 +22,7 @@ export {Platform} from "./platform/web/Platform.js"; export {Client, LoadStatus} from "./matrix/Client.js"; export {RoomStatus} from "./matrix/room/common"; // export everything needed to observe state events on all rooms using session.observeRoomState -export type {RoomStateHandler} from "./matrix/room/common"; +export type {RoomStateHandler} from "./matrix/room/state/types"; export type {MemberChange} from "./matrix/room/members/RoomMember"; export type {Transaction} from "./matrix/storage/idb/Transaction"; export type {Room} from "./matrix/room/Room"; diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 6211c456..cd676fc5 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -48,7 +48,7 @@ import {SecretStorage} from "./ssss/SecretStorage"; import {ObservableValue} from "../observable/value/ObservableValue"; import {RetainedObservableValue} from "../observable/value/RetainedObservableValue"; import {CallHandler} from "./calls/CallHandler"; -import {RoomStateHandlerSet} from "./RoomStateHandlerSet"; +import {RoomStateHandlerSet} from "./room/state/RoomStateHandlerSet"; const PICKLE_KEY = "DEFAULT_KEY"; const PUSHER_KEY = "pusher"; diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 2386076b..e585bb40 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -35,7 +35,7 @@ import type {Options as GroupCallOptions} from "./group/GroupCall"; import type {Transaction} from "../storage/idb/Transaction"; import type {CallEntry} from "../storage/idb/stores/CallStore"; import type {Clock} from "../../platform/web/dom/Clock"; -import type {RoomStateHandler} from "../room/common"; +import type {RoomStateHandler} from "../room/state/types"; export type Options = Omit & { clock: Clock diff --git a/src/matrix/room/BaseRoom.js b/src/matrix/room/BaseRoom.js index cc65a320..b8f172d0 100644 --- a/src/matrix/room/BaseRoom.js +++ b/src/matrix/room/BaseRoom.js @@ -31,6 +31,8 @@ import {ensureLogItem} from "../../logging/utils"; import {PowerLevels} from "./PowerLevels.js"; import {RetainedObservableValue} from "../../observable/value/RetainedObservableValue"; import {TimelineReader} from "./timeline/persistence/TimelineReader"; +import {ObservedStateTypeMap} from "./state/ObservedStateTypeMap"; +import {ObservedStateKeyValue} from "./state/ObservedStateKeyValue"; const EVENT_ENCRYPTED_TYPE = "m.room.encrypted"; @@ -53,11 +55,35 @@ export class BaseRoom extends EventEmitter { this._getSyncToken = getSyncToken; this._platform = platform; this._observedEvents = null; + this._roomStateObservers = new Set(); this._powerLevels = null; this._powerLevelLoading = null; this._observedMembers = null; } + async observeStateType(type, txn = undefined) { + const map = new ObservedStateTypeMap(type); + await this._addStateObserver(map, txn); + return map; + } + + async observeStateTypeAndKey(type, stateKey, txn = undefined) { + const value = new ObservedStateKeyValue(type, stateKey); + await this._addStateObserver(value, txn); + return value; + } + + async _addStateObserver(stateObserver, txn) { + if (!txn) { + txn = await this._storage.readTxn([this._storage.storeNames.roomState]); + } + await stateObserver.load(this.id, txn); + this._roomStateObservers.add(stateObserver); + stateObserver.setRemoveCallback(() => { + this._roomStateObservers.delete(stateObserver); + }); + } + async _eventIdsToEntries(eventIds, txn) { const retryEntries = []; await Promise.all(eventIds.map(async eventId => { diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js index a2eadfeb..796474d3 100644 --- a/src/matrix/room/Room.js +++ b/src/matrix/room/Room.js @@ -182,6 +182,7 @@ export class Room extends BaseRoom { const powerLevelsEvent = this._getPowerLevelsEvent(roomResponse); this._runRoomStateHandlers(roomResponse, txn, log); return { + roomResponse, summaryChanges, roomEncryption, newEntries, @@ -204,7 +205,7 @@ export class Room extends BaseRoom { const { summaryChanges, newEntries, updatedEntries, newLiveKey, removedPendingEvents, memberChanges, powerLevelsEvent, - heroChanges, roomEncryption + heroChanges, roomEncryption, roomResponse } = changes; log.set("id", this.id); this._syncWriter.afterSync(newLiveKey); @@ -264,6 +265,7 @@ export class Room extends BaseRoom { if (removedPendingEvents) { this._sendQueue.emitRemovals(removedPendingEvents); } + this._emitSyncRoomState(roomResponse); } _updateObservedMembers(memberChanges) { @@ -457,8 +459,14 @@ export class Room extends BaseRoom { this._roomStateHandler.handleRoomState(this, event, txn, log); }); } + + /** local room state observers, run during after sync step */ + _emitSyncRoomState(roomResponse) { + iterateResponseStateEvents(roomResponse, event => { + for (const handler of this._roomStateObservers) { + handler.handleStateEvent(event); } - } + }); } /** @package */ diff --git a/src/matrix/room/common.ts b/src/matrix/room/common.ts index 38070925..7556cfb0 100644 --- a/src/matrix/room/common.ts +++ b/src/matrix/room/common.ts @@ -47,9 +47,6 @@ export enum RoomType { Public } -export interface RoomStateHandler { - handleRoomState(room: Room, stateEvent: StateEvent, txn: Transaction, log: ILogItem); - updateRoomMembers(room: Room, memberChanges: Map); type RoomResponse = { state?: { events?: Array diff --git a/src/matrix/room/state/ObservedStateKeyValue.ts b/src/matrix/room/state/ObservedStateKeyValue.ts new file mode 100644 index 00000000..41cc3c7b --- /dev/null +++ b/src/matrix/room/state/ObservedStateKeyValue.ts @@ -0,0 +1,55 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import type {StateObserver} from "./types"; +import type {StateEvent} from "../../storage/types"; +import type {Transaction} from "../../storage/idb/Transaction"; +import {BaseObservableValue} from "../../../observable/value/BaseObservableValue"; + +/** + * Observable value for a state event with a given type and state key. + * Unsubscribes when last subscription is removed */ +export class ObservedStateKeyValue extends BaseObservableValue implements StateObserver { + private event?: StateEvent; + private removeCallback?: () => void; + + constructor(private readonly type: string, private readonly stateKey: string) { + super(); + } + /** @internal */ + async load(roomId: string, txn: Transaction): Promise { + this.event = (await txn.roomState.get(roomId, this.type, this.stateKey))?.event; + } + /** @internal */ + handleStateEvent(event: StateEvent) { + if (event.type === this.type && event.state_key === this.stateKey) { + this.event = event; + this.emit(this.get()); + } + } + + get(): StateEvent | undefined { + return this.event; + } + + setRemoveCallback(callback: () => void) { + this.removeCallback = callback; + } + + onUnsubscribeLast() { + this.removeCallback?.(); + } +} diff --git a/src/matrix/room/state/ObservedStateTypeMap.ts b/src/matrix/room/state/ObservedStateTypeMap.ts new file mode 100644 index 00000000..e8fa6f7b --- /dev/null +++ b/src/matrix/room/state/ObservedStateTypeMap.ts @@ -0,0 +1,53 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import type {StateObserver} from "./types"; +import type {StateEvent} from "../../storage/types"; +import type {Transaction} from "../../storage/idb/Transaction"; +import {ObservableMap} from "../../../observable/map/ObservableMap"; + +/** + * Observable map for a given type with state keys as map keys. + * Unsubscribes when last subscription is removed */ +export class ObservedStateTypeMap extends ObservableMap implements StateObserver { + private removeCallback?: () => void; + + constructor(private readonly type: string) { + super(); + } + /** @internal */ + async load(roomId: string, txn: Transaction): Promise { + const events = await txn.roomState.getAllForType(roomId, this.type); + for (let i = 0; i < events.length; ++i) { + const {event} = events[i]; + this.add(event.state_key, event); + } + } + /** @internal */ + handleStateEvent(event: StateEvent) { + if (event.type === this.type) { + this.set(event.state_key, event); + } + } + + setRemoveCallback(callback: () => void) { + this.removeCallback = callback; + } + + onUnsubscribeLast() { + this.removeCallback?.(); + } +} diff --git a/src/matrix/RoomStateHandlerSet.ts b/src/matrix/room/state/RoomStateHandlerSet.ts similarity index 75% rename from src/matrix/RoomStateHandlerSet.ts rename to src/matrix/room/state/RoomStateHandlerSet.ts index cf202097..986cb0f9 100644 --- a/src/matrix/RoomStateHandlerSet.ts +++ b/src/matrix/room/state/RoomStateHandlerSet.ts @@ -14,13 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -import type {ILogItem} from "../logging/types"; -import type {StateEvent} from "./storage/types"; -import type {Transaction} from "./storage/idb/Transaction"; -import type {Room} from "./room/Room"; -import type {MemberChange} from "./room/members/RoomMember"; -import type {RoomStateHandler} from "./room/common"; -import {BaseObservable} from "../observable/BaseObservable"; +import type {ILogItem} from "../../../logging/types"; +import type {StateEvent} from "../../storage/types"; +import type {Transaction} from "../../storage/idb/Transaction"; +import type {Room} from "../Room"; +import type {MemberChange} from "../members/RoomMember"; +import type {RoomStateHandler} from "./types"; +import {BaseObservable} from "../../../observable/BaseObservable"; /** keeps track of all handlers registered with Session.observeRoomState */ export class RoomStateHandlerSet extends BaseObservable implements RoomStateHandler { diff --git a/src/matrix/room/state/types.ts b/src/matrix/room/state/types.ts new file mode 100644 index 00000000..ef99c727 --- /dev/null +++ b/src/matrix/room/state/types.ts @@ -0,0 +1,38 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import type {Room} from "../Room"; +import type {StateEvent} from "../../storage/types"; +import type {Transaction} from "../../storage/idb/Transaction"; +import type {ILogItem} from "../../../logging/types"; +import type {MemberChange} from "../members/RoomMember"; + +/** used for Session.observeRoomState, which observes in all room, but without loading from storage + * It receives the sync write transaction, so other stores can be updated as part of the same transaction. */ +export interface RoomStateHandler { + handleRoomState(room: Room, stateEvent: StateEvent, syncWriteTxn: Transaction, log: ILogItem); + updateRoomMembers(room: Room, memberChanges: Map); +} + +/** + * used for Room.observeStateType and Room.observeStateTypeAndKey + * @internal + * */ +export interface StateObserver { + handleStateEvent(event: StateEvent); + load(roomId: string, txn: Transaction): Promise; + setRemoveCallback(callback: () => void); +} From 6225574df61650e3d792c9640b77e18e0af6ddac Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 12 May 2022 17:52:17 +0200 Subject: [PATCH 135/323] write test for ObservedStateKeyValue --- .../room/state/ObservedStateKeyValue.ts | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/src/matrix/room/state/ObservedStateKeyValue.ts b/src/matrix/room/state/ObservedStateKeyValue.ts index 41cc3c7b..ce380458 100644 --- a/src/matrix/room/state/ObservedStateKeyValue.ts +++ b/src/matrix/room/state/ObservedStateKeyValue.ts @@ -53,3 +53,52 @@ export class ObservedStateKeyValue extends BaseObservableValue { + const storage = await createMockStorage(); + const writeTxn = await storage.readWriteTxn([storage.storeNames.roomState]); + writeTxn.roomState.set("!abc", { + event_id: "$abc", + type: "m.room.member", + state_key: "@alice", + sender: "@alice", + origin_server_ts: 5, + content: {} + }); + await writeTxn.complete(); + const txn = await storage.readTxn([storage.storeNames.roomState]); + const value = new ObservedStateKeyValue("m.room.member", "@alice"); + await value.load("!abc", txn); + const updates: Array = []; + assert.strictEqual(value.get()?.origin_server_ts, 5); + const unsubscribe = value.subscribe(value => updates.push(value)); + value.handleStateEvent({ + event_id: "$abc", + type: "m.room.member", + state_key: "@bob", + sender: "@alice", + origin_server_ts: 10, + content: {} + }); + assert.strictEqual(updates.length, 0); + value.handleStateEvent({ + event_id: "$abc", + type: "m.room.member", + state_key: "@alice", + sender: "@alice", + origin_server_ts: 10, + content: {} + }); + assert.strictEqual(updates.length, 1); + assert.strictEqual(updates[0]?.origin_server_ts, 10); + let removed = false; + value.setRemoveCallback(() => removed = true); + unsubscribe(); + assert(removed); + } + } +} From b2d787b96c4c5fcf41d912b94389faeb6ae0f250 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 17 May 2022 15:55:15 +0200 Subject: [PATCH 136/323] fix wrong extension in import --- src/matrix/room/Room.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js index 796474d3..38982786 100644 --- a/src/matrix/room/Room.js +++ b/src/matrix/room/Room.js @@ -23,7 +23,7 @@ import {WrappedError} from "../error.js" import {Heroes} from "./members/Heroes.js"; import {AttachmentUpload} from "./AttachmentUpload.js"; import {DecryptionSource} from "../e2ee/common.js"; -import {iterateResponseStateEvents} from "./common.js"; +import {iterateResponseStateEvents} from "./common"; import {PowerLevels, EVENT_TYPE as POWERLEVELS_EVENT_TYPE } from "./PowerLevels.js"; const EVENT_ENCRYPTED_TYPE = "m.room.encrypted"; From 6f0ebeacb7eb3def10caa4c83fce8088a18c884e Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 1 Jun 2022 15:27:00 +0200 Subject: [PATCH 137/323] fetch single device key in DeviceTracker --- src/matrix/e2ee/DeviceTracker.js | 52 ++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/matrix/e2ee/DeviceTracker.js b/src/matrix/e2ee/DeviceTracker.js index f8c3bca8..49d9ffab 100644 --- a/src/matrix/e2ee/DeviceTracker.js +++ b/src/matrix/e2ee/DeviceTracker.js @@ -309,6 +309,7 @@ export class DeviceTracker { return await this._devicesForUserIds(roomId, userIds, txn, hsApi, log); } + /** gets devices for the given user ids that are in the given room */ async devicesForRoomMembers(roomId, userIds, hsApi, log) { const txn = await this._storage.readTxn([ this._storage.storeNames.userIdentities, @@ -316,6 +317,57 @@ export class DeviceTracker { return await this._devicesForUserIds(roomId, userIds, txn, hsApi, log); } + /** gets a single device */ + async deviceForId(userId, deviceId, hsApi, log) { + const txn = await this._storage.readTxn([ + this._storage.storeNames.deviceIdentities, + ]); + let device = await txn.deviceIdentities.get(userId, deviceId); + if (device) { + log.set("existingDevice", true); + } else { + //// BEGIN EXTRACT (deviceKeysMap) + const deviceKeyResponse = await hsApi.queryKeys({ + "timeout": 10000, + "device_keys": { + [userId]: [deviceId] + }, + "token": this._getSyncToken() + }, {log}).response(); + // verify signature + const verifiedKeysPerUser = log.wrap("verify", log => this._filterVerifiedDeviceKeys(deviceKeyResponse["device_keys"], log)); + //// END EXTRACT + + // there should only be one device in here, but still check the HS sends us the right one + const verifiedKeys = verifiedKeysPerUser + .find(vkpu => vkpu.userId === userId).verifiedKeys + .find(vk => vk["device_id"] === deviceId); + device = deviceKeysAsDeviceIdentity(verifiedKeys); + const txn = await this._storage.readWriteTxn([ + this._storage.storeNames.deviceIdentities, + ]); + // check again we don't have the device already. + // when updating all keys for a user we allow updating the + // device when the key hasn't changed so the device display name + // can be updated, but here we don't. + const existingDevice = await txn.deviceIdentities.get(userId, deviceId); + if (existingDevice) { + device = existingDevice; + log.set("existingDeviceAfterFetch", true); + } else { + try { + txn.deviceIdentities.set(device); + log.set("newDevice", true); + } catch (err) { + txn.abort(); + throw err; + } + await txn.complete(); + } + } + return device; + } + /** * @param {string} roomId [description] * @param {Array} userIds a set of user ids to try and find the identity for. Will be check to belong to roomId. From 50ae51e8937892ca838f3f2160c6114d14a8462b Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 1 Jun 2022 15:28:49 +0200 Subject: [PATCH 138/323] encrypt call signalling message only for given device --- src/matrix/Session.js | 16 ++++++++-------- src/matrix/calls/group/GroupCall.ts | 6 +++--- src/matrix/calls/group/Member.ts | 24 ++++++++++++------------ 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index cd676fc5..26ab1702 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -79,18 +79,18 @@ export class Session { this._callHandler = new CallHandler({ clock: this._platform.clock, hsApi: this._hsApi, - encryptDeviceMessage: async (roomId, userId, message, log) => { + encryptDeviceMessage: async (roomId, userId, deviceId, message, log) => { if (!this._deviceTracker || !this._olmEncryption) { throw new Error("encryption is not enabled"); } - // TODO: just get the devices we're sending the message to, not all the room devices - // although we probably already fetched all devices to send messages in the likely e2ee room - const devices = await log.wrap("get device keys", async log => { - await this._deviceTracker.trackRoom(this.rooms.get(roomId), log); - return this._deviceTracker.devicesForRoomMembers(roomId, [userId], this._hsApi, log); + const device = await log.wrap("get device key", async log => { + return this._deviceTracker.deviceForId(userId, deviceId, this._hsApi, log); }); - const encryptedMessage = await this._olmEncryption.encrypt(message.type, message.content, devices, this._hsApi, log); - return encryptedMessage; + if (!device) { + throw new Error(`Could not find device key ${deviceId} for ${userId} in ${roomId}`); + } + const encryptedMessages = await this._olmEncryption.encrypt(message.type, message.content, [device], this._hsApi, log); + return encryptedMessages; }, storage: this._storage, webRTC: this._platform.webRTC, diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index af3966a0..b2b52e9b 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -55,7 +55,7 @@ function getDeviceFromMemberKey(key: string): string { export type Options = Omit & { emitUpdate: (call: GroupCall, params?: any) => void; - encryptDeviceMessage: (roomId: string, userId: string, message: SignallingMessage, log: ILogItem) => Promise, + encryptDeviceMessage: (roomId: string, userId: string, deviceId: string, message: SignallingMessage, log: ILogItem) => Promise, storage: Storage, logger: ILogger, }; @@ -93,8 +93,8 @@ export class GroupCall extends EventEmitter<{change: never}> { this._memberOptions = Object.assign({}, options, { confId: this.id, emitUpdate: member => this._members.update(getMemberKey(member.userId, member.deviceId), member), - encryptDeviceMessage: (userId: string, message: SignallingMessage, log) => { - return this.options.encryptDeviceMessage(this.roomId, userId, message, log); + encryptDeviceMessage: (userId: string, deviceId: string, message: SignallingMessage, log) => { + return this.options.encryptDeviceMessage(this.roomId, userId, deviceId, message, log); } }); } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 69e1eeea..e3abc49e 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -36,7 +36,7 @@ export type Options = Omit, log: ILogItem) => Promise, + encryptDeviceMessage: (userId: string, deviceId: string, message: SignallingMessage, log: ILogItem) => Promise, emitUpdate: (participant: Member, params?: any) => void, } @@ -217,20 +217,20 @@ export class Member { groupMessage.content.party_id = this.options.ownDeviceId; groupMessage.content.sender_session_id = this.options.sessionId; groupMessage.content.dest_session_id = this.sessionId; - // const encryptedMessages = await this.options.encryptDeviceMessage(this.member.userId, groupMessage, log); - // const payload = formatToDeviceMessagesPayload(encryptedMessages); - const payload = { - messages: { - [this.member.userId]: { - [this.deviceId]: groupMessage.content - } - } - }; + let payload; + let type: string = message.type; + const encryptedMessages = await this.options.encryptDeviceMessage(this.member.userId, this.deviceId, groupMessage, log); + if (encryptedMessages) { + payload = formatToDeviceMessagesPayload(encryptedMessages); + type = "m.room.encrypted"; + } else { + // device needs deviceId and userId + payload = formatToDeviceMessagesPayload([{content: groupMessage.content, device: this}]); + } // TODO: remove this for release log.set("payload", groupMessage.content); const request = this.options.hsApi.sendToDevice( - message.type, - //"m.room.encrypted", + type, payload, makeTxnId(), {log} From 9efe294a79bdf50e6fd7ba1ffa0a32c23fb58d3b Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 1 Jun 2022 15:29:24 +0200 Subject: [PATCH 139/323] fetch and verify keys on olm call signalling message --- src/matrix/DeviceMessageHandler.js | 62 ++++++++++++++---------------- src/matrix/Session.js | 11 +++++- src/matrix/Sync.js | 2 +- src/matrix/e2ee/README.md | 3 ++ 4 files changed, 41 insertions(+), 37 deletions(-) diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index c33236aa..507e07ef 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -38,7 +38,6 @@ export class DeviceMessageHandler { async prepareSync(toDeviceEvents, lock, txn, log) { log.set("messageTypes", countBy(toDeviceEvents, e => e.type)); - this._handleUnencryptedCallEvents(toDeviceEvents, log); const encryptedEvents = toDeviceEvents.filter(e => e.type === "m.room.encrypted"); if (!this._olmDecryption) { log.log("can't decrypt, encryption not enabled", log.level.Warn); @@ -54,20 +53,6 @@ export class DeviceMessageHandler { } const newRoomKeys = this._megolmDecryption.roomKeysFromDeviceMessages(olmDecryptChanges.results, log); - // const callMessages = olmDecryptChanges.results.filter(dr => this._callHandler.handlesDeviceMessageEventType(dr.event?.type)); - // // load devices by sender key - // await Promise.all(callMessages.map(async dr => { - // dr.setDevice(await this._getDevice(dr.senderCurve25519Key, txn)); - // })); - // // TODO: pass this in the prep and run it in afterSync or afterSyncComplete (as callHandler can send events as well)? - // for (const dr of callMessages) { - // if (dr.device) { - // this._callHandler.handleDeviceMessage(dr.event, dr.device.userId, dr.device.deviceId, log); - // } else { - // console.error("could not deliver message because don't have device for sender key", dr.event); - // } - // } - // TODO: somehow include rooms that received a call to_device message in the sync state? // or have updates flow through event emitter? // well, we don't really need to update the room other then when a call starts or stops @@ -76,33 +61,42 @@ export class DeviceMessageHandler { } } - _handleUnencryptedCallEvents(toDeviceEvents, log) { - const callMessages = toDeviceEvents.filter(e => this._callHandler.handlesDeviceMessageEventType(e.type)); - for (const event of callMessages) { - const userId = event.sender; - const deviceId = event.content.device_id; - this._callHandler.handleDeviceMessage(event, userId, deviceId, log); - } - } - /** check that prep is not undefined before calling this */ async writeSync(prep, txn) { // write olm changes prep.olmDecryptChanges.write(txn); const didWriteValues = await Promise.all(prep.newRoomKeys.map(key => this._megolmDecryption.writeRoomKey(key, txn))); - return didWriteValues.some(didWrite => !!didWrite); + const hasNewRoomKeys = didWriteValues.some(didWrite => !!didWrite); + return { + hasNewRoomKeys, + decryptionResults: prep.olmDecryptChanges.results + }; } - - async _getDevice(senderKey, txn) { - let device = this._senderDeviceCache.get(senderKey); - if (!device) { - device = await txn.deviceIdentities.getByCurve25519Key(senderKey); - if (device) { - this._senderDeviceCache.set(device); - } + async afterSyncCompleted(decryptionResults, deviceTracker, hsApi, log) { + // if we don't have a device, we need to fetch the device keys the message claims + // and check the keys, and we should only do network requests during + // sync processing in the afterSyncCompleted step. + const callMessages = decryptionResults.filter(dr => this._callHandler.handlesDeviceMessageEventType(dr.event?.type)); + if (callMessages.length) { + await log.wrap("process call signalling messages", async log => { + for (const dr of callMessages) { + // serialize device loading, so subsequent messages for the same device take advantage of the cache + const device = await deviceTracker.deviceForId(dr.event.sender, dr.event.content.device_id, hsApi, log); + dr.setDevice(device); + if (dr.isVerified) { + this._callHandler.handleDeviceMessage(dr.event, dr.userId, dr.deviceId, log); + } else { + log.log({ + l: "could not verify olm fingerprint key matches, ignoring", + ed25519Key: dr.device.ed25519Key, + claimedEd25519Key: dr.claimedEd25519Key, + deviceId, userId, + }); + } + } + }); } - return device; } } diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 26ab1702..c80cf527 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -685,7 +685,9 @@ export class Session { async writeSync(syncResponse, syncFilterId, preparation, txn, log) { const changes = { syncInfo: null, - e2eeAccountChanges: null + e2eeAccountChanges: null, + hasNewRoomKeys: false, + deviceMessageDecryptionResults: null, }; const syncToken = syncResponse.next_batch; if (syncToken !== this.syncToken) { @@ -706,7 +708,9 @@ export class Session { } if (preparation) { - changes.hasNewRoomKeys = await log.wrap("deviceMsgs", log => this._deviceMessageHandler.writeSync(preparation, txn, log)); + const {hasNewRoomKeys, decryptionResults} = await log.wrap("deviceMsgs", log => this._deviceMessageHandler.writeSync(preparation, txn, log)); + changes.hasNewRoomKeys = hasNewRoomKeys; + changes.deviceMessageDecryptionResults = decryptionResults; } // store account data @@ -747,6 +751,9 @@ export class Session { if (changes.hasNewRoomKeys) { this._keyBackup.get()?.flush(log); } + if (changes.deviceMessageDecryptionResults) { + await this._deviceMessageHandler.afterSyncCompleted(changes.deviceMessageDecryptionResults, this._deviceTracker, this._hsApi, log); + } } _tryReplaceRoomBeingCreated(roomId, log) { diff --git a/src/matrix/Sync.js b/src/matrix/Sync.js index 4f907563..f77fa79a 100644 --- a/src/matrix/Sync.js +++ b/src/matrix/Sync.js @@ -160,7 +160,7 @@ export class Sync { const isCatchupSync = this._status.get() === SyncStatus.CatchupSync; const sessionPromise = (async () => { try { - await log.wrap("session", log => this._session.afterSyncCompleted(sessionChanges, isCatchupSync, log), log.level.Detail); + await log.wrap("session", log => this._session.afterSyncCompleted(sessionChanges, isCatchupSync, log)); } catch (err) {} // error is logged, but don't fail sessionPromise })(); diff --git a/src/matrix/e2ee/README.md b/src/matrix/e2ee/README.md index fab53880..fdb4866c 100644 --- a/src/matrix/e2ee/README.md +++ b/src/matrix/e2ee/README.md @@ -41,5 +41,8 @@ Runs before any room.prepareSync, so the new room keys can be passed to each roo - e2ee account - generate more otks if needed - upload new otks if needed or device keys if not uploaded before + - device message handler: + - fetch keys we don't know about yet for (call) to_device messages identity + - pass signalling messages to call handler - rooms - share new room keys if needed From 3edfbd2cf6f468b90e198079f2aefa4670c731f4 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 1 Jun 2022 15:30:25 +0200 Subject: [PATCH 140/323] await hangup here, so log doesn't terminate early --- src/matrix/calls/group/Member.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index e3abc49e..6a8158b6 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -157,7 +157,7 @@ export class Member { connection.logItem.wrap("disconnect", async log => { disconnectLogItem = log; if (hangup) { - connection.peerCall?.hangup(CallErrorCode.UserHangup, log); + await connection.peerCall?.hangup(CallErrorCode.UserHangup, log); } else { await connection.peerCall?.close(undefined, log); } From 83eef2be9d76417c47e54cc2ac61768b7c7624ba Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 1 Jun 2022 15:30:41 +0200 Subject: [PATCH 141/323] log lack of persisted storage in ... persisted logs! --- src/matrix/storage/idb/StorageFactory.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/storage/idb/StorageFactory.ts b/src/matrix/storage/idb/StorageFactory.ts index 5cb1b6e5..264d8670 100644 --- a/src/matrix/storage/idb/StorageFactory.ts +++ b/src/matrix/storage/idb/StorageFactory.ts @@ -67,7 +67,7 @@ export class StorageFactory { requestPersistedStorage().then(persisted => { // Firefox lies here though, and returns true even if the user denied the request if (!persisted) { - console.warn("no persisted storage, database can be evicted by browser"); + log.log("no persisted storage, database can be evicted by browser", log.level.Warn); } }); From a014740e72d3343c616a39e2a8d03537fa518b07 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 1 Jun 2022 15:55:43 +0200 Subject: [PATCH 142/323] don't throw when we can't encrypt, just fall back to sending unencrypted --- src/matrix/Session.js | 16 ++++++++++------ src/matrix/calls/group/GroupCall.ts | 2 +- src/matrix/calls/group/Member.ts | 2 +- src/matrix/e2ee/DeviceTracker.js | 5 ++++- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index c80cf527..f82ad555 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -81,16 +81,20 @@ export class Session { hsApi: this._hsApi, encryptDeviceMessage: async (roomId, userId, deviceId, message, log) => { if (!this._deviceTracker || !this._olmEncryption) { - throw new Error("encryption is not enabled"); + log.set("encryption_disabled", true); + return; } const device = await log.wrap("get device key", async log => { - return this._deviceTracker.deviceForId(userId, deviceId, this._hsApi, log); + const device = this._deviceTracker.deviceForId(userId, deviceId, this._hsApi, log); + if (!device) { + log.set("not_found", true); + } + return device; }); - if (!device) { - throw new Error(`Could not find device key ${deviceId} for ${userId} in ${roomId}`); + if (device) { + const encryptedMessages = await this._olmEncryption.encrypt(message.type, message.content, [device], this._hsApi, log); + return encryptedMessages; } - const encryptedMessages = await this._olmEncryption.encrypt(message.type, message.content, [device], this._hsApi, log); - return encryptedMessages; }, storage: this._storage, webRTC: this._platform.webRTC, diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index b2b52e9b..ddc0b517 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -55,7 +55,7 @@ function getDeviceFromMemberKey(key: string): string { export type Options = Omit & { emitUpdate: (call: GroupCall, params?: any) => void; - encryptDeviceMessage: (roomId: string, userId: string, deviceId: string, message: SignallingMessage, log: ILogItem) => Promise, + encryptDeviceMessage: (roomId: string, userId: string, deviceId: string, message: SignallingMessage, log: ILogItem) => Promise, storage: Storage, logger: ILogger, }; diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 6a8158b6..4c30a6cf 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -36,7 +36,7 @@ export type Options = Omit, log: ILogItem) => Promise, + encryptDeviceMessage: (userId: string, deviceId: string, message: SignallingMessage, log: ILogItem) => Promise, emitUpdate: (participant: Member, params?: any) => void, } diff --git a/src/matrix/e2ee/DeviceTracker.js b/src/matrix/e2ee/DeviceTracker.js index 49d9ffab..0d0b0e8b 100644 --- a/src/matrix/e2ee/DeviceTracker.js +++ b/src/matrix/e2ee/DeviceTracker.js @@ -338,10 +338,13 @@ export class DeviceTracker { const verifiedKeysPerUser = log.wrap("verify", log => this._filterVerifiedDeviceKeys(deviceKeyResponse["device_keys"], log)); //// END EXTRACT - // there should only be one device in here, but still check the HS sends us the right one const verifiedKeys = verifiedKeysPerUser .find(vkpu => vkpu.userId === userId).verifiedKeys .find(vk => vk["device_id"] === deviceId); + // user hasn't uploaded keys for device? + if (!verifiedKeys) { + return undefined; + } device = deviceKeysAsDeviceIdentity(verifiedKeys); const txn = await this._storage.readWriteTxn([ this._storage.storeNames.deviceIdentities, From a139571e201271b19944369eafec8c41d25e6751 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Jun 2022 10:58:04 +0200 Subject: [PATCH 143/323] move setting seq on outbound messages to member, is specific to_device --- src/matrix/calls/PeerCall.ts | 9 --------- src/matrix/calls/callEventTypes.ts | 2 +- src/matrix/calls/group/Member.ts | 2 ++ 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index e6ce3571..ea7e9d06 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -71,7 +71,6 @@ export class PeerCall implements IDisposable { private localMedia?: LocalMedia; private localMuteSettings?: MuteSettings; // TODO: this should go in member - private seq: number = 0; // A queue for candidates waiting to go out. // We try to amalgamate candidates into a single candidate message where // possible @@ -238,7 +237,6 @@ export class PeerCall implements IDisposable { const content: MCallSDPStreamMetadataChanged = { call_id: this.callId, version: 1, - seq: this.seq++, [SDPStreamMetadataKey]: this.getSDPMetadata() }; await this.sendSignallingMessage({type: EventType.SDPStreamMetadataChangedPrefix, content}, log); @@ -263,7 +261,6 @@ export class PeerCall implements IDisposable { const content: MCallSDPStreamMetadataChanged = { call_id: this.callId, version: 1, - seq: this.seq++, [SDPStreamMetadataKey]: this.getSDPMetadata() }; await this.sendSignallingMessage({type: EventType.SDPStreamMetadataChangedPrefix, content}, log); @@ -350,7 +347,6 @@ export class PeerCall implements IDisposable { const content = { call_id: callId, version: 1, - seq: this.seq++, }; // TODO: Don't send UserHangup reason to older clients if (reason) { @@ -398,7 +394,6 @@ export class PeerCall implements IDisposable { offer, [SDPStreamMetadataKey]: this.getSDPMetadata(), version: 1, - seq: this.seq++, lifetime: CALL_TIMEOUT_MS }; await this.sendSignallingMessage({type: EventType.Invite, content}, log); @@ -409,7 +404,6 @@ export class PeerCall implements IDisposable { description: offer, [SDPStreamMetadataKey]: this.getSDPMetadata(), version: 1, - seq: this.seq++, lifetime: CALL_TIMEOUT_MS }; await this.sendSignallingMessage({type: EventType.Negotiate, content}, log); @@ -674,7 +668,6 @@ export class PeerCall implements IDisposable { description: this.peerConnection.localDescription!, [SDPStreamMetadataKey]: this.getSDPMetadata(), version: 1, - seq: this.seq++, lifetime: CALL_TIMEOUT_MS }; await this.sendSignallingMessage({type: EventType.Negotiate, content}, log); @@ -689,7 +682,6 @@ export class PeerCall implements IDisposable { const answerContent: MCallAnswer = { call_id: this.callId, version: 1, - seq: this.seq++, answer: { sdp: localDescription.sdp, type: localDescription.type, @@ -755,7 +747,6 @@ export class PeerCall implements IDisposable { content: { call_id: this.callId, version: 1, - seq: this.seq++, candidates }, }, log); diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index 0490b44f..09376c85 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -65,7 +65,6 @@ export interface CallReplacesTarget { export type MCallBase = { call_id: string; version: string | number; - seq: number; } export type MGroupCallBase = MCallBase & { @@ -74,6 +73,7 @@ export type MGroupCallBase = MCallBase & { sender_session_id: string; dest_session_id: string; party_id: string; // Should not need this? + seq: number; } export type MCallAnswer = Base & { diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 4c30a6cf..90765569 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -53,6 +53,7 @@ const errorCodesWithoutRetry = [ class MemberConnection { public retryCount: number = 0; public peerCall?: PeerCall; + public outboundSeqCounter: number = 0; constructor( public localMedia: LocalMedia, @@ -212,6 +213,7 @@ export class Member { /** @internal */ sendSignallingMessage = async (message: SignallingMessage, log: ILogItem): Promise => { const groupMessage = message as SignallingMessage; + groupMessage.content.seq = ++this.connection!.outboundSeqCounter; groupMessage.content.conf_id = this.options.confId; groupMessage.content.device_id = this.options.ownDeviceId; groupMessage.content.party_id = this.options.ownDeviceId; From 513c05945988aabf5835fa0bdde2da9b96caad09 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Jun 2022 10:59:14 +0200 Subject: [PATCH 144/323] buffer messages as long as seq numbers in between haven't been received --- src/matrix/calls/group/Member.ts | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 90765569..a40100d2 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -18,7 +18,7 @@ import {PeerCall, CallState} from "../PeerCall"; import {makeTxnId, makeId} from "../../common"; import {EventType, CallErrorCode} from "../callEventTypes"; import {formatToDeviceMessagesPayload} from "../../common"; - +import {sortedIndex} from "../../../utils/sortedIndex"; import type {MuteSettings} from "../common"; import type {Options as PeerCallOptions, RemoteMedia} from "../PeerCall"; import type {LocalMedia} from "../LocalMedia"; @@ -53,6 +53,8 @@ const errorCodesWithoutRetry = [ class MemberConnection { public retryCount: number = 0; public peerCall?: PeerCall; + public lastProcessedSeqNr: number | undefined; + public queuedSignallingMessages: SignallingMessage[] = []; public outboundSeqCounter: number = 0; constructor( @@ -253,11 +255,19 @@ export class Member { if (message.type === EventType.Invite && !connection.peerCall) { connection.peerCall = this._createPeerCall(message.content.call_id); } + const idx = sortedIndex(connection.queuedSignallingMessages, message, (a, b) => a.content.seq - b.content.seq); + connection.queuedSignallingMessages.splice(idx, 0, message); if (connection.peerCall) { - const item = connection.peerCall.handleIncomingSignallingMessage(message, this.deviceId, connection.logItem); - syncLog.refDetached(item); - } else { - // TODO: need to buffer events until invite comes? + while ( + connection.queuedSignallingMessages.length && ( + connection.lastProcessedSeqNr === undefined || + connection.queuedSignallingMessages[0].content.seq === connection.lastProcessedSeqNr + 1 + )) { + const dequeuedMessage = connection.queuedSignallingMessages.shift()!; + const item = connection.peerCall!.handleIncomingSignallingMessage(dequeuedMessage, this.deviceId, connection.logItem); + connection.lastProcessedSeqNr = dequeuedMessage.content.seq; + syncLog.refDetached(item); + } } } else { syncLog.log({l: "member not connected", userId: this.userId, deviceId: this.deviceId}); From a530944f7d9023c16f40883be88ad6e68489ff75 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Jun 2022 11:11:32 +0200 Subject: [PATCH 145/323] add logging to seq queueing --- src/matrix/calls/group/Member.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index a40100d2..541cdf13 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -257,18 +257,26 @@ export class Member { } const idx = sortedIndex(connection.queuedSignallingMessages, message, (a, b) => a.content.seq - b.content.seq); connection.queuedSignallingMessages.splice(idx, 0, message); + let hasBeenDequeued = false; if (connection.peerCall) { while ( connection.queuedSignallingMessages.length && ( - connection.lastProcessedSeqNr === undefined || - connection.queuedSignallingMessages[0].content.seq === connection.lastProcessedSeqNr + 1 - )) { + connection.lastProcessedSeqNr === undefined || + connection.queuedSignallingMessages[0].content.seq === connection.lastProcessedSeqNr + 1 + ) + ) { const dequeuedMessage = connection.queuedSignallingMessages.shift()!; + if (dequeuedMessage === message) { + hasBeenDequeued = true; + } const item = connection.peerCall!.handleIncomingSignallingMessage(dequeuedMessage, this.deviceId, connection.logItem); - connection.lastProcessedSeqNr = dequeuedMessage.content.seq; syncLog.refDetached(item); + connection.lastProcessedSeqNr = dequeuedMessage.content.seq; } } + if (!hasBeenDequeued) { + syncLog.refDetached(connection.logItem.log({l: "queued signalling message", type: message.type})); + } } else { syncLog.log({l: "member not connected", userId: this.userId, deviceId: this.deviceId}); } From a52740ed1b4fd7fcacc8bd2f9a9c78ceb17b3d9f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Jun 2022 15:55:08 +0200 Subject: [PATCH 146/323] give room state handler access to member sync to get sender profile info --- src/matrix/room/Room.js | 14 ++++++++------ src/matrix/room/state/RoomStateHandlerSet.ts | 7 +++++-- src/matrix/room/state/types.ts | 5 +++-- .../room/timeline/persistence/MemberWriter.js | 6 +++++- src/matrix/room/timeline/persistence/SyncWriter.js | 2 +- 5 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js index 38982786..425401d8 100644 --- a/src/matrix/room/Room.js +++ b/src/matrix/room/Room.js @@ -123,7 +123,7 @@ export class Room extends BaseRoom { txn.roomState.removeAllForRoom(this.id); txn.roomMembers.removeAllForRoom(this.id); } - const {entries: newEntries, updatedEntries, newLiveKey, memberChanges} = + const {entries: newEntries, updatedEntries, newLiveKey, memberChanges, memberSync} = await log.wrap("syncWriter", log => this._syncWriter.writeSync( roomResponse, isRejoin, summaryChanges.hasFetchedMembers, txn, log), log.level.Detail); if (decryptChanges) { @@ -180,7 +180,7 @@ export class Room extends BaseRoom { removedPendingEvents = await this._sendQueue.removeRemoteEchos(roomResponse.timeline.events, txn, log); } const powerLevelsEvent = this._getPowerLevelsEvent(roomResponse); - this._runRoomStateHandlers(roomResponse, txn, log); + await this._runRoomStateHandlers(roomResponse, memberSync, txn, log); return { roomResponse, summaryChanges, @@ -453,14 +453,16 @@ export class Room extends BaseRoom { return this._sendQueue.pendingEvents; } - /** global room state handlers, run during write sync step */ - _runRoomStateHandlers(roomResponse, txn, log) { + /** global room state handlers, run during writeSync step */ + _runRoomStateHandlers(roomResponse, memberSync, txn, log) { + const promises = []; iterateResponseStateEvents(roomResponse, event => { - this._roomStateHandler.handleRoomState(this, event, txn, log); + promises.push(this._roomStateHandler.handleRoomState(this, event, memberSync, txn, log)); }); + return Promise.all(promises); } - /** local room state observers, run during after sync step */ + /** local room state observers, run during afterSync step */ _emitSyncRoomState(roomResponse) { iterateResponseStateEvents(roomResponse, event => { for (const handler of this._roomStateObservers) { diff --git a/src/matrix/room/state/RoomStateHandlerSet.ts b/src/matrix/room/state/RoomStateHandlerSet.ts index 986cb0f9..491b02ce 100644 --- a/src/matrix/room/state/RoomStateHandlerSet.ts +++ b/src/matrix/room/state/RoomStateHandlerSet.ts @@ -20,14 +20,17 @@ import type {Transaction} from "../../storage/idb/Transaction"; import type {Room} from "../Room"; import type {MemberChange} from "../members/RoomMember"; import type {RoomStateHandler} from "./types"; +import type {MemberSync} from "../timeline/persistence/MemberWriter.js"; import {BaseObservable} from "../../../observable/BaseObservable"; /** keeps track of all handlers registered with Session.observeRoomState */ export class RoomStateHandlerSet extends BaseObservable implements RoomStateHandler { - handleRoomState(room: Room, stateEvent: StateEvent, txn: Transaction, log: ILogItem) { + async handleRoomState(room: Room, stateEvent: StateEvent, memberSync: MemberSync, txn: Transaction, log: ILogItem): Promise { + const promises: Promise[] = []; for(let h of this._handlers) { - h.handleRoomState(room, stateEvent, txn, log); + promises.push(h.handleRoomState(room, stateEvent, memberSync, txn, log)); } + await Promise.all(promises); } updateRoomMembers(room: Room, memberChanges: Map) { for(let h of this._handlers) { diff --git a/src/matrix/room/state/types.ts b/src/matrix/room/state/types.ts index ef99c727..2e7167d2 100644 --- a/src/matrix/room/state/types.ts +++ b/src/matrix/room/state/types.ts @@ -19,12 +19,13 @@ import type {StateEvent} from "../../storage/types"; import type {Transaction} from "../../storage/idb/Transaction"; import type {ILogItem} from "../../../logging/types"; import type {MemberChange} from "../members/RoomMember"; +import type {MemberSync} from "../timeline/persistence/MemberWriter"; /** used for Session.observeRoomState, which observes in all room, but without loading from storage * It receives the sync write transaction, so other stores can be updated as part of the same transaction. */ export interface RoomStateHandler { - handleRoomState(room: Room, stateEvent: StateEvent, syncWriteTxn: Transaction, log: ILogItem); - updateRoomMembers(room: Room, memberChanges: Map); + handleRoomState(room: Room, stateEvent: StateEvent, memberSync: MemberSync, syncWriteTxn: Transaction, log: ILogItem): Promise; + updateRoomMembers(room: Room, memberChanges: Map): void; } /** diff --git a/src/matrix/room/timeline/persistence/MemberWriter.js b/src/matrix/room/timeline/persistence/MemberWriter.js index 1cdcb7d5..9345324e 100644 --- a/src/matrix/room/timeline/persistence/MemberWriter.js +++ b/src/matrix/room/timeline/persistence/MemberWriter.js @@ -56,7 +56,11 @@ export class MemberWriter { } } -class MemberSync { +/** Represents the member changes in a given sync. + * Used to write the changes to storage and historical member + * information for events in the same sync. + **/ +export class MemberSync { constructor(memberWriter, stateEvents, timelineEvents, hasFetchedMembers) { this._memberWriter = memberWriter; this._timelineEvents = timelineEvents; diff --git a/src/matrix/room/timeline/persistence/SyncWriter.js b/src/matrix/room/timeline/persistence/SyncWriter.js index dc5ae3a8..76c7bec7 100644 --- a/src/matrix/room/timeline/persistence/SyncWriter.js +++ b/src/matrix/room/timeline/persistence/SyncWriter.js @@ -244,7 +244,7 @@ export class SyncWriter { const {currentKey, entries, updatedEntries} = await this._writeTimeline(timelineEvents, timeline, memberSync, this._lastLiveKey, txn, log); const memberChanges = await memberSync.write(txn); - return {entries, updatedEntries, newLiveKey: currentKey, memberChanges}; + return {entries, updatedEntries, newLiveKey: currentKey, memberChanges, memberSync}; } afterSync(newLiveKey) { From 90b6a5ccb625066325b4a45e4ae3fc1c1fb3b026 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Jun 2022 15:56:23 +0200 Subject: [PATCH 147/323] update call member info with room member info --- src/domain/session/room/CallViewModel.ts | 5 +++- src/matrix/calls/CallHandler.ts | 38 ++++++++++++++++-------- src/matrix/calls/group/GroupCall.ts | 19 ++++++++++-- src/matrix/calls/group/Member.ts | 9 +++++- 4 files changed, 53 insertions(+), 18 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 08bc3691..a770ddbe 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -117,7 +117,10 @@ class OwnMemberViewModel extends ViewModel implements IStreamV } } -type MemberOptions = BaseOptions & {member: Member, mediaRepository: MediaRepository}; +type MemberOptions = BaseOptions & { + member: Member, + mediaRepository: MediaRepository +}; export class CallMemberViewModel extends ViewModel implements IStreamViewModel { get stream(): Stream | undefined { diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index e585bb40..7524200c 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -22,6 +22,7 @@ import {EventType, CallIntent} from "./callEventTypes"; import {GroupCall} from "./group/GroupCall"; import {makeId} from "../common"; import {CALL_LOG_TYPE} from "./common"; +import {EVENT_TYPE as MEMBER_EVENT_TYPE, RoomMember} from "../room/members/RoomMember"; import type {LocalMedia} from "./LocalMedia"; import type {Room} from "../room/Room"; @@ -36,6 +37,7 @@ import type {Transaction} from "../storage/idb/Transaction"; import type {CallEntry} from "../storage/idb/stores/CallStore"; import type {Clock} from "../../platform/web/dom/Clock"; import type {RoomStateHandler} from "../room/state/types"; +import type {MemberSync} from "../room/timeline/persistence/MemberWriter"; export type Options = Omit & { clock: Clock @@ -77,7 +79,7 @@ export class CallHandler implements RoomStateHandler { const names = this.options.storage.storeNames; const txn = await this.options.storage.readTxn([ names.calls, - names.roomState + names.roomState, ]); return txn; } @@ -97,15 +99,17 @@ export class CallHandler implements RoomStateHandler { })); const roomIds = Array.from(new Set(callEntries.map(e => e.roomId))); await Promise.all(roomIds.map(async roomId => { - // const ownCallsMemberEvent = await txn.roomState.get(roomId, EventType.GroupCallMember, this.options.ownUserId); - // if (ownCallsMemberEvent) { - // this.handleCallMemberEvent(ownCallsMemberEvent.event, log); - // } + // TODO: don't load all members until we need them const callsMemberEvents = await txn.roomState.getAllForType(roomId, EventType.GroupCallMember); - for (const entry of callsMemberEvents) { - this.handleCallMemberEvent(entry.event, roomId, log); - } - // TODO: we should be loading the other members as well at some point + await Promise.all(callsMemberEvents.map(async entry => { + const roomMemberState = await txn.roomState.get(roomId, MEMBER_EVENT_TYPE, entry.event.sender); + if (roomMemberState) { + const roomMember = RoomMember.fromMemberEvent(roomMemberState.event); + if (roomMember) { + this.handleCallMemberEvent(entry.event, roomMember, roomId, log); + } + } + })); })); log.set("newSize", this._calls.size); }); @@ -144,12 +148,15 @@ export class CallHandler implements RoomStateHandler { // TODO: check and poll turn server credentials here /** @internal */ - handleRoomState(room: Room, event: StateEvent, txn: Transaction, log: ILogItem) { + async handleRoomState(room: Room, event: StateEvent, memberSync: MemberSync, txn: Transaction, log: ILogItem) { if (event.type === EventType.GroupCall) { this.handleCallEvent(event, room.id, txn, log); } if (event.type === EventType.GroupCallMember) { - this.handleCallMemberEvent(event, room.id, log); + const member: RoomMember | undefined = await memberSync.lookupMemberAtEvent(event.sender, event, txn); + if (member) { // should always have a member? + this.handleCallMemberEvent(event, member, room.id, log); + } } } @@ -157,6 +164,11 @@ export class CallHandler implements RoomStateHandler { updateRoomMembers(room: Room, memberChanges: Map) { // TODO: also have map for roomId to calls, so we can easily update members // we will also need this to get the call for a room + for (const call of this._calls.values()) { + if (call.roomId === room.id) { + call.updateRoomMembers(memberChanges); + } + } } /** @internal */ @@ -193,7 +205,7 @@ export class CallHandler implements RoomStateHandler { } } - private handleCallMemberEvent(event: StateEvent, roomId: string, log: ILogItem) { + private handleCallMemberEvent(event: StateEvent, member: RoomMember, roomId: string, log: ILogItem) { const userId = event.state_key; const roomMemberKey = getRoomMemberKey(roomId, userId) const calls = event.content["m.calls"] ?? []; @@ -201,7 +213,7 @@ export class CallHandler implements RoomStateHandler { const callId = call["m.call_id"]; const groupCall = this._calls.get(callId); // TODO: also check the member when receiving the m.call event - groupCall?.updateMembership(userId, call, log); + groupCall?.updateMembership(userId, member, call, log); }; const newCallIdsMemberOf = new Set(calls.map(call => call["m.call_id"])); let previousCallIdsMemberOf = this.roomMemberToCallIds.get(roomMemberKey); diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index ddc0b517..ebacf844 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -18,7 +18,7 @@ import {ObservableMap} from "../../../observable/map/ObservableMap"; import {Member} from "./Member"; import {LocalMedia} from "../LocalMedia"; import {MuteSettings, CALL_LOG_TYPE} from "../common"; -import {RoomMember} from "../../room/members/RoomMember"; +import {MemberChange, RoomMember} from "../../room/members/RoomMember"; import {EventEmitter} from "../../../utils/EventEmitter"; import {EventType, CallIntent} from "../callEventTypes"; @@ -258,7 +258,20 @@ export class GroupCall extends EventEmitter<{change: never}> { } /** @internal */ - updateMembership(userId: string, callMembership: CallMembership, syncLog: ILogItem) { + updateRoomMembers(memberChanges: Map) { + for (const change of memberChanges.values()) { + const {member} = change; + for (const callMember of this._members.values()) { + // find all call members for a room member (can be multiple, for every device) + if (callMember.userId === member.userId) { + callMember.updateRoomMember(member); + } + } + } + } + + /** @internal */ + updateMembership(userId: string, roomMember: RoomMember, callMembership: CallMembership, syncLog: ILogItem) { syncLog.wrap({l: "update call membership", t: CALL_LOG_TYPE, id: this.id, userId}, log => { const devices = callMembership["m.devices"]; const previousDeviceIds = this.getDeviceIdsForUserId(userId); @@ -290,7 +303,7 @@ export class GroupCall extends EventEmitter<{change: never}> { } log.set("add", true); member = new Member( - RoomMember.fromUserId(this.roomId, userId, "join"), + roomMember, device, this._memberOptions, ); this._members.add(memberKey, member); diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 541cdf13..2a4eefb7 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -68,7 +68,7 @@ export class Member { private connection?: MemberConnection; constructor( - public readonly member: RoomMember, + public member: RoomMember, private callDeviceMembership: CallDeviceMembership, private readonly options: Options, ) {} @@ -179,6 +179,13 @@ export class Member { this.connection.logItem.refDetached(causeItem); } } + + /** @internal */ + updateRoomMember(roomMember: RoomMember) { + this.member = roomMember; + // TODO: this emits an update during the writeSync phase, which we usually try to avoid + this.options.emitUpdate(this); + } /** @internal */ emitUpdateFromPeerCall = (peerCall: PeerCall, params: any, log: ILogItem): void => { From 0c20beb1c0a43dff5275d8711c565b718cf7c36d Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Jun 2022 15:58:03 +0200 Subject: [PATCH 148/323] always pass mediaRepo to call vm --- src/domain/session/room/RoomViewModel.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index c60ce649..f8b70508 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -67,8 +67,9 @@ export class RoomViewModel extends ViewModel { this.emitChange("callViewModel"); })); const call = this._callObservable.get(); + // TODO: cleanup this duplication to create CallViewModel if (call) { - this._callViewModel = new CallViewModel(this.childOptions({call})); + this._callViewModel = this.track(new CallViewModel(this.childOptions({call, mediaRepository: this._room.mediaRepository}))); } } From d66d810fe2a4daa3dcbed4d2f97fbce5a4a393e1 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Jun 2022 15:58:26 +0200 Subject: [PATCH 149/323] pass updates to avatar view --- src/platform/web/ui/session/room/CallView.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/platform/web/ui/session/room/CallView.ts b/src/platform/web/ui/session/room/CallView.ts index 92dfe6b9..ac31973c 100644 --- a/src/platform/web/ui/session/room/CallView.ts +++ b/src/platform/web/ui/session/room/CallView.ts @@ -49,7 +49,7 @@ class StreamView extends TemplateView { t.div({className: { StreamView_avatar: true, hidden: vm => !vm.isCameraMuted - }}, t.view(new AvatarView(vm, 64), {parentProvidesUpdates: true})), + }}, t.view(new AvatarView(vm, 96), {parentProvidesUpdates: true})), t.div({ className: { StreamView_muteStatus: true, @@ -60,4 +60,10 @@ class StreamView extends TemplateView { }) ]); } + + update(value, props) { + super.update(value); + // update the AvatarView as we told it to not subscribe itself with parentProvidesUpdates + this.updateSubViews(value, props); + } } From f452c3ff4c56f1ae4fb42a46107f10ff02b33668 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Jun 2022 15:58:38 +0200 Subject: [PATCH 150/323] enable 96px avatars --- src/platform/web/ui/css/avatar.css | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/platform/web/ui/css/avatar.css b/src/platform/web/ui/css/avatar.css index 2ee9ca0c..a8f8080d 100644 --- a/src/platform/web/ui/css/avatar.css +++ b/src/platform/web/ui/css/avatar.css @@ -46,6 +46,14 @@ limitations under the License. font-size: calc(var(--avatar-size) * 0.6); } +.hydrogen .avatar.size-96 { + --avatar-size: 96px; + width: var(--avatar-size); + height: var(--avatar-size); + line-height: var(--avatar-size); + font-size: calc(var(--avatar-size) * 0.6); +} + .hydrogen .avatar.size-64 { --avatar-size: 64px; width: var(--avatar-size); From 8ba1d085f6a1ed011f464d9e38ea3d414fe991d7 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Jun 2022 15:58:50 +0200 Subject: [PATCH 151/323] fix refactor mistake in logging --- src/matrix/DeviceMessageHandler.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index 507e07ef..05276549 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -91,7 +91,8 @@ export class DeviceMessageHandler { l: "could not verify olm fingerprint key matches, ignoring", ed25519Key: dr.device.ed25519Key, claimedEd25519Key: dr.claimedEd25519Key, - deviceId, userId, + deviceId: device.deviceId, + userId: device.userId, }); } } From c8b5c6dd41e89a20a3a3913a4efbed9046c8f7b8 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Jun 2022 17:30:17 +0200 Subject: [PATCH 152/323] expose own user on BaseRoom so we don't have to pass session around everywhere we need this --- src/matrix/room/BaseRoom.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/matrix/room/BaseRoom.js b/src/matrix/room/BaseRoom.js index b8f172d0..9ca75a24 100644 --- a/src/matrix/room/BaseRoom.js +++ b/src/matrix/room/BaseRoom.js @@ -446,6 +446,10 @@ export class BaseRoom extends EventEmitter { return this._summary.data.membership; } + get user() { + return this._user; + } + isDirectMessageForUserId(userId) { if (this._summary.data.dmUserId === userId) { return true; From 5280467e6612de359b332ad28133cede54f569a3 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Jun 2022 17:30:43 +0200 Subject: [PATCH 153/323] return type is actual subclass options, not the options of ViewModel --- src/domain/ViewModel.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/ViewModel.ts b/src/domain/ViewModel.ts index 0bc52f6e..debf8e34 100644 --- a/src/domain/ViewModel.ts +++ b/src/domain/ViewModel.ts @@ -47,7 +47,7 @@ export class ViewModel extends EventEmitter<{change this._options = options; } - childOptions(explicitOptions: T): T & Options { + childOptions(explicitOptions: T): T & O { return Object.assign({}, this._options, explicitOptions); } From f8b01ac3ccd09fe3527aa7704360d76565f10cd2 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Jun 2022 17:31:17 +0200 Subject: [PATCH 154/323] show profile info for own call member by observing member on room --- src/domain/session/room/CallViewModel.ts | 55 ++++++++++++++++++------ src/domain/session/room/RoomViewModel.js | 4 +- 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index a770ddbe..265648bf 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -20,15 +20,18 @@ import {getStreamVideoTrack, getStreamAudioTrack} from "../../../matrix/calls/co import {avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../avatar"; import {EventObservableValue} from "../../../observable/value/EventObservableValue"; import {ObservableValueMap} from "../../../observable/map/ObservableValueMap"; +import type {Room} from "../../../matrix/room/Room"; import type {GroupCall} from "../../../matrix/calls/group/GroupCall"; import type {Member} from "../../../matrix/calls/group/Member"; +import type {RoomMember} from "../../../matrix/room/members/RoomMember"; import type {BaseObservableList} from "../../../observable/list/BaseObservableList"; +import type {BaseObservableValue} from "../../../observable/value/BaseObservableValue"; import type {Stream} from "../../../platform/types/MediaDevices"; import type {MediaRepository} from "../../../matrix/net/MediaRepository"; type Options = BaseOptions & { call: GroupCall, - mediaRepository: MediaRepository + room: Room, }; export class CallViewModel extends ViewModel { @@ -37,10 +40,10 @@ export class CallViewModel extends ViewModel { constructor(options: Options) { super(options); const ownMemberViewModelMap = new ObservableValueMap("self", new EventObservableValue(this.call, "change")) - .mapValues(call => new OwnMemberViewModel(this.childOptions({call: this.call, mediaRepository: this.getOption("mediaRepository")})), () => {}); + .mapValues((call, emitChange) => new OwnMemberViewModel(this.childOptions({call, emitChange})), () => {}); this.memberViewModels = this.call.members .filterValues(member => member.isConnected) - .mapValues(member => new CallMemberViewModel(this.childOptions({member, mediaRepository: this.getOption("mediaRepository")}))) + .mapValues(member => new CallMemberViewModel(this.childOptions({member, mediaRepository: this.getOption("room").mediaRepository}))) .join(ownMemberViewModelMap) .sortValues((a, b) => a.compare(b)); } @@ -74,12 +77,22 @@ export class CallViewModel extends ViewModel { } } -type OwnMemberOptions = BaseOptions & { - call: GroupCall, - mediaRepository: MediaRepository -} +class OwnMemberViewModel extends ViewModel implements IStreamViewModel { + private memberObservable: undefined | BaseObservableValue; + + constructor(options: Options) { + super(options); + this.init(); + } + + async init() { + const room = this.getOption("room"); + this.memberObservable = await room.observeMember(room.user.id); + this.track(this.memberObservable!.subscribe(() => { + this.emitChange(undefined); + })); + } -class OwnMemberViewModel extends ViewModel implements IStreamViewModel { get stream(): Stream | undefined { return this.call.localMedia?.userMedia; } @@ -97,19 +110,37 @@ class OwnMemberViewModel extends ViewModel implements IStreamV } get avatarLetter(): string { - return "I"; + const member = this.memberObservable?.get(); + if (member) { + return avatarInitials(member.name); + } else { + return ""; + } } get avatarColorNumber(): number { - return 3; + const member = this.memberObservable?.get(); + if (member) { + return getIdentifierColorNumber(member.userId); + } else { + return 0; + } } avatarUrl(size: number): string | undefined { - return undefined; + const member = this.memberObservable?.get(); + if (member) { + return getAvatarHttpUrl(member.avatarUrl, size, this.platform, this.getOption("room").mediaRepository); + } } get avatarTitle(): string { - return "Me"; + const member = this.memberObservable?.get(); + if (member) { + return member.name; + } else { + return ""; + } } compare(other: OwnMemberViewModel | CallMemberViewModel): number { diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index f8b70508..077a996e 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -62,14 +62,14 @@ export class RoomViewModel extends ViewModel { } this._callViewModel = this.disposeTracked(this._callViewModel); if (call) { - this._callViewModel = this.track(new CallViewModel(this.childOptions({call, mediaRepository: this._room.mediaRepository}))); + this._callViewModel = this.track(new CallViewModel(this.childOptions({call, room: this._room}))); } this.emitChange("callViewModel"); })); const call = this._callObservable.get(); // TODO: cleanup this duplication to create CallViewModel if (call) { - this._callViewModel = this.track(new CallViewModel(this.childOptions({call, mediaRepository: this._room.mediaRepository}))); + this._callViewModel = this.track(new CallViewModel(this.childOptions({call, room: this._room}))); } } From ed5fdb8154513b67d07d8fc3f7494a63e9f8c3e7 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 3 Jun 2022 12:43:51 +0200 Subject: [PATCH 155/323] don't withhold member event for call just because we don't have profile --- src/matrix/calls/CallHandler.ts | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 7524200c..ca58c4a8 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -102,13 +102,18 @@ export class CallHandler implements RoomStateHandler { // TODO: don't load all members until we need them const callsMemberEvents = await txn.roomState.getAllForType(roomId, EventType.GroupCallMember); await Promise.all(callsMemberEvents.map(async entry => { - const roomMemberState = await txn.roomState.get(roomId, MEMBER_EVENT_TYPE, entry.event.sender); + const userId = entry.event.sender; + const roomMemberState = await txn.roomState.get(roomId, MEMBER_EVENT_TYPE, userId); + let roomMember; if (roomMemberState) { - const roomMember = RoomMember.fromMemberEvent(roomMemberState.event); - if (roomMember) { - this.handleCallMemberEvent(entry.event, roomMember, roomId, log); - } + roomMember = RoomMember.fromMemberEvent(roomMemberState.event); } + if (!roomMember) { + // we'll be missing the member here if we received a call and it's members + // as pre-gap state and the members weren't active in the timeline we got. + roomMember = RoomMember.fromUserId(roomId, userId, "join"); + } + this.handleCallMemberEvent(entry.event, roomMember, roomId, log); })); })); log.set("newSize", this._calls.size); @@ -153,10 +158,13 @@ export class CallHandler implements RoomStateHandler { this.handleCallEvent(event, room.id, txn, log); } if (event.type === EventType.GroupCallMember) { - const member: RoomMember | undefined = await memberSync.lookupMemberAtEvent(event.sender, event, txn); - if (member) { // should always have a member? - this.handleCallMemberEvent(event, member, room.id, log); + let member = await memberSync.lookupMemberAtEvent(event.sender, event, txn); + if (!member) { + // we'll be missing the member here if we received a call and it's members + // as pre-gap state and the members weren't active in the timeline we got. + member = RoomMember.fromUserId(room.id, event.sender, "join"); } + this.handleCallMemberEvent(event, member, room.id, log); } } From 1fab314dd51c4b7356d047a07bdd04bd3d35ad44 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 3 Jun 2022 15:49:45 +0200 Subject: [PATCH 156/323] return user id for own avatar in call if member hasn't been found --- src/domain/session/room/CallViewModel.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 265648bf..cd974605 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -114,17 +114,12 @@ class OwnMemberViewModel extends ViewModel implements IStreamViewModel if (member) { return avatarInitials(member.name); } else { - return ""; + return this.getOption("room").user.id; } } get avatarColorNumber(): number { - const member = this.memberObservable?.get(); - if (member) { - return getIdentifierColorNumber(member.userId); - } else { - return 0; - } + return getIdentifierColorNumber(this.getOption("room").user.id); } avatarUrl(size: number): string | undefined { @@ -139,7 +134,7 @@ class OwnMemberViewModel extends ViewModel implements IStreamViewModel if (member) { return member.name; } else { - return ""; + return this.getOption("room").user.id; } } From bfdea03bbdff2c44fda85bc922146786164739eb Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 3 Jun 2022 15:50:02 +0200 Subject: [PATCH 157/323] start with seq 1, like Element Call does --- src/matrix/calls/group/Member.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 2a4eefb7..cf6bb6f0 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -222,7 +222,7 @@ export class Member { /** @internal */ sendSignallingMessage = async (message: SignallingMessage, log: ILogItem): Promise => { const groupMessage = message as SignallingMessage; - groupMessage.content.seq = ++this.connection!.outboundSeqCounter; + groupMessage.content.seq = this.connection!.outboundSeqCounter++; groupMessage.content.conf_id = this.options.confId; groupMessage.content.device_id = this.options.ownDeviceId; groupMessage.content.party_id = this.options.ownDeviceId; From 10caba68727707bbfd11d607aedd3d42cadfd210 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 9 Jun 2022 15:33:59 +0200 Subject: [PATCH 158/323] improve calls view --- src/domain/ViewModel.ts | 2 +- src/domain/session/room/CallViewModel.ts | 52 ++++- .../web/ui/css/themes/element/call.css | 206 ++++++++++++++++++ .../ui/css/themes/element/icons/cam-muted.svg | 1 + .../css/themes/element/icons/cam-unmuted.svg | 1 + .../ui/css/themes/element/icons/hangup.svg | 3 + .../ui/css/themes/element/icons/mic-muted.svg | 19 ++ .../css/themes/element/icons/mic-unmuted.svg | 16 ++ .../css/themes/element/icons/video-call.svg | 4 + .../css/themes/element/icons/voice-call.svg | 3 + .../web/ui/css/themes/element/theme.css | 54 +---- src/platform/web/ui/session/room/CallView.ts | 65 +++++- 12 files changed, 359 insertions(+), 67 deletions(-) create mode 100644 src/platform/web/ui/css/themes/element/call.css create mode 100644 src/platform/web/ui/css/themes/element/icons/cam-muted.svg create mode 100644 src/platform/web/ui/css/themes/element/icons/cam-unmuted.svg create mode 100644 src/platform/web/ui/css/themes/element/icons/hangup.svg create mode 100644 src/platform/web/ui/css/themes/element/icons/mic-muted.svg create mode 100644 src/platform/web/ui/css/themes/element/icons/mic-unmuted.svg create mode 100644 src/platform/web/ui/css/themes/element/icons/video-call.svg create mode 100644 src/platform/web/ui/css/themes/element/icons/voice-call.svg diff --git a/src/domain/ViewModel.ts b/src/domain/ViewModel.ts index debf8e34..257624ea 100644 --- a/src/domain/ViewModel.ts +++ b/src/domain/ViewModel.ts @@ -115,7 +115,7 @@ export class ViewModel extends EventEmitter<{change return result; } - emitChange(changedProps: any): void { + emitChange(changedProps?: any): void { if (this._options.emitChange) { this._options.emitChange(changedProps); } else { diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index cd974605..3e7fd951 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -46,12 +46,32 @@ export class CallViewModel extends ViewModel { .mapValues(member => new CallMemberViewModel(this.childOptions({member, mediaRepository: this.getOption("room").mediaRepository}))) .join(ownMemberViewModelMap) .sortValues((a, b) => a.compare(b)); + this.track(this.memberViewModels.subscribe({ + onRemove: () => { + this.emitChange(); // update memberCount + }, + onAdd: () => { + this.emitChange(); // update memberCount + }, + onUpdate: () => {}, + onReset: () => {}, + onMove: () => {} + })) } - private get call(): GroupCall { - return this.getOption("call"); + get isCameraMuted(): boolean { + return isLocalCameraMuted(this.call); } + get isMicrophoneMuted(): boolean { + return isLocalMicrophoneMuted(this.call); + } + + get memberCount(): number { + return this.memberViewModels.length; + } + + get name(): string { return this.call.name; } @@ -60,19 +80,27 @@ export class CallViewModel extends ViewModel { return this.call.id; } - get stream(): Stream | undefined { - return this.call.localMedia?.userMedia; + private get call(): GroupCall { + return this.getOption("call"); } - leave() { + hangup() { if (this.call.hasJoined) { this.call.leave(); } } - async toggleVideo() { + async toggleCamera() { if (this.call.muteSettings) { this.call.setMuted(this.call.muteSettings.toggleCamera()); + this.emitChange(); + } + } + + async toggleMicrophone() { + if (this.call.muteSettings) { + this.call.setMuted(this.call.muteSettings.toggleMicrophone()); + this.emitChange(); } } } @@ -102,11 +130,11 @@ class OwnMemberViewModel extends ViewModel implements IStreamViewModel } get isCameraMuted(): boolean { - return isMuted(this.call.muteSettings?.camera, !!getStreamVideoTrack(this.stream)); + return isLocalCameraMuted(this.call); } get isMicrophoneMuted(): boolean { - return isMuted(this.call.muteSettings?.microphone, !!getStreamAudioTrack(this.stream)); + return isLocalMicrophoneMuted(this.call); } get avatarLetter(): string { @@ -209,3 +237,11 @@ function isMuted(muted: boolean | undefined, hasTrack: boolean) { return !hasTrack; } } + +function isLocalCameraMuted(call: GroupCall): boolean { + return isMuted(call.muteSettings?.camera, !!getStreamVideoTrack(call.localMedia?.userMedia)); +} + +function isLocalMicrophoneMuted(call: GroupCall): boolean { + return isMuted(call.muteSettings?.microphone, !!getStreamAudioTrack(call.localMedia?.userMedia)); +} diff --git a/src/platform/web/ui/css/themes/element/call.css b/src/platform/web/ui/css/themes/element/call.css new file mode 100644 index 00000000..4398f9c6 --- /dev/null +++ b/src/platform/web/ui/css/themes/element/call.css @@ -0,0 +1,206 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.CallView { + height: 40vh; + display: grid; +} + +.CallView > * { + grid-column: 1; + grid-row: 1; +} + +.CallView_members { + display: grid; + gap: 12px; + background: var(--background-color-secondary--darker-60); + padding: 12px; + margin: 0; + min-height: 0; + list-style: none; + align-self: stretch; +} + +.StreamView { + display: grid; + border-radius: 8px; + overflow: hidden; + background-color: black; +} + +.StreamView > * { + grid-column: 1; + grid-row: 1; +} + +.StreamView video { + width: 100%; + height: 100%; + object-fit: contain; +} + +.StreamView_avatar { + align-self: center; + justify-self: center; +} + +.StreamView_muteStatus { + align-self: start; + justify-self: end; + width: 24px; + height: 24px; + background-position: center; + background-repeat: no-repeat; + background-size: 14px; + display: block; + background-color: var(--text-color); + border-radius: 4px; + margin: 4px; +} + +.StreamView_muteStatus.microphoneMuted { + background-image: url("./icons/mic-muted.svg?primary=text-color--lighter-80"); +} + +.StreamView_muteStatus.cameraMuted { + background-image: url("./icons/cam-muted.svg?primary=text-color--lighter-80"); +} + +.CallView_buttons { + align-self: end; + justify-self: center; + display: flex; + gap: 12px; + margin-bottom: 16px; +} + +.CallView_buttons button { + border-radius: 100%; + width: 48px; + height: 48px; + border: none; + background-color: var(--accent-color); + background-position: center; + background-repeat: no-repeat; +} + +.CallView_buttons .CallView_hangup { + background-color: var(--error-color); + background-image: url("./icons/hangup.svg?primary=background-color-primary"); +} + +.CallView_buttons .CallView_mutedMicrophone { + background-color: var(--background-color-primary); + background-image: url("./icons/mic-muted.svg?primary=text-color"); +} + +.CallView_buttons .CallView_unmutedMicrophone { + background-image: url("./icons/mic-unmuted.svg?primary=background-color-primary"); +} + +.CallView_buttons .CallView_mutedCamera { + background-color: var(--background-color-primary); + background-image: url("./icons/cam-muted.svg?primary=text-color"); +} + +.CallView_buttons .CallView_unmutedCamera { + background-image: url("./icons/cam-unmuted.svg?primary=background-color-primary"); +} + +.CallView_members.size1 { + grid-template-columns: 1fr; + grid-template-rows: 1fr; +} + +.CallView_members.size2 { + grid-template-columns: 1fr; + grid-template-rows: repeat(2, 1fr); +} + +/* square */ +.CallView_members.square.size3, +.CallView_members.square.size4 { + grid-template-columns: repeat(2, 1fr); + grid-template-rows: repeat(2, 1fr); +} +.CallView_members.square.size5, +.CallView_members.square.size6 { + grid-template-columns: repeat(3, 1fr); + grid-template-rows: repeat(2, 1fr); +} +.CallView_members.square.size7, +.CallView_members.square.size8, +.CallView_members.square.size9 { + grid-template-columns: repeat(3, 1fr); + grid-template-rows: repeat(3, 1fr); +} +.CallView_members.square.size10 { + grid-template-columns: repeat(3, 1fr); + grid-template-rows: repeat(4, 1fr); +} +/** tall */ +.CallView_members.tall.size3 { + grid-template-columns: 1fr; + grid-template-rows: repeat(3, 1fr); +} +.CallView_members.tall.size4 { + grid-template-columns: 1fr; + grid-template-rows: repeat(4, 1fr); +} +.CallView_members.tall.size5, +.CallView_members.tall.size6 { + grid-template-rows: repeat(3, 1fr); + grid-template-columns: repeat(2, 1fr); +} +.CallView_members.tall.size7, +.CallView_members.tall.size8 { + grid-template-rows: repeat(4, 1fr); + grid-template-columns: repeat(2, 1fr); +} +.CallView_members.tall.size9, +.CallView_members.tall.size10 { + grid-template-rows: repeat(5, 1fr); + grid-template-columns: repeat(2, 1fr); +} +/** wide */ +.CallView_members.wide.size2 { + grid-template-columns: repeat(2, 1fr); + grid-template-rows: 1fr; +} +.CallView_members.wide.size3 { + grid-template-rows: 1fr; + grid-template-columns: repeat(3, 1fr); +} +.CallView_members.wide.size4 { + grid-template-rows: 1fr; + grid-template-columns: repeat(4, 1fr); +} +.CallView_members.wide.size5, +.CallView_members.wide.size6 { + grid-template-rows: repeat(2, 1fr); + grid-template-columns: repeat(3, 1fr); +} +.CallView_members.wide.size7, +.CallView_members.wide.size8 { + grid-template-rows: repeat(2, 1fr); + grid-template-columns: repeat(4, 1fr); +} +.CallView_members.wide.size9, +.CallView_members.wide.size10 { + grid-template-rows: repeat(3, 1fr); + grid-template-columns: repeat(4, 1fr); +} diff --git a/src/platform/web/ui/css/themes/element/icons/cam-muted.svg b/src/platform/web/ui/css/themes/element/icons/cam-muted.svg new file mode 100644 index 00000000..6a739ae2 --- /dev/null +++ b/src/platform/web/ui/css/themes/element/icons/cam-muted.svg @@ -0,0 +1 @@ + diff --git a/src/platform/web/ui/css/themes/element/icons/cam-unmuted.svg b/src/platform/web/ui/css/themes/element/icons/cam-unmuted.svg new file mode 100644 index 00000000..9497e075 --- /dev/null +++ b/src/platform/web/ui/css/themes/element/icons/cam-unmuted.svg @@ -0,0 +1 @@ + diff --git a/src/platform/web/ui/css/themes/element/icons/hangup.svg b/src/platform/web/ui/css/themes/element/icons/hangup.svg new file mode 100644 index 00000000..c56fe7a4 --- /dev/null +++ b/src/platform/web/ui/css/themes/element/icons/hangup.svg @@ -0,0 +1,3 @@ + + + diff --git a/src/platform/web/ui/css/themes/element/icons/mic-muted.svg b/src/platform/web/ui/css/themes/element/icons/mic-muted.svg new file mode 100644 index 00000000..35669ee0 --- /dev/null +++ b/src/platform/web/ui/css/themes/element/icons/mic-muted.svg @@ -0,0 +1,19 @@ + + + + + + diff --git a/src/platform/web/ui/css/themes/element/icons/mic-unmuted.svg b/src/platform/web/ui/css/themes/element/icons/mic-unmuted.svg new file mode 100644 index 00000000..94b81510 --- /dev/null +++ b/src/platform/web/ui/css/themes/element/icons/mic-unmuted.svg @@ -0,0 +1,16 @@ + + + + + diff --git a/src/platform/web/ui/css/themes/element/icons/video-call.svg b/src/platform/web/ui/css/themes/element/icons/video-call.svg new file mode 100644 index 00000000..bc3688b5 --- /dev/null +++ b/src/platform/web/ui/css/themes/element/icons/video-call.svg @@ -0,0 +1,4 @@ + + + + diff --git a/src/platform/web/ui/css/themes/element/icons/voice-call.svg b/src/platform/web/ui/css/themes/element/icons/voice-call.svg new file mode 100644 index 00000000..02a79969 --- /dev/null +++ b/src/platform/web/ui/css/themes/element/icons/voice-call.svg @@ -0,0 +1,3 @@ + + + diff --git a/src/platform/web/ui/css/themes/element/theme.css b/src/platform/web/ui/css/themes/element/theme.css index db0ad66b..6403bb60 100644 --- a/src/platform/web/ui/css/themes/element/theme.css +++ b/src/platform/web/ui/css/themes/element/theme.css @@ -18,6 +18,7 @@ limitations under the License. @import url('../../main.css'); @import url('inter.css'); @import url('timeline.css'); +@import url('call.css'); :root { font-size: 10px; @@ -1155,56 +1156,3 @@ button.RoomDetailsView_row::after { background-position: center; background-size: 36px; } - -.CallView { - max-height: 50vh; - overflow-y: auto; -} - -.CallView ul { - display: flex; - margin: 0; - gap: 12px; - padding: 0; - flex-wrap: wrap; - justify-content: center; -} - -.StreamView { - width: 360px; - min-height: 200px; - border: 2px var(--accent-color) solid; - display: grid; - border-radius: 8px; - overflow: hidden; - background-color: black; -} - -.StreamView > * { - grid-column: 1; - grid-row: 1; -} - -.StreamView video { - width: 100%; -} - -.StreamView_avatar { - align-self: center; - justify-self: center; -} - -.StreamView_muteStatus { - align-self: end; - justify-self: start; - color: var(--text-color--lighter-80); -} - -.StreamView_muteStatus.microphoneMuted::before { - content: "mic muted"; -} - -.StreamView_muteStatus.cameraMuted::before { - content: "cam muted"; -} - diff --git a/src/platform/web/ui/session/room/CallView.ts b/src/platform/web/ui/session/room/CallView.ts index ac31973c..5ccdaa84 100644 --- a/src/platform/web/ui/session/room/CallView.ts +++ b/src/platform/web/ui/session/room/CallView.ts @@ -17,26 +17,81 @@ limitations under the License. import {TemplateView, Builder} from "../../general/TemplateView"; import {AvatarView} from "../../AvatarView"; import {ListView} from "../../general/ListView"; +import {classNames} from "../../general/html"; import {Stream} from "../../../../types/MediaDevices"; import type {CallViewModel, CallMemberViewModel, IStreamViewModel} from "../../../../../domain/session/room/CallViewModel"; export class CallView extends TemplateView { + private resizeObserver?: ResizeObserver; + render(t: Builder, vm: CallViewModel): Element { + const members = t.view(new ListView({ + className: "CallView_members", + list: vm.memberViewModels + }, vm => new StreamView(vm))) as HTMLElement; + this.bindMembersCssClasses(t, members); return t.div({class: "CallView"}, [ - t.p(vm => `Call ${vm.name} (${vm.id})`), - t.view(new ListView({list: vm.memberViewModels}, vm => new StreamView(vm))), - t.div({class: "buttons"}, [ - t.button({onClick: () => vm.leave()}, "Leave"), - t.button({onClick: () => vm.toggleVideo()}, "Toggle video"), + members, + //t.p(vm => `Call ${vm.name}`), + t.div({class: "CallView_buttons"}, [ + t.button({className: { + "CallView_mutedMicrophone": vm => vm.isMicrophoneMuted, + "CallView_unmutedMicrophone": vm => !vm.isMicrophoneMuted, + }, onClick: () => vm.toggleMicrophone()}), + t.button({className: { + "CallView_mutedCamera": vm => vm.isCameraMuted, + "CallView_unmutedCamera": vm => !vm.isCameraMuted, + }, onClick: () => vm.toggleCamera()}), + t.button({className: "CallView_hangup", onClick: () => vm.hangup()}), ]) ]); } + + private bindMembersCssClasses(t, members) { + t.mapSideEffect(vm => vm.memberCount, count => { + members.classList.forEach((c, _, list) => { + if (c.startsWith("size")) { + list.remove(c); + } + }); + members.classList.add(`size${count}`); + }); + // update classes describing aspect ratio categories + if (typeof ResizeObserver === "function") { + const set = (c, flag) => { + if (flag) { + members.classList.add(c); + } else { + members.classList.remove(c); + } + }; + this.resizeObserver = new ResizeObserver(() => { + const ar = members.clientWidth / members.clientHeight; + const isTall = ar < 0.5; + const isSquare = !isTall && ar < 1.8 + const isWide = !isTall && !isSquare; + set("tall", isTall); + set("square", isSquare); + set("wide", isWide); + }); + this.resizeObserver!.observe(members); + } + } + + public unmount() { + if (this.resizeObserver) { + this.resizeObserver.unobserve((this.root()! as Element).querySelector(".CallView_members")); + this.resizeObserver = undefined; + } + super.unmount(); + } } class StreamView extends TemplateView { render(t: Builder, vm: IStreamViewModel): Element { const video = t.video({ autoplay: true, + disablePictureInPicture: true, className: { hidden: vm => vm.isCameraMuted } From 41288683fc9547ef1cf80cfe56ac472bf6afd712 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 10 Jun 2022 17:10:23 +0200 Subject: [PATCH 159/323] allow unmuting when we don't yet have a mic/cam track --- scripts/postcss/css-url-to-variables.js | 8 ++- src/domain/session/room/CallViewModel.ts | 55 +++++++++---------- src/matrix/calls/PeerCall.ts | 12 ++-- src/matrix/calls/common.ts | 28 +++++++++- src/matrix/calls/group/GroupCall.ts | 25 +++++++-- .../web/ui/css/themes/element/call.css | 9 +++ src/platform/web/ui/session/room/CallView.ts | 14 ++++- 7 files changed, 105 insertions(+), 46 deletions(-) diff --git a/scripts/postcss/css-url-to-variables.js b/scripts/postcss/css-url-to-variables.js index 1d4666f4..9e2e0077 100644 --- a/scripts/postcss/css-url-to-variables.js +++ b/scripts/postcss/css-url-to-variables.js @@ -26,7 +26,13 @@ const idToPrepend = "icon-url"; function findAndReplaceUrl(decl) { const value = decl.value; - const parsed = valueParser(value); + let parsed; + try { + parsed = valueParser(value); + } catch (err) { + console.log(`Error trying to parse ${decl}`); + throw err; + } parsed.walk(node => { if (node.type !== "function" || node.value !== "url") { return; diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 3e7fd951..5b6f05b4 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -60,18 +60,17 @@ export class CallViewModel extends ViewModel { } get isCameraMuted(): boolean { - return isLocalCameraMuted(this.call); + return this.call.muteSettings?.camera ?? true; } get isMicrophoneMuted(): boolean { - return isLocalMicrophoneMuted(this.call); + return this.call.muteSettings?.microphone ?? true; } get memberCount(): number { return this.memberViewModels.length; } - get name(): string { return this.call.name; } @@ -84,22 +83,36 @@ export class CallViewModel extends ViewModel { return this.getOption("call"); } - hangup() { + async hangup() { if (this.call.hasJoined) { - this.call.leave(); + await this.call.leave(); } } async toggleCamera() { - if (this.call.muteSettings) { - this.call.setMuted(this.call.muteSettings.toggleCamera()); + const {localMedia, muteSettings} = this.call; + if (muteSettings && localMedia) { + // unmute but no track? + if (muteSettings.camera && !getStreamVideoTrack(localMedia.userMedia)) { + const stream = await this.platform.mediaDevices.getMediaTracks(!muteSettings.microphone, true); + await this.call.setMedia(localMedia.withUserMedia(stream)); + } else { + await this.call.setMuted(muteSettings.toggleCamera()); + } this.emitChange(); } } async toggleMicrophone() { - if (this.call.muteSettings) { - this.call.setMuted(this.call.muteSettings.toggleMicrophone()); + const {localMedia, muteSettings} = this.call; + if (muteSettings && localMedia) { + // unmute but no track? + if (muteSettings.microphone && !getStreamAudioTrack(localMedia.userMedia)) { + const stream = await this.platform.mediaDevices.getMediaTracks(true, !muteSettings.camera); + await this.call.setMedia(localMedia.withUserMedia(stream)); + } else { + await this.call.setMuted(muteSettings.toggleMicrophone()); + } this.emitChange(); } } @@ -130,11 +143,11 @@ class OwnMemberViewModel extends ViewModel implements IStreamViewModel } get isCameraMuted(): boolean { - return isLocalCameraMuted(this.call); + return this.call.muteSettings?.camera ?? true; } get isMicrophoneMuted(): boolean { - return isLocalMicrophoneMuted(this.call); + return this.call.muteSettings?.microphone ?? true; } get avatarLetter(): string { @@ -186,11 +199,11 @@ export class CallMemberViewModel extends ViewModel implements ISt } get isCameraMuted(): boolean { - return isMuted(this.member.remoteMuteSettings?.camera, !!getStreamVideoTrack(this.stream)); + return this.member.remoteMuteSettings?.camera ?? true; } get isMicrophoneMuted(): boolean { - return isMuted(this.member.remoteMuteSettings?.microphone, !!getStreamAudioTrack(this.stream)); + return this.member.remoteMuteSettings?.microphone ?? true; } get avatarLetter(): string { @@ -229,19 +242,3 @@ export interface IStreamViewModel extends AvatarSource, ViewModel { get isCameraMuted(): boolean; get isMicrophoneMuted(): boolean; } - -function isMuted(muted: boolean | undefined, hasTrack: boolean) { - if (muted) { - return true; - } else { - return !hasTrack; - } -} - -function isLocalCameraMuted(call: GroupCall): boolean { - return isMuted(call.muteSettings?.camera, !!getStreamVideoTrack(call.localMedia?.userMedia)); -} - -function isLocalMicrophoneMuted(call: GroupCall): boolean { - return isMuted(call.muteSettings?.microphone, !!getStreamAudioTrack(call.localMedia?.userMedia)); -} diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index ea7e9d06..809decbf 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -887,8 +887,8 @@ export class PeerCall implements IDisposable { const streamId = this.localMedia.userMedia.id; metadata[streamId] = { purpose: SDPStreamMetadataPurpose.Usermedia, - audio_muted: this.localMuteSettings?.microphone || !getStreamAudioTrack(this.localMedia.userMedia), - video_muted: this.localMuteSettings?.camera || !getStreamVideoTrack(this.localMedia.userMedia), + audio_muted: this.localMuteSettings?.microphone ?? false, + video_muted: this.localMuteSettings?.camera ?? false, }; } if (this.localMedia?.screenShare) { @@ -936,7 +936,7 @@ export class PeerCall implements IDisposable { this.updateRemoteMedia(log); } } - }) + }); }; stream.addEventListener("removetrack", listener); const disposeListener = () => { @@ -971,8 +971,10 @@ export class PeerCall implements IDisposable { videoReceiver.track.enabled = !metaData.video_muted; } this._remoteMuteSettings = new MuteSettings( - metaData.audio_muted || !audioReceiver?.track, - metaData.video_muted || !videoReceiver?.track + metaData.audio_muted ?? false, + metaData.video_muted ?? false, + !!audioReceiver?.track ?? false, + !!videoReceiver?.track ?? false ); log.log({ l: "setting userMedia", diff --git a/src/matrix/calls/common.ts b/src/matrix/calls/common.ts index db49a168..66db6edc 100644 --- a/src/matrix/calls/common.ts +++ b/src/matrix/calls/common.ts @@ -25,14 +25,36 @@ export function getStreamVideoTrack(stream: Stream | undefined): Track | undefin } export class MuteSettings { - constructor (public readonly microphone: boolean = false, public readonly camera: boolean = false) {} + constructor ( + private readonly isMicrophoneMuted: boolean = false, + private readonly isCameraMuted: boolean = false, + private hasMicrophoneTrack: boolean = false, + private hasCameraTrack: boolean = false, + ) {} + + updateTrackInfo(userMedia: Stream | undefined) { + this.hasMicrophoneTrack = !!getStreamAudioTrack(userMedia); + this.hasCameraTrack = !!getStreamVideoTrack(userMedia); + } + + get microphone(): boolean { + return !this.hasMicrophoneTrack || this.isMicrophoneMuted; + } + + get camera(): boolean { + return !this.hasCameraTrack || this.isCameraMuted; + } toggleCamera(): MuteSettings { - return new MuteSettings(this.microphone, !this.camera); + return new MuteSettings(this.microphone, !this.camera, this.hasMicrophoneTrack, this.hasCameraTrack); } toggleMicrophone(): MuteSettings { - return new MuteSettings(!this.microphone, this.camera); + return new MuteSettings(!this.microphone, this.camera, this.hasMicrophoneTrack, this.hasCameraTrack); + } + + equals(other: MuteSettings) { + return this.microphone === other.microphone && this.camera === other.camera; } } diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index ebacf844..19e40dbf 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -137,11 +137,13 @@ export class GroupCall extends EventEmitter<{change: never}> { ownSessionId: this.options.sessionId }); const membersLogItem = logItem.child("member connections"); + const localMuteSettings = new MuteSettings(); + localMuteSettings.updateTrackInfo(localMedia.userMedia); const joinedData = new JoinedData( logItem, membersLogItem, localMedia, - new MuteSettings() + localMuteSettings ); this.joinedData = joinedData; await joinedData.logItem.wrap("join", async log => { @@ -163,9 +165,14 @@ export class GroupCall extends EventEmitter<{change: never}> { if ((this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) && this.joinedData) { const oldMedia = this.joinedData.localMedia; this.joinedData.localMedia = localMedia; + // reflect the fact we gained or lost local tracks in the local mute settings + // and update the track info so PeerCall can use it to send up to date metadata, + this.joinedData.localMuteSettings.updateTrackInfo(localMedia.userMedia); + this.emitChange(); //allow listeners to see new media/mute settings await Promise.all(Array.from(this._members.values()).map(m => { return m.setMedia(localMedia, oldMedia); })); + oldMedia?.stopExcept(localMedia); } } @@ -175,11 +182,19 @@ export class GroupCall extends EventEmitter<{change: never}> { if (!joinedData) { return; } + const prevMuteSettings = joinedData.localMuteSettings; + // we still update the mute settings if nothing changed because + // you might be muted because you don't have a track or because + // you actively chosen to mute + // (which we want to respect in the future when you add a track) joinedData.localMuteSettings = muteSettings; - await Promise.all(Array.from(this._members.values()).map(m => { - return m.setMuted(joinedData.localMuteSettings); - })); - this.emitChange(); + joinedData.localMuteSettings.updateTrackInfo(joinedData.localMedia.userMedia); + if (!prevMuteSettings.equals(muteSettings)) { + await Promise.all(Array.from(this._members.values()).map(m => { + return m.setMuted(joinedData.localMuteSettings); + })); + this.emitChange(); + } } get muteSettings(): MuteSettings | undefined { diff --git a/src/platform/web/ui/css/themes/element/call.css b/src/platform/web/ui/css/themes/element/call.css index 4398f9c6..9bdc4abb 100644 --- a/src/platform/web/ui/css/themes/element/call.css +++ b/src/platform/web/ui/css/themes/element/call.css @@ -40,6 +40,7 @@ limitations under the License. border-radius: 8px; overflow: hidden; background-color: black; + display: block; } .StreamView > * { @@ -98,11 +99,19 @@ limitations under the License. background-repeat: no-repeat; } +.CallView_buttons button:disabled { + background-color: var(--accent-color--lighter-10); +} + .CallView_buttons .CallView_hangup { background-color: var(--error-color); background-image: url("./icons/hangup.svg?primary=background-color-primary"); } +.CallView_buttons .CallView_hangup:disabled { + background-color: var(--error-color--lighter-10); +} + .CallView_buttons .CallView_mutedMicrophone { background-color: var(--background-color-primary); background-image: url("./icons/mic-muted.svg?primary=text-color"); diff --git a/src/platform/web/ui/session/room/CallView.ts b/src/platform/web/ui/session/room/CallView.ts index 5ccdaa84..6961dc53 100644 --- a/src/platform/web/ui/session/room/CallView.ts +++ b/src/platform/web/ui/session/room/CallView.ts @@ -37,12 +37,12 @@ export class CallView extends TemplateView { t.button({className: { "CallView_mutedMicrophone": vm => vm.isMicrophoneMuted, "CallView_unmutedMicrophone": vm => !vm.isMicrophoneMuted, - }, onClick: () => vm.toggleMicrophone()}), + }, onClick: disableTargetCallback(() => vm.toggleMicrophone())}), t.button({className: { "CallView_mutedCamera": vm => vm.isCameraMuted, "CallView_unmutedCamera": vm => !vm.isCameraMuted, - }, onClick: () => vm.toggleCamera()}), - t.button({className: "CallView_hangup", onClick: () => vm.hangup()}), + }, onClick: disableTargetCallback(() => vm.toggleCamera())}), + t.button({className: "CallView_hangup", onClick: disableTargetCallback(() => vm.hangup())}), ]) ]); } @@ -122,3 +122,11 @@ class StreamView extends TemplateView { this.updateSubViews(value, props); } } + +function disableTargetCallback(callback: (evt: Event) => Promise): (evt: Event) => Promise { + return async (evt: Event) => { + (evt.target as HTMLElement)?.setAttribute("disabled", "disabled"); + await callback(evt); + (evt.target as HTMLElement)?.removeAttribute("disabled"); + } +} From 595a15c533d3a22bd79487a8ac42fd932f9d7c38 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 14 Jun 2022 12:12:10 +0200 Subject: [PATCH 160/323] make overlay buttons on call view clickable on chrome --- src/platform/web/ui/css/themes/element/call.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/platform/web/ui/css/themes/element/call.css b/src/platform/web/ui/css/themes/element/call.css index 9bdc4abb..59e4046b 100644 --- a/src/platform/web/ui/css/themes/element/call.css +++ b/src/platform/web/ui/css/themes/element/call.css @@ -87,6 +87,10 @@ limitations under the License. display: flex; gap: 12px; margin-bottom: 16px; + /** Chrome (v100) requires this to make the buttons clickable + * where they overlap with the video element, even though + * the buttons come later in the DOM. */ + z-index: 1; } .CallView_buttons button { From d4ee19c4e4d915714b52da05276e6010e51029d2 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 14 Jun 2022 12:24:38 +0200 Subject: [PATCH 161/323] fix video elements not respecting parent height in callview grid --- src/platform/web/ui/css/themes/element/call.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/platform/web/ui/css/themes/element/call.css b/src/platform/web/ui/css/themes/element/call.css index 59e4046b..10388fb4 100644 --- a/src/platform/web/ui/css/themes/element/call.css +++ b/src/platform/web/ui/css/themes/element/call.css @@ -40,7 +40,6 @@ limitations under the License. border-radius: 8px; overflow: hidden; background-color: black; - display: block; } .StreamView > * { @@ -51,6 +50,7 @@ limitations under the License. .StreamView video { width: 100%; height: 100%; + min-height: 0; object-fit: contain; } From 8a90c48d1e0c844827e0e08306c3f915e0acb3b2 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Sat, 25 Jun 2022 05:56:43 +0200 Subject: [PATCH 162/323] debugging unmuting not working --- src/domain/session/room/CallViewModel.ts | 1 + src/matrix/calls/LocalMedia.ts | 12 +++++++++--- src/matrix/calls/PeerCall.ts | 15 +++++++++------ src/matrix/calls/group/GroupCall.ts | 13 ++++++++----- src/matrix/calls/group/Member.ts | 2 +- src/platform/web/dom/MediaDevices.ts | 3 +++ 6 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 5b6f05b4..020c9f17 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -109,6 +109,7 @@ export class CallViewModel extends ViewModel { // unmute but no track? if (muteSettings.microphone && !getStreamAudioTrack(localMedia.userMedia)) { const stream = await this.platform.mediaDevices.getMediaTracks(true, !muteSettings.camera); + console.log("got tracks", Array.from(stream.getTracks()).map((t: MediaStreamTrack) => { return {kind: t.kind, id: t.id};})) await this.call.setMedia(localMedia.withUserMedia(stream)); } else { await this.call.setMuted(muteSettings.toggleMicrophone()); diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index b79fe098..922e6861 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -45,13 +45,19 @@ export class LocalMedia { const cloneOrAdoptStream = (oldOriginalStream: Stream | undefined, oldCloneStream: Stream | undefined, newStream: Stream | undefined): Stream | undefined => { let stream; if (oldOriginalStream?.id === newStream?.id) { - stream = oldCloneStream; + return oldCloneStream; } else { - stream = newStream?.clone(); + const clonedStream = newStream?.clone(); + // clonedStream.addEventListener("removetrack", evt => { + // console.log(`removing track ${evt.track.id} (${evt.track.kind}) from clonedStream ${clonedStream.id}`); + // }); + // console.log("got cloned tracks", Array.from(clonedStream.getTracks()).map((t: MediaStreamTrack) => { return {kind: t.kind, id: t.id};}), "from new stream", Array.from(newStream.getTracks()).map((t: MediaStreamTrack) => { return {kind: t.kind, id: t.id};})); + // console.log("stopping old audio stream", getStreamAudioTrack(oldCloneStream)?.id); + // console.log("stopping old video stream", getStreamVideoTrack(oldCloneStream)?.id); getStreamAudioTrack(oldCloneStream)?.stop(); getStreamVideoTrack(oldCloneStream)?.stop(); + return clonedStream; } - return stream; } return new LocalMedia( cloneOrAdoptStream(oldOriginal?.userMedia, oldClone?.userMedia, this.userMedia), diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 809decbf..1e22a2ff 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -383,8 +383,7 @@ export class PeerCall implements IDisposable { const offer = this.peerConnection.localDescription!; // Get rid of any candidates waiting to be sent: they'll be included in the local // description we just got and will send in the offer. - log.log(`Discarding ${ - this.candidateSendQueue.length} candidates that will be sent in offer`); + log.set("includedCandidates", this.candidateSendQueue.length); this.candidateSendQueue = []; // need to queue this @@ -414,16 +413,20 @@ export class PeerCall implements IDisposable { this.sendCandidateQueue(log); - await log.wrap("invite timeout", async log => { - if (this._state === CallState.InviteSent) { + if (this._state === CallState.InviteSent) { + const timeoutLog = this.logItem.child("invite timeout"); + log.refDetached(timeoutLog); + // don't await this, as it would block other negotationneeded events from being processed + // as they are processed serially + timeoutLog.run(async log => { try { await this.delay(CALL_TIMEOUT_MS); } catch (err) { return; } // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between if (this._state === CallState.InviteSent) { this._hangup(CallErrorCode.InviteTimeout, log); } - } - }); + }).catch(err => {}); // prevent error from being unhandled, it will be logged already by run above + } }; private async handleInviteGlare(content: MCallInvite, partyId: PartyId, log: ILogItem): Promise { diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 19e40dbf..087dc1ff 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -149,11 +149,14 @@ export class GroupCall extends EventEmitter<{change: never}> { await joinedData.logItem.wrap("join", async log => { this._state = GroupCallState.Joining; this.emitChange(); - const memberContent = await this._createJoinPayload(); - // send m.call.member state event - const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCallMember, this.options.ownUserId, memberContent, {log}); - await request.response(); - this.emitChange(); + await log.wrap("update member state", async log => { + const memberContent = await this._createJoinPayload(); + log.set("payload", memberContent); + // send m.call.member state event + const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCallMember, this.options.ownUserId, memberContent, {log}); + await request.response(); + this.emitChange(); + }); // send invite to all members that are < my userId for (const [,member] of this._members) { this.connectToMember(member, joinedData, log); diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index cf6bb6f0..3003c3df 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -293,7 +293,7 @@ export class Member { async setMedia(localMedia: LocalMedia, previousMedia: LocalMedia): Promise { const {connection} = this; if (connection) { - connection.localMedia = connection.localMedia.replaceClone(connection.localMedia, previousMedia); + connection.localMedia = localMedia.replaceClone(connection.localMedia, previousMedia); await connection.peerCall?.setMedia(connection.localMedia, connection.logItem); } } diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index c34ab85b..d6439faa 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -30,6 +30,9 @@ export class MediaDevicesWrapper implements IMediaDevices { async getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise { const stream = await this.mediaDevices.getUserMedia(this.getUserMediaContraints(audio, video)); + stream.addEventListener("removetrack", evt => { + console.log(`removing track ${evt.track.id} (${evt.track.kind}) from stream ${stream.id}`); + }); return stream as Stream; } From 5527e2b22cd1adf1fb91e42c10c9385e0572eb99 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 5 Jul 2022 11:02:36 +0200 Subject: [PATCH 163/323] also remove deferred log items from open list when finishing them otherwise they end up in the logs twice when exporting --- src/logging/Logger.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/logging/Logger.ts b/src/logging/Logger.ts index 395181ef..37a3326c 100644 --- a/src/logging/Logger.ts +++ b/src/logging/Logger.ts @@ -159,6 +159,11 @@ export class Logger implements ILogger { this._openItems.clear(); } + /** @internal */ + _removeItemFromOpenList(item: LogItem): void { + this._openItems.delete(item); + } + /** @internal */ _persistItem(item: LogItem, filter?: LogFilter, forced?: boolean): void { for (var i = 0; i < this.reporters.length; i += 1) { @@ -186,6 +191,7 @@ class DeferredPersistRootLogItem extends LogItem { finish() { super.finish(); (this._logger as Logger)._persistItem(this, undefined, false); + (this._logger as Logger)._removeItemFromOpenList(this); } forceFinish() { From 206ac6e2dd2e85dbee2c908cad3a3c89fd41f6ed Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 5 Jul 2022 18:22:36 +0200 Subject: [PATCH 164/323] WIP: prevent stream id from changing when upgrading call --- src/matrix/calls/PeerCall.ts | 84 ++++++++++++++++++------------ src/matrix/calls/group/Member.ts | 1 + src/platform/types/MediaDevices.ts | 2 + 3 files changed, 53 insertions(+), 34 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 1e22a2ff..29eccf2c 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -999,49 +999,55 @@ export class PeerCall implements IDisposable { private updateLocalMedia(localMedia: LocalMedia, logItem: ILogItem): Promise { return logItem.wrap("updateLocalMedia", async log => { - const oldMedia = this.localMedia; - this.localMedia = localMedia; + const senders = this.peerConnection.getSenders(); const applyStream = async (oldStream: Stream | undefined, stream: Stream | undefined, streamPurpose: SDPStreamMetadataPurpose) => { const applyTrack = async (oldTrack: Track | undefined, newTrack: Track | undefined) => { - if (!oldTrack && newTrack) { - log.wrap(`adding ${streamPurpose} ${newTrack.kind} track`, log => { - const sender = this.peerConnection.addTrack(newTrack, stream!); - this.options.webRTC.prepareSenderForPurpose(this.peerConnection, sender, streamPurpose); - }); - } else if (oldTrack) { - const sender = this.peerConnection.getSenders().find(s => s.track && s.track.id === oldTrack.id); - if (sender) { - if (newTrack && oldTrack.id !== newTrack.id) { - try { - await log.wrap(`replacing ${streamPurpose} ${newTrack.kind} track`, log => { - return sender.replaceTrack(newTrack); - }); - } catch (err) { - // can't replace the track without renegotiating{ - log.wrap(`adding and removing ${streamPurpose} ${newTrack.kind} track`, log => { - this.peerConnection.removeTrack(sender); - const newSender = this.peerConnection.addTrack(newTrack); - this.options.webRTC.prepareSenderForPurpose(this.peerConnection, newSender, streamPurpose); - }); - } - } else if (!newTrack) { - log.wrap(`removing ${streamPurpose} ${sender.track!.kind} track`, log => { - this.peerConnection.removeTrack(sender); - }); - } else { - log.log(`${streamPurpose} ${oldTrack.kind} track hasn't changed`); - } + const oldSender = senders.find(s => s.track === oldTrack); + const streamToKeep = (oldStream ?? stream)!; + if (streamToKeep !== stream) { + if (oldTrack) { + streamToKeep.removeTrack(oldTrack); } - // TODO: should we do something if we didn't find the sender? e.g. some other code already removed the sender but didn't update localMedia + if (newTrack) { + streamToKeep.addTrack(newTrack); + } + } + if (newTrack && oldSender) { + try { + await log.wrap(`attempting to replace ${streamPurpose} ${newTrack.kind} track`, log => { + return oldSender.replaceTrack(newTrack); + }); + // replaceTrack succeeded, nothing left to do + return; + } catch (err) {} + } + if(oldSender) { + log.wrap(`removing ${streamPurpose} ${oldSender.track!.kind} track`, log => { + this.peerConnection.removeTrack(oldSender); + }); + } + if (newTrack) { + log.wrap(`adding ${streamPurpose} ${newTrack.kind} track`, log => { + const newSender = this.peerConnection.addTrack(newTrack, streamToKeep); + this.options.webRTC.prepareSenderForPurpose(this.peerConnection, newSender, streamPurpose); + }); } } - + if (!oldStream && !stream) { + return; + } await applyTrack(getStreamAudioTrack(oldStream), getStreamAudioTrack(stream)); await applyTrack(getStreamVideoTrack(oldStream), getStreamVideoTrack(stream)); }; - await applyStream(oldMedia?.userMedia, localMedia?.userMedia, SDPStreamMetadataPurpose.Usermedia); - await applyStream(oldMedia?.screenShare, localMedia?.screenShare, SDPStreamMetadataPurpose.Screenshare); + await applyStream(this.localMedia?.userMedia, localMedia?.userMedia, SDPStreamMetadataPurpose.Usermedia); + await applyStream(this.localMedia?.screenShare, localMedia?.screenShare, SDPStreamMetadataPurpose.Screenshare); + // we explicitly don't replace this.localMedia if already set + // as we need to keep the old stream so the stream id doesn't change + // instead we add and remove tracks in the stream in applyTrack + if (!this.localMedia) { + this.localMedia = localMedia; + } // TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method }); } @@ -1158,3 +1164,13 @@ function enableTransceiver(transceiver: Transceiver, enabled: boolean, exclusive } } } + +/** + * tests to write: + * + * upgradeCall: adding a track with setMedia calls the correct methods on the peerConnection + * upgradeCall: removing a track with setMedia calls the correct methods on the peerConnection + * upgradeCall: replacing compatible track with setMedia calls the correct methods on the peerConnection + * upgradeCall: replacing incompatible track (sender.replaceTrack throws) with setMedia calls the correct methods on the peerConnection + * + * */ diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 3003c3df..b6461e6a 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -293,6 +293,7 @@ export class Member { async setMedia(localMedia: LocalMedia, previousMedia: LocalMedia): Promise { const {connection} = this; if (connection) { + // TODO: see if we can simplify this connection.localMedia = localMedia.replaceClone(connection.localMedia, previousMedia); await connection.peerCall?.setMedia(connection.localMedia, connection.logItem); } diff --git a/src/platform/types/MediaDevices.ts b/src/platform/types/MediaDevices.ts index 1b5f7afd..c5439ac4 100644 --- a/src/platform/types/MediaDevices.ts +++ b/src/platform/types/MediaDevices.ts @@ -56,6 +56,8 @@ export interface Stream { clone(): Stream; addEventListener(type: K, listener: (this: Stream, ev: StreamEventMap[K]) => any, options?: boolean | AddEventListenerOptions): void; removeEventListener(type: K, listener: (this: Stream, ev: StreamEventMap[K]) => any, options?: boolean | EventListenerOptions): void; + addTrack(track: Track); + removeTrack(track: Track); } export enum TrackKind { From f187a51c97ea09c8c8af90136eb8fe64b6f14337 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 7 Jul 2022 15:36:30 +0200 Subject: [PATCH 165/323] stop replaced track in PeerCall --- src/matrix/calls/LocalMedia.ts | 11 +---------- src/matrix/calls/PeerCall.ts | 1 + 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 922e6861..90a257fb 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -47,16 +47,7 @@ export class LocalMedia { if (oldOriginalStream?.id === newStream?.id) { return oldCloneStream; } else { - const clonedStream = newStream?.clone(); - // clonedStream.addEventListener("removetrack", evt => { - // console.log(`removing track ${evt.track.id} (${evt.track.kind}) from clonedStream ${clonedStream.id}`); - // }); - // console.log("got cloned tracks", Array.from(clonedStream.getTracks()).map((t: MediaStreamTrack) => { return {kind: t.kind, id: t.id};}), "from new stream", Array.from(newStream.getTracks()).map((t: MediaStreamTrack) => { return {kind: t.kind, id: t.id};})); - // console.log("stopping old audio stream", getStreamAudioTrack(oldCloneStream)?.id); - // console.log("stopping old video stream", getStreamVideoTrack(oldCloneStream)?.id); - getStreamAudioTrack(oldCloneStream)?.stop(); - getStreamVideoTrack(oldCloneStream)?.stop(); - return clonedStream; + return newStream?.clone(); } } return new LocalMedia( diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 29eccf2c..118e5de4 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -1007,6 +1007,7 @@ export class PeerCall implements IDisposable { if (streamToKeep !== stream) { if (oldTrack) { streamToKeep.removeTrack(oldTrack); + oldTrack.stop(); } if (newTrack) { streamToKeep.addTrack(newTrack); From 2f08cd89843062a22c545d2e45d97bd9843d7ae6 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 7 Jul 2022 15:36:49 +0200 Subject: [PATCH 166/323] clone localMedia in Member when connection, like we do for setMedia --- src/matrix/calls/group/GroupCall.ts | 3 +-- src/matrix/calls/group/Member.ts | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 087dc1ff..32052cc3 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -510,8 +510,7 @@ export class GroupCall extends EventEmitter<{change: never}> { const logItem = joinedData.membersLogItem.child({l: "member", id: memberKey}); logItem.set("sessionId", member.sessionId); log.wrap({l: "connect", id: memberKey}, log => { - // Safari can't send a MediaStream to multiple sources, so clone it - const connectItem = member.connect(joinedData.localMedia.clone(), joinedData.localMuteSettings, logItem); + const connectItem = member.connect(joinedData.localMedia, joinedData.localMuteSettings, logItem); if (connectItem) { log.refDetached(connectItem); } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index b6461e6a..9f370d6f 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -116,7 +116,8 @@ export class Member { if (this.connection) { return; } - const connection = new MemberConnection(localMedia, localMuteSettings, memberLogItem); + // Safari can't send a MediaStream to multiple sources, so clone it + const connection = new MemberConnection(localMedia.clone(), localMuteSettings, memberLogItem); this.connection = connection; let connectLogItem; connection.logItem.wrap("connect", async log => { @@ -293,7 +294,6 @@ export class Member { async setMedia(localMedia: LocalMedia, previousMedia: LocalMedia): Promise { const {connection} = this; if (connection) { - // TODO: see if we can simplify this connection.localMedia = localMedia.replaceClone(connection.localMedia, previousMedia); await connection.peerCall?.setMedia(connection.localMedia, connection.logItem); } From e9649ec7c2dbc0f69ef3719f0b77ede611feb31b Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 7 Jul 2022 15:47:09 +0200 Subject: [PATCH 167/323] different streams never have the same id, even for same devices --- src/matrix/calls/LocalMedia.ts | 16 +++------------- src/matrix/calls/group/GroupCall.ts | 3 +-- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 90a257fb..f02cd11b 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -40,8 +40,6 @@ export class LocalMedia { /** @internal */ replaceClone(oldClone: LocalMedia | undefined, oldOriginal: LocalMedia | undefined): LocalMedia { - let userMedia; - let screenShare; const cloneOrAdoptStream = (oldOriginalStream: Stream | undefined, oldCloneStream: Stream | undefined, newStream: Stream | undefined): Stream | undefined => { let stream; if (oldOriginalStream?.id === newStream?.id) { @@ -63,16 +61,8 @@ export class LocalMedia { } dispose() { - this.stopExcept(undefined); - } - - stopExcept(newMedia: LocalMedia | undefined) { - if(newMedia?.userMedia?.id !== this.userMedia?.id) { - getStreamAudioTrack(this.userMedia)?.stop(); - getStreamVideoTrack(this.userMedia)?.stop(); - } - if(newMedia?.screenShare?.id !== this.screenShare?.id) { - getStreamVideoTrack(this.screenShare)?.stop(); - } + getStreamAudioTrack(this.userMedia)?.stop(); + getStreamVideoTrack(this.userMedia)?.stop(); + getStreamVideoTrack(this.screenShare)?.stop(); } } diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 32052cc3..07aa2a08 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -175,8 +175,7 @@ export class GroupCall extends EventEmitter<{change: never}> { await Promise.all(Array.from(this._members.values()).map(m => { return m.setMedia(localMedia, oldMedia); })); - - oldMedia?.stopExcept(localMedia); + oldMedia?.dispose(); } } From 3346f68d25e4ba6b6d4d09a8b961b67627cd354c Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 12 Jul 2022 11:59:52 +0200 Subject: [PATCH 168/323] WIP --- src/matrix/calls/PeerCall.ts | 62 +++++++++++++++++--------------- src/matrix/calls/group/Member.ts | 2 +- src/platform/web/dom/WebRTC.ts | 15 +++++++- 3 files changed, 49 insertions(+), 30 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 118e5de4..fc5b48a6 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -177,6 +177,7 @@ export class PeerCall implements IDisposable { if (this._state !== CallState.Fledgling) { return; } + log.set("signalingState", this.peerConnection.signalingState); this.direction = CallDirection.Outbound; this.setState(CallState.CreateOffer, log); this.localMuteSettings = localMuteSettings; @@ -309,35 +310,39 @@ export class PeerCall implements IDisposable { }, async log => { logItem = log; - switch (message.type) { - case EventType.Invite: - if (this.callId !== message.content.call_id) { - await this.handleInviteGlare(message.content, partyId, log); - } else { + const callIdMatches = this.callId === message.content.call_id; + + if (message.type === EventType.Invite && !callIdMatches) { + await this.handleInviteGlare(message.content, partyId, log); + } else if (callIdMatches) { + switch (message.type) { + case EventType.Invite: await this.handleFirstInvite(message.content, partyId, log); - } - break; - case EventType.Answer: - await this.handleAnswer(message.content, partyId, log); - break; - case EventType.Negotiate: - await this.onNegotiateReceived(message.content, log); - break; - case EventType.Candidates: - await this.handleRemoteIceCandidates(message.content, partyId, log); - break; - case EventType.SDPStreamMetadataChanged: - case EventType.SDPStreamMetadataChangedPrefix: - this.updateRemoteSDPStreamMetadata(message.content[SDPStreamMetadataKey], log); - break; - case EventType.Hangup: - // TODO: this is a bit hacky, double check its what we need - log.set("reason", message.content.reason); - this.terminate(CallParty.Remote, message.content.reason ?? CallErrorCode.UserHangup, log); - break; - default: - log.log(`Unknown event type for call: ${message.type}`); - break; + break; + case EventType.Answer: + await this.handleAnswer(message.content, partyId, log); + break; + case EventType.Negotiate: + await this.onNegotiateReceived(message.content, log); + break; + case EventType.Candidates: + await this.handleRemoteIceCandidates(message.content, partyId, log); + break; + case EventType.SDPStreamMetadataChanged: + case EventType.SDPStreamMetadataChangedPrefix: + this.updateRemoteSDPStreamMetadata(message.content[SDPStreamMetadataKey], log); + break; + case EventType.Hangup: + // TODO: this is a bit hacky, double check its what we need + log.set("reason", message.content.reason); + this.terminate(CallParty.Remote, message.content.reason ?? CallErrorCode.UserHangup, log); + break; + default: + log.log(`Unknown event type for call: ${message.type}`); + break; + } + } else if (!callIdMatches) { + log.set("wrongCallId", true); } }); return logItem; @@ -879,6 +884,7 @@ export class PeerCall implements IDisposable { this.setState(CallState.Ended, log); this.localMedia = undefined; + // TODO: change signalingState to connectionState? if (this.peerConnection && this.peerConnection.signalingState !== 'closed') { this.peerConnection.close(); } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 9f370d6f..9e87fa22 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -283,7 +283,7 @@ export class Member { } } if (!hasBeenDequeued) { - syncLog.refDetached(connection.logItem.log({l: "queued signalling message", type: message.type})); + syncLog.refDetached(connection.logItem.log({l: "queued signalling message", type: message.type, seq: message.content.seq})); } } else { syncLog.log({l: "member not connected", userId: this.userId, deviceId: this.deviceId}); diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 61d481d4..05e032ca 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -24,11 +24,24 @@ const SPEAKING_SAMPLE_COUNT = 8; // samples export class DOMWebRTC implements WebRTC { createPeerConnection(forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize): PeerConnection { - return new RTCPeerConnection({ + const peerConn = new RTCPeerConnection({ iceTransportPolicy: forceTURN ? 'relay' : undefined, iceServers: turnServers, iceCandidatePoolSize: iceCandidatePoolSize, }) as PeerConnection; + return new Proxy(peerConn, { + get(target, prop, receiver) { + if (prop === "close") { + console.trace("calling peerConnection.close"); + } + const value = target[prop]; + if (typeof value === "function") { + return value.bind(target); + } else { + return value; + } + } + }); } prepareSenderForPurpose(peerConnection: PeerConnection, sender: Sender, purpose: SDPStreamMetadataPurpose): void { From bb2e63b05ba568918f6e7caf56dc33e7f0b60f6f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 20 Sep 2022 17:27:39 +0200 Subject: [PATCH 169/323] don't queue messages for different callIds so last seq doesn't corrupt this includes handling invite glares differently --- src/matrix/calls/PeerCall.ts | 137 ++++++++++++++++++------------- src/matrix/calls/group/Member.ts | 71 +++++++++++----- 2 files changed, 131 insertions(+), 77 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index fc5b48a6..d004dedc 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -53,6 +53,12 @@ export type Options = { sendSignallingMessage: (message: SignallingMessage, log: ILogItem) => Promise; }; +export enum IncomingMessageAction { + InviteGlare, + Handle, + Ignore +}; + export class RemoteMedia { constructor(public userMedia?: Stream | undefined, public screenShare?: Stream | undefined) {} } @@ -299,6 +305,17 @@ export class PeerCall implements IDisposable { await this.sendHangupWithCallId(this.callId, errorCode, log); } + getMessageAction(message: SignallingMessage): IncomingMessageAction { + const callIdMatches = this.callId === message.content.call_id; + if (message.type === EventType.Invite && !callIdMatches) { + return IncomingMessageAction.InviteGlare; + } if (callIdMatches) { + return IncomingMessageAction.Handle; + } else { + return IncomingMessageAction.Ignore; + } + } + handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId, log: ILogItem): ILogItem { // return logItem item immediately so it can be references in sync manner let logItem; @@ -309,40 +326,35 @@ export class PeerCall implements IDisposable { payload: message.content }, async log => { logItem = log; - - const callIdMatches = this.callId === message.content.call_id; - - if (message.type === EventType.Invite && !callIdMatches) { - await this.handleInviteGlare(message.content, partyId, log); - } else if (callIdMatches) { - switch (message.type) { - case EventType.Invite: - await this.handleFirstInvite(message.content, partyId, log); - break; - case EventType.Answer: - await this.handleAnswer(message.content, partyId, log); - break; - case EventType.Negotiate: - await this.onNegotiateReceived(message.content, log); - break; - case EventType.Candidates: - await this.handleRemoteIceCandidates(message.content, partyId, log); - break; - case EventType.SDPStreamMetadataChanged: - case EventType.SDPStreamMetadataChangedPrefix: - this.updateRemoteSDPStreamMetadata(message.content[SDPStreamMetadataKey], log); - break; - case EventType.Hangup: - // TODO: this is a bit hacky, double check its what we need - log.set("reason", message.content.reason); - this.terminate(CallParty.Remote, message.content.reason ?? CallErrorCode.UserHangup, log); - break; - default: - log.log(`Unknown event type for call: ${message.type}`); - break; - } - } else if (!callIdMatches) { + if (this.getMessageAction(message) !== IncomingMessageAction.Handle) { log.set("wrongCallId", true); + return; + } + switch (message.type) { + case EventType.Invite: + await this.handleFirstInvite(message.content, partyId, log); + break; + case EventType.Answer: + await this.handleAnswer(message.content, partyId, log); + break; + case EventType.Negotiate: + await this.onNegotiateReceived(message.content, log); + break; + case EventType.Candidates: + await this.handleRemoteIceCandidates(message.content, partyId, log); + break; + case EventType.SDPStreamMetadataChanged: + case EventType.SDPStreamMetadataChangedPrefix: + this.updateRemoteSDPStreamMetadata(message.content[SDPStreamMetadataKey], log); + break; + case EventType.Hangup: + // TODO: this is a bit hacky, double check its what we need + log.set("reason", message.content.reason); + this.terminate(CallParty.Remote, message.content.reason ?? CallErrorCode.UserHangup, log); + break; + default: + log.log(`Unknown event type for call: ${message.type}`); + break; } }); return logItem; @@ -434,30 +446,43 @@ export class PeerCall implements IDisposable { } }; - private async handleInviteGlare(content: MCallInvite, partyId: PartyId, log: ILogItem): Promise { - // this is only called when the ids are different - const newCallId = content.call_id; - if (this.callId! > newCallId) { - log.log( - "Glare detected: answering incoming call " + newCallId + - " and canceling outgoing call " - ); - // How do we interrupt `call()`? well, perhaps we need to not just await InviteSent but also CreateAnswer? - if (this._state === CallState.Fledgling || this._state === CallState.CreateOffer) { - // TODO: don't send invite! - } else { - await this.sendHangupWithCallId(this.callId, CallErrorCode.Replaced, log); - } - await this.handleInvite(content, partyId, log); - // TODO: need to skip state check - await this.answer(this.localMedia!, this.localMuteSettings!, log); - } else { - log.log( - "Glare detected: rejecting incoming call " + newCallId + - " and keeping outgoing call " - ); - await this.sendHangupWithCallId(newCallId, CallErrorCode.Replaced, log); + /** + * @returns {boolean} whether or not this call should be replaced + * */ + handleInviteGlare(message: SignallingMessage, partyId: PartyId, log: ILogItem): {shouldReplace: boolean, log?: ILogItem} { + if (message.type !== EventType.Invite) { + return {shouldReplace: false}; } + + const {content} = message; + const newCallId = content.call_id; + const shouldReplace = this.callId! > newCallId; + + let logItem; + log.wrap("handling call glare", async log => { + logItem = log; + if (shouldReplace) { + log.log( + "Glare detected: answering incoming call " + newCallId + + " and canceling outgoing call " + ); + // TODO: How do we interrupt `call()`? well, perhaps we need to not just await InviteSent but also CreateAnswer? + if (this._state !== CallState.Fledgling && this._state !== CallState.CreateOffer) { + await this.sendHangupWithCallId(this.callId, CallErrorCode.Replaced, log); + } + // since this method isn't awaited, we dispose ourselves once we hung up + this.close(CallErrorCode.Replaced, log); + this.dispose(); + } else { + log.log( + "Glare detected: rejecting incoming call " + newCallId + + " and keeping outgoing call " + ); + await this.sendHangupWithCallId(newCallId, CallErrorCode.Replaced, log); + } + }); + + return {shouldReplace, log: logItem}; } private handleHangupReceived(content: MCallHangupReject, log: ILogItem) { diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 9e87fa22..98e2aa4d 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {PeerCall, CallState} from "../PeerCall"; +import {PeerCall, CallState, IncomingMessageAction} from "../PeerCall"; import {makeTxnId, makeId} from "../../common"; import {EventType, CallErrorCode} from "../callEventTypes"; import {formatToDeviceMessagesPayload} from "../../common"; @@ -251,7 +251,7 @@ export class Member { } /** @internal */ - handleDeviceMessage(message: SignallingMessage, syncLog: ILogItem): void{ + handleDeviceMessage(message: SignallingMessage, syncLog: ILogItem): void { const {connection} = this; if (connection) { const destSessionId = message.content.dest_session_id; @@ -260,36 +260,65 @@ export class Member { syncLog.refDetached(logItem); return; } + // if there is no peerCall, we either create it with an invite and Handle is implied or we'll ignore it + let action = IncomingMessageAction.Handle; + if (connection.peerCall) { + action = connection.peerCall.getMessageAction(message); + // deal with glare and replacing the call before creating new calls + if (action === IncomingMessageAction.InviteGlare) { + const {shouldReplace, log} = connection.peerCall.handleInviteGlare(message, this.deviceId, connection.logItem); + if (log) { + syncLog.refDetached(log); + } + if (shouldReplace) { + connection.peerCall = undefined; + } + } + } if (message.type === EventType.Invite && !connection.peerCall) { connection.peerCall = this._createPeerCall(message.content.call_id); } - const idx = sortedIndex(connection.queuedSignallingMessages, message, (a, b) => a.content.seq - b.content.seq); - connection.queuedSignallingMessages.splice(idx, 0, message); - let hasBeenDequeued = false; - if (connection.peerCall) { - while ( - connection.queuedSignallingMessages.length && ( - connection.lastProcessedSeqNr === undefined || - connection.queuedSignallingMessages[0].content.seq === connection.lastProcessedSeqNr + 1 - ) - ) { - const dequeuedMessage = connection.queuedSignallingMessages.shift()!; - if (dequeuedMessage === message) { - hasBeenDequeued = true; + if (action === IncomingMessageAction.Handle) { + const idx = sortedIndex(connection.queuedSignallingMessages, message, (a, b) => a.content.seq - b.content.seq); + connection.queuedSignallingMessages.splice(idx, 0, message); + if (connection.peerCall) { + const hasNewMessageBeenDequeued = this.dequeueSignallingMessages(connection, connection.peerCall, message, syncLog); + if (!hasNewMessageBeenDequeued) { + syncLog.refDetached(connection.logItem.log({l: "queued signalling message", type: message.type, seq: message.content.seq})); } - const item = connection.peerCall!.handleIncomingSignallingMessage(dequeuedMessage, this.deviceId, connection.logItem); - syncLog.refDetached(item); - connection.lastProcessedSeqNr = dequeuedMessage.content.seq; } - } - if (!hasBeenDequeued) { - syncLog.refDetached(connection.logItem.log({l: "queued signalling message", type: message.type, seq: message.content.seq})); + } else if (action === IncomingMessageAction.Ignore && connection.peerCall) { + const logItem = connection.logItem.log({l: "ignoring to_device event with wrong call_id", callId: message.content.call_id, type: message.type}); + syncLog.refDetached(logItem); } } else { syncLog.log({l: "member not connected", userId: this.userId, deviceId: this.deviceId}); } } + private dequeueSignallingMessages(connection: MemberConnection, peerCall: PeerCall, newMessage: SignallingMessage, syncLog: ILogItem): boolean { + let hasNewMessageBeenDequeued = false; + while ( + connection.queuedSignallingMessages.length && ( + connection.lastProcessedSeqNr === undefined || + connection.queuedSignallingMessages[0].content.seq === connection.lastProcessedSeqNr + 1 + ) + ) { + const message = connection.queuedSignallingMessages.shift()!; + if (message === newMessage) { + hasNewMessageBeenDequeued = true; + } + // ignore items in the queue that should not be handled and prevent + // the lastProcessedSeqNr being corrupted with the `seq` for other call ids + if (peerCall.getMessageAction(message) === IncomingMessageAction.Handle) { + const item = peerCall.handleIncomingSignallingMessage(message, this.deviceId, connection.logItem); + syncLog.refDetached(item); + connection.lastProcessedSeqNr = message.content.seq; + } + } + return hasNewMessageBeenDequeued; + } + /** @internal */ async setMedia(localMedia: LocalMedia, previousMedia: LocalMedia): Promise { const {connection} = this; From 7ce5cdfc4ae4840028af2a3ea96ea431920c71bd Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 22 Sep 2022 13:19:56 +0200 Subject: [PATCH 170/323] prevent concurrent calls of OlmEncryption.encrypt to OOM olm wasm heap This is being triggered by connecting to many call members at once, while encrypting the signaling messages. This keeps many olm.Session objects into memory at the same time, which makes olm run out of wasm heap memory. --- src/matrix/e2ee/olm/Encryption.ts | 38 ++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/src/matrix/e2ee/olm/Encryption.ts b/src/matrix/e2ee/olm/Encryption.ts index dcc9f0b1..5fd1f25b 100644 --- a/src/matrix/e2ee/olm/Encryption.ts +++ b/src/matrix/e2ee/olm/Encryption.ts @@ -21,6 +21,7 @@ import {createSessionEntry} from "./Session"; import type {OlmMessage, OlmPayload, OlmEncryptedMessageContent} from "./types"; import type {Account} from "../Account"; import type {LockMap} from "../../../utils/LockMap"; +import {Lock, MultiLock, ILock} from "../../../utils/Lock"; import type {Storage} from "../../storage/idb/Storage"; import type {Transaction} from "../../storage/idb/Transaction"; import type {DeviceIdentity} from "../../storage/idb/stores/DeviceIdentityStore"; @@ -62,6 +63,9 @@ const OTK_ALGORITHM = "signed_curve25519"; const MAX_BATCH_SIZE = 20; export class Encryption { + + private _batchLocks: Array; + constructor( private readonly account: Account, private readonly pickleKey: string, @@ -71,14 +75,42 @@ export class Encryption { private readonly ownUserId: string, private readonly olmUtil: Olm.Utility, private readonly senderKeyLock: LockMap - ) {} + ) { + this._batchLocks = new Array(MAX_BATCH_SIZE); + for (let i = 0; i < MAX_BATCH_SIZE; i += 1) { + this._batchLocks[i] = new Lock(); + } + } + + /** A hack to prevent olm OOMing when `encrypt` is called several times concurrently, + * which is the case when encrypting voip signalling message to send over to_device. + * A better fix will be to extract the common bits from megolm/KeyLoader in a super class + * and have some sort of olm/SessionLoader that is shared between encryption and decryption + * and only keeps the olm session in wasm memory for a brief moment, like we already do for RoomKeys, + * and get the benefit of an optimal cache at the same time. + * */ + private async _takeBatchLock(amount: number): Promise { + const locks = this._batchLocks.filter(l => !l.isTaken).slice(0, amount); + if (locks.length < amount) { + const takenLocks = this._batchLocks.filter(l => l.isTaken).slice(0, amount - locks.length); + locks.push(...takenLocks); + } + await Promise.all(locks.map(l => l.take())); + return new MultiLock(locks); + } async encrypt(type: string, content: Record, devices: DeviceIdentity[], hsApi: HomeServerApi, log: ILogItem): Promise { let messages: EncryptedMessage[] = []; for (let i = 0; i < devices.length ; i += MAX_BATCH_SIZE) { const batchDevices = devices.slice(i, i + MAX_BATCH_SIZE); - const batchMessages = await this._encryptForMaxDevices(type, content, batchDevices, hsApi, log); - messages = messages.concat(batchMessages); + const batchLock = await this._takeBatchLock(batchDevices.length); + try { + const batchMessages = await this._encryptForMaxDevices(type, content, batchDevices, hsApi, log); + messages = messages.concat(batchMessages); + } + finally { + batchLock.release(); + } } return messages; } From 8aa62b257326cdf94ac0c168d83174cfa169b06c Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 25 Sep 2022 21:39:38 +0100 Subject: [PATCH 171/323] don't ignore end-of-ICE candidates --- src/matrix/calls/PeerCall.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index d004dedc..9a67fae1 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -818,14 +818,16 @@ export class PeerCall implements IDisposable { private async addIceCandidates(candidates: RTCIceCandidate[], log: ILogItem): Promise { for (const candidate of candidates) { + let logItem; if ( (candidate.sdpMid === null || candidate.sdpMid === undefined) && (candidate.sdpMLineIndex === null || candidate.sdpMLineIndex === undefined) ) { - log.log(`Ignoring remote ICE candidate with no sdpMid or sdpMLineIndex`); - continue; + logItem = log.log(`Got remote ICE end-of-ICE candidates`); + } + else { + logItem = log.log(`Adding remote ICE ${candidate.sdpMid} candidate: ${candidate.candidate}`); } - const logItem = log.log(`Adding remote ICE ${candidate.sdpMid} candidate: ${candidate.candidate}`); try { await this.peerConnection.addIceCandidate(candidate); } catch (err) { From 31e67142bdc1ffa2d503c7492371a703a26e07b2 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 25 Sep 2022 22:11:35 +0100 Subject: [PATCH 172/323] typo --- src/matrix/calls/PeerCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 9a67fae1..1ee945c5 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -823,7 +823,7 @@ export class PeerCall implements IDisposable { (candidate.sdpMid === null || candidate.sdpMid === undefined) && (candidate.sdpMLineIndex === null || candidate.sdpMLineIndex === undefined) ) { - logItem = log.log(`Got remote ICE end-of-ICE candidates`); + logItem = log.log(`Got remote end-of-ICE candidates`); } else { logItem = log.log(`Adding remote ICE ${candidate.sdpMid} candidate: ${candidate.candidate}`); From af2098327bc023062da0b6ac6607f1051eea9549 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 26 Sep 2022 00:47:12 +0100 Subject: [PATCH 173/323] first cut at implementing TURN --- src/matrix/Session.js | 13 +++++++++++++ src/matrix/calls/CallHandler.ts | 6 ++++++ src/matrix/net/HomeServerApi.ts | 6 +++++- 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index f82ad555..de05f040 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -513,6 +513,7 @@ export class Session { // TODO: what can we do if this throws? await txn.complete(); } + await this._updateTurnServers(); // enable session backup, this requests the latest backup version if (!this._keyBackup.get()) { if (dehydratedDevice) { @@ -559,6 +560,18 @@ export class Session { } } + async _updateTurnServers() { + const turnServersData = await this._hsApi.getTurnServers().response(); + this._callHandler.setTurnServers({ + urls: turnServerData.uris, + username: turnServerData.username, + credential: turnServerData.password, + }); + if (turnServersData.ttl > 0) { + setTimeout(this._updateTurnServers, turnServersData.ttl * 1000); + } + } + async _getPendingEventsByRoom(txn) { const pendingEvents = await txn.pendingEvents.getAll(); return pendingEvents.reduce((groups, pe) => { diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index ca58c4a8..93491e07 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -75,6 +75,12 @@ export class CallHandler implements RoomStateHandler { this._loadCallEntries(callEntries, txn); } + async setTurnServers(turnServers: RTCIceServer) { + this.options.turnServers = turnServers; + this.groupCallOptions.turnServers = turnServers; + // TODO: we should update any ongoing peerconnections if the TURN server details have changed + } + private async _getLoadTxn(): Promise { const names = this.options.storage.storeNames; const txn = await this.options.storage.readTxn([ diff --git a/src/matrix/net/HomeServerApi.ts b/src/matrix/net/HomeServerApi.ts index 30406c34..828a1c82 100644 --- a/src/matrix/net/HomeServerApi.ts +++ b/src/matrix/net/HomeServerApi.ts @@ -305,10 +305,14 @@ export class HomeServerApi { createRoom(payload: Record, options?: BaseRequestOptions): IHomeServerRequest { return this._post(`/createRoom`, {}, payload, options); } - + setAccountData(ownUserId: string, type: string, content: Record, options?: BaseRequestOptions): IHomeServerRequest { return this._put(`/user/${encodeURIComponent(ownUserId)}/account_data/${encodeURIComponent(type)}`, {}, content, options); } + + getTurnServer(options?: BaseRequestOptions): IHomeServerRequest { + return this._get(`/voip/turnServer`, undefined, undefined, options); + } } import {Request as MockRequest} from "../../mocks/Request.js"; From eab87e5157c795d255ae3760515b29ddd477aec8 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 26 Sep 2022 00:52:41 +0100 Subject: [PATCH 174/323] fix plurals --- src/matrix/Session.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index de05f040..527c0e00 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -561,14 +561,14 @@ export class Session { } async _updateTurnServers() { - const turnServersData = await this._hsApi.getTurnServers().response(); + const turnServerData = await this._hsApi.getTurnServer().response(); this._callHandler.setTurnServers({ urls: turnServerData.uris, username: turnServerData.username, credential: turnServerData.password, }); - if (turnServersData.ttl > 0) { - setTimeout(this._updateTurnServers, turnServersData.ttl * 1000); + if (turnServerData.ttl > 0) { + setTimeout(this._updateTurnServers, turnServerData.ttl * 1000); } } From 3d7852a469bd4584798dad39e834cbe38451f763 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 26 Sep 2022 01:07:24 +0100 Subject: [PATCH 175/323] set array of turnservers --- src/matrix/Session.js | 4 ++-- src/matrix/calls/CallHandler.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 527c0e00..3c49182d 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -562,11 +562,11 @@ export class Session { async _updateTurnServers() { const turnServerData = await this._hsApi.getTurnServer().response(); - this._callHandler.setTurnServers({ + this._callHandler.setTurnServers([{ urls: turnServerData.uris, username: turnServerData.username, credential: turnServerData.password, - }); + }]); if (turnServerData.ttl > 0) { setTimeout(this._updateTurnServers, turnServerData.ttl * 1000); } diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 93491e07..183ec5a1 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -75,7 +75,7 @@ export class CallHandler implements RoomStateHandler { this._loadCallEntries(callEntries, txn); } - async setTurnServers(turnServers: RTCIceServer) { + async setTurnServers(turnServers: RTCIceServer[]) { this.options.turnServers = turnServers; this.groupCallOptions.turnServers = turnServers; // TODO: we should update any ongoing peerconnections if the TURN server details have changed From 6570ec46f4a9ba500215b7b15f65e858f74cb0a2 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 26 Sep 2022 09:02:22 +0100 Subject: [PATCH 176/323] setTurnServers doesn't need to be async --- src/matrix/calls/CallHandler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 183ec5a1..17aa0e55 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -75,7 +75,7 @@ export class CallHandler implements RoomStateHandler { this._loadCallEntries(callEntries, txn); } - async setTurnServers(turnServers: RTCIceServer[]) { + setTurnServers(turnServers: RTCIceServer[]) { this.options.turnServers = turnServers; this.groupCallOptions.turnServers = turnServers; // TODO: we should update any ongoing peerconnections if the TURN server details have changed From f4e633beb4faebe1eaced7ea64f3903ecc57f08b Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:24:14 +0200 Subject: [PATCH 177/323] support onfirstsubscribe callback in ROV will use it to start polling for turnServer updates --- src/observable/value/RetainedObservableValue.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/observable/value/RetainedObservableValue.ts b/src/observable/value/RetainedObservableValue.ts index edfb6c15..16058f8e 100644 --- a/src/observable/value/RetainedObservableValue.ts +++ b/src/observable/value/RetainedObservableValue.ts @@ -17,15 +17,17 @@ limitations under the License. import {ObservableValue} from "./ObservableValue"; export class RetainedObservableValue extends ObservableValue { - private _freeCallback: () => void; - constructor(initialValue: T, freeCallback: () => void) { + constructor(initialValue: T, private freeCallback: () => void, private startCallback: () => void = () => {}) { super(initialValue); - this._freeCallback = freeCallback; + } + + onSubscribeFirst() { + this.startCallback(); } onUnsubscribeLast() { super.onUnsubscribeLast(); - this._freeCallback(); + this.freeCallback(); } } From 035ead0d5b47c7ebb742f74aae0e4949f556fcb9 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:24:41 +0200 Subject: [PATCH 178/323] implement polling of voip turnServer settings from HS in separate class --- src/matrix/calls/TurnServerSource.ts | 222 +++++++++++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100644 src/matrix/calls/TurnServerSource.ts diff --git a/src/matrix/calls/TurnServerSource.ts b/src/matrix/calls/TurnServerSource.ts new file mode 100644 index 00000000..cc6923af --- /dev/null +++ b/src/matrix/calls/TurnServerSource.ts @@ -0,0 +1,222 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {RetainedObservableValue} from "../../observable/value/RetainedObservableValue"; + +import type {HomeServerApi} from "../net/HomeServerApi"; +import type {IHomeServerRequest} from "../net/HomeServerRequest"; +import type {BaseObservableValue} from "../../observable/value/BaseObservableValue"; +import type {ObservableValue} from "../../observable/value/ObservableValue"; +import type {Clock, Timeout} from "../../platform/web/dom/Clock"; +import type {ILogItem} from "../../logging/types"; + +type TurnServerSettings = { + urls: string[], + username: string, + password: string, + ttl: number +}; + +const DEFAULT_TTL = 5 * 60; // 5min +const DEFAULT_SETTINGS: RTCIceServer = { + urls: ["stun:turn.matrix.org"], + username: "", + credential: "", +}; + +export class TurnServerSource { + private currentObservable?: ObservableValue; + private pollTimeout?: Timeout; + private pollRequest?: IHomeServerRequest; + private isPolling = false; + + constructor( + private hsApi: HomeServerApi, + private clock: Clock, + private defaultSettings: RTCIceServer = DEFAULT_SETTINGS + ) {} + + getSettings(log: ILogItem): Promise> { + return log.wrap("get turn server", async log => { + if (!this.isPolling) { + const settings = await this.doRequest(log); + const iceServer = settings ? toIceServer(settings) : this.defaultSettings; + if (this.currentObservable) { + this.currentObservable.set(iceServer); + } else { + this.currentObservable = new RetainedObservableValue(iceServer, + () => { + this.stopPollLoop(); + }, + () => { + // start loop on first subscribe + this.runLoop(this.currentObservable!, settings?.ttl ?? DEFAULT_TTL); + }); + } + } + return this.currentObservable!; + }); + } + + private async runLoop(observable: ObservableValue, initialTtl: number): Promise { + let ttl = initialTtl; + this.isPolling = true; + while(this.isPolling) { + try { + this.pollTimeout = this.clock.createTimeout(ttl * 1000); + await this.pollTimeout.elapsed(); + this.pollTimeout = undefined; + const settings = await this.doRequest(undefined); + if (settings) { + const iceServer = toIceServer(settings); + if (shouldUpdate(observable, iceServer)) { + observable.set(iceServer); + } + if (settings.ttl > 0) { + ttl = settings.ttl; + } else { + // stop polling is settings are good indefinitely + this.stopPollLoop(); + } + } else { + ttl = DEFAULT_TTL; + } + } catch (err) { + if (err.name === "AbortError") { + /* ignore, the loop will exit because isPolling is false */ + } else { + // TODO: log error + } + } + } + } + + private async doRequest(log: ILogItem | undefined): Promise { + try { + this.pollRequest = this.hsApi.getTurnServer({log}); + const settings = await this.pollRequest.response(); + return settings; + } catch (err) { + if (err.name === "HomeServerError") { + return undefined; + } + throw err; + } finally { + this.pollRequest = undefined; + } + } + + stopPollLoop() { + this.isPolling = false; + this.currentObservable = undefined; + this.pollTimeout?.dispose(); + this.pollTimeout = undefined; + this.pollRequest?.abort(); + this.pollRequest = undefined; + } + + dispose() { + this.stopPollLoop(); + } +} + +function shouldUpdate(observable: BaseObservableValue, settings: RTCIceServer): boolean { + const currentSettings = observable.get(); + if (!currentSettings) { + return true; + } + // same length and new settings doesn't contain any uri the old settings don't contain + const currentUrls = Array.isArray(currentSettings.urls) ? currentSettings.urls : [currentSettings.urls]; + const newUrls = Array.isArray(settings.urls) ? settings.urls : [settings.urls]; + const arraysEqual = currentUrls.length === newUrls.length && + !newUrls.some(uri => !currentUrls.includes(uri)); + return !arraysEqual || settings.username !== currentSettings.username || + settings.credential !== currentSettings.credential; +} + +function toIceServer(settings: TurnServerSettings): RTCIceServer { + return { + urls: settings.urls, + username: settings.username, + credential: settings.password, + credentialType: "password" + } +} + +export function tests() { + return { + "shouldUpdate returns false for same object": assert => { + const observable = {get() { + return { + urls: ["a", "b"], + username: "alice", + credential: "f00", + }; + }}; + const same = { + urls: ["a", "b"], + username: "alice", + credential: "f00", + }; + assert.equal(false, shouldUpdate(observable as any as BaseObservableValue, same)); + }, + "shouldUpdate returns true for 1 different uri": assert => { + const observable = {get() { + return { + urls: ["a", "c"], + username: "alice", + credential: "f00", + }; + }}; + const same = { + urls: ["a", "b"], + username: "alice", + credential: "f00", + }; + assert.equal(true, shouldUpdate(observable as any as BaseObservableValue, same)); + }, + "shouldUpdate returns true for different user": assert => { + const observable = {get() { + return { + urls: ["a", "b"], + username: "alice", + credential: "f00", + }; + }}; + const same = { + urls: ["a", "b"], + username: "bob", + credential: "f00", + }; + assert.equal(true, shouldUpdate(observable as any as BaseObservableValue, same)); + }, + "shouldUpdate returns true for different password": assert => { + const observable = {get() { + return { + urls: ["a", "b"], + username: "alice", + credential: "f00", + }; + }}; + const same = { + urls: ["a", "b"], + username: "alice", + credential: "b4r", + }; + assert.equal(true, shouldUpdate(observable as any as BaseObservableValue, same)); + } + } +} From 3a4c38086c066c31534263d98c19e875b3627a0f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:25:24 +0200 Subject: [PATCH 179/323] fetch turn server settings when joining a call, and pass down --- src/matrix/calls/CallHandler.ts | 8 ++------ src/matrix/calls/PeerCall.ts | 3 ++- src/matrix/calls/group/GroupCall.ts | 19 +++++++++++++++---- src/matrix/calls/group/Member.ts | 22 ++++++++++++++++------ 4 files changed, 35 insertions(+), 17 deletions(-) diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 17aa0e55..354c9e6a 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -23,6 +23,7 @@ import {GroupCall} from "./group/GroupCall"; import {makeId} from "../common"; import {CALL_LOG_TYPE} from "./common"; import {EVENT_TYPE as MEMBER_EVENT_TYPE, RoomMember} from "../room/members/RoomMember"; +import {TurnServerSource} from "./TurnServerSource"; import type {LocalMedia} from "./LocalMedia"; import type {Room} from "../room/Room"; @@ -57,6 +58,7 @@ export class CallHandler implements RoomStateHandler { constructor(private readonly options: Options) { this.groupCallOptions = Object.assign({}, this.options, { + turnServerSource: new TurnServerSource(this.options.hsApi, this.options.clock), emitUpdate: (groupCall, params) => this._calls.update(groupCall.id, params), createTimeout: this.options.clock.createTimeout, sessionId: this.sessionId @@ -75,12 +77,6 @@ export class CallHandler implements RoomStateHandler { this._loadCallEntries(callEntries, txn); } - setTurnServers(turnServers: RTCIceServer[]) { - this.options.turnServers = turnServers; - this.groupCallOptions.turnServers = turnServers; - // TODO: we should update any ongoing peerconnections if the TURN server details have changed - } - private async _getLoadTxn(): Promise { const names = this.options.storage.storeNames; const txn = await this.options.storage.readTxn([ diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index d004dedc..ef906d97 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -15,6 +15,7 @@ limitations under the License. */ import {ObservableMap} from "../../observable/map/ObservableMap"; +import {BaseObservableValue} from "../../observable/value/BaseObservableValue"; import {recursivelyAssign} from "../../utils/recursivelyAssign"; import {Disposables, Disposable, IDisposable} from "../../utils/Disposables"; import {WebRTC, PeerConnection, Transceiver, TransceiverDirection, Sender, Receiver, PeerConnectionEventMap} from "../../platform/types/WebRTC"; @@ -47,7 +48,7 @@ import type { export type Options = { webRTC: WebRTC, forceTURN: boolean, - turnServers: RTCIceServer[], + turnServer: BaseObservableValue, createTimeout: TimeoutCreator, emitUpdate: (peerCall: PeerCall, params: any, log: ILogItem) => void; sendSignallingMessage: (message: SignallingMessage, log: ILogItem) => Promise; diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 07aa2a08..4c038be9 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -23,6 +23,7 @@ import {EventEmitter} from "../../../utils/EventEmitter"; import {EventType, CallIntent} from "../callEventTypes"; import type {Options as MemberOptions} from "./Member"; +import type {TurnServerSource} from "../TurnServerSource"; import type {BaseObservableMap} from "../../../observable/map/BaseObservableMap"; import type {Track} from "../../../platform/types/MediaDevices"; import type {SignallingMessage, MGroupCallBase, CallMembership} from "../callEventTypes"; @@ -32,6 +33,7 @@ import type {Platform} from "../../../platform/web/Platform"; import type {EncryptedMessage} from "../../e2ee/olm/Encryption"; import type {ILogItem, ILogger} from "../../../logging/types"; import type {Storage} from "../../storage/idb/Storage"; +import type {BaseObservableValue} from "../../../observable/value/BaseObservableValue"; export enum GroupCallState { Fledgling = "fledgling", @@ -53,11 +55,12 @@ function getDeviceFromMemberKey(key: string): string { return JSON.parse(`[${key}]`)[1]; } -export type Options = Omit & { +export type Options = Omit & { emitUpdate: (call: GroupCall, params?: any) => void; encryptDeviceMessage: (roomId: string, userId: string, deviceId: string, message: SignallingMessage, log: ILogItem) => Promise, storage: Storage, logger: ILogger, + turnServerSource: TurnServerSource }; class JoinedData { @@ -65,7 +68,8 @@ class JoinedData { public readonly logItem: ILogItem, public readonly membersLogItem: ILogItem, public localMedia: LocalMedia, - public localMuteSettings: MuteSettings + public localMuteSettings: MuteSettings, + public turnServer: BaseObservableValue ) {} dispose() { @@ -136,6 +140,7 @@ export class GroupCall extends EventEmitter<{change: never}> { id: this.id, ownSessionId: this.options.sessionId }); + const turnServer = await this.options.turnServerSource.getSettings(logItem); const membersLogItem = logItem.child("member connections"); const localMuteSettings = new MuteSettings(); localMuteSettings.updateTrackInfo(localMedia.userMedia); @@ -143,7 +148,8 @@ export class GroupCall extends EventEmitter<{change: never}> { logItem, membersLogItem, localMedia, - localMuteSettings + localMuteSettings, + turnServer ); this.joinedData = joinedData; await joinedData.logItem.wrap("join", async log => { @@ -509,7 +515,12 @@ export class GroupCall extends EventEmitter<{change: never}> { const logItem = joinedData.membersLogItem.child({l: "member", id: memberKey}); logItem.set("sessionId", member.sessionId); log.wrap({l: "connect", id: memberKey}, log => { - const connectItem = member.connect(joinedData.localMedia, joinedData.localMuteSettings, logItem); + const connectItem = member.connect( + joinedData.localMedia, + joinedData.localMuteSettings, + joinedData.turnServer, + logItem + ); if (connectItem) { log.refDetached(connectItem); } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 98e2aa4d..7cd900f2 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -19,6 +19,7 @@ import {makeTxnId, makeId} from "../../common"; import {EventType, CallErrorCode} from "../callEventTypes"; import {formatToDeviceMessagesPayload} from "../../common"; import {sortedIndex} from "../../../utils/sortedIndex"; + import type {MuteSettings} from "../common"; import type {Options as PeerCallOptions, RemoteMedia} from "../PeerCall"; import type {LocalMedia} from "../LocalMedia"; @@ -28,8 +29,9 @@ import type {GroupCall} from "./GroupCall"; import type {RoomMember} from "../../room/members/RoomMember"; import type {EncryptedMessage} from "../../e2ee/olm/Encryption"; import type {ILogItem} from "../../../logging/types"; +import type {BaseObservableValue} from "../../../observable/value/BaseObservableValue"; -export type Options = Omit & { +export type Options = Omit & { confId: string, ownUserId: string, ownDeviceId: string, @@ -60,6 +62,7 @@ class MemberConnection { constructor( public localMedia: LocalMedia, public localMuteSettings: MuteSettings, + public turnServer: BaseObservableValue, public readonly logItem: ILogItem ) {} } @@ -112,12 +115,17 @@ export class Member { } /** @internal */ - connect(localMedia: LocalMedia, localMuteSettings: MuteSettings, memberLogItem: ILogItem): ILogItem | undefined { + connect(localMedia: LocalMedia, localMuteSettings: MuteSettings, turnServer: BaseObservableValue, memberLogItem: ILogItem): ILogItem | undefined { if (this.connection) { return; } // Safari can't send a MediaStream to multiple sources, so clone it - const connection = new MemberConnection(localMedia.clone(), localMuteSettings, memberLogItem); + const connection = new MemberConnection( + localMedia.clone(), + localMuteSettings, + turnServer, + memberLogItem + ); this.connection = connection; let connectLogItem; connection.logItem.wrap("connect", async log => { @@ -204,7 +212,7 @@ export class Member { if (hangupReason && !errorCodesWithoutRetry.includes(hangupReason)) { connection.retryCount += 1; const {retryCount} = connection; - connection.logItem.wrap({l: "retry connection", retryCount}, async retryLog => { + connection.logItem.wrap({l: "retry connection", retryCount}, async retryLog => { log.refDetached(retryLog); if (retryCount <= 3) { await this.callIfNeeded(retryLog); @@ -337,9 +345,11 @@ export class Member { } private _createPeerCall(callId: string): PeerCall { + const connection = this.connection!; return new PeerCall(callId, Object.assign({}, this.options, { emitUpdate: this.emitUpdateFromPeerCall, - sendSignallingMessage: this.sendSignallingMessage - }), this.connection!.logItem); + sendSignallingMessage: this.sendSignallingMessage, + turnServer: connection.turnServer + }), connection.logItem); } } From 917ad52302a78d7367c19a56287bc4284e411f5a Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:25:53 +0200 Subject: [PATCH 180/323] use observable for turnserver in peercall and subscribe if changing config is supported --- src/matrix/calls/PeerCall.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index ef906d97..ccff5868 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -115,8 +115,18 @@ export class PeerCall implements IDisposable { ) { logItem.log({l: "create PeerCall", id: callId}); this._remoteMedia = new RemoteMedia(); - this.peerConnection = options.webRTC.createPeerConnection(this.options.forceTURN, this.options.turnServers, 0); - + this.peerConnection = options.webRTC.createPeerConnection( + this.options.forceTURN, + [this.options.turnServer.get()], + 0 + ); + // update turn servers when they change (see TurnServerSource) if possible + if (typeof this.peerConnection["setConfiguration"] === "function") { + this.disposables.track(this.options.turnServer.subscribe(turnServer => { + this.logItem.log({l: "updating turn server", turnServer}) + this.peerConnection["setConfiguration"]({iceServers: [turnServer]}); + })); + } const listen = (type: K, listener: (this: PeerConnection, ev: PeerConnectionEventMap[K]) => any, options?: boolean | EventListenerOptions): void => { this.peerConnection.addEventListener(type, listener); const dispose = () => { From f74c4e642590395e18f135d6266578123c7a7350 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:26:16 +0200 Subject: [PATCH 181/323] remove previous approach --- src/matrix/Session.js | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 3c49182d..f82ad555 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -513,7 +513,6 @@ export class Session { // TODO: what can we do if this throws? await txn.complete(); } - await this._updateTurnServers(); // enable session backup, this requests the latest backup version if (!this._keyBackup.get()) { if (dehydratedDevice) { @@ -560,18 +559,6 @@ export class Session { } } - async _updateTurnServers() { - const turnServerData = await this._hsApi.getTurnServer().response(); - this._callHandler.setTurnServers([{ - urls: turnServerData.uris, - username: turnServerData.username, - credential: turnServerData.password, - }]); - if (turnServerData.ttl > 0) { - setTimeout(this._updateTurnServers, turnServerData.ttl * 1000); - } - } - async _getPendingEventsByRoom(txn) { const pendingEvents = await txn.pendingEvents.getAll(); return pendingEvents.reduce((groups, pe) => { From e6bf49a6ccbee379613848d6f25fd7fa9981f817 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:27:41 +0200 Subject: [PATCH 182/323] remove previous hardcoded turnServers setting --- src/matrix/Session.js | 3 --- src/matrix/calls/CallHandler.ts | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index f82ad555..7d187b30 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -101,9 +101,6 @@ export class Session { ownDeviceId: sessionInfo.deviceId, ownUserId: sessionInfo.userId, logger: this._platform.logger, - turnServers: [{ - urls: ["stun:turn.matrix.org"], - }], forceTURN: false, }); this._roomStateHandler = new RoomStateHandlerSet(); diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 354c9e6a..7b8265c5 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -40,7 +40,7 @@ import type {Clock} from "../../platform/web/dom/Clock"; import type {RoomStateHandler} from "../room/state/types"; import type {MemberSync} from "../room/timeline/persistence/MemberWriter"; -export type Options = Omit & { +export type Options = Omit & { clock: Clock }; From 05c2da95c4ffa99b2581e7b1a3077a5fcb41901d Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:38:04 +0200 Subject: [PATCH 183/323] fix typo --- src/matrix/calls/TurnServerSource.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/matrix/calls/TurnServerSource.ts b/src/matrix/calls/TurnServerSource.ts index cc6923af..cac08bdf 100644 --- a/src/matrix/calls/TurnServerSource.ts +++ b/src/matrix/calls/TurnServerSource.ts @@ -24,7 +24,7 @@ import type {Clock, Timeout} from "../../platform/web/dom/Clock"; import type {ILogItem} from "../../logging/types"; type TurnServerSettings = { - urls: string[], + uris: string[], username: string, password: string, ttl: number @@ -54,6 +54,7 @@ export class TurnServerSource { if (!this.isPolling) { const settings = await this.doRequest(log); const iceServer = settings ? toIceServer(settings) : this.defaultSettings; + log.set("iceServer", iceServer); if (this.currentObservable) { this.currentObservable.set(iceServer); } else { @@ -149,7 +150,7 @@ function shouldUpdate(observable: BaseObservableValue, function toIceServer(settings: TurnServerSettings): RTCIceServer { return { - urls: settings.urls, + urls: settings.uris, username: settings.username, credential: settings.password, credentialType: "password" From d36b9be24f1d041c2c668f213041165a23921fd0 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:41:30 +0200 Subject: [PATCH 184/323] fix screwing up whitespace --- src/matrix/calls/group/Member.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 7cd900f2..b1c7b430 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -212,7 +212,7 @@ export class Member { if (hangupReason && !errorCodesWithoutRetry.includes(hangupReason)) { connection.retryCount += 1; const {retryCount} = connection; - connection.logItem.wrap({l: "retry connection", retryCount}, async retryLog => { + connection.logItem.wrap({l: "retry connection", retryCount}, async retryLog => { log.refDetached(retryLog); if (retryCount <= 3) { await this.callIfNeeded(retryLog); From ac319bdafd747b1dbf3575dc2aeba59145db50cb Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:45:41 +0200 Subject: [PATCH 185/323] we can assume setConfiguration is available --- src/matrix/calls/PeerCall.ts | 12 +++++------- src/platform/types/WebRTC.ts | 1 + 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index ccff5868..a7ae7b04 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -120,13 +120,11 @@ export class PeerCall implements IDisposable { [this.options.turnServer.get()], 0 ); - // update turn servers when they change (see TurnServerSource) if possible - if (typeof this.peerConnection["setConfiguration"] === "function") { - this.disposables.track(this.options.turnServer.subscribe(turnServer => { - this.logItem.log({l: "updating turn server", turnServer}) - this.peerConnection["setConfiguration"]({iceServers: [turnServer]}); - })); - } + // update turn servers when they change (see TurnServerSource) + this.disposables.track(this.options.turnServer.subscribe(turnServer => { + this.logItem.log({l: "updating turn server", turnServer}) + this.peerConnection.setConfiguration({iceServers: [turnServer]}); + })); const listen = (type: K, listener: (this: PeerConnection, ev: PeerConnectionEventMap[K]) => any, options?: boolean | EventListenerOptions): void => { this.peerConnection.addEventListener(type, listener); const dispose = () => { diff --git a/src/platform/types/WebRTC.ts b/src/platform/types/WebRTC.ts index 39ad49c5..236e8354 100644 --- a/src/platform/types/WebRTC.ts +++ b/src/platform/types/WebRTC.ts @@ -148,6 +148,7 @@ export interface PeerConnection { addEventListener(type: K, listener: (this: PeerConnection, ev: PeerConnectionEventMap[K]) => any, options?: boolean | AddEventListenerOptions): void; removeEventListener(type: K, listener: (this: PeerConnection, ev: PeerConnectionEventMap[K]) => any, options?: boolean | EventListenerOptions): void; getStats(selector?: Track | null): Promise; + setConfiguration(configuration?: RTCConfiguration): void; } From 24ebf6c559154aa6dcbc4685613b1b57e561f7de Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:47:48 +0200 Subject: [PATCH 186/323] cleanup --- src/matrix/calls/TurnServerSource.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/matrix/calls/TurnServerSource.ts b/src/matrix/calls/TurnServerSource.ts index cac08bdf..ca0f6848 100644 --- a/src/matrix/calls/TurnServerSource.ts +++ b/src/matrix/calls/TurnServerSource.ts @@ -64,7 +64,7 @@ export class TurnServerSource { }, () => { // start loop on first subscribe - this.runLoop(this.currentObservable!, settings?.ttl ?? DEFAULT_TTL); + this.runLoop(settings?.ttl ?? DEFAULT_TTL); }); } } @@ -72,7 +72,7 @@ export class TurnServerSource { }); } - private async runLoop(observable: ObservableValue, initialTtl: number): Promise { + private async runLoop(initialTtl: number): Promise { let ttl = initialTtl; this.isPolling = true; while(this.isPolling) { @@ -83,8 +83,8 @@ export class TurnServerSource { const settings = await this.doRequest(undefined); if (settings) { const iceServer = toIceServer(settings); - if (shouldUpdate(observable, iceServer)) { - observable.set(iceServer); + if (shouldUpdate(this.currentObservable!, iceServer)) { + this.currentObservable!.set(iceServer); } if (settings.ttl > 0) { ttl = settings.ttl; From eccbab1491d6245ae95084ff120907be66cfd7f2 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:57:20 +0200 Subject: [PATCH 187/323] cleanup polling and also hangup on all ongoing calls on dispose session --- src/matrix/Session.js | 2 ++ src/matrix/calls/CallHandler.ts | 10 +++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 7d187b30..31b57b57 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -488,6 +488,8 @@ export class Session { this._megolmDecryption = undefined; this._e2eeAccount?.dispose(); this._e2eeAccount = undefined; + this._callHandler?.dispose(); + this._callHandler = undefined; for (const room of this._rooms.values()) { room.dispose(); } diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 7b8265c5..67cf3dc7 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -55,10 +55,12 @@ export class CallHandler implements RoomStateHandler { private roomMemberToCallIds: Map> = new Map(); private groupCallOptions: GroupCallOptions; private sessionId = makeId("s"); + private turnServerSource: TurnServerSource; constructor(private readonly options: Options) { + this.turnServerSource = new TurnServerSource(this.options.hsApi, this.options.clock); this.groupCallOptions = Object.assign({}, this.options, { - turnServerSource: new TurnServerSource(this.options.hsApi, this.options.clock), + turnServerSource: this.turnServerSource, emitUpdate: (groupCall, params) => this._calls.update(groupCall.id, params), createTimeout: this.options.clock.createTimeout, sessionId: this.sessionId @@ -243,5 +245,11 @@ export class CallHandler implements RoomStateHandler { this.roomMemberToCallIds.set(roomMemberKey, newCallIdsMemberOf); } } + + dispose() { + this.turnServerSource.dispose(); + const joinedCalls = Array.from(this._calls.values()).filter(c => c.hasJoined); + Promise.all(joinedCalls.map(c => c.leave())).then(() => {}, () => {}); + } } From c660d82d6a47620a78ac18b9eed443b103cec3c6 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 26 Sep 2022 16:00:25 +0200 Subject: [PATCH 188/323] make internal method private --- src/matrix/calls/TurnServerSource.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/TurnServerSource.ts b/src/matrix/calls/TurnServerSource.ts index ca0f6848..1066f7a6 100644 --- a/src/matrix/calls/TurnServerSource.ts +++ b/src/matrix/calls/TurnServerSource.ts @@ -120,7 +120,7 @@ export class TurnServerSource { } } - stopPollLoop() { + private stopPollLoop() { this.isPolling = false; this.currentObservable = undefined; this.pollTimeout?.dispose(); From 6f0e781b4970274a7d3452b2821faa745f488f56 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 26 Sep 2022 19:15:13 +0200 Subject: [PATCH 189/323] WIP --- src/matrix/Session.js | 1 + src/matrix/calls/CallHandler.ts | 11 +- src/matrix/calls/PeerCall.ts | 8 +- src/matrix/calls/callEventTypes.ts | 4 +- src/matrix/calls/common.ts | 2 + src/matrix/calls/group/GroupCall.ts | 149 ++++++++++++++++++++-------- src/matrix/calls/group/Member.ts | 48 ++++++++- src/matrix/e2ee/DeviceTracker.js | 2 +- 8 files changed, 175 insertions(+), 50 deletions(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 31b57b57..291002c5 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -78,6 +78,7 @@ export class Session { this._user = new User(sessionInfo.userId); this._callHandler = new CallHandler({ clock: this._platform.clock, + random: this._platform.random, hsApi: this._hsApi, encryptDeviceMessage: async (roomId, userId, deviceId, message, log) => { if (!this._deviceTracker || !this._olmEncryption) { diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 67cf3dc7..2b914a2f 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -55,12 +55,10 @@ export class CallHandler implements RoomStateHandler { private roomMemberToCallIds: Map> = new Map(); private groupCallOptions: GroupCallOptions; private sessionId = makeId("s"); - private turnServerSource: TurnServerSource; constructor(private readonly options: Options) { - this.turnServerSource = new TurnServerSource(this.options.hsApi, this.options.clock); this.groupCallOptions = Object.assign({}, this.options, { - turnServerSource: this.turnServerSource, + turnServerSource: new TurnServerSource(this.options.hsApi, this.options.clock), emitUpdate: (groupCall, params) => this._calls.update(groupCall.id, params), createTimeout: this.options.clock.createTimeout, sessionId: this.sessionId @@ -247,9 +245,10 @@ export class CallHandler implements RoomStateHandler { } dispose() { - this.turnServerSource.dispose(); - const joinedCalls = Array.from(this._calls.values()).filter(c => c.hasJoined); - Promise.all(joinedCalls.map(c => c.leave())).then(() => {}, () => {}); + this.groupCallOptions.turnServerSource.dispose(); + for(const call of this._calls.values()) { + call.dispose(); + } } } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 0f16b207..d7d815fa 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -1098,7 +1098,13 @@ export class PeerCall implements IDisposable { private async delay(timeoutMs: number): Promise { // Allow a short time for initial candidates to be gathered const timeout = this.disposables.track(this.options.createTimeout(timeoutMs)); - await timeout.elapsed(); + try { + await timeout.elapsed(); + } catch (err) { + if (err.name !== "AbortError") { + throw err; + } + } this.disposables.untrack(timeout); } diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index 09376c85..ca5c870c 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -24,7 +24,9 @@ export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata"; export interface CallDeviceMembership { device_id: string, - session_id: string + session_id: string, + ["m.expires_ts"]?: number, + feeds?: Array<{purpose: string}> } export interface CallMembership { diff --git a/src/matrix/calls/common.ts b/src/matrix/calls/common.ts index 66db6edc..dde55ddc 100644 --- a/src/matrix/calls/common.ts +++ b/src/matrix/calls/common.ts @@ -59,3 +59,5 @@ export class MuteSettings { } export const CALL_LOG_TYPE = "call"; +//export const CALL_MEMBER_VALIDITY_PERIOD_MS = 3600 * 1000; // 1h +export const CALL_MEMBER_VALIDITY_PERIOD_MS = 60 * 1000; diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 4c038be9..cb027be8 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -15,9 +15,9 @@ limitations under the License. */ import {ObservableMap} from "../../../observable/map/ObservableMap"; -import {Member} from "./Member"; +import {Member, isMemberExpired, memberExpiresAt} from "./Member"; import {LocalMedia} from "../LocalMedia"; -import {MuteSettings, CALL_LOG_TYPE} from "../common"; +import {MuteSettings, CALL_LOG_TYPE, CALL_MEMBER_VALIDITY_PERIOD_MS} from "../common"; import {MemberChange, RoomMember} from "../../room/members/RoomMember"; import {EventEmitter} from "../../../utils/EventEmitter"; import {EventType, CallIntent} from "../callEventTypes"; @@ -26,7 +26,7 @@ import type {Options as MemberOptions} from "./Member"; import type {TurnServerSource} from "../TurnServerSource"; import type {BaseObservableMap} from "../../../observable/map/BaseObservableMap"; import type {Track} from "../../../platform/types/MediaDevices"; -import type {SignallingMessage, MGroupCallBase, CallMembership} from "../callEventTypes"; +import type {SignallingMessage, MGroupCallBase, CallMembership, CallMemberContent, CallDeviceMembership} from "../callEventTypes"; import type {Room} from "../../room/Room"; import type {StateEvent} from "../../storage/types"; import type {Platform} from "../../../platform/web/Platform"; @@ -34,6 +34,7 @@ import type {EncryptedMessage} from "../../e2ee/olm/Encryption"; import type {ILogItem, ILogger} from "../../../logging/types"; import type {Storage} from "../../storage/idb/Storage"; import type {BaseObservableValue} from "../../../observable/value/BaseObservableValue"; +import type {Clock, Timeout} from "../../../platform/web/dom/Clock"; export enum GroupCallState { Fledgling = "fledgling", @@ -59,11 +60,14 @@ export type Options = Omit void; encryptDeviceMessage: (roomId: string, userId: string, deviceId: string, message: SignallingMessage, log: ILogItem) => Promise, storage: Storage, + random: () => number, logger: ILogger, turnServerSource: TurnServerSource }; class JoinedData { + public renewMembershipTimeout?: Timeout; + constructor( public readonly logItem: ILogItem, public readonly membersLogItem: ILogItem, @@ -75,6 +79,7 @@ class JoinedData { dispose() { this.localMedia.dispose(); this.logItem.finish(); + this.renewMembershipTimeout?.dispose(); } } @@ -96,7 +101,17 @@ export class GroupCall extends EventEmitter<{change: never}> { this._state = newCall ? GroupCallState.Fledgling : GroupCallState.Created; this._memberOptions = Object.assign({}, options, { confId: this.id, - emitUpdate: member => this._members.update(getMemberKey(member.userId, member.deviceId), member), + emitUpdate: member => { + if (!member.isExpired) { + this._members.update(getMemberKey(member.userId, member.deviceId), member); + } else if (!member.isConnected) { + // don't just kick out connected members, even if their timestamp expired + this._members.remove(getMemberKey(member.userId, member.deviceId)); + } + if (member.isExpired && member.isConnected) { + console.trace("was about to kick a connected but expired member"); + } + }, encryptDeviceMessage: (userId: string, deviceId: string, message: SignallingMessage, log) => { return this.options.encryptDeviceMessage(this.roomId, userId, deviceId, message, log); } @@ -156,7 +171,7 @@ export class GroupCall extends EventEmitter<{change: never}> { this._state = GroupCallState.Joining; this.emitChange(); await log.wrap("update member state", async log => { - const memberContent = await this._createJoinPayload(); + const memberContent = await this._createMemberPayload(true); log.set("payload", memberContent); // send m.call.member state event const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCallMember, this.options.ownUserId, memberContent, {log}); @@ -220,7 +235,9 @@ export class GroupCall extends EventEmitter<{change: never}> { } await joinedData.logItem.wrap("leave", async log => { try { - const memberContent = await this._leaveCallMemberContent(); + joinedData.renewMembershipTimeout?.dispose(); + joinedData.renewMembershipTimeout = undefined; + const memberContent = await this._createMemberPayload(false); // send m.call.member state event if (memberContent) { const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCallMember, this.options.ownUserId, memberContent, {log}); @@ -296,19 +313,32 @@ export class GroupCall extends EventEmitter<{change: never}> { /** @internal */ updateMembership(userId: string, roomMember: RoomMember, callMembership: CallMembership, syncLog: ILogItem) { syncLog.wrap({l: "update call membership", t: CALL_LOG_TYPE, id: this.id, userId}, log => { + const now = this.options.clock.now(); const devices = callMembership["m.devices"]; const previousDeviceIds = this.getDeviceIdsForUserId(userId); for (const device of devices) { const deviceId = device.device_id; const memberKey = getMemberKey(userId, deviceId); - log.wrap({l: "update device membership", id: memberKey, sessionId: device.session_id}, log => { - if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) { + if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) { + log.wrap("update own membership", log => { + // TODO: should we check if new device is expired? + if (this.hasJoined) { + this.joinedData!.logItem.refDetached(log); + this._setupRenewMembershipTimeout(device, log); + } if (this._state === GroupCallState.Joining) { - log.set("update_own", true); + log.set("joined", true); this._state = GroupCallState.Joined; this.emitChange(); } - } else { + }); + } else { + log.wrap({l: "update device membership", id: memberKey, sessionId: device.session_id}, log => { + if (isMemberExpired(device, now)) { + log.set("expired", true); + this._members.remove(memberKey); + return; + } let member = this._members.get(memberKey); const sessionIdChanged = member && member.sessionId !== device.session_id; if (member && !sessionIdChanged) { @@ -328,6 +358,7 @@ export class GroupCall extends EventEmitter<{change: never}> { member = new Member( roomMember, device, this._memberOptions, + log ); this._members.add(memberKey, member); if (this.joinedData) { @@ -337,8 +368,8 @@ export class GroupCall extends EventEmitter<{change: never}> { // flush pending messages, either after having created the member, // or updated the session id with updateCallInfo this.flushPendingIncomingDeviceMessages(member, log); - } - }); + }); + } } const newDeviceIds = new Set(devices.map(call => call.device_id)); @@ -466,14 +497,14 @@ export class GroupCall extends EventEmitter<{change: never}> { } } - private async _createJoinPayload() { + private async _createMemberPayload(includeOwn: boolean): Promise { const {storage} = this.options; const txn = await storage.readTxn([storage.storeNames.roomState]); const stateEvent = await txn.roomState.get(this.roomId, EventType.GroupCallMember, this.options.ownUserId); - const stateContent = stateEvent?.event?.content ?? { + const stateContent: CallMemberContent = stateEvent?.event?.content as CallMemberContent ?? { ["m.calls"]: [] }; - const callsInfo = stateContent["m.calls"]; + let callsInfo = stateContent["m.calls"]; let callInfo = callsInfo.find(c => c["m.call_id"] === this.id); if (!callInfo) { callInfo = { @@ -482,32 +513,29 @@ export class GroupCall extends EventEmitter<{change: never}> { }; callsInfo.push(callInfo); } - callInfo["m.devices"] = callInfo["m.devices"].filter(d => d["device_id"] !== this.options.ownDeviceId); - callInfo["m.devices"].push({ - ["device_id"]: this.options.ownDeviceId, - ["session_id"]: this.options.sessionId, - feeds: [{purpose: "m.usermedia"}] - }); - return stateContent; - } - - private async _leaveCallMemberContent(): Promise | undefined> { - const {storage} = this.options; - const txn = await storage.readTxn([storage.storeNames.roomState]); - const stateEvent = await txn.roomState.get(this.roomId, EventType.GroupCallMember, this.options.ownUserId); - if (stateEvent) { - const content = stateEvent.event.content; - const callInfo = content["m.calls"]?.find(c => c["m.call_id"] === this.id); - if (callInfo) { - const devicesInfo = callInfo["m.devices"]; - const deviceIndex = devicesInfo.findIndex(d => d["device_id"] === this.options.ownDeviceId); - if (deviceIndex !== -1) { - devicesInfo.splice(deviceIndex, 1); - return content; - } + const now = this.options.clock.now(); + callInfo["m.devices"] = callInfo["m.devices"].filter(d => { + // remove our own device (to add it again below) + if (d["device_id"] === this.options.ownDeviceId) { + return false; } - + // also remove any expired devices (+ the validity period added again) + if (memberExpiresAt(d) === undefined || isMemberExpired(d, now, CALL_MEMBER_VALIDITY_PERIOD_MS)) { + return false; + } + return true; + }); + if (includeOwn) { + callInfo["m.devices"].push({ + ["device_id"]: this.options.ownDeviceId, + ["session_id"]: this.options.sessionId, + ["m.expires_ts"]: now + CALL_MEMBER_VALIDITY_PERIOD_MS, + feeds: [{purpose: "m.usermedia"}] + }); } + // filter out empty call membership + stateContent["m.calls"] = callsInfo.filter(c => c["m.devices"].length !== 0); + return stateContent; } private connectToMember(member: Member, joinedData: JoinedData, log: ILogItem) { @@ -524,11 +552,52 @@ export class GroupCall extends EventEmitter<{change: never}> { if (connectItem) { log.refDetached(connectItem); } - }) + }); } protected emitChange() { this.emit("change"); this.options.emitUpdate(this); } + + private _setupRenewMembershipTimeout(callDeviceMembership: CallDeviceMembership, log: ILogItem) { + const {joinedData} = this; + if (!joinedData) { + return; + } + joinedData.renewMembershipTimeout?.dispose(); + joinedData.renewMembershipTimeout = undefined; + const expiresAt = memberExpiresAt(callDeviceMembership); + if (typeof expiresAt !== "number") { + return; + } + const expiresFromNow = expiresAt - this.options.clock.now(); + // renew 1 to 5 minutes (8.3% of 1h) before expiring + // do it a bit beforehand and somewhat random to not collide with + // other clients trying to renew as well + const timeToRenewBeforeExpiration = Math.max(4000, Math.ceil((0.2 +(this.options.random() * 0.8)) * (0.08333 * CALL_MEMBER_VALIDITY_PERIOD_MS))); + const renewFromNow = Math.max(0, expiresFromNow - timeToRenewBeforeExpiration); + log.set("expiresIn", expiresFromNow); + log.set("renewIn", renewFromNow); + joinedData.renewMembershipTimeout = this.options.clock.createTimeout(renewFromNow); + joinedData.renewMembershipTimeout.elapsed().then( + () => { + joinedData.logItem.wrap("renew membership", async log => { + const memberContent = await this._createMemberPayload(true); + log.set("payload", memberContent); + // send m.call.member state event + const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCallMember, this.options.ownUserId, memberContent, {log}); + await request.response(); + }); + }, + () => { /* assume we're swallowing AbortError from dispose above */ } + ); + } + + dispose() { + this.joinedData?.dispose(); + for (const member of this._members.values()) { + member.dispose(); + } + } } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index b1c7b430..8e266fac 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -30,6 +30,7 @@ import type {RoomMember} from "../../room/members/RoomMember"; import type {EncryptedMessage} from "../../e2ee/olm/Encryption"; import type {ILogItem} from "../../../logging/types"; import type {BaseObservableValue} from "../../../observable/value/BaseObservableValue"; +import type {Clock, Timeout} from "../../../platform/web/dom/Clock"; export type Options = Omit & { confId: string, @@ -40,6 +41,7 @@ export type Options = Omit, log: ILogItem) => Promise, emitUpdate: (participant: Member, params?: any) => void, + clock: Clock } const errorCodesWithoutRetry = [ @@ -69,12 +71,33 @@ class MemberConnection { export class Member { private connection?: MemberConnection; + private expireTimeout?: Timeout; constructor( public member: RoomMember, private callDeviceMembership: CallDeviceMembership, private readonly options: Options, - ) {} + updateMemberLog: ILogItem + ) { + this._renewExpireTimeout(updateMemberLog); + } + + private _renewExpireTimeout(log: ILogItem) { + this.expireTimeout?.dispose(); + this.expireTimeout = undefined; + const expiresAt = memberExpiresAt(this.callDeviceMembership); + if (typeof expiresAt !== "number") { + return; + } + const expiresFromNow = Math.max(0, expiresAt - this.options.clock.now()); + log?.set("expiresIn", expiresFromNow); + // add 10ms to make sure isExpired returns true + this.expireTimeout = this.options.clock.createTimeout(expiresFromNow + 10); + this.expireTimeout.elapsed().then( + () => { this.options.emitUpdate(this, "isExpired"); }, + (err) => { /* ignore abort error */ }, + ); + } /** * Gives access the log item for this item once joined to the group call. @@ -89,6 +112,11 @@ export class Member { return this.connection?.peerCall?.remoteMedia; } + get isExpired(): boolean { + // never consider a peer we're connected to, to be expired + return !this.isConnected && isMemberExpired(this.callDeviceMembership, this.options.clock.now()); + } + get remoteMuteSettings(): MuteSettings | undefined { return this.connection?.peerCall?.remoteMuteSettings; } @@ -184,6 +212,7 @@ export class Member { /** @internal */ updateCallInfo(callDeviceMembership: CallDeviceMembership, causeItem: ILogItem) { this.callDeviceMembership = callDeviceMembership; + this._renewExpireTimeout(causeItem); if (this.connection) { this.connection.logItem.refDetached(causeItem); } @@ -352,4 +381,21 @@ export class Member { turnServer: connection.turnServer }), connection.logItem); } + + dispose() { + this.expireTimeout?.dispose(); + this.connection?.peerCall?.dispose(); + } +} + +export function memberExpiresAt(callDeviceMembership: CallDeviceMembership): number | undefined { + const expiresAt = callDeviceMembership["m.expires_ts"]; + if (Number.isSafeInteger(expiresAt)) { + return expiresAt; + } +} + +export function isMemberExpired(callDeviceMembership: CallDeviceMembership, now: number, margin: number = 0) { + const expiresAt = memberExpiresAt(callDeviceMembership); + return typeof expiresAt === "number" && ((expiresAt + margin) <= now); } diff --git a/src/matrix/e2ee/DeviceTracker.js b/src/matrix/e2ee/DeviceTracker.js index 0d0b0e8b..e0eea4fc 100644 --- a/src/matrix/e2ee/DeviceTracker.js +++ b/src/matrix/e2ee/DeviceTracker.js @@ -337,7 +337,7 @@ export class DeviceTracker { // verify signature const verifiedKeysPerUser = log.wrap("verify", log => this._filterVerifiedDeviceKeys(deviceKeyResponse["device_keys"], log)); //// END EXTRACT - + // TODO: what if verifiedKeysPerUser is empty or does not contain userId? const verifiedKeys = verifiedKeysPerUser .find(vkpu => vkpu.userId === userId).verifiedKeys .find(vk => vk["device_id"] === deviceId); From 05bb360c004f89ae48908e9f8497903df7c45cb7 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 30 Sep 2022 11:12:04 +0200 Subject: [PATCH 190/323] allow to dequeue signalling messages with repeated seq (from other call) they will just be ignored by peerCall.getMessageAction() but we don't want to block dequeuing on these --- src/matrix/calls/group/Member.ts | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 8e266fac..a21de30f 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -67,6 +67,22 @@ class MemberConnection { public turnServer: BaseObservableValue, public readonly logItem: ILogItem ) {} + + get canDequeueNextSignallingMessage() { + if (this.queuedSignallingMessages.length === 0) { + return false; + } + if (this.lastProcessedSeqNr === undefined) { + return true; + } + const first = this.queuedSignallingMessages[0]; + // allow messages with both a seq we've just seen and + // the next one to be dequeued as it can happen + // that messages for other callIds (which could repeat seq) + // are present in the queue + return first.content.seq === this.lastProcessedSeqNr || + first.content.seq === this.lastProcessedSeqNr + 1; + } } export class Member { @@ -335,12 +351,7 @@ export class Member { private dequeueSignallingMessages(connection: MemberConnection, peerCall: PeerCall, newMessage: SignallingMessage, syncLog: ILogItem): boolean { let hasNewMessageBeenDequeued = false; - while ( - connection.queuedSignallingMessages.length && ( - connection.lastProcessedSeqNr === undefined || - connection.queuedSignallingMessages[0].content.seq === connection.lastProcessedSeqNr + 1 - ) - ) { + while (connection.canDequeueNextSignallingMessage) { const message = connection.queuedSignallingMessages.shift()!; if (message === newMessage) { hasNewMessageBeenDequeued = true; From 8f8a964b3bc0159f98991f45e6972f90414b45fb Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 30 Sep 2022 11:15:48 +0200 Subject: [PATCH 191/323] allow to actually process the invite in the new peer call --- src/matrix/calls/group/Member.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index a21de30f..d4cc93f4 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -325,6 +325,7 @@ export class Member { } if (shouldReplace) { connection.peerCall = undefined; + action = IncomingMessageAction.Handle; } } } From 167a19a85babc34845700eb19caf687ae19666d3 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 30 Sep 2022 11:16:09 +0200 Subject: [PATCH 192/323] untracking from a disposed Disposables is actually not alarming it happens under normal conditions when an abortable action is aborted and you untrack afterwards. --- src/utils/Disposables.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/Disposables.ts b/src/utils/Disposables.ts index f7c7eb53..f27846be 100644 --- a/src/utils/Disposables.ts +++ b/src/utils/Disposables.ts @@ -50,8 +50,8 @@ export class Disposables { } untrack(disposable: Disposable): undefined { - if (this.isDisposed) { - console.warn("Disposables already disposed, cannot untrack"); + // already disposed + if (!this._disposables) { return undefined; } const idx = this._disposables!.indexOf(disposable); From 282cba0ff16016fe75a10c257e1a5310460dc8ec Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 30 Sep 2022 11:17:07 +0200 Subject: [PATCH 193/323] call points for PeerCall.delay rely on AbortError actually being thrown + fix this in one call point where it wasn't the case --- src/matrix/calls/PeerCall.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index d7d815fa..a64a1524 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -771,7 +771,8 @@ export class PeerCall implements IDisposable { const {flushCandidatesLog} = this; // MSC2746 recommends these values (can be quite long when calling because the // callee will need a while to answer the call) - await this.delay(this.direction === CallDirection.Inbound ? 500 : 2000); + try { await this.delay(this.direction === CallDirection.Inbound ? 500 : 2000); } + catch (err) { return; } this.sendCandidateQueue(flushCandidatesLog); this.flushCandidatesLog = undefined; } @@ -1100,12 +1101,9 @@ export class PeerCall implements IDisposable { const timeout = this.disposables.track(this.options.createTimeout(timeoutMs)); try { await timeout.elapsed(); - } catch (err) { - if (err.name !== "AbortError") { - throw err; - } + } finally { + this.disposables.untrack(timeout); } - this.disposables.untrack(timeout); } private sendSignallingMessage(message: SignallingMessage, log: ILogItem) { From b5f686b10e052586bbeef935e10d9280301bdeb1 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 30 Sep 2022 11:17:54 +0200 Subject: [PATCH 194/323] also allow clean up while still in Joining state, otherwise we get stuck in joined state but without joinedData --- src/matrix/calls/group/GroupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index cb027be8..0c9d3e90 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -437,7 +437,7 @@ export class GroupCall extends EventEmitter<{change: never}> { /** @internal */ disconnect(log: ILogItem) { - if (this._state === GroupCallState.Joined) { + if (this.hasJoined) { for (const [,member] of this._members) { const disconnectLogItem = member.disconnect(true); if (disconnectLogItem) { From f2564ed5cc4ad9965d6c4670c56def57ba13a91d Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 30 Sep 2022 11:20:00 +0200 Subject: [PATCH 195/323] also emit updates for expired, connected members that we didn't kick --- src/matrix/calls/group/GroupCall.ts | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 0c9d3e90..db908004 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -102,14 +102,15 @@ export class GroupCall extends EventEmitter<{change: never}> { this._memberOptions = Object.assign({}, options, { confId: this.id, emitUpdate: member => { - if (!member.isExpired) { - this._members.update(getMemberKey(member.userId, member.deviceId), member); - } else if (!member.isConnected) { - // don't just kick out connected members, even if their timestamp expired - this._members.remove(getMemberKey(member.userId, member.deviceId)); - } - if (member.isExpired && member.isConnected) { - console.trace("was about to kick a connected but expired member"); + const memberKey = getMemberKey(member.userId, member.deviceId); + // only remove expired members from the call if we don't have a peer conn with them + if (member.isExpired && !member.isConnected) { + this._members.remove(memberKey); + } else { + if (member.isExpired && member.isConnected) { + console.trace("was about to kick a connected but expired member"); + } + this._members.update(memberKey, member); } }, encryptDeviceMessage: (userId: string, deviceId: string, message: SignallingMessage, log) => { From b694d13348e4f23141c49fa3a18b470a4893f834 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 30 Sep 2022 11:20:33 +0200 Subject: [PATCH 196/323] can be readonly --- src/matrix/calls/group/GroupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index db908004..729c152e 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -73,7 +73,7 @@ class JoinedData { public readonly membersLogItem: ILogItem, public localMedia: LocalMedia, public localMuteSettings: MuteSettings, - public turnServer: BaseObservableValue + public readonly turnServer: BaseObservableValue ) {} dispose() { From 1dddabc038b8a39e7d2c0a9f18928c1e67ae8884 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 30 Sep 2022 11:21:26 +0200 Subject: [PATCH 197/323] make removing own membership a bit clearer in the logs --- src/matrix/calls/group/GroupCall.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 729c152e..60099c03 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -432,8 +432,9 @@ export class GroupCall extends EventEmitter<{change: never}> { } private removeOwnDevice(log: ILogItem) { - log.set("leave_own", true); - this.disconnect(log); + log.wrap("remove own membership", log => { + this.disconnect(log); + }); } /** @internal */ From 56ecd39f267de4fb770e9ecae2ca3f307ef0ee95 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 30 Sep 2022 17:46:57 +0200 Subject: [PATCH 198/323] don't assume joinedData is set here although not entirely sure why it wouldn't be --- src/matrix/calls/group/GroupCall.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 60099c03..ac41a772 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -324,7 +324,9 @@ export class GroupCall extends EventEmitter<{change: never}> { log.wrap("update own membership", log => { // TODO: should we check if new device is expired? if (this.hasJoined) { - this.joinedData!.logItem.refDetached(log); + if (this.joinedData) { + this.joinedData.logItem.refDetached(log); + } this._setupRenewMembershipTimeout(device, log); } if (this._state === GroupCallState.Joining) { From 7eb8015ace99a3ce02455ade28af2eab91703544 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 6 Oct 2022 15:20:05 +0200 Subject: [PATCH 199/323] ensure member.dispose is called when removing member so expiration timer is always stopped --- src/matrix/calls/group/GroupCall.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index ac41a772..c787b00a 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -105,6 +105,7 @@ export class GroupCall extends EventEmitter<{change: never}> { const memberKey = getMemberKey(member.userId, member.deviceId); // only remove expired members from the call if we don't have a peer conn with them if (member.isExpired && !member.isConnected) { + member.dispose(); this._members.remove(memberKey); } else { if (member.isExpired && member.isConnected) { @@ -339,6 +340,7 @@ export class GroupCall extends EventEmitter<{change: never}> { log.wrap({l: "update device membership", id: memberKey, sessionId: device.session_id}, log => { if (isMemberExpired(device, now)) { log.set("expired", true); + this._members.get(memberKey)?.dispose(); this._members.remove(memberKey); return; } @@ -354,6 +356,7 @@ export class GroupCall extends EventEmitter<{change: never}> { if (disconnectLogItem) { log.refDetached(disconnectLogItem); } + member.dispose(); this._members.remove(memberKey); member = undefined; } @@ -462,11 +465,12 @@ export class GroupCall extends EventEmitter<{change: never}> { const member = this._members.get(memberKey); if (member) { log.set("leave", true); - this._members.remove(memberKey); const disconnectLogItem = member.disconnect(false); if (disconnectLogItem) { log.refDetached(disconnectLogItem); } + member.dispose(); + this._members.remove(memberKey); } this.emitChange(); }); From 2ecfb8f13911cd1ec35fcb92ba1227ecf5162d40 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 6 Oct 2022 15:22:15 +0200 Subject: [PATCH 200/323] unify dispose logic in Member --- src/matrix/calls/group/Member.ts | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index d4cc93f4..5f6325b5 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -83,6 +83,12 @@ class MemberConnection { return first.content.seq === this.lastProcessedSeqNr || first.content.seq === this.lastProcessedSeqNr + 1; } + + dispose() { + this.peerCall?.dispose(); + this.localMedia.dispose(); + this.logItem.finish(); + } } export class Member { @@ -210,18 +216,15 @@ export class Member { return; } let disconnectLogItem; + // if if not sending the hangup, still log disconnect connection.logItem.wrap("disconnect", async log => { disconnectLogItem = log; - if (hangup) { - await connection.peerCall?.hangup(CallErrorCode.UserHangup, log); - } else { - await connection.peerCall?.close(undefined, log); + if (hangup && connection.peerCall) { + await connection.peerCall.hangup(CallErrorCode.UserHangup, log); } - connection.peerCall?.dispose(); - connection.localMedia?.dispose(); - this.connection = undefined; }); - connection.logItem.finish(); + connection.dispose(); + this.connection = undefined; return disconnectLogItem; } @@ -395,8 +398,10 @@ export class Member { } dispose() { + this.connection?.dispose(); + this.connection = undefined; this.expireTimeout?.dispose(); - this.connection?.peerCall?.dispose(); + this.expireTimeout = undefined; } } From c87fd4dfed64346449876faacf48f8fecaf25109 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 6 Oct 2022 15:59:45 +0200 Subject: [PATCH 201/323] cleanup --- src/matrix/calls/group/GroupCall.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index c787b00a..64249176 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -548,8 +548,11 @@ export class GroupCall extends EventEmitter<{change: never}> { private connectToMember(member: Member, joinedData: JoinedData, log: ILogItem) { const memberKey = getMemberKey(member.userId, member.deviceId); - const logItem = joinedData.membersLogItem.child({l: "member", id: memberKey}); - logItem.set("sessionId", member.sessionId); + const logItem = joinedData.membersLogItem.child({ + l: "member", + id: memberKey, + sessionId: member.sessionId + }); log.wrap({l: "connect", id: memberKey}, log => { const connectItem = member.connect( joinedData.localMedia, From 7ae9c3af0d7cec33cb82b8a9a87ad82d8220a985 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 6 Oct 2022 16:00:38 +0200 Subject: [PATCH 202/323] renew at least 10s before own membership expires --- src/matrix/calls/group/GroupCall.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 64249176..bdb3844b 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -583,10 +583,10 @@ export class GroupCall extends EventEmitter<{change: never}> { return; } const expiresFromNow = expiresAt - this.options.clock.now(); - // renew 1 to 5 minutes (8.3% of 1h) before expiring + // renew 1 to 5 minutes (8.3% of 1h, but min 10s) before expiring // do it a bit beforehand and somewhat random to not collide with // other clients trying to renew as well - const timeToRenewBeforeExpiration = Math.max(4000, Math.ceil((0.2 +(this.options.random() * 0.8)) * (0.08333 * CALL_MEMBER_VALIDITY_PERIOD_MS))); + const timeToRenewBeforeExpiration = Math.max(10000, Math.ceil((0.2 +(this.options.random() * 0.8)) * (0.08333 * CALL_MEMBER_VALIDITY_PERIOD_MS))); const renewFromNow = Math.max(0, expiresFromNow - timeToRenewBeforeExpiration); log.set("expiresIn", expiresFromNow); log.set("renewIn", renewFromNow); From a8ac504efdc0966badb670fd4186f601d8f7d957 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 6 Oct 2022 16:10:36 +0200 Subject: [PATCH 203/323] explain when joinedData is set --- src/matrix/calls/group/GroupCall.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index bdb3844b..c90c7d2c 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -88,6 +88,7 @@ export class GroupCall extends EventEmitter<{change: never}> { private _memberOptions: MemberOptions; private _state: GroupCallState; private bufferedDeviceMessages = new Map>>(); + /** Set between calling join and leave. */ private joinedData?: JoinedData; constructor( From a07be730f94bc290df5b65af345e33f73ee05e2f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 6 Oct 2022 16:10:50 +0200 Subject: [PATCH 204/323] bring member validity period back to 1h --- src/matrix/calls/common.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/matrix/calls/common.ts b/src/matrix/calls/common.ts index dde55ddc..015e0df8 100644 --- a/src/matrix/calls/common.ts +++ b/src/matrix/calls/common.ts @@ -59,5 +59,4 @@ export class MuteSettings { } export const CALL_LOG_TYPE = "call"; -//export const CALL_MEMBER_VALIDITY_PERIOD_MS = 3600 * 1000; // 1h -export const CALL_MEMBER_VALIDITY_PERIOD_MS = 60 * 1000; +export const CALL_MEMBER_VALIDITY_PERIOD_MS = 3600 * 1000; // 1h From 4a36fd96e464727c06a1c45ed64a52cf1d5f610c Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 6 Oct 2022 16:11:16 +0200 Subject: [PATCH 205/323] update bundled logviewer --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index e3f82fa1..f470bd9f 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ }, "homepage": "https://github.com/vector-im/hydrogen-web/#readme", "devDependencies": { - "@matrixdotorg/structured-logviewer": "^0.0.1", + "@matrixdotorg/structured-logviewer": "^0.0.3", "@typescript-eslint/eslint-plugin": "^4.29.2", "@typescript-eslint/parser": "^4.29.2", "acorn": "^8.6.0", diff --git a/yarn.lock b/yarn.lock index 0408a6e0..9b9e5453 100644 --- a/yarn.lock +++ b/yarn.lock @@ -56,10 +56,10 @@ version "3.2.8" resolved "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.8.tgz#8d53636d045e1776e2a2ec6613e57330dd9ce856" -"@matrixdotorg/structured-logviewer@^0.0.1": - version "0.0.1" - resolved "https://registry.yarnpkg.com/@matrixdotorg/structured-logviewer/-/structured-logviewer-0.0.1.tgz#9c29470b552f874afbb1df16c6e8e9e0c55cbf59" - integrity sha512-IdPYxAFDEoEs2G1ImKCkCxFI3xF1DDctP3N9JOtHRvIPbPPdTT9DyNqKTewCb5zwjNB1mGBrnWyURnHDiOOL3w== +"@matrixdotorg/structured-logviewer@^0.0.3": + version "0.0.3" + resolved "https://registry.yarnpkg.com/@matrixdotorg/structured-logviewer/-/structured-logviewer-0.0.3.tgz#1555111159d83cde0cfd5ba1a571e1faa1a90871" + integrity sha512-QqFglx0M8ix0IoRsJXDg1If26ltbYfuLjJ0MQrJYze3yz4ayEESRpQEA0YxJRVVtbco5M94tmrDpikokTFnn3A== "@nodelib/fs.scandir@2.1.5": version "2.1.5" From 400df6a4fff84ac8fa0f7460541abcd1659e62ce Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 7 Oct 2022 10:07:10 +0200 Subject: [PATCH 206/323] ensure removing expired members from call is always logged also return the log item from logger.log so we can ref it --- src/logging/Logger.ts | 3 ++- src/logging/types.ts | 2 +- src/matrix/calls/group/GroupCall.ts | 11 +++++++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/logging/Logger.ts b/src/logging/Logger.ts index 37a3326c..d3aa1d6d 100644 --- a/src/logging/Logger.ts +++ b/src/logging/Logger.ts @@ -30,10 +30,11 @@ export class Logger implements ILogger { this._platform = platform; } - log(labelOrValues: LabelOrValues, logLevel: LogLevel = LogLevel.Info): void { + log(labelOrValues: LabelOrValues, logLevel: LogLevel = LogLevel.Info): ILogItem { const item = new LogItem(labelOrValues, logLevel, this); item.end = item.start; this._persistItem(item, undefined, false); + return item; } /** Prefer `run()` or `log()` above this method; only use it if you have a long-running operation diff --git a/src/logging/types.ts b/src/logging/types.ts index 0ed3b0dc..0f4f6150 100644 --- a/src/logging/types.ts +++ b/src/logging/types.ts @@ -68,7 +68,7 @@ export interface ILogItemCreator { */ export interface ILogger { - log(labelOrValues: LabelOrValues, logLevel?: LogLevel): void; + log(labelOrValues: LabelOrValues, logLevel?: LogLevel): ILogItem; child(labelOrValues: LabelOrValues, logLevel?: LogLevel, filterCreator?: FilterCreator): ILogItem; wrapOrRun(item: ILogItem | undefined, labelOrValues: LabelOrValues, callback: LogCallback, logLevel?: LogLevel, filterCreator?: FilterCreator): T; runDetached(labelOrValues: LabelOrValues, callback: LogCallback, logLevel?: LogLevel, filterCreator?: FilterCreator): ILogItem; diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index c90c7d2c..a8b6605f 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -104,14 +104,17 @@ export class GroupCall extends EventEmitter<{change: never}> { confId: this.id, emitUpdate: member => { const memberKey = getMemberKey(member.userId, member.deviceId); - // only remove expired members from the call if we don't have a peer conn with them + // only remove expired members to whom we're not already connected if (member.isExpired && !member.isConnected) { + const logItem = this.options.logger.log({ + l: "removing expired member from call", + memberKey, + callId: this.id + }) + member.logItem?.refDetached(logItem); member.dispose(); this._members.remove(memberKey); } else { - if (member.isExpired && member.isConnected) { - console.trace("was about to kick a connected but expired member"); - } this._members.update(memberKey, member); } }, From 793686393414006c98f6a7dd73e5c092f2200362 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 7 Oct 2022 10:07:46 +0200 Subject: [PATCH 207/323] remove options on dispose in Member to ensure callback can't be called anymore, as we don't check that the member argument is the one currently in GroupCall._members. --- src/matrix/calls/group/Member.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 5f6325b5..8763bf5e 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -98,7 +98,7 @@ export class Member { constructor( public member: RoomMember, private callDeviceMembership: CallDeviceMembership, - private readonly options: Options, + private options: Options, updateMemberLog: ILogItem ) { this._renewExpireTimeout(updateMemberLog); @@ -402,6 +402,8 @@ export class Member { this.connection = undefined; this.expireTimeout?.dispose(); this.expireTimeout = undefined; + // ensure the emitUpdate callback can't be called anymore + this.options = undefined as any as Options; } } From 4350537004c3e9af6d2a6c633a1bbd440d321fd7 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 7 Oct 2022 10:58:12 +0200 Subject: [PATCH 208/323] cleanup --- src/matrix/calls/group/GroupCall.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index a8b6605f..84fb9be9 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -344,8 +344,12 @@ export class GroupCall extends EventEmitter<{change: never}> { log.wrap({l: "update device membership", id: memberKey, sessionId: device.session_id}, log => { if (isMemberExpired(device, now)) { log.set("expired", true); - this._members.get(memberKey)?.dispose(); - this._members.remove(memberKey); + const member = this._members.get(memberKey); + if (member) { + member.dispose(); + this._members.remove(memberKey); + log.set("removed", true); + } return; } let member = this._members.get(memberKey); From 7d10bec4acb65c8fd2c696a0994a0f98c6467982 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 7 Oct 2022 10:59:13 +0200 Subject: [PATCH 209/323] remove comment not relevant anymore --- src/matrix/calls/group/GroupCall.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 84fb9be9..f2561f32 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -327,7 +327,6 @@ export class GroupCall extends EventEmitter<{change: never}> { const memberKey = getMemberKey(userId, deviceId); if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) { log.wrap("update own membership", log => { - // TODO: should we check if new device is expired? if (this.hasJoined) { if (this.joinedData) { this.joinedData.logItem.refDetached(log); From e9b8cfbd9e328437f2484d0a384a4a76b6e2493f Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 18 Oct 2022 12:56:58 -0700 Subject: [PATCH 210/323] Member should be expired if there's no expires at key --- src/matrix/calls/group/Member.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 8763bf5e..0aba3b54 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -416,5 +416,5 @@ export function memberExpiresAt(callDeviceMembership: CallDeviceMembership): num export function isMemberExpired(callDeviceMembership: CallDeviceMembership, now: number, margin: number = 0) { const expiresAt = memberExpiresAt(callDeviceMembership); - return typeof expiresAt === "number" && ((expiresAt + margin) <= now); + return typeof expiresAt === "number" ? ((expiresAt + margin) <= now) : true; } From c0306b45a606023793aaa869a175e7661bad2824 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 18 Oct 2022 13:43:19 -0700 Subject: [PATCH 211/323] Fix typechecking --- src/logging/NullLogger.ts | 4 +++- src/platform/web/ui/session/room/CallView.ts | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/logging/NullLogger.ts b/src/logging/NullLogger.ts index f6f877b3..bf7b3f40 100644 --- a/src/logging/NullLogger.ts +++ b/src/logging/NullLogger.ts @@ -21,7 +21,9 @@ function noop (): void {} export class NullLogger implements ILogger { public readonly item: ILogItem = new NullLogItem(this); - log(): void {} + log(): ILogItem { + return this.item; + } addReporter() {} diff --git a/src/platform/web/ui/session/room/CallView.ts b/src/platform/web/ui/session/room/CallView.ts index 6961dc53..619afc2e 100644 --- a/src/platform/web/ui/session/room/CallView.ts +++ b/src/platform/web/ui/session/room/CallView.ts @@ -80,7 +80,7 @@ export class CallView extends TemplateView { public unmount() { if (this.resizeObserver) { - this.resizeObserver.unobserve((this.root()! as Element).querySelector(".CallView_members")); + this.resizeObserver.unobserve((this.root()! as Element).querySelector(".CallView_members")!); this.resizeObserver = undefined; } super.unmount(); From 32835e26b9b2435dc7db32d76a82cf698ee11216 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 10 Nov 2022 14:08:34 +0100 Subject: [PATCH 212/323] adjust query for finding all state events of a type we were looking for state events with state key between \0 and \u10FFFF but an empty state key is also valid and would come before \0, so allow empty state keys at the beginning of the range and include them in the result by opening the lower bound. --- src/matrix/storage/idb/stores/RoomStateStore.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/matrix/storage/idb/stores/RoomStateStore.ts b/src/matrix/storage/idb/stores/RoomStateStore.ts index 99315e9e..e7661378 100644 --- a/src/matrix/storage/idb/stores/RoomStateStore.ts +++ b/src/matrix/storage/idb/stores/RoomStateStore.ts @@ -43,9 +43,9 @@ export class RoomStateStore { getAllForType(roomId: string, type: string): Promise { const range = this._roomStateStore.IDBKeyRange.bound( - encodeKey(roomId, type, MIN_UNICODE), + encodeKey(roomId, type, ""), encodeKey(roomId, type, MAX_UNICODE), - true, + false, true ); return this._roomStateStore.selectAll(range); From e26eb30b82fb04e36d2cf8e510ee2ae02dda1efb Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 5 Jan 2023 15:47:01 +0100 Subject: [PATCH 213/323] rename m.expires_ts to expires_ts to be compatible with MSC/Element --- src/matrix/calls/callEventTypes.ts | 2 +- src/matrix/calls/group/GroupCall.ts | 2 +- src/matrix/calls/group/Member.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index ca5c870c..8fd7b23d 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -25,7 +25,7 @@ export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata"; export interface CallDeviceMembership { device_id: string, session_id: string, - ["m.expires_ts"]?: number, + ["expires_ts"]?: number, feeds?: Array<{purpose: string}> } diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index f2561f32..0931d4ea 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -544,7 +544,7 @@ export class GroupCall extends EventEmitter<{change: never}> { callInfo["m.devices"].push({ ["device_id"]: this.options.ownDeviceId, ["session_id"]: this.options.sessionId, - ["m.expires_ts"]: now + CALL_MEMBER_VALIDITY_PERIOD_MS, + ["expires_ts"]: now + CALL_MEMBER_VALIDITY_PERIOD_MS, feeds: [{purpose: "m.usermedia"}] }); } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 0aba3b54..9debef22 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -408,7 +408,7 @@ export class Member { } export function memberExpiresAt(callDeviceMembership: CallDeviceMembership): number | undefined { - const expiresAt = callDeviceMembership["m.expires_ts"]; + const expiresAt = callDeviceMembership["expires_ts"]; if (Number.isSafeInteger(expiresAt)) { return expiresAt; } From fcb4f2a62dd1569eb1bf8c4c3e44358cbe08f8a3 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 6 Jan 2023 12:05:11 +0100 Subject: [PATCH 214/323] fix microphone audio being played back through local video preview --- src/domain/session/room/CallViewModel.ts | 2 +- src/matrix/calls/LocalMedia.ts | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 020c9f17..9c0eae9a 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -136,7 +136,7 @@ class OwnMemberViewModel extends ViewModel implements IStreamViewModel } get stream(): Stream | undefined { - return this.call.localMedia?.userMedia; + return this.call.localMedia?.userMediaPreview; } private get call(): GroupCall { diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index f02cd11b..25d6862f 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -20,11 +20,24 @@ import {SDPStreamMetadata} from "./callEventTypes"; import {getStreamVideoTrack, getStreamAudioTrack} from "./common"; export class LocalMedia { + // the userMedia stream without audio, to play in the UI + // without our own audio being played back to us + public readonly userMediaPreview?: Stream; + constructor( public readonly userMedia?: Stream, public readonly screenShare?: Stream, public readonly dataChannelOptions?: RTCDataChannelInit, - ) {} + ) { + if (userMedia && userMedia.getVideoTracks().length > 0) { + this.userMediaPreview = userMedia.clone(); + const audioTrack = getStreamAudioTrack(this.userMediaPreview); + if (audioTrack) { + audioTrack.stop(); + this.userMediaPreview.removeTrack(audioTrack); + } + } + } withUserMedia(stream: Stream) { return new LocalMedia(stream, this.screenShare, this.dataChannelOptions); @@ -63,6 +76,7 @@ export class LocalMedia { dispose() { getStreamAudioTrack(this.userMedia)?.stop(); getStreamVideoTrack(this.userMedia)?.stop(); + getStreamVideoTrack(this.userMediaPreview)?.stop(); getStreamVideoTrack(this.screenShare)?.stop(); } } From cb0f803276106d608238aebeedc62256644d83fd Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 6 Jan 2023 12:05:34 +0100 Subject: [PATCH 215/323] make this code a bit clearer --- src/matrix/calls/group/GroupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 0931d4ea..d65ea838 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -216,8 +216,8 @@ export class GroupCall extends EventEmitter<{change: never}> { // you might be muted because you don't have a track or because // you actively chosen to mute // (which we want to respect in the future when you add a track) + muteSettings.updateTrackInfo(joinedData.localMedia.userMedia); joinedData.localMuteSettings = muteSettings; - joinedData.localMuteSettings.updateTrackInfo(joinedData.localMedia.userMedia); if (!prevMuteSettings.equals(muteSettings)) { await Promise.all(Array.from(this._members.values()).map(m => { return m.setMuted(joinedData.localMuteSettings); From 47a8b4d72805574f8d12c3de80b2ec298e56d71d Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 6 Jan 2023 17:01:57 +0100 Subject: [PATCH 216/323] WIP --- src/utils/ErrorBoundary.ts | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 src/utils/ErrorBoundary.ts diff --git a/src/utils/ErrorBoundary.ts b/src/utils/ErrorBoundary.ts new file mode 100644 index 00000000..e9f297c5 --- /dev/null +++ b/src/utils/ErrorBoundary.ts @@ -0,0 +1,38 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {ObservableValue, BaseObservableValue} from "../observable/ObservableValue"; + +export class ErrorBoundary { + constructor(private readonly errorCallback: (Error) => void) {} + + try(callback: () => T): T | undefined; + try(callback: () => Promise): Promise | undefined { + try { + let result: T | Promise = callback(); + if (result instanceof Promise) { + result = result.catch(err => { + this.errorCallback(err); + return undefined; + }); + } + return result; + } catch (err) { + this.errorCallback(err); + return undefined; + } + } +} From 50f46a21bd6c0583b70a08a8e5d95b40289ad71a Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 9 Jan 2023 10:36:28 +0100 Subject: [PATCH 217/323] finish ErrorBoundary --- src/utils/ErrorBoundary.ts | 56 ++++++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 6 deletions(-) diff --git a/src/utils/ErrorBoundary.ts b/src/utils/ErrorBoundary.ts index e9f297c5..520d2a35 100644 --- a/src/utils/ErrorBoundary.ts +++ b/src/utils/ErrorBoundary.ts @@ -14,25 +14,69 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {ObservableValue, BaseObservableValue} from "../observable/ObservableValue"; +export const ErrorValue = Symbol("ErrorBoundary:Error"); export class ErrorBoundary { + private _error?: Error; + constructor(private readonly errorCallback: (Error) => void) {} - try(callback: () => T): T | undefined; - try(callback: () => Promise): Promise | undefined { + /** + * Executes callback() and then runs errorCallback() on error. + * This will never throw but instead return `errorValue` if an error occured. + */ + try(callback: () => T): T | typeof ErrorValue; + try(callback: () => Promise): Promise | typeof ErrorValue { try { - let result: T | Promise = callback(); + let result: T | Promise = callback(); if (result instanceof Promise) { result = result.catch(err => { + this._error = err; this.errorCallback(err); - return undefined; + return ErrorValue; }); } return result; } catch (err) { + this._error = err; this.errorCallback(err); - return undefined; + return ErrorValue; } } + + get error(): Error | undefined { + return this._error; + } } + +export function tests() { + return { + "catches sync error": assert => { + let emitted = false; + const boundary = new ErrorBoundary(() => emitted = true); + const result = boundary.try(() => { + throw new Error("fail!"); + }); + assert(emitted); + assert.strictEqual(result, ErrorValue); + }, + "return value of callback is forwarded": assert => { + let emitted = false; + const boundary = new ErrorBoundary(() => emitted = true); + const result = boundary.try(() => { + return "hello"; + }); + assert(!emitted); + assert.strictEqual(result, "hello"); + }, + "catches async error": async assert => { + let emitted = false; + const boundary = new ErrorBoundary(() => emitted = true); + const result = await boundary.try(async () => { + throw new Error("fail!"); + }); + assert(emitted); + assert.strictEqual(result, ErrorValue); + } + } +} \ No newline at end of file From f6c94ecc5aa514d5d1d42557a36e97579daebb2a Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Mon, 16 Jan 2023 13:41:47 +0530 Subject: [PATCH 218/323] WIP --- src/domain/session/room/CallViewModel.ts | 2 +- src/matrix/calls/LocalMedia.ts | 31 +++++++++++++----------- src/matrix/calls/PeerCall.ts | 27 ++------------------- src/matrix/calls/common.ts | 25 +++++++++++++++++++ src/matrix/calls/group/GroupCall.ts | 15 +++++++++++- 5 files changed, 59 insertions(+), 41 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 9c0eae9a..37f30840 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -136,7 +136,7 @@ class OwnMemberViewModel extends ViewModel implements IStreamViewModel } get stream(): Stream | undefined { - return this.call.localMedia?.userMediaPreview; + return this.call.localPreviewMedia?.userMedia; } private get call(): GroupCall { diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 25d6862f..7a673ba9 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -20,24 +20,12 @@ import {SDPStreamMetadata} from "./callEventTypes"; import {getStreamVideoTrack, getStreamAudioTrack} from "./common"; export class LocalMedia { - // the userMedia stream without audio, to play in the UI - // without our own audio being played back to us - public readonly userMediaPreview?: Stream; constructor( public readonly userMedia?: Stream, public readonly screenShare?: Stream, public readonly dataChannelOptions?: RTCDataChannelInit, - ) { - if (userMedia && userMedia.getVideoTracks().length > 0) { - this.userMediaPreview = userMedia.clone(); - const audioTrack = getStreamAudioTrack(this.userMediaPreview); - if (audioTrack) { - audioTrack.stop(); - this.userMediaPreview.removeTrack(audioTrack); - } - } - } + ) {} withUserMedia(stream: Stream) { return new LocalMedia(stream, this.screenShare, this.dataChannelOptions); @@ -51,6 +39,22 @@ export class LocalMedia { return new LocalMedia(this.userMedia, this.screenShare, options); } + /** + * Create an instance of LocalMedia without audio track (for user preview) + */ + asPreview(): LocalMedia { + const media = new LocalMedia(this.userMedia, this.screenShare, this.dataChannelOptions); + const userMedia = media.userMedia; + if (userMedia && userMedia.getVideoTracks().length > 0) { + const audioTrack = getStreamAudioTrack(userMedia); + if (audioTrack) { + audioTrack.stop(); + userMedia.removeTrack(audioTrack); + } + } + return media; + } + /** @internal */ replaceClone(oldClone: LocalMedia | undefined, oldOriginal: LocalMedia | undefined): LocalMedia { const cloneOrAdoptStream = (oldOriginalStream: Stream | undefined, oldCloneStream: Stream | undefined, newStream: Stream | undefined): Stream | undefined => { @@ -76,7 +80,6 @@ export class LocalMedia { dispose() { getStreamAudioTrack(this.userMedia)?.stop(); getStreamVideoTrack(this.userMedia)?.stop(); - getStreamVideoTrack(this.userMediaPreview)?.stop(); getStreamVideoTrack(this.screenShare)?.stop(); } } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index a64a1524..ae200ea2 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -20,7 +20,7 @@ import {recursivelyAssign} from "../../utils/recursivelyAssign"; import {Disposables, Disposable, IDisposable} from "../../utils/Disposables"; import {WebRTC, PeerConnection, Transceiver, TransceiverDirection, Sender, Receiver, PeerConnectionEventMap} from "../../platform/types/WebRTC"; import {MediaDevices, Track, TrackKind, Stream, StreamTrackEvent} from "../../platform/types/MediaDevices"; -import {getStreamVideoTrack, getStreamAudioTrack, MuteSettings} from "./common"; +import {getStreamVideoTrack, getStreamAudioTrack, MuteSettings, mute} from "./common"; import { SDPStreamMetadataKey, SDPStreamMetadataPurpose, @@ -266,14 +266,7 @@ export class PeerCall implements IDisposable { log.set("microphoneMuted", localMuteSettings.microphone); if (this.localMedia) { - const userMediaAudio = getStreamAudioTrack(this.localMedia.userMedia); - if (userMediaAudio) { - this.muteTrack(userMediaAudio, this.localMuteSettings.microphone, log); - } - const userMediaVideo = getStreamVideoTrack(this.localMedia.userMedia); - if (userMediaVideo) { - this.muteTrack(userMediaVideo, this.localMuteSettings.camera, log); - } + mute(this.localMedia, localMuteSettings, log); const content: MCallSDPStreamMetadataChanged = { call_id: this.callId, version: 1, @@ -290,22 +283,6 @@ export class PeerCall implements IDisposable { }); } - private muteTrack(track: Track, muted: boolean, log: ILogItem): void { - log.wrap({l: "track", kind: track.kind, id: track.id}, log => { - const enabled = !muted; - log.set("enabled", enabled); - const transceiver = this.findTransceiverForTrack(track); - if (transceiver) { - if (transceiver.sender.track) { - transceiver.sender.track.enabled = enabled; - } - log.set("fromDirection", transceiver.direction); - // enableSenderOnTransceiver(transceiver, enabled); - log.set("toDirection", transceiver.direction); - } - }); - } - private async _hangup(errorCode: CallErrorCode, log: ILogItem): Promise { if (this._state === CallState.Ended) { return; diff --git a/src/matrix/calls/common.ts b/src/matrix/calls/common.ts index 015e0df8..b99ed303 100644 --- a/src/matrix/calls/common.ts +++ b/src/matrix/calls/common.ts @@ -14,7 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */ +import {ILogItem} from "../../logging/types"; import type {Track, Stream} from "../../platform/types/MediaDevices"; +import {LocalMedia} from "./LocalMedia"; export function getStreamAudioTrack(stream: Stream | undefined): Track | undefined { return stream?.getAudioTracks()[0]; @@ -24,6 +26,29 @@ export function getStreamVideoTrack(stream: Stream | undefined): Track | undefin return stream?.getVideoTracks()[0]; } +export function mute(localMedia: LocalMedia, localMuteSettings: MuteSettings, log: ILogItem) { + return log.wrap("setMuted", log => { + log.set("cameraMuted", localMuteSettings.camera); + log.set("microphoneMuted", localMuteSettings.microphone); + + // Mute audio + const userMediaAudio = getStreamAudioTrack(localMedia.userMedia); + if (userMediaAudio) { + const enabled = !localMuteSettings.microphone; + log.set("microphone enabled", enabled); + userMediaAudio.enabled = enabled; + } + + // Mute video + const userMediaVideo = getStreamVideoTrack(localMedia.userMedia); + if (userMediaVideo) { + const enabled = !localMuteSettings.camera; + log.set("camera enabled", enabled); + userMediaVideo.enabled = enabled; + } + }); +} + export class MuteSettings { constructor ( private readonly isMicrophoneMuted: boolean = false, diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index d65ea838..79f0e66c 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -17,7 +17,7 @@ limitations under the License. import {ObservableMap} from "../../../observable/map/ObservableMap"; import {Member, isMemberExpired, memberExpiresAt} from "./Member"; import {LocalMedia} from "../LocalMedia"; -import {MuteSettings, CALL_LOG_TYPE, CALL_MEMBER_VALIDITY_PERIOD_MS} from "../common"; +import {MuteSettings, CALL_LOG_TYPE, CALL_MEMBER_VALIDITY_PERIOD_MS, mute} from "../common"; import {MemberChange, RoomMember} from "../../room/members/RoomMember"; import {EventEmitter} from "../../../utils/EventEmitter"; import {EventType, CallIntent} from "../callEventTypes"; @@ -72,12 +72,14 @@ class JoinedData { public readonly logItem: ILogItem, public readonly membersLogItem: ILogItem, public localMedia: LocalMedia, + public localPreviewMedia: LocalMedia, public localMuteSettings: MuteSettings, public readonly turnServer: BaseObservableValue ) {} dispose() { this.localMedia.dispose(); + this.localPreviewMedia.dispose(); this.logItem.finish(); this.renewMembershipTimeout?.dispose(); } @@ -125,6 +127,7 @@ export class GroupCall extends EventEmitter<{change: never}> { } get localMedia(): LocalMedia | undefined { return this.joinedData?.localMedia; } + get localPreviewMedia(): LocalMedia | undefined { return this.joinedData?.localPreviewMedia; } get members(): BaseObservableMap { return this._members; } get isTerminated(): boolean { @@ -165,10 +168,12 @@ export class GroupCall extends EventEmitter<{change: never}> { const membersLogItem = logItem.child("member connections"); const localMuteSettings = new MuteSettings(); localMuteSettings.updateTrackInfo(localMedia.userMedia); + const localPreviewMedia = localMedia.asPreview(); const joinedData = new JoinedData( logItem, membersLogItem, localMedia, + localPreviewMedia, localMuteSettings, turnServer ); @@ -219,6 +224,14 @@ export class GroupCall extends EventEmitter<{change: never}> { muteSettings.updateTrackInfo(joinedData.localMedia.userMedia); joinedData.localMuteSettings = muteSettings; if (!prevMuteSettings.equals(muteSettings)) { + // Mute our copies of LocalMedias; + // otherwise the camera lights will still be on. + if (this.localPreviewMedia) { + mute(this.localPreviewMedia, muteSettings, this.joinedData!.logItem); + } + if (this.localMedia) { + mute(this.localMedia, muteSettings, this.joinedData!.logItem); + } await Promise.all(Array.from(this._members.values()).map(m => { return m.setMuted(joinedData.localMuteSettings); })); From c064336e359a2d271a30a6af3aa73e8eeb83c035 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Mon, 16 Jan 2023 15:28:50 +0530 Subject: [PATCH 219/323] Create localPreviewMedia in GroupCall.setMedia --- src/matrix/calls/group/GroupCall.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 79f0e66c..f3f37a2b 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -200,6 +200,8 @@ export class GroupCall extends EventEmitter<{change: never}> { if ((this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) && this.joinedData) { const oldMedia = this.joinedData.localMedia; this.joinedData.localMedia = localMedia; + this.localPreviewMedia?.dispose(); + this.joinedData.localPreviewMedia = localMedia.asPreview(); // reflect the fact we gained or lost local tracks in the local mute settings // and update the track info so PeerCall can use it to send up to date metadata, this.joinedData.localMuteSettings.updateTrackInfo(localMedia.userMedia); From ce5c5779bdf59f4ab13182c7c7b7b81b4d6af556 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 17 Jan 2023 14:24:20 +0530 Subject: [PATCH 220/323] Set mute -> mute --- src/matrix/calls/common.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/common.ts b/src/matrix/calls/common.ts index b99ed303..5036eae4 100644 --- a/src/matrix/calls/common.ts +++ b/src/matrix/calls/common.ts @@ -27,7 +27,7 @@ export function getStreamVideoTrack(stream: Stream | undefined): Track | undefin } export function mute(localMedia: LocalMedia, localMuteSettings: MuteSettings, log: ILogItem) { - return log.wrap("setMuted", log => { + return log.wrap("mute", log => { log.set("cameraMuted", localMuteSettings.camera); log.set("microphoneMuted", localMuteSettings.microphone); From 1cd6540476dfa80b9b6e53d7ed83213d4079fe64 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 17 Jan 2023 14:32:02 +0530 Subject: [PATCH 221/323] Don't use getter here --- src/matrix/calls/group/GroupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index f3f37a2b..cda5a40f 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -200,7 +200,7 @@ export class GroupCall extends EventEmitter<{change: never}> { if ((this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) && this.joinedData) { const oldMedia = this.joinedData.localMedia; this.joinedData.localMedia = localMedia; - this.localPreviewMedia?.dispose(); + this.joinedData.localPreviewMedia?.dispose(); this.joinedData.localPreviewMedia = localMedia.asPreview(); // reflect the fact we gained or lost local tracks in the local mute settings // and update the track info so PeerCall can use it to send up to date metadata, From f0d2c191849ed0154c3ac57418a624fb997d4d71 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 9 Jan 2023 13:50:03 +0100 Subject: [PATCH 222/323] allow an explicit error value again in ErrorBoundary --- src/utils/ErrorBoundary.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/utils/ErrorBoundary.ts b/src/utils/ErrorBoundary.ts index 520d2a35..d13065f9 100644 --- a/src/utils/ErrorBoundary.ts +++ b/src/utils/ErrorBoundary.ts @@ -25,22 +25,22 @@ export class ErrorBoundary { * Executes callback() and then runs errorCallback() on error. * This will never throw but instead return `errorValue` if an error occured. */ - try(callback: () => T): T | typeof ErrorValue; - try(callback: () => Promise): Promise | typeof ErrorValue { + try(callback: () => T, errorValue?: E): T | typeof errorValue; + try(callback: () => Promise, errorValue?: E): Promise | typeof errorValue { try { - let result: T | Promise = callback(); + let result: T | Promise = callback(); if (result instanceof Promise) { result = result.catch(err => { this._error = err; this.errorCallback(err); - return ErrorValue; + return errorValue; }); } return result; } catch (err) { this._error = err; this.errorCallback(err); - return ErrorValue; + return errorValue; } } @@ -56,9 +56,9 @@ export function tests() { const boundary = new ErrorBoundary(() => emitted = true); const result = boundary.try(() => { throw new Error("fail!"); - }); + }, 0); assert(emitted); - assert.strictEqual(result, ErrorValue); + assert.strictEqual(result, 0); }, "return value of callback is forwarded": assert => { let emitted = false; @@ -74,9 +74,9 @@ export function tests() { const boundary = new ErrorBoundary(() => emitted = true); const result = await boundary.try(async () => { throw new Error("fail!"); - }); + }, 0); assert(emitted); - assert.strictEqual(result, ErrorValue); + assert.strictEqual(result, 0); } } } \ No newline at end of file From b1687d71155ffaf50a4b1ed541c2bd57d7a16602 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 9 Jan 2023 13:50:27 +0100 Subject: [PATCH 223/323] introduce error boundary in call member --- src/matrix/calls/group/Member.ts | 185 +++++++++++++++++-------------- 1 file changed, 103 insertions(+), 82 deletions(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 9debef22..4ceaf46f 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -19,6 +19,7 @@ import {makeTxnId, makeId} from "../../common"; import {EventType, CallErrorCode} from "../callEventTypes"; import {formatToDeviceMessagesPayload} from "../../common"; import {sortedIndex} from "../../../utils/sortedIndex"; +import { ErrorBoundary } from "../../../utils/ErrorBoundary"; import type {MuteSettings} from "../common"; import type {Options as PeerCallOptions, RemoteMedia} from "../PeerCall"; @@ -94,6 +95,9 @@ class MemberConnection { export class Member { private connection?: MemberConnection; private expireTimeout?: Timeout; + private errorBoundary = new ErrorBoundary(err => { + this.options.emitUpdate(this, "error"); + }); constructor( public member: RoomMember, @@ -104,6 +108,10 @@ export class Member { this._renewExpireTimeout(updateMemberLog); } + get error(): Error | undefined { + return this.errorBoundary.error; + } + private _renewExpireTimeout(log: ILogItem) { this.expireTimeout?.dispose(); this.expireTimeout = undefined; @@ -166,23 +174,26 @@ export class Member { /** @internal */ connect(localMedia: LocalMedia, localMuteSettings: MuteSettings, turnServer: BaseObservableValue, memberLogItem: ILogItem): ILogItem | undefined { - if (this.connection) { - return; - } - // Safari can't send a MediaStream to multiple sources, so clone it - const connection = new MemberConnection( - localMedia.clone(), - localMuteSettings, - turnServer, - memberLogItem - ); - this.connection = connection; - let connectLogItem; - connection.logItem.wrap("connect", async log => { - connectLogItem = log; - await this.callIfNeeded(log); + return this.errorBoundary.try(() => { + if (this.connection) { + return; + } + // Safari can't send a MediaStream to multiple sources, so clone it + const connection = new MemberConnection( + localMedia.clone(), + localMuteSettings, + turnServer, + memberLogItem + ); + this.connection = connection; + let connectLogItem: ILogItem | undefined; + connection.logItem.wrap("connect", async log => { + connectLogItem = log; + await this.callIfNeeded(log); + }); + throw new Error("connect failed!"); + return connectLogItem; }); - return connectLogItem; } private callIfNeeded(log: ILogItem): Promise { @@ -211,30 +222,34 @@ export class Member { /** @internal */ disconnect(hangup: boolean): ILogItem | undefined { - const {connection} = this; - if (!connection) { - return; - } - let disconnectLogItem; - // if if not sending the hangup, still log disconnect - connection.logItem.wrap("disconnect", async log => { - disconnectLogItem = log; - if (hangup && connection.peerCall) { - await connection.peerCall.hangup(CallErrorCode.UserHangup, log); + return this.errorBoundary.try(() => { + const {connection} = this; + if (!connection) { + return; } + let disconnectLogItem; + // if if not sending the hangup, still log disconnect + connection.logItem.wrap("disconnect", async log => { + disconnectLogItem = log; + if (hangup && connection.peerCall) { + await connection.peerCall.hangup(CallErrorCode.UserHangup, log); + } + }); + connection.dispose(); + this.connection = undefined; + return disconnectLogItem; }); - connection.dispose(); - this.connection = undefined; - return disconnectLogItem; } /** @internal */ updateCallInfo(callDeviceMembership: CallDeviceMembership, causeItem: ILogItem) { - this.callDeviceMembership = callDeviceMembership; - this._renewExpireTimeout(causeItem); - if (this.connection) { - this.connection.logItem.refDetached(causeItem); - } + this.errorBoundary.try(() => { + this.callDeviceMembership = callDeviceMembership; + this._renewExpireTimeout(causeItem); + if (this.connection) { + this.connection.logItem.refDetached(causeItem); + } + }); } /** @internal */ @@ -308,49 +323,51 @@ export class Member { /** @internal */ handleDeviceMessage(message: SignallingMessage, syncLog: ILogItem): void { - const {connection} = this; - if (connection) { - const destSessionId = message.content.dest_session_id; - if (destSessionId !== this.options.sessionId) { - const logItem = connection.logItem.log({l: "ignoring to_device event with wrong session_id", destSessionId, type: message.type}); - syncLog.refDetached(logItem); - return; - } - // if there is no peerCall, we either create it with an invite and Handle is implied or we'll ignore it - let action = IncomingMessageAction.Handle; - if (connection.peerCall) { - action = connection.peerCall.getMessageAction(message); - // deal with glare and replacing the call before creating new calls - if (action === IncomingMessageAction.InviteGlare) { - const {shouldReplace, log} = connection.peerCall.handleInviteGlare(message, this.deviceId, connection.logItem); - if (log) { - syncLog.refDetached(log); - } - if (shouldReplace) { - connection.peerCall = undefined; - action = IncomingMessageAction.Handle; - } + this.errorBoundary.try(() => { + const {connection} = this; + if (connection) { + const destSessionId = message.content.dest_session_id; + if (destSessionId !== this.options.sessionId) { + const logItem = connection.logItem.log({l: "ignoring to_device event with wrong session_id", destSessionId, type: message.type}); + syncLog.refDetached(logItem); + return; } - } - if (message.type === EventType.Invite && !connection.peerCall) { - connection.peerCall = this._createPeerCall(message.content.call_id); - } - if (action === IncomingMessageAction.Handle) { - const idx = sortedIndex(connection.queuedSignallingMessages, message, (a, b) => a.content.seq - b.content.seq); - connection.queuedSignallingMessages.splice(idx, 0, message); + // if there is no peerCall, we either create it with an invite and Handle is implied or we'll ignore it + let action = IncomingMessageAction.Handle; if (connection.peerCall) { - const hasNewMessageBeenDequeued = this.dequeueSignallingMessages(connection, connection.peerCall, message, syncLog); - if (!hasNewMessageBeenDequeued) { - syncLog.refDetached(connection.logItem.log({l: "queued signalling message", type: message.type, seq: message.content.seq})); + action = connection.peerCall.getMessageAction(message); + // deal with glare and replacing the call before creating new calls + if (action === IncomingMessageAction.InviteGlare) { + const {shouldReplace, log} = connection.peerCall.handleInviteGlare(message, this.deviceId, connection.logItem); + if (log) { + syncLog.refDetached(log); + } + if (shouldReplace) { + connection.peerCall = undefined; + action = IncomingMessageAction.Handle; + } } } - } else if (action === IncomingMessageAction.Ignore && connection.peerCall) { - const logItem = connection.logItem.log({l: "ignoring to_device event with wrong call_id", callId: message.content.call_id, type: message.type}); - syncLog.refDetached(logItem); + if (message.type === EventType.Invite && !connection.peerCall) { + connection.peerCall = this._createPeerCall(message.content.call_id); + } + if (action === IncomingMessageAction.Handle) { + const idx = sortedIndex(connection.queuedSignallingMessages, message, (a, b) => a.content.seq - b.content.seq); + connection.queuedSignallingMessages.splice(idx, 0, message); + if (connection.peerCall) { + const hasNewMessageBeenDequeued = this.dequeueSignallingMessages(connection, connection.peerCall, message, syncLog); + if (!hasNewMessageBeenDequeued) { + syncLog.refDetached(connection.logItem.log({l: "queued signalling message", type: message.type, seq: message.content.seq})); + } + } + } else if (action === IncomingMessageAction.Ignore && connection.peerCall) { + const logItem = connection.logItem.log({l: "ignoring to_device event with wrong call_id", callId: message.content.call_id, type: message.type}); + syncLog.refDetached(logItem); + } + } else { + syncLog.log({l: "member not connected", userId: this.userId, deviceId: this.deviceId}); } - } else { - syncLog.log({l: "member not connected", userId: this.userId, deviceId: this.deviceId}); - } + }); } private dequeueSignallingMessages(connection: MemberConnection, peerCall: PeerCall, newMessage: SignallingMessage, syncLog: ILogItem): boolean { @@ -373,19 +390,23 @@ export class Member { /** @internal */ async setMedia(localMedia: LocalMedia, previousMedia: LocalMedia): Promise { - const {connection} = this; - if (connection) { - connection.localMedia = localMedia.replaceClone(connection.localMedia, previousMedia); - await connection.peerCall?.setMedia(connection.localMedia, connection.logItem); - } + return this.errorBoundary.try(async () => { + const {connection} = this; + if (connection) { + connection.localMedia = localMedia.replaceClone(connection.localMedia, previousMedia); + await connection.peerCall?.setMedia(connection.localMedia, connection.logItem); + } + }); } async setMuted(muteSettings: MuteSettings): Promise { - const {connection} = this; - if (connection) { - connection.localMuteSettings = muteSettings; - await connection.peerCall?.setMuted(muteSettings, connection.logItem); - } + return this.errorBoundary.try(async () => { + const {connection} = this; + if (connection) { + connection.localMuteSettings = muteSettings; + await connection.peerCall?.setMuted(muteSettings, connection.logItem); + } + }); } private _createPeerCall(callId: string): PeerCall { From 7f9edbb742bb78b6a782bce5f0670a46e4bcc3f5 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 9 Jan 2023 14:03:14 +0100 Subject: [PATCH 224/323] this can be private --- src/matrix/calls/group/GroupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index cda5a40f..01ff4d31 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -276,7 +276,7 @@ export class GroupCall extends EventEmitter<{change: never}> { }); } - terminate(log?: ILogItem): Promise { + private terminate(log?: ILogItem): Promise { return this.options.logger.wrapOrRun(log, {l: "terminate call", t: CALL_LOG_TYPE}, async log => { if (this._state === GroupCallState.Fledgling) { return; From 1e4180a71ff4f4ef5218eae861035a4bc5ca6387 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 9 Jan 2023 14:03:52 +0100 Subject: [PATCH 225/323] add error boundary to GroupCall --- src/matrix/calls/group/GroupCall.ts | 321 +++++++++++++++------------- 1 file changed, 177 insertions(+), 144 deletions(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 01ff4d31..53a87d5f 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -21,6 +21,7 @@ import {MuteSettings, CALL_LOG_TYPE, CALL_MEMBER_VALIDITY_PERIOD_MS, mute} from import {MemberChange, RoomMember} from "../../room/members/RoomMember"; import {EventEmitter} from "../../../utils/EventEmitter"; import {EventType, CallIntent} from "../callEventTypes"; +import { ErrorBoundary } from "../../../utils/ErrorBoundary"; import type {Options as MemberOptions} from "./Member"; import type {TurnServerSource} from "../TurnServerSource"; @@ -92,6 +93,9 @@ export class GroupCall extends EventEmitter<{change: never}> { private bufferedDeviceMessages = new Map>>(); /** Set between calling join and leave. */ private joinedData?: JoinedData; + private errorBoundary = new ErrorBoundary(err => { + this.emitChange(); + }); constructor( public readonly id: string, @@ -154,6 +158,10 @@ export class GroupCall extends EventEmitter<{change: never}> { return this.joinedData?.logItem; } + get error(): Error | undefined { + return this.errorBoundary.error; + } + async join(localMedia: LocalMedia): Promise { if (this._state !== GroupCallState.Created || this.joinedData) { return; @@ -206,6 +214,9 @@ export class GroupCall extends EventEmitter<{change: never}> { // and update the track info so PeerCall can use it to send up to date metadata, this.joinedData.localMuteSettings.updateTrackInfo(localMedia.userMedia); this.emitChange(); //allow listeners to see new media/mute settings + // TODO: if setMedia fails on one of the members, we should revert to the old media + // on the members processed so far, and show an error that we could not set the new media + // for this, we will need to remove the usage of the errorBoundary in member.setMedia. await Promise.all(Array.from(this._members.values()).map(m => { return m.setMedia(localMedia, oldMedia); })); @@ -234,6 +245,9 @@ export class GroupCall extends EventEmitter<{change: never}> { if (this.localMedia) { mute(this.localMedia, muteSettings, this.joinedData!.logItem); } + // TODO: if setMuted fails on one of the members, we should revert to the old media + // on the members processed so far, and show an error that we could not set the new media + // for this, we will need to remove the usage of the errorBoundary in member.setMuted. await Promise.all(Array.from(this._members.values()).map(m => { return m.setMuted(joinedData.localMuteSettings); })); @@ -271,7 +285,14 @@ export class GroupCall extends EventEmitter<{change: never}> { log.set("already_left", true); } } finally { - this.disconnect(log); + // disconnect is called both from the sync loop and from methods like this one that + // are called from the view model. We want errors during the sync loop being caught + // by the errorboundary, but since leave is called from the view model, we want + // the error to be thrown. So here we check if disconnect succeeded, and if not + // we rethrow the error put into the errorBoundary. + if(!this.disconnect(log)) { + throw this.errorBoundary.error; + } } }); } @@ -308,126 +329,134 @@ export class GroupCall extends EventEmitter<{change: never}> { /** @internal */ updateCallEvent(callContent: Record, syncLog: ILogItem) { - syncLog.wrap({l: "update call", t: CALL_LOG_TYPE, id: this.id}, log => { - this.callContent = callContent; - if (this._state === GroupCallState.Creating) { - this._state = GroupCallState.Created; - } - log.set("status", this._state); - this.emitChange(); + this.errorBoundary.try(() => { + syncLog.wrap({l: "update call", t: CALL_LOG_TYPE, id: this.id}, log => { + this.callContent = callContent; + if (this._state === GroupCallState.Creating) { + this._state = GroupCallState.Created; + } + log.set("status", this._state); + this.emitChange(); + }); }); } /** @internal */ updateRoomMembers(memberChanges: Map) { - for (const change of memberChanges.values()) { - const {member} = change; - for (const callMember of this._members.values()) { - // find all call members for a room member (can be multiple, for every device) - if (callMember.userId === member.userId) { - callMember.updateRoomMember(member); + this.errorBoundary.try(() => { + for (const change of memberChanges.values()) { + const {member} = change; + for (const callMember of this._members.values()) { + // find all call members for a room member (can be multiple, for every device) + if (callMember.userId === member.userId) { + callMember.updateRoomMember(member); + } } } - } - } - - /** @internal */ - updateMembership(userId: string, roomMember: RoomMember, callMembership: CallMembership, syncLog: ILogItem) { - syncLog.wrap({l: "update call membership", t: CALL_LOG_TYPE, id: this.id, userId}, log => { - const now = this.options.clock.now(); - const devices = callMembership["m.devices"]; - const previousDeviceIds = this.getDeviceIdsForUserId(userId); - for (const device of devices) { - const deviceId = device.device_id; - const memberKey = getMemberKey(userId, deviceId); - if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) { - log.wrap("update own membership", log => { - if (this.hasJoined) { - if (this.joinedData) { - this.joinedData.logItem.refDetached(log); - } - this._setupRenewMembershipTimeout(device, log); - } - if (this._state === GroupCallState.Joining) { - log.set("joined", true); - this._state = GroupCallState.Joined; - this.emitChange(); - } - }); - } else { - log.wrap({l: "update device membership", id: memberKey, sessionId: device.session_id}, log => { - if (isMemberExpired(device, now)) { - log.set("expired", true); - const member = this._members.get(memberKey); - if (member) { - member.dispose(); - this._members.remove(memberKey); - log.set("removed", true); - } - return; - } - let member = this._members.get(memberKey); - const sessionIdChanged = member && member.sessionId !== device.session_id; - if (member && !sessionIdChanged) { - log.set("update", true); - member.updateCallInfo(device, log); - } else { - if (member && sessionIdChanged) { - log.set("removedSessionId", member.sessionId); - const disconnectLogItem = member.disconnect(false); - if (disconnectLogItem) { - log.refDetached(disconnectLogItem); - } - member.dispose(); - this._members.remove(memberKey); - member = undefined; - } - log.set("add", true); - member = new Member( - roomMember, - device, this._memberOptions, - log - ); - this._members.add(memberKey, member); - if (this.joinedData) { - this.connectToMember(member, this.joinedData, log); - } - } - // flush pending messages, either after having created the member, - // or updated the session id with updateCallInfo - this.flushPendingIncomingDeviceMessages(member, log); - }); - } - } - - const newDeviceIds = new Set(devices.map(call => call.device_id)); - // remove user as member of any calls not present anymore - for (const previousDeviceId of previousDeviceIds) { - if (!newDeviceIds.has(previousDeviceId)) { - this.removeMemberDevice(userId, previousDeviceId, log); - } - } - if (userId === this.options.ownUserId && !newDeviceIds.has(this.options.ownDeviceId)) { - this.removeOwnDevice(log); - } + }); + } + + /** @internal */ + updateMembership(userId: string, roomMember: RoomMember, callMembership: CallMembership, syncLog: ILogItem) { + this.errorBoundary.try(() => { + syncLog.wrap({l: "update call membership", t: CALL_LOG_TYPE, id: this.id, userId}, log => { + const now = this.options.clock.now(); + const devices = callMembership["m.devices"]; + const previousDeviceIds = this.getDeviceIdsForUserId(userId); + for (const device of devices) { + const deviceId = device.device_id; + const memberKey = getMemberKey(userId, deviceId); + if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) { + log.wrap("update own membership", log => { + if (this.hasJoined) { + if (this.joinedData) { + this.joinedData.logItem.refDetached(log); + } + this._setupRenewMembershipTimeout(device, log); + } + if (this._state === GroupCallState.Joining) { + log.set("joined", true); + this._state = GroupCallState.Joined; + this.emitChange(); + } + }); + } else { + log.wrap({l: "update device membership", id: memberKey, sessionId: device.session_id}, log => { + if (isMemberExpired(device, now)) { + log.set("expired", true); + const member = this._members.get(memberKey); + if (member) { + member.dispose(); + this._members.remove(memberKey); + log.set("removed", true); + } + return; + } + let member = this._members.get(memberKey); + const sessionIdChanged = member && member.sessionId !== device.session_id; + if (member && !sessionIdChanged) { + log.set("update", true); + member.updateCallInfo(device, log); + } else { + if (member && sessionIdChanged) { + log.set("removedSessionId", member.sessionId); + const disconnectLogItem = member.disconnect(false); + if (disconnectLogItem) { + log.refDetached(disconnectLogItem); + } + member.dispose(); + this._members.remove(memberKey); + member = undefined; + } + log.set("add", true); + member = new Member( + roomMember, + device, this._memberOptions, + log + ); + this._members.add(memberKey, member); + if (this.joinedData) { + this.connectToMember(member, this.joinedData, log); + } + } + // flush pending messages, either after having created the member, + // or updated the session id with updateCallInfo + this.flushPendingIncomingDeviceMessages(member, log); + }); + } + } + + const newDeviceIds = new Set(devices.map(call => call.device_id)); + // remove user as member of any calls not present anymore + for (const previousDeviceId of previousDeviceIds) { + if (!newDeviceIds.has(previousDeviceId)) { + this.removeMemberDevice(userId, previousDeviceId, log); + } + } + if (userId === this.options.ownUserId && !newDeviceIds.has(this.options.ownDeviceId)) { + this.removeOwnDevice(log); + } + }); }); } /** @internal */ removeMembership(userId: string, syncLog: ILogItem) { - const deviceIds = this.getDeviceIdsForUserId(userId); - syncLog.wrap({ - l: "remove call member", - t: CALL_LOG_TYPE, - id: this.id, - userId - }, log => { - for (const deviceId of deviceIds) { - this.removeMemberDevice(userId, deviceId, log); - } - if (userId === this.options.ownUserId) { - this.removeOwnDevice(log); - } + this.errorBoundary.try(() => { + const deviceIds = this.getDeviceIdsForUserId(userId); + syncLog.wrap({ + l: "remove call member", + t: CALL_LOG_TYPE, + id: this.id, + userId + }, log => { + for (const deviceId of deviceIds) { + this.removeMemberDevice(userId, deviceId, log); + } + if (userId === this.options.ownUserId) { + this.removeOwnDevice(log); + } + }); }); } @@ -465,19 +494,21 @@ export class GroupCall extends EventEmitter<{change: never}> { } /** @internal */ - disconnect(log: ILogItem) { - if (this.hasJoined) { - for (const [,member] of this._members) { - const disconnectLogItem = member.disconnect(true); - if (disconnectLogItem) { - log.refDetached(disconnectLogItem); + disconnect(log: ILogItem): boolean { + return this.errorBoundary.try(() => { + if (this.hasJoined) { + for (const [,member] of this._members) { + const disconnectLogItem = member.disconnect(true); + if (disconnectLogItem) { + log.refDetached(disconnectLogItem); + } } + this._state = GroupCallState.Created; } - this._state = GroupCallState.Created; - } - this.joinedData?.dispose(); - this.joinedData = undefined; - this.emitChange(); + this.joinedData?.dispose(); + this.joinedData = undefined; + this.emitChange(); + }, false) || true; } /** @internal */ @@ -500,31 +531,33 @@ export class GroupCall extends EventEmitter<{change: never}> { /** @internal */ handleDeviceMessage(message: SignallingMessage, userId: string, deviceId: string, syncLog: ILogItem) { - // TODO: return if we are not membering to the call - const key = getMemberKey(userId, deviceId); - let member = this._members.get(key); - if (member && message.content.sender_session_id === member.sessionId) { - member.handleDeviceMessage(message, syncLog); - } else { - const item = syncLog.log({ - l: "call: buffering to_device message, member not found", - t: CALL_LOG_TYPE, - id: this.id, - userId, - deviceId, - sessionId: message.content.sender_session_id, - type: message.type - }); - syncLog.refDetached(item); - // we haven't received the m.call.member yet for this caller (or with this session id). - // buffer the device messages or create the member/call as it should arrive in a moment - let messages = this.bufferedDeviceMessages.get(key); - if (!messages) { - messages = new Set(); - this.bufferedDeviceMessages.set(key, messages); + this.errorBoundary.try(() => { + // TODO: return if we are not membering to the call + const key = getMemberKey(userId, deviceId); + let member = this._members.get(key); + if (member && message.content.sender_session_id === member.sessionId) { + member.handleDeviceMessage(message, syncLog); + } else { + const item = syncLog.log({ + l: "call: buffering to_device message, member not found", + t: CALL_LOG_TYPE, + id: this.id, + userId, + deviceId, + sessionId: message.content.sender_session_id, + type: message.type + }); + syncLog.refDetached(item); + // we haven't received the m.call.member yet for this caller (or with this session id). + // buffer the device messages or create the member/call as it should arrive in a moment + let messages = this.bufferedDeviceMessages.get(key); + if (!messages) { + messages = new Set(); + this.bufferedDeviceMessages.set(key, messages); + } + messages.add(message); } - messages.add(message); - } + }); } private async _createMemberPayload(includeOwn: boolean): Promise { From fef7af3b31fa4fb48f2d72a038493beac5f915cc Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 9 Jan 2023 14:04:13 +0100 Subject: [PATCH 226/323] report errors from ErrorBoundary on GroupCall and Member in UI UI is still very crude fwiw --- src/domain/session/room/CallViewModel.ts | 19 ++++++++++++++++++- .../session/room/timeline/tiles/CallTile.js | 11 ++++++++--- .../web/ui/css/themes/element/call.css | 16 ++++++++++++++++ src/platform/web/ui/session/room/CallView.ts | 8 +++++++- 4 files changed, 49 insertions(+), 5 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 37f30840..358b26f0 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -39,7 +39,11 @@ export class CallViewModel extends ViewModel { constructor(options: Options) { super(options); - const ownMemberViewModelMap = new ObservableValueMap("self", new EventObservableValue(this.call, "change")) + const callObservable = new EventObservableValue(this.call, "change"); + this.track(callObservable.subscribe(() => { + this.emitChange(); + })); + const ownMemberViewModelMap = new ObservableValueMap("self", callObservable) .mapValues((call, emitChange) => new OwnMemberViewModel(this.childOptions({call, emitChange})), () => {}); this.memberViewModels = this.call.members .filterValues(member => member.isConnected) @@ -79,6 +83,10 @@ export class CallViewModel extends ViewModel { return this.call.id; } + get error(): string | undefined { + return this.call.error?.message; + } + private get call(): GroupCall { return this.getOption("call"); } @@ -135,6 +143,10 @@ class OwnMemberViewModel extends ViewModel implements IStreamViewModel })); } + get error(): string | undefined { + return undefined; + } + get stream(): Stream | undefined { return this.call.localPreviewMedia?.userMedia; } @@ -195,6 +207,10 @@ export class CallMemberViewModel extends ViewModel implements ISt return this.member.remoteMedia?.userMedia; } + get error(): string | undefined { + return this.member.error?.message; + } + private get member(): Member { return this.getOption("member"); } @@ -242,4 +258,5 @@ export interface IStreamViewModel extends AvatarSource, ViewModel { get stream(): Stream | undefined; get isCameraMuted(): boolean; get isMicrophoneMuted(): boolean; + get error(): string | undefined; } diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index 0bc12698..a54af5d8 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -74,9 +74,14 @@ export class CallTile extends SimpleTile { async join() { if (this.canJoin) { - const stream = await this.platform.mediaDevices.getMediaTracks(false, true); - const localMedia = new LocalMedia().withUserMedia(stream); - await this._call.join(localMedia); + try { + const stream = await this.platform.mediaDevices.getMediaTracks(false, true); + const localMedia = new LocalMedia().withUserMedia(stream); + await this._call.join(localMedia); + } catch (err) { + this._error = err; + this.emitChange("error"); + } } } diff --git a/src/platform/web/ui/css/themes/element/call.css b/src/platform/web/ui/css/themes/element/call.css index 10388fb4..f4d674b4 100644 --- a/src/platform/web/ui/css/themes/element/call.css +++ b/src/platform/web/ui/css/themes/element/call.css @@ -24,6 +24,14 @@ limitations under the License. grid-row: 1; } +.CallView_error { + color: red; + font-weight: bold; + align-self: start; + justify-self: center; + margin: 16px; +} + .CallView_members { display: grid; gap: 12px; @@ -59,6 +67,14 @@ limitations under the License. justify-self: center; } +.StreamView_error { + color: red; + font-weight: bold; + align-self: start; + justify-self: center; + margin: 16px; +} + .StreamView_muteStatus { align-self: start; justify-self: end; diff --git a/src/platform/web/ui/session/room/CallView.ts b/src/platform/web/ui/session/room/CallView.ts index 619afc2e..87cac99f 100644 --- a/src/platform/web/ui/session/room/CallView.ts +++ b/src/platform/web/ui/session/room/CallView.ts @@ -43,7 +43,10 @@ export class CallView extends TemplateView { "CallView_unmutedCamera": vm => !vm.isCameraMuted, }, onClick: disableTargetCallback(() => vm.toggleCamera())}), t.button({className: "CallView_hangup", onClick: disableTargetCallback(() => vm.hangup())}), - ]) + ]), + t.if(vm => !!vm.error, t => { + return t.div({className: "CallView_error"}, vm => vm.error); + }) ]); } @@ -112,6 +115,9 @@ class StreamView extends TemplateView { microphoneMuted: vm => vm.isMicrophoneMuted && !vm.isCameraMuted, cameraMuted: vm => vm.isCameraMuted, } + }), + t.if(vm => !!vm.error, t => { + return t.div({className: "StreamView_error"}, vm => vm.error); }) ]); } From bd3499056ae358d1de1f172dc33e2d3f939a827d Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 10 Jan 2023 12:05:30 +0100 Subject: [PATCH 227/323] provider higher-level rageshake fn for opened session Co-authored-by: R Midhun Suresh --- src/domain/rageshake.ts | 33 +++++++++++++++++-- .../session/settings/SettingsViewModel.js | 30 ++++------------- 2 files changed, 38 insertions(+), 25 deletions(-) diff --git a/src/domain/rageshake.ts b/src/domain/rageshake.ts index cb06e638..08e82d8c 100644 --- a/src/domain/rageshake.ts +++ b/src/domain/rageshake.ts @@ -16,11 +16,15 @@ limitations under the License. import type {BlobHandle} from "../platform/web/dom/BlobHandle"; import type {RequestFunction} from "../platform/types/types"; +import type {Platform} from "../platform/web/Platform"; +import type {ILogger} from "../logging/types"; +import type { IDBLogPersister } from "../logging/IDBLogPersister"; +import type { Session } from "../matrix/Session"; // see https://github.com/matrix-org/rageshake#readme type RageshakeData = { // A textual description of the problem. Included in the details.log.gz file. - text: string | undefined; + text?: string; // Application user-agent. Included in the details.log.gz file. userAgent: string; // Identifier for the application (eg 'riot-web'). Should correspond to a mapping configured in the configuration file for github issue reporting to work. @@ -28,7 +32,7 @@ type RageshakeData = { // Application version. Included in the details.log.gz file. version: string; // Label to attach to the github issue, and include in the details file. - label: string | undefined; + label?: string; }; export async function submitLogsToRageshakeServer(data: RageshakeData, logsBlob: BlobHandle, submitUrl: string, request: RequestFunction): Promise { @@ -63,3 +67,28 @@ export async function submitLogsToRageshakeServer(data: RageshakeData, logsBlob: // we don't bother with reading report_url from the body as the rageshake server doesn't always return it // and would have to have CORS setup properly for us to be able to read it. } + +/** @throws {Error} */ +export async function submitLogsFromSessionToDefaultServer(session: Session, platform: Platform): Promise { + const {bugReportEndpointUrl} = platform.config; + if (!bugReportEndpointUrl) { + throw new Error("no server configured to submit logs"); + } + const logReporters = (platform.logger as ILogger).reporters; + const exportReporter = logReporters.find(r => !!r["export"]) as IDBLogPersister | undefined; + if (!exportReporter) { + throw new Error("No logger that can export configured"); + } + const logExport = await exportReporter.export(); + await submitLogsToRageshakeServer( + { + app: "hydrogen", + userAgent: platform.description, + version: platform.version, + text: `Submit logs from settings for user ${session.userId} on device ${session.deviceId}`, + }, + logExport.asBlob(), + bugReportEndpointUrl, + platform.request + ); +} \ No newline at end of file diff --git a/src/domain/session/settings/SettingsViewModel.js b/src/domain/session/settings/SettingsViewModel.js index 0d61bcac..d60a6327 100644 --- a/src/domain/session/settings/SettingsViewModel.js +++ b/src/domain/session/settings/SettingsViewModel.js @@ -16,7 +16,7 @@ limitations under the License. import {ViewModel} from "../../ViewModel"; import {KeyBackupViewModel} from "./KeyBackupViewModel.js"; -import {submitLogsToRageshakeServer} from "../../../domain/rageshake"; +import {submitLogsFromSessionToDefaultServer} from "../../../domain/rageshake"; class PushNotificationStatus { constructor() { @@ -175,29 +175,13 @@ export class SettingsViewModel extends ViewModel { } async sendLogsToServer() { - const {bugReportEndpointUrl} = this.platform.config; - if (bugReportEndpointUrl) { - this._logsFeedbackMessage = this.i18n`Sending logs…`; + this._logsFeedbackMessage = this.i18n`Sending logs…`; + try { + await submitLogsFromSessionToDefaultServer(this._session, this.platform); + this._logsFeedbackMessage = this.i18n`Logs sent succesfully!`; + } catch (err) { + this._logsFeedbackMessage = err.message; this.emitChange(); - try { - const logExport = await this.logger.export(); - await submitLogsToRageshakeServer( - { - app: "hydrogen", - userAgent: this.platform.description, - version: DEFINE_VERSION, - text: `Submit logs from settings for user ${this._session.userId} on device ${this._session.deviceId}`, - }, - logExport.asBlob(), - bugReportEndpointUrl, - this.platform.request - ); - this._logsFeedbackMessage = this.i18n`Logs sent succesfully!`; - this.emitChange(); - } catch (err) { - this._logsFeedbackMessage = err.message; - this.emitChange(); - } } } From b8bc6edbc0240731b883437a8a062f1f0d36c645 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 10 Jan 2023 12:06:49 +0100 Subject: [PATCH 228/323] add ErrorView(Model) to easily report errors and submit logs from UI --- src/domain/ErrorViewModel.ts | 44 +++++++++++++++ .../web/ui/css/themes/element/error.css | 50 +++++++++++++++++ .../web/ui/css/themes/element/theme.css | 1 + src/platform/web/ui/general/ErrorView.ts | 56 +++++++++++++++++++ 4 files changed, 151 insertions(+) create mode 100644 src/domain/ErrorViewModel.ts create mode 100644 src/platform/web/ui/css/themes/element/error.css create mode 100644 src/platform/web/ui/general/ErrorView.ts diff --git a/src/domain/ErrorViewModel.ts b/src/domain/ErrorViewModel.ts new file mode 100644 index 00000000..9045af80 --- /dev/null +++ b/src/domain/ErrorViewModel.ts @@ -0,0 +1,44 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { ViewModel, Options as BaseOptions } from "./ViewModel"; +import {submitLogsFromSessionToDefaultServer} from "./rageshake"; +import type { Session } from "../matrix/Session"; + +type Options = { + error: Error + session: Session, + onClose: () => void +} & BaseOptions; + +export class ErrorViewModel extends ViewModel { + get message(): string { + return this.getOption("error")?.message; + } + + close() { + this.getOption("onClose")(); + } + + async submitLogs(): Promise { + try { + await submitLogsFromSessionToDefaultServer(this.getOption("session"), this.platform); + return true; + } catch (err) { + return false; + } + } +} \ No newline at end of file diff --git a/src/platform/web/ui/css/themes/element/error.css b/src/platform/web/ui/css/themes/element/error.css new file mode 100644 index 00000000..f4e4fb6f --- /dev/null +++ b/src/platform/web/ui/css/themes/element/error.css @@ -0,0 +1,50 @@ +.ErrorView_block { + background: var(--error-color); + color: var(--fixed-white); + margin: 16px; +} + +.ErrorView_inline { + color: var(--error-color); + margin: 4px; +} + +.ErrorView { + font-weight: bold; + margin: 16px; + border-radius: 8px; + padding: 12px; + display: flex; + gap: 4px; +} + +.ErrorView_message { + flex-basis: 0; + flex-grow: 1; + margin: 0px; + word-break: break-all; + word-break: break-word; +} + +.ErrorView_submit { + align-self: end; +} + +.ErrorView_close { + align-self: start; + width: 16px; + height: 16px; + border: none; + background: none; + background-repeat: no-repeat; + background-size: contain; + cursor: pointer; +} + +.ErrorView_block .ErrorView_close { + background-image: url('icons/clear.svg?primary=fixed-white'); +} + +.ErrorView_inline .ErrorView_close { + background-image: url('icons/clear.svg?primary=text-color'); +} \ No newline at end of file diff --git a/src/platform/web/ui/css/themes/element/theme.css b/src/platform/web/ui/css/themes/element/theme.css index 6403bb60..84a27f13 100644 --- a/src/platform/web/ui/css/themes/element/theme.css +++ b/src/platform/web/ui/css/themes/element/theme.css @@ -19,6 +19,7 @@ limitations under the License. @import url('inter.css'); @import url('timeline.css'); @import url('call.css'); +@import url('error.css'); :root { font-size: 10px; diff --git a/src/platform/web/ui/general/ErrorView.ts b/src/platform/web/ui/general/ErrorView.ts new file mode 100644 index 00000000..f44361c3 --- /dev/null +++ b/src/platform/web/ui/general/ErrorView.ts @@ -0,0 +1,56 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {TemplateView, Builder} from "./TemplateView"; +import { disableTargetCallback } from "./utils"; + +import type { ViewNode } from "./types"; +import type {ErrorViewModel} from "../../../../domain/ErrorViewModel"; + + +export class ErrorView extends TemplateView { + constructor(vm: ErrorViewModel, private readonly options: {inline: boolean} = {inline: false}) { + super(vm); + } + override render(t: Builder, vm: ErrorViewModel): ViewNode { + const submitLogsButton = t.button({ + className: "ErrorView_submit", + onClick: disableTargetCallback(async () => { + if (await vm.submitLogs()) { + alert("Logs submitted!"); + } else { + alert("Could not submit logs"); + } + }) + }, "Submit logs"); + const closeButton = t.button({ + className: "ErrorView_close", + onClick: () => vm.close(), + title: "Dismiss error" + }); + return t.div({ + className: { + "ErrorView": true, + "ErrorView_inline": this.options.inline, + "ErrorView_block": !this.options.inline + }}, [ + t.p({className: "ErrorView_message"}, vm.message), + submitLogsButton, + closeButton + ]); + } +} + From f15e849f5496bca0b571c9f47d5f069882e38e82 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 10 Jan 2023 12:08:10 +0100 Subject: [PATCH 229/323] user error view model in room, also when starting call --- src/domain/session/room/RoomViewModel.js | 68 +++++++++++--------- src/platform/web/ui/session/room/RoomView.js | 5 +- 2 files changed, 40 insertions(+), 33 deletions(-) diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 077a996e..d744e5d5 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -23,6 +23,7 @@ import {avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../ import {ViewModel} from "../../ViewModel"; import {imageToInfo} from "../common.js"; import {LocalMedia} from "../../../matrix/calls/LocalMedia"; +import { ErrorViewModel } from "../../ErrorViewModel"; // TODO: remove fallback so default isn't included in bundle for SDK users that have their custom tileClassForEntry // this is a breaking SDK change though to make this option mandatory import {tileClassForEntry as defaultTileClassForEntry} from "./timeline/tiles/index"; @@ -36,8 +37,7 @@ export class RoomViewModel extends ViewModel { this._tileClassForEntry = tileClassForEntry ?? defaultTileClassForEntry; this._tileOptions = undefined; this._onRoomChange = this._onRoomChange.bind(this); - this._timelineError = null; - this._sendError = null; + this._errorViewModel = null; this._composerVM = null; if (room.isArchived) { this._composerVM = new ArchivedViewModel(this.childOptions({archivedRoom: room})); @@ -73,6 +73,14 @@ export class RoomViewModel extends ViewModel { } } + _reportError(error) { + this._errorViewModel = new ErrorViewModel(this.childOptions({error, onClose: () => { + this._errorViewModel = null; + this.emitChange("errorViewModel"); + }})); + this.emitChange("errorViewModel"); + } + async load() { this._room.on("change", this._onRoomChange); try { @@ -88,10 +96,8 @@ export class RoomViewModel extends ViewModel { timeline, }))); this.emitChange("timelineViewModel"); - } catch (err) { - console.error(`room.openTimeline(): ${err.message}:\n${err.stack}`); - this._timelineError = err; - this.emitChange("error"); + } catch (error) { + this._reportError(error); } this._clearUnreadAfterDelay(); } @@ -143,14 +149,8 @@ export class RoomViewModel extends ViewModel { get timelineViewModel() { return this._timelineVM; } get isEncrypted() { return this._room.isEncrypted; } - get error() { - if (this._timelineError) { - return `Something went wrong loading the timeline: ${this._timelineError.message}`; - } - if (this._sendError) { - return `Something went wrong sending your message: ${this._sendError.message}`; - } - return ""; + get errorViewModel() { + return this._errorViewModel; } get avatarLetter() { @@ -215,11 +215,8 @@ export class RoomViewModel extends ViewModel { } else { await this._room.sendEvent("m.room.message", {msgtype, body: message}); } - } catch (err) { - console.error(`room.sendMessage(): ${err.message}:\n${err.stack}`); - this._sendError = err; - this._timelineError = null; - this.emitChange("error"); + } catch (error) { + this._reportError(error); return false; } return true; @@ -289,10 +286,8 @@ export class RoomViewModel extends ViewModel { attachments["info.thumbnail_url"] = this._room.createAttachment(thumbnail.blob, file.name); await this._room.sendEvent("m.room.message", content, attachments); - } catch (err) { - this._sendError = err; - this.emitChange("error"); - console.error(err.stack); + } catch (error) { + this._reportError(error); } } @@ -331,10 +326,8 @@ export class RoomViewModel extends ViewModel { this._room.createAttachment(thumbnail.blob, file.name); } await this._room.sendEvent("m.room.message", content, attachments); - } catch (err) { - this._sendError = err; - this.emitChange("error"); - console.error(err.stack); + } catch (error) { + this._reportError(error); } } @@ -364,15 +357,28 @@ export class RoomViewModel extends ViewModel { } async startCall() { + let localMedia; try { - const session = this.getOption("session"); const stream = await this.platform.mediaDevices.getMediaTracks(false, true); - const localMedia = new LocalMedia().withUserMedia(stream); + localMedia = new LocalMedia().withUserMedia(stream); + } catch (err) { + this._reportError(new Error(`Could not get local audio and/or video stream: ${err.message}`)); + return; + } + const session = this.getOption("session"); + let call; + try { // this will set the callViewModel above as a call will be added to callHandler.calls - const call = await session.callHandler.createCall(this._room.id, "m.video", "A call " + Math.round(this.platform.random() * 100)); + call = await session.callHandler.createCall(this._room.id, "m.video", "A call " + Math.round(this.platform.random() * 100)); + } catch (err) { + this._reportError(new Error(`Could not create call: ${err.message}`)); + return; + } + try { await call.join(localMedia); } catch (err) { - console.error(err.stack); + this._reportError(new Error(`Could not join call: ${err.message}`)); + return; } } } diff --git a/src/platform/web/ui/session/room/RoomView.js b/src/platform/web/ui/session/room/RoomView.js index 4f9e40d9..9478f5e6 100644 --- a/src/platform/web/ui/session/room/RoomView.js +++ b/src/platform/web/ui/session/room/RoomView.js @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {TemplateView} from "../../general/TemplateView"; +import {TemplateView} from "../../general/TemplateView"; import {Popup} from "../../general/Popup.js"; import {Menu} from "../../general/Menu.js"; import {TimelineView} from "./TimelineView"; @@ -24,6 +24,7 @@ import {MessageComposer} from "./MessageComposer.js"; import {RoomArchivedView} from "./RoomArchivedView.js"; import {AvatarView} from "../../AvatarView.js"; import {CallView} from "./CallView"; +import { ErrorView } from "../../general/ErrorView"; export class RoomView extends TemplateView { constructor(vm, viewClassForTile) { @@ -53,7 +54,7 @@ export class RoomView extends TemplateView { }) ]), t.div({className: "RoomView_body"}, [ - t.div({className: "RoomView_error"}, vm => vm.error), + t.if(vm => vm.errorViewModel, t => t.div({className: "RoomView_error"}, t.view(new ErrorView(vm.errorViewModel)))), t.mapView(vm => vm.callViewModel, callViewModel => callViewModel ? new CallView(callViewModel) : null), t.mapView(vm => vm.timelineViewModel, timelineViewModel => { return timelineViewModel ? From 42ee2d294b1f9f4f3afd3d6ccb737d70ccf95bb6 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 10 Jan 2023 12:09:10 +0100 Subject: [PATCH 230/323] use error view model from call tile --- .../session/room/timeline/tiles/CallTile.js | 26 +++++++++++++++---- .../ui/session/room/timeline/CallTileView.ts | 21 ++++++++++----- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index a54af5d8..a3008cb9 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -16,7 +16,7 @@ limitations under the License. import {SimpleTile} from "./SimpleTile.js"; import {LocalMedia} from "../../../../../matrix/calls/LocalMedia"; - +import {ErrorViewModel} from "../../../../ErrorViewModel" // TODO: timeline entries for state events with the same state key and type // should also update previous entries in the timeline, so we can update the name of the call, whether it is terminated, etc ... @@ -28,6 +28,7 @@ export class CallTile extends SimpleTile { const calls = this.getOption("session").callHandler.calls; this._call = calls.get(this._entry.stateKey); this._callSubscription = undefined; + this._errorViewModel = undefined; if (this._call) { this._callSubscription = this._call.disposableOn("change", () => { // unsubscribe when terminated @@ -60,6 +61,10 @@ export class CallTile extends SimpleTile { return this._call && this._call.hasJoined; } + get errorViewModel() { + return this._errorViewModel; + } + get label() { if (this._call) { if (this._call.hasJoined) { @@ -78,16 +83,19 @@ export class CallTile extends SimpleTile { const stream = await this.platform.mediaDevices.getMediaTracks(false, true); const localMedia = new LocalMedia().withUserMedia(stream); await this._call.join(localMedia); - } catch (err) { - this._error = err; - this.emitChange("error"); + } catch (error) { + this._reportError(error); } } } async leave() { if (this.canLeave) { - this._call.leave(); + try { + this._call.leave(); + } catch (err) { + this._reportError(err); + } } } @@ -96,4 +104,12 @@ export class CallTile extends SimpleTile { this._callSubscription = this._callSubscription(); } } + + _reportError(error) { + this._errorViewModel = new ErrorViewModel(this.childOptions({error, onClose: () => { + this._errorViewModel = undefined; + this.emitChange("errorViewModel"); + }})); + this.emitChange("errorViewModel"); + } } diff --git a/src/platform/web/ui/session/room/timeline/CallTileView.ts b/src/platform/web/ui/session/room/timeline/CallTileView.ts index e0ca00bd..5bede510 100644 --- a/src/platform/web/ui/session/room/timeline/CallTileView.ts +++ b/src/platform/web/ui/session/room/timeline/CallTileView.ts @@ -14,17 +14,24 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {TemplateView} from "../../../general/TemplateView"; +import {Builder, TemplateView} from "../../../general/TemplateView"; import type {CallTile} from "../../../../../../domain/session/room/timeline/tiles/CallTile"; +import {ErrorView} from "../../../general/ErrorView"; export class CallTileView extends TemplateView { - render(t, vm) { + render(t: Builder, vm: CallTile) { return t.li( - {className: "AnnouncementView"}, - t.div([ - vm => vm.label, - t.button({className: "CallTileView_join", hidden: vm => !vm.canJoin}, "Join"), - t.button({className: "CallTileView_leave", hidden: vm => !vm.canLeave}, "Leave") + {className: "CallTileView AnnouncementView"}, + t.div( + [ + t.if(vm => vm.errorViewModel, t => { + return t.div({className: "CallTileView_error"}, t.view(new ErrorView(vm.errorViewModel, {inline: true}))); + }), + t.div([ + vm => vm.label, + t.button({className: "CallTileView_join", hidden: vm => !vm.canJoin}, "Join"), + t.button({className: "CallTileView_leave", hidden: vm => !vm.canLeave}, "Leave") + ]) ]) ); } From 64d6db556a0e3a6a0c5fed3dc8a27345962e928f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 12 Jan 2023 14:36:39 +0100 Subject: [PATCH 231/323] fix updates from call and member classes in VM this fixes this.emitChange sending the update over the collection in the call member VM, which is how updates are subscribed to by the UI. It also adds a callback to the VM for when the member sends an update, so we can check later on if the error on the member has been set. --- src/domain/session/room/CallViewModel.ts | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 358b26f0..1080015f 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -40,14 +40,19 @@ export class CallViewModel extends ViewModel { constructor(options: Options) { super(options); const callObservable = new EventObservableValue(this.call, "change"); - this.track(callObservable.subscribe(() => { - this.emitChange(); - })); + this.track(callObservable.subscribe(() => this.onUpdate())); const ownMemberViewModelMap = new ObservableValueMap("self", callObservable) .mapValues((call, emitChange) => new OwnMemberViewModel(this.childOptions({call, emitChange})), () => {}); this.memberViewModels = this.call.members .filterValues(member => member.isConnected) - .mapValues(member => new CallMemberViewModel(this.childOptions({member, mediaRepository: this.getOption("room").mediaRepository}))) + .mapValues( + (member, emitChange) => new CallMemberViewModel(this.childOptions({ + member, + emitChange, + mediaRepository: this.getOption("room").mediaRepository + })), + (vm: CallMemberViewModel) => vm.onUpdate(), + ) .join(ownMemberViewModelMap) .sortValues((a, b) => a.compare(b)); this.track(this.memberViewModels.subscribe({ @@ -91,6 +96,9 @@ export class CallViewModel extends ViewModel { return this.getOption("call"); } + private onUpdate() { + } + async hangup() { if (this.call.hasJoined) { await this.call.leave(); @@ -241,6 +249,8 @@ export class CallMemberViewModel extends ViewModel implements ISt return this.member.member.name; } + onUpdate() { + } compare(other: OwnMemberViewModel | CallMemberViewModel): number { if (other instanceof OwnMemberViewModel) { return -other.compare(this); From 4070d422cdd0883af3c4709cab3afc781c13c14b Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 12 Jan 2023 14:37:28 +0100 Subject: [PATCH 232/323] use error view (model) in call view (model) --- src/domain/ErrorViewModel.ts | 4 ++ src/domain/session/room/CallViewModel.ts | 68 ++++++++++++++++---- src/platform/web/ui/session/room/CallView.ts | 9 +-- 3 files changed, 65 insertions(+), 16 deletions(-) diff --git a/src/domain/ErrorViewModel.ts b/src/domain/ErrorViewModel.ts index 9045af80..51a34565 100644 --- a/src/domain/ErrorViewModel.ts +++ b/src/domain/ErrorViewModel.ts @@ -29,6 +29,10 @@ export class ErrorViewModel extends ViewModel { return this.getOption("error")?.message; } + get error(): Error { + return this.getOption("error"); + } + close() { this.getOption("onClose")(); } diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 1080015f..e87af12a 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -20,6 +20,7 @@ import {getStreamVideoTrack, getStreamAudioTrack} from "../../../matrix/calls/co import {avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../avatar"; import {EventObservableValue} from "../../../observable/value/EventObservableValue"; import {ObservableValueMap} from "../../../observable/map/ObservableValueMap"; +import { ErrorViewModel } from "../../ErrorViewModel"; import type {Room} from "../../../matrix/room/Room"; import type {GroupCall} from "../../../matrix/calls/group/GroupCall"; import type {Member} from "../../../matrix/calls/group/Member"; @@ -28,14 +29,17 @@ import type {BaseObservableList} from "../../../observable/list/BaseObservableLi import type {BaseObservableValue} from "../../../observable/value/BaseObservableValue"; import type {Stream} from "../../../platform/types/MediaDevices"; import type {MediaRepository} from "../../../matrix/net/MediaRepository"; +import type { Session } from "../../../matrix/Session"; type Options = BaseOptions & { call: GroupCall, room: Room, + session: Session }; export class CallViewModel extends ViewModel { public readonly memberViewModels: BaseObservableList; + private _errorViewModel?: ErrorViewModel; constructor(options: Options) { super(options); @@ -88,8 +92,8 @@ export class CallViewModel extends ViewModel { return this.call.id; } - get error(): string | undefined { - return this.call.error?.message; + get errorViewModel(): ErrorViewModel | undefined { + return this._errorViewModel; } private get call(): GroupCall { @@ -97,11 +101,18 @@ export class CallViewModel extends ViewModel { } private onUpdate() { + if (this.call.error) { + this._reportError(this.call.error); + } } async hangup() { - if (this.call.hasJoined) { - await this.call.leave(); + try { + if (this.call.hasJoined) { + await this.call.leave(); + } + } catch (err) { + this._reportError(err); } } @@ -125,7 +136,6 @@ export class CallViewModel extends ViewModel { // unmute but no track? if (muteSettings.microphone && !getStreamAudioTrack(localMedia.userMedia)) { const stream = await this.platform.mediaDevices.getMediaTracks(true, !muteSettings.camera); - console.log("got tracks", Array.from(stream.getTracks()).map((t: MediaStreamTrack) => { return {kind: t.kind, id: t.id};})) await this.call.setMedia(localMedia.withUserMedia(stream)); } else { await this.call.setMuted(muteSettings.toggleMicrophone()); @@ -133,6 +143,21 @@ export class CallViewModel extends ViewModel { this.emitChange(); } } + + private _reportError(error: Error) { + if (this._errorViewModel?.error === error) { + return; + } + this.disposeTracked(this._errorViewModel); + this._errorViewModel = new ErrorViewModel(this.childOptions({ + error, + onClose: () => { + this._errorViewModel = this.disposeTracked(this._errorViewModel); + this.emitChange("errorViewModel"); + } + })); + this.emitChange("errorViewModel"); + } } class OwnMemberViewModel extends ViewModel implements IStreamViewModel { @@ -151,7 +176,7 @@ class OwnMemberViewModel extends ViewModel implements IStreamViewModel })); } - get error(): string | undefined { + get errorViewModel(): ErrorViewModel | undefined { return undefined; } @@ -207,22 +232,25 @@ class OwnMemberViewModel extends ViewModel implements IStreamViewModel type MemberOptions = BaseOptions & { member: Member, - mediaRepository: MediaRepository + mediaRepository: MediaRepository, + session: Session }; export class CallMemberViewModel extends ViewModel implements IStreamViewModel { + private _errorViewModel?: ErrorViewModel; + get stream(): Stream | undefined { return this.member.remoteMedia?.userMedia; } - get error(): string | undefined { - return this.member.error?.message; - } - private get member(): Member { return this.getOption("member"); } + get errorViewModel(): ErrorViewModel | undefined { + return this._errorViewModel; + } + get isCameraMuted(): boolean { return this.member.remoteMuteSettings?.camera ?? true; } @@ -250,7 +278,23 @@ export class CallMemberViewModel extends ViewModel implements ISt } onUpdate() { + this.mapMemberSyncErrorIfNeeded(); } + + private mapMemberSyncErrorIfNeeded() { + if (this.member.error && (!this._errorViewModel || this._errorViewModel.error !== this.member.error)) { + this.disposeTracked(this._errorViewModel); + this._errorViewModel = this.track(new ErrorViewModel(this.childOptions({ + error: this.member.error, + onClose: () => { + this._errorViewModel = this.disposeTracked(this._errorViewModel); + this.emitChange("errorViewModel"); + }, + }))); + this.emitChange("errorViewModel"); + } + } + compare(other: OwnMemberViewModel | CallMemberViewModel): number { if (other instanceof OwnMemberViewModel) { return -other.compare(this); @@ -268,5 +312,5 @@ export interface IStreamViewModel extends AvatarSource, ViewModel { get stream(): Stream | undefined; get isCameraMuted(): boolean; get isMicrophoneMuted(): boolean; - get error(): string | undefined; + get errorViewModel(): ErrorViewModel | undefined; } diff --git a/src/platform/web/ui/session/room/CallView.ts b/src/platform/web/ui/session/room/CallView.ts index 87cac99f..eacb3144 100644 --- a/src/platform/web/ui/session/room/CallView.ts +++ b/src/platform/web/ui/session/room/CallView.ts @@ -20,6 +20,7 @@ import {ListView} from "../../general/ListView"; import {classNames} from "../../general/html"; import {Stream} from "../../../../types/MediaDevices"; import type {CallViewModel, CallMemberViewModel, IStreamViewModel} from "../../../../../domain/session/room/CallViewModel"; +import { ErrorView } from "../../general/ErrorView"; export class CallView extends TemplateView { private resizeObserver?: ResizeObserver; @@ -44,8 +45,8 @@ export class CallView extends TemplateView { }, onClick: disableTargetCallback(() => vm.toggleCamera())}), t.button({className: "CallView_hangup", onClick: disableTargetCallback(() => vm.hangup())}), ]), - t.if(vm => !!vm.error, t => { - return t.div({className: "CallView_error"}, vm => vm.error); + t.if(vm => !!vm.errorViewModel, t => { + return t.div({className: "CallView_error"}, t.view(new ErrorView(vm.errorViewModel!))); }) ]); } @@ -116,8 +117,8 @@ class StreamView extends TemplateView { cameraMuted: vm => vm.isCameraMuted, } }), - t.if(vm => !!vm.error, t => { - return t.div({className: "StreamView_error"}, vm => vm.error); + t.if(vm => !!vm.errorViewModel, t => { + return t.div({className: "StreamView_error"}, t.view(new ErrorView(vm.errorViewModel!))); }) ]); } From 7b32a2729e09c31778e9fb9f3c82f09586ddd499 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 13 Jan 2023 10:39:44 +0100 Subject: [PATCH 233/323] don't allow other click handlers to run in parent elements --- src/platform/web/ui/general/ErrorView.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/platform/web/ui/general/ErrorView.ts b/src/platform/web/ui/general/ErrorView.ts index f44361c3..eb1e7838 100644 --- a/src/platform/web/ui/general/ErrorView.ts +++ b/src/platform/web/ui/general/ErrorView.ts @@ -28,7 +28,8 @@ export class ErrorView extends TemplateView { override render(t: Builder, vm: ErrorViewModel): ViewNode { const submitLogsButton = t.button({ className: "ErrorView_submit", - onClick: disableTargetCallback(async () => { + onClick: disableTargetCallback(async evt => { + evt.stopPropagation(); if (await vm.submitLogs()) { alert("Logs submitted!"); } else { @@ -38,7 +39,10 @@ export class ErrorView extends TemplateView { }, "Submit logs"); const closeButton = t.button({ className: "ErrorView_close", - onClick: () => vm.close(), + onClick: evt => { + evt.stopPropagation(); + vm.close(); + }, title: "Dismiss error" }); return t.div({ From 75839007ea41b7a398a2adb95651629d5b8648ff Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 13 Jan 2023 10:40:04 +0100 Subject: [PATCH 234/323] make buttons clickable in the first place --- src/platform/web/ui/css/themes/element/call.css | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/platform/web/ui/css/themes/element/call.css b/src/platform/web/ui/css/themes/element/call.css index f4d674b4..c0f1550e 100644 --- a/src/platform/web/ui/css/themes/element/call.css +++ b/src/platform/web/ui/css/themes/element/call.css @@ -68,11 +68,12 @@ limitations under the License. } .StreamView_error { - color: red; - font-weight: bold; align-self: start; justify-self: center; - margin: 16px; + /** Chrome (v100) requires this to make the buttons clickable + * where they overlap with the video element, even though + * the buttons come later in the DOM. */ + z-index: 1; } .StreamView_muteStatus { From 2bba9f8675bfefc0122db7f0b1e8d7dc41f7c0ec Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 13 Jan 2023 10:40:15 +0100 Subject: [PATCH 235/323] some layout improvements --- src/platform/web/ui/css/themes/element/error.css | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/platform/web/ui/css/themes/element/error.css b/src/platform/web/ui/css/themes/element/error.css index f4e4fb6f..ce252056 100644 --- a/src/platform/web/ui/css/themes/element/error.css +++ b/src/platform/web/ui/css/themes/element/error.css @@ -15,7 +15,7 @@ border-radius: 8px; padding: 12px; display: flex; - gap: 4px; + gap: 8px; } .ErrorView_message { @@ -24,6 +24,7 @@ margin: 0px; word-break: break-all; word-break: break-word; + align-self: center; } .ErrorView_submit { From d3b5a7066383a8d8d0e3addc6ac1e6f232bdd8ab Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 13 Jan 2023 12:42:52 +0100 Subject: [PATCH 236/323] join errors thrown from matrix layer with sync errors caught by error boundary. this adds a new base view model that facilitates reporting errors with the ErrorViewModel --- src/domain/ErrorReportViewModel.ts | 62 +++++++++++ src/domain/session/room/CallViewModel.ts | 101 ++++++------------ src/domain/session/room/RoomViewModel.js | 27 ++--- .../session/room/timeline/tiles/CallTile.js | 31 ++---- .../session/room/timeline/tiles/SimpleTile.js | 4 +- 5 files changed, 114 insertions(+), 111 deletions(-) create mode 100644 src/domain/ErrorReportViewModel.ts diff --git a/src/domain/ErrorReportViewModel.ts b/src/domain/ErrorReportViewModel.ts new file mode 100644 index 00000000..eb6ed0aa --- /dev/null +++ b/src/domain/ErrorReportViewModel.ts @@ -0,0 +1,62 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { ViewModel } from "./ViewModel"; +import type { Options as BaseOptions } from "./ViewModel"; +import type { Session } from "../matrix/Session"; +import { ErrorViewModel } from "./ErrorViewModel"; +import type { LogCallback, LabelOrValues } from "../logging/types"; + +export type Options = BaseOptions & { + session: Session +}; + +/** Base class for view models that need to report errors to the UI. */ +export class ErrorReportViewModel extends ViewModel { + private _errorViewModel?: ErrorViewModel; + + get errorViewModel(): ErrorViewModel | undefined { + return this._errorViewModel; + } + + protected reportError(error: Error) { + if (this._errorViewModel?.error === error) { + return; + } + this.disposeTracked(this._errorViewModel); + this._errorViewModel = this.track(new ErrorViewModel(this.childOptions({ + error, + onClose: () => { + this._errorViewModel = this.disposeTracked(this._errorViewModel); + this.emitChange("errorViewModel"); + } + }))); + this.emitChange("errorViewModel"); + } + + protected logAndCatch(labelOrValues: LabelOrValues, callback: LogCallback, errorValue: T = undefined as unknown as T): T { + try { + const result = this.logger.run(labelOrValues, callback); + if (result instanceof Promise) { + result.catch(err => this.reportError(err)); + } + return result; + } catch (err) { + this.reportError(err); + return errorValue; + } + } +} diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index e87af12a..ee0ad502 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -15,7 +15,8 @@ limitations under the License. */ import {AvatarSource} from "../../AvatarSource"; -import {ViewModel, Options as BaseOptions} from "../../ViewModel"; +import type {ViewModel} from "../../ViewModel"; +import {ErrorReportViewModel, Options as BaseOptions} from "../../ErrorReportViewModel"; import {getStreamVideoTrack, getStreamAudioTrack} from "../../../matrix/calls/common"; import {avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../avatar"; import {EventObservableValue} from "../../../observable/value/EventObservableValue"; @@ -34,12 +35,10 @@ import type { Session } from "../../../matrix/Session"; type Options = BaseOptions & { call: GroupCall, room: Room, - session: Session }; -export class CallViewModel extends ViewModel { +export class CallViewModel extends ErrorReportViewModel { public readonly memberViewModels: BaseObservableList; - private _errorViewModel?: ErrorViewModel; constructor(options: Options) { super(options); @@ -92,75 +91,58 @@ export class CallViewModel extends ViewModel { return this.call.id; } - get errorViewModel(): ErrorViewModel | undefined { - return this._errorViewModel; - } - private get call(): GroupCall { return this.getOption("call"); } private onUpdate() { if (this.call.error) { - this._reportError(this.call.error); + this.reportError(this.call.error); } } async hangup() { - try { + this.logAndCatch("Call.hangup", async log => { if (this.call.hasJoined) { await this.call.leave(); } - } catch (err) { - this._reportError(err); - } + }); } async toggleCamera() { - const {localMedia, muteSettings} = this.call; - if (muteSettings && localMedia) { - // unmute but no track? - if (muteSettings.camera && !getStreamVideoTrack(localMedia.userMedia)) { - const stream = await this.platform.mediaDevices.getMediaTracks(!muteSettings.microphone, true); - await this.call.setMedia(localMedia.withUserMedia(stream)); - } else { - await this.call.setMuted(muteSettings.toggleCamera()); + this.logAndCatch("Call.toggleCamera", async log => { + const {localMedia, muteSettings} = this.call; + if (muteSettings && localMedia) { + // unmute but no track? + if (muteSettings.camera && !getStreamVideoTrack(localMedia.userMedia)) { + const stream = await this.platform.mediaDevices.getMediaTracks(!muteSettings.microphone, true); + await this.call.setMedia(localMedia.withUserMedia(stream)); + } else { + await this.call.setMuted(muteSettings.toggleCamera()); + } + this.emitChange(); } - this.emitChange(); - } + }); } async toggleMicrophone() { - const {localMedia, muteSettings} = this.call; - if (muteSettings && localMedia) { - // unmute but no track? - if (muteSettings.microphone && !getStreamAudioTrack(localMedia.userMedia)) { - const stream = await this.platform.mediaDevices.getMediaTracks(true, !muteSettings.camera); - await this.call.setMedia(localMedia.withUserMedia(stream)); - } else { - await this.call.setMuted(muteSettings.toggleMicrophone()); + this.logAndCatch("Call.toggleMicrophone", async log => { + const {localMedia, muteSettings} = this.call; + if (muteSettings && localMedia) { + // unmute but no track? + if (muteSettings.microphone && !getStreamAudioTrack(localMedia.userMedia)) { + const stream = await this.platform.mediaDevices.getMediaTracks(true, !muteSettings.camera); + await this.call.setMedia(localMedia.withUserMedia(stream)); + } else { + await this.call.setMuted(muteSettings.toggleMicrophone()); + } + this.emitChange(); } - this.emitChange(); - } - } - - private _reportError(error: Error) { - if (this._errorViewModel?.error === error) { - return; - } - this.disposeTracked(this._errorViewModel); - this._errorViewModel = new ErrorViewModel(this.childOptions({ - error, - onClose: () => { - this._errorViewModel = this.disposeTracked(this._errorViewModel); - this.emitChange("errorViewModel"); - } - })); - this.emitChange("errorViewModel"); + }); } } -class OwnMemberViewModel extends ViewModel implements IStreamViewModel { +class OwnMemberViewModel extends ErrorReportViewModel implements IStreamViewModel { private memberObservable: undefined | BaseObservableValue; constructor(options: Options) { @@ -233,12 +215,9 @@ class OwnMemberViewModel extends ViewModel implements IStreamViewModel type MemberOptions = BaseOptions & { member: Member, mediaRepository: MediaRepository, - session: Session }; -export class CallMemberViewModel extends ViewModel implements IStreamViewModel { - private _errorViewModel?: ErrorViewModel; - +export class CallMemberViewModel extends ErrorReportViewModel implements IStreamViewModel { get stream(): Stream | undefined { return this.member.remoteMedia?.userMedia; } @@ -247,10 +226,6 @@ export class CallMemberViewModel extends ViewModel implements ISt return this.getOption("member"); } - get errorViewModel(): ErrorViewModel | undefined { - return this._errorViewModel; - } - get isCameraMuted(): boolean { return this.member.remoteMuteSettings?.camera ?? true; } @@ -282,16 +257,8 @@ export class CallMemberViewModel extends ViewModel implements ISt } private mapMemberSyncErrorIfNeeded() { - if (this.member.error && (!this._errorViewModel || this._errorViewModel.error !== this.member.error)) { - this.disposeTracked(this._errorViewModel); - this._errorViewModel = this.track(new ErrorViewModel(this.childOptions({ - error: this.member.error, - onClose: () => { - this._errorViewModel = this.disposeTracked(this._errorViewModel); - this.emitChange("errorViewModel"); - }, - }))); - this.emitChange("errorViewModel"); + if (this.member.error) { + this.reportError(this.member.error); } } diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index d744e5d5..a5dfdc8f 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -20,7 +20,7 @@ import {ComposerViewModel} from "./ComposerViewModel.js" import {CallViewModel} from "./CallViewModel" import {PickMapObservableValue} from "../../../observable/value/PickMapObservableValue"; import {avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../avatar"; -import {ViewModel} from "../../ViewModel"; +import {ErrorReportViewModel} from "../../ErrorReportViewModel"; import {imageToInfo} from "../common.js"; import {LocalMedia} from "../../../matrix/calls/LocalMedia"; import { ErrorViewModel } from "../../ErrorViewModel"; @@ -28,7 +28,7 @@ import { ErrorViewModel } from "../../ErrorViewModel"; // this is a breaking SDK change though to make this option mandatory import {tileClassForEntry as defaultTileClassForEntry} from "./timeline/tiles/index"; -export class RoomViewModel extends ViewModel { +export class RoomViewModel extends ErrorReportViewModel { constructor(options) { super(options); const {room, tileClassForEntry} = options; @@ -37,7 +37,6 @@ export class RoomViewModel extends ViewModel { this._tileClassForEntry = tileClassForEntry ?? defaultTileClassForEntry; this._tileOptions = undefined; this._onRoomChange = this._onRoomChange.bind(this); - this._errorViewModel = null; this._composerVM = null; if (room.isArchived) { this._composerVM = new ArchivedViewModel(this.childOptions({archivedRoom: room})); @@ -73,14 +72,6 @@ export class RoomViewModel extends ViewModel { } } - _reportError(error) { - this._errorViewModel = new ErrorViewModel(this.childOptions({error, onClose: () => { - this._errorViewModel = null; - this.emitChange("errorViewModel"); - }})); - this.emitChange("errorViewModel"); - } - async load() { this._room.on("change", this._onRoomChange); try { @@ -97,7 +88,7 @@ export class RoomViewModel extends ViewModel { }))); this.emitChange("timelineViewModel"); } catch (error) { - this._reportError(error); + this.reportError(error); } this._clearUnreadAfterDelay(); } @@ -216,7 +207,7 @@ export class RoomViewModel extends ViewModel { await this._room.sendEvent("m.room.message", {msgtype, body: message}); } } catch (error) { - this._reportError(error); + this.reportError(error); return false; } return true; @@ -287,7 +278,7 @@ export class RoomViewModel extends ViewModel { this._room.createAttachment(thumbnail.blob, file.name); await this._room.sendEvent("m.room.message", content, attachments); } catch (error) { - this._reportError(error); + this.reportError(error); } } @@ -327,7 +318,7 @@ export class RoomViewModel extends ViewModel { } await this._room.sendEvent("m.room.message", content, attachments); } catch (error) { - this._reportError(error); + this.reportError(error); } } @@ -362,7 +353,7 @@ export class RoomViewModel extends ViewModel { const stream = await this.platform.mediaDevices.getMediaTracks(false, true); localMedia = new LocalMedia().withUserMedia(stream); } catch (err) { - this._reportError(new Error(`Could not get local audio and/or video stream: ${err.message}`)); + this.reportError(new Error(`Could not get local audio and/or video stream: ${err.message}`)); return; } const session = this.getOption("session"); @@ -371,13 +362,13 @@ export class RoomViewModel extends ViewModel { // this will set the callViewModel above as a call will be added to callHandler.calls call = await session.callHandler.createCall(this._room.id, "m.video", "A call " + Math.round(this.platform.random() * 100)); } catch (err) { - this._reportError(new Error(`Could not create call: ${err.message}`)); + this.reportError(new Error(`Could not create call: ${err.message}`)); return; } try { await call.join(localMedia); } catch (err) { - this._reportError(new Error(`Could not join call: ${err.message}`)); + this.reportError(new Error(`Could not join call: ${err.message}`)); return; } } diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index a3008cb9..44c39eef 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -28,7 +28,6 @@ export class CallTile extends SimpleTile { const calls = this.getOption("session").callHandler.calls; this._call = calls.get(this._entry.stateKey); this._callSubscription = undefined; - this._errorViewModel = undefined; if (this._call) { this._callSubscription = this._call.disposableOn("change", () => { // unsubscribe when terminated @@ -61,10 +60,6 @@ export class CallTile extends SimpleTile { return this._call && this._call.hasJoined; } - get errorViewModel() { - return this._errorViewModel; - } - get label() { if (this._call) { if (this._call.hasJoined) { @@ -78,25 +73,21 @@ export class CallTile extends SimpleTile { } async join() { - if (this.canJoin) { - try { + await this.logAndCatch("Call.join", async log => { + if (this.canJoin) { const stream = await this.platform.mediaDevices.getMediaTracks(false, true); const localMedia = new LocalMedia().withUserMedia(stream); await this._call.join(localMedia); - } catch (error) { - this._reportError(error); } - } + }); } async leave() { - if (this.canLeave) { - try { - this._call.leave(); - } catch (err) { - this._reportError(err); + await this.logAndCatch("Call.leave", async log => { + if (this.canLeave) { + await this._call.leave(); } - } + }); } dispose() { @@ -104,12 +95,4 @@ export class CallTile extends SimpleTile { this._callSubscription = this._callSubscription(); } } - - _reportError(error) { - this._errorViewModel = new ErrorViewModel(this.childOptions({error, onClose: () => { - this._errorViewModel = undefined; - this.emitChange("errorViewModel"); - }})); - this.emitChange("errorViewModel"); - } } diff --git a/src/domain/session/room/timeline/tiles/SimpleTile.js b/src/domain/session/room/timeline/tiles/SimpleTile.js index 18e6ba17..aa26da81 100644 --- a/src/domain/session/room/timeline/tiles/SimpleTile.js +++ b/src/domain/session/room/timeline/tiles/SimpleTile.js @@ -15,10 +15,10 @@ limitations under the License. */ import {UpdateAction} from "../UpdateAction.js"; -import {ViewModel} from "../../../../ViewModel"; +import {ErrorReportViewModel} from "../../../../ErrorReportViewModel"; import {SendStatus} from "../../../../../matrix/room/sending/PendingEvent.js"; -export class SimpleTile extends ViewModel { +export class SimpleTile extends ErrorReportViewModel { constructor(entry, options) { super(options); this._entry = entry; From 80be2b74570df5e4d78cdb160ae5271ab07a5eb0 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 13 Jan 2023 14:20:29 +0100 Subject: [PATCH 237/323] fix missing import --- src/domain/session/room/RoomViewModel.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index a5dfdc8f..c864222e 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -21,6 +21,7 @@ import {CallViewModel} from "./CallViewModel" import {PickMapObservableValue} from "../../../observable/value/PickMapObservableValue"; import {avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../avatar"; import {ErrorReportViewModel} from "../../ErrorReportViewModel"; +import {ViewModel} from "../../ViewModel"; import {imageToInfo} from "../common.js"; import {LocalMedia} from "../../../matrix/calls/LocalMedia"; import { ErrorViewModel } from "../../ErrorViewModel"; From 29a7b0451e9b6f4167ae36298841677dd87a6a5f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 13 Jan 2023 14:20:33 +0100 Subject: [PATCH 238/323] prevent errors in promises from being uncaught by returning a promise that has the error swallowed --- src/domain/ErrorReportViewModel.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/domain/ErrorReportViewModel.ts b/src/domain/ErrorReportViewModel.ts index eb6ed0aa..68fe580c 100644 --- a/src/domain/ErrorReportViewModel.ts +++ b/src/domain/ErrorReportViewModel.ts @@ -49,9 +49,12 @@ export class ErrorReportViewModel extends ViewModel protected logAndCatch(labelOrValues: LabelOrValues, callback: LogCallback, errorValue: T = undefined as unknown as T): T { try { - const result = this.logger.run(labelOrValues, callback); + let result = this.logger.run(labelOrValues, callback); if (result instanceof Promise) { - result.catch(err => this.reportError(err)); + result = result.catch(err => { + this.reportError(err); + return errorValue; + }) as unknown as T; } return result; } catch (err) { From cec9f6b691b7473c98d6cbaf8cb2b86cf8494e70 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 13 Jan 2023 21:26:01 +0100 Subject: [PATCH 239/323] WIP for docs on error handling --- doc/error-handling.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/error-handling.md diff --git a/doc/error-handling.md b/doc/error-handling.md new file mode 100644 index 00000000..5e136735 --- /dev/null +++ b/doc/error-handling.md @@ -0,0 +1,13 @@ +# Error handling + +Ideally, every error that is unexpected and can't be automatically recovered from without degrading the experience is shown in the UI. This is the task of the view model, and you can use `ErrorReportViewModel`, a dedicated base view model class for this purpose. It exposes a child view model, `ErrorViewModel`, when `reportError` is called which can be paired with `ErrorView` in the view to present an error message from which debug logs can also be sent. + +Methods on classes from the `matrix` layer can often throw errors and those errors should be caught in the view model and reported with `reportError`. As a convenience method, there is also `logAndCatch` which calls a callback within a log item and also a try catch that reports the error. + +## Sync errors + +There are some errors that are throw during background processes though, most notably the sync loop. These processes are not triggered by the view model directly, and hence there is not always a method call they can wrap in a try/catch. For this, there is the `ErrorBoundary` utility class. Since almost all aspects of the client can be updated through the sync loop, it is not too helpful if there is only one try/catch around the whole sync and we stop sync if something goes wrong. + +Instead, it's more helpful to split up the error handling into different scopes, where errors are stored and not rethrown when leaving the scope. One example is to have a scope per room. In this way, we can isolate an error occuring during sync to a specific room, and report it in the UI of that room. + +There is an extra complication though. The `writeSync` sync lifecycle step should not swallow any errors, or data loss can occur. This is because the whole `writeSync` lifecycle step is writes all changes (for all rooms, the session, ...) for a sync response in one transaction. This includes the sync token. So if there is an error in `writeSync` of a given room preventing storing all changes the sync response would cause, \ No newline at end of file From f853871722254a460855369ed2cc19a8ee868cb0 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 16 Jan 2023 10:11:04 +0100 Subject: [PATCH 240/323] some rewording on error handling doc --- doc/error-handling.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/error-handling.md b/doc/error-handling.md index 5e136735..21c67ee5 100644 --- a/doc/error-handling.md +++ b/doc/error-handling.md @@ -1,13 +1,13 @@ # Error handling -Ideally, every error that is unexpected and can't be automatically recovered from without degrading the experience is shown in the UI. This is the task of the view model, and you can use `ErrorReportViewModel`, a dedicated base view model class for this purpose. It exposes a child view model, `ErrorViewModel`, when `reportError` is called which can be paired with `ErrorView` in the view to present an error message from which debug logs can also be sent. +Ideally, every error that is unexpected and can't be automatically recovered from without degrading the experience is shown in the UI. This is the task of the view model, and you can use `ErrorReportViewModel` for this purpose, a dedicated base view model class. It exposes a child view model, `ErrorViewModel`, when `reportError` is called which can be paired with `ErrorView` in the view to present an error message from which debug logs can also be sent. -Methods on classes from the `matrix` layer can often throw errors and those errors should be caught in the view model and reported with `reportError`. As a convenience method, there is also `logAndCatch` which calls a callback within a log item and also a try catch that reports the error. +Methods on classes from the `matrix` layer can often throw errors and those errors should be caught in the view model and reported with `reportError`. As a convenience method, there is also `logAndCatch` when inheriting from `ErrorReportViewModel` which combines logging and error reporting; it calls a callback within a log item and also a try catch that reports the error. -## Sync errors +## Sync errors & ErrorBoundary -There are some errors that are throw during background processes though, most notably the sync loop. These processes are not triggered by the view model directly, and hence there is not always a method call they can wrap in a try/catch. For this, there is the `ErrorBoundary` utility class. Since almost all aspects of the client can be updated through the sync loop, it is not too helpful if there is only one try/catch around the whole sync and we stop sync if something goes wrong. +There are some errors that are thrown during background processes though, most notably the sync loop. These processes are not triggered by the view model directly, and hence there is not always a method call they can wrap in a try/catch. For this, there is the `ErrorBoundary` utility class. Since almost all aspects of the client can be updated through the sync loop, it is also not too helpful if there is only one try/catch around the whole sync and we stop sync if something goes wrong. Instead, it's more helpful to split up the error handling into different scopes, where errors are stored and not rethrown when leaving the scope. One example is to have a scope per room. In this way, we can isolate an error occuring during sync to a specific room, and report it in the UI of that room. -There is an extra complication though. The `writeSync` sync lifecycle step should not swallow any errors, or data loss can occur. This is because the whole `writeSync` lifecycle step is writes all changes (for all rooms, the session, ...) for a sync response in one transaction. This includes the sync token. So if there is an error in `writeSync` of a given room preventing storing all changes the sync response would cause, \ No newline at end of file +There is an extra complication though. The `writeSync` sync lifecycle step should not swallow any errors, or data loss can occur. This is because the whole `writeSync` lifecycle step writes all changes (for all rooms, the session, ...) for a sync response in one transaction (including the sync token), and aborts the transaction and stops sync if there is an error thrown during this step. So if there is an error in `writeSync` of a given room, it's fair to assume not all changes it was planning to write were passed to the transaction, as it got interrupted by the exception. Therefore, if we would swallow the error, data loss can occur as we'd not get another chance to write these changes to disk as we would have advanced the sync token. Therefore, code in the `writeSync` lifecycle step should be written defensively but always throw. From 1de92af2eb4622ae14a7b51816e1db99add2f3c5 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 17 Jan 2023 08:25:08 +0000 Subject: [PATCH 241/323] Update src/domain/session/room/CallViewModel.ts Co-authored-by: R Midhun Suresh --- src/domain/session/room/CallViewModel.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index ee0ad502..98473888 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -21,7 +21,7 @@ import {getStreamVideoTrack, getStreamAudioTrack} from "../../../matrix/calls/co import {avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../avatar"; import {EventObservableValue} from "../../../observable/value/EventObservableValue"; import {ObservableValueMap} from "../../../observable/map/ObservableValueMap"; -import { ErrorViewModel } from "../../ErrorViewModel"; +import {ErrorViewModel} from "../../ErrorViewModel"; import type {Room} from "../../../matrix/room/Room"; import type {GroupCall} from "../../../matrix/calls/group/GroupCall"; import type {Member} from "../../../matrix/calls/group/Member"; From 7d80fbda4cadc14b96d4a46f7958f747d79fb2e7 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 17 Jan 2023 08:25:16 +0000 Subject: [PATCH 242/323] Update src/domain/session/room/CallViewModel.ts Co-authored-by: R Midhun Suresh --- src/domain/session/room/CallViewModel.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 98473888..df2da3c6 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -30,7 +30,7 @@ import type {BaseObservableList} from "../../../observable/list/BaseObservableLi import type {BaseObservableValue} from "../../../observable/value/BaseObservableValue"; import type {Stream} from "../../../platform/types/MediaDevices"; import type {MediaRepository} from "../../../matrix/net/MediaRepository"; -import type { Session } from "../../../matrix/Session"; +import type {Session} from "../../../matrix/Session"; type Options = BaseOptions & { call: GroupCall, From f421cdd4f2e2c400c243b683ce2a623b5219010b Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 17 Jan 2023 08:25:33 +0000 Subject: [PATCH 243/323] Update src/domain/session/room/RoomViewModel.js Co-authored-by: R Midhun Suresh --- src/domain/session/room/RoomViewModel.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index c864222e..421041e9 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -24,7 +24,7 @@ import {ErrorReportViewModel} from "../../ErrorReportViewModel"; import {ViewModel} from "../../ViewModel"; import {imageToInfo} from "../common.js"; import {LocalMedia} from "../../../matrix/calls/LocalMedia"; -import { ErrorViewModel } from "../../ErrorViewModel"; +import {ErrorViewModel} from "../../ErrorViewModel"; // TODO: remove fallback so default isn't included in bundle for SDK users that have their custom tileClassForEntry // this is a breaking SDK change though to make this option mandatory import {tileClassForEntry as defaultTileClassForEntry} from "./timeline/tiles/index"; From cc653884a515e04d38d37f814b9447f38659fb06 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 17 Jan 2023 09:28:23 +0100 Subject: [PATCH 244/323] remove getter that is now in parent class --- src/domain/session/room/RoomViewModel.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 421041e9..0c815949 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -141,10 +141,6 @@ export class RoomViewModel extends ErrorReportViewModel { get timelineViewModel() { return this._timelineVM; } get isEncrypted() { return this._room.isEncrypted; } - get errorViewModel() { - return this._errorViewModel; - } - get avatarLetter() { return avatarInitials(this.name); } From a2c44484b21a96dfcc47121fd2af936a08b29cb2 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 17 Jan 2023 09:29:03 +0100 Subject: [PATCH 245/323] newline --- src/domain/rageshake.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/rageshake.ts b/src/domain/rageshake.ts index 08e82d8c..cc8aec07 100644 --- a/src/domain/rageshake.ts +++ b/src/domain/rageshake.ts @@ -91,4 +91,4 @@ export async function submitLogsFromSessionToDefaultServer(session: Session, pla bugReportEndpointUrl, platform.request ); -} \ No newline at end of file +} From 0dbb7d4e50d7f947cd1d2b4e978c91b988a16c9d Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 17 Jan 2023 11:37:49 +0100 Subject: [PATCH 246/323] use logAndCatch in RoomViewModel, everything reporting errors also logs --- src/domain/session/room/RoomViewModel.js | 127 +++++++++++------------ 1 file changed, 61 insertions(+), 66 deletions(-) diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 0c815949..a3542ebb 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -74,9 +74,9 @@ export class RoomViewModel extends ErrorReportViewModel { } async load() { - this._room.on("change", this._onRoomChange); - try { - const timeline = await this._room.openTimeline(); + this.logAndCatch("Room.load", async log => { + this._room.on("change", this._onRoomChange); + const timeline = await this._room.openTimeline(log); this._tileOptions = this.childOptions({ session: this.getOption("session"), roomVM: this, @@ -88,13 +88,11 @@ export class RoomViewModel extends ErrorReportViewModel { timeline, }))); this.emitChange("timelineViewModel"); - } catch (error) { - this.reportError(error); - } - this._clearUnreadAfterDelay(); + await this._clearUnreadAfterDelay(log); + }); } - async _clearUnreadAfterDelay() { + async _clearUnreadAfterDelay(log) { if (this._room.isArchived || this._clearUnreadTimout) { return; } @@ -104,7 +102,9 @@ export class RoomViewModel extends ErrorReportViewModel { await this._room.clearUnread(); this._clearUnreadTimout = null; } catch (err) { - if (err.name !== "AbortError") { + if (err.name === "AbortError") { + log.set("clearUnreadCancelled", true); + } else { throw err; } } @@ -190,62 +190,61 @@ export class RoomViewModel extends ErrorReportViewModel { } } - async _sendMessage(message, replyingTo) { - if (!this._room.isArchived && message) { - try { + _sendMessage(message, replyingTo) { + return this.logAndCatch("sendMessage", async log => { + let success = false; + if (!this._room.isArchived && message) { let msgtype = "m.text"; if (message.startsWith("/me ")) { message = message.substr(4).trim(); msgtype = "m.emote"; } if (replyingTo) { + log.set("replyingTo", replyingTo.eventId); + // TODO: reply should not send? it should just return the content for the reply and we send it ourselves await replyingTo.reply(msgtype, message); } else { - await this._room.sendEvent("m.room.message", {msgtype, body: message}); + await this._room.sendEvent("m.room.message", {msgtype, body: message}, undefined, log); } - } catch (error) { - this.reportError(error); - return false; + success = true; } - return true; - } - return false; + log.set("success", success); + return success; + }, false); } - async _pickAndSendFile() { - try { + _pickAndSendFile() { + return this.logAndCatch("sendFile", async log => { const file = await this.platform.openFile(); if (!file) { + log.set("cancelled", true); return; } - return this._sendFile(file); - } catch (err) { - console.error(err); - } + return this._sendFile(file, log); + }); } - async _sendFile(file) { + async _sendFile(file, log) { const content = { body: file.name, msgtype: "m.file" }; await this._room.sendEvent("m.room.message", content, { "url": this._room.createAttachment(file.blob, file.name) - }); + }, log); } - async _pickAndSendVideo() { - try { + _pickAndSendVideo() { + return this.logAndCatch("sendVideo", async log => { if (!this.platform.hasReadPixelPermission()) { - alert("Please allow canvas image data access, so we can scale your images down."); - return; + throw new Error("Please allow canvas image data access, so we can scale your images down."); } const file = await this.platform.openFile("video/*"); if (!file) { return; } if (!file.blob.mimeType.startsWith("video/")) { - return this._sendFile(file); + return this._sendFile(file, log); } let video; try { @@ -273,24 +272,23 @@ export class RoomViewModel extends ErrorReportViewModel { content.info.thumbnail_info = imageToInfo(thumbnail); attachments["info.thumbnail_url"] = this._room.createAttachment(thumbnail.blob, file.name); - await this._room.sendEvent("m.room.message", content, attachments); - } catch (error) { - this.reportError(error); - } + await this._room.sendEvent("m.room.message", content, attachments, log); + }); } async _pickAndSendPicture() { - try { + this.logAndCatch("sendPicture", async log => { if (!this.platform.hasReadPixelPermission()) { alert("Please allow canvas image data access, so we can scale your images down."); return; } const file = await this.platform.openFile("image/*"); if (!file) { + log.set("cancelled", true); return; } if (!file.blob.mimeType.startsWith("image/")) { - return this._sendFile(file); + return this._sendFile(file, log); } let image = await this.platform.loadImage(file.blob); const limit = await this.platform.settingsStorage.getInt("sentImageSizeLimit"); @@ -313,10 +311,8 @@ export class RoomViewModel extends ErrorReportViewModel { attachments["info.thumbnail_url"] = this._room.createAttachment(thumbnail.blob, file.name); } - await this._room.sendEvent("m.room.message", content, attachments); - } catch (error) { - this.reportError(error); - } + await this._room.sendEvent("m.room.message", content, attachments, log); + }); } get room() { @@ -344,30 +340,29 @@ export class RoomViewModel extends ErrorReportViewModel { } } - async startCall() { - let localMedia; - try { - const stream = await this.platform.mediaDevices.getMediaTracks(false, true); - localMedia = new LocalMedia().withUserMedia(stream); - } catch (err) { - this.reportError(new Error(`Could not get local audio and/or video stream: ${err.message}`)); - return; - } - const session = this.getOption("session"); - let call; - try { - // this will set the callViewModel above as a call will be added to callHandler.calls - call = await session.callHandler.createCall(this._room.id, "m.video", "A call " + Math.round(this.platform.random() * 100)); - } catch (err) { - this.reportError(new Error(`Could not create call: ${err.message}`)); - return; - } - try { - await call.join(localMedia); - } catch (err) { - this.reportError(new Error(`Could not join call: ${err.message}`)); - return; - } + startCall() { + return this.logAndCatch("startCall", async log => { + let localMedia; + try { + const stream = await this.platform.mediaDevices.getMediaTracks(false, true); + localMedia = new LocalMedia().withUserMedia(stream); + } catch (err) { + throw new Error(`Could not get local audio and/or video stream: ${err.message}`); + } + const session = this.getOption("session"); + let call; + try { + // this will set the callViewModel above as a call will be added to callHandler.calls + call = await session.callHandler.createCall(this._room.id, "m.video", "A call " + Math.round(this.platform.random() * 100)); + } catch (err) { + throw new Error(`Could not create call: ${err.message}`); + } + try { + await call.join(localMedia); + } catch (err) { + throw new Error(`Could not join call: ${err.message}`); + } + }); } } From e33209b747140c51d83a585e01358251adb70f5a Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 17 Jan 2023 12:23:20 +0100 Subject: [PATCH 247/323] start logging in view model and pass it on to model methods (calls+room) --- src/domain/session/room/CallViewModel.ts | 4 +- src/domain/session/room/RoomViewModel.js | 30 ++-- .../session/room/timeline/tiles/CallTile.js | 9 +- src/matrix/calls/CallHandler.ts | 141 ++++++++++-------- src/matrix/calls/group/GroupCall.ts | 95 ++++++------ 5 files changed, 151 insertions(+), 128 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index df2da3c6..dd942878 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -102,9 +102,9 @@ export class CallViewModel extends ErrorReportViewModel { } async hangup() { - this.logAndCatch("Call.hangup", async log => { + this.logAndCatch("CallViewModel.hangup", async log => { if (this.call.hasJoined) { - await this.call.leave(); + await this.call.leave(log); } }); } diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index a3542ebb..5a29bd85 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -24,7 +24,6 @@ import {ErrorReportViewModel} from "../../ErrorReportViewModel"; import {ViewModel} from "../../ViewModel"; import {imageToInfo} from "../common.js"; import {LocalMedia} from "../../../matrix/calls/LocalMedia"; -import {ErrorViewModel} from "../../ErrorViewModel"; // TODO: remove fallback so default isn't included in bundle for SDK users that have their custom tileClassForEntry // this is a breaking SDK change though to make this option mandatory import {tileClassForEntry as defaultTileClassForEntry} from "./timeline/tiles/index"; @@ -74,7 +73,7 @@ export class RoomViewModel extends ErrorReportViewModel { } async load() { - this.logAndCatch("Room.load", async log => { + this.logAndCatch("RoomViewModel.load", async log => { this._room.on("change", this._onRoomChange); const timeline = await this._room.openTimeline(log); this._tileOptions = this.childOptions({ @@ -99,7 +98,7 @@ export class RoomViewModel extends ErrorReportViewModel { this._clearUnreadTimout = this.clock.createTimeout(2000); try { await this._clearUnreadTimout.elapsed(); - await this._room.clearUnread(); + await this._room.clearUnread(log); this._clearUnreadTimout = null; } catch (err) { if (err.name === "AbortError") { @@ -111,7 +110,9 @@ export class RoomViewModel extends ErrorReportViewModel { } focus() { - this._clearUnreadAfterDelay(); + this.logAndCatch("RoomViewModel.focus", async log => { + this._clearUnreadAfterDelay(log); + }); } dispose() { @@ -191,7 +192,7 @@ export class RoomViewModel extends ErrorReportViewModel { } _sendMessage(message, replyingTo) { - return this.logAndCatch("sendMessage", async log => { + return this.logAndCatch("RoomViewModel.sendMessage", async log => { let success = false; if (!this._room.isArchived && message) { let msgtype = "m.text"; @@ -214,7 +215,7 @@ export class RoomViewModel extends ErrorReportViewModel { } _pickAndSendFile() { - return this.logAndCatch("sendFile", async log => { + return this.logAndCatch("RoomViewModel.sendFile", async log => { const file = await this.platform.openFile(); if (!file) { log.set("cancelled", true); @@ -235,7 +236,7 @@ export class RoomViewModel extends ErrorReportViewModel { } _pickAndSendVideo() { - return this.logAndCatch("sendVideo", async log => { + return this.logAndCatch("RoomViewModel.sendVideo", async log => { if (!this.platform.hasReadPixelPermission()) { throw new Error("Please allow canvas image data access, so we can scale your images down."); } @@ -277,7 +278,7 @@ export class RoomViewModel extends ErrorReportViewModel { } async _pickAndSendPicture() { - this.logAndCatch("sendPicture", async log => { + this.logAndCatch("RoomViewModel.sendPicture", async log => { if (!this.platform.hasReadPixelPermission()) { alert("Please allow canvas image data access, so we can scale your images down."); return; @@ -341,7 +342,8 @@ export class RoomViewModel extends ErrorReportViewModel { } startCall() { - return this.logAndCatch("startCall", async log => { + return this.logAndCatch("RoomViewModel.startCall", async log => { + log.set("roomId", this._room.id); let localMedia; try { const stream = await this.platform.mediaDevices.getMediaTracks(false, true); @@ -353,12 +355,18 @@ export class RoomViewModel extends ErrorReportViewModel { let call; try { // this will set the callViewModel above as a call will be added to callHandler.calls - call = await session.callHandler.createCall(this._room.id, "m.video", "A call " + Math.round(this.platform.random() * 100)); + call = await session.callHandler.createCall( + this._room.id, + "m.video", + "A call " + Math.round(this.platform.random() * 100), + undefined, + log + ); } catch (err) { throw new Error(`Could not create call: ${err.message}`); } try { - await call.join(localMedia); + await call.join(localMedia, log); } catch (err) { throw new Error(`Could not join call: ${err.message}`); } diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index 44c39eef..c792496e 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -16,7 +16,6 @@ limitations under the License. import {SimpleTile} from "./SimpleTile.js"; import {LocalMedia} from "../../../../../matrix/calls/LocalMedia"; -import {ErrorViewModel} from "../../../../ErrorViewModel" // TODO: timeline entries for state events with the same state key and type // should also update previous entries in the timeline, so we can update the name of the call, whether it is terminated, etc ... @@ -73,19 +72,19 @@ export class CallTile extends SimpleTile { } async join() { - await this.logAndCatch("Call.join", async log => { + await this.logAndCatch("CallTile.join", async log => { if (this.canJoin) { const stream = await this.platform.mediaDevices.getMediaTracks(false, true); const localMedia = new LocalMedia().withUserMedia(stream); - await this._call.join(localMedia); + await this._call.join(localMedia, log); } }); } async leave() { - await this.logAndCatch("Call.leave", async log => { + await this.logAndCatch("CallTile.leave", async log => { if (this.canLeave) { - await this._call.leave(); + await this._call.leave(log); } }); } diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 2b914a2f..51afcda9 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -65,16 +65,26 @@ export class CallHandler implements RoomStateHandler { }); } - async loadCalls(intent: CallIntent = CallIntent.Ring) { - const txn = await this._getLoadTxn(); - const callEntries = await txn.calls.getByIntent(intent); - this._loadCallEntries(callEntries, txn); + loadCalls(intent?: CallIntent, log?: ILogItem) { + return this.options.logger.wrapOrRun(log, "CallHandler.loadCalls", async log => { + if (!intent) { + intent = CallIntent.Ring; + } + log.set("intent", intent); + const txn = await this._getLoadTxn(); + const callEntries = await txn.calls.getByIntent(intent); + await this._loadCallEntries(callEntries, txn, log); + }); } - async loadCallsForRoom(intent: CallIntent, roomId: string) { - const txn = await this._getLoadTxn(); - const callEntries = await txn.calls.getByIntentAndRoom(intent, roomId); - this._loadCallEntries(callEntries, txn); + loadCallsForRoom(intent: CallIntent, roomId: string, log?: ILogItem) { + return this.options.logger.wrapOrRun(log, "CallHandler.loadCallsForRoom", async log => { + log.set("intent", intent); + log.set("roomId", roomId); + const txn = await this._getLoadTxn(); + const callEntries = await txn.calls.getByIntentAndRoom(intent, roomId); + await this._loadCallEntries(callEntries, txn, log); + }); } private async _getLoadTxn(): Promise { @@ -86,68 +96,71 @@ export class CallHandler implements RoomStateHandler { return txn; } - private async _loadCallEntries(callEntries: CallEntry[], txn: Transaction): Promise { - return this.options.logger.run({l: "loading calls", t: CALL_LOG_TYPE}, async log => { - log.set("entries", callEntries.length); - await Promise.all(callEntries.map(async callEntry => { - if (this._calls.get(callEntry.callId)) { - return; + private async _loadCallEntries(callEntries: CallEntry[], txn: Transaction, log: ILogItem): Promise { + log.set("entries", callEntries.length); + await Promise.all(callEntries.map(async callEntry => { + if (this._calls.get(callEntry.callId)) { + return; + } + const event = await txn.roomState.get(callEntry.roomId, EventType.GroupCall, callEntry.callId); + if (event) { + const call = new GroupCall(event.event.state_key, false, event.event.content, event.roomId, this.groupCallOptions); + this._calls.set(call.id, call); + } + })); + const roomIds = Array.from(new Set(callEntries.map(e => e.roomId))); + await Promise.all(roomIds.map(async roomId => { + // TODO: don't load all members until we need them + const callsMemberEvents = await txn.roomState.getAllForType(roomId, EventType.GroupCallMember); + await Promise.all(callsMemberEvents.map(async entry => { + const userId = entry.event.sender; + const roomMemberState = await txn.roomState.get(roomId, MEMBER_EVENT_TYPE, userId); + let roomMember; + if (roomMemberState) { + roomMember = RoomMember.fromMemberEvent(roomMemberState.event); } - const event = await txn.roomState.get(callEntry.roomId, EventType.GroupCall, callEntry.callId); - if (event) { - const call = new GroupCall(event.event.state_key, false, event.event.content, event.roomId, this.groupCallOptions); - this._calls.set(call.id, call); + if (!roomMember) { + // we'll be missing the member here if we received a call and it's members + // as pre-gap state and the members weren't active in the timeline we got. + roomMember = RoomMember.fromUserId(roomId, userId, "join"); } + this.handleCallMemberEvent(entry.event, roomMember, roomId, log); })); - const roomIds = Array.from(new Set(callEntries.map(e => e.roomId))); - await Promise.all(roomIds.map(async roomId => { - // TODO: don't load all members until we need them - const callsMemberEvents = await txn.roomState.getAllForType(roomId, EventType.GroupCallMember); - await Promise.all(callsMemberEvents.map(async entry => { - const userId = entry.event.sender; - const roomMemberState = await txn.roomState.get(roomId, MEMBER_EVENT_TYPE, userId); - let roomMember; - if (roomMemberState) { - roomMember = RoomMember.fromMemberEvent(roomMemberState.event); - } - if (!roomMember) { - // we'll be missing the member here if we received a call and it's members - // as pre-gap state and the members weren't active in the timeline we got. - roomMember = RoomMember.fromUserId(roomId, userId, "join"); - } - this.handleCallMemberEvent(entry.event, roomMember, roomId, log); - })); - })); - log.set("newSize", this._calls.size); - }); + })); + log.set("newSize", this._calls.size); } - async createCall(roomId: string, type: "m.video" | "m.voice", name: string, intent: CallIntent = CallIntent.Ring): Promise { - const call = new GroupCall(makeId("conf-"), true, { - "m.name": name, - "m.intent": intent - }, roomId, this.groupCallOptions); - this._calls.set(call.id, call); + createCall(roomId: string, type: "m.video" | "m.voice", name: string, intent?: CallIntent, log?: ILogItem): Promise { + return this.options.logger.wrapOrRun(log, "CallHandler.createCall", async log => { + if (!intent) { + intent = CallIntent.Ring; + } + const call = new GroupCall(makeId("conf-"), true, { + "m.name": name, + "m.intent": intent + }, roomId, this.groupCallOptions); + this._calls.set(call.id, call); - try { - await call.create(type); - // store call info so it will ring again when reopening the app - const txn = await this.options.storage.readWriteTxn([this.options.storage.storeNames.calls]); - txn.calls.add({ - intent: call.intent, - callId: call.id, - timestamp: this.options.clock.now(), - roomId: roomId - }); - await txn.complete(); - } catch (err) { - //if (err.name === "ConnectionError") { - // if we're offline, give up and remove the call again - this._calls.remove(call.id); - //} - throw err; - } - return call; + try { + await call.create(type, log); + // store call info so it will ring again when reopening the app + const txn = await this.options.storage.readWriteTxn([this.options.storage.storeNames.calls]); + txn.calls.add({ + intent: call.intent, + callId: call.id, + timestamp: this.options.clock.now(), + roomId: roomId + }); + await txn.complete(); + } catch (err) { + //if (err.name === "ConnectionError") { + // if we're offline, give up and remove the call again + this._calls.remove(call.id); + //} + throw err; + } + return call; + }); } get calls(): BaseObservableMap { return this._calls; } diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 53a87d5f..d700f2a0 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -162,45 +162,48 @@ export class GroupCall extends EventEmitter<{change: never}> { return this.errorBoundary.error; } - async join(localMedia: LocalMedia): Promise { - if (this._state !== GroupCallState.Created || this.joinedData) { - return; - } - const logItem = this.options.logger.child({ - l: "answer call", - t: CALL_LOG_TYPE, - id: this.id, - ownSessionId: this.options.sessionId - }); - const turnServer = await this.options.turnServerSource.getSettings(logItem); - const membersLogItem = logItem.child("member connections"); - const localMuteSettings = new MuteSettings(); - localMuteSettings.updateTrackInfo(localMedia.userMedia); - const localPreviewMedia = localMedia.asPreview(); - const joinedData = new JoinedData( - logItem, - membersLogItem, - localMedia, - localPreviewMedia, - localMuteSettings, - turnServer - ); - this.joinedData = joinedData; - await joinedData.logItem.wrap("join", async log => { - this._state = GroupCallState.Joining; - this.emitChange(); - await log.wrap("update member state", async log => { - const memberContent = await this._createMemberPayload(true); - log.set("payload", memberContent); - // send m.call.member state event - const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCallMember, this.options.ownUserId, memberContent, {log}); - await request.response(); - this.emitChange(); - }); - // send invite to all members that are < my userId - for (const [,member] of this._members) { - this.connectToMember(member, joinedData, log); + join(localMedia: LocalMedia, log?: ILogItem): Promise { + return this.options.logger.wrapOrRun(log, "Call.join", async joinLog => { + if (this._state !== GroupCallState.Created || this.joinedData) { + return; } + const logItem = this.options.logger.child({ + l: "Call.connection", + t: CALL_LOG_TYPE, + id: this.id, + ownSessionId: this.options.sessionId + }); + const turnServer = await this.options.turnServerSource.getSettings(logItem); + const membersLogItem = logItem.child("member connections"); + const localMuteSettings = new MuteSettings(); + localMuteSettings.updateTrackInfo(localMedia.userMedia); + const localPreviewMedia = localMedia.asPreview(); + const joinedData = new JoinedData( + logItem, + membersLogItem, + localMedia, + localPreviewMedia, + localMuteSettings, + turnServer + ); + this.joinedData = joinedData; + await joinedData.logItem.wrap("join", async log => { + joinLog.refDetached(log); + this._state = GroupCallState.Joining; + this.emitChange(); + await log.wrap("update member state", async log => { + const memberContent = await this._createMemberPayload(true); + log.set("payload", memberContent); + // send m.call.member state event + const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCallMember, this.options.ownUserId, memberContent, {log}); + await request.response(); + this.emitChange(); + }); + // send invite to all members that are < my userId + for (const [,member] of this._members) { + this.connectToMember(member, joinedData, log); + } + }); }); } @@ -263,12 +266,12 @@ export class GroupCall extends EventEmitter<{change: never}> { return this._state === GroupCallState.Joining || this._state === GroupCallState.Joined; } - async leave(): Promise { - const {joinedData} = this; - if (!joinedData) { - return; - } - await joinedData.logItem.wrap("leave", async log => { + async leave(log?: ILogItem): Promise { + await this.options.logger.wrapOrRun(log, "Call.leave", async log => { + const {joinedData} = this; + if (!joinedData) { + return; + } try { joinedData.renewMembershipTimeout?.dispose(); joinedData.renewMembershipTimeout = undefined; @@ -310,8 +313,8 @@ export class GroupCall extends EventEmitter<{change: never}> { } /** @internal */ - create(type: "m.video" | "m.voice", log?: ILogItem): Promise { - return this.options.logger.wrapOrRun(log, {l: "create call", t: CALL_LOG_TYPE}, async log => { + create(type: "m.video" | "m.voice", log: ILogItem): Promise { + return log.wrap({l: "create call", t: CALL_LOG_TYPE}, async log => { if (this._state !== GroupCallState.Fledgling) { return; } From dfaaf6d23470b4d74707fba84ef5939be22213f0 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 17 Jan 2023 12:32:05 +0100 Subject: [PATCH 248/323] cleanup reply code a bit, have only 1 path to send message --- doc/impl-thoughts/RELATIONS.md | 2 +- src/domain/session/room/RoomViewModel.js | 7 ++++--- src/domain/session/room/timeline/tiles/BaseMessageTile.js | 4 ++-- src/matrix/room/timeline/entries/BaseEventEntry.js | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/doc/impl-thoughts/RELATIONS.md b/doc/impl-thoughts/RELATIONS.md index 5d91c28e..ac4e43ce 100644 --- a/doc/impl-thoughts/RELATIONS.md +++ b/doc/impl-thoughts/RELATIONS.md @@ -237,7 +237,7 @@ room.sendEvent(eventEntry.eventType, replacement); ## Replies ```js -const reply = eventEntry.reply({}); +const reply = eventEntry.createReplyContent({}); room.sendEvent("m.room.message", reply); ``` diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 5a29bd85..8c551172 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -200,13 +200,14 @@ export class RoomViewModel extends ErrorReportViewModel { message = message.substr(4).trim(); msgtype = "m.emote"; } + let content; if (replyingTo) { log.set("replyingTo", replyingTo.eventId); - // TODO: reply should not send? it should just return the content for the reply and we send it ourselves - await replyingTo.reply(msgtype, message); + content = await replyingTo.createReplyContent(msgtype, message); } else { - await this._room.sendEvent("m.room.message", {msgtype, body: message}, undefined, log); + content = {msgtype, body: message}; } + await this._room.sendEvent("m.room.message", content, undefined, log); success = true; } log.set("success", success); diff --git a/src/domain/session/room/timeline/tiles/BaseMessageTile.js b/src/domain/session/room/timeline/tiles/BaseMessageTile.js index 5a28e1b3..44e0c5b0 100644 --- a/src/domain/session/room/timeline/tiles/BaseMessageTile.js +++ b/src/domain/session/room/timeline/tiles/BaseMessageTile.js @@ -146,8 +146,8 @@ export class BaseMessageTile extends SimpleTile { this._roomVM.startReply(this._entry); } - reply(msgtype, body, log = null) { - return this._room.sendEvent("m.room.message", this._entry.reply(msgtype, body), null, log); + createReplyContent(msgtype, body) { + return this._entry.createReplyContent(msgtype, body); } redact(reason, log) { diff --git a/src/matrix/room/timeline/entries/BaseEventEntry.js b/src/matrix/room/timeline/entries/BaseEventEntry.js index 44fdcaec..39698557 100644 --- a/src/matrix/room/timeline/entries/BaseEventEntry.js +++ b/src/matrix/room/timeline/entries/BaseEventEntry.js @@ -181,7 +181,7 @@ export class BaseEventEntry extends BaseEntry { return createAnnotation(this.id, key); } - reply(msgtype, body) { + createReplyContent(msgtype, body) { return createReplyContent(this, msgtype, body); } From bf9c868c8bc8f71cb9ed99301ccd0f580bcfb98d Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 17 Jan 2023 17:18:07 +0100 Subject: [PATCH 249/323] make it clearer that logAndCatch is probably what you want --- doc/error-handling.md | 6 ++++-- src/domain/ErrorReportViewModel.ts | 7 +++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/doc/error-handling.md b/doc/error-handling.md index 21c67ee5..ac2c4e59 100644 --- a/doc/error-handling.md +++ b/doc/error-handling.md @@ -2,12 +2,14 @@ Ideally, every error that is unexpected and can't be automatically recovered from without degrading the experience is shown in the UI. This is the task of the view model, and you can use `ErrorReportViewModel` for this purpose, a dedicated base view model class. It exposes a child view model, `ErrorViewModel`, when `reportError` is called which can be paired with `ErrorView` in the view to present an error message from which debug logs can also be sent. -Methods on classes from the `matrix` layer can often throw errors and those errors should be caught in the view model and reported with `reportError`. As a convenience method, there is also `logAndCatch` when inheriting from `ErrorReportViewModel` which combines logging and error reporting; it calls a callback within a log item and also a try catch that reports the error. +Methods on classes from the `matrix` layer can often throw errors and those errors should be caught in the view model and reported to the UI. When inheriting from `ErrorReportViewModel`, there is the low-level `reportError` method, but typically you'd use the convenience method `logAndCatch`. The latter makes it easy to get both error handlikng and logging right. You would typically use `logAndCatch` for every public method in the view model (e.g methods called from the view or from the parent view model). It calls a callback within a log item and also a try catch that reports the error. ## Sync errors & ErrorBoundary There are some errors that are thrown during background processes though, most notably the sync loop. These processes are not triggered by the view model directly, and hence there is not always a method call they can wrap in a try/catch. For this, there is the `ErrorBoundary` utility class. Since almost all aspects of the client can be updated through the sync loop, it is also not too helpful if there is only one try/catch around the whole sync and we stop sync if something goes wrong. -Instead, it's more helpful to split up the error handling into different scopes, where errors are stored and not rethrown when leaving the scope. One example is to have a scope per room. In this way, we can isolate an error occuring during sync to a specific room, and report it in the UI of that room. +Instead, it's more helpful to split up the error handling into different scopes, where errors are stored and not rethrown when leaving the scope. One example is to have a scope per room. In this way, we can isolate an error occuring during sync to a specific room, and report it in the UI of that room. This is typically where you would use `reportError` from `ErrorReportViewModel` rather than `logAndCatch`. You observe changes from your model in the view model (see docs on updates), and if the `error` property is set (by the `ErrorBoundary`), you call reportError with it. You can do this repeatedly without problems, if the same error is already reported, it's a No-Op. + +### `writeSync` and preventing data loss when dealing with errors. There is an extra complication though. The `writeSync` sync lifecycle step should not swallow any errors, or data loss can occur. This is because the whole `writeSync` lifecycle step writes all changes (for all rooms, the session, ...) for a sync response in one transaction (including the sync token), and aborts the transaction and stops sync if there is an error thrown during this step. So if there is an error in `writeSync` of a given room, it's fair to assume not all changes it was planning to write were passed to the transaction, as it got interrupted by the exception. Therefore, if we would swallow the error, data loss can occur as we'd not get another chance to write these changes to disk as we would have advanced the sync token. Therefore, code in the `writeSync` lifecycle step should be written defensively but always throw. diff --git a/src/domain/ErrorReportViewModel.ts b/src/domain/ErrorReportViewModel.ts index 68fe580c..da46b2df 100644 --- a/src/domain/ErrorReportViewModel.ts +++ b/src/domain/ErrorReportViewModel.ts @@ -32,6 +32,10 @@ export class ErrorReportViewModel extends ViewModel return this._errorViewModel; } + /** Typically you'd want to use `logAndCatch` when implementing a view model method. + * Use `reportError` when showing errors on your model that were set by + * background processes using `ErrorBoundary` or you have some other + * special low-level need to write your try/catch yourself. */ protected reportError(error: Error) { if (this._errorViewModel?.error === error) { return; @@ -47,6 +51,9 @@ export class ErrorReportViewModel extends ViewModel this.emitChange("errorViewModel"); } + /** Combines logging and error reporting in one method. + * Wrap the implementation of public view model methods + * with this to ensure errors are logged and reported.*/ protected logAndCatch(labelOrValues: LabelOrValues, callback: LogCallback, errorValue: T = undefined as unknown as T): T { try { let result = this.logger.run(labelOrValues, callback); From 3842f450dda28992767c62f382793fc8dc8f5d06 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 19 Jan 2023 11:30:23 +0100 Subject: [PATCH 250/323] ensure errors caught by boundary are logged in calls code --- src/matrix/calls/group/GroupCall.ts | 5 +++++ src/matrix/calls/group/Member.ts | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index d700f2a0..25e201e5 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -94,6 +94,11 @@ export class GroupCall extends EventEmitter<{change: never}> { /** Set between calling join and leave. */ private joinedData?: JoinedData; private errorBoundary = new ErrorBoundary(err => { + if (this.joinedData) { + // in case the error happens in code that does not log, + // log it here to make sure it isn't swallowed + this.joinedData.logItem.log("error at boundary").catch(err); + } this.emitChange(); }); diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 4ceaf46f..257628f5 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -97,6 +97,11 @@ export class Member { private expireTimeout?: Timeout; private errorBoundary = new ErrorBoundary(err => { this.options.emitUpdate(this, "error"); + if (this.connection) { + // in case the error happens in code that does not log, + // log it here to make sure it isn't swallowed + this.connection.logItem.log("error at boundary").catch(err); + } }); constructor( From e6b17cc74aeaa6a06e3799914fb4a6e7a0e79412 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 19 Jan 2023 11:30:52 +0100 Subject: [PATCH 251/323] fix callback type --- src/utils/ErrorBoundary.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/ErrorBoundary.ts b/src/utils/ErrorBoundary.ts index d13065f9..a4ac459c 100644 --- a/src/utils/ErrorBoundary.ts +++ b/src/utils/ErrorBoundary.ts @@ -19,7 +19,7 @@ export const ErrorValue = Symbol("ErrorBoundary:Error"); export class ErrorBoundary { private _error?: Error; - constructor(private readonly errorCallback: (Error) => void) {} + constructor(private readonly errorCallback: (err: Error) => void) {} /** * Executes callback() and then runs errorCallback() on error. @@ -79,4 +79,4 @@ export function tests() { assert.strictEqual(result, 0); } } -} \ No newline at end of file +} From 24088506785034a3164aac9f85811b561dcd8e19 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 19 Jan 2023 11:33:21 +0100 Subject: [PATCH 252/323] emit change before logging --- src/matrix/calls/group/GroupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 25e201e5..f6436eeb 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -94,12 +94,12 @@ export class GroupCall extends EventEmitter<{change: never}> { /** Set between calling join and leave. */ private joinedData?: JoinedData; private errorBoundary = new ErrorBoundary(err => { + this.emitChange(); if (this.joinedData) { // in case the error happens in code that does not log, // log it here to make sure it isn't swallowed this.joinedData.logItem.log("error at boundary").catch(err); } - this.emitChange(); }); constructor( From daad19c0604305ed00e40cb06be42a987e3d56c8 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 19 Jan 2023 11:37:39 +0100 Subject: [PATCH 253/323] swallow errors in errorCallback in ErrorBoundary nothing should be able to make ErrorBoundary.try throw --- src/utils/ErrorBoundary.ts | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/utils/ErrorBoundary.ts b/src/utils/ErrorBoundary.ts index a4ac459c..a74da479 100644 --- a/src/utils/ErrorBoundary.ts +++ b/src/utils/ErrorBoundary.ts @@ -32,18 +32,26 @@ export class ErrorBoundary { if (result instanceof Promise) { result = result.catch(err => { this._error = err; - this.errorCallback(err); + this.reportError(err); return errorValue; }); } return result; } catch (err) { this._error = err; - this.errorCallback(err); + this.reportError(err); return errorValue; } } + private reportError(err: Error) { + try { + this.errorCallback(err); + } catch (err) { + console.error("error in ErrorBoundary callback", err); + } + } + get error(): Error | undefined { return this._error; } @@ -77,6 +85,15 @@ export function tests() { }, 0); assert(emitted); assert.strictEqual(result, 0); + }, + "exception in error callback is swallowed": async assert => { + let emitted = false; + const boundary = new ErrorBoundary(() => { throw new Error("bug in errorCallback"); }); + assert.doesNotThrow(() => { + boundary.try(() => { + throw new Error("fail!"); + }); + }); } } } From b2feaf2b4e83d72718140da5892ce82e6d645e01 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 20 Jan 2023 16:50:07 +0100 Subject: [PATCH 254/323] fix mixing up params, causing us to call onUpdate on params rather than vm --- src/domain/session/room/CallViewModel.ts | 2 +- src/matrix/calls/group/GroupCall.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 0ba774a2..484543b7 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -55,7 +55,7 @@ export class CallViewModel extends ErrorReportViewModel vm.onUpdate(), + (param, vm) => vm?.onUpdate(), ) as BaseObservableMap; this.memberViewModels = otherMemberViewModels .join(ownMemberViewModelMap) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index efb321b2..0b8c7db5 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -126,7 +126,7 @@ export class GroupCall extends EventEmitter<{change: never}> { member.dispose(); this._members.remove(memberKey); } else { - this._members.update(memberKey, member); + this._members.update(memberKey); } }, encryptDeviceMessage: (userId: string, deviceId: string, message: SignallingMessage, log) => { From 6645f8f43b1ad2ffd532462bdf4584101dae43f1 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 20 Jan 2023 16:50:15 +0100 Subject: [PATCH 255/323] remove debug throw --- src/matrix/calls/group/Member.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 71c8b7f9..e097804c 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -196,7 +196,6 @@ export class Member { connectLogItem = log; await this.callIfNeeded(log); }); - throw new Error("connect failed!"); return connectLogItem; }); } From 590e9500c179b7d5238cdd60abfc7b9e24163716 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 20 Jan 2023 17:33:16 +0100 Subject: [PATCH 256/323] clone localMedia,so we don't remove audio track from stream sent to peer --- src/matrix/calls/LocalMedia.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 7a673ba9..adecfc18 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -43,7 +43,7 @@ export class LocalMedia { * Create an instance of LocalMedia without audio track (for user preview) */ asPreview(): LocalMedia { - const media = new LocalMedia(this.userMedia, this.screenShare, this.dataChannelOptions); + const media = this.clone(); const userMedia = media.userMedia; if (userMedia && userMedia.getVideoTracks().length > 0) { const audioTrack = getStreamAudioTrack(userMedia); From 2bd37970ba8a8267545378a407057c317fa6aa1a Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 20 Jan 2023 17:34:15 +0100 Subject: [PATCH 257/323] each LocalMedia own their streams, so a copy should have their own clone --- src/matrix/calls/LocalMedia.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index adecfc18..b7a14a9a 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -28,15 +28,15 @@ export class LocalMedia { ) {} withUserMedia(stream: Stream) { - return new LocalMedia(stream, this.screenShare, this.dataChannelOptions); + return new LocalMedia(stream, this.screenShare?.clone(), this.dataChannelOptions); } withScreenShare(stream: Stream) { - return new LocalMedia(this.userMedia, stream, this.dataChannelOptions); + return new LocalMedia(this.userMedia?.clone(), stream, this.dataChannelOptions); } withDataChannel(options: RTCDataChannelInit): LocalMedia { - return new LocalMedia(this.userMedia, this.screenShare, options); + return new LocalMedia(this.userMedia?.clone(), this.screenShare?.clone(), options); } /** From 3bb889ed9c1efb6c093811adb9bc1e79a885f74d Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 23 Jan 2023 14:53:08 +0100 Subject: [PATCH 258/323] WIP --- .../session/room/timeline/tiles/CallTile.js | 61 ++++++++++++++++--- src/matrix/calls/CallHandler.ts | 4 +- src/matrix/calls/callEventTypes.ts | 5 ++ src/matrix/calls/group/GroupCall.ts | 8 ++- .../web/ui/css/themes/element/timeline.css | 9 ++- .../ui/session/room/timeline/CallTileView.ts | 12 +++- 6 files changed, 83 insertions(+), 16 deletions(-) diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index c792496e..1758e6bb 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -15,7 +15,11 @@ limitations under the License. */ import {SimpleTile} from "./SimpleTile.js"; +import {ViewModel} from "../../../../ViewModel"; import {LocalMedia} from "../../../../../matrix/calls/LocalMedia"; +import {CallType} from "../../../../../matrix/calls/callEventTypes"; +import {avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../../../avatar"; + // TODO: timeline entries for state events with the same state key and type // should also update previous entries in the timeline, so we can update the name of the call, whether it is terminated, etc ... @@ -28,17 +32,28 @@ export class CallTile extends SimpleTile { this._call = calls.get(this._entry.stateKey); this._callSubscription = undefined; if (this._call) { - this._callSubscription = this._call.disposableOn("change", () => { + this._callSubscription = this.track(this._call.disposableOn("change", () => { // unsubscribe when terminated if (this._call.isTerminated) { this._callSubscription = this._callSubscription(); this._call = undefined; } this.emitChange(); - }); + })); + this.memberViewModels = this._setupMembersList(this._call); } } + _setupMembersList(call) { + return call.members.mapValues( + (member, emitChange) => new MemberAvatarViewModel(this.childOptions({ + member, + emitChange, + mediaRepository: this.getOption("room").mediaRepository + })), + ).sortValues((a, b) => a.avatarTitle < b.avatarTitle ? -1 : 1); + } + get confId() { return this._entry.stateKey; } @@ -61,16 +76,24 @@ export class CallTile extends SimpleTile { get label() { if (this._call) { - if (this._call.hasJoined) { - return `Ongoing call (${this.name}, ${this.confId})`; + if (this._type === CallType.Video) { + return `${this.displayName} started a video call`; } else { - return `${this.displayName} started a call (${this.name}, ${this.confId})`; + return `${this.displayName} started a voice call`; } } else { - return `Call finished, started by ${this.displayName} (${this.name}, ${this.confId})`; + if (this._type === CallType.Video) { + return `Video call ended`; + } else { + return `Voice call ended`; + } } } + get _type() { + return this._entry.event.content["m.type"]; + } + async join() { await this.logAndCatch("CallTile.join", async log => { if (this.canJoin) { @@ -88,10 +111,28 @@ export class CallTile extends SimpleTile { } }); } +} - dispose() { - if (this._callSubscription) { - this._callSubscription = this._callSubscription(); - } +class MemberAvatarViewModel extends ViewModel { + get _member() { + return this.getOption("member"); + } + + get avatarLetter() { + return avatarInitials(this._member.member.name); + } + + get avatarColorNumber() { + return getIdentifierColorNumber(this._member.userId); + } + + avatarUrl(size) { + const {avatarUrl} = this._member.member; + const mediaRepository = this.getOption("mediaRepository"); + return getAvatarHttpUrl(avatarUrl, size, this.platform, mediaRepository); + } + + get avatarTitle() { + return this._member.member.name; } } diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 6be6b193..f6533f4f 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -18,7 +18,7 @@ import {ObservableMap} from "../../observable/map"; import {WebRTC, PeerConnection} from "../../platform/types/WebRTC"; import {MediaDevices, Track} from "../../platform/types/MediaDevices"; import {handlesEventType} from "./PeerCall"; -import {EventType, CallIntent} from "./callEventTypes"; +import {EventType, CallIntent, CallType} from "./callEventTypes"; import {GroupCall} from "./group/GroupCall"; import {makeId} from "../common"; import {CALL_LOG_TYPE} from "./common"; @@ -130,7 +130,7 @@ export class CallHandler implements RoomStateHandler { log.set("newSize", this._calls.size); } - createCall(roomId: string, type: "m.video" | "m.voice", name: string, intent?: CallIntent, log?: ILogItem): Promise { + createCall(roomId: string, type: CallType, name: string, intent?: CallIntent, log?: ILogItem): Promise { return this.options.logger.wrapOrRun(log, "CallHandler.createCall", async log => { if (!intent) { intent = CallIntent.Ring; diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index 8fd7b23d..a0eb986c 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -227,3 +227,8 @@ export enum CallIntent { Prompt = "m.prompt", Room = "m.room", }; + +export enum CallType { + Video = "m.video", + Voice = "m.voice", +} diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 0b8c7db5..241e486c 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -20,7 +20,7 @@ import {LocalMedia} from "../LocalMedia"; import {MuteSettings, CALL_LOG_TYPE, CALL_MEMBER_VALIDITY_PERIOD_MS, mute} from "../common"; import {MemberChange, RoomMember} from "../../room/members/RoomMember"; import {EventEmitter} from "../../../utils/EventEmitter"; -import {EventType, CallIntent} from "../callEventTypes"; +import {EventType, CallIntent, CallType} from "../callEventTypes"; import { ErrorBoundary } from "../../../utils/ErrorBoundary"; import type {Options as MemberOptions} from "./Member"; @@ -155,6 +155,10 @@ export class GroupCall extends EventEmitter<{change: never}> { return this.callContent?.["m.intent"]; } + get type(): CallType { + return this.callContent?.["m.type"]; + } + /** * Gives access the log item for this call while joined. * Can be used for call diagnostics while in the call. @@ -318,7 +322,7 @@ export class GroupCall extends EventEmitter<{change: never}> { } /** @internal */ - create(type: "m.video" | "m.voice", log: ILogItem): Promise { + create(type: CallType, log: ILogItem): Promise { return log.wrap({l: "create call", t: CALL_LOG_TYPE}, async log => { if (this._state !== GroupCallState.Fledgling) { return; diff --git a/src/platform/web/ui/css/themes/element/timeline.css b/src/platform/web/ui/css/themes/element/timeline.css index 7ff35eb1..81a81960 100644 --- a/src/platform/web/ui/css/themes/element/timeline.css +++ b/src/platform/web/ui/css/themes/element/timeline.css @@ -440,4 +440,11 @@ only loads when the top comes into view*/ background-color: var(--background-color-primary); border-radius: 8px; text-align: center; - } +} + +.CallTileView_members > * { + margin-left: -16px; +} +.CallTileView_members { + display: flex; +} diff --git a/src/platform/web/ui/session/room/timeline/CallTileView.ts b/src/platform/web/ui/session/room/timeline/CallTileView.ts index 5bede510..c99c2136 100644 --- a/src/platform/web/ui/session/room/timeline/CallTileView.ts +++ b/src/platform/web/ui/session/room/timeline/CallTileView.ts @@ -17,7 +17,16 @@ limitations under the License. import {Builder, TemplateView} from "../../../general/TemplateView"; import type {CallTile} from "../../../../../../domain/session/room/timeline/tiles/CallTile"; import {ErrorView} from "../../../general/ErrorView"; - +import {ListView} from "../../../general/ListView"; +import {AvatarView} from "../../../AvatarView"; +/* +.CallTileView_members > * { + margin-left: -16px; +} +.CallTileView_members { + display: flex; +} +*/ export class CallTileView extends TemplateView { render(t: Builder, vm: CallTile) { return t.li( @@ -29,6 +38,7 @@ export class CallTileView extends TemplateView { }), t.div([ vm => vm.label, + t.view(new ListView({className: "CallTileView_members", list: vm.memberViewModels}, vm => new AvatarView(vm, 24))), t.button({className: "CallTileView_join", hidden: vm => !vm.canJoin}, "Join"), t.button({className: "CallTileView_leave", hidden: vm => !vm.canLeave}, "Leave") ]) From af5cc0f62b709724e661de8220f5c0b89b34886c Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 23 Jan 2023 15:35:28 +0100 Subject: [PATCH 259/323] sort by userId, sorting order needs to be stable --- src/domain/session/room/timeline/tiles/CallTile.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index 1758e6bb..b762cef4 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -51,7 +51,7 @@ export class CallTile extends SimpleTile { emitChange, mediaRepository: this.getOption("room").mediaRepository })), - ).sortValues((a, b) => a.avatarTitle < b.avatarTitle ? -1 : 1); + ).sortValues((a, b) => a.userId.localeCompare(b.userId)); } get confId() { @@ -118,6 +118,10 @@ class MemberAvatarViewModel extends ViewModel { return this.getOption("member"); } + get userId() { + return this._member.userId; + } + get avatarLetter() { return avatarInitials(this._member.member.name); } From 1df8d31ab5f4b812544b6ff18c7a4a389aae9854 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 23 Jan 2023 17:06:33 +0100 Subject: [PATCH 260/323] show call duration in tile --- .../session/room/timeline/tiles/CallTile.js | 30 +++++++++++++++---- src/matrix/calls/CallHandler.ts | 8 ++--- src/matrix/calls/group/GroupCall.ts | 15 ++++++++-- src/platform/types/types.ts | 1 + src/platform/web/dom/TimeFormatter.ts | 24 +++++++++++++++ .../ui/session/room/timeline/CallTileView.ts | 10 ++----- 6 files changed, 68 insertions(+), 20 deletions(-) diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index b762cef4..71297646 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -33,17 +33,27 @@ export class CallTile extends SimpleTile { this._callSubscription = undefined; if (this._call) { this._callSubscription = this.track(this._call.disposableOn("change", () => { - // unsubscribe when terminated - if (this._call.isTerminated) { - this._callSubscription = this._callSubscription(); - this._call = undefined; - } - this.emitChange(); + this._onCallUpdate(); })); + this._onCallUpdate(); this.memberViewModels = this._setupMembersList(this._call); } } + _onCallUpdate() { + // unsubscribe when terminated + if (this._call.isTerminated) { + this._durationInterval = this.disposeTracked(this._durationInterval); + this._callSubscription = this.disposeTracked(this._callSubscription); + this._call = undefined; + } else if (!this._durationInterval) { + this._durationInterval = this.track(this.platform.clock.createInterval(() => { + this.emitChange("duration"); + }, 1000)); + } + this.emitChange(); + } + _setupMembersList(call) { return call.members.mapValues( (member, emitChange) => new MemberAvatarViewModel(this.childOptions({ @@ -57,6 +67,14 @@ export class CallTile extends SimpleTile { get confId() { return this._entry.stateKey; } + + get duration() { + if (this._call && this._call.duration) { + return this.timeFormatter.formatDuration(this._call.duration); + } else { + return ""; + } + } get shape() { return "call"; diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index f6533f4f..c3f5d917 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -104,7 +104,7 @@ export class CallHandler implements RoomStateHandler { } const event = await txn.roomState.get(callEntry.roomId, EventType.GroupCall, callEntry.callId); if (event) { - const call = new GroupCall(event.event.state_key, false, event.event.content, event.roomId, this.groupCallOptions); + const call = new GroupCall(event.event.state_key, false, callEntry.timestamp, event.event.content, event.roomId, this.groupCallOptions); this._calls.set(call.id, call); } })); @@ -135,7 +135,7 @@ export class CallHandler implements RoomStateHandler { if (!intent) { intent = CallIntent.Ring; } - const call = new GroupCall(makeId("conf-"), true, { + const call = new GroupCall(makeId("conf-"), true, undefined, { "m.name": name, "m.intent": intent }, roomId, this.groupCallOptions); @@ -210,14 +210,14 @@ export class CallHandler implements RoomStateHandler { const callId = event.state_key; let call = this._calls.get(callId); if (call) { - call.updateCallEvent(event.content, log); + call.updateCallEvent(event, log); if (call.isTerminated) { call.disconnect(log); this._calls.remove(call.id); txn.calls.remove(call.intent, roomId, call.id); } } else { - call = new GroupCall(event.state_key, false, event.content, roomId, this.groupCallOptions); + call = new GroupCall(event.state_key, false, event.origin_server_ts, event.content, roomId, this.groupCallOptions); this._calls.set(call.id, call); txn.calls.add({ intent: call.intent, diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 241e486c..5748601f 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -105,6 +105,7 @@ export class GroupCall extends EventEmitter<{change: never}> { constructor( public readonly id: string, newCall: boolean, + private startTime: number | undefined, private callContent: Record, public readonly roomId: string, private readonly options: Options, @@ -143,6 +144,12 @@ export class GroupCall extends EventEmitter<{change: never}> { return this.callContent?.["m.terminated"] === true; } + get duration(): number | undefined { + if (typeof this.startTime === "number") { + return (this.options.clock.now() - this.startTime); + } + } + get isRinging(): boolean { return this._state === GroupCallState.Created && this.intent === "m.ring" && !this.isMember(this.options.ownUserId); } @@ -340,10 +347,14 @@ export class GroupCall extends EventEmitter<{change: never}> { } /** @internal */ - updateCallEvent(callContent: Record, syncLog: ILogItem) { + updateCallEvent(event: StateEvent, syncLog: ILogItem) { this.errorBoundary.try(() => { syncLog.wrap({l: "update call", t: CALL_LOG_TYPE, id: this.id}, log => { - this.callContent = callContent; + + if (typeof this.startTime !== "number") { + this.startTime = event.origin_server_ts; + } + this.callContent = event.content; if (this._state === GroupCallState.Creating) { this._state = GroupCallState.Created; } diff --git a/src/platform/types/types.ts b/src/platform/types/types.ts index 77953c4a..0e2f536e 100644 --- a/src/platform/types/types.ts +++ b/src/platform/types/types.ts @@ -55,4 +55,5 @@ export interface ITimeFormatter { formatTime(date: Date): string; formatRelativeDate(date: Date): string; formatMachineReadableDate(date: Date): string; + formatDuration(milliseconds: number): string; } diff --git a/src/platform/web/dom/TimeFormatter.ts b/src/platform/web/dom/TimeFormatter.ts index 2a98a716..cebc6ec4 100644 --- a/src/platform/web/dom/TimeFormatter.ts +++ b/src/platform/web/dom/TimeFormatter.ts @@ -22,6 +22,9 @@ enum TimeScope { Day = 24 * 60 * 60 * 1000, } +const MINUTES_IN_MS = 60 * 1000; +const HOURS_IN_MS = MINUTES_IN_MS * 60; + export class TimeFormatter implements ITimeFormatter { private todayMidnight: Date; @@ -75,6 +78,27 @@ export class TimeFormatter implements ITimeFormatter { return this.otherYearFormatter.format(date); } } + + formatDuration(milliseconds: number): string { + let hours = 0; + let minutes = 0; + if (milliseconds > HOURS_IN_MS) { + hours = Math.floor(milliseconds / HOURS_IN_MS); + milliseconds -= hours * HOURS_IN_MS; + } + if (milliseconds > MINUTES_IN_MS) { + minutes = Math.floor(milliseconds / MINUTES_IN_MS); + milliseconds -= minutes * MINUTES_IN_MS; + } + const seconds = Math.floor(milliseconds / 1000); + if (hours) { + return `${hours}h ${minutes}m ${seconds}s`; + } + if (minutes) { + return `${minutes}m ${seconds}s`; + } + return `${seconds}s`; + } } function capitalizeFirstLetter(str: string) { diff --git a/src/platform/web/ui/session/room/timeline/CallTileView.ts b/src/platform/web/ui/session/room/timeline/CallTileView.ts index c99c2136..a5c6ae09 100644 --- a/src/platform/web/ui/session/room/timeline/CallTileView.ts +++ b/src/platform/web/ui/session/room/timeline/CallTileView.ts @@ -19,14 +19,7 @@ import type {CallTile} from "../../../../../../domain/session/room/timeline/tile import {ErrorView} from "../../../general/ErrorView"; import {ListView} from "../../../general/ListView"; import {AvatarView} from "../../../AvatarView"; -/* -.CallTileView_members > * { - margin-left: -16px; -} -.CallTileView_members { - display: flex; -} -*/ + export class CallTileView extends TemplateView { render(t: Builder, vm: CallTile) { return t.li( @@ -38,6 +31,7 @@ export class CallTileView extends TemplateView { }), t.div([ vm => vm.label, + vm => vm.duration, t.view(new ListView({className: "CallTileView_members", list: vm.memberViewModels}, vm => new AvatarView(vm, 24))), t.button({className: "CallTileView_join", hidden: vm => !vm.canJoin}, "Join"), t.button({className: "CallTileView_leave", hidden: vm => !vm.canLeave}, "Leave") From 5035d2357349032aa28b8a3c866f8a1de861d03f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 23 Jan 2023 17:42:33 +0100 Subject: [PATCH 261/323] add all elements to call tile that need to be shown, style buttons --- .../session/room/timeline/tiles/CallTile.js | 28 +++++++++++++------ .../web/ui/css/themes/element/timeline.css | 14 ++++++++++ .../ui/session/room/timeline/CallTileView.ts | 22 ++++++++++----- 3 files changed, 49 insertions(+), 15 deletions(-) diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index 71297646..8887b834 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -64,6 +64,14 @@ export class CallTile extends SimpleTile { ).sortValues((a, b) => a.userId.localeCompare(b.userId)); } + get memberCount() { + // TODO: emit updates for this property + if (this._call) { + return this._call.members.size; + } + return 0; + } + get confId() { return this._entry.stateKey; } @@ -80,10 +88,6 @@ export class CallTile extends SimpleTile { return "call"; } - get name() { - return this._entry.content["m.name"]; - } - get canJoin() { return this._call && !this._call.hasJoined; } @@ -92,15 +96,15 @@ export class CallTile extends SimpleTile { return this._call && this._call.hasJoined; } - get label() { + get title() { if (this._call) { - if (this._type === CallType.Video) { + if (this.type === CallType.Video) { return `${this.displayName} started a video call`; } else { return `${this.displayName} started a voice call`; } } else { - if (this._type === CallType.Video) { + if (this.type === CallType.Video) { return `Video call ended`; } else { return `Voice call ended`; @@ -108,7 +112,15 @@ export class CallTile extends SimpleTile { } } - get _type() { + get typeLabel() { + if (this.type === CallType.Video) { + return `Video call`; + } else { + return `Voice call`; + } + } + + get type() { return this._entry.event.content["m.type"]; } diff --git a/src/platform/web/ui/css/themes/element/timeline.css b/src/platform/web/ui/css/themes/element/timeline.css index 81a81960..5dc86276 100644 --- a/src/platform/web/ui/css/themes/element/timeline.css +++ b/src/platform/web/ui/css/themes/element/timeline.css @@ -442,9 +442,23 @@ only loads when the top comes into view*/ text-align: center; } +.CallTileView > div > div { + display: flex; + flex-direction: column; + gap: 4px; +} + .CallTileView_members > * { margin-left: -16px; } .CallTileView_members { display: flex; } + +.CallTileView_memberCount::before { + content: url('./icons/room-members.svg?primary=text-color'); + background-repeat: no-repeat; + background-size: 24px; + width: 24px; + height: 24px; +} diff --git a/src/platform/web/ui/session/room/timeline/CallTileView.ts b/src/platform/web/ui/session/room/timeline/CallTileView.ts index a5c6ae09..a9a872b3 100644 --- a/src/platform/web/ui/session/room/timeline/CallTileView.ts +++ b/src/platform/web/ui/session/room/timeline/CallTileView.ts @@ -30,11 +30,19 @@ export class CallTileView extends TemplateView { return t.div({className: "CallTileView_error"}, t.view(new ErrorView(vm.errorViewModel, {inline: true}))); }), t.div([ - vm => vm.label, - vm => vm.duration, - t.view(new ListView({className: "CallTileView_members", list: vm.memberViewModels}, vm => new AvatarView(vm, 24))), - t.button({className: "CallTileView_join", hidden: vm => !vm.canJoin}, "Join"), - t.button({className: "CallTileView_leave", hidden: vm => !vm.canLeave}, "Leave") + t.div({className: "CallTileView_title"}, vm => vm.title), + t.div({className: "CallTileView_subtitle"}, [ + vm.typeLabel, " • ", + t.span({className: "CallTileView_memberCount"}, vm => vm.memberCount) + ]), + t.view(new ListView({className: "CallTileView_members", tagName: "div", list: vm.memberViewModels}, vm => { + return new AvatarView(vm, 24); + })), + t.div(vm => vm.duration), + t.div([ + t.button({className: "CallTileView_join button-action primary", hidden: vm => !vm.canJoin}, "Join"), + t.button({className: "CallTileView_leave button-action primary destructive", hidden: vm => !vm.canLeave}, "Leave") + ]) ]) ]) ); @@ -42,9 +50,9 @@ export class CallTileView extends TemplateView { /* This is called by the parent ListView, which just has 1 listener for the whole list */ onClick(evt) { - if (evt.target.className === "CallTileView_join") { + if (evt.target.classList.contains("CallTileView_join")) { this.value.join(); - } else if (evt.target.className === "CallTileView_leave") { + } else if (evt.target.classList.contains("CallTileView_leave")) { this.value.leave(); } } From fa5cb684b0917949cb40fae9267a6e980cd86393 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 24 Jan 2023 11:53:49 +0530 Subject: [PATCH 262/323] WIP --- src/domain/avatar.ts | 8 ++ src/domain/session/SessionViewModel.js | 9 ++ .../toast/BaseToastNotificationViewModel.ts | 34 ++++++ .../toast/CallToastNotificationViewModel.ts | 97 +++++++++++++++++ .../session/toast/ToastCollectionViewModel.ts | 88 +++++++++++++++ src/matrix/calls/CallHandler.ts | 6 +- src/matrix/calls/group/GroupCall.ts | 1 + .../web/ui/css/themes/element/theme.css | 100 ++++++++++++++++++ src/platform/web/ui/session/SessionView.js | 2 + .../toast/CallToastNotificationView.ts | 51 +++++++++ .../ui/session/toast/ToastCollectionView.ts | 33 ++++++ 11 files changed, 426 insertions(+), 3 deletions(-) create mode 100644 src/domain/session/toast/BaseToastNotificationViewModel.ts create mode 100644 src/domain/session/toast/CallToastNotificationViewModel.ts create mode 100644 src/domain/session/toast/ToastCollectionViewModel.ts create mode 100644 src/platform/web/ui/session/toast/CallToastNotificationView.ts create mode 100644 src/platform/web/ui/session/toast/ToastCollectionView.ts diff --git a/src/domain/avatar.ts b/src/domain/avatar.ts index ecaccebe..aaf700ef 100644 --- a/src/domain/avatar.ts +++ b/src/domain/avatar.ts @@ -58,3 +58,11 @@ export function getAvatarHttpUrl(avatarUrl: string | undefined, cssSize: number, } return undefined; } + +// move to AvatarView.js when converting to typescript +export interface IAvatarContract { + avatarLetter: string; + avatarColorNumber: number; + avatarUrl: (size: number) => string | undefined; + avatarTitle: string; +} diff --git a/src/domain/session/SessionViewModel.js b/src/domain/session/SessionViewModel.js index eb371ca8..1247f5d7 100644 --- a/src/domain/session/SessionViewModel.js +++ b/src/domain/session/SessionViewModel.js @@ -30,6 +30,7 @@ import {ViewModel} from "../ViewModel"; import {RoomViewModelObservable} from "./RoomViewModelObservable.js"; import {RightPanelViewModel} from "./rightpanel/RightPanelViewModel.js"; import {SyncStatus} from "../../matrix/Sync.js"; +import {ToastCollectionViewModel} from "./toast/ToastCollectionViewModel"; export class SessionViewModel extends ViewModel { constructor(options) { @@ -47,6 +48,10 @@ export class SessionViewModel extends ViewModel { this._gridViewModel = null; this._createRoomViewModel = null; this._joinRoomViewModel = null; + this._toastCollectionViewModel = this.track(new ToastCollectionViewModel(this.childOptions({ + callHandler: this._client.session.callHandler, + session: this._client.session, + }))); this._setupNavigation(); this._setupForcedLogoutOnAccessTokenInvalidation(); } @@ -173,6 +178,10 @@ export class SessionViewModel extends ViewModel { return this._joinRoomViewModel; } + get toastCollectionViewModel() { + return this._toastCollectionViewModel; + } + _updateGrid(roomIds) { const changed = !(this._gridViewModel && roomIds); const currentRoomId = this.navigation.path.get("room"); diff --git a/src/domain/session/toast/BaseToastNotificationViewModel.ts b/src/domain/session/toast/BaseToastNotificationViewModel.ts new file mode 100644 index 00000000..70fefea7 --- /dev/null +++ b/src/domain/session/toast/BaseToastNotificationViewModel.ts @@ -0,0 +1,34 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { ErrorReportViewModel } from "../../ErrorReportViewModel"; +import {Options as BaseOptions} from "../../ViewModel"; +import type {Session} from "../../../matrix/Session.js"; + +export type BaseClassOptions = { + dismiss: () => void; + session: Session; +} & BaseOptions; + +export abstract class BaseToastNotificationViewModel extends ErrorReportViewModel { + constructor(options: T) { + super(options); + } + + dismiss() { + this.getOption("dismiss")(); + } +} diff --git a/src/domain/session/toast/CallToastNotificationViewModel.ts b/src/domain/session/toast/CallToastNotificationViewModel.ts new file mode 100644 index 00000000..0e72c3f9 --- /dev/null +++ b/src/domain/session/toast/CallToastNotificationViewModel.ts @@ -0,0 +1,97 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import type {GroupCall} from "../../../matrix/calls/group/GroupCall"; +import type {Room} from "../../../matrix/room/Room.js"; +import {IAvatarContract, avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../avatar"; +import {LocalMedia} from "../../../matrix/calls/LocalMedia"; +import {BaseClassOptions, BaseToastNotificationViewModel} from "./BaseToastNotificationViewModel"; + +type Options = { + call: GroupCall; + room: Room; +} & BaseClassOptions; + + +export class CallToastNotificationViewModel extends BaseToastNotificationViewModel implements IAvatarContract { + constructor(options: Options) { + super(options); + this.track( + this.call.members.subscribe({ + onAdd: (_, __) => { + this.emitChange("memberCount"); + }, + onUpdate: (_, __) => { + this.emitChange("memberCount"); + }, + onRemove: (_, __) => { + this.emitChange("memberCount"); + }, + onReset: () => { /** noop */ }, + }) + ); + // Dismiss the toast if the room is opened manually + this.track( + this.navigation.observe("room").subscribe(roomId => { + if (roomId === this.call.roomId) { + this.dismiss(); + } + })); + } + + async join() { + await this.logAndCatch("CallToastNotificationViewModel.join", async (log) => { + const stream = await this.platform.mediaDevices.getMediaTracks(false, true); + const localMedia = new LocalMedia().withUserMedia(stream); + await this.call.join(localMedia, log); + const url = this.urlCreator.openRoomActionUrl(this.call.roomId); + this.urlCreator.pushUrl(url); + }); + } + + get call(): GroupCall { + return this.getOption("call"); + } + + private get room(): Room { + return this.getOption("room"); + } + + get roomName(): string { + return this.room.name; + } + + get memberCount(): number { + return this.call.members.size; + } + + get avatarLetter() { + return avatarInitials(this.roomName); + } + + get avatarColorNumber() { + return getIdentifierColorNumber(this.room.avatarColorId); + } + + avatarUrl(size: number) { + return getAvatarHttpUrl(this.room.avatarUrl, size, this.platform, this.room.mediaRepository); + } + + get avatarTitle() { + return this.roomName; + } +} + + diff --git a/src/domain/session/toast/ToastCollectionViewModel.ts b/src/domain/session/toast/ToastCollectionViewModel.ts new file mode 100644 index 00000000..f2fe65e3 --- /dev/null +++ b/src/domain/session/toast/ToastCollectionViewModel.ts @@ -0,0 +1,88 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {ObservableArray, ObservableMap} from "../../../observable"; +import {ViewModel, Options as BaseOptions} from "../../ViewModel"; +import type {GroupCall} from "../../../matrix/calls/group/GroupCall"; +import type {Room} from "../../../matrix/room/Room.js"; +import type {CallHandler} from "../../../matrix/calls/CallHandler"; +import type {Session} from "../../../matrix/Session.js"; +import {CallToastNotificationViewModel} from "./CallToastNotificationViewModel"; + +type Options = { + callHandler: CallHandler; + session: Session; +} & BaseOptions; + +export class ToastCollectionViewModel extends ViewModel { + public readonly toastViewModels: ObservableArray = new ObservableArray(); + + constructor(options: Options) { + super(options); + const callsObservableMap = this.getOption("callHandler").calls; + this.track(callsObservableMap.subscribe(this)); + } + + onAdd(_, call) { + if (this.shouldShowNotification(call)) { + const room = this._findRoomForCall(call); + const dismiss = () => { + const idx = this.toastViewModels.array.findIndex(vm => vm.call === call); + if (idx !== -1) { + this.toastViewModels.remove(idx); + } + }; + this.toastViewModels.append( + new CallToastNotificationViewModel(this.childOptions({ call, room, dismiss })) + ); + } + } + + onRemove(_, call) { + const idx = this.toastViewModels.array.findIndex(vm => vm.call === call); + if (idx !== -1) { + this.toastViewModels.remove(idx); + } + } + + onUpdate(_, call) { + const idx = this.toastViewModels.array.findIndex(vm => vm.call === call); + if (idx !== -1) { + // todo: is this correct? + this.toastViewModels.update(idx, this.toastViewModels.at(idx)!); + } + } + + onReset() { + for (let i = 0; i < this.toastViewModels.length; ++i) { + this.toastViewModels.remove(i); + } + } + + private _findRoomForCall(call: GroupCall): Room { + const id = call.roomId; + const rooms = this.getOption("session").rooms; + return rooms.get(id); + } + + private shouldShowNotification(call: GroupCall): boolean { + const currentlyOpenedRoomId = this.navigation.path.get("room")?.value; + if (!call.isLoadedFromStorage && call.roomId !== currentlyOpenedRoomId) { + return true; + } + return false; + } +} diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 6be6b193..e153c761 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -104,7 +104,7 @@ export class CallHandler implements RoomStateHandler { } const event = await txn.roomState.get(callEntry.roomId, EventType.GroupCall, callEntry.callId); if (event) { - const call = new GroupCall(event.event.state_key, false, event.event.content, event.roomId, this.groupCallOptions); + const call = new GroupCall(event.event.state_key, true, false, event.event.content, event.roomId, this.groupCallOptions); this._calls.set(call.id, call); } })); @@ -135,7 +135,7 @@ export class CallHandler implements RoomStateHandler { if (!intent) { intent = CallIntent.Ring; } - const call = new GroupCall(makeId("conf-"), true, { + const call = new GroupCall(makeId("conf-"), false, true, { "m.name": name, "m.intent": intent }, roomId, this.groupCallOptions); @@ -217,7 +217,7 @@ export class CallHandler implements RoomStateHandler { txn.calls.remove(call.intent, roomId, call.id); } } else { - call = new GroupCall(event.state_key, false, event.content, roomId, this.groupCallOptions); + call = new GroupCall(event.state_key, false, false, event.content, roomId, this.groupCallOptions); this._calls.set(call.id, call); txn.calls.add({ intent: call.intent, diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 0b8c7db5..108e1fb9 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -104,6 +104,7 @@ export class GroupCall extends EventEmitter<{change: never}> { constructor( public readonly id: string, + public readonly isLoadedFromStorage: boolean, newCall: boolean, private callContent: Record, public readonly roomId: string, diff --git a/src/platform/web/ui/css/themes/element/theme.css b/src/platform/web/ui/css/themes/element/theme.css index 1c9a233b..3aec2a8a 100644 --- a/src/platform/web/ui/css/themes/element/theme.css +++ b/src/platform/web/ui/css/themes/element/theme.css @@ -1224,3 +1224,103 @@ button.RoomDetailsView_row::after { .JoinRoomView_status .spinner { margin-right: 5px; } + +/* Toast */ +.ToastCollectionView { + display: flex; + position: fixed; + flex-direction: column; + z-index: 1000; + left: 44px; + top: 52px; +} + +.ToastCollectionView ul { + margin: 0; + padding: 0; +} + +.CallToastNotificationView:not(:first-child) { + margin-top: 12px; +} + +.CallToastNotificationView { + display: grid; + grid-template-rows: 40px 1fr 1fr 48px; + row-gap: 4px; + width: 260px; + background-color: var(--background-color-secondary); + border-radius: 8px; + color: var(--text-color); + box-shadow: 2px 2px 10px rgba(0, 0, 0, 0.5); +} + +.CallToastNotificationView__top { + display: grid; + grid-template-columns: auto 176px auto; + align-items: center; + justify-items: center; +} +.CallToastNotificationView__dismiss-btn { + background: center var(--background-color-secondary--darker-5) url("./icons/dismiss.svg?primary=text-color") no-repeat; + border-radius: 100%; + height: 15px; + width: 15px; +} + +.CallToastNotificationView__name { + font-weight: 600; + width: 100%; +} + +.CallToastNotificationView__description { + margin-left: 42px; +} + +.CallToastNotificationView__call-type::before { + content: url("./icons/video-call.svg?primary=light-text-color"); + display: flex; + width: 20px; + height: 20px; + padding-right: 5px; +} + +.CallToastNotificationView__call-type::after { + content: ""; + width: 4px; + height: 4px; + background-color: var(--text-color); + border-radius: 100%; + align-self: center; + margin: 5px; +} + +.CallToastNotificationView__member-count::before { + content: url("./icons/room-members.svg?primary=light-text-color"); + display: flex; + width: 20px; + height: 20px; + padding-right: 5px; +} + +.CallToastNotificationView__member-count, +.CallToastNotificationView__call-type { + display: flex; + align-items: center; +} + +.CallToastNotificationView__info { + display: flex; + margin-left: 42px; +} + +.CallToastNotificationView__action { + display: flex; + justify-content: end; + margin-right: 10px; +} + +.CallToastNotificationView__action .button-action { + width: 100px; + height: 40px; +} diff --git a/src/platform/web/ui/session/SessionView.js b/src/platform/web/ui/session/SessionView.js index 7bcd8c0f..9f84e872 100644 --- a/src/platform/web/ui/session/SessionView.js +++ b/src/platform/web/ui/session/SessionView.js @@ -30,6 +30,7 @@ import {CreateRoomView} from "./CreateRoomView.js"; import {RightPanelView} from "./rightpanel/RightPanelView.js"; import {viewClassForTile} from "./room/common"; import {JoinRoomView} from "./JoinRoomView"; +import {ToastCollectionView} from "./toast/ToastCollectionView"; export class SessionView extends TemplateView { render(t, vm) { @@ -40,6 +41,7 @@ export class SessionView extends TemplateView { "right-shown": vm => !!vm.rightPanelViewModel }, }, [ + t.view(new ToastCollectionView(vm.toastCollectionViewModel)), t.view(new SessionStatusView(vm.sessionStatusViewModel)), t.view(new LeftPanelView(vm.leftPanelViewModel)), t.mapView(vm => vm.activeMiddleViewModel, () => { diff --git a/src/platform/web/ui/session/toast/CallToastNotificationView.ts b/src/platform/web/ui/session/toast/CallToastNotificationView.ts new file mode 100644 index 00000000..50adcc7b --- /dev/null +++ b/src/platform/web/ui/session/toast/CallToastNotificationView.ts @@ -0,0 +1,51 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {AvatarView} from "../../AvatarView.js"; +import {ErrorView} from "../../general/ErrorView"; +import {TemplateView, Builder} from "../../general/TemplateView"; +import type {CallToastNotificationViewModel} from "../../../../../domain/session/toast/CallToastNotificationViewModel"; + +export class CallToastNotificationView extends TemplateView { + render(t: Builder, vm: CallToastNotificationViewModel) { + return t.div({ className: "CallToastNotificationView" }, [ + t.div({ className: "CallToastNotificationView__top" }, [ + t.view(new AvatarView(vm, 24)), + t.span({ className: "CallToastNotificationView__name" }, (vm) => vm.roomName), + t.button({ + className: "button-action CallToastNotificationView__dismiss-btn", + onClick: () => vm.dismiss(), + }), + ]), + t.div({ className: "CallToastNotificationView__description" }, [ + t.span(vm.i18n`Video call started`) + ]), + t.div({ className: "CallToastNotificationView__info" }, [ + t.span({className: "CallToastNotificationView__call-type"}, vm.i18n`Video`), + t.span({className: "CallToastNotificationView__member-count"}, (vm) => vm.memberCount), + ]), + t.div({ className: "CallToastNotificationView__action" }, [ + t.button({ + className: "button-action primary", + onClick: () => vm.join(), + }, vm.i18n`Join`), + ]), + t.if(vm => !!vm.errorViewModel, t => { + return t.div({className: "CallView_error"}, t.view(new ErrorView(vm.errorViewModel!))); + }), + ]); + } +} diff --git a/src/platform/web/ui/session/toast/ToastCollectionView.ts b/src/platform/web/ui/session/toast/ToastCollectionView.ts new file mode 100644 index 00000000..3dc99c77 --- /dev/null +++ b/src/platform/web/ui/session/toast/ToastCollectionView.ts @@ -0,0 +1,33 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {CallToastNotificationView} from "./CallToastNotificationView"; +import {ListView} from "../../general/ListView"; +import {TemplateView, Builder} from "../../general/TemplateView"; +import type {CallToastNotificationViewModel} from "../../../../../domain/session/toast/CallToastNotificationViewModel"; +import type {ToastCollectionViewModel} from "../../../../../domain/session/toast/ToastCollectionViewModel"; + +export class ToastCollectionView extends TemplateView { + render(t: Builder, vm: ToastCollectionViewModel) { + const view = new ListView({ + list: vm.toastViewModels, + parentProvidesUpdates: false, + }, (vm: CallToastNotificationViewModel) => new CallToastNotificationView(vm)); + return t.div({ className: "ToastCollectionView" }, [ + t.view(view), + ]); + } +} From d2b1fc7fef1eace1a14ea43bc4d4b54b804f0d87 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 24 Jan 2023 11:58:49 +0530 Subject: [PATCH 263/323] No need to pass in callHandler separately because session is already passed --- src/domain/session/SessionViewModel.js | 1 - src/domain/session/toast/ToastCollectionViewModel.ts | 7 +++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/domain/session/SessionViewModel.js b/src/domain/session/SessionViewModel.js index 1247f5d7..072e1b74 100644 --- a/src/domain/session/SessionViewModel.js +++ b/src/domain/session/SessionViewModel.js @@ -49,7 +49,6 @@ export class SessionViewModel extends ViewModel { this._createRoomViewModel = null; this._joinRoomViewModel = null; this._toastCollectionViewModel = this.track(new ToastCollectionViewModel(this.childOptions({ - callHandler: this._client.session.callHandler, session: this._client.session, }))); this._setupNavigation(); diff --git a/src/domain/session/toast/ToastCollectionViewModel.ts b/src/domain/session/toast/ToastCollectionViewModel.ts index f2fe65e3..3860830d 100644 --- a/src/domain/session/toast/ToastCollectionViewModel.ts +++ b/src/domain/session/toast/ToastCollectionViewModel.ts @@ -14,16 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {ObservableArray, ObservableMap} from "../../../observable"; +import {ObservableArray} from "../../../observable"; import {ViewModel, Options as BaseOptions} from "../../ViewModel"; import type {GroupCall} from "../../../matrix/calls/group/GroupCall"; import type {Room} from "../../../matrix/room/Room.js"; -import type {CallHandler} from "../../../matrix/calls/CallHandler"; import type {Session} from "../../../matrix/Session.js"; import {CallToastNotificationViewModel} from "./CallToastNotificationViewModel"; type Options = { - callHandler: CallHandler; session: Session; } & BaseOptions; @@ -32,7 +30,8 @@ export class ToastCollectionViewModel extends ViewModel { constructor(options: Options) { super(options); - const callsObservableMap = this.getOption("callHandler").calls; + const session = this.getOption("session"); + const callsObservableMap = session.callHandler.calls; this.track(callsObservableMap.subscribe(this)); } From e3a8c184f6088d57f49bd9584e06df8a06d26eb9 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 24 Jan 2023 12:03:04 +0530 Subject: [PATCH 264/323] Fix formatting --- src/domain/session/toast/BaseToastNotificationViewModel.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/session/toast/BaseToastNotificationViewModel.ts b/src/domain/session/toast/BaseToastNotificationViewModel.ts index 70fefea7..0ace7fdf 100644 --- a/src/domain/session/toast/BaseToastNotificationViewModel.ts +++ b/src/domain/session/toast/BaseToastNotificationViewModel.ts @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { ErrorReportViewModel } from "../../ErrorReportViewModel"; +import {ErrorReportViewModel} from "../../ErrorReportViewModel"; import {Options as BaseOptions} from "../../ViewModel"; import type {Session} from "../../../matrix/Session.js"; From afee565eb7cd116015a96289035780f7693ffbbd Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 24 Jan 2023 12:05:17 +0530 Subject: [PATCH 265/323] Also emit on reset event --- src/domain/session/toast/CallToastNotificationViewModel.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/domain/session/toast/CallToastNotificationViewModel.ts b/src/domain/session/toast/CallToastNotificationViewModel.ts index 0e72c3f9..ebc66167 100644 --- a/src/domain/session/toast/CallToastNotificationViewModel.ts +++ b/src/domain/session/toast/CallToastNotificationViewModel.ts @@ -39,7 +39,9 @@ export class CallToastNotificationViewModel extends BaseToastNotificationViewMod onRemove: (_, __) => { this.emitChange("memberCount"); }, - onReset: () => { /** noop */ }, + onReset: () => { + this.emitChange("memberCount"); + }, }) ); // Dismiss the toast if the room is opened manually From b86fdd476f51ed4fb0372ddd84297a9926949011 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 24 Jan 2023 12:12:35 +0530 Subject: [PATCH 266/323] Add return types --- .../session/toast/BaseToastNotificationViewModel.ts | 2 +- .../session/toast/CallToastNotificationViewModel.ts | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/domain/session/toast/BaseToastNotificationViewModel.ts b/src/domain/session/toast/BaseToastNotificationViewModel.ts index 0ace7fdf..01ec688b 100644 --- a/src/domain/session/toast/BaseToastNotificationViewModel.ts +++ b/src/domain/session/toast/BaseToastNotificationViewModel.ts @@ -28,7 +28,7 @@ export abstract class BaseToastNotificationViewModel { await this.logAndCatch("CallToastNotificationViewModel.join", async (log) => { const stream = await this.platform.mediaDevices.getMediaTracks(false, true); const localMedia = new LocalMedia().withUserMedia(stream); @@ -79,19 +79,19 @@ export class CallToastNotificationViewModel extends BaseToastNotificationViewMod return this.call.members.size; } - get avatarLetter() { + get avatarLetter(): string { return avatarInitials(this.roomName); } - get avatarColorNumber() { + get avatarColorNumber(): number { return getIdentifierColorNumber(this.room.avatarColorId); } - avatarUrl(size: number) { + avatarUrl(size: number): string | undefined { return getAvatarHttpUrl(this.room.avatarUrl, size, this.platform, this.room.mediaRepository); } - get avatarTitle() { + get avatarTitle(): string { return this.roomName; } } From eb7fcc6da2eaabcf216b96d1c4fd59e15ea7f012 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 24 Jan 2023 13:24:15 +0530 Subject: [PATCH 267/323] Add return types --- .../session/toast/ToastCollectionViewModel.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/domain/session/toast/ToastCollectionViewModel.ts b/src/domain/session/toast/ToastCollectionViewModel.ts index 3860830d..30e424ae 100644 --- a/src/domain/session/toast/ToastCollectionViewModel.ts +++ b/src/domain/session/toast/ToastCollectionViewModel.ts @@ -14,12 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. */ +import {CallToastNotificationViewModel} from "./CallToastNotificationViewModel"; import {ObservableArray} from "../../../observable"; import {ViewModel, Options as BaseOptions} from "../../ViewModel"; import type {GroupCall} from "../../../matrix/calls/group/GroupCall"; import type {Room} from "../../../matrix/room/Room.js"; import type {Session} from "../../../matrix/Session.js"; -import {CallToastNotificationViewModel} from "./CallToastNotificationViewModel"; type Options = { session: Session; @@ -35,8 +35,8 @@ export class ToastCollectionViewModel extends ViewModel { this.track(callsObservableMap.subscribe(this)); } - onAdd(_, call) { - if (this.shouldShowNotification(call)) { + onAdd(_, call: GroupCall) { + if (this._shouldShowNotification(call)) { const room = this._findRoomForCall(call); const dismiss = () => { const idx = this.toastViewModels.array.findIndex(vm => vm.call === call); @@ -50,17 +50,16 @@ export class ToastCollectionViewModel extends ViewModel { } } - onRemove(_, call) { + onRemove(_, call: GroupCall) { const idx = this.toastViewModels.array.findIndex(vm => vm.call === call); if (idx !== -1) { this.toastViewModels.remove(idx); } } - onUpdate(_, call) { + onUpdate(_, call: GroupCall) { const idx = this.toastViewModels.array.findIndex(vm => vm.call === call); if (idx !== -1) { - // todo: is this correct? this.toastViewModels.update(idx, this.toastViewModels.at(idx)!); } } @@ -77,7 +76,7 @@ export class ToastCollectionViewModel extends ViewModel { return rooms.get(id); } - private shouldShowNotification(call: GroupCall): boolean { + private _shouldShowNotification(call: GroupCall): boolean { const currentlyOpenedRoomId = this.navigation.path.get("room")?.value; if (!call.isLoadedFromStorage && call.roomId !== currentlyOpenedRoomId) { return true; From 26476324dce53b908d051003a20917dc8f8d5e37 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 24 Jan 2023 18:07:48 +0530 Subject: [PATCH 268/323] lockfile --- yarn.lock | 908 +----------------------------------------------------- 1 file changed, 8 insertions(+), 900 deletions(-) diff --git a/yarn.lock b/yarn.lock index 4b7e83f3..876917a8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -23,43 +23,6 @@ chalk "^2.0.0" js-tokens "^4.0.0" -"@colors/colors@1.5.0": - version "1.5.0" - resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.5.0.tgz#bb504579c1cae923e6576a4f5da43d25f97bdbd9" - integrity sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ== - -"@cypress/request@^2.88.10": - version "2.88.10" - resolved "https://registry.yarnpkg.com/@cypress/request/-/request-2.88.10.tgz#b66d76b07f860d3a4b8d7a0604d020c662752cce" - integrity sha512-Zp7F+R93N0yZyG34GutyTNr+okam7s/Fzc1+i3kcqOP8vk6OuajuE9qZJ6Rs+10/1JFtXFYMdyarnU1rZuJesg== - dependencies: - aws-sign2 "~0.7.0" - aws4 "^1.8.0" - caseless "~0.12.0" - combined-stream "~1.0.6" - extend "~3.0.2" - forever-agent "~0.6.1" - form-data "~2.3.2" - http-signature "~1.3.6" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.19" - performance-now "^2.1.0" - qs "~6.5.2" - safe-buffer "^5.1.2" - tough-cookie "~2.5.0" - tunnel-agent "^0.6.0" - uuid "^8.3.2" - -"@cypress/xvfb@^1.2.4": - version "1.2.4" - resolved "https://registry.yarnpkg.com/@cypress/xvfb/-/xvfb-1.2.4.tgz#2daf42e8275b39f4aa53c14214e557bd14e7748a" - integrity sha512-skbBzPggOVYCbnGgV+0dmBdW/s77ZkAOXIC1knS8NagwDjBrNC1LuXtQJeiN6l+m7lzmHtaoUw/ctJKdqkG57Q== - dependencies: - debug "^3.1.0" - lodash.once "^4.1.1" - "@eslint/eslintrc@^0.4.3": version "0.4.3" resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-0.4.3.tgz#9e42981ef035beb3dd49add17acb96e8ff6f394c" @@ -142,28 +105,6 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-18.7.13.tgz#23e6c5168333480d454243378b69e861ab5c011a" integrity sha512-46yIhxSe5xEaJZXWdIBP7GU4HDTG8/eo0qd9atdiL+lFpA03y8KS+lkTN834TWJj5767GbWv4n/P6efyTFt1Dw== -"@types/node@^14.14.31": - version "14.18.26" - resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.26.tgz#239e19f8b4ea1a9eb710528061c1d733dc561996" - integrity sha512-0b+utRBSYj8L7XAp0d+DX7lI4cSmowNaaTkk6/1SKzbKkG+doLuPusB9EOvzLJ8ahJSk03bTLIL6cWaEd4dBKA== - -"@types/sinonjs__fake-timers@8.1.1": - version "8.1.1" - resolved "https://registry.yarnpkg.com/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-8.1.1.tgz#b49c2c70150141a15e0fa7e79cf1f92a72934ce3" - integrity sha512-0kSuKjAS0TrGLJ0M/+8MaFkGsQhZpB6pxOmvS3K8FYI72K//YmdfoW9X2qPsAKh1mkwxGD5zib9s1FIFed6E8g== - -"@types/sizzle@^2.3.2": - version "2.3.3" - resolved "https://registry.yarnpkg.com/@types/sizzle/-/sizzle-2.3.3.tgz#ff5e2f1902969d305225a047c8a0fd5c915cebef" - integrity sha512-JYM8x9EGF163bEyhdJBpR2QX1R5naCJHC8ucJylJ3w9/CVBaskdQ8WqBf8MmQrd1kRvp/a4TS8HJ+bxzR7ZJYQ== - -"@types/yauzl@^2.9.1": - version "2.10.0" - resolved "https://registry.yarnpkg.com/@types/yauzl/-/yauzl-2.10.0.tgz#b3248295276cf8c6f153ebe6a9aba0c988cb2599" - integrity sha512-Cn6WYCm0tXv8p6k+A8PvbDG763EDpBoTzHdA+Q/MF6H3sapGjCm9NzoaJncJS9tUKSuCoDs9XHxYYsQDgxR6kw== - dependencies: - "@types/node" "*" - "@typescript-eslint/eslint-plugin@^4.29.2": version "4.29.2" resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-4.29.2.tgz#f54dc0a32b8f61c6024ab8755da05363b733838d" @@ -258,14 +199,6 @@ aes-js@^3.1.2: resolved "https://registry.yarnpkg.com/aes-js/-/aes-js-3.1.2.tgz#db9aabde85d5caabbfc0d4f2a4446960f627146a" integrity sha512-e5pEa2kBnBOgR4Y/p20pskXI74UEz7de8ZGVo58asOtvSVG5YAbJeELPZxOmt+Bnz3rX753YKhfIn4X4l1PPRQ== -aggregate-error@^3.0.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/aggregate-error/-/aggregate-error-3.1.0.tgz#92670ff50f5359bdb7a3e0d40d0ec30c5737687a" - integrity sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA== - dependencies: - clean-stack "^2.0.0" - indent-string "^4.0.0" - ajv@^6.10.0, ajv@^6.12.4: version "6.12.6" resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" @@ -296,23 +229,11 @@ ansi-colors@^4.1.1: resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-4.1.1.tgz#cbb9ae256bf750af1eab344f229aa27fe94ba348" integrity sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA== -ansi-escapes@^4.3.0: - version "4.3.2" - resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-4.3.2.tgz#6b2291d1db7d98b6521d5f1efa42d0f3a9feb65e" - integrity sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ== - dependencies: - type-fest "^0.21.3" - ansi-regex@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75" integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg== -ansi-regex@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" - integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== - ansi-styles@^3.2.1: version "3.2.1" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d" @@ -327,11 +248,6 @@ ansi-styles@^4.0.0, ansi-styles@^4.1.0: dependencies: color-convert "^2.0.1" -arch@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/arch/-/arch-2.2.0.tgz#1bc47818f305764f23ab3306b0bfc086c5a29d11" - integrity sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ== - argparse@^1.0.7: version "1.0.10" resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" @@ -344,48 +260,11 @@ array-union@^2.1.0: resolved "https://registry.yarnpkg.com/array-union/-/array-union-2.1.0.tgz#b798420adbeb1de828d84acd8a2e23d3efe85e8d" integrity sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw== -asn1@~0.2.3: - version "0.2.6" - resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.6.tgz#0d3a7bb6e64e02a90c0303b31f292868ea09a08d" - integrity sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ== - dependencies: - safer-buffer "~2.1.0" - -assert-plus@1.0.0, assert-plus@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525" - integrity sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw== - astral-regex@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/astral-regex/-/astral-regex-2.0.0.tgz#483143c567aeed4785759c0865786dc77d7d2e31" integrity sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ== -async@^3.2.0: - version "3.2.4" - resolved "https://registry.yarnpkg.com/async/-/async-3.2.4.tgz#2d22e00f8cddeb5fde5dd33522b56d1cf569a81c" - integrity sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ== - -asynckit@^0.4.0: - version "0.4.0" - resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" - integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q== - -at-least-node@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/at-least-node/-/at-least-node-1.0.0.tgz#602cd4b46e844ad4effc92a8011a3c46e0238dc2" - integrity sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg== - -aws-sign2@~0.7.0: - version "0.7.0" - resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" - integrity sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA== - -aws4@^1.8.0: - version "1.11.0" - resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59" - integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA== - balanced-match@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" @@ -408,28 +287,6 @@ base64-arraybuffer@^0.2.0: resolved "https://registry.yarnpkg.com/base64-arraybuffer/-/base64-arraybuffer-0.2.0.tgz#4b944fac0191aa5907afe2d8c999ccc57ce80f45" integrity sha512-7emyCsu1/xiBXgQZrscw/8KPRT44I4Yq9Pe6EGs3aPRTsWuggML1/1DTuZUuIaJPIm1FTDUVXl4x/yW8s0kQDQ== -base64-js@^1.3.1: - version "1.5.1" - resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" - integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== - -bcrypt-pbkdf@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e" - integrity sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w== - dependencies: - tweetnacl "^0.14.3" - -blob-util@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/blob-util/-/blob-util-2.0.2.tgz#3b4e3c281111bb7f11128518006cdc60b403a1eb" - integrity sha512-T7JQa+zsXXEa6/8ZhHcQEW1UFfVM49Ts65uBkFL6fz2QmrElqmbajIDJvuA0tEhRe5eIjpV9ZF+0RfZR9voJFQ== - -bluebird@^3.7.2: - version "3.7.2" - resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" - integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== - boolbase@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e" @@ -457,34 +314,11 @@ bs58@^4.0.1: dependencies: base-x "^3.0.2" -buffer-crc32@~0.2.3: - version "0.2.13" - resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242" - integrity sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ== - -buffer@^5.6.0: - version "5.7.1" - resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0" - integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ== - dependencies: - base64-js "^1.3.1" - ieee754 "^1.1.13" - -cachedir@^2.3.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/cachedir/-/cachedir-2.3.0.tgz#0c75892a052198f0b21c7c1804d8331edfcae0e8" - integrity sha512-A+Fezp4zxnit6FanDmv9EqXNAi3vt9DWp51/71UEhXukb7QUuvtv9344h91dyAxuTLoSYJFU299qzR3tzwPAhw== - callsites@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73" integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ== -caseless@~0.12.0: - version "0.12.0" - resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" - integrity sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw== - chalk@^2.0.0: version "2.4.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" @@ -502,53 +336,6 @@ chalk@^4.0.0: ansi-styles "^4.1.0" supports-color "^7.1.0" -chalk@^4.1.0: - version "4.1.2" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" - integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== - dependencies: - ansi-styles "^4.1.0" - supports-color "^7.1.0" - -check-more-types@^2.24.0: - version "2.24.0" - resolved "https://registry.yarnpkg.com/check-more-types/-/check-more-types-2.24.0.tgz#1420ffb10fd444dcfc79b43891bbfffd32a84600" - integrity sha512-Pj779qHxV2tuapviy1bSZNEL1maXr13bPYpsvSDB68HlYcYuhlDrmGd63i0JHMCLKzc7rUSNIrpdJlhVlNwrxA== - -ci-info@^3.2.0: - version "3.3.2" - resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-3.3.2.tgz#6d2967ffa407466481c6c90b6e16b3098f080128" - integrity sha512-xmDt/QIAdeZ9+nfdPsaBCpMvHNLFiLdjj59qjqn+6iPe6YmHGQ35sBnQ8uslRBXFmXkiZQOJRjvQeoGppoTjjg== - -clean-stack@^2.0.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-2.2.0.tgz#ee8472dbb129e727b31e8a10a427dee9dfe4008b" - integrity sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A== - -cli-cursor@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-3.1.0.tgz#264305a7ae490d1d03bf0c9ba7c925d1753af307" - integrity sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw== - dependencies: - restore-cursor "^3.1.0" - -cli-table3@~0.6.1: - version "0.6.2" - resolved "https://registry.yarnpkg.com/cli-table3/-/cli-table3-0.6.2.tgz#aaf5df9d8b5bf12634dc8b3040806a0c07120d2a" - integrity sha512-QyavHCaIC80cMivimWu4aWHilIpiDpfm3hGmqAmXVL1UsnbLuBSMd21hTX6VY4ZSDSM73ESLeF8TOYId3rBTbw== - dependencies: - string-width "^4.2.0" - optionalDependencies: - "@colors/colors" "1.5.0" - -cli-truncate@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/cli-truncate/-/cli-truncate-2.1.0.tgz#c39e28bf05edcde5be3b98992a22deed5a2b93c7" - integrity sha512-n8fOixwDD6b/ObinzTrp1ZKFzbgvKZvuz/TvejnLn1aQfC6r52XEx85FmuC+3HI+JM7coBRXUvNqEU2PHVrHpg== - dependencies: - slice-ansi "^3.0.0" - string-width "^4.2.0" - color-convert@^1.9.0: version "1.9.3" resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" @@ -573,28 +360,11 @@ color-name@~1.1.4: resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== -colorette@^2.0.16: - version "2.0.19" - resolved "https://registry.yarnpkg.com/colorette/-/colorette-2.0.19.tgz#cdf044f47ad41a0f4b56b3a0d5b4e6e1a2d5a798" - integrity sha512-3tlv/dIP7FWvj3BsbHrGLJ6l/oKh1O3TcgBqMn+yyCagOxc23fyzDS6HypQbgxWbkpDnf52p1LuR4eWDQ/K9WQ== - colors@^1.3.3: version "1.4.0" resolved "https://registry.yarnpkg.com/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78" integrity sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA== -combined-stream@^1.0.6, combined-stream@~1.0.6: - version "1.0.8" - resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" - integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== - dependencies: - delayed-stream "~1.0.0" - -commander@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/commander/-/commander-5.1.0.tgz#46abbd1652f8e059bddaef99bbdcb2ad9cf179ae" - integrity sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg== - commander@^6.1.0: version "6.2.1" resolved "https://registry.yarnpkg.com/commander/-/commander-6.2.1.tgz#0792eb682dfbc325999bb2b84fddddba110ac73c" @@ -605,11 +375,6 @@ commander@^7.2.0: resolved "https://registry.yarnpkg.com/commander/-/commander-7.2.0.tgz#a36cb57d0b501ce108e4d20559a150a391d97ab7" integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw== -common-tags@^1.8.0: - version "1.8.2" - resolved "https://registry.yarnpkg.com/common-tags/-/common-tags-1.8.2.tgz#94ebb3c076d26032745fd54face7f688ef5ac9c6" - integrity sha512-gk/Z852D2Wtb//0I+kRFNKKE9dIIVirjoqPoA1wJU+XePVXZfGeBpk45+A1rKO4Q43prqWBNY/MiIeRLbPWUaA== - concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" @@ -625,12 +390,7 @@ core-js@^3.6.5: resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.19.3.tgz#6df8142a996337503019ff3235a7022d7cdf4559" integrity sha512-LeLBMgEGSsG7giquSzvgBrTS7V5UL6ks3eQlUSbN8dJStlLFiRzUm5iqsRyzUB8carhfKjkJ2vzKqE6z1Vga9g== -core-util-is@1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" - integrity sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ== - -cross-spawn@^7.0.0, cross-spawn@^7.0.2: +cross-spawn@^7.0.2: version "7.0.3" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== @@ -675,73 +435,6 @@ cuint@^0.2.2: resolved "https://registry.yarnpkg.com/cuint/-/cuint-0.2.2.tgz#408086d409550c2631155619e9fa7bcadc3b991b" integrity sha1-QICG1AlVDCYxFVYZ6fp7ytw7mRs= -cypress@^10.6.0: - version "10.6.0" - resolved "https://registry.yarnpkg.com/cypress/-/cypress-10.6.0.tgz#13f46867febf2c3715874ed5dce9c2e946b175fe" - integrity sha512-6sOpHjostp8gcLO34p6r/Ci342lBs8S5z9/eb3ZCQ22w2cIhMWGUoGKkosabPBfKcvRS9BE4UxybBtlIs8gTQA== - dependencies: - "@cypress/request" "^2.88.10" - "@cypress/xvfb" "^1.2.4" - "@types/node" "^14.14.31" - "@types/sinonjs__fake-timers" "8.1.1" - "@types/sizzle" "^2.3.2" - arch "^2.2.0" - blob-util "^2.0.2" - bluebird "^3.7.2" - buffer "^5.6.0" - cachedir "^2.3.0" - chalk "^4.1.0" - check-more-types "^2.24.0" - cli-cursor "^3.1.0" - cli-table3 "~0.6.1" - commander "^5.1.0" - common-tags "^1.8.0" - dayjs "^1.10.4" - debug "^4.3.2" - enquirer "^2.3.6" - eventemitter2 "^6.4.3" - execa "4.1.0" - executable "^4.1.1" - extract-zip "2.0.1" - figures "^3.2.0" - fs-extra "^9.1.0" - getos "^3.2.1" - is-ci "^3.0.0" - is-installed-globally "~0.4.0" - lazy-ass "^1.6.0" - listr2 "^3.8.3" - lodash "^4.17.21" - log-symbols "^4.0.0" - minimist "^1.2.6" - ospath "^1.2.2" - pretty-bytes "^5.6.0" - proxy-from-env "1.0.0" - request-progress "^3.0.0" - semver "^7.3.2" - supports-color "^8.1.1" - tmp "~0.2.1" - untildify "^4.0.0" - yauzl "^2.10.0" - -dashdash@^1.12.0: - version "1.14.1" - resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" - integrity sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g== - dependencies: - assert-plus "^1.0.0" - -dayjs@^1.10.4: - version "1.11.5" - resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.5.tgz#00e8cc627f231f9499c19b38af49f56dc0ac5e93" - integrity sha512-CAdX5Q3YW3Gclyo5Vpqkgpj8fSdLQcRuzfX6mC6Phy0nfJ0eGYOeS7m4mt2plDWLAtA4TqTakvbboHvUxfe4iA== - -debug@^3.1.0: - version "3.2.7" - resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a" - integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== - dependencies: - ms "^2.1.1" - debug@^4.0.1, debug@^4.1.1: version "4.3.1" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee" @@ -756,13 +449,6 @@ debug@^4.3.1: dependencies: ms "2.1.2" -debug@^4.3.2: - version "4.3.4" - resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" - integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== - dependencies: - ms "2.1.2" - deep-is@^0.1.3: version "0.1.3" resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34" @@ -773,11 +459,6 @@ deep-is@~0.1.3: resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.4.tgz#a6f2dce612fadd2ef1f519b73551f17e85199831" integrity sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ== -delayed-stream@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" - integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== - dir-glob@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f" @@ -839,27 +520,12 @@ domutils@^2.6.0: domelementtype "^2.2.0" domhandler "^4.2.0" -ecc-jsbn@~0.1.1: - version "0.1.2" - resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" - integrity sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw== - dependencies: - jsbn "~0.1.0" - safer-buffer "^2.1.0" - emoji-regex@^8.0.0: version "8.0.0" resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== -end-of-stream@^1.1.0: - version "1.4.4" - resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" - integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q== - dependencies: - once "^1.4.0" - -enquirer@^2.3.5, enquirer@^2.3.6: +enquirer@^2.3.5: version "2.3.6" resolved "https://registry.yarnpkg.com/enquirer/-/enquirer-2.3.6.tgz#2a7fe5dd634a1e4125a975ec994ff5456dc3734d" integrity sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg== @@ -1259,59 +925,11 @@ esutils@^2.0.2: resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64" integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g== -eventemitter2@^6.4.3: - version "6.4.7" - resolved "https://registry.yarnpkg.com/eventemitter2/-/eventemitter2-6.4.7.tgz#a7f6c4d7abf28a14c1ef3442f21cb306a054271d" - integrity sha512-tYUSVOGeQPKt/eC1ABfhHy5Xd96N3oIijJvN3O9+TsC28T5V9yX9oEfEK5faP0EFSNVOG97qtAS68GBrQB2hDg== - -execa@4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/execa/-/execa-4.1.0.tgz#4e5491ad1572f2f17a77d388c6c857135b22847a" - integrity sha512-j5W0//W7f8UxAn8hXVnwG8tLwdiUy4FJLcSupCg6maBYZDpyBvTApK7KyuI4bKj8KOh1r2YH+6ucuYtJv1bTZA== - dependencies: - cross-spawn "^7.0.0" - get-stream "^5.0.0" - human-signals "^1.1.1" - is-stream "^2.0.0" - merge-stream "^2.0.0" - npm-run-path "^4.0.0" - onetime "^5.1.0" - signal-exit "^3.0.2" - strip-final-newline "^2.0.0" - -executable@^4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/executable/-/executable-4.1.1.tgz#41532bff361d3e57af4d763b70582db18f5d133c" - integrity sha512-8iA79xD3uAch729dUG8xaaBBFGaEa0wdD2VkYLFHwlqosEj/jT66AzcreRDSgV7ehnNLBW2WR5jIXwGKjVdTLg== - dependencies: - pify "^2.2.0" - -extend@^3.0.1, extend@~3.0.2: +extend@^3.0.1: version "3.0.2" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== -extract-zip@2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-2.0.1.tgz#663dca56fe46df890d5f131ef4a06d22bb8ba13a" - integrity sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg== - dependencies: - debug "^4.1.1" - get-stream "^5.1.0" - yauzl "^2.10.0" - optionalDependencies: - "@types/yauzl" "^2.9.1" - -extsprintf@1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" - integrity sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g== - -extsprintf@^1.2.0: - version "1.4.1" - resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.1.tgz#8d172c064867f235c0c84a596806d279bf4bcc07" - integrity sha512-Wrk35e8ydCKDj/ArClo1VrPVmN8zph5V4AtHwIuHhvMXsKf73UT3BOD+azBIW+3wOJ4FhEH7zyaJCFvChjYvMA== - fake-indexeddb@^3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/fake-indexeddb/-/fake-indexeddb-3.1.2.tgz#8073a12ed3b254f7afc064f3cc2629f0110a5303" @@ -1353,20 +971,6 @@ fastq@^1.6.0: dependencies: reusify "^1.0.4" -fd-slicer@~1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/fd-slicer/-/fd-slicer-1.1.0.tgz#25c7c89cb1f9077f8891bbe61d8f390eae256f1e" - integrity sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g== - dependencies: - pend "~1.2.0" - -figures@^3.2.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/figures/-/figures-3.2.0.tgz#625c18bd293c604dc4a8ddb2febf0c88341746af" - integrity sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg== - dependencies: - escape-string-regexp "^1.0.5" - file-entry-cache@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-6.0.1.tgz#211b2dd9659cb0394b073e7323ac3c933d522027" @@ -1394,30 +998,6 @@ flatted@^3.1.0: resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.1.1.tgz#c4b489e80096d9df1dfc97c79871aea7c617c469" integrity sha512-zAoAQiudy+r5SvnSw3KJy5os/oRJYHzrzja/tBDqrZtNhUw8bt6y8OBzMWcjWr+8liV8Eb6yOhw8WZ7VFZ5ZzA== -forever-agent@~0.6.1: - version "0.6.1" - resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" - integrity sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw== - -form-data@~2.3.2: - version "2.3.3" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6" - integrity sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ== - dependencies: - asynckit "^0.4.0" - combined-stream "^1.0.6" - mime-types "^2.1.12" - -fs-extra@^9.1.0: - version "9.1.0" - resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-9.1.0.tgz#5954460c764a8da2094ba3554bf839e6b9a7c86d" - integrity sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ== - dependencies: - at-least-node "^1.0.0" - graceful-fs "^4.2.0" - jsonfile "^6.0.1" - universalify "^2.0.0" - fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" @@ -1438,27 +1018,6 @@ functional-red-black-tree@^1.0.1: resolved "https://registry.yarnpkg.com/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz#1b0ab3bd553b2a0d6399d29c0e3ea0b252078327" integrity sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc= -get-stream@^5.0.0, get-stream@^5.1.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-5.2.0.tgz#4966a1795ee5ace65e706c4b7beb71257d6e22d3" - integrity sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA== - dependencies: - pump "^3.0.0" - -getos@^3.2.1: - version "3.2.1" - resolved "https://registry.yarnpkg.com/getos/-/getos-3.2.1.tgz#0134d1f4e00eb46144c5a9c0ac4dc087cbb27dc5" - integrity sha512-U56CfOK17OKgTVqozZjUKNdkfEv6jk5WISBJ8SHoagjE6L69zOwl3Z+O8myjY9MEW3i2HPWQBt/LTbCgcC973Q== - dependencies: - async "^3.2.0" - -getpass@^0.1.1: - version "0.1.7" - resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" - integrity sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng== - dependencies: - assert-plus "^1.0.0" - glob-parent@^5.1.2: version "5.1.2" resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4" @@ -1478,13 +1037,6 @@ glob@^7.1.3: once "^1.3.0" path-is-absolute "^1.0.0" -global-dirs@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/global-dirs/-/global-dirs-3.0.0.tgz#70a76fe84ea315ab37b1f5576cbde7d48ef72686" - integrity sha512-v8ho2DS5RiCjftj1nD9NmnfaOzTdud7RRnVd9kFNOjqZbISlx5DQ+OrTkywgd0dIt7oFCvKetZSHoHcP3sDdiA== - dependencies: - ini "2.0.0" - globals@^13.6.0: version "13.8.0" resolved "https://registry.yarnpkg.com/globals/-/globals-13.8.0.tgz#3e20f504810ce87a8d72e55aecf8435b50f4c1b3" @@ -1511,11 +1063,6 @@ globby@^11.0.3: merge2 "^1.3.0" slash "^3.0.0" -graceful-fs@^4.1.6, graceful-fs@^4.2.0: - version "4.2.10" - resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.10.tgz#147d3a006da4ca3ce14728c7aefc287c367d7a6c" - integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA== - has-flag@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" @@ -1538,25 +1085,6 @@ he@1.2.0: resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f" integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw== -http-signature@~1.3.6: - version "1.3.6" - resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.3.6.tgz#cb6fbfdf86d1c974f343be94e87f7fc128662cf9" - integrity sha512-3adrsD6zqo4GsTqtO7FyrejHNv+NgiIfAfv68+jVlFmSr9OGy7zrxONceFRLKvnnZA5jbxQBX1u9PpB6Wi32Gw== - dependencies: - assert-plus "^1.0.0" - jsprim "^2.0.2" - sshpk "^1.14.1" - -human-signals@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-1.1.1.tgz#c5b1cd14f50aeae09ab6c59fe63ba3395fe4dfa3" - integrity sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw== - -ieee754@^1.1.13: - version "1.2.1" - resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" - integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== - ignore@^4.0.6: version "4.0.6" resolved "https://registry.yarnpkg.com/ignore/-/ignore-4.0.6.tgz#750e3db5862087b4737ebac8207ffd1ef27b25fc" @@ -1589,11 +1117,6 @@ imurmurhash@^0.1.4: resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" integrity sha1-khi5srkoojixPcT7a21XbyMUU+o= -indent-string@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-4.0.0.tgz#624f8f4497d619b2d9768531d58f4122854d7251" - integrity sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg== - inflight@^1.0.4: version "1.0.6" resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" @@ -1607,18 +1130,6 @@ inherits@2: resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== -ini@2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/ini/-/ini-2.0.0.tgz#e5fd556ecdd5726be978fa1001862eacb0a94bc5" - integrity sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA== - -is-ci@^3.0.0: - version "3.0.1" - resolved "https://registry.yarnpkg.com/is-ci/-/is-ci-3.0.1.tgz#db6ecbed1bd659c43dac0f45661e7674103d1867" - integrity sha512-ZYvCgrefwqoQ6yTyYUbQu64HsITZ3NfKX1lzaEYdkTDcfKzzCI/wthRRYKkdjHKFVgNiXKAKm65Zo1pk2as/QQ== - dependencies: - ci-info "^3.2.0" - is-core-module@^2.8.1: version "2.8.1" resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.8.1.tgz#f59fdfca701d5879d0a6b100a40aa1560ce27211" @@ -1643,54 +1154,21 @@ is-glob@^4.0.0, is-glob@^4.0.1: dependencies: is-extglob "^2.1.1" -is-installed-globally@~0.4.0: - version "0.4.0" - resolved "https://registry.yarnpkg.com/is-installed-globally/-/is-installed-globally-0.4.0.tgz#9a0fd407949c30f86eb6959ef1b7994ed0b7b520" - integrity sha512-iwGqO3J21aaSkC7jWnHP/difazwS7SFeIqxv6wEtLU8Y5KlzFTjyqcSIT0d8s4+dDhKytsk9PJZ2BkS5eZwQRQ== - dependencies: - global-dirs "^3.0.0" - is-path-inside "^3.0.2" - is-number@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/is-number/-/is-number-7.0.0.tgz#7535345b896734d5f80c4d06c50955527a14f12b" integrity sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng== -is-path-inside@^3.0.2: - version "3.0.3" - resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.3.tgz#d231362e53a07ff2b0e0ea7fed049161ffd16283" - integrity sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ== - is-plain-obj@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/is-plain-obj/-/is-plain-obj-2.1.0.tgz#45e42e37fccf1f40da8e5f76ee21515840c09287" integrity sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA== -is-stream@^2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.1.tgz#fac1e3d53b97ad5a9d0ae9cef2389f5810a5c077" - integrity sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg== - -is-typedarray@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" - integrity sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA== - -is-unicode-supported@^0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz#3f26c76a809593b52bfa2ecb5710ed2779b522a7" - integrity sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw== - isexe@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" integrity sha1-6PvzdNxVb/iUehDcsFctYz8s+hA= -isstream@~0.1.2: - version "0.1.2" - resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" - integrity sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g== - js-tokens@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" @@ -1704,11 +1182,6 @@ js-yaml@^3.13.1: argparse "^1.0.7" esprima "^4.0.0" -jsbn@~0.1.0: - version "0.1.1" - resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" - integrity sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg== - json-schema-traverse@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" @@ -1719,45 +1192,11 @@ json-schema-traverse@^1.0.0: resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz#ae7bcb3656ab77a73ba5c49bf654f38e6b6860e2" integrity sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug== -json-schema@0.4.0: - version "0.4.0" - resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.4.0.tgz#f7de4cf6efab838ebaeb3236474cbba5a1930ab5" - integrity sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA== - json-stable-stringify-without-jsonify@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651" integrity sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE= -json-stringify-safe@~5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" - integrity sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA== - -jsonfile@^6.0.1: - version "6.1.0" - resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.1.0.tgz#bc55b2634793c679ec6403094eb13698a6ec0aae" - integrity sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ== - dependencies: - universalify "^2.0.0" - optionalDependencies: - graceful-fs "^4.1.6" - -jsprim@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-2.0.2.tgz#77ca23dbcd4135cd364800d22ff82c2185803d4d" - integrity sha512-gqXddjPqQ6G40VdnI6T6yObEC+pDNvyP95wdQhkWkg7crHH3km5qP1FsOXEkzEQwnz6gz5qGTn1c2Y52wP3OyQ== - dependencies: - assert-plus "1.0.0" - extsprintf "1.3.0" - json-schema "0.4.0" - verror "1.10.0" - -lazy-ass@^1.6.0: - version "1.6.0" - resolved "https://registry.yarnpkg.com/lazy-ass/-/lazy-ass-1.6.0.tgz#7999655e8646c17f089fdd187d150d3324d54513" - integrity sha512-cc8oEVoctTvsFZ/Oje/kGnHbpWHYBe8IAJe4C0QNc3t8uM/0Y8+erSz/7Y1ALuXTEZTMvxXwO6YbX1ey3ujiZw== - levn@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/levn/-/levn-0.4.1.tgz#ae4562c007473b932a6200d403268dd2fffc6ade" @@ -1774,20 +1213,6 @@ levn@~0.3.0: prelude-ls "~1.1.2" type-check "~0.3.2" -listr2@^3.8.3: - version "3.14.0" - resolved "https://registry.yarnpkg.com/listr2/-/listr2-3.14.0.tgz#23101cc62e1375fd5836b248276d1d2b51fdbe9e" - integrity sha512-TyWI8G99GX9GjE54cJ+RrNMcIFBfwMPxc3XTFiAYGN4s10hWROGtOg7+O6u6LE3mNkyld7RSLE6nrKBvTfcs3g== - dependencies: - cli-truncate "^2.1.0" - colorette "^2.0.16" - log-update "^4.0.0" - p-map "^4.0.0" - rfdc "^1.3.0" - rxjs "^7.5.1" - through "^2.3.8" - wrap-ansi "^7.0.0" - lodash.clonedeep@^4.5.0: version "4.5.0" resolved "https://registry.yarnpkg.com/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz#e23f3f9c4f8fbdde872529c1071857a086e5ccef" @@ -1798,39 +1223,16 @@ lodash.merge@^4.6.2: resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a" integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== -lodash.once@^4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac" - integrity sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg== - lodash.truncate@^4.4.2: version "4.4.2" resolved "https://registry.yarnpkg.com/lodash.truncate/-/lodash.truncate-4.4.2.tgz#5a350da0b1113b837ecfffd5812cbe58d6eae193" integrity sha1-WjUNoLERO4N+z//VgSy+WNbq4ZM= -lodash@^4.17.21, lodash@^4.7.0: +lodash@^4.7.0: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== -log-symbols@^4.0.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-4.1.0.tgz#3fbdbb95b4683ac9fc785111e792e558d4abd503" - integrity sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg== - dependencies: - chalk "^4.1.0" - is-unicode-supported "^0.1.0" - -log-update@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/log-update/-/log-update-4.0.0.tgz#589ecd352471f2a1c0c570287543a64dfd20e0a1" - integrity sha512-9fkkDevMefjg0mmzWFBW8YkFP91OrizzkW3diF7CpG+S2EYdy4+TVfGwz1zeF8x7hCx1ovSPTOE9Ngib74qqUg== - dependencies: - ansi-escapes "^4.3.0" - cli-cursor "^3.1.0" - slice-ansi "^4.0.0" - wrap-ansi "^6.2.0" - lru-cache@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-6.0.0.tgz#6d6fe6570ebd96aaf90fcad1dafa3b2566db3a94" @@ -1855,11 +1257,6 @@ merge-options@^3.0.4: dependencies: is-plain-obj "^2.1.0" -merge-stream@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/merge-stream/-/merge-stream-2.0.0.tgz#52823629a14dd00c9770fb6ad47dc6310f2c1f60" - integrity sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w== - merge2@^1.3.0: version "1.4.1" resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae" @@ -1873,23 +1270,6 @@ micromatch@^4.0.4: braces "^3.0.1" picomatch "^2.2.3" -mime-db@1.52.0: - version "1.52.0" - resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" - integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== - -mime-types@^2.1.12, mime-types@~2.1.19: - version "2.1.35" - resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" - integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== - dependencies: - mime-db "1.52.0" - -mimic-fn@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" - integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg== - minimatch@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" @@ -1897,21 +1277,11 @@ minimatch@^3.0.4: dependencies: brace-expansion "^1.1.7" -minimist@^1.2.6: - version "1.2.6" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44" - integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q== - ms@2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -ms@^2.1.1: - version "2.1.3" - resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" - integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== - nanoid@^3.3.3: version "3.3.4" resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.4.tgz#730b67e3cd09e2deacf03c027c81c9d9dbc5e8ab" @@ -1930,13 +1300,6 @@ node-html-parser@^4.0.0: css-select "^4.1.3" he "1.2.0" -npm-run-path@^4.0.0: - version "4.0.1" - resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-4.0.1.tgz#b7ecd1e5ed53da8e37a55e1c2269e0b97ed748ea" - integrity sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw== - dependencies: - path-key "^3.0.0" - nth-check@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-2.0.0.tgz#1bb4f6dac70072fc313e8c9cd1417b5074c0a125" @@ -1951,20 +1314,13 @@ off-color@^2.0.0: dependencies: core-js "^3.6.5" -once@^1.3.0, once@^1.3.1, once@^1.4.0: +once@^1.3.0: version "1.4.0" resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" integrity sha1-WDsap3WWHUsROsF9nFC6753Xa9E= dependencies: wrappy "1" -onetime@^5.1.0: - version "5.1.2" - resolved "https://registry.yarnpkg.com/onetime/-/onetime-5.1.2.tgz#d0e96ebb56b07476df1dd9c4806e5237985ca45e" - integrity sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg== - dependencies: - mimic-fn "^2.1.0" - optionator@^0.8.1: version "0.8.3" resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.8.3.tgz#84fa1d036fe9d3c7e21d99884b601167ec8fb495" @@ -1989,18 +1345,6 @@ optionator@^0.9.1: type-check "^0.4.0" word-wrap "^1.2.3" -ospath@^1.2.2: - version "1.2.2" - resolved "https://registry.yarnpkg.com/ospath/-/ospath-1.2.2.tgz#1276639774a3f8ef2572f7fe4280e0ea4550c07b" - integrity sha512-o6E5qJV5zkAbIDNhGSIlyOhScKXgQrSRMilfph0clDfM0nEnBOlKlH4sWDmG95BW/CvwNz0vmm7dJVtU2KlMiA== - -p-map@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/p-map/-/p-map-4.0.0.tgz#bb2f95a5eda2ec168ec9274e06a747c3e2904d2b" - integrity sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ== - dependencies: - aggregate-error "^3.0.0" - parent-module@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/parent-module/-/parent-module-1.0.1.tgz#691d2709e78c79fae3a156622452d00762caaaa2" @@ -2013,7 +1357,7 @@ path-is-absolute@^1.0.0: resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" integrity sha1-F0uSaHNVNP+8es5r9TpanhtcX18= -path-key@^3.0.0, path-key@^3.1.0: +path-key@^3.1.0: version "3.1.1" resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375" integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q== @@ -2028,16 +1372,6 @@ path-type@^4.0.0: resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b" integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw== -pend@~1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50" - integrity sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg== - -performance-now@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" - integrity sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow== - picocolors@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" @@ -2048,11 +1382,6 @@ picomatch@^2.2.3: resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.0.tgz#f1f061de8f6a4bf022892e2d128234fb98302972" integrity sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw== -pify@^2.2.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c" - integrity sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog== - playwright-core@1.27.1: version "1.27.1" resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.27.1.tgz#840ef662e55a3ed759d8b5d3d00a5f885a7184f4" @@ -2096,44 +1425,16 @@ prelude-ls@~1.1.2: resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54" integrity sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ= -pretty-bytes@^5.6.0: - version "5.6.0" - resolved "https://registry.yarnpkg.com/pretty-bytes/-/pretty-bytes-5.6.0.tgz#356256f643804773c82f64723fe78c92c62beaeb" - integrity sha512-FFw039TmrBqFK8ma/7OL3sDz/VytdtJr044/QUJtH0wK9lb9jLq9tJyIxUwtQJHwar2BqtiA4iCWSwo9JLkzFg== - progress@^2.0.0: version "2.0.3" resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== -proxy-from-env@1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.0.0.tgz#33c50398f70ea7eb96d21f7b817630a55791c7ee" - integrity sha512-F2JHgJQ1iqwnHDcQjVBsq3n/uoaFL+iPW/eAeL7kVxy/2RrWaN4WroKjjvbsoRtv0ftelNyC01bjRhn/bhcf4A== - -psl@^1.1.28: - version "1.9.0" - resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" - integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag== - -pump@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64" - integrity sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww== - dependencies: - end-of-stream "^1.1.0" - once "^1.3.1" - punycode@^2.1.0, punycode@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== -qs@~6.5.2: - version "6.5.3" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.3.tgz#3aeeffc91967ef6e35c0e488ef46fb296ab76aad" - integrity sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA== - queue-microtask@^1.2.2: version "1.2.3" resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243" @@ -2159,13 +1460,6 @@ regexpp@^3.1.0: resolved "https://registry.yarnpkg.com/regexpp/-/regexpp-3.1.0.tgz#206d0ad0a5648cffbdb8ae46438f3dc51c9f78e2" integrity sha512-ZOIzd8yVsQQA7j8GCSlPGXwg5PfmA1mrq0JP4nGhh54LaKN3xdai/vHUDu74pKwV8OxseMS65u2NImosQcSD0Q== -request-progress@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/request-progress/-/request-progress-3.0.0.tgz#4ca754081c7fec63f505e4faa825aa06cd669dbe" - integrity sha512-MnWzEHHaxHO2iWiQuHrUPBi/1WeBf5PkxQqNyNvLl9VAYSdXkP8tQ3pBSeCPD+yw0v0Aq1zosWLz0BdeXpWwZg== - dependencies: - throttleit "^1.0.0" - require-from-string@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/require-from-string/-/require-from-string-2.0.2.tgz#89a7fdd938261267318eafe14f9c32e598c36909" @@ -2185,25 +1479,12 @@ resolve@^1.22.0: path-parse "^1.0.7" supports-preserve-symlinks-flag "^1.0.0" -restore-cursor@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/restore-cursor/-/restore-cursor-3.1.0.tgz#39f67c54b3a7a58cea5236d95cf0034239631f7e" - integrity sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA== - dependencies: - onetime "^5.1.0" - signal-exit "^3.0.2" - reusify@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76" integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw== -rfdc@^1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/rfdc/-/rfdc-1.3.0.tgz#d0b7c441ab2720d05dc4cf26e01c89631d9da08b" - integrity sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA== - -rimraf@^3.0.0, rimraf@^3.0.2: +rimraf@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA== @@ -2224,23 +1505,11 @@ run-parallel@^1.1.9: dependencies: queue-microtask "^1.2.2" -rxjs@^7.5.1: - version "7.5.6" - resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-7.5.6.tgz#0446577557862afd6903517ce7cae79ecb9662bc" - integrity sha512-dnyv2/YsXhnm461G+R/Pe5bWP41Nm6LBXEYWI6eiFP4fiwx6WRI/CD0zbdVAudd9xwLEF2IDcKXLHit0FYjUzw== - dependencies: - tslib "^2.1.0" - -safe-buffer@^5.0.1, safe-buffer@^5.1.2: +safe-buffer@^5.0.1: version "5.2.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== -safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: - version "2.1.2" - resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" - integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== - semver@^7.2.1, semver@^7.3.5: version "7.3.5" resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.5.tgz#0b621c879348d8998e4b0e4be94b3f12e6018ef7" @@ -2248,13 +1517,6 @@ semver@^7.2.1, semver@^7.3.5: dependencies: lru-cache "^6.0.0" -semver@^7.3.2: - version "7.3.7" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.7.tgz#12c5b649afdbf9049707796e22a4028814ce523f" - integrity sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g== - dependencies: - lru-cache "^6.0.0" - setimmediate@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/setimmediate/-/setimmediate-1.0.5.tgz#290cbb232e306942d7d7ea9b83732ab7856f8285" @@ -2272,25 +1534,11 @@ shebang-regex@^3.0.0: resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172" integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== -signal-exit@^3.0.2: - version "3.0.7" - resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9" - integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ== - slash@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634" integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q== -slice-ansi@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-3.0.0.tgz#31ddc10930a1b7e0b67b08c96c2f49b77a789787" - integrity sha512-pSyv7bSTC7ig9Dcgbw9AuRNUb5k5V6oDudjZoMBSr13qpLBG7tB+zgCkARjq7xIUgdz5P1Qe8u+rSGdouOOIyQ== - dependencies: - ansi-styles "^4.0.0" - astral-regex "^2.0.0" - is-fullwidth-code-point "^3.0.0" - slice-ansi@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-4.0.0.tgz#500e8dd0fd55b05815086255b3195adf2a45fe6b" @@ -2315,35 +1563,11 @@ sprintf-js@~1.0.2: resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw= -sshpk@^1.14.1: - version "1.17.0" - resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.17.0.tgz#578082d92d4fe612b13007496e543fa0fbcbe4c5" - integrity sha512-/9HIEs1ZXGhSPE8X6Ccm7Nam1z8KcoCqPdI7ecm1N33EzAetWahvQWVqLZtaZQ+IDKX4IyA2o0gBzqIMkAagHQ== - dependencies: - asn1 "~0.2.3" - assert-plus "^1.0.0" - bcrypt-pbkdf "^1.0.0" - dashdash "^1.12.0" - ecc-jsbn "~0.1.1" - getpass "^0.1.1" - jsbn "~0.1.0" - safer-buffer "^2.0.2" - tweetnacl "~0.14.0" - stable@^0.1.8: version "0.1.8" resolved "https://registry.yarnpkg.com/stable/-/stable-0.1.8.tgz#836eb3c8382fe2936feaf544631017ce7d47a3cf" integrity sha512-ji9qxRnOVfcuLDySj9qzhGSEFVobyt1kIOSkj1qZzYLzq7Tos/oUUWvotUPQLlrsidqsK6tBH89Bc9kL5zHA6w== -string-width@^4.1.0: - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - string-width@^4.2.0: version "4.2.2" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.2.tgz#dafd4f9559a7585cfba529c6a0a4f73488ebd4c5" @@ -2360,18 +1584,6 @@ strip-ansi@^6.0.0: dependencies: ansi-regex "^5.0.0" -strip-ansi@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - -strip-final-newline@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/strip-final-newline/-/strip-final-newline-2.0.0.tgz#89b852fb2fcbe936f6f4b3187afb0a12c1ab58ad" - integrity sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA== - strip-json-comments@^3.1.0, strip-json-comments@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006" @@ -2391,13 +1603,6 @@ supports-color@^7.1.0: dependencies: has-flag "^4.0.0" -supports-color@^8.1.1: - version "8.1.1" - resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-8.1.1.tgz#cd6fc17e28500cff56c1b86c0a7fd4a54a73005c" - integrity sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q== - dependencies: - has-flag "^4.0.0" - supports-preserve-symlinks-flag@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09" @@ -2438,23 +1643,6 @@ text-table@^0.2.0: resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" integrity sha1-f17oI66AUgfACvLfSoTsP8+lcLQ= -throttleit@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/throttleit/-/throttleit-1.0.0.tgz#9e785836daf46743145a5984b6268d828528ac6c" - integrity sha512-rkTVqu6IjfQ/6+uNuuc3sZek4CEYxTJom3IktzgdSxcZqdARuebbA/f4QmAxMQIxqq9ZLEUkSYqvuk1I6VKq4g== - -through@^2.3.8: - version "2.3.8" - resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" - integrity sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg== - -tmp@~0.2.1: - version "0.2.1" - resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.2.1.tgz#8457fc3037dcf4719c251367a1af6500ee1ccf14" - integrity sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ== - dependencies: - rimraf "^3.0.0" - to-regex-range@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-5.0.1.tgz#1648c44aae7c8d988a326018ed72f5b4dd0392e4" @@ -2462,14 +1650,6 @@ to-regex-range@^5.0.1: dependencies: is-number "^7.0.0" -tough-cookie@~2.5.0: - version "2.5.0" - resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" - integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== - dependencies: - psl "^1.1.28" - punycode "^2.1.1" - tr46@^2.0.2: version "2.1.0" resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240" @@ -2482,11 +1662,6 @@ tslib@^1.8.1: resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== -tslib@^2.1.0: - version "2.4.0" - resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.4.0.tgz#7cecaa7f073ce680a05847aa77be941098f36dc3" - integrity sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ== - tsutils@^3.21.0: version "3.21.0" resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.21.0.tgz#b48717d394cea6c1e096983eed58e9d61715b623" @@ -2494,18 +1669,6 @@ tsutils@^3.21.0: dependencies: tslib "^1.8.1" -tunnel-agent@^0.6.0: - version "0.6.0" - resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" - integrity sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w== - dependencies: - safe-buffer "^5.0.1" - -tweetnacl@^0.14.3, tweetnacl@~0.14.0: - version "0.14.5" - resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" - integrity sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA== - type-check@^0.4.0, type-check@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1" @@ -2525,11 +1688,6 @@ type-fest@^0.20.2: resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.20.2.tgz#1bf207f4b28f91583666cb5fbd327887301cd5f4" integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ== -type-fest@^0.21.3: - version "0.21.3" - resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.21.3.tgz#d260a24b0198436e133fa26a524a6d65fa3b2e37" - integrity sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w== - typescript@^4.7.0: version "4.7.4" resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.7.4.tgz#1a88596d1cf47d59507a1bcdfb5b9dfe4d488235" @@ -2554,16 +1712,6 @@ typeson@^6.0.0: resolved "https://registry.yarnpkg.com/typeson/-/typeson-6.1.0.tgz#5b2a53705a5f58ff4d6f82f965917cabd0d7448b" integrity sha512-6FTtyGr8ldU0pfbvW/eOZrEtEkczHRUtduBnA90Jh9kMPCiFNnXIon3vF41N0S4tV1HHQt4Hk1j4srpESziCaA== -universalify@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.0.tgz#75a4984efedc4b08975c5aeb73f530d02df25717" - integrity sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ== - -untildify@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" - integrity sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw== - uri-js@^4.2.2: version "4.4.1" resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.4.1.tgz#9b1a52595225859e55f669d928f88c6c57f2a77e" @@ -2571,25 +1719,11 @@ uri-js@^4.2.2: dependencies: punycode "^2.1.0" -uuid@^8.3.2: - version "8.3.2" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" - integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== - v8-compile-cache@^2.0.3: version "2.3.0" resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz#2de19618c66dc247dcfb6f99338035d8245a2cee" integrity sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA== -verror@1.10.0: - version "1.10.0" - resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" - integrity sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw== - dependencies: - assert-plus "^1.0.0" - core-util-is "1.0.2" - extsprintf "^1.2.0" - vite@^2.9.8: version "2.9.8" resolved "https://registry.yarnpkg.com/vite/-/vite-2.9.8.tgz#2c2cb0790beb0fbe4b8c0995b80fe691a91c2545" @@ -2633,24 +1767,6 @@ word-wrap@^1.2.3, word-wrap@~1.2.3: resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ== -wrap-ansi@^6.2.0: - version "6.2.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-6.2.0.tgz#e9393ba07102e6c91a3b221478f0257cd2856e53" - integrity sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrappy@1: version "1.0.2" resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" @@ -2667,11 +1783,3 @@ yallist@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== - -yauzl@^2.10.0: - version "2.10.0" - resolved "https://registry.yarnpkg.com/yauzl/-/yauzl-2.10.0.tgz#c7eb17c93e112cb1086fa6d8e51fb0667b79a5f9" - integrity sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g== - dependencies: - buffer-crc32 "~0.2.3" - fd-slicer "~1.1.0" From 374fb08c98b702908a824ee89962f6158c6f17e9 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 24 Jan 2023 18:08:29 +0530 Subject: [PATCH 269/323] Fix typescript errors --- .../toast/BaseToastNotificationViewModel.ts | 9 ++++---- .../toast/CallToastNotificationViewModel.ts | 22 ++++++++++++------- .../session/toast/ToastCollectionViewModel.ts | 3 ++- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/domain/session/toast/BaseToastNotificationViewModel.ts b/src/domain/session/toast/BaseToastNotificationViewModel.ts index 01ec688b..41e20e42 100644 --- a/src/domain/session/toast/BaseToastNotificationViewModel.ts +++ b/src/domain/session/toast/BaseToastNotificationViewModel.ts @@ -17,14 +17,15 @@ limitations under the License. import {ErrorReportViewModel} from "../../ErrorReportViewModel"; import {Options as BaseOptions} from "../../ViewModel"; import type {Session} from "../../../matrix/Session.js"; +import {SegmentType} from "../../navigation"; -export type BaseClassOptions = { +export type BaseClassOptions = { dismiss: () => void; session: Session; -} & BaseOptions; +} & BaseOptions; -export abstract class BaseToastNotificationViewModel extends ErrorReportViewModel { - constructor(options: T) { +export abstract class BaseToastNotificationViewModel = BaseClassOptions> extends ErrorReportViewModel { + constructor(options: O) { super(options); } diff --git a/src/domain/session/toast/CallToastNotificationViewModel.ts b/src/domain/session/toast/CallToastNotificationViewModel.ts index 4c005cc5..3281b86a 100644 --- a/src/domain/session/toast/CallToastNotificationViewModel.ts +++ b/src/domain/session/toast/CallToastNotificationViewModel.ts @@ -18,15 +18,21 @@ import type {Room} from "../../../matrix/room/Room.js"; import {IAvatarContract, avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../avatar"; import {LocalMedia} from "../../../matrix/calls/LocalMedia"; import {BaseClassOptions, BaseToastNotificationViewModel} from "./BaseToastNotificationViewModel"; +import {SegmentType} from "../../navigation"; -type Options = { +type Options = { call: GroupCall; room: Room; -} & BaseClassOptions; +} & BaseClassOptions; +// Since we access the room segment below, the segment type +// needs to at least contain the room segment! +type MinimumNeededSegmentType = { + "room": string; +}; -export class CallToastNotificationViewModel extends BaseToastNotificationViewModel implements IAvatarContract { - constructor(options: Options) { +export class CallToastNotificationViewModel = Options> extends BaseToastNotificationViewModel implements IAvatarContract { + constructor(options: O) { super(options); this.track( this.call.members.subscribe({ @@ -46,8 +52,8 @@ export class CallToastNotificationViewModel extends BaseToastNotificationViewMod ); // Dismiss the toast if the room is opened manually this.track( - this.navigation.observe("room").subscribe(roomId => { - if (roomId === this.call.roomId) { + this.navigation.observe("room").subscribe((roomId) => { + if ((roomId as unknown as string) === this.call.roomId) { this.dismiss(); } })); @@ -58,8 +64,8 @@ export class CallToastNotificationViewModel extends BaseToastNotificationViewMod const stream = await this.platform.mediaDevices.getMediaTracks(false, true); const localMedia = new LocalMedia().withUserMedia(stream); await this.call.join(localMedia, log); - const url = this.urlCreator.openRoomActionUrl(this.call.roomId); - this.urlCreator.pushUrl(url); + const url = this.urlRouter.openRoomActionUrl(this.call.roomId); + this.urlRouter.pushUrl(url); }); } diff --git a/src/domain/session/toast/ToastCollectionViewModel.ts b/src/domain/session/toast/ToastCollectionViewModel.ts index 30e424ae..18b58b76 100644 --- a/src/domain/session/toast/ToastCollectionViewModel.ts +++ b/src/domain/session/toast/ToastCollectionViewModel.ts @@ -20,12 +20,13 @@ import {ViewModel, Options as BaseOptions} from "../../ViewModel"; import type {GroupCall} from "../../../matrix/calls/group/GroupCall"; import type {Room} from "../../../matrix/room/Room.js"; import type {Session} from "../../../matrix/Session.js"; +import type {SegmentType} from "../../navigation"; type Options = { session: Session; } & BaseOptions; -export class ToastCollectionViewModel extends ViewModel { +export class ToastCollectionViewModel extends ViewModel { public readonly toastViewModels: ObservableArray = new ObservableArray(); constructor(options: Options) { From c2fab59f582943e59f89bce7c8c1afce580c1375 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 24 Jan 2023 23:15:32 +0100 Subject: [PATCH 270/323] ensure call isn't cleared by onCallUpdate when setting up member list --- src/domain/session/room/timeline/tiles/CallTile.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index 8887b834..4583de49 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -29,14 +29,15 @@ export class CallTile extends SimpleTile { constructor(entry, options) { super(entry, options); const calls = this.getOption("session").callHandler.calls; - this._call = calls.get(this._entry.stateKey); this._callSubscription = undefined; - if (this._call) { + const call = calls.get(this._entry.stateKey); + if (call && !call.isTerminated) { + this._call = call; + this.memberViewModels = this._setupMembersList(this._call); this._callSubscription = this.track(this._call.disposableOn("change", () => { this._onCallUpdate(); })); this._onCallUpdate(); - this.memberViewModels = this._setupMembersList(this._call); } } From 98416f8c35bf3e8da6b25628bd06c334de92d02f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 24 Jan 2023 23:17:04 +0100 Subject: [PATCH 271/323] also calculate days in formatDuration --- src/platform/web/dom/TimeFormatter.ts | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/platform/web/dom/TimeFormatter.ts b/src/platform/web/dom/TimeFormatter.ts index cebc6ec4..916b18bb 100644 --- a/src/platform/web/dom/TimeFormatter.ts +++ b/src/platform/web/dom/TimeFormatter.ts @@ -24,6 +24,7 @@ enum TimeScope { const MINUTES_IN_MS = 60 * 1000; const HOURS_IN_MS = MINUTES_IN_MS * 60; +const DAYS_IN_MS = HOURS_IN_MS * 24; export class TimeFormatter implements ITimeFormatter { @@ -80,8 +81,13 @@ export class TimeFormatter implements ITimeFormatter { } formatDuration(milliseconds: number): string { + let days = 0; let hours = 0; let minutes = 0; + if (milliseconds > DAYS_IN_MS) { + days = Math.floor(milliseconds / DAYS_IN_MS); + milliseconds -= days * DAYS_IN_MS; + } if (milliseconds > HOURS_IN_MS) { hours = Math.floor(milliseconds / HOURS_IN_MS); milliseconds -= hours * HOURS_IN_MS; @@ -91,13 +97,18 @@ export class TimeFormatter implements ITimeFormatter { milliseconds -= minutes * MINUTES_IN_MS; } const seconds = Math.floor(milliseconds / 1000); - if (hours) { - return `${hours}h ${minutes}m ${seconds}s`; + let result = ""; + if (days) { + result = `${days}d `; } - if (minutes) { - return `${minutes}m ${seconds}s`; + if (hours || days) { + result += `${hours}h `; } - return `${seconds}s`; + if (minutes || hours || days) { + result += `${minutes}m `; + } + result += `${seconds}s`; + return result; } } From 47d9773fc89c0edb948c371f76df4435ae895334 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 24 Jan 2023 23:17:20 +0100 Subject: [PATCH 272/323] more style changes --- .../web/ui/css/themes/element/timeline.css | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/platform/web/ui/css/themes/element/timeline.css b/src/platform/web/ui/css/themes/element/timeline.css index 5dc86276..16c758a3 100644 --- a/src/platform/web/ui/css/themes/element/timeline.css +++ b/src/platform/web/ui/css/themes/element/timeline.css @@ -449,16 +449,26 @@ only loads when the top comes into view*/ } .CallTileView_members > * { - margin-left: -16px; + margin-right: -16px; } + .CallTileView_members { display: flex; } +.CallTileView_title { + font-weight: bold; +} + +.CallTileView_subtitle { + font-size: 12px; +} + .CallTileView_memberCount::before { content: url('./icons/room-members.svg?primary=text-color'); - background-repeat: no-repeat; - background-size: 24px; - width: 24px; - height: 24px; + width: 16px; + height: 16px; + display: inline-flex; + vertical-align: bottom; + margin-right: 4px; } From e140a4ba64f54a4f897aa8ff8367c9047bffb091 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 23 Jan 2023 15:06:06 +0100 Subject: [PATCH 273/323] element call puts string in terminated, not a boolean --- src/matrix/calls/group/GroupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 0b8c7db5..d323f6b2 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -140,7 +140,7 @@ export class GroupCall extends EventEmitter<{change: never}> { get members(): BaseObservableMap { return this._members; } get isTerminated(): boolean { - return this.callContent?.["m.terminated"] === true; + return !!this.callContent?.["m.terminated"]; } get isRinging(): boolean { From 043ad988669842d8ea71d822714af0263fcb9a62 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 23 Jan 2023 15:06:24 +0100 Subject: [PATCH 274/323] element call also terminates prompt calls, so do so too --- src/matrix/calls/group/GroupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index d323f6b2..6407584a 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -286,7 +286,7 @@ export class GroupCall extends EventEmitter<{change: never}> { const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCallMember, this.options.ownUserId, memberContent, {log}); await request.response(); // our own user isn't included in members, so not in the count - if (this.intent === CallIntent.Ring && this._members.size === 0) { + if ((this.intent === CallIntent.Ring || this.intent === CallIntent.Prompt) && this._members.size === 0) { await this.terminate(log); } } else { From 7f422882dd0ed52e3df418318ed42f999fe252dc Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 23 Jan 2023 15:08:38 +0100 Subject: [PATCH 275/323] make call error view clickable above video elements --- src/platform/web/ui/css/themes/element/call.css | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/platform/web/ui/css/themes/element/call.css b/src/platform/web/ui/css/themes/element/call.css index c0f1550e..3efe3c56 100644 --- a/src/platform/web/ui/css/themes/element/call.css +++ b/src/platform/web/ui/css/themes/element/call.css @@ -25,11 +25,13 @@ limitations under the License. } .CallView_error { - color: red; - font-weight: bold; align-self: start; justify-self: center; margin: 16px; + /** Chrome (v100) requires this to make the buttons clickable + * where they overlap with the video element, even though + * the buttons come later in the DOM. */ + z-index: 1; } .CallView_members { From a278086c3787366cf8738cc015c03629d08d93a4 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 26 Jan 2023 09:58:24 +0100 Subject: [PATCH 276/323] fix error at time unit boundary --- src/platform/web/dom/TimeFormatter.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/platform/web/dom/TimeFormatter.ts b/src/platform/web/dom/TimeFormatter.ts index 916b18bb..090f8076 100644 --- a/src/platform/web/dom/TimeFormatter.ts +++ b/src/platform/web/dom/TimeFormatter.ts @@ -84,15 +84,15 @@ export class TimeFormatter implements ITimeFormatter { let days = 0; let hours = 0; let minutes = 0; - if (milliseconds > DAYS_IN_MS) { + if (milliseconds >= DAYS_IN_MS) { days = Math.floor(milliseconds / DAYS_IN_MS); milliseconds -= days * DAYS_IN_MS; } - if (milliseconds > HOURS_IN_MS) { + if (milliseconds >= HOURS_IN_MS) { hours = Math.floor(milliseconds / HOURS_IN_MS); milliseconds -= hours * HOURS_IN_MS; } - if (milliseconds > MINUTES_IN_MS) { + if (milliseconds >= MINUTES_IN_MS) { minutes = Math.floor(milliseconds / MINUTES_IN_MS); milliseconds -= minutes * MINUTES_IN_MS; } From 0f91f2065cb069dffd6e985dd53adbc88923ef15 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 26 Jan 2023 10:25:22 +0100 Subject: [PATCH 277/323] extract to src/utils as it doesn't assume the DOM --- src/platform/web/dom/TimeFormatter.ts | 39 ++------------------------- src/utils/timeFormatting.ts | 36 +++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 37 deletions(-) create mode 100644 src/utils/timeFormatting.ts diff --git a/src/platform/web/dom/TimeFormatter.ts b/src/platform/web/dom/TimeFormatter.ts index 090f8076..c25e902b 100644 --- a/src/platform/web/dom/TimeFormatter.ts +++ b/src/platform/web/dom/TimeFormatter.ts @@ -16,15 +16,7 @@ limitations under the License. import type { ITimeFormatter } from "../../types/types"; import {Clock} from "./Clock"; - -enum TimeScope { - Minute = 60 * 1000, - Day = 24 * 60 * 60 * 1000, -} - -const MINUTES_IN_MS = 60 * 1000; -const HOURS_IN_MS = MINUTES_IN_MS * 60; -const DAYS_IN_MS = HOURS_IN_MS * 24; +import {formatDuration, TimeScope} from "../../../utils/timeFormatting"; export class TimeFormatter implements ITimeFormatter { @@ -81,34 +73,7 @@ export class TimeFormatter implements ITimeFormatter { } formatDuration(milliseconds: number): string { - let days = 0; - let hours = 0; - let minutes = 0; - if (milliseconds >= DAYS_IN_MS) { - days = Math.floor(milliseconds / DAYS_IN_MS); - milliseconds -= days * DAYS_IN_MS; - } - if (milliseconds >= HOURS_IN_MS) { - hours = Math.floor(milliseconds / HOURS_IN_MS); - milliseconds -= hours * HOURS_IN_MS; - } - if (milliseconds >= MINUTES_IN_MS) { - minutes = Math.floor(milliseconds / MINUTES_IN_MS); - milliseconds -= minutes * MINUTES_IN_MS; - } - const seconds = Math.floor(milliseconds / 1000); - let result = ""; - if (days) { - result = `${days}d `; - } - if (hours || days) { - result += `${hours}h `; - } - if (minutes || hours || days) { - result += `${minutes}m `; - } - result += `${seconds}s`; - return result; + return formatDuration(milliseconds); } } diff --git a/src/utils/timeFormatting.ts b/src/utils/timeFormatting.ts new file mode 100644 index 00000000..8d052e95 --- /dev/null +++ b/src/utils/timeFormatting.ts @@ -0,0 +1,36 @@ +export enum TimeScope { + Minute = 60 * 1000, + Hours = 60 * TimeScope.Minute, + Day = 24 * TimeScope.Hours, +} + +export function formatDuration(milliseconds: number): string { + let days = 0; + let hours = 0; + let minutes = 0; + if (milliseconds >= TimeScope.Day) { + days = Math.floor(milliseconds / TimeScope.Day); + milliseconds -= days * TimeScope.Day; + } + if (milliseconds >= TimeScope.Hours) { + hours = Math.floor(milliseconds / TimeScope.Hours); + milliseconds -= hours * TimeScope.Hours; + } + if (milliseconds >= TimeScope.Minute) { + minutes = Math.floor(milliseconds / TimeScope.Minute); + milliseconds -= minutes * TimeScope.Minute; + } + const seconds = Math.floor(milliseconds / 1000); + let result = ""; + if (days) { + result = `${days}d `; + } + if (hours || days) { + result += `${hours}h `; + } + if (minutes || hours || days) { + result += `${minutes}m `; + } + result += `${seconds}s`; + return result; +} From cd76619953fde153c257b0b6486129eeeafeb859 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 26 Jan 2023 10:30:29 +0100 Subject: [PATCH 278/323] add header --- src/utils/timeFormatting.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/utils/timeFormatting.ts b/src/utils/timeFormatting.ts index 8d052e95..2f2ae8c0 100644 --- a/src/utils/timeFormatting.ts +++ b/src/utils/timeFormatting.ts @@ -1,3 +1,19 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + export enum TimeScope { Minute = 60 * 1000, Hours = 60 * TimeScope.Minute, From e1fc2b46c491cd963de419a15041d2a29fb840e0 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 26 Jan 2023 11:40:40 +0100 Subject: [PATCH 279/323] add observeSize operator on ObservableMap --- src/observable/map/BaseObservableMap.ts | 27 ++++--- .../value/MapSizeObservableValue.ts | 71 +++++++++++++++++++ src/observable/value/index.ts | 1 + 3 files changed, 88 insertions(+), 11 deletions(-) create mode 100644 src/observable/value/MapSizeObservableValue.ts diff --git a/src/observable/map/BaseObservableMap.ts b/src/observable/map/BaseObservableMap.ts index f81a94f8..c82e531c 100644 --- a/src/observable/map/BaseObservableMap.ts +++ b/src/observable/map/BaseObservableMap.ts @@ -18,6 +18,7 @@ import {BaseObservable} from "../BaseObservable"; import {JoinedMap} from "./index"; import {MappedMap} from "./index"; import {FilteredMap} from "./index"; +import {BaseObservableValue, MapSizeObservableValue} from "../value/index"; import {SortedMapList} from "../list/SortedMapList.js"; @@ -66,19 +67,23 @@ export abstract class BaseObservableMap extends BaseObservable>(...otherMaps: Array): JoinedMap { return new JoinedMap([this as BaseObservableMap].concat(otherMaps)); - } + } - mapValues(mapper: Mapper, updater?: Updater): MappedMap { - return new MappedMap(this, mapper, updater); - } + mapValues(mapper: Mapper, updater?: Updater): MappedMap { + return new MappedMap(this, mapper, updater); + } - sortValues(comparator: Comparator): SortedMapList { - return new SortedMapList(this, comparator); - } + sortValues(comparator: Comparator): SortedMapList { + return new SortedMapList(this, comparator); + } - filterValues(filter: Filter): FilteredMap { - return new FilteredMap(this, filter); - } + filterValues(filter: Filter): FilteredMap { + return new FilteredMap(this, filter); + } + + observeSize(): BaseObservableValue { + return new MapSizeObservableValue(this); + } abstract [Symbol.iterator](): Iterator<[K, V]>; abstract get size(): number; @@ -94,4 +99,4 @@ export type Updater = (params: any, mappedValue?: MappedV, value?: V export type Comparator = (a: V, b: V) => number; -export type Filter = (v: V, k: K) => boolean; \ No newline at end of file +export type Filter = (v: V, k: K) => boolean; diff --git a/src/observable/value/MapSizeObservableValue.ts b/src/observable/value/MapSizeObservableValue.ts new file mode 100644 index 00000000..13bac604 --- /dev/null +++ b/src/observable/value/MapSizeObservableValue.ts @@ -0,0 +1,71 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {BaseObservableValue} from "./index"; +import {BaseObservableMap} from "../map/index"; +import type {SubscriptionHandle} from "../BaseObservable"; + +export class MapSizeObservableValue extends BaseObservableValue { + private subscription?: SubscriptionHandle; + + constructor(private readonly map: BaseObservableMap) + { + super(); + } + + onSubscribeFirst(): void { + this.subscription = this.map.subscribe({ + onAdd: (key: K, value: V) => { + this.emit(this.get()); + }, + onRemove: (key: K, value: V) => { + this.emit(this.get()); + }, + onUpdate: (key: K, value: V) => {}, + onReset: () => { + this.emit(this.get()); + }, + }); + } + + onUnsubscribeLast(): void { + this.subscription = this.subscription?.(); + } + + get(): number { + return this.map.size; + } +} + +import {ObservableMap} from "../map/index"; + +export function tests() { + return { + "emits update on add and remove": assert => { + const map = new ObservableMap(); + const size = new MapSizeObservableValue(map); + const updates: number[] = []; + size.subscribe(size => { + updates.push(size); + }); + map.add("hello", 1); + map.add("world", 2); + map.remove("world"); + map.remove("hello"); + assert.deepEqual(updates, [1, 2, 1, 0]); + } + }; +} diff --git a/src/observable/value/index.ts b/src/observable/value/index.ts index ceb53628..1b0f1347 100644 --- a/src/observable/value/index.ts +++ b/src/observable/value/index.ts @@ -12,4 +12,5 @@ export {EventObservableValue} from './EventObservableValue'; export {FlatMapObservableValue} from './FlatMapObservableValue'; export {PickMapObservableValue} from './PickMapObservableValue'; export {RetainedObservableValue} from './RetainedObservableValue'; +export {MapSizeObservableValue} from './MapSizeObservableValue'; export {ObservableValue} from './ObservableValue'; From 59ebcf99fbd1fc1c3f4cfdae50f9e0bf78c01529 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 26 Jan 2023 11:56:30 +0100 Subject: [PATCH 280/323] use observeSize to emit update on memberCount rather than custom handler --- .../toast/CallToastNotificationViewModel.ts | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/src/domain/session/toast/CallToastNotificationViewModel.ts b/src/domain/session/toast/CallToastNotificationViewModel.ts index 3281b86a..5c788334 100644 --- a/src/domain/session/toast/CallToastNotificationViewModel.ts +++ b/src/domain/session/toast/CallToastNotificationViewModel.ts @@ -34,22 +34,9 @@ type MinimumNeededSegmentType = { export class CallToastNotificationViewModel = Options> extends BaseToastNotificationViewModel implements IAvatarContract { constructor(options: O) { super(options); - this.track( - this.call.members.subscribe({ - onAdd: (_, __) => { - this.emitChange("memberCount"); - }, - onUpdate: (_, __) => { - this.emitChange("memberCount"); - }, - onRemove: (_, __) => { - this.emitChange("memberCount"); - }, - onReset: () => { - this.emitChange("memberCount"); - }, - }) - ); + this.track(this.call.members.observeSize().subscribe(() => { + this.emitChange("memberCount"); + })); // Dismiss the toast if the room is opened manually this.track( this.navigation.observe("room").subscribe((roomId) => { From 90ba35da7a19939b967b84bfe5e2e6fb0f34241e Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 26 Jan 2023 14:50:43 +0100 Subject: [PATCH 281/323] listen for members.size changes in CallTile and emit update so memberCount binding updates also be consistent to not emit updates on call object when changing members map --- src/domain/session/room/timeline/tiles/CallTile.js | 4 ++++ src/matrix/calls/group/GroupCall.ts | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index 4583de49..ffb9ef59 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -30,6 +30,7 @@ export class CallTile extends SimpleTile { super(entry, options); const calls = this.getOption("session").callHandler.calls; this._callSubscription = undefined; + this._memberSizeSubscription = undefined; const call = calls.get(this._entry.stateKey); if (call && !call.isTerminated) { this._call = call; @@ -37,6 +38,9 @@ export class CallTile extends SimpleTile { this._callSubscription = this.track(this._call.disposableOn("change", () => { this._onCallUpdate(); })); + this._memberSizeSubscription = this.track(this._call.members.observeSize().subscribe(() => { + this.emitChange("memberCount"); + })); this._onCallUpdate(); } } diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 7f9ba96e..e0099f4e 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -549,7 +549,6 @@ export class GroupCall extends EventEmitter<{change: never}> { member.dispose(); this._members.remove(memberKey); } - this.emitChange(); }); } From d65bbcf168a56ad70ff022cba7f535ac1cdca50f Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Fri, 27 Jan 2023 13:38:45 +0530 Subject: [PATCH 282/323] Fix scaling on chrome --- src/platform/web/ui/css/themes/element/theme.css | 10 ++++++++-- src/platform/web/ui/css/themes/element/timeline.css | 5 ++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/platform/web/ui/css/themes/element/theme.css b/src/platform/web/ui/css/themes/element/theme.css index 3aec2a8a..7bea169e 100644 --- a/src/platform/web/ui/css/themes/element/theme.css +++ b/src/platform/web/ui/css/themes/element/theme.css @@ -1278,7 +1278,10 @@ button.RoomDetailsView_row::after { } .CallToastNotificationView__call-type::before { - content: url("./icons/video-call.svg?primary=light-text-color"); + content: ""; + background-image: url("./icons/video-call.svg?primary=light-text-color"); + background-repeat: no-repeat; + background-size: 20px 20px; display: flex; width: 20px; height: 20px; @@ -1296,7 +1299,10 @@ button.RoomDetailsView_row::after { } .CallToastNotificationView__member-count::before { - content: url("./icons/room-members.svg?primary=light-text-color"); + content: ""; + background-image: url("./icons/room-members.svg?primary=light-text-color"); + background-repeat: no-repeat; + background-size: 20px 20px; display: flex; width: 20px; height: 20px; diff --git a/src/platform/web/ui/css/themes/element/timeline.css b/src/platform/web/ui/css/themes/element/timeline.css index 16c758a3..8606400d 100644 --- a/src/platform/web/ui/css/themes/element/timeline.css +++ b/src/platform/web/ui/css/themes/element/timeline.css @@ -465,7 +465,10 @@ only loads when the top comes into view*/ } .CallTileView_memberCount::before { - content: url('./icons/room-members.svg?primary=text-color'); + content: ""; + background-image: url('./icons/room-members.svg?primary=text-color'); + background-repeat: no-repeat; + background-size: 16px 16px; width: 16px; height: 16px; display: inline-flex; From 43dea3bfdc1af23df1efc43808715c8b5ba733c2 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 27 Jan 2023 10:01:53 +0100 Subject: [PATCH 283/323] detect calls using a foci --- src/matrix/calls/callEventTypes.ts | 6 ++++++ src/matrix/calls/group/GroupCall.ts | 9 +++++++++ src/matrix/calls/group/Member.ts | 5 +++++ 3 files changed, 20 insertions(+) diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index a0eb986c..879d0c11 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -22,11 +22,17 @@ export enum EventType { // TODO: Change to "sdp_stream_metadata" when MSC3077 is merged export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata"; +export interface FocusConfig { + user_id: string, + device_id: string +} + export interface CallDeviceMembership { device_id: string, session_id: string, ["expires_ts"]?: number, feeds?: Array<{purpose: string}> + ["m.foci.active"]?: Array } export interface CallMembership { diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index e0099f4e..16f09dd2 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -145,6 +145,15 @@ export class GroupCall extends EventEmitter<{change: never}> { return !!this.callContent?.["m.terminated"]; } + get usesFoci(): boolean { + for (const member of this._members.values()) { + if (member.usesFoci) { + return true; + } + } + return false; + } + get duration(): number | undefined { if (typeof this.startTime === "number") { return (this.options.clock.now() - this.startTime); diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index e097804c..ab99c6b8 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -117,6 +117,11 @@ export class Member { return this.errorBoundary.error; } + get usesFoci(): boolean { + const activeFoci = this.callDeviceMembership["m.foci.active"]; + return Array.isArray(activeFoci) && activeFoci.length > 0; + } + private _renewExpireTimeout(log: ILogItem) { this.expireTimeout?.dispose(); this.expireTimeout = undefined; From 365157449e9e052c6045f6a8693dadcdc42566be Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 27 Jan 2023 10:02:13 +0100 Subject: [PATCH 284/323] cleanup loops here to not get keys --- src/matrix/calls/group/GroupCall.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 16f09dd2..90c4d982 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -226,7 +226,7 @@ export class GroupCall extends EventEmitter<{change: never}> { this.emitChange(); }); // send invite to all members that are < my userId - for (const [,member] of this._members) { + for (const member of this._members.values()) { this.connectToMember(member, joinedData, log); } }); @@ -530,7 +530,7 @@ export class GroupCall extends EventEmitter<{change: never}> { disconnect(log: ILogItem): boolean { return this.errorBoundary.try(() => { if (this.hasJoined) { - for (const [,member] of this._members) { + for (const member of this._members.values()) { const disconnectLogItem = member.disconnect(true); if (disconnectLogItem) { log.refDetached(disconnectLogItem); From c8bb5fffb0fa7cce87a9d02fd4dbb6f8826af2a6 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 27 Jan 2023 10:12:22 +0100 Subject: [PATCH 285/323] don't allow to join a call using a foci --- src/domain/session/room/timeline/tiles/CallTile.js | 2 +- src/matrix/calls/group/GroupCall.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index ffb9ef59..37be05be 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -94,7 +94,7 @@ export class CallTile extends SimpleTile { } get canJoin() { - return this._call && !this._call.hasJoined; + return this._call && !this._call.hasJoined && !this._call.usesFoci; } get canLeave() { diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 90c4d982..9ef2e068 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -190,7 +190,7 @@ export class GroupCall extends EventEmitter<{change: never}> { join(localMedia: LocalMedia, log?: ILogItem): Promise { return this.options.logger.wrapOrRun(log, "Call.join", async joinLog => { - if (this._state !== GroupCallState.Created || this.joinedData) { + if (this._state !== GroupCallState.Created || this.joinedData || this.usesFoci) { return; } const logItem = this.options.logger.child({ From 825602a04a2d89dd6cb7ec75cab3de821a6b77fc Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 27 Jan 2023 10:12:36 +0100 Subject: [PATCH 286/323] dispose local media here when returning early as join takes ownership --- src/matrix/calls/group/GroupCall.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 9ef2e068..0460c212 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -191,6 +191,7 @@ export class GroupCall extends EventEmitter<{change: never}> { join(localMedia: LocalMedia, log?: ILogItem): Promise { return this.options.logger.wrapOrRun(log, "Call.join", async joinLog => { if (this._state !== GroupCallState.Created || this.joinedData || this.usesFoci) { + localMedia.dispose(); return; } const logItem = this.options.logger.child({ From f5838b21baae8fed048af4e8aa376a00f1c8d50f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 27 Jan 2023 10:12:51 +0100 Subject: [PATCH 287/323] show message in tile when call uses foci, explaining we can't join --- src/domain/session/room/timeline/tiles/CallTile.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index 37be05be..05762bc0 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -118,11 +118,14 @@ export class CallTile extends SimpleTile { } get typeLabel() { + if (this._call && this._call.usesFoci) { + return `This call uses a stream-forwarding unit, which isn't supported yet, so you can't join this call.`; + } if (this.type === CallType.Video) { - return `Video call`; - } else { - return `Voice call`; - } + return `Video call`; + } else { + return `Voice call`; + } } get type() { From 1d7db53f30cb4f6ff438fe86936df84fee797737 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 27 Jan 2023 10:22:01 +0100 Subject: [PATCH 288/323] don't show toast for foci calls --- src/domain/session/toast/ToastCollectionViewModel.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/session/toast/ToastCollectionViewModel.ts b/src/domain/session/toast/ToastCollectionViewModel.ts index 18b58b76..709f5b81 100644 --- a/src/domain/session/toast/ToastCollectionViewModel.ts +++ b/src/domain/session/toast/ToastCollectionViewModel.ts @@ -79,7 +79,7 @@ export class ToastCollectionViewModel extends ViewModel { private _shouldShowNotification(call: GroupCall): boolean { const currentlyOpenedRoomId = this.navigation.path.get("room")?.value; - if (!call.isLoadedFromStorage && call.roomId !== currentlyOpenedRoomId) { + if (!call.isLoadedFromStorage && call.roomId !== currentlyOpenedRoomId && !call.usesFoci) { return true; } return false; From 903a157de2bbf6c61545cccfbb91468199c7f2e5 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 31 Jan 2023 16:48:48 +0530 Subject: [PATCH 289/323] Don't show toast if room is not available --- src/domain/session/toast/ToastCollectionViewModel.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/domain/session/toast/ToastCollectionViewModel.ts b/src/domain/session/toast/ToastCollectionViewModel.ts index 709f5b81..78961029 100644 --- a/src/domain/session/toast/ToastCollectionViewModel.ts +++ b/src/domain/session/toast/ToastCollectionViewModel.ts @@ -39,6 +39,9 @@ export class ToastCollectionViewModel extends ViewModel { onAdd(_, call: GroupCall) { if (this._shouldShowNotification(call)) { const room = this._findRoomForCall(call); + if (!room) { + return; + } const dismiss = () => { const idx = this.toastViewModels.array.findIndex(vm => vm.call === call); if (idx !== -1) { From 9a6d15a72b64c7a93af73319b494833d92fc09c8 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 31 Jan 2023 16:49:58 +0530 Subject: [PATCH 290/323] Fix unwanted toast appearing when opening client --- src/matrix/calls/CallHandler.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index af01beef..b4a1c5cf 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -229,7 +229,8 @@ export class CallHandler implements RoomStateHandler { this._calls.remove(call.id); txn.calls.remove(call.intent, roomId, call.id); } - } else { + } else if(!event.content["m.terminated"]) { + // We don't have this call already and it isn't terminated, so create the call: call = new GroupCall( event.state_key, // id false, // isLoadedFromStorage From de57e0798269b87c899528f78c0375677cde67a3 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Thu, 2 Feb 2023 15:26:38 +0530 Subject: [PATCH 291/323] Wait for room to come through sync --- .../session/toast/ToastCollectionViewModel.ts | 20 ++++++++------ src/matrix/Session.js | 26 ++++++++++++------- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/src/domain/session/toast/ToastCollectionViewModel.ts b/src/domain/session/toast/ToastCollectionViewModel.ts index 78961029..d1595b2d 100644 --- a/src/domain/session/toast/ToastCollectionViewModel.ts +++ b/src/domain/session/toast/ToastCollectionViewModel.ts @@ -21,6 +21,7 @@ import type {GroupCall} from "../../../matrix/calls/group/GroupCall"; import type {Room} from "../../../matrix/room/Room.js"; import type {Session} from "../../../matrix/Session.js"; import type {SegmentType} from "../../navigation"; +import { RoomStatus } from "../../../lib"; type Options = { session: Session; @@ -36,12 +37,9 @@ export class ToastCollectionViewModel extends ViewModel { this.track(callsObservableMap.subscribe(this)); } - onAdd(_, call: GroupCall) { + async onAdd(_, call: GroupCall) { if (this._shouldShowNotification(call)) { - const room = this._findRoomForCall(call); - if (!room) { - return; - } + const room = await this._findRoomForCall(call); const dismiss = () => { const idx = this.toastViewModels.array.findIndex(vm => vm.call === call); if (idx !== -1) { @@ -74,10 +72,16 @@ export class ToastCollectionViewModel extends ViewModel { } } - private _findRoomForCall(call: GroupCall): Room { + private async _findRoomForCall(call: GroupCall): Promise { const id = call.roomId; - const rooms = this.getOption("session").rooms; - return rooms.get(id); + const session = this.getOption("session"); + const rooms = session.rooms; + // Make sure that we know of this room, + // otherwise wait for it to come through sync + const observable = await session.observeRoomStatus(id); + await observable.waitFor(s => s === RoomStatus.Joined).promise; + const room = rooms.get(id); + return room; } private _shouldShowNotification(call: GroupCall): boolean { diff --git a/src/matrix/Session.js b/src/matrix/Session.js index d640e2df..87680a49 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -132,6 +132,7 @@ export class Session { this._createRoomEncryption = this._createRoomEncryption.bind(this); this._forgetArchivedRoom = this._forgetArchivedRoom.bind(this); this.needsKeyBackup = new ObservableValue(false); + this._pendingObserveCalls = []; } get fingerprintKey() { @@ -776,7 +777,7 @@ export class Session { } } - applyRoomCollectionChangesAfterSync(inviteStates, roomStates, archivedRoomStates, log) { + async applyRoomCollectionChangesAfterSync(inviteStates, roomStates, archivedRoomStates, log) { // update the collections after sync for (const rs of roomStates) { if (rs.shouldAdd) { @@ -796,6 +797,8 @@ export class Session { // now all the collections are updated, update the room status // so any listeners to the status will find the collections // completely up to date + await Promise.all(this._pendingObserveCalls); + this._pendingObserveCalls = []; if (this._observedRoomStatus.size !== 0) { for (const ars of archivedRoomStates) { if (ars.shouldAdd) { @@ -938,16 +941,21 @@ export class Session { } async observeRoomStatus(roomId) { - let observable = this._observedRoomStatus.get(roomId); - if (!observable) { - const status = await this.getRoomStatus(roomId); - observable = new RetainedObservableValue(status, () => { - this._observedRoomStatus.delete(roomId); - }); + const op = async () => { + let observable = this._observedRoomStatus.get(roomId); + if (!observable) { + const status = await this.getRoomStatus(roomId); + observable = new RetainedObservableValue(status, () => { + this._observedRoomStatus.delete(roomId); + }); - this._observedRoomStatus.set(roomId, observable); + this._observedRoomStatus.set(roomId, observable); + } + return observable; } - return observable; + const promise = op(); + this._pendingObserveCalls.push(promise); + return await promise; } observeRoomState(roomStateHandler) { From 09e67ec21c42783bc847a8592cd4bd2bd3e618f5 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Thu, 2 Feb 2023 18:39:19 +0530 Subject: [PATCH 292/323] Deal with race in a better way --- src/matrix/Session.js | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 87680a49..f1b273ff 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -132,7 +132,6 @@ export class Session { this._createRoomEncryption = this._createRoomEncryption.bind(this); this._forgetArchivedRoom = this._forgetArchivedRoom.bind(this); this.needsKeyBackup = new ObservableValue(false); - this._pendingObserveCalls = []; } get fingerprintKey() { @@ -797,8 +796,6 @@ export class Session { // now all the collections are updated, update the room status // so any listeners to the status will find the collections // completely up to date - await Promise.all(this._pendingObserveCalls); - this._pendingObserveCalls = []; if (this._observedRoomStatus.size !== 0) { for (const ars of archivedRoomStates) { if (ars.shouldAdd) { @@ -941,21 +938,24 @@ export class Session { } async observeRoomStatus(roomId) { - const op = async () => { - let observable = this._observedRoomStatus.get(roomId); - if (!observable) { - const status = await this.getRoomStatus(roomId); - observable = new RetainedObservableValue(status, () => { - this._observedRoomStatus.delete(roomId); - }); - - this._observedRoomStatus.set(roomId, observable); + let observable = this._observedRoomStatus.get(roomId); + if (!observable) { + let status = undefined; + // Create and set the observable with value = undefined, so that + // we don't loose any sync changes that come in while we are busy + // calculating the current room status. + observable = new RetainedObservableValue(status, () => { + this._observedRoomStatus.delete(roomId); + }); + this._observedRoomStatus.set(roomId, observable); + status = await this.getRoomStatus(roomId); + // If observable.value is not undefined anymore, then some + // change has come through the sync. + if (observable.get() === undefined) { + observable.set(status); } - return observable; } - const promise = op(); - this._pendingObserveCalls.push(promise); - return await promise; + return observable; } observeRoomState(roomStateHandler) { From 7eae171ac9e05a4c06650f2b6b03b9478853b122 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Mon, 6 Feb 2023 17:12:39 +0530 Subject: [PATCH 293/323] Emit after sending signalling message --- src/matrix/calls/PeerCall.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 36136b97..fcb71db1 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -284,11 +284,12 @@ export class PeerCall implements IDisposable { } private async _hangup(errorCode: CallErrorCode, log: ILogItem): Promise { - if (this._state === CallState.Ended) { + if (this._state === CallState.Ended || this._state === CallState.Ending) { return; } - this.terminate(CallParty.Local, errorCode, log); + this.setState(CallState.Ending, log); await this.sendHangupWithCallId(this.callId, errorCode, log); + this.terminate(CallParty.Local, errorCode, log); } getMessageAction(message: SignallingMessage): IncomingMessageAction { @@ -1130,6 +1131,7 @@ export enum CallState { Connecting = 'connecting', Connected = 'connected', Ringing = 'ringing', + Ending = 'ending', Ended = 'ended', } From cadeae98bc69dc35ba4f966e298fc34b5f7e7fdd Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 14:13:49 +0100 Subject: [PATCH 294/323] prevent ignored signaling messages from blocking the queue signaling messages get ignored when they are not for the currently active call id. In that case we currently don't advance the lastProcessedSeqNr counter, as we had a problem before where the counter would be brought out of sync with seq numbers for other call ids. However when we've previously processed a signalling message (e.g. the counter is not undefined) and the first message in the queue is to be ignored, it will prevent the subsequent messages from being dequeued as their seq number is more than 1 away from the last processed seq. This adds an additional counter for ignored seq numbers that is also used to see if the next message is only 1 away from the next seq value. I am adding logging as well here to have a better overview in the future --- src/matrix/calls/group/Member.ts | 51 +++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index ab99c6b8..f82c5ddd 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -59,6 +59,9 @@ class MemberConnection { public retryCount: number = 0; public peerCall?: PeerCall; public lastProcessedSeqNr: number | undefined; + // XXX: Not needed anymore when seq is scoped to call_id + // see https://github.com/matrix-org/matrix-spec-proposals/pull/3401#discussion_r1097482166 + public lastIgnoredSeqNr: number | undefined; public queuedSignallingMessages: SignallingMessage[] = []; public outboundSeqCounter: number = 0; @@ -73,16 +76,24 @@ class MemberConnection { if (this.queuedSignallingMessages.length === 0) { return false; } + const first = this.queuedSignallingMessages[0]; + const firstSeq = first.content.seq; + // prevent not being able to jump over seq values of ignored messages for other call ids + // as they don't increase lastProcessedSeqNr. + if (this.lastIgnoredSeqNr !== undefined && firstSeq === this.lastIgnoredSeqNr + 1) { + return true; + } if (this.lastProcessedSeqNr === undefined) { return true; } - const first = this.queuedSignallingMessages[0]; // allow messages with both a seq we've just seen and // the next one to be dequeued as it can happen // that messages for other callIds (which could repeat seq) // are present in the queue - return first.content.seq === this.lastProcessedSeqNr || - first.content.seq === this.lastProcessedSeqNr + 1; + // XXX: Not needed anymore when seq is scoped to call_id + // see https://github.com/matrix-org/matrix-spec-proposals/pull/3401#discussion_r1097482166 + return firstSeq === this.lastProcessedSeqNr || + firstSeq === this.lastProcessedSeqNr + 1; } dispose() { @@ -382,17 +393,29 @@ export class Member { private dequeueSignallingMessages(connection: MemberConnection, peerCall: PeerCall, newMessage: SignallingMessage, syncLog: ILogItem): boolean { let hasNewMessageBeenDequeued = false; while (connection.canDequeueNextSignallingMessage) { - const message = connection.queuedSignallingMessages.shift()!; - if (message === newMessage) { - hasNewMessageBeenDequeued = true; - } - // ignore items in the queue that should not be handled and prevent - // the lastProcessedSeqNr being corrupted with the `seq` for other call ids - if (peerCall.getMessageAction(message) === IncomingMessageAction.Handle) { - const item = peerCall.handleIncomingSignallingMessage(message, this.deviceId, connection.logItem); - syncLog.refDetached(item); - connection.lastProcessedSeqNr = message.content.seq; - } + syncLog.wrap("dequeue message", log => { + const message = connection.queuedSignallingMessages.shift()!; + if (message === newMessage) { + log.set("isNewMsg", true); + hasNewMessageBeenDequeued = true; + } + const seq = message.content?.seq; + log.set("seq", seq); + // ignore items in the queue that should not be handled and prevent + // the lastProcessedSeqNr being corrupted with the `seq` for other call ids + // XXX: Not needed anymore when seq is scoped to call_id + // see https://github.com/matrix-org/matrix-spec-proposals/pull/3401#discussion_r1097482166 + const action = peerCall.getMessageAction(message); + if (action === IncomingMessageAction.Handle) { + const item = peerCall.handleIncomingSignallingMessage(message, this.deviceId, connection.logItem); + log.refDetached(item); + connection.lastProcessedSeqNr = seq; + } else { + log.set("type", message.type); + log.set("ignored", true); + connection.lastIgnoredSeqNr = seq; + } + }); } return hasNewMessageBeenDequeued; } From 39e9a43a1b365227b5a5bb04e86cfff4763de212 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 14:42:44 +0100 Subject: [PATCH 295/323] be strict about the first seq being 0 otherwise if first 2 messages are delivered in reverse order, the queue gets blocked --- src/matrix/calls/group/Member.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index f82c5ddd..fe6e40a4 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -84,7 +84,7 @@ class MemberConnection { return true; } if (this.lastProcessedSeqNr === undefined) { - return true; + return firstSeq === 0; } // allow messages with both a seq we've just seen and // the next one to be dequeued as it can happen From 5f4ad30d03efaa3f0a1203e9964bac0bdf7f4ef0 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 14:43:28 +0100 Subject: [PATCH 296/323] don't block if it does happen that we have processed a message too early allow dequeueing if the first seq in the queue is actually lower than what we already processed. Normally should not happen, but the bug fixed in the previous commit was aggravated by this behavior, so be more lenient here. --- src/matrix/calls/group/Member.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index fe6e40a4..81b1dc7d 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -92,8 +92,7 @@ class MemberConnection { // are present in the queue // XXX: Not needed anymore when seq is scoped to call_id // see https://github.com/matrix-org/matrix-spec-proposals/pull/3401#discussion_r1097482166 - return firstSeq === this.lastProcessedSeqNr || - firstSeq === this.lastProcessedSeqNr + 1; + return firstSeq <= (this.lastProcessedSeqNr + 1); } dispose() { From 6d800ff35913e2dd494124cda4b2d1d7bd28b00a Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 7 Feb 2023 19:15:36 +0530 Subject: [PATCH 297/323] WIP --- src/matrix/calls/PeerCall.ts | 72 ++++++++++++++++++----------- src/matrix/calls/group/GroupCall.ts | 29 ++++++------ src/matrix/calls/group/Member.ts | 26 +++++++---- src/utils/ErrorBoundary.ts | 4 +- 4 files changed, 76 insertions(+), 55 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index fcb71db1..905756d7 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -44,6 +44,8 @@ import type { SDPStreamMetadata, SignallingMessage } from "./callEventTypes"; +import type { ErrorBoundary } from "../../utils/ErrorBoundary"; +import { AbortError } from "../../utils/error"; export type Options = { webRTC: WebRTC, @@ -51,6 +53,7 @@ export type Options = { turnServer: BaseObservableValue, createTimeout: TimeoutCreator, emitUpdate: (peerCall: PeerCall, params: any, log: ILogItem) => void; + errorBoundary: ErrorBoundary; sendSignallingMessage: (message: SignallingMessage, log: ILogItem) => Promise; }; @@ -125,31 +128,34 @@ export class PeerCall implements IDisposable { this.logItem.log({l: "updating turn server", turnServer}) this.peerConnection.setConfiguration({iceServers: [turnServer]}); })); - const listen = (type: K, listener: (this: PeerConnection, ev: PeerConnectionEventMap[K]) => any, options?: boolean | EventListenerOptions): void => { - this.peerConnection.addEventListener(type, listener); + const listen = (type: K, listener: (ev: PeerConnectionEventMap[K]) => any, options?: boolean | EventListenerOptions): void => { + const newListener = (e) => { + this.options.errorBoundary.try(() => listener(e)); + }; + this.peerConnection.addEventListener(type, newListener); const dispose = () => { - this.peerConnection.removeEventListener(type, listener); + this.peerConnection.removeEventListener(type, newListener); }; this.disposables.track(dispose); }; - listen("iceconnectionstatechange", () => { + listen("iceconnectionstatechange", async () => { const state = this.peerConnection.iceConnectionState; - logItem.wrap({l: "onIceConnectionStateChange", status: state}, log => { - this.onIceConnectionStateChange(state, log); + await logItem.wrap({l: "onIceConnectionStateChange", status: state}, async log => { + await this.onIceConnectionStateChange(state, log); }); }); - listen("icecandidate", event => { - logItem.wrap("onLocalIceCandidate", log => { + listen("icecandidate", async (event) => { + await logItem.wrap("onLocalIceCandidate", async log => { if (event.candidate) { - this.handleLocalIceCandidate(event.candidate, log); + await this.handleLocalIceCandidate(event.candidate, log); } }); }); - listen("icegatheringstatechange", () => { + listen("icegatheringstatechange", async () => { const state = this.peerConnection.iceGatheringState; - logItem.wrap({l: "onIceGatheringStateChange", status: state}, log => { - this.handleIceGatheringState(state, log); + await logItem.wrap({l: "onIceGatheringStateChange", status: state}, async log => { + await this.handleIceGatheringState(state, log); }); }); listen("track", event => { @@ -422,14 +428,18 @@ export class PeerCall implements IDisposable { log.refDetached(timeoutLog); // don't await this, as it would block other negotationneeded events from being processed // as they are processed serially - timeoutLog.run(async log => { - try { await this.delay(CALL_TIMEOUT_MS); } - catch (err) { return; } - // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between - if (this._state === CallState.InviteSent) { - this._hangup(CallErrorCode.InviteTimeout, log); - } - }).catch(err => {}); // prevent error from being unhandled, it will be logged already by run above + try { + await timeoutLog.run(async log => { + await this.delay(CALL_TIMEOUT_MS); + // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between + if (this._state === CallState.InviteSent) { + await this._hangup(CallErrorCode.InviteTimeout, log); + } + }); + } + catch (e) { + // prevent error from being unhandled, it will be logged already by run above + } } }; @@ -579,7 +589,7 @@ export class PeerCall implements IDisposable { } } - private handleIceGatheringState(state: RTCIceGatheringState, log: ILogItem) { + private async handleIceGatheringState(state: RTCIceGatheringState, log: ILogItem) { if (state === 'complete' && !this.sentEndOfCandidates) { // If we didn't get an empty-string candidate to signal the end of candidates, // create one ourselves now gathering has finished. @@ -591,12 +601,12 @@ export class PeerCall implements IDisposable { const c = { candidate: '', } as RTCIceCandidate; - this.queueCandidate(c, log); + await this.queueCandidate(c, log); this.sentEndOfCandidates = true; } } - private handleLocalIceCandidate(candidate: RTCIceCandidate, log: ILogItem) { + private async handleLocalIceCandidate(candidate: RTCIceCandidate, log: ILogItem) { log.set("sdpMid", candidate.sdpMid); log.set("candidate", candidate.candidate); @@ -606,7 +616,7 @@ export class PeerCall implements IDisposable { // As with the offer, note we need to make a copy of this object, not // pass the original: that broke in Chrome ~m43. if (candidate.candidate !== '' || !this.sentEndOfCandidates) { - this.queueCandidate(candidate, log); + await this.queueCandidate(candidate, log); if (candidate.candidate === '') { this.sentEndOfCandidates = true; } @@ -841,13 +851,19 @@ export class PeerCall implements IDisposable { logStats = true; this.iceDisconnectedTimeout?.abort(); this.iceDisconnectedTimeout = undefined; - this._hangup(CallErrorCode.IceFailed, log); + await this._hangup(CallErrorCode.IceFailed, log); } else if (state == 'disconnected') { logStats = true; this.iceDisconnectedTimeout = this.options.createTimeout(30 * 1000); - this.iceDisconnectedTimeout.elapsed().then(() => { - this._hangup(CallErrorCode.IceFailed, log); - }, () => { /* ignore AbortError */ }); + try { + await this.iceDisconnectedTimeout.elapsed() + await this._hangup(CallErrorCode.IceFailed, log); + } + catch (e){ + if (!(e instanceof AbortError)) { + throw e; + } + } } if (logStats) { const stats = await this.peerConnection.getStats(); diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 0460c212..d540d2e0 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -133,7 +133,8 @@ export class GroupCall extends EventEmitter<{change: never}> { }, encryptDeviceMessage: (userId: string, deviceId: string, message: SignallingMessage, log) => { return this.options.encryptDeviceMessage(this.roomId, userId, deviceId, message, log); - } + }, + groupCallErrorBoundary: this.errorBoundary, }); } @@ -392,8 +393,8 @@ export class GroupCall extends EventEmitter<{change: never}> { /** @internal */ updateMembership(userId: string, roomMember: RoomMember, callMembership: CallMembership, syncLog: ILogItem) { - this.errorBoundary.try(() => { - syncLog.wrap({l: "update call membership", t: CALL_LOG_TYPE, id: this.id, userId}, log => { + this.errorBoundary.try(async () => { + await syncLog.wrap({l: "update call membership", t: CALL_LOG_TYPE, id: this.id, userId}, async log => { const now = this.options.clock.now(); const devices = callMembership["m.devices"]; const previousDeviceIds = this.getDeviceIdsForUserId(userId); @@ -415,7 +416,7 @@ export class GroupCall extends EventEmitter<{change: never}> { } }); } else { - log.wrap({l: "update device membership", id: memberKey, sessionId: device.session_id}, log => { + await log.wrap({l: "update device membership", id: memberKey, sessionId: device.session_id}, async log => { if (isMemberExpired(device, now)) { log.set("expired", true); const member = this._members.get(memberKey); @@ -434,7 +435,7 @@ export class GroupCall extends EventEmitter<{change: never}> { } else { if (member && sessionIdChanged) { log.set("removedSessionId", member.sessionId); - const disconnectLogItem = member.disconnect(false); + const disconnectLogItem = await member.disconnect(false); if (disconnectLogItem) { log.refDetached(disconnectLogItem); } @@ -528,11 +529,11 @@ export class GroupCall extends EventEmitter<{change: never}> { } /** @internal */ - disconnect(log: ILogItem): boolean { - return this.errorBoundary.try(() => { + disconnect(log: ILogItem): Promise | true { + return this.errorBoundary.try(async () => { if (this.hasJoined) { for (const member of this._members.values()) { - const disconnectLogItem = member.disconnect(true); + const disconnectLogItem = await member.disconnect(true); if (disconnectLogItem) { log.refDetached(disconnectLogItem); } @@ -546,13 +547,13 @@ export class GroupCall extends EventEmitter<{change: never}> { } /** @internal */ - private removeMemberDevice(userId: string, deviceId: string, log: ILogItem) { + private async removeMemberDevice(userId: string, deviceId: string, log: ILogItem) { const memberKey = getMemberKey(userId, deviceId); - log.wrap({l: "remove device member", id: memberKey}, log => { + await log.wrap({l: "remove device member", id: memberKey}, async log => { const member = this._members.get(memberKey); if (member) { log.set("leave", true); - const disconnectLogItem = member.disconnect(false); + const disconnectLogItem = await member.disconnect(false); if (disconnectLogItem) { log.refDetached(disconnectLogItem); } @@ -634,15 +635,15 @@ export class GroupCall extends EventEmitter<{change: never}> { return stateContent; } - private connectToMember(member: Member, joinedData: JoinedData, log: ILogItem) { + private async connectToMember(member: Member, joinedData: JoinedData, log: ILogItem) { const memberKey = getMemberKey(member.userId, member.deviceId); const logItem = joinedData.membersLogItem.child({ l: "member", id: memberKey, sessionId: member.sessionId }); - log.wrap({l: "connect", id: memberKey}, log => { - const connectItem = member.connect( + await log.wrap({l: "connect", id: memberKey}, async log => { + const connectItem = await member.connect( joinedData.localMedia, joinedData.localMuteSettings, joinedData.turnServer, diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index ab99c6b8..88e0369b 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -42,6 +42,7 @@ export type Options = Omit, log: ILogItem) => Promise, emitUpdate: (participant: Member, params?: any) => void, + groupCallErrorBoundary: ErrorBoundary, clock: Clock } @@ -183,8 +184,8 @@ export class Member { } /** @internal */ - connect(localMedia: LocalMedia, localMuteSettings: MuteSettings, turnServer: BaseObservableValue, memberLogItem: ILogItem): ILogItem | undefined { - return this.errorBoundary.try(() => { + connect(localMedia: LocalMedia, localMuteSettings: MuteSettings, turnServer: BaseObservableValue, memberLogItem: ILogItem): Promise | undefined { + return this.errorBoundary.try(async () => { if (this.connection) { return; } @@ -197,7 +198,7 @@ export class Member { ); this.connection = connection; let connectLogItem: ILogItem | undefined; - connection.logItem.wrap("connect", async log => { + await connection.logItem.wrap("connect", async log => { connectLogItem = log; await this.callIfNeeded(log); }); @@ -230,15 +231,15 @@ export class Member { } /** @internal */ - disconnect(hangup: boolean): ILogItem | undefined { - return this.errorBoundary.try(() => { + disconnect(hangup: boolean): Promise | undefined { + return this.errorBoundary.try(async () => { const {connection} = this; if (!connection) { return; } - let disconnectLogItem; + let disconnectLogItem: ILogItem | undefined; // if if not sending the hangup, still log disconnect - connection.logItem.wrap("disconnect", async log => { + await connection.logItem.wrap("disconnect", async log => { disconnectLogItem = log; if (hangup && connection.peerCall) { await connection.peerCall.hangup(CallErrorCode.UserHangup, log); @@ -269,7 +270,7 @@ export class Member { } /** @internal */ - emitUpdateFromPeerCall = (peerCall: PeerCall, params: any, log: ILogItem): void => { + emitUpdateFromPeerCall = async (peerCall: PeerCall, params: any, log: ILogItem): Promise => { const connection = this.connection!; if (peerCall.state === CallState.Ringing) { connection.logItem.wrap("ringing, answer peercall", answerLog => { @@ -284,12 +285,12 @@ export class Member { if (hangupReason && !errorCodesWithoutRetry.includes(hangupReason)) { connection.retryCount += 1; const {retryCount} = connection; - connection.logItem.wrap({l: "retry connection", retryCount}, async retryLog => { + await connection.logItem.wrap({l: "retry connection", retryCount}, async retryLog => { log.refDetached(retryLog); if (retryCount <= 3) { await this.callIfNeeded(retryLog); } else { - const disconnectLogItem = this.disconnect(false); + const disconnectLogItem = await this.disconnect(false); if (disconnectLogItem) { retryLog.refDetached(disconnectLogItem); } @@ -303,6 +304,10 @@ export class Member { /** @internal */ sendSignallingMessage = async (message: SignallingMessage, log: ILogItem): Promise => { const groupMessage = message as SignallingMessage; + if (this.connection?.peerCall?.state === CallState.CreateOffer) { + // @ts-ignore + this.connection!.foobar.barfpp; + } groupMessage.content.seq = this.connection!.outboundSeqCounter++; groupMessage.content.conf_id = this.options.confId; groupMessage.content.device_id = this.options.ownDeviceId; @@ -421,6 +426,7 @@ export class Member { private _createPeerCall(callId: string): PeerCall { const connection = this.connection!; return new PeerCall(callId, Object.assign({}, this.options, { + errorBoundary: this.options.groupCallErrorBoundary, emitUpdate: this.emitUpdateFromPeerCall, sendSignallingMessage: this.sendSignallingMessage, turnServer: connection.turnServer diff --git a/src/utils/ErrorBoundary.ts b/src/utils/ErrorBoundary.ts index a74da479..a0f6fca7 100644 --- a/src/utils/ErrorBoundary.ts +++ b/src/utils/ErrorBoundary.ts @@ -14,8 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -export const ErrorValue = Symbol("ErrorBoundary:Error"); - export class ErrorBoundary { private _error?: Error; @@ -23,7 +21,7 @@ export class ErrorBoundary { /** * Executes callback() and then runs errorCallback() on error. - * This will never throw but instead return `errorValue` if an error occured. + * This will never throw but instead return `errorValue` if an error occurred. */ try(callback: () => T, errorValue?: E): T | typeof errorValue; try(callback: () => Promise, errorValue?: E): Promise | typeof errorValue { From 496dfee5937ed62cad7a79794ee2e36cc4fc304e Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 7 Feb 2023 19:25:48 +0530 Subject: [PATCH 298/323] Catch error in promise chain --- src/matrix/calls/PeerCall.ts | 3 +++ src/matrix/calls/group/GroupCall.ts | 1 + src/matrix/calls/group/Member.ts | 4 ---- src/utils/ErrorBoundary.ts | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 905756d7..d81bf047 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -176,6 +176,9 @@ export class PeerCall implements IDisposable { }); }; this.responsePromiseChain = this.responsePromiseChain?.then(promiseCreator) ?? promiseCreator(); + this.responsePromiseChain.catch((e) => + this.options.errorBoundary.reportError(e) + ); }); } diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index d540d2e0..d728e7c2 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -99,6 +99,7 @@ export class GroupCall extends EventEmitter<{change: never}> { // in case the error happens in code that does not log, // log it here to make sure it isn't swallowed this.joinedData.logItem.log("error at boundary").catch(err); + console.error(err); } }); diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 88e0369b..0c31bc32 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -304,10 +304,6 @@ export class Member { /** @internal */ sendSignallingMessage = async (message: SignallingMessage, log: ILogItem): Promise => { const groupMessage = message as SignallingMessage; - if (this.connection?.peerCall?.state === CallState.CreateOffer) { - // @ts-ignore - this.connection!.foobar.barfpp; - } groupMessage.content.seq = this.connection!.outboundSeqCounter++; groupMessage.content.conf_id = this.options.confId; groupMessage.content.device_id = this.options.ownDeviceId; diff --git a/src/utils/ErrorBoundary.ts b/src/utils/ErrorBoundary.ts index a0f6fca7..750385c7 100644 --- a/src/utils/ErrorBoundary.ts +++ b/src/utils/ErrorBoundary.ts @@ -42,7 +42,7 @@ export class ErrorBoundary { } } - private reportError(err: Error) { + reportError(err: Error) { try { this.errorCallback(err); } catch (err) { From e39dd176a4b1edb5a583acb22e9ca0ff6e0aa326 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 14:58:32 +0100 Subject: [PATCH 299/323] remove debug logging --- src/matrix/calls/group/Member.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 81b1dc7d..23b35bd8 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -372,6 +372,7 @@ export class Member { } if (action === IncomingMessageAction.Handle) { const idx = sortedIndex(connection.queuedSignallingMessages, message, (a, b) => a.content.seq - b.content.seq); + console.log(`splice ${message.type} at ${idx}`); connection.queuedSignallingMessages.splice(idx, 0, message); if (connection.peerCall) { const hasNewMessageBeenDequeued = this.dequeueSignallingMessages(connection, connection.peerCall, message, syncLog); From f67fb7add6ef0d06c44e9023ceb32254ee368bb1 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 14:58:57 +0100 Subject: [PATCH 300/323] add unit test for this particular error case --- src/matrix/calls/group/Member.ts | 114 ++++++++++++++++++++++++++++++- 1 file changed, 112 insertions(+), 2 deletions(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 23b35bd8..d1463b51 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -21,13 +21,13 @@ import {formatToDeviceMessagesPayload} from "../../common"; import {sortedIndex} from "../../../utils/sortedIndex"; import { ErrorBoundary } from "../../../utils/ErrorBoundary"; -import type {MuteSettings} from "../common"; +import {MuteSettings} from "../common"; import type {Options as PeerCallOptions, RemoteMedia} from "../PeerCall"; import type {LocalMedia} from "../LocalMedia"; import type {HomeServerApi} from "../../net/HomeServerApi"; import type {MCallBase, MGroupCallBase, SignallingMessage, CallDeviceMembership} from "../callEventTypes"; import type {GroupCall} from "./GroupCall"; -import type {RoomMember} from "../../room/members/RoomMember"; +import {RoomMember} from "../../room/members/RoomMember"; import type {EncryptedMessage} from "../../e2ee/olm/Encryption"; import type {ILogItem} from "../../../logging/types"; import type {BaseObservableValue} from "../../../observable/value"; @@ -471,3 +471,113 @@ export function isMemberExpired(callDeviceMembership: CallDeviceMembership, now: const expiresAt = memberExpiresAt(callDeviceMembership); return typeof expiresAt === "number" ? ((expiresAt + margin) <= now) : true; } + +import {ObservableValue} from "../../../observable/value"; +import {Clock as MockClock} from "../../../mocks/Clock"; +import {Instance as NullLoggerInstance} from "../../../logging/NullLogger"; + +export function tests() { + + class MockMedia { + clone(): MockMedia { return this; } + } + + class MockPeerConn { + addEventListener() {} + removeEventListener() {} + setConfiguration() {} + setRemoteDescription() {} + getReceivers() { return [{}]; } // non-empty array + getSenders() { return []; } + addTrack() { return {}; } + removeTrack() {} + close() {} + } + return { + "test queue doesn't get blocked by enqueued, then ignored device message": assert => { + // XXX we might want to refactor the queue code a bit so it's easier to test + // without having to provide so many mocks + const clock = new MockClock(); + // setup logging + const logger = NullLoggerInstance; + // const logger = new Logger({platform: {clock, random: Math.random}}); + // logger.addReporter(new ConsoleReporter()); + + // create member + const callDeviceMembership = { + ["device_id"]: "BVPIHSKXFC", + ["session_id"]: "s1d5863f41ec5a5", + ["expires_ts"]: 123, + feeds: [{purpose: "m.usermedia"}] + }; + const roomMember = RoomMember.fromUserId("!abc", "@bruno4:matrix.org", "join"); + const turnServer = new ObservableValue({}); + const options = { + confId: "conf", + ownUserId: "@foobaraccount2:matrix.org", + ownDeviceId: "CMLEZSARRT", + sessionId: "s1cece7088b9d35", + clock, + emitUpdate: () => {}, + webRTC: { + prepareSenderForPurpose: () => {}, + createPeerConnection() { + return new MockPeerConn(); + } + } + } as Options; + const member = new Member(roomMember, callDeviceMembership, options, logger.child("member")); + member.connect(new MockMedia() as LocalMedia, new MuteSettings(), turnServer, logger.child("connect")); + // pretend we've already received 3 messages + member.connection.lastProcessedSeqNr = 2; + // send hangup with seq=3, this will enqueue the message because there is no peerCall + // as it's up to @bruno4:matrix.org to send the invite + const hangup = { + type: EventType.Hangup, + content: { + "call_id": "c0ac1b0e37afe73", + "version": 1, + "reason": "invite_timeout", + "seq": 3, + "conf_id": "conf-16a120796440a6", + "device_id": "BVPIHSKXFC", + "party_id": "BVPIHSKXFC", + "sender_session_id": "s1d5863f41ec5a5", + "dest_session_id": "s1cece7088b9d35" + } + }; + member.handleDeviceMessage(hangup, logger.child("handle hangup")); + // Send an invite with seq=4, this will create a new peer call with a the call id + // when dequeueing the hangup from before, it'll get ignored because it is + // for the previous call id. + const invite = { + type: EventType.Invite, + content: { + "call_id": "c1175b12d559fb1", + "offer": { + "type": "offer", + "sdp": "..." + }, + "org.matrix.msc3077.sdp_stream_metadata": { + "60087b60-487e-4fa0-8229-b232c18e1332": { + "purpose": "m.usermedia", + "audio_muted": false, + "video_muted": false + } + }, + "version": 1, + "lifetime": 60000, + "seq": 4, + "conf_id": "conf-16a120796440a6", + "device_id": "BVPIHSKXFC", + "party_id": "BVPIHSKXFC", + "sender_session_id": "s1d5863f41ec5a5", + "dest_session_id": "s1cece7088b9d35" + } + }; + member.handleDeviceMessage(invite, logger.child("handle invite")); + assert.equal(member.connection.queuedSignallingMessages.length, 0); + // logger.reporters[0].printOpenItems(); + } + }; +} From 02108c69dc782c2737e4ddde9725c731238a8011 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 15:00:22 +0100 Subject: [PATCH 301/323] remove debug logging --- src/matrix/calls/group/Member.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index d1463b51..e0c9c6ca 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -372,7 +372,6 @@ export class Member { } if (action === IncomingMessageAction.Handle) { const idx = sortedIndex(connection.queuedSignallingMessages, message, (a, b) => a.content.seq - b.content.seq); - console.log(`splice ${message.type} at ${idx}`); connection.queuedSignallingMessages.splice(idx, 0, message); if (connection.peerCall) { const hasNewMessageBeenDequeued = this.dequeueSignallingMessages(connection, connection.peerCall, message, syncLog); From ddb5865ccb0defe5c80238e4565c3e6f45bdd9d6 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 15:00:56 +0100 Subject: [PATCH 302/323] actually forgot to dispose peerCall here when replacing --- src/matrix/calls/group/Member.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index e0c9c6ca..d88a216f 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -362,6 +362,7 @@ export class Member { syncLog.refDetached(log); } if (shouldReplace) { + connection.peerCall.dispose(); connection.peerCall = undefined; action = IncomingMessageAction.Handle; } From 7f9d64c9725c607eaac81a8744c7f95a161a28b5 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 19:42:20 +0100 Subject: [PATCH 303/323] improve logging of arrival of to_device call signalling messages --- src/matrix/calls/group/Member.ts | 88 ++++++++++++++++---------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index d88a216f..daada4af 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -343,64 +343,65 @@ export class Member { /** @internal */ handleDeviceMessage(message: SignallingMessage, syncLog: ILogItem): void { this.errorBoundary.try(() => { - const {connection} = this; - if (connection) { - const destSessionId = message.content.dest_session_id; - if (destSessionId !== this.options.sessionId) { - const logItem = connection.logItem.log({l: "ignoring to_device event with wrong session_id", destSessionId, type: message.type}); - syncLog.refDetached(logItem); - return; - } - // if there is no peerCall, we either create it with an invite and Handle is implied or we'll ignore it - let action = IncomingMessageAction.Handle; - if (connection.peerCall) { - action = connection.peerCall.getMessageAction(message); - // deal with glare and replacing the call before creating new calls - if (action === IncomingMessageAction.InviteGlare) { - const {shouldReplace, log} = connection.peerCall.handleInviteGlare(message, this.deviceId, connection.logItem); - if (log) { - syncLog.refDetached(log); - } - if (shouldReplace) { - connection.peerCall.dispose(); - connection.peerCall = undefined; - action = IncomingMessageAction.Handle; + syncLog.wrap({l: "Member.handleDeviceMessage", type: message.type, seq: message.content?.seq}, log => { + const {connection} = this; + if (connection) { + const destSessionId = message.content.dest_session_id; + if (destSessionId !== this.options.sessionId) { + const logItem = connection.logItem.log({l: "ignoring to_device event with wrong session_id", destSessionId, type: message.type}); + log.refDetached(logItem); + return; + } + // if there is no peerCall, we either create it with an invite and Handle is implied or we'll ignore it + if (connection.peerCall) { + const action = connection.peerCall.getMessageAction(message); + // deal with glare and replacing the call before creating new calls + if (action === IncomingMessageAction.InviteGlare) { + const {shouldReplace, log} = connection.peerCall.handleInviteGlare(message, this.deviceId, connection.logItem); + if (log) { + log.refDetached(log); + } + if (shouldReplace) { + connection.peerCall.dispose(); + connection.peerCall = undefined; + } } } - } - if (message.type === EventType.Invite && !connection.peerCall) { - connection.peerCall = this._createPeerCall(message.content.call_id); - } - if (action === IncomingMessageAction.Handle) { + // create call on invite + if (message.type === EventType.Invite && !connection.peerCall) { + connection.peerCall = this._createPeerCall(message.content.call_id); + } + // enqueue const idx = sortedIndex(connection.queuedSignallingMessages, message, (a, b) => a.content.seq - b.content.seq); connection.queuedSignallingMessages.splice(idx, 0, message); + // dequeue as much as we can + let hasNewMessageBeenDequeued = false; if (connection.peerCall) { - const hasNewMessageBeenDequeued = this.dequeueSignallingMessages(connection, connection.peerCall, message, syncLog); - if (!hasNewMessageBeenDequeued) { - syncLog.refDetached(connection.logItem.log({l: "queued signalling message", type: message.type, seq: message.content.seq})); - } + hasNewMessageBeenDequeued = this.dequeueSignallingMessages(connection, connection.peerCall, message, log); } - } else if (action === IncomingMessageAction.Ignore && connection.peerCall) { - const logItem = connection.logItem.log({l: "ignoring to_device event with wrong call_id", callId: message.content.call_id, type: message.type}); - syncLog.refDetached(logItem); + if (!hasNewMessageBeenDequeued) { + log.refDetached(connection.logItem.log({l: "queued message", type: message.type, seq: message.content.seq, idx})); + } + } else { + // TODO: the right thing to do here would be to at least enqueue the message rather than drop it, + // and if it's up to the other end to send the invite and the type is an invite to actually + // call connect and assume the m.call.member state update is on its way? + syncLog.log({l: "member not connected", userId: this.userId, deviceId: this.deviceId}); } - } else { - syncLog.log({l: "member not connected", userId: this.userId, deviceId: this.deviceId}); - } + }); }); } private dequeueSignallingMessages(connection: MemberConnection, peerCall: PeerCall, newMessage: SignallingMessage, syncLog: ILogItem): boolean { let hasNewMessageBeenDequeued = false; while (connection.canDequeueNextSignallingMessage) { - syncLog.wrap("dequeue message", log => { - const message = connection.queuedSignallingMessages.shift()!; - if (message === newMessage) { - log.set("isNewMsg", true); - hasNewMessageBeenDequeued = true; - } + const message = connection.queuedSignallingMessages.shift()!; + const isNewMsg = message === newMessage; + hasNewMessageBeenDequeued = hasNewMessageBeenDequeued || isNewMsg; + syncLog.wrap(isNewMsg ? "process message" : "dequeue message", log => { const seq = message.content?.seq; log.set("seq", seq); + log.set("type", message.type); // ignore items in the queue that should not be handled and prevent // the lastProcessedSeqNr being corrupted with the `seq` for other call ids // XXX: Not needed anymore when seq is scoped to call_id @@ -411,7 +412,6 @@ export class Member { log.refDetached(item); connection.lastProcessedSeqNr = seq; } else { - log.set("type", message.type); log.set("ignored", true); connection.lastIgnoredSeqNr = seq; } From 0fa9d193d970e5da96018bc56ca691f9bc14d30a Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 19:42:43 +0100 Subject: [PATCH 304/323] fix comment typo --- src/matrix/calls/PeerCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 36136b97..2b28fbef 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -303,7 +303,7 @@ export class PeerCall implements IDisposable { } handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId, log: ILogItem): ILogItem { - // return logItem item immediately so it can be references in sync manner + // return logItem item immediately so it can be referenced by the sync log let logItem; log.wrap({ l: "receive signalling message", From 7114428b234198164359c60e7147923181687d92 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 19:57:42 +0100 Subject: [PATCH 305/323] now that the dom handler uses an error boundary, don't need this anymore --- src/matrix/calls/PeerCall.ts | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index d81bf047..f7af6546 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -431,18 +431,14 @@ export class PeerCall implements IDisposable { log.refDetached(timeoutLog); // don't await this, as it would block other negotationneeded events from being processed // as they are processed serially - try { - await timeoutLog.run(async log => { - await this.delay(CALL_TIMEOUT_MS); - // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between - if (this._state === CallState.InviteSent) { - await this._hangup(CallErrorCode.InviteTimeout, log); - } - }); - } - catch (e) { - // prevent error from being unhandled, it will be logged already by run above - } + await timeoutLog.run(async log => { + try { await this.delay(CALL_TIMEOUT_MS); } + catch (err) { return; } // return when delay is cancelled by throwing an AbortError + // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between + if (this._state === CallState.InviteSent) { + await this._hangup(CallErrorCode.InviteTimeout, log); + } + }); } }; From 5c2889aa5bd3139327ad5d3b23e880f357150e03 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 19:59:13 +0100 Subject: [PATCH 306/323] show DOM errors on the member error boundary rather than one for call gives a bit more context --- src/matrix/calls/group/GroupCall.ts | 3 +-- src/matrix/calls/group/Member.ts | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index d728e7c2..b82b3276 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -134,8 +134,7 @@ export class GroupCall extends EventEmitter<{change: never}> { }, encryptDeviceMessage: (userId: string, deviceId: string, message: SignallingMessage, log) => { return this.options.encryptDeviceMessage(this.roomId, userId, deviceId, message, log); - }, - groupCallErrorBoundary: this.errorBoundary, + } }); } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 0c31bc32..27842672 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -42,7 +42,6 @@ export type Options = Omit, log: ILogItem) => Promise, emitUpdate: (participant: Member, params?: any) => void, - groupCallErrorBoundary: ErrorBoundary, clock: Clock } @@ -422,7 +421,7 @@ export class Member { private _createPeerCall(callId: string): PeerCall { const connection = this.connection!; return new PeerCall(callId, Object.assign({}, this.options, { - errorBoundary: this.options.groupCallErrorBoundary, + errorBoundary: this.errorBoundary, emitUpdate: this.emitUpdateFromPeerCall, sendSignallingMessage: this.sendSignallingMessage, turnServer: connection.turnServer From 3ff91639c682b1f0746cfad0ba4fc4f0bc01416c Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 20:07:16 +0100 Subject: [PATCH 307/323] log signalingState on negotiationneeded --- src/matrix/calls/PeerCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 2b28fbef..a4ea984d 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -165,7 +165,7 @@ export class PeerCall implements IDisposable { }); listen("negotiationneeded", () => { const promiseCreator = () => { - return logItem.wrap("onNegotiationNeeded", log => { + return logItem.wrap({l: "onNegotiationNeeded", signalingState: this.peerConnection.signalingState}, log => { return this.handleNegotiation(log); }); }; From 5ad3f0c6712cac4f53ab7f1305fb38cce450d0d1 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 20:11:35 +0100 Subject: [PATCH 308/323] look at signalingState when even is fired, not later when it may have changed --- src/matrix/calls/PeerCall.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index a4ea984d..dc85084a 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -164,8 +164,9 @@ export class PeerCall implements IDisposable { }); }); listen("negotiationneeded", () => { + const signalingState = this.peerConnection.signalingState; const promiseCreator = () => { - return logItem.wrap({l: "onNegotiationNeeded", signalingState: this.peerConnection.signalingState}, log => { + return logItem.wrap({l: "onNegotiationNeeded", signalingState}, log => { return this.handleNegotiation(log); }); }; From dd89aab516ab8454c0d84eb56e165384e2e07890 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 20:19:06 +0100 Subject: [PATCH 309/323] fix typescript error in unit test --- src/logging/NullLogger.ts | 8 ++++---- src/matrix/calls/group/Member.ts | 13 ++++++++----- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/logging/NullLogger.ts b/src/logging/NullLogger.ts index bf7b3f40..83e5fc00 100644 --- a/src/logging/NullLogger.ts +++ b/src/logging/NullLogger.ts @@ -21,7 +21,7 @@ function noop (): void {} export class NullLogger implements ILogger { public readonly item: ILogItem = new NullLogItem(this); - log(): ILogItem { + log(labelOrValues: LabelOrValues): ILogItem { return this.item; } @@ -37,7 +37,7 @@ export class NullLogger implements ILogger { forceFinish(): void {} - child(): ILogItem { + child(labelOrValues: LabelOrValues): ILogItem { return this.item; } @@ -83,11 +83,11 @@ export class NullLogItem implements ILogItem { } - log(): ILogItem { + log(labelOrValues: LabelOrValues): ILogItem { return this; } - set(): ILogItem { return this; } + set(labelOrValues: LabelOrValues): ILogItem { return this; } runDetached(_: LabelOrValues, callback: LogCallback): ILogItem { new Promise(r => r(callback(this))).then(noop, noop); diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index daada4af..0a0e697c 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -511,7 +511,8 @@ export function tests() { feeds: [{purpose: "m.usermedia"}] }; const roomMember = RoomMember.fromUserId("!abc", "@bruno4:matrix.org", "join"); - const turnServer = new ObservableValue({}); + const turnServer = new ObservableValue({}) as ObservableValue; + // @ts-ignore const options = { confId: "conf", ownUserId: "@foobaraccount2:matrix.org", @@ -529,7 +530,8 @@ export function tests() { const member = new Member(roomMember, callDeviceMembership, options, logger.child("member")); member.connect(new MockMedia() as LocalMedia, new MuteSettings(), turnServer, logger.child("connect")); // pretend we've already received 3 messages - member.connection.lastProcessedSeqNr = 2; + // @ts-ignore + member.connection!.lastProcessedSeqNr = 2; // send hangup with seq=3, this will enqueue the message because there is no peerCall // as it's up to @bruno4:matrix.org to send the invite const hangup = { @@ -545,7 +547,7 @@ export function tests() { "sender_session_id": "s1d5863f41ec5a5", "dest_session_id": "s1cece7088b9d35" } - }; + } as SignallingMessage; member.handleDeviceMessage(hangup, logger.child("handle hangup")); // Send an invite with seq=4, this will create a new peer call with a the call id // when dequeueing the hangup from before, it'll get ignored because it is @@ -574,9 +576,10 @@ export function tests() { "sender_session_id": "s1d5863f41ec5a5", "dest_session_id": "s1cece7088b9d35" } - }; + } as SignallingMessage; member.handleDeviceMessage(invite, logger.child("handle invite")); - assert.equal(member.connection.queuedSignallingMessages.length, 0); + // @ts-ignore + assert.equal(member.connection!.queuedSignallingMessages.length, 0); // logger.reporters[0].printOpenItems(); } }; From 93661690e134e340a7363a330e39828430c11425 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 22:18:13 +0100 Subject: [PATCH 310/323] fix bg color of calls --- src/platform/web/ui/css/themes/element/call.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/platform/web/ui/css/themes/element/call.css b/src/platform/web/ui/css/themes/element/call.css index 3efe3c56..9b525dc2 100644 --- a/src/platform/web/ui/css/themes/element/call.css +++ b/src/platform/web/ui/css/themes/element/call.css @@ -37,7 +37,7 @@ limitations under the License. .CallView_members { display: grid; gap: 12px; - background: var(--background-color-secondary--darker-60); + background: var(--background-color-secondary--darker-5); padding: 12px; margin: 0; min-height: 0; From 928419502e6ce58bb03c1955dea8d7c36246fd43 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 22:44:43 +0100 Subject: [PATCH 311/323] no point in ref'ing an item on its parent --- src/matrix/calls/group/GroupCall.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index b82b3276..a47a3ff2 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -581,7 +581,6 @@ export class GroupCall extends EventEmitter<{change: never}> { sessionId: message.content.sender_session_id, type: message.type }); - syncLog.refDetached(item); // we haven't received the m.call.member yet for this caller (or with this session id). // buffer the device messages or create the member/call as it should arrive in a moment let messages = this.bufferedDeviceMessages.get(key); From fe5794a4bcfaee0b8bdbaa348391464085cdb47e Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 23:26:02 +0100 Subject: [PATCH 312/323] don't clear options as error boundary may fire after dispose --- src/matrix/calls/group/Member.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index db5482ee..5d67bafe 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -457,7 +457,7 @@ export class Member { this.expireTimeout?.dispose(); this.expireTimeout = undefined; // ensure the emitUpdate callback can't be called anymore - this.options = undefined as any as Options; + this.options.emitUpdate = () => {}; } } From c4944599cfe7268e32d43f8322cd113b37a8b52c Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 9 Feb 2023 11:52:00 +0100 Subject: [PATCH 313/323] add feature set to keep track of enabled features already include the calls feature --- src/features.ts | 50 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 src/features.ts diff --git a/src/features.ts b/src/features.ts new file mode 100644 index 00000000..e503f0e0 --- /dev/null +++ b/src/features.ts @@ -0,0 +1,50 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import type {SettingsStorage} from "./platform/web/dom/SettingsStorage"; + +export enum FeatureFlag { + Calls = 1 << 0, +} + +export class FeatureSet { + constructor(public readonly flags: number = 0) {} + + withFeature(flag: FeatureFlag): FeatureSet { + return new FeatureSet(this.flags | flag); + } + + withoutFeature(flag: FeatureFlag): FeatureSet { + return new FeatureSet(this.flags ^ flag); + } + + isFeatureEnabled(flag: FeatureFlag): boolean { + return (this.flags & flag) !== 0; + } + + get calls(): boolean { + return this.isFeatureEnabled(FeatureFlag.Calls); + } + + static async load(settingsStorage: SettingsStorage): Promise { + const flags = await settingsStorage.getInt("enabled_features") || 0; + return new FeatureSet(flags); + } + + async store(settingsStorage: SettingsStorage): Promise { + await settingsStorage.setInt("enabled_features", this.flags); + } +} From f65b43f612f0b3377bd895ed200ff4201739fbdd Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 9 Feb 2023 11:52:39 +0100 Subject: [PATCH 314/323] load features at startup and pass them along in all view models --- src/domain/ViewModel.ts | 6 ++++++ src/platform/web/main.js | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/src/domain/ViewModel.ts b/src/domain/ViewModel.ts index 34a05855..409e61c9 100644 --- a/src/domain/ViewModel.ts +++ b/src/domain/ViewModel.ts @@ -30,6 +30,7 @@ import type {Navigation} from "./navigation/Navigation"; import type {SegmentType} from "./navigation/index"; import type {IURLRouter} from "./navigation/URLRouter"; import type { ITimeFormatter } from "../platform/types/types"; +import type { FeatureSet } from "../features"; export type Options = { platform: Platform; @@ -37,6 +38,7 @@ export type Options = { urlRouter: IURLRouter; navigation: Navigation; emitChange?: (params: any) => void; + features: FeatureSet } @@ -142,6 +144,10 @@ export class ViewModel = Op return this._options.urlRouter; } + get features(): FeatureSet { + return this._options.features; + } + get navigation(): Navigation { // typescript needs a little help here return this._options.navigation as unknown as Navigation; diff --git a/src/platform/web/main.js b/src/platform/web/main.js index 2b28187e..d9c6fe8a 100644 --- a/src/platform/web/main.js +++ b/src/platform/web/main.js @@ -18,6 +18,8 @@ limitations under the License. // import {RecordRequester, ReplayRequester} from "./matrix/net/request/replay"; import {RootViewModel} from "../../domain/RootViewModel.js"; import {createNavigation, createRouter} from "../../domain/navigation/index"; +import {FeatureSet} from "../../features"; + // Don't use a default export here, as we use multiple entries during legacy build, // which does not support default exports, // see https://github.com/rollup/plugins/tree/master/packages/multi-entry @@ -33,6 +35,7 @@ export async function main(platform) { // const request = recorder.request; // window.getBrawlFetchLog = () => recorder.log(); await platform.init(); + const features = await FeatureSet.load(platform.settingsStorage); const navigation = createNavigation(); platform.setNavigation(navigation); const urlRouter = createRouter({navigation, history: platform.history}); @@ -43,6 +46,7 @@ export async function main(platform) { // so we call it that in the view models urlRouter: urlRouter, navigation, + features }); await vm.load(); platform.createAndMountRootView(vm); From d5929d9ebe6dd561637df57e7c450d58133747e9 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 9 Feb 2023 11:53:39 +0100 Subject: [PATCH 315/323] make features available in Client and Session --- src/domain/login/LoginViewModel.ts | 2 +- src/matrix/Client.js | 5 ++++- src/matrix/Session.js | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/domain/login/LoginViewModel.ts b/src/domain/login/LoginViewModel.ts index 75d57880..f43361d0 100644 --- a/src/domain/login/LoginViewModel.ts +++ b/src/domain/login/LoginViewModel.ts @@ -55,7 +55,7 @@ export class LoginViewModel extends ViewModel { const {ready, defaultHomeserver, loginToken} = options; this._ready = ready; this._loginToken = loginToken; - this._client = new Client(this.platform); + this._client = new Client(this.platform, this.features); this._homeserver = defaultHomeserver; this._initViewModels(); } diff --git a/src/matrix/Client.js b/src/matrix/Client.js index 5b53958b..fabb489b 100644 --- a/src/matrix/Client.js +++ b/src/matrix/Client.js @@ -31,6 +31,7 @@ import {TokenLoginMethod} from "./login/TokenLoginMethod"; import {SSOLoginHelper} from "./login/SSOLoginHelper"; import {getDehydratedDevice} from "./e2ee/Dehydration.js"; import {Registration} from "./registration/Registration"; +import {FeatureSet} from "../features"; export const LoadStatus = createEnum( "NotLoading", @@ -53,7 +54,7 @@ export const LoginFailure = createEnum( ); export class Client { - constructor(platform) { + constructor(platform, features = new FeatureSet(0)) { this._platform = platform; this._sessionStartedByReconnector = false; this._status = new ObservableValue(LoadStatus.NotLoading); @@ -68,6 +69,7 @@ export class Client { this._olmPromise = platform.loadOlm(); this._workerPromise = platform.loadOlmWorker(); this._accountSetup = undefined; + this._features = features; } createNewSessionId() { @@ -278,6 +280,7 @@ export class Client { olmWorker, mediaRepository, platform: this._platform, + features: this._features }); await this._session.load(log); if (dehydratedDevice) { diff --git a/src/matrix/Session.js b/src/matrix/Session.js index f1b273ff..3bcb0ff7 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -54,7 +54,7 @@ const PUSHER_KEY = "pusher"; export class Session { // sessionInfo contains deviceId, userId and homeserver - constructor({storage, hsApi, sessionInfo, olm, olmWorker, platform, mediaRepository}) { + constructor({storage, hsApi, sessionInfo, olm, olmWorker, platform, mediaRepository, features}) { this._platform = platform; this._storage = storage; this._hsApi = hsApi; From f86663fe7be2c05305fa2923ad8562529b50c3fd Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 9 Feb 2023 11:56:22 +0100 Subject: [PATCH 316/323] pass features to tilesCreator (and all options really for comfort) --- src/domain/session/room/RoomViewModel.js | 2 +- src/domain/session/room/timeline/TilesCollection.js | 2 +- src/domain/session/room/timeline/tiles/BaseMessageTile.js | 2 +- src/domain/session/room/timeline/tiles/index.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index a741050d..65afd348 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -209,7 +209,7 @@ export class RoomViewModel extends ErrorReportViewModel { _createTile(entry) { if (this._tileOptions) { - const Tile = this._tileOptions.tileClassForEntry(entry); + const Tile = this._tileOptions.tileClassForEntry(entry, this._tileOptions); if (Tile) { return new Tile(entry, this._tileOptions); } diff --git a/src/domain/session/room/timeline/TilesCollection.js b/src/domain/session/room/timeline/TilesCollection.js index 458697ca..c5bddc2c 100644 --- a/src/domain/session/room/timeline/TilesCollection.js +++ b/src/domain/session/room/timeline/TilesCollection.js @@ -34,7 +34,7 @@ export class TilesCollection extends BaseObservableList { } _createTile(entry) { - const Tile = this._tileOptions.tileClassForEntry(entry); + const Tile = this._tileOptions.tileClassForEntry(entry, this._tileOptions); if (Tile) { return new Tile(entry, this._tileOptions); } diff --git a/src/domain/session/room/timeline/tiles/BaseMessageTile.js b/src/domain/session/room/timeline/tiles/BaseMessageTile.js index 62c9632e..da5304ed 100644 --- a/src/domain/session/room/timeline/tiles/BaseMessageTile.js +++ b/src/domain/session/room/timeline/tiles/BaseMessageTile.js @@ -126,7 +126,7 @@ export class BaseMessageTile extends SimpleTile { if (action?.shouldReplace || !this._replyTile) { this.disposeTracked(this._replyTile); const tileClassForEntry = this._options.tileClassForEntry; - const ReplyTile = tileClassForEntry(replyEntry); + const ReplyTile = tileClassForEntry(replyEntry, this._options); if (ReplyTile) { this._replyTile = new ReplyTile(replyEntry, this._options); } diff --git a/src/domain/session/room/timeline/tiles/index.ts b/src/domain/session/room/timeline/tiles/index.ts index b4805635..32bc9c76 100644 --- a/src/domain/session/room/timeline/tiles/index.ts +++ b/src/domain/session/room/timeline/tiles/index.ts @@ -47,7 +47,7 @@ export type Options = ViewModelOptions & { }; export type TileConstructor = new (entry: TimelineEntry, options: Options) => ITile; -export function tileClassForEntry(entry: TimelineEntry): TileConstructor | undefined { +export function tileClassForEntry(entry: TimelineEntry, options: Options): TileConstructor | undefined { if (entry.isGap) { return GapTile; } else if (entry.isPending && entry.pendingEvent.isMissingAttachments) { From 22a81822664ffd81be919ca4e3077587eed5f23a Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 9 Feb 2023 11:57:30 +0100 Subject: [PATCH 317/323] feature-gate calls everywhere in the app --- src/domain/session/SessionViewModel.js | 8 +- src/domain/session/room/RoomViewModel.js | 7 ++ .../session/room/timeline/tiles/index.ts | 2 +- .../session/toast/ToastCollectionViewModel.ts | 6 +- src/matrix/Session.js | 103 ++++++++++-------- src/platform/web/ui/session/room/RoomView.js | 6 +- 6 files changed, 76 insertions(+), 56 deletions(-) diff --git a/src/domain/session/SessionViewModel.js b/src/domain/session/SessionViewModel.js index 072e1b74..7d1dac3c 100644 --- a/src/domain/session/SessionViewModel.js +++ b/src/domain/session/SessionViewModel.js @@ -130,9 +130,11 @@ export class SessionViewModel extends ViewModel { start() { this._sessionStatusViewModel.start(); - this._client.session.callHandler.loadCalls("m.ring"); - // TODO: only do this when opening the room - this._client.session.callHandler.loadCalls("m.prompt"); + if (this.features.calls) { + this._client.session.callHandler.loadCalls("m.ring"); + // TODO: only do this when opening the room + this._client.session.callHandler.loadCalls("m.prompt"); + } } get activeMiddleViewModel() { diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 65afd348..5cd610f2 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -50,6 +50,9 @@ export class RoomViewModel extends ErrorReportViewModel { } _setupCallViewModel() { + if (!this.features.calls) { + return; + } // pick call for this room with lowest key const calls = this.getOption("session").callHandler.calls; this._callObservable = new PickMapObservableValue(calls.filterValues(c => { @@ -421,6 +424,10 @@ export class RoomViewModel extends ErrorReportViewModel { startCall() { return this.logAndCatch("RoomViewModel.startCall", async log => { + if (!this.features.calls) { + log.set("feature_disbled", true); + return; + } log.set("roomId", this._room.id); let localMedia; try { diff --git a/src/domain/session/room/timeline/tiles/index.ts b/src/domain/session/room/timeline/tiles/index.ts index 32bc9c76..e86d61cb 100644 --- a/src/domain/session/room/timeline/tiles/index.ts +++ b/src/domain/session/room/timeline/tiles/index.ts @@ -92,7 +92,7 @@ export function tileClassForEntry(entry: TimelineEntry, options: Options): TileC case "org.matrix.msc3401.call": { // if prevContent is present, it's an update to a call event, which we don't render // as the original event is updated through the call object which receive state event updates - if (entry.stateKey && !entry.prevContent) { + if (options.features.calls && entry.stateKey && !entry.prevContent) { return CallTile; } return undefined; diff --git a/src/domain/session/toast/ToastCollectionViewModel.ts b/src/domain/session/toast/ToastCollectionViewModel.ts index d1595b2d..df4da88f 100644 --- a/src/domain/session/toast/ToastCollectionViewModel.ts +++ b/src/domain/session/toast/ToastCollectionViewModel.ts @@ -33,8 +33,10 @@ export class ToastCollectionViewModel extends ViewModel { constructor(options: Options) { super(options); const session = this.getOption("session"); - const callsObservableMap = session.callHandler.calls; - this.track(callsObservableMap.subscribe(this)); + if (this.features.calls) { + const callsObservableMap = session.callHandler.calls; + this.track(callsObservableMap.subscribe(this)); + } } async onAdd(_, call: GroupCall) { diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 3bcb0ff7..b3bd6f98 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -75,6 +75,61 @@ export class Session { }; this._roomsBeingCreated = new ObservableMap(); this._user = new User(sessionInfo.userId); + this._roomStateHandler = new RoomStateHandlerSet(); + this._deviceMessageHandler = new DeviceMessageHandler({storage, callHandler: this._callHandler}); + this._olm = olm; + this._olmUtil = null; + this._e2eeAccount = null; + this._deviceTracker = null; + this._olmEncryption = null; + this._keyLoader = null; + this._megolmEncryption = null; + this._megolmDecryption = null; + this._getSyncToken = () => this.syncToken; + this._olmWorker = olmWorker; + this._keyBackup = new ObservableValue(undefined); + this._observedRoomStatus = new Map(); + + if (olm) { + this._olmUtil = new olm.Utility(); + this._deviceTracker = new DeviceTracker({ + storage, + getSyncToken: this._getSyncToken, + olmUtil: this._olmUtil, + ownUserId: sessionInfo.userId, + ownDeviceId: sessionInfo.deviceId, + }); + } + this._createRoomEncryption = this._createRoomEncryption.bind(this); + this._forgetArchivedRoom = this._forgetArchivedRoom.bind(this); + this.needsKeyBackup = new ObservableValue(false); + + if (features.calls) { + this._setupCallHandler(); + } + } + + get fingerprintKey() { + return this._e2eeAccount?.identityKeys.ed25519; + } + + get hasSecretStorageKey() { + return this._hasSecretStorageKey; + } + + get deviceId() { + return this._sessionInfo.deviceId; + } + + get userId() { + return this._sessionInfo.userId; + } + + get callHandler() { + return this._callHandler; + } + + _setupCallHandler() { this._callHandler = new CallHandler({ clock: this._platform.clock, random: this._platform.random, @@ -103,55 +158,7 @@ export class Session { logger: this._platform.logger, forceTURN: false, }); - this._roomStateHandler = new RoomStateHandlerSet(); this.observeRoomState(this._callHandler); - this._deviceMessageHandler = new DeviceMessageHandler({storage, callHandler: this._callHandler}); - this._olm = olm; - this._olmUtil = null; - this._e2eeAccount = null; - this._deviceTracker = null; - this._olmEncryption = null; - this._keyLoader = null; - this._megolmEncryption = null; - this._megolmDecryption = null; - this._getSyncToken = () => this.syncToken; - this._olmWorker = olmWorker; - this._keyBackup = new ObservableValue(undefined); - this._observedRoomStatus = new Map(); - - if (olm) { - this._olmUtil = new olm.Utility(); - this._deviceTracker = new DeviceTracker({ - storage, - getSyncToken: this._getSyncToken, - olmUtil: this._olmUtil, - ownUserId: sessionInfo.userId, - ownDeviceId: sessionInfo.deviceId, - }); - } - this._createRoomEncryption = this._createRoomEncryption.bind(this); - this._forgetArchivedRoom = this._forgetArchivedRoom.bind(this); - this.needsKeyBackup = new ObservableValue(false); - } - - get fingerprintKey() { - return this._e2eeAccount?.identityKeys.ed25519; - } - - get hasSecretStorageKey() { - return this._hasSecretStorageKey; - } - - get deviceId() { - return this._sessionInfo.deviceId; - } - - get userId() { - return this._sessionInfo.userId; - } - - get callHandler() { - return this._callHandler; } // called once this._e2eeAccount is assigned diff --git a/src/platform/web/ui/session/room/RoomView.js b/src/platform/web/ui/session/room/RoomView.js index 892cb25e..727fb44d 100644 --- a/src/platform/web/ui/session/room/RoomView.js +++ b/src/platform/web/ui/session/room/RoomView.js @@ -73,8 +73,10 @@ export class RoomView extends TemplateView { } else { const vm = this.value; const options = []; - options.push(Menu.option(vm.i18n`Room details`, () => vm.openDetailsPanel())) - options.push(Menu.option(vm.i18n`Start call`, () => vm.startCall())) + options.push(Menu.option(vm.i18n`Room details`, () => vm.openDetailsPanel())); + if (vm.features.calls) { + options.push(Menu.option(vm.i18n`Start call`, () => vm.startCall())); + } if (vm.canLeave) { options.push(Menu.option(vm.i18n`Leave room`, () => this._confirmToLeaveRoom()).setDestructive()); } From 4a46c98d124239fcab319f470eacdc18fb25071b Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 9 Feb 2023 11:57:45 +0100 Subject: [PATCH 318/323] don't assume the call handler is always set in device message handler --- src/matrix/DeviceMessageHandler.js | 46 ++++++++++++++++-------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index 05276549..f6e7cad7 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -74,29 +74,31 @@ export class DeviceMessageHandler { } async afterSyncCompleted(decryptionResults, deviceTracker, hsApi, log) { - // if we don't have a device, we need to fetch the device keys the message claims - // and check the keys, and we should only do network requests during - // sync processing in the afterSyncCompleted step. - const callMessages = decryptionResults.filter(dr => this._callHandler.handlesDeviceMessageEventType(dr.event?.type)); - if (callMessages.length) { - await log.wrap("process call signalling messages", async log => { - for (const dr of callMessages) { - // serialize device loading, so subsequent messages for the same device take advantage of the cache - const device = await deviceTracker.deviceForId(dr.event.sender, dr.event.content.device_id, hsApi, log); - dr.setDevice(device); - if (dr.isVerified) { - this._callHandler.handleDeviceMessage(dr.event, dr.userId, dr.deviceId, log); - } else { - log.log({ - l: "could not verify olm fingerprint key matches, ignoring", - ed25519Key: dr.device.ed25519Key, - claimedEd25519Key: dr.claimedEd25519Key, - deviceId: device.deviceId, - userId: device.userId, - }); + if (this._callHandler) { + // if we don't have a device, we need to fetch the device keys the message claims + // and check the keys, and we should only do network requests during + // sync processing in the afterSyncCompleted step. + const callMessages = decryptionResults.filter(dr => this._callHandler.handlesDeviceMessageEventType(dr.event?.type)); + if (callMessages.length) { + await log.wrap("process call signalling messages", async log => { + for (const dr of callMessages) { + // serialize device loading, so subsequent messages for the same device take advantage of the cache + const device = await deviceTracker.deviceForId(dr.event.sender, dr.event.content.device_id, hsApi, log); + dr.setDevice(device); + if (dr.isVerified) { + this._callHandler.handleDeviceMessage(dr.event, dr.userId, dr.deviceId, log); + } else { + log.log({ + l: "could not verify olm fingerprint key matches, ignoring", + ed25519Key: dr.device.ed25519Key, + claimedEd25519Key: dr.claimedEd25519Key, + deviceId: device.deviceId, + userId: device.userId, + }); + } } - } - }); + }); + } } } } From f9fa59609f04b334c26ce7d023427575ef07e0ec Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 10 Feb 2023 08:48:10 +0100 Subject: [PATCH 319/323] fix local variable usage after extracting method --- src/matrix/Session.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index b3bd6f98..bd4de880 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -153,8 +153,8 @@ export class Session { }, storage: this._storage, webRTC: this._platform.webRTC, - ownDeviceId: sessionInfo.deviceId, - ownUserId: sessionInfo.userId, + ownDeviceId: this._sessionInfo.deviceId, + ownUserId: this._sessionInfo.userId, logger: this._platform.logger, forceTURN: false, }); From bb477b6aad0005471edaced6e65d9fa4d883263f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 10 Feb 2023 08:48:31 +0100 Subject: [PATCH 320/323] fix not passing features to client construction --- src/domain/RootViewModel.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/RootViewModel.js b/src/domain/RootViewModel.js index 524dfe13..2896fba6 100644 --- a/src/domain/RootViewModel.js +++ b/src/domain/RootViewModel.js @@ -158,7 +158,7 @@ export class RootViewModel extends ViewModel { } _showSessionLoader(sessionId) { - const client = new Client(this.platform); + const client = new Client(this.platform, this.features); client.startWithExistingSession(sessionId); this._setSection(() => { this._sessionLoadViewModel = new SessionLoadViewModel(this.childOptions({ From da1b7d410895e9279c8b4017375a74e2dfe258f2 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 10 Feb 2023 09:27:18 +0100 Subject: [PATCH 321/323] UI in settings for toggling features --- .../session/settings/FeaturesViewModel.ts | 70 +++++++++++++++++++ .../session/settings/SettingsViewModel.js | 6 ++ src/platform/web/Platform.js | 4 ++ .../web/ui/css/themes/element/theme.css | 18 +++++ .../web/ui/session/settings/FeaturesView.ts | 52 ++++++++++++++ .../web/ui/session/settings/SettingsView.js | 7 ++ 6 files changed, 157 insertions(+) create mode 100644 src/domain/session/settings/FeaturesViewModel.ts create mode 100644 src/platform/web/ui/session/settings/FeaturesView.ts diff --git a/src/domain/session/settings/FeaturesViewModel.ts b/src/domain/session/settings/FeaturesViewModel.ts new file mode 100644 index 00000000..69018bd2 --- /dev/null +++ b/src/domain/session/settings/FeaturesViewModel.ts @@ -0,0 +1,70 @@ +/* +Copyright 2020 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {ViewModel} from "../../ViewModel"; +import type {Options as BaseOptions} from "../../ViewModel"; +import {FeatureFlag, FeatureSet} from "../../../features"; +import type {SegmentType} from "../../navigation/index"; + +export class FeaturesViewModel extends ViewModel { + public readonly featureViewModels: ReadonlyArray; + + constructor(options) { + super(options); + this.featureViewModels = [ + new FeatureViewModel(this.childOptions({ + name: this.i18n`Audio/video calls (experimental)`, + description: this.i18n`Allows starting and participating in A/V calls compatible with Element Call (MSC3401). Look for the start call option in the room menu ((...) in the right corner) to start a call.`, + feature: FeatureFlag.Calls + })), + ]; + } +} + +type FeatureOptions = BaseOptions & { + feature: FeatureFlag, + description: string, + name: string +}; + +export class FeatureViewModel extends ViewModel { + get enabled(): boolean { + return this.features.isFeatureEnabled(this.getOption("feature")); + } + + async enableFeature(enabled: boolean): Promise { + let newFeatures; + if (enabled) { + newFeatures = this.features.withFeature(this.getOption("feature")); + } else { + newFeatures = this.features.withoutFeature(this.getOption("feature")); + } + await newFeatures.store(this.platform.settingsStorage); + this.platform.restart(); + } + + get id(): string { + return `${this.getOption("feature")}`; + } + + get name(): string { + return this.getOption("name"); + } + + get description(): string { + return this.getOption("description"); + } +} diff --git a/src/domain/session/settings/SettingsViewModel.js b/src/domain/session/settings/SettingsViewModel.js index 952c910b..f8420a53 100644 --- a/src/domain/session/settings/SettingsViewModel.js +++ b/src/domain/session/settings/SettingsViewModel.js @@ -16,6 +16,7 @@ limitations under the License. import {ViewModel} from "../../ViewModel"; import {KeyBackupViewModel} from "./KeyBackupViewModel.js"; +import {FeaturesViewModel} from "./FeaturesViewModel"; import {submitLogsFromSessionToDefaultServer} from "../../../domain/rageshake"; class PushNotificationStatus { @@ -53,6 +54,7 @@ export class SettingsViewModel extends ViewModel { this.pushNotifications = new PushNotificationStatus(); this._activeTheme = undefined; this._logsFeedbackMessage = undefined; + this._featuresViewModel = new FeaturesViewModel(this.childOptions()); } get _session() { @@ -125,6 +127,10 @@ export class SettingsViewModel extends ViewModel { return this._keyBackupViewModel; } + get featuresViewModel() { + return this._featuresViewModel; + } + get storageQuota() { return this._formatBytes(this._estimate?.quota); } diff --git a/src/platform/web/Platform.js b/src/platform/web/Platform.js index bfb585a3..be8c9970 100644 --- a/src/platform/web/Platform.js +++ b/src/platform/web/Platform.js @@ -283,6 +283,10 @@ export class Platform { } } + restart() { + document.location.reload(); + } + openFile(mimeType = null) { const input = document.createElement("input"); input.setAttribute("type", "file"); diff --git a/src/platform/web/ui/css/themes/element/theme.css b/src/platform/web/ui/css/themes/element/theme.css index 7bea169e..fd8e69b0 100644 --- a/src/platform/web/ui/css/themes/element/theme.css +++ b/src/platform/web/ui/css/themes/element/theme.css @@ -751,6 +751,24 @@ a { margin-bottom: 0; } +.FeatureView { + display: flex; + gap: 8px; +} + +.FeaturesView ul { + list-style: none; + padding: 8px 16px; +} + +.FeaturesView input[type="checkbox"] { + align-self: start; +} + +.FeatureView h4 { + margin: 0; +} + .error { color: var(--error-color); font-weight: 600; diff --git a/src/platform/web/ui/session/settings/FeaturesView.ts b/src/platform/web/ui/session/settings/FeaturesView.ts new file mode 100644 index 00000000..625fc362 --- /dev/null +++ b/src/platform/web/ui/session/settings/FeaturesView.ts @@ -0,0 +1,52 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {TemplateView, TemplateBuilder} from "../../general/TemplateView"; +import {ViewNode} from "../../general/types"; +import {disableTargetCallback} from "../../general/utils"; +import type {FeaturesViewModel, FeatureViewModel} from "../../../../../domain/session/settings/FeaturesViewModel"; + +export class FeaturesView extends TemplateView { + render(t, vm: FeaturesViewModel): ViewNode { + return t.div({ + className: "FeaturesView", + }, [ + t.p("Enable experimental features here that are still in development. These are not yet ready for primetime, so expect bugs."), + // we don't use a binding/ListView because this is a static list + t.ul(vm.featureViewModels.map(vm => { + return t.li(t.view(new FeatureView(vm))); + })) + ]); + } +} + +class FeatureView extends TemplateView { + render(t, vm): ViewNode { + let id = `feature_${vm.id}`; + return t.div({className: "FeatureView"}, [ + t.input({ + type: "checkbox", + id, + checked: vm => vm.enabled, + onChange: evt => vm.enableFeature(evt.target.checked) + }), + t.div({class: "FeatureView_container"}, [ + t.h4(t.label({for: id}, vm.name)), + t.p(vm.description) + ]) + ]); + } +} diff --git a/src/platform/web/ui/session/settings/SettingsView.js b/src/platform/web/ui/session/settings/SettingsView.js index d7c48351..0d0d6941 100644 --- a/src/platform/web/ui/session/settings/SettingsView.js +++ b/src/platform/web/ui/session/settings/SettingsView.js @@ -17,6 +17,7 @@ limitations under the License. import {TemplateView} from "../../general/TemplateView"; import {disableTargetCallback} from "../../general/utils"; import {KeyBackupSettingsView} from "./KeyBackupSettingsView.js" +import {FeaturesView} from "./FeaturesView" export class SettingsView extends TemplateView { render(t, vm) { @@ -110,6 +111,12 @@ export class SettingsView extends TemplateView { logButtons.push(t.button({onClick: disableTargetCallback(() => vm.sendLogsToServer())}, `Submit logs to ${vm.logsServer}`)); } logButtons.push(t.button({onClick: () => vm.exportLogs()}, "Download logs")); + + settingNodes.push( + t.h3("Experimental features"), + t.view(new FeaturesView(vm.featuresViewModel)) + ); + settingNodes.push( t.h3("Application"), row(t, vm.i18n`Version`, version), From 83d1ea05a1c9739d049bcb91e0663e06980c3fde Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 10 Feb 2023 11:22:13 +0100 Subject: [PATCH 322/323] fix unit test --- src/matrix/Session.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index bd4de880..d4c68a8d 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -1015,6 +1015,7 @@ export class Session { } } +import {FeatureSet} from "../features"; export function tests() { function createStorageMock(session, pendingEvents = []) { return { @@ -1058,7 +1059,8 @@ export function tests() { clock: { createTimeout: () => undefined } - } + }, + features: new FeatureSet(0) }); await session.load(); let syncSet = false; From c946319891dde161a64b32812f7c33f7797d098c Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 10 Feb 2023 11:33:19 +0100 Subject: [PATCH 323/323] remove double experimal warning in UI --- src/domain/session/settings/FeaturesViewModel.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/session/settings/FeaturesViewModel.ts b/src/domain/session/settings/FeaturesViewModel.ts index 69018bd2..6017cc6a 100644 --- a/src/domain/session/settings/FeaturesViewModel.ts +++ b/src/domain/session/settings/FeaturesViewModel.ts @@ -26,7 +26,7 @@ export class FeaturesViewModel extends ViewModel { super(options); this.featureViewModels = [ new FeatureViewModel(this.childOptions({ - name: this.i18n`Audio/video calls (experimental)`, + name: this.i18n`Audio/video calls`, description: this.i18n`Allows starting and participating in A/V calls compatible with Element Call (MSC3401). Look for the start call option in the room menu ((...) in the right corner) to start a call.`, feature: FeatureFlag.Calls })),