From 46ebd55092ad26dfbb822a2abd4b4f70a5f491cf Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Mon, 14 Feb 2022 17:14:21 +0100 Subject: [PATCH 001/435] WIP --- src/matrix/DeviceMessageHandler.js | 26 ++++- src/matrix/Session.js | 6 +- src/matrix/calls/CallHandler.ts | 156 ++++++++++++++++++++++++++++ src/matrix/e2ee/DecryptionResult.ts | 8 ++ src/matrix/room/Room.js | 22 ++++ src/utils/LRUCache.ts | 2 +- 6 files changed, 216 insertions(+), 4 deletions(-) create mode 100644 src/matrix/calls/CallHandler.ts diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index 6ac5ac07..c6bce31e 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -16,12 +16,15 @@ limitations under the License. import {OLM_ALGORITHM} from "./e2ee/common.js"; import {countBy, groupBy} from "../utils/groupBy"; +import {LRUCache} from "../../utils/LRUCache"; export class DeviceMessageHandler { - constructor({storage}) { + constructor({storage, callHandler}) { this._storage = storage; this._olmDecryption = null; this._megolmDecryption = null; + this._callHandler = callHandler; + this._senderDeviceCache = new LRUCache(10, di => di.curve25519Key); } enableEncryption({olmDecryption, megolmDecryption}) { @@ -49,6 +52,15 @@ export class DeviceMessageHandler { log.child("decrypt_error").catch(err); } const newRoomKeys = this._megolmDecryption.roomKeysFromDeviceMessages(olmDecryptChanges.results, log); + const callMessages = olmDecryptChanges.results.filter(dr => this._callHandler.handlesDeviceMessageEventType(dr.event?.type)); + await Promise.all(callMessages.map(async dr => { + dr.setDevice(await this._getDevice(dr.senderCurve25519Key, txn)); + this._callHandler.handleDeviceMessage(dr.device.userId, dr.device.deviceId, dr.event.type, dr.event.content, log); + })); + // TODO: somehow include rooms that received a call to_device message in the sync state? + // or have updates flow through event emitter? + // well, we don't really need to update the room other then when a call starts or stops + // any changes within the call will be emitted on the call object? return new SyncPreparation(olmDecryptChanges, newRoomKeys); } } @@ -60,6 +72,18 @@ export class DeviceMessageHandler { const didWriteValues = await Promise.all(prep.newRoomKeys.map(key => this._megolmDecryption.writeRoomKey(key, txn))); return didWriteValues.some(didWrite => !!didWrite); } + + + async _getDevice(senderKey, txn) { + let device = this._senderDeviceCache.get(senderKey); + if (!device) { + device = await txn.deviceIdentities.getByCurve25519Key(senderKey); + if (device) { + this._senderDeviceCache.set(device); + } + } + return device; + } } class SyncPreparation { diff --git a/src/matrix/Session.js b/src/matrix/Session.js index ae1dea61..3d9b13c8 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -73,7 +73,7 @@ export class Session { }; this._roomsBeingCreated = new ObservableMap(); this._user = new User(sessionInfo.userId); - this._deviceMessageHandler = new DeviceMessageHandler({storage}); + this._deviceMessageHandler = new DeviceMessageHandler({storage, callHandler: this._callHandler}); this._olm = olm; this._olmUtil = null; this._e2eeAccount = null; @@ -100,6 +100,7 @@ export class Session { this._createRoomEncryption = this._createRoomEncryption.bind(this); this._forgetArchivedRoom = this._forgetArchivedRoom.bind(this); this.needsKeyBackup = new ObservableValue(false); + this._callHandler = new CallHandler(this._platform, this._hsApi); } get fingerprintKey() { @@ -562,7 +563,8 @@ export class Session { pendingEvents, user: this._user, createRoomEncryption: this._createRoomEncryption, - platform: this._platform + platform: this._platform, + callHandler: this._callHandler }); } diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts new file mode 100644 index 00000000..55571c5a --- /dev/null +++ b/src/matrix/calls/CallHandler.ts @@ -0,0 +1,156 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {ObservableMap} from "../../observable/map/ObservableMap"; + +import type {Room} from "../room/Room"; +import type {StateEvent} from "../storage/types"; +import type {ILogItem} from "../../logging/types"; + +const GROUP_CALL_TYPE = "m.call"; +const GROUP_CALL_MEMBER_TYPE = "m.call.member"; + +enum CallSetupMessageType { + Invite = "m.call.invite", + Answer = "m.call.answer", + Candidates = "m.call.candidates", + Hangup = "m.call.hangup", +} + +const CALL_ID = "m.call_id"; +const CALL_TERMINATED = "m.terminated"; + +export class CallHandler { + public readonly groupCalls: ObservableMap = new ObservableMap(); + + constructor() { + + } + + handleRoomState(room: Room, events: StateEvent[], log: ILogItem) { + // first update call events + for (const event of events) { + if (event.type === GROUP_CALL_TYPE) { + const callId = event.state_key; + let call = this.groupCalls.get(callId); + if (call) { + call.updateCallEvent(event); + if (call.isTerminated) { + this.groupCalls.remove(call.id); + } + } else { + call = new GroupCall(event, room); + this.groupCalls.set(call.id, call); + } + } + } + // then update participants + for (const event of events) { + if (event.type === GROUP_CALL_MEMBER_TYPE) { + const participant = event.state_key; + const sources = event.content["m.sources"]; + for (const source of sources) { + const call = this.groupCalls.get(source[CALL_ID]); + if (call && !call.isTerminated) { + call.addParticipant(participant, source); + } + } + } + } + } + + handlesDeviceMessageEventType(eventType: string | undefined): boolean { + return eventType === CallSetupMessageType.Invite || + eventType === CallSetupMessageType.Candidates || + eventType === CallSetupMessageType.Answer || + eventType === CallSetupMessageType.Hangup; + } + + handleDeviceMessage(senderUserId: string, senderDeviceId: string, eventType: string, content: Record, log: ILogItem) { + const callId = content[CALL_ID]; + const call = this.groupCalls.get(callId); + call?.handleDeviceMessage(senderUserId, senderDeviceId, eventType, content, log); + } +} + +function peerCallKey(senderUserId: string, senderDeviceId: string) { + return JSON.stringify(senderUserId) + JSON.stringify(senderDeviceId); +} + +class GroupCall { + private peerCalls: Map + + constructor(private callEvent: StateEvent, private readonly room: Room) { + + } + + updateCallEvent(callEvent: StateEvent) { + this.callEvent = callEvent; + } + + addParticipant(userId, source) { + + } + + handleDeviceMessage(senderUserId: string, senderDeviceId: string, eventType: string, content: Record, log: ILogItem) { + const peerCall = this.peerCalls.get(peerCallKey(senderUserId, senderDeviceId)); + peerCall?.handleIncomingSignallingMessage() + } + + get id(): string { + return this.callEvent.state_key; + } + + get isTerminated(): boolean { + return !!this.callEvent.content[CALL_TERMINATED]; + } + + private createPeerCall(userId: string, deviceId: string): PeerCall { + + } +} + +/** + * Does WebRTC signalling for a single PeerConnection, and deals with WebRTC wrappers from platform + * */ + + +// when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already +// do for sharing keys will be best as that already deals with room tracking. +type SendSignallingMessageCallback = (type: CallSetupMessageType, content: Record) => Promise; + +class PeerCall { + constructor(private readonly sendSignallingMessage: SendSignallingMessageCallback) { + + } + + handleIncomingSignallingMessage(type: CallSetupMessageType, content: Record) { + switch (type) { + case CallSetupMessageType.Invite: + case CallSetupMessageType.Answer: + case CallSetupMessageType.Candidates: + case CallSetupMessageType.Hangup: + } + } +} + +class MediaSource { + +} + +class PeerConnection { + +} diff --git a/src/matrix/e2ee/DecryptionResult.ts b/src/matrix/e2ee/DecryptionResult.ts index 7735856a..8fe1a6ed 100644 --- a/src/matrix/e2ee/DecryptionResult.ts +++ b/src/matrix/e2ee/DecryptionResult.ts @@ -69,6 +69,14 @@ export class DecryptionResult { } } + get userId(): string | undefined { + return this.device?.userId; + } + + get deviceId(): string | undefined { + return this.device?.deviceId; + } + get isVerificationUnknown(): boolean { // verification is unknown if we haven't yet fetched the devices for the room return !this.device && !this.roomTracked; diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js index 12c17580..b9ec82a3 100644 --- a/src/matrix/room/Room.js +++ b/src/matrix/room/Room.js @@ -30,6 +30,7 @@ const EVENT_ENCRYPTED_TYPE = "m.room.encrypted"; export class Room extends BaseRoom { constructor(options) { super(options); + this._callHandler = options.callHandler; // TODO: pass pendingEvents to start like pendingOperations? const {pendingEvents} = options; const relationWriter = new RelationWriter({ @@ -92,6 +93,8 @@ export class Room extends BaseRoom { } } + this._updateCallHandler(roomResponse); + return { roomEncryption, summaryChanges, @@ -442,6 +445,25 @@ export class Room extends BaseRoom { return this._sendQueue.pendingEvents; } + _updateCallHandler(roomResponse) { + if (this._callHandler) { + const stateEvents = roomResponse.state?.events; + if (stateEvents) { + for (const e of stateEvents) { + this._callHandler.handleRoomState(this, e); + } + } + let timelineEvents = roomResponse.timeline?.events; + if (timelineEvents) { + for (const e of timelineEvents) { + if (typeof e.state_key === "string") { + this._callHandler.handleRoomState(this, e); + } + } + } + } + } + /** @package */ writeIsTrackingMembers(value, txn) { return this._summary.writeIsTrackingMembers(value, txn); diff --git a/src/utils/LRUCache.ts b/src/utils/LRUCache.ts index c5a7cd06..bab9bf51 100644 --- a/src/utils/LRUCache.ts +++ b/src/utils/LRUCache.ts @@ -71,7 +71,7 @@ export class BaseLRUCache { export class LRUCache extends BaseLRUCache { private _keyFn: (T) => K; - constructor(limit, keyFn: (T) => K) { + constructor(limit: number, keyFn: (T) => K) { super(limit); this._keyFn = keyFn; } From b12bc52c4a96448e0332f58ede8c57526e233a33 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Tue, 15 Feb 2022 17:05:20 +0100 Subject: [PATCH 002/435] WIP2 --- src/matrix/calls/Call.ts | 2493 ++++++++++++++++++++++++++++ src/matrix/calls/CallFeed.ts | 274 +++ src/matrix/calls/CallHandler.ts | 1 + src/matrix/calls/callEventTypes.ts | 92 + 4 files changed, 2860 insertions(+) create mode 100644 src/matrix/calls/Call.ts create mode 100644 src/matrix/calls/CallFeed.ts create mode 100644 src/matrix/calls/callEventTypes.ts diff --git a/src/matrix/calls/Call.ts b/src/matrix/calls/Call.ts new file mode 100644 index 00000000..d6ac4612 --- /dev/null +++ b/src/matrix/calls/Call.ts @@ -0,0 +1,2493 @@ +/* +Copyright 2015, 2016 OpenMarket Ltd +Copyright 2017 New Vector Ltd +Copyright 2019, 2020 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { + MCallReplacesEvent, + MCallAnswer, + MCallInviteNegotiate, + CallCapabilities, + SDPStreamMetadataPurpose, + SDPStreamMetadata, + SDPStreamMetadataKey, + MCallSDPStreamMetadataChanged, + MCallSelectAnswer, + MCAllAssertedIdentity, + MCallCandidates, + MCallBase, + MCallHangupReject, +} from './callEventTypes'; +import { CallFeed } from './CallFeed'; + +// events: hangup, error(err), replaced(call), state(state, oldState) + +/** + * Fires whenever an error occurs when call.js encounters an issue with setting up the call. + *

+ * The error given will have a code equal to either `MatrixCall.ERR_LOCAL_OFFER_FAILED` or + * `MatrixCall.ERR_NO_USER_MEDIA`. `ERR_LOCAL_OFFER_FAILED` is emitted when the local client + * fails to create an offer. `ERR_NO_USER_MEDIA` is emitted when the user has denied access + * to their audio/video hardware. + * + * @event module:webrtc/call~MatrixCall#"error" + * @param {Error} err The error raised by MatrixCall. + * @example + * matrixCall.on("error", function(err){ + * console.error(err.code, err); + * }); + */ + +interface CallOpts { + roomId?: string; + invitee?: string; + client?: any; // Fix when client is TSified + forceTURN?: boolean; + turnServers?: Array; + opponentDeviceId?: string; + opponentSessionId?: string; + groupCallId?: string; +} + +interface TurnServer { + urls: Array; + username?: string; + password?: string; + ttl?: number; +} + +interface AssertedIdentity { + id: string; + displayName: string; +} + +export enum CallState { + Fledgling = 'fledgling', + InviteSent = 'invite_sent', + WaitLocalMedia = 'wait_local_media', + CreateOffer = 'create_offer', + CreateAnswer = 'create_answer', + Connecting = 'connecting', + Connected = 'connected', + Ringing = 'ringing', + Ended = 'ended', +} + +export enum CallType { + Voice = 'voice', + Video = 'video', +} + +export enum CallDirection { + Inbound = 'inbound', + Outbound = 'outbound', +} + +export enum CallParty { + Local = 'local', + Remote = 'remote', +} + +export enum CallEvent { + Hangup = 'hangup', + State = 'state', + Error = 'error', + Replaced = 'replaced', + + // The value of isLocalOnHold() has changed + LocalHoldUnhold = 'local_hold_unhold', + // The value of isRemoteOnHold() has changed + RemoteHoldUnhold = 'remote_hold_unhold', + // backwards compat alias for LocalHoldUnhold: remove in a major version bump + HoldUnhold = 'hold_unhold', + // Feeds have changed + FeedsChanged = 'feeds_changed', + + AssertedIdentityChanged = 'asserted_identity_changed', + + LengthChanged = 'length_changed', + + DataChannel = 'datachannel', + + SendVoipEvent = "send_voip_event", +} + +export enum CallErrorCode { + /** The user chose to end the call */ + UserHangup = 'user_hangup', + + /** An error code when the local client failed to create an offer. */ + LocalOfferFailed = 'local_offer_failed', + /** + * An error code when there is no local mic/camera to use. This may be because + * the hardware isn't plugged in, or the user has explicitly denied access. + */ + NoUserMedia = 'no_user_media', + + /** + * Error code used when a call event failed to send + * because unknown devices were present in the room + */ + UnknownDevices = 'unknown_devices', + + /** + * Error code used when we fail to send the invite + * for some reason other than there being unknown devices + */ + SendInvite = 'send_invite', + + /** + * An answer could not be created + */ + CreateAnswer = 'create_answer', + + /** + * Error code used when we fail to send the answer + * for some reason other than there being unknown devices + */ + SendAnswer = 'send_answer', + + /** + * The session description from the other side could not be set + */ + SetRemoteDescription = 'set_remote_description', + + /** + * The session description from this side could not be set + */ + SetLocalDescription = 'set_local_description', + + /** + * A different device answered the call + */ + AnsweredElsewhere = 'answered_elsewhere', + + /** + * No media connection could be established to the other party + */ + IceFailed = 'ice_failed', + + /** + * The invite timed out whilst waiting for an answer + */ + InviteTimeout = 'invite_timeout', + + /** + * The call was replaced by another call + */ + Replaced = 'replaced', + + /** + * Signalling for the call could not be sent (other than the initial invite) + */ + SignallingFailed = 'signalling_timeout', + + /** + * The remote party is busy + */ + UserBusy = 'user_busy', + + /** + * We transferred the call off to somewhere else + */ + Transfered = 'transferred', + + /** + * A call from the same user was found with a new session id + */ + NewSession = 'new_session', +} + +/** + * The version field that we set in m.call.* events + */ +const VOIP_PROTO_VERSION = 1; + +/** The fallback ICE server to use for STUN or TURN protocols. */ +const FALLBACK_ICE_SERVER = 'stun:turn.matrix.org'; + +/** The length of time a call can be ringing for. */ +const CALL_TIMEOUT_MS = 60000; + +const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); + +export class CallError extends Error { + code: string; + + constructor(code: CallErrorCode, msg: string, err: Error) { + // Still don't think there's any way to have proper nested errors + super(msg + ": " + err); + + this.code = code; + } +} + +export function genCallID(): string { + return Date.now().toString() + randomString(16); +} + +/** + * Construct a new Matrix Call. + * @constructor + * @param {Object} opts Config options. + * @param {string} opts.roomId The room ID for this call. + * @param {Object} opts.webRtc The WebRTC globals from the browser. + * @param {boolean} opts.forceTURN whether relay through TURN should be forced. + * @param {Object} opts.URL The URL global. + * @param {Array} opts.turnServers Optional. A list of TURN servers. + * @param {MatrixClient} opts.client The Matrix Client instance to send events to. + */ +export class MatrixCall extends EventEmitter { + public roomId: string; + public callId: string; + public invitee?: string; + public state = CallState.Fledgling; + public hangupParty: CallParty; + public hangupReason: string; + public direction: CallDirection; + public ourPartyId: string; + public peerConn?: RTCPeerConnection; + + private client: MatrixClient; + private forceTURN: boolean; + private turnServers: Array; + // A queue for candidates waiting to go out. + // We try to amalgamate candidates into a single candidate message where + // possible + private candidateSendQueue: Array = []; + private candidateSendTries = 0; + private sentEndOfCandidates = false; + private feeds: Array = []; + private usermediaSenders: Array = []; + private screensharingSenders: Array = []; + private inviteOrAnswerSent = false; + private waitForLocalAVStream: boolean; + private successor: MatrixCall; + private opponentMember: RoomMember; + private opponentVersion: number | string; + // The party ID of the other side: undefined if we haven't chosen a partner + // yet, null if we have but they didn't send a party ID. + private opponentPartyId: string; + private opponentCaps: CallCapabilities; + private inviteTimeout: number; + private iceDisconnectedTimeout: number; + + // The logic of when & if a call is on hold is nontrivial and explained in is*OnHold + // This flag represents whether we want the other party to be on hold + private remoteOnHold = false; + + // the stats for the call at the point it ended. We can't get these after we + // tear the call down, so we just grab a snapshot before we stop the call. + // The typescript definitions have this type as 'any' :( + private callStatsAtEnd: any[]; + + // Perfect negotiation state: https://www.w3.org/TR/webrtc/#perfect-negotiation-example + private makingOffer = false; + private ignoreOffer: boolean; + + private responsePromiseChain?: Promise; + + // If candidates arrive before we've picked an opponent (which, in particular, + // will happen if the opponent sends candidates eagerly before the user answers + // the call) we buffer them up here so we can then add the ones from the party we pick + private remoteCandidateBuffer = new Map(); + + private remoteAssertedIdentity: AssertedIdentity; + + private remoteSDPStreamMetadata: SDPStreamMetadata; + + private callLengthInterval: number; + private callLength = 0; + + private opponentDeviceId: string; + private opponentSessionId: string; + public groupCallId: string; + + constructor(opts: CallOpts) { + super(); + this.roomId = opts.roomId; + this.invitee = opts.invitee; + this.client = opts.client; + this.forceTURN = opts.forceTURN; + this.ourPartyId = this.client.deviceId; + this.opponentDeviceId = opts.opponentDeviceId; + this.opponentSessionId = opts.opponentSessionId; + this.groupCallId = opts.groupCallId; + // Array of Objects with urls, username, credential keys + this.turnServers = opts.turnServers || []; + if (this.turnServers.length === 0 && this.client.isFallbackICEServerAllowed()) { + this.turnServers.push({ + urls: [FALLBACK_ICE_SERVER], + }); + } + for (const server of this.turnServers) { + utils.checkObjectHasKeys(server, ["urls"]); + } + this.callId = genCallID(); + } + + /** + * Place a voice call to this room. + * @throws If you have not specified a listener for 'error' events. + */ + public async placeVoiceCall(): Promise { + await this.placeCall(true, false); + } + + /** + * Place a video call to this room. + * @throws If you have not specified a listener for 'error' events. + */ + public async placeVideoCall(): Promise { + await this.placeCall(true, true); + } + + /** + * Create a datachannel using this call's peer connection. + * @param label A human readable label for this datachannel + * @param options An object providing configuration options for the data channel. + */ + public createDataChannel(label: string, options: RTCDataChannelInit) { + const dataChannel = this.peerConn.createDataChannel(label, options); + this.emit(CallEvent.DataChannel, dataChannel); + return dataChannel; + } + + public getOpponentMember(): RoomMember { + return this.opponentMember; + } + + public getOpponentSessionId(): string { + return this.opponentSessionId; + } + + public opponentCanBeTransferred(): boolean { + return Boolean(this.opponentCaps && this.opponentCaps["m.call.transferee"]); + } + + public opponentSupportsDTMF(): boolean { + return Boolean(this.opponentCaps && this.opponentCaps["m.call.dtmf"]); + } + + public getRemoteAssertedIdentity(): AssertedIdentity { + return this.remoteAssertedIdentity; + } + + public get type(): CallType { + return (this.hasLocalUserMediaVideoTrack || this.hasRemoteUserMediaVideoTrack) + ? CallType.Video + : CallType.Voice; + } + + public get hasLocalUserMediaVideoTrack(): boolean { + return this.localUsermediaStream?.getVideoTracks().length > 0; + } + + public get hasRemoteUserMediaVideoTrack(): boolean { + return this.getRemoteFeeds().some((feed) => { + return ( + feed.purpose === SDPStreamMetadataPurpose.Usermedia && + feed.stream.getVideoTracks().length > 0 + ); + }); + } + + public get hasLocalUserMediaAudioTrack(): boolean { + return this.localUsermediaStream?.getAudioTracks().length > 0; + } + + public get hasRemoteUserMediaAudioTrack(): boolean { + return this.getRemoteFeeds().some((feed) => { + return ( + feed.purpose === SDPStreamMetadataPurpose.Usermedia && + feed.stream.getAudioTracks().length > 0 + ); + }); + } + + public get localUsermediaFeed(): CallFeed { + return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); + } + + public get localScreensharingFeed(): CallFeed { + return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); + } + + public get localUsermediaStream(): MediaStream { + return this.localUsermediaFeed?.stream; + } + + public get localScreensharingStream(): MediaStream { + return this.localScreensharingFeed?.stream; + } + + public get remoteUsermediaFeed(): CallFeed { + return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); + } + + public get remoteScreensharingFeed(): CallFeed { + return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); + } + + public get remoteUsermediaStream(): MediaStream { + return this.remoteUsermediaFeed?.stream; + } + + public get remoteScreensharingStream(): MediaStream { + return this.remoteScreensharingFeed?.stream; + } + + private getFeedByStreamId(streamId: string): CallFeed { + return this.getFeeds().find((feed) => feed.stream.id === streamId); + } + + /** + * Returns an array of all CallFeeds + * @returns {Array} CallFeeds + */ + public getFeeds(): Array { + return this.feeds; + } + + /** + * Returns an array of all local CallFeeds + * @returns {Array} local CallFeeds + */ + public getLocalFeeds(): Array { + return this.feeds.filter((feed) => feed.isLocal()); + } + + /** + * Returns an array of all remote CallFeeds + * @returns {Array} remote CallFeeds + */ + public getRemoteFeeds(): Array { + return this.feeds.filter((feed) => !feed.isLocal()); + } + + /** + * Generates and returns localSDPStreamMetadata + * @returns {SDPStreamMetadata} localSDPStreamMetadata + */ + private getLocalSDPStreamMetadata(updateStreamIds = false): SDPStreamMetadata { + const metadata: SDPStreamMetadata = {}; + for (const localFeed of this.getLocalFeeds()) { + if (updateStreamIds) { + localFeed.sdpMetadataStreamId = localFeed.stream.id; + } + + metadata[localFeed.sdpMetadataStreamId] = { + purpose: localFeed.purpose, + audio_muted: localFeed.isAudioMuted(), + video_muted: localFeed.isVideoMuted(), + }; + } + return metadata; + } + + /** + * Returns true if there are no incoming feeds, + * otherwise returns false + * @returns {boolean} no incoming feeds + */ + public noIncomingFeeds(): boolean { + return !this.feeds.some((feed) => !feed.isLocal()); + } + + private pushRemoteFeed(stream: MediaStream): void { + // Fallback to old behavior if the other side doesn't support SDPStreamMetadata + if (!this.opponentSupportsSDPStreamMetadata()) { + this.pushRemoteFeedWithoutMetadata(stream); + return; + } + + const userId = this.getOpponentMember().userId; + const purpose = this.remoteSDPStreamMetadata[stream.id].purpose; + const audioMuted = this.remoteSDPStreamMetadata[stream.id].audio_muted; + const videoMuted = this.remoteSDPStreamMetadata[stream.id].video_muted; + + if (!purpose) { + logger.warn(`Ignoring stream with id ${stream.id} because we didn't get any metadata about it`); + return; + } + + // Try to find a feed with the same purpose as the new stream, + // if we find it replace the old stream with the new one + const existingFeed = this.getRemoteFeeds().find((feed) => feed.purpose === purpose); + if (existingFeed) { + existingFeed.setNewStream(stream); + } else { + this.feeds.push(new CallFeed({ + client: this.client, + roomId: this.roomId, + userId, + stream, + purpose, + audioMuted, + videoMuted, + })); + this.emit(CallEvent.FeedsChanged, this.feeds); + } + + logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}", purpose=${purpose})`); + } + + /** + * This method is used ONLY if the other client doesn't support sending SDPStreamMetadata + */ + private pushRemoteFeedWithoutMetadata(stream: MediaStream): void { + const userId = this.getOpponentMember().userId; + // We can guess the purpose here since the other client can only send one stream + const purpose = SDPStreamMetadataPurpose.Usermedia; + const oldRemoteStream = this.feeds.find((feed) => !feed.isLocal())?.stream; + + // Note that we check by ID and always set the remote stream: Chrome appears + // to make new stream objects when transceiver directionality is changed and the 'active' + // status of streams change - Dave + // If we already have a stream, check this stream has the same id + if (oldRemoteStream && stream.id !== oldRemoteStream.id) { + logger.warn(`Ignoring new stream ID ${stream.id}: we already have stream ID ${oldRemoteStream.id}`); + return; + } + + // Try to find a feed with the same stream id as the new stream, + // if we find it replace the old stream with the new one + const feed = this.getFeedByStreamId(stream.id); + if (feed) { + feed.setNewStream(stream); + } else { + this.feeds.push(new CallFeed({ + client: this.client, + roomId: this.roomId, + audioMuted: false, + videoMuted: false, + userId, + stream, + purpose, + })); + this.emit(CallEvent.FeedsChanged, this.feeds); + } + + logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}")`); + } + + private pushNewLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose, addToPeerConnection = true): void { + const userId = this.client.getUserId(); + + // TODO: Find out what is going on here + // why do we enable audio (and only audio) tracks here? -- matthew + setTracksEnabled(stream.getAudioTracks(), true); + + // We try to replace an existing feed if there already is one with the same purpose + const existingFeed = this.getLocalFeeds().find((feed) => feed.purpose === purpose); + if (existingFeed) { + existingFeed.setNewStream(stream); + } else { + this.pushLocalFeed( + new CallFeed({ + client: this.client, + roomId: this.roomId, + audioMuted: stream.getAudioTracks().length === 0, + videoMuted: stream.getVideoTracks().length === 0, + userId, + stream, + purpose, + }), + addToPeerConnection, + ); + this.emit(CallEvent.FeedsChanged, this.feeds); + } + } + + /** + * Pushes supplied feed to the call + * @param {CallFeed} callFeed to push + * @param {boolean} addToPeerConnection whether to add the tracks to the peer connection + */ + public pushLocalFeed(callFeed: CallFeed, addToPeerConnection = true): void { + if (this.feeds.some((feed) => callFeed.stream.id === feed.stream.id)) { + logger.info(`Ignoring duplicate local stream ${callFeed.stream.id} in call ${this.callId}`); + return; + } + + this.feeds.push(callFeed); + + if (addToPeerConnection) { + const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia ? + this.usermediaSenders : this.screensharingSenders; + // Empty the array + senderArray.splice(0, senderArray.length); + + for (const track of callFeed.stream.getTracks()) { + logger.info( + `Adding track (` + + `id="${track.id}", ` + + `kind="${track.kind}", ` + + `streamId="${callFeed.stream.id}", ` + + `streamPurpose="${callFeed.purpose}"` + + `) to peer connection`, + ); + senderArray.push(this.peerConn.addTrack(track, callFeed.stream)); + } + } + + logger.info( + `Pushed local stream `+ + `(id="${callFeed.stream.id}", `+ + `active="${callFeed.stream.active}", `+ + `purpose="${callFeed.purpose}")`, + ); + + this.emit(CallEvent.FeedsChanged, this.feeds); + } + + /** + * Removes local call feed from the call and its tracks from the peer + * connection + * @param callFeed to remove + */ + public removeLocalFeed(callFeed: CallFeed): void { + const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia + ? this.usermediaSenders + : this.screensharingSenders; + + for (const sender of senderArray) { + this.peerConn.removeTrack(sender); + } + + if (callFeed.purpose === SDPStreamMetadataPurpose.Screenshare) { + this.client.getMediaHandler().stopScreensharingStream(callFeed.stream); + } + + // Empty the array + senderArray.splice(0, senderArray.length); + this.deleteFeed(callFeed); + } + + private deleteAllFeeds(): void { + for (const feed of this.feeds) { + if (!feed.isLocal() || !this.groupCallId) { + feed.dispose(); + } + } + + this.feeds = []; + this.emit(CallEvent.FeedsChanged, this.feeds); + } + + private deleteFeedByStream(stream: MediaStream): void { + const feed = this.getFeedByStreamId(stream.id); + if (!feed) { + logger.warn(`Didn't find the feed with stream id ${stream.id} to delete`); + return; + } + this.deleteFeed(feed); + } + + private deleteFeed(feed: CallFeed): void { + feed.dispose(); + this.feeds.splice(this.feeds.indexOf(feed), 1); + this.emit(CallEvent.FeedsChanged, this.feeds); + } + + // The typescript definitions have this type as 'any' :( + public async getCurrentCallStats(): Promise { + if (this.callHasEnded()) { + return this.callStatsAtEnd; + } + + return this.collectCallStats(); + } + + private async collectCallStats(): Promise { + // This happens when the call fails before it starts. + // For example when we fail to get capture sources + if (!this.peerConn) return; + + const statsReport = await this.peerConn.getStats(); + const stats = []; + for (const item of statsReport) { + stats.push(item[1]); + } + + return stats; + } + + /** + * Configure this call from an invite event. Used by MatrixClient. + * @param {MatrixEvent} event The m.call.invite event + */ + public async initWithInvite(event: MatrixEvent): Promise { + const invite = event.getContent(); + this.direction = CallDirection.Inbound; + + // make sure we have valid turn creds. Unless something's gone wrong, it should + // poll and keep the credentials valid so this should be instant. + const haveTurnCreds = await this.client.checkTurnServers(); + if (!haveTurnCreds) { + logger.warn("Failed to get TURN credentials! Proceeding with call anyway..."); + } + + const sdpStreamMetadata = invite[SDPStreamMetadataKey]; + if (sdpStreamMetadata) { + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); + } else { + logger.debug("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); + } + + this.peerConn = this.createPeerConnection(); + // we must set the party ID before await-ing on anything: the call event + // handler will start giving us more call events (eg. candidates) so if + // we haven't set the party ID, we'll ignore them. + this.chooseOpponent(event); + try { + await this.peerConn.setRemoteDescription(invite.offer); + await this.addBufferedIceCandidates(); + } catch (e) { + logger.debug("Failed to set remote description", e); + this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); + return; + } + + const remoteStream = this.feeds.find((feed) => !feed.isLocal())?.stream; + + // According to previous comments in this file, firefox at some point did not + // add streams until media started arriving on them. Testing latest firefox + // (81 at time of writing), this is no longer a problem, so let's do it the correct way. + if (!remoteStream || remoteStream.getTracks().length === 0) { + logger.error("No remote stream or no tracks after setting remote description!"); + this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); + return; + } + + this.setState(CallState.Ringing); + + if (event.getLocalAge()) { + setTimeout(() => { + if (this.state == CallState.Ringing) { + logger.debug("Call invite has expired. Hanging up."); + this.hangupParty = CallParty.Remote; // effectively + this.setState(CallState.Ended); + this.stopAllMedia(); + if (this.peerConn.signalingState != 'closed') { + this.peerConn.close(); + } + this.emit(CallEvent.Hangup); + } + }, invite.lifetime - event.getLocalAge()); + } + } + + /** + * Configure this call from a hangup or reject event. Used by MatrixClient. + * @param {MatrixEvent} event The m.call.hangup event + */ + public initWithHangup(event: MatrixEvent): void { + // perverse as it may seem, sometimes we want to instantiate a call with a + // hangup message (because when getting the state of the room on load, events + // come in reverse order and we want to remember that a call has been hung up) + this.setState(CallState.Ended); + } + + private shouldAnswerWithMediaType( + wantedValue: boolean | undefined, valueOfTheOtherSide: boolean | undefined, type: "audio" | "video", + ): boolean { + if (wantedValue && !valueOfTheOtherSide) { + // TODO: Figure out how to do this + logger.warn(`Unable to answer with ${type} because the other side isn't sending it either.`); + return false; + } else if ( + !utils.isNullOrUndefined(wantedValue) && + wantedValue !== valueOfTheOtherSide && + !this.opponentSupportsSDPStreamMetadata() + ) { + logger.warn( + `Unable to answer with ${type}=${wantedValue} because the other side doesn't support it. ` + + `Answering with ${type}=${valueOfTheOtherSide}.`, + ); + return valueOfTheOtherSide; + } + return wantedValue ?? valueOfTheOtherSide; + } + + /** + * Answer a call. + */ + public async answer(audio?: boolean, video?: boolean): Promise { + if (this.inviteOrAnswerSent) return; + // TODO: Figure out how to do this + if (audio === false && video === false) throw new Error("You CANNOT answer a call without media"); + + if (!this.localUsermediaStream && !this.waitForLocalAVStream) { + const prevState = this.state; + const answerWithAudio = this.shouldAnswerWithMediaType(audio, this.hasRemoteUserMediaAudioTrack, "audio"); + const answerWithVideo = this.shouldAnswerWithMediaType(video, this.hasRemoteUserMediaVideoTrack, "video"); + + this.setState(CallState.WaitLocalMedia); + this.waitForLocalAVStream = true; + + try { + const stream = await this.client.getMediaHandler().getUserMediaStream( + answerWithAudio, answerWithVideo, + ); + this.waitForLocalAVStream = false; + const usermediaFeed = new CallFeed({ + client: this.client, + roomId: this.roomId, + userId: this.client.getUserId(), + stream, + purpose: SDPStreamMetadataPurpose.Usermedia, + audioMuted: stream.getAudioTracks().length === 0, + videoMuted: stream.getVideoTracks().length === 0, + }); + + const feeds = [usermediaFeed]; + + if (this.localScreensharingFeed) { + feeds.push(this.localScreensharingFeed); + } + + this.answerWithCallFeeds(feeds); + } catch (e) { + if (answerWithVideo) { + // Try to answer without video + logger.warn("Failed to getUserMedia(), trying to getUserMedia() without video"); + this.setState(prevState); + this.waitForLocalAVStream = false; + await this.answer(answerWithAudio, false); + } else { + this.getUserMediaFailed(e); + return; + } + } + } else if (this.waitForLocalAVStream) { + this.setState(CallState.WaitLocalMedia); + } + } + + public answerWithCallFeeds(callFeeds: CallFeed[]): void { + if (this.inviteOrAnswerSent) return; + + logger.debug(`Answering call ${this.callId}`); + + this.queueGotCallFeedsForAnswer(callFeeds); + } + + /** + * Replace this call with a new call, e.g. for glare resolution. Used by + * MatrixClient. + * @param {MatrixCall} newCall The new call. + */ + public replacedBy(newCall: MatrixCall): void { + if (this.state === CallState.WaitLocalMedia) { + logger.debug("Telling new call to wait for local media"); + newCall.waitForLocalAVStream = true; + } else if ([CallState.CreateOffer, CallState.InviteSent].includes(this.state)) { + if (newCall.direction === CallDirection.Outbound) { + newCall.queueGotCallFeedsForAnswer([]); + } else { + logger.debug("Handing local stream to new call"); + newCall.queueGotCallFeedsForAnswer(this.getLocalFeeds().map(feed => feed.clone())); + } + } + this.successor = newCall; + this.emit(CallEvent.Replaced, newCall); + this.hangup(CallErrorCode.Replaced, true); + } + + /** + * Hangup a call. + * @param {string} reason The reason why the call is being hung up. + * @param {boolean} suppressEvent True to suppress emitting an event. + */ + public hangup(reason: CallErrorCode, suppressEvent: boolean): void { + if (this.callHasEnded()) return; + + logger.debug("Ending call " + this.callId); + this.terminate(CallParty.Local, reason, !suppressEvent); + // We don't want to send hangup here if we didn't even get to sending an invite + if (this.state === CallState.WaitLocalMedia) return; + const content = {}; + // Don't send UserHangup reason to older clients + if ((this.opponentVersion && this.opponentVersion >= 1) || reason !== CallErrorCode.UserHangup) { + content["reason"] = reason; + } + this.sendVoipEvent(EventType.CallHangup, content); + } + + /** + * Reject a call + * This used to be done by calling hangup, but is a separate method and protocol + * event as of MSC2746. + */ + public reject(): void { + if (this.state !== CallState.Ringing) { + throw Error("Call must be in 'ringing' state to reject!"); + } + + if (this.opponentVersion < 1) { + logger.info( + `Opponent version is less than 1 (${this.opponentVersion}): sending hangup instead of reject`, + ); + this.hangup(CallErrorCode.UserHangup, true); + return; + } + + logger.debug("Rejecting call: " + this.callId); + this.terminate(CallParty.Local, CallErrorCode.UserHangup, true); + this.sendVoipEvent(EventType.CallReject, {}); + } + + /** + * Adds an audio and/or video track - upgrades the call + * @param {boolean} audio should add an audio track + * @param {boolean} video should add an video track + */ + private async upgradeCall( + audio: boolean, video: boolean, + ): Promise { + // We don't do call downgrades + if (!audio && !video) return; + if (!this.opponentSupportsSDPStreamMetadata()) return; + + try { + const upgradeAudio = audio && !this.hasLocalUserMediaAudioTrack; + const upgradeVideo = video && !this.hasLocalUserMediaVideoTrack; + logger.debug(`Upgrading call: audio?=${upgradeAudio} video?=${upgradeVideo}`); + + const stream = await this.client.getMediaHandler().getUserMediaStream(upgradeAudio, upgradeVideo); + if (upgradeAudio && upgradeVideo) { + if (this.hasLocalUserMediaAudioTrack) return; + if (this.hasLocalUserMediaVideoTrack) return; + + this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); + } else if (upgradeAudio) { + if (this.hasLocalUserMediaAudioTrack) return; + + const audioTrack = stream.getAudioTracks()[0]; + this.localUsermediaStream.addTrack(audioTrack); + this.peerConn.addTrack(audioTrack, this.localUsermediaStream); + } else if (upgradeVideo) { + if (this.hasLocalUserMediaVideoTrack) return; + + const videoTrack = stream.getVideoTracks()[0]; + this.localUsermediaStream.addTrack(videoTrack); + this.peerConn.addTrack(videoTrack, this.localUsermediaStream); + } + } catch (error) { + logger.error("Failed to upgrade the call", error); + this.emit(CallEvent.Error, + new CallError(CallErrorCode.NoUserMedia, "Failed to get camera access: ", error), + ); + } + } + + /** + * Returns true if this.remoteSDPStreamMetadata is defined, otherwise returns false + * @returns {boolean} can screenshare + */ + public opponentSupportsSDPStreamMetadata(): boolean { + return Boolean(this.remoteSDPStreamMetadata); + } + + /** + * If there is a screensharing stream returns true, otherwise returns false + * @returns {boolean} is screensharing + */ + public isScreensharing(): boolean { + return Boolean(this.localScreensharingStream); + } + + /** + * Starts/stops screensharing + * @param enabled the desired screensharing state + * @param {string} desktopCapturerSourceId optional id of the desktop capturer source to use + * @returns {boolean} new screensharing state + */ + public async setScreensharingEnabled( + enabled: boolean, desktopCapturerSourceId?: string, + ): Promise { + // Skip if there is nothing to do + if (enabled && this.isScreensharing()) { + logger.warn(`There is already a screensharing stream - there is nothing to do!`); + return true; + } else if (!enabled && !this.isScreensharing()) { + logger.warn(`There already isn't a screensharing stream - there is nothing to do!`); + return false; + } + + // Fallback to replaceTrack() + if (!this.opponentSupportsSDPStreamMetadata()) { + return await this.setScreensharingEnabledWithoutMetadataSupport(enabled, desktopCapturerSourceId); + } + + logger.debug(`Set screensharing enabled? ${enabled}`); + if (enabled) { + try { + const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId); + if (!stream) return false; + this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare); + return true; + } catch (err) { + logger.error("Failed to get screen-sharing stream:", err); + return false; + } + } else { + for (const sender of this.screensharingSenders) { + this.peerConn.removeTrack(sender); + } + this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream); + this.deleteFeedByStream(this.localScreensharingStream); + return false; + } + } + + /** + * Starts/stops screensharing + * Should be used ONLY if the opponent doesn't support SDPStreamMetadata + * @param enabled the desired screensharing state + * @param {string} desktopCapturerSourceId optional id of the desktop capturer source to use + * @returns {boolean} new screensharing state + */ + private async setScreensharingEnabledWithoutMetadataSupport( + enabled: boolean, desktopCapturerSourceId?: string, + ): Promise { + logger.debug(`Set screensharing enabled? ${enabled} using replaceTrack()`); + if (enabled) { + try { + const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId); + if (!stream) return false; + + const track = stream.getTracks().find((track) => { + return track.kind === "video"; + }); + const sender = this.usermediaSenders.find((sender) => { + return sender.track?.kind === "video"; + }); + sender.replaceTrack(track); + + this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare, false); + + return true; + } catch (err) { + logger.error("Failed to get screen-sharing stream:", err); + return false; + } + } else { + const track = this.localUsermediaStream.getTracks().find((track) => { + return track.kind === "video"; + }); + const sender = this.usermediaSenders.find((sender) => { + return sender.track?.kind === "video"; + }); + sender.replaceTrack(track); + + this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream); + this.deleteFeedByStream(this.localScreensharingStream); + + return false; + } + } + + /** + * Request a new local usermedia stream with the current device id. + */ + public async updateLocalUsermediaStream(stream: MediaStream) { + const callFeed = this.localUsermediaFeed; + callFeed.setNewStream(stream); + const micShouldBeMuted = callFeed.isAudioMuted() || this.remoteOnHold; + const vidShouldBeMuted = callFeed.isVideoMuted() || this.remoteOnHold; + setTracksEnabled(stream.getAudioTracks(), !micShouldBeMuted); + setTracksEnabled(stream.getVideoTracks(), !vidShouldBeMuted); + + const newSenders = []; + + for (const track of stream.getTracks()) { + const oldSender = this.usermediaSenders.find((sender) => { + return sender.track?.kind === track.kind; + }); + + let newSender: RTCRtpSender; + + try { + logger.info( + `Replacing track (` + + `id="${track.id}", ` + + `kind="${track.kind}", ` + + `streamId="${stream.id}", ` + + `streamPurpose="${callFeed.purpose}"` + + `) to peer connection`, + ); + await oldSender.replaceTrack(track); + newSender = oldSender; + } catch (error) { + logger.info( + `Adding track (` + + `id="${track.id}", ` + + `kind="${track.kind}", ` + + `streamId="${stream.id}", ` + + `streamPurpose="${callFeed.purpose}"` + + `) to peer connection`, + ); + newSender = this.peerConn.addTrack(track, stream); + } + + newSenders.push(newSender); + } + + this.usermediaSenders = newSenders; + } + + /** + * Set whether our outbound video should be muted or not. + * @param {boolean} muted True to mute the outbound video. + * @returns the new mute state + */ + public async setLocalVideoMuted(muted: boolean): Promise { + if (!await this.client.getMediaHandler().hasVideoDevice()) { + return this.isLocalVideoMuted(); + } + + if (!this.hasLocalUserMediaVideoTrack && !muted) { + await this.upgradeCall(false, true); + return this.isLocalVideoMuted(); + } + this.localUsermediaFeed?.setVideoMuted(muted); + this.updateMuteStatus(); + return this.isLocalVideoMuted(); + } + + /** + * Check if local video is muted. + * + * If there are multiple video tracks, all of the tracks need to be muted + * for this to return true. This means if there are no video tracks, this will + * return true. + * @return {Boolean} True if the local preview video is muted, else false + * (including if the call is not set up yet). + */ + public isLocalVideoMuted(): boolean { + return this.localUsermediaFeed?.isVideoMuted(); + } + + /** + * Set whether the microphone should be muted or not. + * @param {boolean} muted True to mute the mic. + * @returns the new mute state + */ + public async setMicrophoneMuted(muted: boolean): Promise { + if (!await this.client.getMediaHandler().hasAudioDevice()) { + return this.isMicrophoneMuted(); + } + + if (!this.hasLocalUserMediaAudioTrack && !muted) { + await this.upgradeCall(true, false); + return this.isMicrophoneMuted(); + } + this.localUsermediaFeed?.setAudioMuted(muted); + this.updateMuteStatus(); + return this.isMicrophoneMuted(); + } + + /** + * Check if the microphone is muted. + * + * If there are multiple audio tracks, all of the tracks need to be muted + * for this to return true. This means if there are no audio tracks, this will + * return true. + * @return {Boolean} True if the mic is muted, else false (including if the call + * is not set up yet). + */ + public isMicrophoneMuted(): boolean { + return this.localUsermediaFeed?.isAudioMuted(); + } + + /** + * @returns true if we have put the party on the other side of the call on hold + * (that is, we are signalling to them that we are not listening) + */ + public isRemoteOnHold(): boolean { + return this.remoteOnHold; + } + + public setRemoteOnHold(onHold: boolean): void { + if (this.isRemoteOnHold() === onHold) return; + this.remoteOnHold = onHold; + + for (const transceiver of this.peerConn.getTransceivers()) { + // We don't send hold music or anything so we're not actually + // sending anything, but sendrecv is fairly standard for hold and + // it makes it a lot easier to figure out who's put who on hold. + transceiver.direction = onHold ? 'sendonly' : 'sendrecv'; + } + this.updateMuteStatus(); + + this.emit(CallEvent.RemoteHoldUnhold, this.remoteOnHold); + } + + /** + * Indicates whether we are 'on hold' to the remote party (ie. if true, + * they cannot hear us). + * @returns true if the other party has put us on hold + */ + public isLocalOnHold(): boolean { + if (this.state !== CallState.Connected) return false; + + let callOnHold = true; + + // We consider a call to be on hold only if *all* the tracks are on hold + // (is this the right thing to do?) + for (const transceiver of this.peerConn.getTransceivers()) { + const trackOnHold = ['inactive', 'recvonly'].includes(transceiver.currentDirection); + + if (!trackOnHold) callOnHold = false; + } + + return callOnHold; + } + + /** + * Sends a DTMF digit to the other party + * @param digit The digit (nb. string - '#' and '*' are dtmf too) + */ + public sendDtmfDigit(digit: string): void { + for (const sender of this.peerConn.getSenders()) { + if (sender.track.kind === 'audio' && sender.dtmf) { + sender.dtmf.insertDTMF(digit); + return; + } + } + + throw new Error("Unable to find a track to send DTMF on"); + } + + private updateMuteStatus(): void { + this.sendVoipEvent(EventType.CallSDPStreamMetadataChangedPrefix, { + [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(), + }); + + const micShouldBeMuted = this.localUsermediaFeed?.isAudioMuted() || this.remoteOnHold; + const vidShouldBeMuted = this.localUsermediaFeed?.isVideoMuted() || this.remoteOnHold; + + setTracksEnabled(this.localUsermediaStream.getAudioTracks(), !micShouldBeMuted); + setTracksEnabled(this.localUsermediaStream.getVideoTracks(), !vidShouldBeMuted); + } + + private gotCallFeedsForInvite(callFeeds: CallFeed[], requestScreenshareFeed = false): void { + if (this.successor) { + this.successor.queueGotCallFeedsForAnswer(callFeeds); + return; + } + if (this.callHasEnded()) { + this.stopAllMedia(); + return; + } + + for (const feed of callFeeds) { + this.pushLocalFeed(feed); + } + + if (requestScreenshareFeed) { + this.peerConn.addTransceiver("video", { + direction: "recvonly", + }); + } + + this.setState(CallState.CreateOffer); + + logger.debug("gotUserMediaForInvite"); + // Now we wait for the negotiationneeded event + } + + private async sendAnswer(): Promise { + const answerContent = { + answer: { + sdp: this.peerConn.localDescription.sdp, + // type is now deprecated as of Matrix VoIP v1, but + // required to still be sent for backwards compat + type: this.peerConn.localDescription.type, + }, + [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(true), + } as MCallAnswer; + + answerContent.capabilities = { + 'm.call.transferee': this.client.supportsCallTransfer, + 'm.call.dtmf': false, + }; + + // We have just taken the local description from the peerConn which will + // contain all the local candidates added so far, so we can discard any candidates + // we had queued up because they'll be in the answer. + logger.info(`Discarding ${this.candidateSendQueue.length} candidates that will be sent in answer`); + this.candidateSendQueue = []; + + try { + await this.sendVoipEvent(EventType.CallAnswer, answerContent); + // If this isn't the first time we've tried to send the answer, + // we may have candidates queued up, so send them now. + this.inviteOrAnswerSent = true; + } catch (error) { + // We've failed to answer: back to the ringing state + this.setState(CallState.Ringing); + this.client.cancelPendingEvent(error.event); + + let code = CallErrorCode.SendAnswer; + let message = "Failed to send answer"; + if (error.name == 'UnknownDeviceError') { + code = CallErrorCode.UnknownDevices; + message = "Unknown devices present in the room"; + } + this.emit(CallEvent.Error, new CallError(code, message, error)); + throw error; + } + + // error handler re-throws so this won't happen on error, but + // we don't want the same error handling on the candidate queue + this.sendCandidateQueue(); + } + + private queueGotCallFeedsForAnswer(callFeeds: CallFeed[]): void { + // Ensure only one negotiate/answer event is being processed at a time. + if (this.responsePromiseChain) { + this.responsePromiseChain = + this.responsePromiseChain.then(() => this.gotCallFeedsForAnswer(callFeeds)); + } else { + this.responsePromiseChain = this.gotCallFeedsForAnswer(callFeeds); + } + } + + private async gotCallFeedsForAnswer(callFeeds: CallFeed[]): Promise { + if (this.callHasEnded()) return; + + this.waitForLocalAVStream = false; + + for (const feed of callFeeds) { + this.pushLocalFeed(feed); + } + + this.setState(CallState.CreateAnswer); + + let myAnswer; + try { + this.getRidOfRTXCodecs(); + myAnswer = await this.peerConn.createAnswer(); + } catch (err) { + logger.debug("Failed to create answer: ", err); + this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true); + return; + } + + try { + await this.peerConn.setLocalDescription(myAnswer); + this.setState(CallState.Connecting); + + // Allow a short time for initial candidates to be gathered + await new Promise(resolve => { + setTimeout(resolve, 200); + }); + + this.sendAnswer(); + } catch (err) { + logger.debug("Error setting local description!", err); + this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); + return; + } + + // HACK: Safari doesn't like it when we reuse MediaStreams. In most cases + // we can get around this by calling MediaStream.clone(), however inbound + // calls seem to still be broken unless we getUserMedia again and replace + // all MediaStreams using sender.replaceTrack + if (isSafari) { + await new Promise(resolve => { + setTimeout(resolve, 200); + }); + + if (this.state === CallState.Ended) { + return; + } + + const callFeed = this.localUsermediaFeed; + const stream = callFeed.stream; + + if (!stream.active) { + throw new Error(`Call ${this.callId} has an inactive stream ${ + stream.id} and its tracks cannot be replaced`); + } + + const newSenders = []; + + for (const track of this.localUsermediaStream.getTracks()) { + const oldSender = this.usermediaSenders.find((sender) => { + return sender.track?.kind === track.kind; + }); + + if (track.readyState === "ended") { + throw new Error(`Call ${this.callId} tried to replace track ${track.id} in the ended state`); + } + + let newSender: RTCRtpSender; + + try { + logger.info( + `Replacing track (` + + `id="${track.id}", ` + + `kind="${track.kind}", ` + + `streamId="${stream.id}", ` + + `streamPurpose="${callFeed.purpose}"` + + `) to peer connection`, + ); + await oldSender.replaceTrack(track); + newSender = oldSender; + } catch (error) { + logger.info( + `Adding track (` + + `id="${track.id}", ` + + `kind="${track.kind}", ` + + `streamId="${stream.id}", ` + + `streamPurpose="${callFeed.purpose}"` + + `) to peer connection`, + ); + newSender = this.peerConn.addTrack(track, stream); + } + + newSenders.push(newSender); + } + + this.usermediaSenders = newSenders; + } + } + + /** + * Internal + * @param {Object} event + */ + private gotLocalIceCandidate = (event: RTCPeerConnectionIceEvent): Promise => { + if (event.candidate) { + logger.debug( + "Call " + this.callId + " got local ICE " + event.candidate.sdpMid + " candidate: " + + event.candidate.candidate, + ); + + if (this.callHasEnded()) return; + + // As with the offer, note we need to make a copy of this object, not + // pass the original: that broke in Chrome ~m43. + if (event.candidate.candidate !== '' || !this.sentEndOfCandidates) { + this.queueCandidate(event.candidate); + + if (event.candidate.candidate === '') this.sentEndOfCandidates = true; + } + } + }; + + private onIceGatheringStateChange = (event: Event): void => { + logger.debug("ice gathering state changed to " + this.peerConn.iceGatheringState); + if (this.peerConn.iceGatheringState === 'complete' && !this.sentEndOfCandidates) { + // If we didn't get an empty-string candidate to signal the end of candidates, + // create one ourselves now gathering has finished. + // We cast because the interface lists all the properties as required but we + // only want to send 'candidate' + // XXX: We probably want to send either sdpMid or sdpMLineIndex, as it's not strictly + // correct to have a candidate that lacks both of these. We'd have to figure out what + // previous candidates had been sent with and copy them. + const c = { + candidate: '', + } as RTCIceCandidate; + this.queueCandidate(c); + this.sentEndOfCandidates = true; + } + }; + + public async onRemoteIceCandidatesReceived(ev: MatrixEvent): Promise { + if (this.callHasEnded()) { + //debuglog("Ignoring remote ICE candidate because call has ended"); + return; + } + + const content = ev.getContent(); + const candidates = content.candidates; + if (!candidates) { + logger.info("Ignoring candidates event with no candidates!"); + return; + } + + const fromPartyId = content.version === 0 ? null : content.party_id || null; + + if (this.opponentPartyId === undefined) { + // we haven't picked an opponent yet so save the candidates + logger.info(`Buffering ${candidates.length} candidates until we pick an opponent`); + const bufferedCandidates = this.remoteCandidateBuffer.get(fromPartyId) || []; + bufferedCandidates.push(...candidates); + this.remoteCandidateBuffer.set(fromPartyId, bufferedCandidates); + return; + } + + if (!this.partyIdMatches(content)) { + logger.info( + `Ignoring candidates from party ID ${content.party_id}: ` + + `we have chosen party ID ${this.opponentPartyId}`, + ); + + return; + } + + await this.addIceCandidates(candidates); + } + + /** + * Used by MatrixClient. + * @param {Object} msg + */ + public async onAnswerReceived(event: MatrixEvent): Promise { + const content = event.getContent(); + logger.debug(`Got answer for call ID ${this.callId} from party ID ${content.party_id}`); + + if (this.callHasEnded()) { + logger.debug(`Ignoring answer because call ID ${this.callId} has ended`); + return; + } + + if (this.opponentPartyId !== undefined) { + logger.info( + `Ignoring answer from party ID ${content.party_id}: ` + + `we already have an answer/reject from ${this.opponentPartyId}`, + ); + return; + } + + this.chooseOpponent(event); + await this.addBufferedIceCandidates(); + + this.setState(CallState.Connecting); + + const sdpStreamMetadata = content[SDPStreamMetadataKey]; + if (sdpStreamMetadata) { + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); + } else { + logger.warn("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); + } + + try { + await this.peerConn.setRemoteDescription(content.answer); + } catch (e) { + logger.debug("Failed to set remote description", e); + this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); + return; + } + + // If the answer we selected has a party_id, send a select_answer event + // We do this after setting the remote description since otherwise we'd block + // call setup on it + if (this.opponentPartyId !== null) { + try { + await this.sendVoipEvent(EventType.CallSelectAnswer, { + selected_party_id: this.opponentPartyId, + }); + } catch (err) { + // This isn't fatal, and will just mean that if another party has raced to answer + // the call, they won't know they got rejected, so we carry on & don't retry. + logger.warn("Failed to send select_answer event", err); + } + } + } + + public async onSelectAnswerReceived(event: MatrixEvent): Promise { + if (this.direction !== CallDirection.Inbound) { + logger.warn("Got select_answer for an outbound call: ignoring"); + return; + } + + const selectedPartyId = event.getContent().selected_party_id; + + if (selectedPartyId === undefined || selectedPartyId === null) { + logger.warn("Got nonsensical select_answer with null/undefined selected_party_id: ignoring"); + return; + } + + if (selectedPartyId !== this.ourPartyId) { + logger.info(`Got select_answer for party ID ${selectedPartyId}: we are party ID ${this.ourPartyId}.`); + // The other party has picked somebody else's answer + this.terminate(CallParty.Remote, CallErrorCode.AnsweredElsewhere, true); + } + } + + public async onNegotiateReceived(event: MatrixEvent): Promise { + const content = event.getContent(); + const description = content.description; + if (!description || !description.sdp || !description.type) { + logger.info("Ignoring invalid m.call.negotiate event"); + return; + } + // Politeness always follows the direction of the call: in a glare situation, + // we pick either the inbound or outbound call, so one side will always be + // inbound and one outbound + const polite = this.direction === CallDirection.Inbound; + + // Here we follow the perfect negotiation logic from + // https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation + const offerCollision = ( + (description.type === 'offer') && + (this.makingOffer || this.peerConn.signalingState !== 'stable') + ); + + this.ignoreOffer = !polite && offerCollision; + if (this.ignoreOffer) { + logger.info("Ignoring colliding negotiate event because we're impolite"); + return; + } + + const prevLocalOnHold = this.isLocalOnHold(); + + const sdpStreamMetadata = content[SDPStreamMetadataKey]; + if (sdpStreamMetadata) { + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); + } else { + logger.warn("Received negotiation event without SDPStreamMetadata!"); + } + + try { + await this.peerConn.setRemoteDescription(description); + + if (description.type === 'offer') { + this.getRidOfRTXCodecs(); + await this.peerConn.setLocalDescription(); + + this.sendVoipEvent(EventType.CallNegotiate, { + description: this.peerConn.localDescription, + [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(true), + }); + } + } catch (err) { + logger.warn("Failed to complete negotiation", err); + } + + const newLocalOnHold = this.isLocalOnHold(); + if (prevLocalOnHold !== newLocalOnHold) { + this.emit(CallEvent.LocalHoldUnhold, newLocalOnHold); + // also this one for backwards compat + this.emit(CallEvent.HoldUnhold, newLocalOnHold); + } + } + + private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { + this.remoteSDPStreamMetadata = utils.recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); + for (const feed of this.getRemoteFeeds()) { + const streamId = feed.stream.id; + feed.setAudioMuted(this.remoteSDPStreamMetadata[streamId]?.audio_muted); + feed.setVideoMuted(this.remoteSDPStreamMetadata[streamId]?.video_muted); + feed.purpose = this.remoteSDPStreamMetadata[streamId]?.purpose; + } + } + + public onSDPStreamMetadataChangedReceived(event: MatrixEvent): void { + const content = event.getContent(); + const metadata = content[SDPStreamMetadataKey]; + this.updateRemoteSDPStreamMetadata(metadata); + } + + public async onAssertedIdentityReceived(event: MatrixEvent): Promise { + const content = event.getContent(); + if (!content.asserted_identity) return; + + this.remoteAssertedIdentity = { + id: content.asserted_identity.id, + displayName: content.asserted_identity.display_name, + }; + this.emit(CallEvent.AssertedIdentityChanged); + } + + private callHasEnded(): boolean { + // This exists as workaround to typescript trying to be clever and erroring + // when putting if (this.state === CallState.Ended) return; twice in the same + // function, even though that function is async. + return this.state === CallState.Ended; + } + + private queueGotLocalOffer(): void { + // Ensure only one negotiate/answer event is being processed at a time. + if (this.responsePromiseChain) { + this.responsePromiseChain = + this.responsePromiseChain.then(() => this.wrappedGotLocalOffer()); + } else { + this.responsePromiseChain = this.wrappedGotLocalOffer(); + } + } + + private async wrappedGotLocalOffer(): Promise { + this.makingOffer = true; + try { + this.getRidOfRTXCodecs(); + await this.gotLocalOffer(); + } catch (e) { + this.getLocalOfferFailed(e); + return; + } finally { + this.makingOffer = false; + } + } + + private async gotLocalOffer(): Promise { + logger.debug("Setting local description"); + + if (this.callHasEnded()) { + logger.debug("Ignoring newly created offer on call ID " + this.callId + + " because the call has ended"); + return; + } + + try { + await this.peerConn.setLocalDescription(); + } catch (err) { + logger.debug("Error setting local description!", err); + this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); + return; + } + + if (this.peerConn.iceGatheringState === 'gathering') { + // Allow a short time for initial candidates to be gathered + await new Promise(resolve => { + setTimeout(resolve, 200); + }); + } + + if (this.callHasEnded()) return; + + const eventType = this.state === CallState.CreateOffer ? EventType.CallInvite : EventType.CallNegotiate; + + const content = { + lifetime: CALL_TIMEOUT_MS, + } as MCallInviteNegotiate; + + if (eventType === EventType.CallInvite && this.invitee) { + content.invitee = this.invitee; + } + + // clunky because TypeScript can't follow the types through if we use an expression as the key + if (this.state === CallState.CreateOffer) { + content.offer = this.peerConn.localDescription; + } else { + content.description = this.peerConn.localDescription; + } + + content.capabilities = { + 'm.call.transferee': this.client.supportsCallTransfer, + 'm.call.dtmf': false, + }; + + content[SDPStreamMetadataKey] = this.getLocalSDPStreamMetadata(true); + + // Get rid of any candidates waiting to be sent: they'll be included in the local + // description we just got and will send in the offer. + logger.info(`Discarding ${this.candidateSendQueue.length} candidates that will be sent in offer`); + this.candidateSendQueue = []; + + try { + await this.sendVoipEvent(eventType, content); + } catch (error) { + logger.error("Failed to send invite", error); + if (error.event) this.client.cancelPendingEvent(error.event); + + let code = CallErrorCode.SignallingFailed; + let message = "Signalling failed"; + if (this.state === CallState.CreateOffer) { + code = CallErrorCode.SendInvite; + message = "Failed to send invite"; + } + if (error.name == 'UnknownDeviceError') { + code = CallErrorCode.UnknownDevices; + message = "Unknown devices present in the room"; + } + + this.emit(CallEvent.Error, new CallError(code, message, error)); + this.terminate(CallParty.Local, code, false); + + // no need to carry on & send the candidate queue, but we also + // don't want to rethrow the error + return; + } + + this.sendCandidateQueue(); + if (this.state === CallState.CreateOffer) { + this.inviteOrAnswerSent = true; + this.setState(CallState.InviteSent); + this.inviteTimeout = setTimeout(() => { + this.inviteTimeout = null; + if (this.state === CallState.InviteSent) { + this.hangup(CallErrorCode.InviteTimeout, false); + } + }, CALL_TIMEOUT_MS); + } + } + + private getLocalOfferFailed = (err: Error): void => { + logger.error("Failed to get local offer", err); + + this.emit( + CallEvent.Error, + new CallError( + CallErrorCode.LocalOfferFailed, + "Failed to get local offer!", err, + ), + ); + this.terminate(CallParty.Local, CallErrorCode.LocalOfferFailed, false); + }; + + private getUserMediaFailed = (err: Error): void => { + if (this.successor) { + this.successor.getUserMediaFailed(err); + return; + } + + logger.warn("Failed to get user media - ending call", err); + + this.emit( + CallEvent.Error, + new CallError( + CallErrorCode.NoUserMedia, + "Couldn't start capturing media! Is your microphone set up and " + + "does this app have permission?", err, + ), + ); + this.terminate(CallParty.Local, CallErrorCode.NoUserMedia, false); + }; + + private onIceConnectionStateChanged = (): void => { + if (this.callHasEnded()) { + return; // because ICE can still complete as we're ending the call + } + logger.debug( + "Call ID " + this.callId + ": ICE connection state changed to: " + this.peerConn.iceConnectionState, + ); + // ideally we'd consider the call to be connected when we get media but + // chrome doesn't implement any of the 'onstarted' events yet + if (this.peerConn.iceConnectionState == 'connected') { + clearTimeout(this.iceDisconnectedTimeout); + this.setState(CallState.Connected); + + if (!this.callLengthInterval) { + this.callLengthInterval = setInterval(() => { + this.callLength++; + this.emit(CallEvent.LengthChanged, this.callLength); + }, 1000); + } + } else if (this.peerConn.iceConnectionState == 'failed') { + // Firefox for Android does not yet have support for restartIce() + if (this.peerConn.restartIce) { + this.peerConn.restartIce(); + } else { + this.hangup(CallErrorCode.IceFailed, false); + } + } else if (this.peerConn.iceConnectionState == 'disconnected') { + this.iceDisconnectedTimeout = setTimeout(() => { + this.hangup(CallErrorCode.IceFailed, false); + }, 30 * 1000); + } + }; + + private onSignallingStateChanged = (): void => { + logger.debug( + "call " + this.callId + ": Signalling state changed to: " + + this.peerConn.signalingState, + ); + }; + + private onTrack = (ev: RTCTrackEvent): void => { + if (ev.streams.length === 0) { + logger.warn(`Streamless ${ev.track.kind} found: ignoring.`); + return; + } + + const stream = ev.streams[0]; + this.pushRemoteFeed(stream); + stream.addEventListener("removetrack", () => { + logger.log(`Removing track streamId: ${stream.id}`); + this.deleteFeedByStream(stream); + }); + }; + + private onDataChannel = (ev: RTCDataChannelEvent): void => { + this.emit(CallEvent.DataChannel, ev.channel); + }; + + /** + * This method removes all video/rtx codecs from screensharing video + * transceivers. This is necessary since they can cause problems. Without + * this the following steps should produce an error: + * Chromium calls Firefox + * Firefox answers + * Firefox starts screen-sharing + * Chromium starts screen-sharing + * Call crashes for Chromium with: + * [96685:23:0518/162603.933321:ERROR:webrtc_video_engine.cc(3296)] RTX codec (PT=97) mapped to PT=96 which is not in the codec list. + * [96685:23:0518/162603.933377:ERROR:webrtc_video_engine.cc(1171)] GetChangedRecvParameters called without any video codecs. + * [96685:23:0518/162603.933430:ERROR:sdp_offer_answer.cc(4302)] Failed to set local video description recv parameters for m-section with mid='2'. (INVALID_PARAMETER) + */ + private getRidOfRTXCodecs(): void { + // RTCRtpReceiver.getCapabilities and RTCRtpSender.getCapabilities don't seem to be supported on FF + if (!RTCRtpReceiver.getCapabilities || !RTCRtpSender.getCapabilities) return; + + const recvCodecs = RTCRtpReceiver.getCapabilities("video").codecs; + const sendCodecs = RTCRtpSender.getCapabilities("video").codecs; + const codecs = [...sendCodecs, ...recvCodecs]; + + for (const codec of codecs) { + if (codec.mimeType === "video/rtx") { + const rtxCodecIndex = codecs.indexOf(codec); + codecs.splice(rtxCodecIndex, 1); + } + } + + for (const trans of this.peerConn.getTransceivers()) { + if ( + this.screensharingSenders.includes(trans.sender) && + ( + trans.sender.track?.kind === "video" || + trans.receiver.track?.kind === "video" + ) + ) { + trans.setCodecPreferences(codecs); + } + } + } + + private onNegotiationNeeded = async (): Promise => { + logger.info("Negotiation is needed!"); + + if (this.state !== CallState.CreateOffer && this.opponentVersion === 0) { + logger.info("Opponent does not support renegotiation: ignoring negotiationneeded event"); + return; + } + + this.queueGotLocalOffer(); + }; + + public onHangupReceived = (msg: MCallHangupReject): void => { + logger.debug("Hangup received for call ID " + this.callId); + + // party ID must match (our chosen partner hanging up the call) or be undefined (we haven't chosen + // a partner yet but we're treating the hangup as a reject as per VoIP v0) + if (this.partyIdMatches(msg) || this.state === CallState.Ringing) { + // default reason is user_hangup + this.terminate(CallParty.Remote, msg.reason || CallErrorCode.UserHangup, true); + } else { + logger.info(`Ignoring message from party ID ${msg.party_id}: our partner is ${this.opponentPartyId}`); + } + }; + + public onRejectReceived = (msg: MCallHangupReject): void => { + logger.debug("Reject received for call ID " + this.callId); + + // No need to check party_id for reject because if we'd received either + // an answer or reject, we wouldn't be in state InviteSent + + const shouldTerminate = ( + // reject events also end the call if it's ringing: it's another of + // our devices rejecting the call. + ([CallState.InviteSent, CallState.Ringing].includes(this.state)) || + // also if we're in the init state and it's an inbound call, since + // this means we just haven't entered the ringing state yet + this.state === CallState.Fledgling && this.direction === CallDirection.Inbound + ); + + if (shouldTerminate) { + this.terminate(CallParty.Remote, msg.reason || CallErrorCode.UserHangup, true); + } else { + logger.debug(`Call is in state: ${this.state}: ignoring reject`); + } + }; + + public onAnsweredElsewhere = (msg: MCallAnswer): void => { + logger.debug("Call ID " + this.callId + " answered elsewhere"); + this.terminate(CallParty.Remote, CallErrorCode.AnsweredElsewhere, true); + }; + + private setState(state: CallState): void { + const oldState = this.state; + this.state = state; + this.emit(CallEvent.State, state, oldState); + } + + /** + * Internal + * @param {string} eventType + * @param {Object} content + * @return {Promise} + */ + private sendVoipEvent(eventType: string, content: object): Promise { + const realContent = Object.assign({}, content, { + version: VOIP_PROTO_VERSION, + call_id: this.callId, + party_id: this.ourPartyId, + conf_id: this.groupCallId, + }); + + if (this.opponentDeviceId) { + this.emit(CallEvent.SendVoipEvent, { + type: "toDevice", + eventType, + userId: this.invitee || this.getOpponentMember().userId, + opponentDeviceId: this.opponentDeviceId, + content: { + ...realContent, + device_id: this.client.deviceId, + sender_session_id: this.client.getSessionId(), + dest_session_id: this.opponentSessionId, + }, + }); + + return this.client.sendToDevice(eventType, { + [this.invitee || this.getOpponentMember().userId]: { + [this.opponentDeviceId]: { + ...realContent, + device_id: this.client.deviceId, + sender_session_id: this.client.getSessionId(), + dest_session_id: this.opponentSessionId, + }, + }, + }); + } else { + this.emit(CallEvent.SendVoipEvent, { + type: "sendEvent", + eventType, + roomId: this.roomId, + content: realContent, + userId: this.invitee || this.getOpponentMember().userId, + }); + + return this.client.sendEvent(this.roomId, eventType, realContent); + } + } + + private queueCandidate(content: RTCIceCandidate): void { + // We partially de-trickle candidates by waiting for `delay` before sending them + // amalgamated, in order to avoid sending too many m.call.candidates events and hitting + // rate limits in Matrix. + // In practice, it'd be better to remove rate limits for m.call.* + + // N.B. this deliberately lets you queue and send blank candidates, which MSC2746 + // currently proposes as the way to indicate that candidate gathering is complete. + // This will hopefully be changed to an explicit rather than implicit notification + // shortly. + this.candidateSendQueue.push(content); + + // Don't send the ICE candidates yet if the call is in the ringing state: this + // means we tried to pick (ie. started generating candidates) and then failed to + // send the answer and went back to the ringing state. Queue up the candidates + // to send if we successfully send the answer. + // Equally don't send if we haven't yet sent the answer because we can send the + // first batch of candidates along with the answer + if (this.state === CallState.Ringing || !this.inviteOrAnswerSent) return; + + // MSC2746 recommends these values (can be quite long when calling because the + // callee will need a while to answer the call) + const delay = this.direction === CallDirection.Inbound ? 500 : 2000; + + if (this.candidateSendTries === 0) { + setTimeout(() => { + this.sendCandidateQueue(); + }, delay); + } + } + + /* + * Transfers this call to another user + */ + public async transfer(targetUserId: string): Promise { + // Fetch the target user's global profile info: their room avatar / displayname + // could be different in whatever room we share with them. + const profileInfo = await this.client.getProfileInfo(targetUserId); + + const replacementId = genCallID(); + + const body = { + replacement_id: genCallID(), + target_user: { + id: targetUserId, + display_name: profileInfo.displayname, + avatar_url: profileInfo.avatar_url, + }, + create_call: replacementId, + } as MCallReplacesEvent; + + await this.sendVoipEvent(EventType.CallReplaces, body); + + await this.terminate(CallParty.Local, CallErrorCode.Transfered, true); + } + + /* + * Transfers this call to the target call, effectively 'joining' the + * two calls (so the remote parties on each call are connected together). + */ + public async transferToCall(transferTargetCall?: MatrixCall): Promise { + const targetProfileInfo = await this.client.getProfileInfo(transferTargetCall.getOpponentMember().userId); + const transfereeProfileInfo = await this.client.getProfileInfo(this.getOpponentMember().userId); + + const newCallId = genCallID(); + + const bodyToTransferTarget = { + // the replacements on each side have their own ID, and it's distinct from the + // ID of the new call (but we can use the same function to generate it) + replacement_id: genCallID(), + target_user: { + id: this.getOpponentMember().userId, + display_name: transfereeProfileInfo.displayname, + avatar_url: transfereeProfileInfo.avatar_url, + }, + await_call: newCallId, + } as MCallReplacesEvent; + + await transferTargetCall.sendVoipEvent(EventType.CallReplaces, bodyToTransferTarget); + + const bodyToTransferee = { + replacement_id: genCallID(), + target_user: { + id: transferTargetCall.getOpponentMember().userId, + display_name: targetProfileInfo.displayname, + avatar_url: targetProfileInfo.avatar_url, + }, + create_call: newCallId, + } as MCallReplacesEvent; + + await this.sendVoipEvent(EventType.CallReplaces, bodyToTransferee); + + await this.terminate(CallParty.Local, CallErrorCode.Replaced, true); + await transferTargetCall.terminate(CallParty.Local, CallErrorCode.Transfered, true); + } + + private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean): Promise { + if (this.callHasEnded()) return; + + this.hangupParty = hangupParty; + this.hangupReason = hangupReason; + this.setState(CallState.Ended); + + if (this.inviteTimeout) { + clearTimeout(this.inviteTimeout); + this.inviteTimeout = null; + } + if (this.callLengthInterval) { + clearInterval(this.callLengthInterval); + this.callLengthInterval = null; + } + + this.callStatsAtEnd = await this.collectCallStats(); + + // Order is important here: first we stopAllMedia() and only then we can deleteAllFeeds() + this.stopAllMedia(); + this.deleteAllFeeds(); + + if (this.peerConn && this.peerConn.signalingState !== 'closed') { + this.peerConn.close(); + } + if (shouldEmit) { + this.emit(CallEvent.Hangup, this); + } + + this.client.callEventHandler.calls.delete(this.callId); + } + + private stopAllMedia(): void { + logger.debug(!this.groupCallId ? "Stopping all media" : "Stopping all media except local feeds" ); + for (const feed of this.feeds) { + if ( + feed.isLocal() && + feed.purpose === SDPStreamMetadataPurpose.Usermedia && + !this.groupCallId + ) { + this.client.getMediaHandler().stopUserMediaStream(feed.stream); + } else if ( + feed.isLocal() && + feed.purpose === SDPStreamMetadataPurpose.Screenshare && + !this.groupCallId + ) { + this.client.getMediaHandler().stopScreensharingStream(feed.stream); + } else if (!feed.isLocal() || !this.groupCallId) { + for (const track of feed.stream.getTracks()) { + track.stop(); + } + } + } + } + + private checkForErrorListener(): void { + if (this.listeners("error").length === 0) { + throw new Error( + "You MUST attach an error listener using call.on('error', function() {})", + ); + } + } + + private async sendCandidateQueue(): Promise { + if (this.candidateSendQueue.length === 0 || this.callHasEnded()) { + return; + } + + const candidates = this.candidateSendQueue; + this.candidateSendQueue = []; + ++this.candidateSendTries; + const content = { + candidates: candidates, + }; + logger.debug("Attempting to send " + candidates.length + " candidates"); + try { + await this.sendVoipEvent(EventType.CallCandidates, content); + // reset our retry count if we have successfully sent our candidates + // otherwise queueCandidate() will refuse to try to flush the queue + this.candidateSendTries = 0; + + // Try to send candidates again just in case we received more candidates while sending. + this.sendCandidateQueue(); + } catch (error) { + // don't retry this event: we'll send another one later as we might + // have more candidates by then. + if (error.event) this.client.cancelPendingEvent(error.event); + + // put all the candidates we failed to send back in the queue + this.candidateSendQueue.push(...candidates); + + if (this.candidateSendTries > 5) { + logger.debug( + "Failed to send candidates on attempt " + this.candidateSendTries + + ". Giving up on this call.", error, + ); + + const code = CallErrorCode.SignallingFailed; + const message = "Signalling failed"; + + this.emit(CallEvent.Error, new CallError(code, message, error)); + this.hangup(code, false); + + return; + } + + const delayMs = 500 * Math.pow(2, this.candidateSendTries); + ++this.candidateSendTries; + logger.debug("Failed to send candidates. Retrying in " + delayMs + "ms", error); + setTimeout(() => { + this.sendCandidateQueue(); + }, delayMs); + } + } + + /** + * Place a call to this room. + * @throws if you have not specified a listener for 'error' events. + * @throws if have passed audio=false. + */ + public async placeCall(audio: boolean, video: boolean): Promise { + if (!audio) { + throw new Error("You CANNOT start a call without audio"); + } + this.setState(CallState.WaitLocalMedia); + + try { + const stream = await this.client.getMediaHandler().getUserMediaStream(audio, video); + const callFeed = new CallFeed({ + client: this.client, + roomId: this.roomId, + userId: this.client.getUserId(), + stream, + purpose: SDPStreamMetadataPurpose.Usermedia, + audioMuted: stream.getAudioTracks().length === 0, + videoMuted: stream.getVideoTracks().length === 0, + }); + await this.placeCallWithCallFeeds([callFeed]); + } catch (e) { + this.getUserMediaFailed(e); + return; + } + } + + /** + * Place a call to this room with call feed. + * @param {CallFeed[]} callFeeds to use + * @throws if you have not specified a listener for 'error' events. + * @throws if have passed audio=false. + */ + public async placeCallWithCallFeeds(callFeeds: CallFeed[], requestScreenshareFeed = false): Promise { + this.checkForErrorListener(); + this.direction = CallDirection.Outbound; + + // XXX Find a better way to do this + this.client.callEventHandler.calls.set(this.callId, this); + + // make sure we have valid turn creds. Unless something's gone wrong, it should + // poll and keep the credentials valid so this should be instant. + const haveTurnCreds = await this.client.checkTurnServers(); + if (!haveTurnCreds) { + logger.warn("Failed to get TURN credentials! Proceeding with call anyway..."); + } + + // create the peer connection now so it can be gathering candidates while we get user + // media (assuming a candidate pool size is configured) + this.peerConn = this.createPeerConnection(); + this.gotCallFeedsForInvite(callFeeds, requestScreenshareFeed); + } + + private createPeerConnection(): RTCPeerConnection { + const pc = new window.RTCPeerConnection({ + iceTransportPolicy: this.forceTURN ? 'relay' : undefined, + iceServers: this.turnServers, + iceCandidatePoolSize: this.client.iceCandidatePoolSize, + }); + + // 'connectionstatechange' would be better, but firefox doesn't implement that. + pc.addEventListener('iceconnectionstatechange', this.onIceConnectionStateChanged); + pc.addEventListener('signalingstatechange', this.onSignallingStateChanged); + pc.addEventListener('icecandidate', this.gotLocalIceCandidate); + pc.addEventListener('icegatheringstatechange', this.onIceGatheringStateChange); + pc.addEventListener('track', this.onTrack); + pc.addEventListener('negotiationneeded', this.onNegotiationNeeded); + pc.addEventListener('datachannel', this.onDataChannel); + + return pc; + } + + private partyIdMatches(msg: MCallBase): boolean { + // They must either match or both be absent (in which case opponentPartyId will be null) + // Also we ignore party IDs on the invite/offer if the version is 0, so we must do the same + // here and use null if the version is 0 (woe betide any opponent sending messages in the + // same call with different versions) + const msgPartyId = msg.version === 0 ? null : msg.party_id || null; + return msgPartyId === this.opponentPartyId; + } + + // Commits to an opponent for the call + // ev: An invite or answer event + private chooseOpponent(ev: MatrixEvent): void { + // I choo-choo-choose you + const msg = ev.getContent(); + + logger.debug(`Choosing party ID ${msg.party_id} for call ID ${this.callId}`); + + this.opponentVersion = msg.version; + if (this.opponentVersion === 0) { + // set to null to indicate that we've chosen an opponent, but because + // they're v0 they have no party ID (even if they sent one, we're ignoring it) + this.opponentPartyId = null; + } else { + // set to their party ID, or if they're naughty and didn't send one despite + // not being v0, set it to null to indicate we picked an opponent with no + // party ID + this.opponentPartyId = msg.party_id || null; + } + this.opponentCaps = msg.capabilities || {} as CallCapabilities; + this.opponentMember = this.client.getRoom(this.roomId).getMember(ev.getSender()); + } + + private async addBufferedIceCandidates(): Promise { + const bufferedCandidates = this.remoteCandidateBuffer.get(this.opponentPartyId); + if (bufferedCandidates) { + logger.info(`Adding ${bufferedCandidates.length} buffered candidates for opponent ${this.opponentPartyId}`); + await this.addIceCandidates(bufferedCandidates); + } + this.remoteCandidateBuffer = null; + } + + private async addIceCandidates(candidates: RTCIceCandidate[]): Promise { + for (const candidate of candidates) { + if ( + (candidate.sdpMid === null || candidate.sdpMid === undefined) && + (candidate.sdpMLineIndex === null || candidate.sdpMLineIndex === undefined) + ) { + logger.debug("Ignoring remote ICE candidate with no sdpMid or sdpMLineIndex"); + continue; + } + logger.debug( + "Call " + this.callId + " got remote ICE " + candidate.sdpMid + " candidate: " + candidate.candidate, + ); + try { + await this.peerConn.addIceCandidate(candidate); + } catch (err) { + if (!this.ignoreOffer) { + logger.info("Failed to add remote ICE candidate", err); + } + } + } + } + + public get hasPeerConnection(): boolean { + return Boolean(this.peerConn); + } +} + +export function setTracksEnabled(tracks: Array, enabled: boolean): void { + for (let i = 0; i < tracks.length; i++) { + tracks[i].enabled = enabled; + } +} + +/** + * DEPRECATED + * Use client.createCall() + * + * Create a new Matrix call for the browser. + * @param {MatrixClient} client The client instance to use. + * @param {string} roomId The room the call is in. + * @param {Object?} options DEPRECATED optional options map. + * @param {boolean} options.forceTURN DEPRECATED whether relay through TURN should be + * forced. This option is deprecated - use opts.forceTURN when creating the matrix client + * since it's only possible to set this option on outbound calls. + * @return {MatrixCall} the call or null if the browser doesn't support calling. + */ +export function createNewMatrixCall(client: any, roomId: string, options?: CallOpts): MatrixCall { + // typeof prevents Node from erroring on an undefined reference + if (typeof(window) === 'undefined' || typeof(document) === 'undefined') { + // NB. We don't log here as apps try to create a call object as a test for + // whether calls are supported, so we shouldn't fill the logs up. + return null; + } + + // Firefox throws on so little as accessing the RTCPeerConnection when operating in + // a secure mode. There's some information at https://bugzilla.mozilla.org/show_bug.cgi?id=1542616 + // though the concern is that the browser throwing a SecurityError will brick the + // client creation process. + try { + const supported = Boolean( + window.RTCPeerConnection || window.RTCSessionDescription || + window.RTCIceCandidate || navigator.mediaDevices, + ); + if (!supported) { + // Adds a lot of noise to test runs, so disable logging there. + if (process.env.NODE_ENV !== "test") { + logger.error("WebRTC is not supported in this browser / environment"); + } + return null; + } + } catch (e) { + logger.error("Exception thrown when trying to access WebRTC", e); + return null; + } + + const optionsForceTURN = options ? options.forceTURN : false; + + const opts: CallOpts = { + client: client, + roomId: roomId, + invitee: options?.invitee, + turnServers: client.getTurnServers(), + // call level options + forceTURN: client.forceTURN || optionsForceTURN, + opponentDeviceId: options?.opponentDeviceId, + opponentSessionId: options?.opponentSessionId, + groupCallId: options?.groupCallId, + }; + const call = new MatrixCall(opts); + + client.reEmitter.reEmit(call, Object.values(CallEvent)); + + return call; +} diff --git a/src/matrix/calls/CallFeed.ts b/src/matrix/calls/CallFeed.ts new file mode 100644 index 00000000..c8cc8662 --- /dev/null +++ b/src/matrix/calls/CallFeed.ts @@ -0,0 +1,274 @@ +/* +Copyright 2021 Šimon Brandner + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { SDPStreamMetadataPurpose } from "./callEventTypes"; + +const POLLING_INTERVAL = 200; // ms +export const SPEAKING_THRESHOLD = -60; // dB +const SPEAKING_SAMPLE_COUNT = 8; // samples + +export interface ICallFeedOpts { + client: MatrixClient; + roomId: string; + userId: string; + stream: MediaStream; + purpose: SDPStreamMetadataPurpose; + audioMuted: boolean; + videoMuted: boolean; +} + +export enum CallFeedEvent { + NewStream = "new_stream", + MuteStateChanged = "mute_state_changed", + VolumeChanged = "volume_changed", + Speaking = "speaking", +} + +export class CallFeed extends EventEmitter { + public stream: MediaStream; + public sdpMetadataStreamId: string; + public userId: string; + public purpose: SDPStreamMetadataPurpose; + public speakingVolumeSamples: number[]; + + private client: MatrixClient; + private roomId: string; + private audioMuted: boolean; + private videoMuted: boolean; + private measuringVolumeActivity = false; + private audioContext: AudioContext; + private analyser: AnalyserNode; + private frequencyBinCount: Float32Array; + private speakingThreshold = SPEAKING_THRESHOLD; + private speaking = false; + private volumeLooperTimeout: number; + + constructor(opts: ICallFeedOpts) { + super(); + + this.client = opts.client; + this.roomId = opts.roomId; + this.userId = opts.userId; + this.purpose = opts.purpose; + this.audioMuted = opts.audioMuted; + this.videoMuted = opts.videoMuted; + this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); + this.sdpMetadataStreamId = opts.stream.id; + + this.updateStream(null, opts.stream); + + if (this.hasAudioTrack) { + this.initVolumeMeasuring(); + } + } + + private get hasAudioTrack(): boolean { + return this.stream.getAudioTracks().length > 0; + } + + private updateStream(oldStream: MediaStream, newStream: MediaStream): void { + if (newStream === oldStream) return; + + if (oldStream) { + oldStream.removeEventListener("addtrack", this.onAddTrack); + this.measureVolumeActivity(false); + } + if (newStream) { + this.stream = newStream; + newStream.addEventListener("addtrack", this.onAddTrack); + + if (this.hasAudioTrack) { + this.initVolumeMeasuring(); + } else { + this.measureVolumeActivity(false); + } + } + + this.emit(CallFeedEvent.NewStream, this.stream); + } + + private initVolumeMeasuring(): void { + const AudioContext = window.AudioContext || window.webkitAudioContext; + if (!this.hasAudioTrack || !AudioContext) return; + + this.audioContext = new AudioContext(); + + this.analyser = this.audioContext.createAnalyser(); + this.analyser.fftSize = 512; + this.analyser.smoothingTimeConstant = 0.1; + + const mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.stream); + mediaStreamAudioSourceNode.connect(this.analyser); + + this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount); + } + + private onAddTrack = (): void => { + this.emit(CallFeedEvent.NewStream, this.stream); + }; + + /** + * Returns callRoom member + * @returns member of the callRoom + */ + public getMember(): RoomMember { + const callRoom = this.client.getRoom(this.roomId); + return callRoom.getMember(this.userId); + } + + /** + * Returns true if CallFeed is local, otherwise returns false + * @returns {boolean} is local? + */ + public isLocal(): boolean { + return this.userId === this.client.getUserId(); + } + + /** + * Returns true if audio is muted or if there are no audio + * tracks, otherwise returns false + * @returns {boolean} is audio muted? + */ + public isAudioMuted(): boolean { + return this.stream.getAudioTracks().length === 0 || this.audioMuted; + } + + /** + * Returns true video is muted or if there are no video + * tracks, otherwise returns false + * @returns {boolean} is video muted? + */ + public isVideoMuted(): boolean { + // We assume only one video track + return this.stream.getVideoTracks().length === 0 || this.videoMuted; + } + + public isSpeaking(): boolean { + return this.speaking; + } + + /** + * Replaces the current MediaStream with a new one. + * This method should be only used by MatrixCall. + * @param newStream new stream with which to replace the current one + */ + public setNewStream(newStream: MediaStream): void { + this.updateStream(this.stream, newStream); + } + + /** + * Set feed's internal audio mute state + * @param muted is the feed's audio muted? + */ + public setAudioMuted(muted: boolean): void { + this.audioMuted = muted; + this.speakingVolumeSamples.fill(-Infinity); + this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); + } + + /** + * Set feed's internal video mute state + * @param muted is the feed's video muted? + */ + public setVideoMuted(muted: boolean): void { + this.videoMuted = muted; + this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); + } + + /** + * Starts emitting volume_changed events where the emitter value is in decibels + * @param enabled emit volume changes + */ + public measureVolumeActivity(enabled: boolean): void { + if (enabled) { + if (!this.audioContext || !this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return; + + this.measuringVolumeActivity = true; + this.volumeLooper(); + } else { + this.measuringVolumeActivity = false; + this.speakingVolumeSamples.fill(-Infinity); + this.emit(CallFeedEvent.VolumeChanged, -Infinity); + } + } + + public setSpeakingThreshold(threshold: number) { + this.speakingThreshold = threshold; + } + + private volumeLooper = () => { + if (!this.analyser) return; + + if (!this.measuringVolumeActivity) return; + + this.analyser.getFloatFrequencyData(this.frequencyBinCount); + + let maxVolume = -Infinity; + for (let i = 0; i < this.frequencyBinCount.length; i++) { + if (this.frequencyBinCount[i] > maxVolume) { + maxVolume = this.frequencyBinCount[i]; + } + } + + this.speakingVolumeSamples.shift(); + this.speakingVolumeSamples.push(maxVolume); + + this.emit(CallFeedEvent.VolumeChanged, maxVolume); + + let newSpeaking = false; + + for (let i = 0; i < this.speakingVolumeSamples.length; i++) { + const volume = this.speakingVolumeSamples[i]; + + if (volume > this.speakingThreshold) { + newSpeaking = true; + break; + } + } + + if (this.speaking !== newSpeaking) { + this.speaking = newSpeaking; + this.emit(CallFeedEvent.Speaking, this.speaking); + } + + this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL); + }; + + public clone(): CallFeed { + const mediaHandler = this.client.getMediaHandler(); + const stream = this.stream.clone(); + + if (this.purpose === SDPStreamMetadataPurpose.Usermedia) { + mediaHandler.userMediaStreams.push(stream); + } else { + mediaHandler.screensharingStreams.push(stream); + } + + return new CallFeed({ + client: this.client, + roomId: this.roomId, + userId: this.userId, + stream, + purpose: this.purpose, + audioMuted: this.audioMuted, + videoMuted: this.videoMuted, + }); + } + + public dispose(): void { + clearTimeout(this.volumeLooperTimeout); + } +} diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 55571c5a..b88c3ce2 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -34,6 +34,7 @@ const CALL_ID = "m.call_id"; const CALL_TERMINATED = "m.terminated"; export class CallHandler { + // group calls by call id public readonly groupCalls: ObservableMap = new ObservableMap(); constructor() { diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts new file mode 100644 index 00000000..cd0fbb9f --- /dev/null +++ b/src/matrix/calls/callEventTypes.ts @@ -0,0 +1,92 @@ +// allow non-camelcase as these are events type that go onto the wire +/* eslint-disable camelcase */ + +import { CallErrorCode } from "./Call"; + +// TODO: Change to "sdp_stream_metadata" when MSC3077 is merged +export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata"; + +export enum SDPStreamMetadataPurpose { + Usermedia = "m.usermedia", + Screenshare = "m.screenshare", +} + +export interface SDPStreamMetadataObject { + purpose: SDPStreamMetadataPurpose; + audio_muted: boolean; + video_muted: boolean; +} + +export interface SDPStreamMetadata { + [key: string]: SDPStreamMetadataObject; +} + +export interface CallCapabilities { + 'm.call.transferee': boolean; + 'm.call.dtmf': boolean; +} + +export interface CallReplacesTarget { + id: string; + display_name: string; + avatar_url: string; +} + +export interface MCallBase { + call_id: string; + version: string | number; + party_id?: string; + sender_session_id?: string; + dest_session_id?: string; +} + +export interface MCallAnswer extends MCallBase { + answer: RTCSessionDescription; + capabilities?: CallCapabilities; + [SDPStreamMetadataKey]: SDPStreamMetadata; +} + +export interface MCallSelectAnswer extends MCallBase { + selected_party_id: string; +} + +export interface MCallInviteNegotiate extends MCallBase { + offer: RTCSessionDescription; + description: RTCSessionDescription; + lifetime: number; + capabilities?: CallCapabilities; + invitee?: string; + sender_session_id?: string; + dest_session_id?: string; + [SDPStreamMetadataKey]: SDPStreamMetadata; +} + +export interface MCallSDPStreamMetadataChanged extends MCallBase { + [SDPStreamMetadataKey]: SDPStreamMetadata; +} + +export interface MCallReplacesEvent extends MCallBase { + replacement_id: string; + target_user: CallReplacesTarget; + create_call: string; + await_call: string; + target_room: string; +} + +export interface MCAllAssertedIdentity extends MCallBase { + asserted_identity: { + id: string; + display_name: string; + avatar_url: string; + }; +} + +export interface MCallCandidates extends MCallBase { + candidates: RTCIceCandidate[]; +} + +export interface MCallHangupReject extends MCallBase { + reason?: CallErrorCode; +} + +/* eslint-enable camelcase */ From 468841eceae09904ac05a638d64e48c509626a34 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 16 Feb 2022 17:01:02 +0100 Subject: [PATCH 003/435] WIP3 --- src/matrix/calls/CallHandler.ts | 97 +++++- src/platform/types/MediaDevices.ts | 42 +++ src/platform/types/WebRTC.ts | 55 ++++ src/platform/web/dom/MediaDevices.ts | 457 +++++++++++++++++++++++++++ src/platform/web/dom/WebRTC.ts | 414 ++++++++++++++++++++++++ 5 files changed, 1056 insertions(+), 9 deletions(-) create mode 100644 src/platform/types/MediaDevices.ts create mode 100644 src/platform/types/WebRTC.ts create mode 100644 src/platform/web/dom/MediaDevices.ts create mode 100644 src/platform/web/dom/WebRTC.ts diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index b88c3ce2..0425304e 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -20,6 +20,9 @@ import type {Room} from "../room/Room"; import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; +import {WebRTC, PeerConnection, PeerConnectionHandler, StreamPurpose} from "../../platform/types/WebRTC"; +import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; + const GROUP_CALL_TYPE = "m.call"; const GROUP_CALL_MEMBER_TYPE = "m.call.member"; @@ -129,13 +132,96 @@ class GroupCall { * */ +class LocalMedia { + private tracks = new Map(); + + setTracks(tracks: Track[]) { + for (const track of tracks) { + this.setTrack(track); + } + } + + setTrack(track: Track) { + let cameraAndMicStreamDontMatch = false; + if (track.type === TrackType.Microphone) { + const {cameraTrack} = this; + if (cameraTrack && track.streamId !== cameraTrack.streamId) { + cameraAndMicStreamDontMatch = true; + } + } + if (track.type === TrackType.Camera) { + const {microphoneTrack} = this; + if (microphoneTrack && track.streamId !== microphoneTrack.streamId) { + cameraAndMicStreamDontMatch = true; + } + } + if (cameraAndMicStreamDontMatch) { + throw new Error("The camera and audio track should have the same stream id"); + } + this.tracks.set(track.type, track); + } + + public get cameraTrack(): Track | undefined { return this.tracks.get(TrackType.Camera); }; + public get screenShareTrack(): Track | undefined { return this.tracks.get(TrackType.ScreenShare); }; + public get microphoneTrack(): AudioTrack | undefined { return this.tracks.get(TrackType.Microphone) as (AudioTrack | undefined); }; + + getSDPMetadata(): any { + const metadata = {}; + const userMediaTrack = this.microphoneTrack ?? this.cameraTrack; + if (userMediaTrack) { + metadata[userMediaTrack.streamId] = { + purpose: StreamPurpose.UserMedia + }; + } + if (this.screenShareTrack) { + metadata[this.screenShareTrack.streamId] = { + purpose: StreamPurpose.ScreenShare + }; + } + return metadata; + } +} + // when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already // do for sharing keys will be best as that already deals with room tracking. type SendSignallingMessageCallback = (type: CallSetupMessageType, content: Record) => Promise; -class PeerCall { - constructor(private readonly sendSignallingMessage: SendSignallingMessageCallback) { +/** Implements a call between two peers with the signalling state keeping, while still delegating the signalling message sending. Used by GroupCall.*/ +class PeerCall implements PeerConnectionHandler { + private readonly peerConnection: PeerConnection; + constructor( + private readonly sendSignallingMessage: SendSignallingMessageCallback, + private localMedia: LocalMedia, + webRTC: WebRTC + ) { + this.peerConnection = webRTC.createPeerConnection(this); + } + + onIceConnectionStateChange(state: RTCIceConnectionState) {} + onLocalIceCandidate(candidate: RTCIceCandidate) {} + onIceGatheringStateChange(state: RTCIceGatheringState) {} + onRemoteTracksChanged(tracks: Track[]) {} + onDataChannelChanged(dataChannel: DataChannel | undefined) {} + onNegotiationNeeded() { + const message = { + offer: this.peerConnection.createOffer(), + sdp_stream_metadata: this.localMedia.getSDPMetadata(), + version: 1 + } + this.sendSignallingMessage(CallSetupMessageType.Invite, message); + } + + setLocalMedia(localMedia: LocalMedia) { + this.localMedia = localMedia; + // TODO: send new metadata + } + + + // request the type of incoming track + getPurposeForStreamId(streamId: string): StreamPurpose { + // look up stream purpose + return StreamPurpose.UserMedia; } handleIncomingSignallingMessage(type: CallSetupMessageType, content: Record) { @@ -148,10 +234,3 @@ class PeerCall { } } -class MediaSource { - -} - -class PeerConnection { - -} diff --git a/src/platform/types/MediaDevices.ts b/src/platform/types/MediaDevices.ts new file mode 100644 index 00000000..ec4895fa --- /dev/null +++ b/src/platform/types/MediaDevices.ts @@ -0,0 +1,42 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +export interface MediaDevices { + // filter out audiooutput + enumerate(): Promise; + getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise; + getScreenShareTrack(): Promise; +} + +export enum TrackType { + ScreenShare, + Camera, + Microphone, +} + +export interface Track { + get type(): TrackType; + get label(): string; + get id(): string; + get streamId(): string; + get muted(): boolean; + setMuted(muted: boolean): void; +} + +export interface AudioTrack extends Track { + get isSpeaking(): boolean; +} + diff --git a/src/platform/types/WebRTC.ts b/src/platform/types/WebRTC.ts new file mode 100644 index 00000000..b34e4214 --- /dev/null +++ b/src/platform/types/WebRTC.ts @@ -0,0 +1,55 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {Track, TrackType} from "./MediaDevices"; + +export enum StreamPurpose { + UserMedia = "m.usermedia", + ScreenShare = "m.screenshare" +} + +export interface WebRTC { + createPeerConnection(handler: PeerConnectionHandler): PeerConnection; +} + +export interface PeerConnectionHandler { + onIceConnectionStateChange(state: RTCIceConnectionState); + onLocalIceCandidate(candidate: RTCIceCandidate); + onIceGatheringStateChange(state: RTCIceGatheringState); + onRemoteTracksChanged(tracks: Track[]); + onDataChannelChanged(dataChannel: DataChannel | undefined); + onNegotiationNeeded(); + // request the type of incoming stream + getPurposeForStreamId(trackId: string): StreamPurpose; +} +// does it make sense to wrap this? +export interface DataChannel { + close(); + send(); +} + +export interface PeerConnection { + get remoteTracks(): Track[] | undefined; + get dataChannel(): DataChannel | undefined; + createOffer(): Promise; + createAnswer(): Promise; + setLocalDescription(description: RTCSessionDescriptionInit); + setRemoteDescription(description: RTCSessionDescriptionInit); + addTrack(track: Track): void; + removeTrack(track: Track): boolean; + replaceTrack(oldTrack: Track, newTrack: Track): Promise; + createDataChannel(): DataChannel; +} diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts new file mode 100644 index 00000000..08ccc636 --- /dev/null +++ b/src/platform/web/dom/MediaDevices.ts @@ -0,0 +1,457 @@ +/* +Copyright 2021 Šimon Brandner +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {MediaDevices as IMediaDevices, TrackType, Track, AudioTrack} from "../../types/MediaDevices"; + +const POLLING_INTERVAL = 200; // ms +export const SPEAKING_THRESHOLD = -60; // dB +const SPEAKING_SAMPLE_COUNT = 8; // samples + +class MediaDevicesWrapper implements IMediaDevices { + constructor(private readonly mediaDevices: MediaDevices) {} + + enumerate(): Promise { + return this.mediaDevices.enumerateDevices(); + } + + async getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise { + const stream = await this.mediaDevices.getUserMedia(this.getUserMediaContraints(audio, video)); + const tracks = stream.getTracks().map(t => { + const type = t.kind === "audio" ? TrackType.Microphone : TrackType.Camera; + return wrapTrack(t, stream, type); + }); + return tracks; + } + + async getScreenShareTrack(): Promise { + const stream = await this.mediaDevices.getDisplayMedia(this.getScreenshareContraints()); + const videoTrack = stream.getTracks().find(t => t.kind === "video"); + if (videoTrack) { + return wrapTrack(videoTrack, stream, TrackType.ScreenShare); + } + return; + } + + private getUserMediaContraints(audio: boolean | MediaDeviceInfo, video: boolean | MediaDeviceInfo): MediaStreamConstraints { + const isWebkit = !!navigator["webkitGetUserMedia"]; + + return { + audio: audio + ? { + deviceId: typeof audio !== "boolean" ? { ideal: audio.deviceId } : undefined, + } + : false, + video: video + ? { + deviceId: typeof video !== "boolean" ? { ideal: video.deviceId } : undefined, + /* We want 640x360. Chrome will give it only if we ask exactly, + FF refuses entirely if we ask exactly, so have to ask for ideal + instead + XXX: Is this still true? + */ + width: isWebkit ? { exact: 640 } : { ideal: 640 }, + height: isWebkit ? { exact: 360 } : { ideal: 360 }, + } + : false, + }; + } + + private getScreenshareContraints(): DisplayMediaStreamConstraints { + return { + audio: false, + video: true, + }; + } +} + +export function wrapTrack(track: MediaStreamTrack, stream: MediaStream, type: TrackType) { + if (track.kind === "audio") { + return new AudioTrackWrapper(track, stream, type); + } else { + return new TrackWrapper(track, stream, type); + } +} + +export class TrackWrapper implements Track { + constructor( + public readonly track: MediaStreamTrack, + public readonly stream: MediaStream, + public readonly type: TrackType + ) {} + + get label(): string { return this.track.label; } + get id(): string { return this.track.id; } + get streamId(): string { return this.stream.id; } + get muted(): boolean { return this.track.muted; } + + setMuted(muted: boolean): void { + this.track.enabled = !muted; + } +} + +export class AudioTrackWrapper extends TrackWrapper { + private measuringVolumeActivity = false; + private audioContext?: AudioContext; + private analyser: AnalyserNode; + private frequencyBinCount: Float32Array; + private speakingThreshold = SPEAKING_THRESHOLD; + private speaking = false; + private volumeLooperTimeout: number; + private speakingVolumeSamples: number[]; + + constructor(track: MediaStreamTrack, stream: MediaStream, type: TrackType) { + super(track, stream, type); + this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); + this.initVolumeMeasuring(); + this.measureVolumeActivity(true); + } + + get isSpeaking(): boolean { return this.speaking; } + /** + * Starts emitting volume_changed events where the emitter value is in decibels + * @param enabled emit volume changes + */ + private measureVolumeActivity(enabled: boolean): void { + if (enabled) { + if (!this.audioContext || !this.analyser || !this.frequencyBinCount) return; + + this.measuringVolumeActivity = true; + this.volumeLooper(); + } else { + this.measuringVolumeActivity = false; + this.speakingVolumeSamples.fill(-Infinity); + this.emit(CallFeedEvent.VolumeChanged, -Infinity); + } + } + + private initVolumeMeasuring(): void { + const AudioContext = window.AudioContext || window["webkitAudioContext"] as undefined | typeof window.AudioContext; + if (!AudioContext) return; + + this.audioContext = new AudioContext(); + + this.analyser = this.audioContext.createAnalyser(); + this.analyser.fftSize = 512; + this.analyser.smoothingTimeConstant = 0.1; + + const mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.stream); + mediaStreamAudioSourceNode.connect(this.analyser); + + this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount); + } + + + public setSpeakingThreshold(threshold: number) { + this.speakingThreshold = threshold; + } + + private volumeLooper = () => { + if (!this.analyser) return; + + if (!this.measuringVolumeActivity) return; + + this.analyser.getFloatFrequencyData(this.frequencyBinCount); + + let maxVolume = -Infinity; + for (let i = 0; i < this.frequencyBinCount.length; i++) { + if (this.frequencyBinCount[i] > maxVolume) { + maxVolume = this.frequencyBinCount[i]; + } + } + + this.speakingVolumeSamples.shift(); + this.speakingVolumeSamples.push(maxVolume); + + this.emit(CallFeedEvent.VolumeChanged, maxVolume); + + let newSpeaking = false; + + for (let i = 0; i < this.speakingVolumeSamples.length; i++) { + const volume = this.speakingVolumeSamples[i]; + + if (volume > this.speakingThreshold) { + newSpeaking = true; + break; + } + } + + if (this.speaking !== newSpeaking) { + this.speaking = newSpeaking; + this.emit(CallFeedEvent.Speaking, this.speaking); + } + + this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL) as unknown as number; + }; + + public dispose(): void { + clearTimeout(this.volumeLooperTimeout); + } +} + +// export interface ICallFeedOpts { +// client: MatrixClient; +// roomId: string; +// userId: string; +// stream: MediaStream; +// purpose: SDPStreamMetadataPurpose; +// audioMuted: boolean; +// videoMuted: boolean; +// } + +// export enum CallFeedEvent { +// NewStream = "new_stream", +// MuteStateChanged = "mute_state_changed", +// VolumeChanged = "volume_changed", +// Speaking = "speaking", +// } + +// export class CallFeed extends EventEmitter { +// public stream: MediaStream; +// public sdpMetadataStreamId: string; +// public userId: string; +// public purpose: SDPStreamMetadataPurpose; +// public speakingVolumeSamples: number[]; + +// private client: MatrixClient; +// private roomId: string; +// private audioMuted: boolean; +// private videoMuted: boolean; +// private measuringVolumeActivity = false; +// private audioContext: AudioContext; +// private analyser: AnalyserNode; +// private frequencyBinCount: Float32Array; +// private speakingThreshold = SPEAKING_THRESHOLD; +// private speaking = false; +// private volumeLooperTimeout: number; + +// constructor(opts: ICallFeedOpts) { +// super(); + +// this.client = opts.client; +// this.roomId = opts.roomId; +// this.userId = opts.userId; +// this.purpose = opts.purpose; +// this.audioMuted = opts.audioMuted; +// this.videoMuted = opts.videoMuted; +// this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); +// this.sdpMetadataStreamId = opts.stream.id; + +// this.updateStream(null, opts.stream); + +// if (this.hasAudioTrack) { +// this.initVolumeMeasuring(); +// } +// } + +// private get hasAudioTrack(): boolean { +// return this.stream.getAudioTracks().length > 0; +// } + +// private updateStream(oldStream: MediaStream, newStream: MediaStream): void { +// if (newStream === oldStream) return; + +// if (oldStream) { +// oldStream.removeEventListener("addtrack", this.onAddTrack); +// this.measureVolumeActivity(false); +// } +// if (newStream) { +// this.stream = newStream; +// newStream.addEventListener("addtrack", this.onAddTrack); + +// if (this.hasAudioTrack) { +// this.initVolumeMeasuring(); +// } else { +// this.measureVolumeActivity(false); +// } +// } + +// this.emit(CallFeedEvent.NewStream, this.stream); +// } + +// private initVolumeMeasuring(): void { +// const AudioContext = window.AudioContext || window.webkitAudioContext; +// if (!this.hasAudioTrack || !AudioContext) return; + +// this.audioContext = new AudioContext(); + +// this.analyser = this.audioContext.createAnalyser(); +// this.analyser.fftSize = 512; +// this.analyser.smoothingTimeConstant = 0.1; + +// const mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.stream); +// mediaStreamAudioSourceNode.connect(this.analyser); + +// this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount); +// } + +// private onAddTrack = (): void => { +// this.emit(CallFeedEvent.NewStream, this.stream); +// }; + +// /** +// * Returns callRoom member +// * @returns member of the callRoom +// */ +// public getMember(): RoomMember { +// const callRoom = this.client.getRoom(this.roomId); +// return callRoom.getMember(this.userId); +// } + +// /** +// * Returns true if CallFeed is local, otherwise returns false +// * @returns {boolean} is local? +// */ +// public isLocal(): boolean { +// return this.userId === this.client.getUserId(); +// } + +// /** +// * Returns true if audio is muted or if there are no audio +// * tracks, otherwise returns false +// * @returns {boolean} is audio muted? +// */ +// public isAudioMuted(): boolean { +// return this.stream.getAudioTracks().length === 0 || this.audioMuted; +// } + +// * +// * Returns true video is muted or if there are no video +// * tracks, otherwise returns false +// * @returns {boolean} is video muted? + +// public isVideoMuted(): boolean { +// // We assume only one video track +// return this.stream.getVideoTracks().length === 0 || this.videoMuted; +// } + +// public isSpeaking(): boolean { +// return this.speaking; +// } + +// /** +// * Replaces the current MediaStream with a new one. +// * This method should be only used by MatrixCall. +// * @param newStream new stream with which to replace the current one +// */ +// public setNewStream(newStream: MediaStream): void { +// this.updateStream(this.stream, newStream); +// } + +// /** +// * Set feed's internal audio mute state +// * @param muted is the feed's audio muted? +// */ +// public setAudioMuted(muted: boolean): void { +// this.audioMuted = muted; +// this.speakingVolumeSamples.fill(-Infinity); +// this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); +// } + +// /** +// * Set feed's internal video mute state +// * @param muted is the feed's video muted? +// */ +// public setVideoMuted(muted: boolean): void { +// this.videoMuted = muted; +// this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); +// } + +// /** +// * Starts emitting volume_changed events where the emitter value is in decibels +// * @param enabled emit volume changes +// */ +// public measureVolumeActivity(enabled: boolean): void { +// if (enabled) { +// if (!this.audioContext || !this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return; + +// this.measuringVolumeActivity = true; +// this.volumeLooper(); +// } else { +// this.measuringVolumeActivity = false; +// this.speakingVolumeSamples.fill(-Infinity); +// this.emit(CallFeedEvent.VolumeChanged, -Infinity); +// } +// } + +// public setSpeakingThreshold(threshold: number) { +// this.speakingThreshold = threshold; +// } + +// private volumeLooper = () => { +// if (!this.analyser) return; + +// if (!this.measuringVolumeActivity) return; + +// this.analyser.getFloatFrequencyData(this.frequencyBinCount); + +// let maxVolume = -Infinity; +// for (let i = 0; i < this.frequencyBinCount.length; i++) { +// if (this.frequencyBinCount[i] > maxVolume) { +// maxVolume = this.frequencyBinCount[i]; +// } +// } + +// this.speakingVolumeSamples.shift(); +// this.speakingVolumeSamples.push(maxVolume); + +// this.emit(CallFeedEvent.VolumeChanged, maxVolume); + +// let newSpeaking = false; + +// for (let i = 0; i < this.speakingVolumeSamples.length; i++) { +// const volume = this.speakingVolumeSamples[i]; + +// if (volume > this.speakingThreshold) { +// newSpeaking = true; +// break; +// } +// } + +// if (this.speaking !== newSpeaking) { +// this.speaking = newSpeaking; +// this.emit(CallFeedEvent.Speaking, this.speaking); +// } + +// this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL); +// }; + +// public clone(): CallFeed { +// const mediaHandler = this.client.getMediaHandler(); +// const stream = this.stream.clone(); + +// if (this.purpose === SDPStreamMetadataPurpose.Usermedia) { +// mediaHandler.userMediaStreams.push(stream); +// } else { +// mediaHandler.screensharingStreams.push(stream); +// } + +// return new CallFeed({ +// client: this.client, +// roomId: this.roomId, +// userId: this.userId, +// stream, +// purpose: this.purpose, +// audioMuted: this.audioMuted, +// videoMuted: this.videoMuted, +// }); +// } + +// public dispose(): void { +// clearTimeout(this.volumeLooperTimeout); +// this.measureVolumeActivity(false); +// } +// } diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts new file mode 100644 index 00000000..712e8cc6 --- /dev/null +++ b/src/platform/web/dom/WebRTC.ts @@ -0,0 +1,414 @@ +/* +Copyright 2021 Šimon Brandner + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {TrackWrapper, wrapTrack} from "./MediaDevices"; +import {Track} from "../../types/MediaDevices"; +import {WebRTC, PeerConnectionHandler, DataChannel, PeerConnection, StreamPurpose} from "../../types/WebRTC"; + +const POLLING_INTERVAL = 200; // ms +export const SPEAKING_THRESHOLD = -60; // dB +const SPEAKING_SAMPLE_COUNT = 8; // samples + +class DOMPeerConnection implements PeerConnection { + private readonly peerConnection: RTCPeerConnection; + private readonly handler: PeerConnectionHandler; + private dataChannelWrapper?: DOMDataChannel; + private _remoteTracks: TrackWrapper[]; + + constructor(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize = 0) { + this.handler = handler; + this.peerConnection = new RTCPeerConnection({ + iceTransportPolicy: forceTURN ? 'relay' : undefined, + iceServers: turnServers, + iceCandidatePoolSize: iceCandidatePoolSize, + }); + this.registerHandler(); + } + + get remoteTracks(): Track[] { return this._remoteTracks; } + get dataChannel(): DataChannel | undefined { return this.dataChannelWrapper; } + + createOffer(): Promise { + return this.peerConnection.createOffer(); + } + + createAnswer(): Promise { + return this.peerConnection.createAnswer(); + } + + setLocalDescription(description: RTCSessionDescriptionInit) { + this.peerConnection.setLocalDescription(description); + } + + setRemoteDescription(description: RTCSessionDescriptionInit) { + this.peerConnection.setRemoteDescription(description); + } + + addTrack(track: Track): void { + if (!(track instanceof TrackWrapper)) { + throw new Error("Not a TrackWrapper"); + } + this.peerConnection.addTrack(track.track, track.stream); + } + + removeTrack(track: Track): boolean { + if (!(track instanceof TrackWrapper)) { + throw new Error("Not a TrackWrapper"); + } + const sender = this.peerConnection.getSenders().find(s => s.track === track.track); + if (sender) { + this.peerConnection.removeTrack(sender); + return true; + } + return false; + } + + async replaceTrack(oldTrack: Track, newTrack: Track): Promise { + if (!(oldTrack instanceof TrackWrapper) || !(newTrack instanceof TrackWrapper)) { + throw new Error("Not a TrackWrapper"); + } + const sender = this.peerConnection.getSenders().find(s => s.track === oldTrack.track); + if (sender) { + await sender.replaceTrack(newTrack.track); + return true; + } + return false; + } + createDataChannel(): DataChannel { + return new DataChannel(this.peerConnection.createDataChannel()); + } + + private registerHandler() { + const pc = this.peerConnection; + pc.addEventListener('negotiationneeded', this); + pc.addEventListener('icecandidate', this); + pc.addEventListener('iceconnectionstatechange', this); + pc.addEventListener('icegatheringstatechange', this); + pc.addEventListener('signalingstatechange', this); + pc.addEventListener('track', this); + pc.addEventListener('datachannel', this); + } + + /** @internal */ + handleEvent(evt: Event) { + switch (evt.type) { + case "iceconnectionstatechange": + this.handleIceConnectionStateChange(); + break; + case "icecandidate": + this.handleLocalIceCandidate(evt as RTCPeerConnectionIceEvent); + break; + case "icegatheringstatechange": + this.handler.onIceGatheringStateChange(this.peerConnection.iceGatheringState); + break; + case "track": + this.handleRemoteTrack(evt as RTCTrackEvent); + break; + case "negotiationneeded": + this.handler.onNegotiationNeeded(); + break; + case "datachannel": + break; + } + } + + private handleLocalIceCandidate(event: RTCPeerConnectionIceEvent) { + if (event.candidate) { + this.handler.onLocalIceCandidate(event.candidate); + } + }; + + private handleIceConnectionStateChange() { + const {iceConnectionState} = this.peerConnection; + if (iceConnectionState === "failed" && this.peerConnection.restartIce) { + this.peerConnection.restartIce(); + } else { + this.handler.onIceConnectionStateChange(iceConnectionState); + } + } + + private handleRemoteTrack(evt: RTCTrackEvent) { + const updatedTracks = evt.streams.flatMap(stream => stream.getTracks().map(track => {return {stream, track};})); + const withoutRemovedTracks = this._remoteTracks.filter(t => !updatedTracks.some(ut => t.track == ut.track)); + const addedTracks = updatedTracks.filter(ut => !this._remoteTracks.some(t => t.track === ut.track)); + const wrappedAddedTracks = addedTracks.map(t => this.wrapRemoteTrack(t.track, t.stream)); + this._remoteTracks = withoutRemovedTracks.concat(...wrappedAddedTracks); + this.handler.onRemoteTracksChanged(this.remoteTracks); + } + private wrapRemoteTrack(track: MediaStreamTrack, stream: MediaStream): TrackWrapper { + let type: TrackType; + if (track.kind === "video") { + const purpose = this.handler.getPurposeForStreamId(stream.id); + type = purpose === StreamPurpose.Usermedia ? TrackType.Camera : TrackType.ScreenShare; + } else { + type = TrackType.Microphone; + } + return wrapTrack(track, stream, type); + } +} + +export interface ICallFeedOpts { + client: MatrixClient; + roomId: string; + userId: string; + stream: MediaStream; + purpose: SDPStreamMetadataPurpose; + audioMuted: boolean; + videoMuted: boolean; +} + +export enum CallFeedEvent { + NewStream = "new_stream", + MuteStateChanged = "mute_state_changed", + VolumeChanged = "volume_changed", + Speaking = "speaking", +} + +export class CallFeed extends EventEmitter { + public stream: MediaStream; + public sdpMetadataStreamId: string; + public userId: string; + public purpose: SDPStreamMetadataPurpose; + public speakingVolumeSamples: number[]; + + private client: MatrixClient; + private roomId: string; + private audioMuted: boolean; + private videoMuted: boolean; + private measuringVolumeActivity = false; + private audioContext: AudioContext; + private analyser: AnalyserNode; + private frequencyBinCount: Float32Array; + private speakingThreshold = SPEAKING_THRESHOLD; + private speaking = false; + private volumeLooperTimeout: number; + + constructor(opts: ICallFeedOpts) { + super(); + + this.client = opts.client; + this.roomId = opts.roomId; + this.userId = opts.userId; + this.purpose = opts.purpose; + this.audioMuted = opts.audioMuted; + this.videoMuted = opts.videoMuted; + this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); + this.sdpMetadataStreamId = opts.stream.id; + + this.updateStream(null, opts.stream); + + if (this.hasAudioTrack) { + this.initVolumeMeasuring(); + } + } + + private get hasAudioTrack(): boolean { + return this.stream.getAudioTracks().length > 0; + } + + private updateStream(oldStream: MediaStream, newStream: MediaStream): void { + if (newStream === oldStream) return; + + if (oldStream) { + oldStream.removeEventListener("addtrack", this.onAddTrack); + this.measureVolumeActivity(false); + } + if (newStream) { + this.stream = newStream; + newStream.addEventListener("addtrack", this.onAddTrack); + + if (this.hasAudioTrack) { + this.initVolumeMeasuring(); + } else { + this.measureVolumeActivity(false); + } + } + + this.emit(CallFeedEvent.NewStream, this.stream); + } + + private initVolumeMeasuring(): void { + const AudioContext = window.AudioContext || window.webkitAudioContext; + if (!this.hasAudioTrack || !AudioContext) return; + + this.audioContext = new AudioContext(); + + this.analyser = this.audioContext.createAnalyser(); + this.analyser.fftSize = 512; + this.analyser.smoothingTimeConstant = 0.1; + + const mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.stream); + mediaStreamAudioSourceNode.connect(this.analyser); + + this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount); + } + + private onAddTrack = (): void => { + this.emit(CallFeedEvent.NewStream, this.stream); + }; + + /** + * Returns callRoom member + * @returns member of the callRoom + */ + public getMember(): RoomMember { + const callRoom = this.client.getRoom(this.roomId); + return callRoom.getMember(this.userId); + } + + /** + * Returns true if CallFeed is local, otherwise returns false + * @returns {boolean} is local? + */ + public isLocal(): boolean { + return this.userId === this.client.getUserId(); + } + + /** + * Returns true if audio is muted or if there are no audio + * tracks, otherwise returns false + * @returns {boolean} is audio muted? + */ + public isAudioMuted(): boolean { + return this.stream.getAudioTracks().length === 0 || this.audioMuted; + } + + /** + * Returns true video is muted or if there are no video + * tracks, otherwise returns false + * @returns {boolean} is video muted? + */ + public isVideoMuted(): boolean { + // We assume only one video track + return this.stream.getVideoTracks().length === 0 || this.videoMuted; + } + + public isSpeaking(): boolean { + return this.speaking; + } + + /** + * Replaces the current MediaStream with a new one. + * This method should be only used by MatrixCall. + * @param newStream new stream with which to replace the current one + */ + public setNewStream(newStream: MediaStream): void { + this.updateStream(this.stream, newStream); + } + + /** + * Set feed's internal audio mute state + * @param muted is the feed's audio muted? + */ + public setAudioMuted(muted: boolean): void { + this.audioMuted = muted; + this.speakingVolumeSamples.fill(-Infinity); + this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); + } + + /** + * Set feed's internal video mute state + * @param muted is the feed's video muted? + */ + public setVideoMuted(muted: boolean): void { + this.videoMuted = muted; + this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); + } + + /** + * Starts emitting volume_changed events where the emitter value is in decibels + * @param enabled emit volume changes + */ + public measureVolumeActivity(enabled: boolean): void { + if (enabled) { + if (!this.audioContext || !this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return; + + this.measuringVolumeActivity = true; + this.volumeLooper(); + } else { + this.measuringVolumeActivity = false; + this.speakingVolumeSamples.fill(-Infinity); + this.emit(CallFeedEvent.VolumeChanged, -Infinity); + } + } + + public setSpeakingThreshold(threshold: number) { + this.speakingThreshold = threshold; + } + + private volumeLooper = () => { + if (!this.analyser) return; + + if (!this.measuringVolumeActivity) return; + + this.analyser.getFloatFrequencyData(this.frequencyBinCount); + + let maxVolume = -Infinity; + for (let i = 0; i < this.frequencyBinCount.length; i++) { + if (this.frequencyBinCount[i] > maxVolume) { + maxVolume = this.frequencyBinCount[i]; + } + } + + this.speakingVolumeSamples.shift(); + this.speakingVolumeSamples.push(maxVolume); + + this.emit(CallFeedEvent.VolumeChanged, maxVolume); + + let newSpeaking = false; + + for (let i = 0; i < this.speakingVolumeSamples.length; i++) { + const volume = this.speakingVolumeSamples[i]; + + if (volume > this.speakingThreshold) { + newSpeaking = true; + break; + } + } + + if (this.speaking !== newSpeaking) { + this.speaking = newSpeaking; + this.emit(CallFeedEvent.Speaking, this.speaking); + } + + this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL); + }; + + public clone(): CallFeed { + const mediaHandler = this.client.getMediaHandler(); + const stream = this.stream.clone(); + + if (this.purpose === SDPStreamMetadataPurpose.Usermedia) { + mediaHandler.userMediaStreams.push(stream); + } else { + mediaHandler.screensharingStreams.push(stream); + } + + return new CallFeed({ + client: this.client, + roomId: this.roomId, + userId: this.userId, + stream, + purpose: this.purpose, + audioMuted: this.audioMuted, + videoMuted: this.videoMuted, + }); + } + + public dispose(): void { + clearTimeout(this.volumeLooperTimeout); + } +} From e5f44aecfbe2fbcc53e3a7fac75b517baa776c7d Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Thu, 17 Feb 2022 16:58:44 +0100 Subject: [PATCH 004/435] WIP4 --- src/matrix/calls/CallHandler.ts | 207 +- src/matrix/calls/PeerCall.ts | 2757 +++++++++++++++++++++++++ src/matrix/calls/TODO.md | 43 + src/matrix/calls/group/Call.ts | 88 + src/matrix/calls/group/Participant.ts | 46 + src/platform/types/MediaDevices.ts | 1 + src/platform/types/WebRTC.ts | 3 +- src/platform/web/dom/MediaDevices.ts | 1 + src/platform/web/dom/WebRTC.ts | 261 +-- 9 files changed, 3026 insertions(+), 381 deletions(-) create mode 100644 src/matrix/calls/PeerCall.ts create mode 100644 src/matrix/calls/TODO.md create mode 100644 src/matrix/calls/group/Call.ts create mode 100644 src/matrix/calls/group/Participant.ts diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 0425304e..a31ff666 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -36,9 +36,9 @@ enum CallSetupMessageType { const CALL_ID = "m.call_id"; const CALL_TERMINATED = "m.terminated"; -export class CallHandler { +export class GroupCallHandler { // group calls by call id - public readonly groupCalls: ObservableMap = new ObservableMap(); + public readonly calls: ObservableMap = new ObservableMap(); constructor() { @@ -49,15 +49,15 @@ export class CallHandler { for (const event of events) { if (event.type === GROUP_CALL_TYPE) { const callId = event.state_key; - let call = this.groupCalls.get(callId); + let call = this.calls.get(callId); if (call) { call.updateCallEvent(event); if (call.isTerminated) { - this.groupCalls.remove(call.id); + this.calls.remove(call.id); } } else { call = new GroupCall(event, room); - this.groupCalls.set(call.id, call); + this.calls.set(call.id, call); } } } @@ -67,7 +67,7 @@ export class CallHandler { const participant = event.state_key; const sources = event.content["m.sources"]; for (const source of sources) { - const call = this.groupCalls.get(source[CALL_ID]); + const call = this.calls.get(source[CALL_ID]); if (call && !call.isTerminated) { call.addParticipant(participant, source); } @@ -85,33 +85,102 @@ export class CallHandler { handleDeviceMessage(senderUserId: string, senderDeviceId: string, eventType: string, content: Record, log: ILogItem) { const callId = content[CALL_ID]; - const call = this.groupCalls.get(callId); + const call = this.calls.get(callId); call?.handleDeviceMessage(senderUserId, senderDeviceId, eventType, content, log); } } -function peerCallKey(senderUserId: string, senderDeviceId: string) { +function participantId(senderUserId: string, senderDeviceId: string | null) { return JSON.stringify(senderUserId) + JSON.stringify(senderDeviceId); } +class GroupParticipant implements PeerCallHandler { + private peerCall?: PeerCall; + + constructor( + private readonly userId: string, + private readonly deviceId: string, + private localMedia: LocalMedia | undefined, + private readonly webRTC: WebRTC, + private readonly hsApi: HomeServerApi + ) {} + + sendInvite() { + this.peerCall = new PeerCall(this, this.webRTC); + this.peerCall.setLocalMedia(this.localMedia); + this.peerCall.sendOffer(); + } + + /** From PeerCallHandler + * @internal */ + override emitUpdate() { + + } + + /** From PeerCallHandler + * @internal */ + override onSendSignallingMessage() { + // TODO: this needs to be encrypted with olm first + this.hsApi.sendToDevice(type, {[this.userId]: {[this.deviceId ?? "*"]: content}}); + } +} + class GroupCall { - private peerCalls: Map + private readonly participants: ObservableMap = new ObservableMap(); + private localMedia?: LocalMedia; - constructor(private callEvent: StateEvent, private readonly room: Room) { + constructor(private readonly ownUserId: string, private callEvent: StateEvent, private readonly room: Room, private readonly webRTC: WebRTC) { } + get id(): string { return this.callEvent.state_key; } + + async participate(tracks: Track[]) { + this.localMedia = LocalMedia.fromTracks(tracks); + for (const [,participant] of this.participants) { + participant.setLocalMedia(this.localMedia.clone()); + } + // send m.call.member state event + + // send invite to all participants that are < my userId + for (const [,participant] of this.participants) { + if (participant.userId < this.ownUserId) { + participant.sendInvite(); + } + } + } + updateCallEvent(callEvent: StateEvent) { this.callEvent = callEvent; } addParticipant(userId, source) { - + const participantId = getParticipantId(userId, source.device_id); + const participant = this.participants.get(participantId); + if (participant) { + participant.updateSource(source); + } else { + participant.add(participantId, new GroupParticipant(userId, source.device_id, this.localMedia?.clone(), this.webRTC)); + } } handleDeviceMessage(senderUserId: string, senderDeviceId: string, eventType: string, content: Record, log: ILogItem) { - const peerCall = this.peerCalls.get(peerCallKey(senderUserId, senderDeviceId)); - peerCall?.handleIncomingSignallingMessage() + const participantId = getParticipantId(senderUserId, senderDeviceId); + let peerCall = this.participants.get(participantId); + let hasDeviceInKey = true; + if (!peerCall) { + hasDeviceInKey = false; + peerCall = this.participants.get(getParticipantId(senderUserId, null)) + } + if (peerCall) { + peerCall.handleIncomingSignallingMessage(eventType, content, senderDeviceId); + if (!hasDeviceInKey && peerCall.opponentPartyId) { + this.participants.delete(getParticipantId(senderUserId, null)); + this.participants.add(getParticipantId(senderUserId, peerCall.opponentPartyId)); + } + } else { + // create peerCall + } } get id(): string { @@ -121,116 +190,4 @@ class GroupCall { get isTerminated(): boolean { return !!this.callEvent.content[CALL_TERMINATED]; } - - private createPeerCall(userId: string, deviceId: string): PeerCall { - - } } - -/** - * Does WebRTC signalling for a single PeerConnection, and deals with WebRTC wrappers from platform - * */ - - -class LocalMedia { - private tracks = new Map(); - - setTracks(tracks: Track[]) { - for (const track of tracks) { - this.setTrack(track); - } - } - - setTrack(track: Track) { - let cameraAndMicStreamDontMatch = false; - if (track.type === TrackType.Microphone) { - const {cameraTrack} = this; - if (cameraTrack && track.streamId !== cameraTrack.streamId) { - cameraAndMicStreamDontMatch = true; - } - } - if (track.type === TrackType.Camera) { - const {microphoneTrack} = this; - if (microphoneTrack && track.streamId !== microphoneTrack.streamId) { - cameraAndMicStreamDontMatch = true; - } - } - if (cameraAndMicStreamDontMatch) { - throw new Error("The camera and audio track should have the same stream id"); - } - this.tracks.set(track.type, track); - } - - public get cameraTrack(): Track | undefined { return this.tracks.get(TrackType.Camera); }; - public get screenShareTrack(): Track | undefined { return this.tracks.get(TrackType.ScreenShare); }; - public get microphoneTrack(): AudioTrack | undefined { return this.tracks.get(TrackType.Microphone) as (AudioTrack | undefined); }; - - getSDPMetadata(): any { - const metadata = {}; - const userMediaTrack = this.microphoneTrack ?? this.cameraTrack; - if (userMediaTrack) { - metadata[userMediaTrack.streamId] = { - purpose: StreamPurpose.UserMedia - }; - } - if (this.screenShareTrack) { - metadata[this.screenShareTrack.streamId] = { - purpose: StreamPurpose.ScreenShare - }; - } - return metadata; - } -} - -// when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already -// do for sharing keys will be best as that already deals with room tracking. -type SendSignallingMessageCallback = (type: CallSetupMessageType, content: Record) => Promise; - -/** Implements a call between two peers with the signalling state keeping, while still delegating the signalling message sending. Used by GroupCall.*/ -class PeerCall implements PeerConnectionHandler { - private readonly peerConnection: PeerConnection; - - constructor( - private readonly sendSignallingMessage: SendSignallingMessageCallback, - private localMedia: LocalMedia, - webRTC: WebRTC - ) { - this.peerConnection = webRTC.createPeerConnection(this); - } - - onIceConnectionStateChange(state: RTCIceConnectionState) {} - onLocalIceCandidate(candidate: RTCIceCandidate) {} - onIceGatheringStateChange(state: RTCIceGatheringState) {} - onRemoteTracksChanged(tracks: Track[]) {} - onDataChannelChanged(dataChannel: DataChannel | undefined) {} - onNegotiationNeeded() { - const message = { - offer: this.peerConnection.createOffer(), - sdp_stream_metadata: this.localMedia.getSDPMetadata(), - version: 1 - } - this.sendSignallingMessage(CallSetupMessageType.Invite, message); - } - - setLocalMedia(localMedia: LocalMedia) { - this.localMedia = localMedia; - // TODO: send new metadata - } - - - // request the type of incoming track - getPurposeForStreamId(streamId: string): StreamPurpose { - // look up stream purpose - return StreamPurpose.UserMedia; - } - - handleIncomingSignallingMessage(type: CallSetupMessageType, content: Record) { - switch (type) { - case CallSetupMessageType.Invite: - case CallSetupMessageType.Answer: - case CallSetupMessageType.Candidates: - case CallSetupMessageType.Hangup: - } - } -} - diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts new file mode 100644 index 00000000..8780258e --- /dev/null +++ b/src/matrix/calls/PeerCall.ts @@ -0,0 +1,2757 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {ObservableMap} from "../../observable/map/ObservableMap"; + +import type {Room} from "../room/Room"; +import type {StateEvent} from "../storage/types"; +import type {ILogItem} from "../../logging/types"; + +import {WebRTC, PeerConnection, PeerConnectionHandler, StreamPurpose} from "../../platform/types/WebRTC"; +import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; + +import { randomString } from '../randomstring'; +import { + MCallReplacesEvent, + MCallAnswer, + MCallInviteNegotiate, + CallCapabilities, + SDPStreamMetadataPurpose, + SDPStreamMetadata, + SDPStreamMetadataKey, + MCallSDPStreamMetadataChanged, + MCallSelectAnswer, + MCAllAssertedIdentity, + MCallCandidates, + MCallBase, + MCallHangupReject, +} from './callEventTypes'; + + +const GROUP_CALL_TYPE = "m.call"; +const GROUP_CALL_MEMBER_TYPE = "m.call.member"; + + +/** + * Fires whenever an error occurs when call.js encounters an issue with setting up the call. + *

+ * The error given will have a code equal to either `MatrixCall.ERR_LOCAL_OFFER_FAILED` or + * `MatrixCall.ERR_NO_USER_MEDIA`. `ERR_LOCAL_OFFER_FAILED` is emitted when the local client + * fails to create an offer. `ERR_NO_USER_MEDIA` is emitted when the user has denied access + * to their audio/video hardware. + * + * @event module:webrtc/call~MatrixCall#"error" + * @param {Error} err The error raised by MatrixCall. + * @example + * matrixCall.on("error", function(err){ + * console.error(err.code, err); + * }); + */ + +// null is used as a special value meaning that the we're in a legacy 1:1 call +// without MSC2746 that doesn't provide an id which device sent the message. +type PartyId = string | null; + +interface CallOpts { + roomId?: string; + invitee?: string; + client?: any; // Fix when client is TSified + forceTURN?: boolean; + turnServers?: Array; + opponentDeviceId?: string; + opponentSessionId?: string; + groupCallId?: string; +} + +interface TurnServer { + urls: Array; + username?: string; + password?: string; + ttl?: number; +} + +interface AssertedIdentity { + id: string; + displayName: string; +} + +export enum CallState { + Fledgling = 'fledgling', + InviteSent = 'invite_sent', + WaitLocalMedia = 'wait_local_media', + CreateOffer = 'create_offer', + CreateAnswer = 'create_answer', + Connecting = 'connecting', + Connected = 'connected', + Ringing = 'ringing', + Ended = 'ended', +} + +export enum CallType { + Voice = 'voice', + Video = 'video', +} + +export enum CallDirection { + Inbound = 'inbound', + Outbound = 'outbound', +} + +export enum CallParty { + Local = 'local', + Remote = 'remote', +} + +export enum CallEvent { + Hangup = 'hangup', + State = 'state', + Error = 'error', + Replaced = 'replaced', + + // The value of isLocalOnHold() has changed + LocalHoldUnhold = 'local_hold_unhold', + // The value of isRemoteOnHold() has changed + RemoteHoldUnhold = 'remote_hold_unhold', + // backwards compat alias for LocalHoldUnhold: remove in a major version bump + HoldUnhold = 'hold_unhold', + // Feeds have changed + FeedsChanged = 'feeds_changed', + + AssertedIdentityChanged = 'asserted_identity_changed', + + LengthChanged = 'length_changed', + + DataChannel = 'datachannel', + + SendVoipEvent = "send_voip_event", +} + +export enum CallErrorCode { + /** The user chose to end the call */ + UserHangup = 'user_hangup', + + /** An error code when the local client failed to create an offer. */ + LocalOfferFailed = 'local_offer_failed', + /** + * An error code when there is no local mic/camera to use. This may be because + * the hardware isn't plugged in, or the user has explicitly denied access. + */ + NoUserMedia = 'no_user_media', + + /** + * Error code used when a call event failed to send + * because unknown devices were present in the room + */ + UnknownDevices = 'unknown_devices', + + /** + * Error code used when we fail to send the invite + * for some reason other than there being unknown devices + */ + SendInvite = 'send_invite', + + /** + * An answer could not be created + */ + CreateAnswer = 'create_answer', + + /** + * Error code used when we fail to send the answer + * for some reason other than there being unknown devices + */ + SendAnswer = 'send_answer', + + /** + * The session description from the other side could not be set + */ + SetRemoteDescription = 'set_remote_description', + + /** + * The session description from this side could not be set + */ + SetLocalDescription = 'set_local_description', + + /** + * A different device answered the call + */ + AnsweredElsewhere = 'answered_elsewhere', + + /** + * No media connection could be established to the other party + */ + IceFailed = 'ice_failed', + + /** + * The invite timed out whilst waiting for an answer + */ + InviteTimeout = 'invite_timeout', + + /** + * The call was replaced by another call + */ + Replaced = 'replaced', + + /** + * Signalling for the call could not be sent (other than the initial invite) + */ + SignallingFailed = 'signalling_timeout', + + /** + * The remote party is busy + */ + UserBusy = 'user_busy', + + /** + * We transferred the call off to somewhere else + */ + Transfered = 'transferred', + + /** + * A call from the same user was found with a new session id + */ + NewSession = 'new_session', +} + +/** + * The version field that we set in m.call.* events + */ +const VOIP_PROTO_VERSION = 1; + +/** The fallback ICE server to use for STUN or TURN protocols. */ +const FALLBACK_ICE_SERVER = 'stun:turn.matrix.org'; + +/** The length of time a call can be ringing for. */ +const CALL_TIMEOUT_MS = 60000; + +const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); + +export class CallError extends Error { + code: string; + + constructor(code: CallErrorCode, msg: string, err: Error) { + // Still don't think there's any way to have proper nested errors + super(msg + ": " + err); + + this.code = code; + } +} + +export function genCallID(): string { + return Date.now().toString() + randomString(16); +} + +enum CallSetupMessageType { + Invite = "m.call.invite", + Answer = "m.call.answer", + Candidates = "m.call.candidates", + Hangup = "m.call.hangup", +} + +const CALL_ID = "m.call_id"; +const CALL_TERMINATED = "m.terminated"; + +class LocalMedia { + constructor( + public readonly cameraTrack?: Track, + public readonly screenShareTrack?: Track, + public readonly microphoneTrack?: AudioTrack + ) {} + + withTracks(tracks: Track[]) { + const cameraTrack = tracks.find(t => t.type === TrackType.Camera) ?? this.cameraTrack; + const screenShareTrack = tracks.find(t => t.type === TrackType.ScreenShare) ?? this.screenShareTrack; + const microphoneTrack = tracks.find(t => t.type === TrackType.Microphone) ?? this.microphoneTrack; + if (cameraTrack && microphoneTrack && cameraTrack.streamId !== microphoneTrack.streamId) { + throw new Error("The camera and audio track should have the same stream id"); + } + return new LocalMedia(cameraTrack, screenShareTrack, microphoneTrack as AudioTrack); + } + + getSDPMetadata(): any { + const metadata = {}; + const userMediaTrack = this.microphoneTrack ?? this.cameraTrack; + if (userMediaTrack) { + metadata[userMediaTrack.streamId] = { + purpose: StreamPurpose.UserMedia, + audio_muted: this.microphoneTrack?.muted ?? false, + video_muted: this.cameraTrack?.muted ?? false, + }; + } + if (this.screenShareTrack) { + metadata[this.screenShareTrack.streamId] = { + purpose: StreamPurpose.ScreenShare + }; + } + return metadata; + } +} + +interface PeerCallHandler { + emitUpdate(); + sendSignallingMessage(type: string, content: Record); +} + +// when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already +// do for sharing keys will be best as that already deals with room tracking. +type SendSignallingMessageCallback = (type: CallSetupMessageType, content: Record) => Promise; +/** + * Does WebRTC signalling for a single PeerConnection, and deals with WebRTC wrappers from platform + * */ +/** Implements a call between two peers with the signalling state keeping, while still delegating the signalling message sending. Used by GroupCall.*/ +class PeerCall implements PeerConnectionHandler { + private readonly peerConnection: PeerConnection; + + + public state = CallState.Fledgling; + public hangupParty: CallParty; + public hangupReason: string; + public direction: CallDirection; + public peerConn?: RTCPeerConnection; + + // A queue for candidates waiting to go out. + // We try to amalgamate candidates into a single candidate message where + // possible + private candidateSendQueue: Array = []; + private candidateSendTries = 0; + private sentEndOfCandidates = false; + + private inviteOrAnswerSent = false; + private waitForLocalAVStream: boolean; + private successor: MatrixCall; + private opponentVersion: number | string; + // The party ID of the other side: undefined if we haven't chosen a partner + // yet, null if we have but they didn't send a party ID. + private opponentPartyId: PartyId; + private opponentCaps: CallCapabilities; + private inviteTimeout: number; + private iceDisconnectedTimeout: number; + + // The logic of when & if a call is on hold is nontrivial and explained in is*OnHold + // This flag represents whether we want the other party to be on hold + private remoteOnHold = false; + + // the stats for the call at the point it ended. We can't get these after we + // tear the call down, so we just grab a snapshot before we stop the call. + // The typescript definitions have this type as 'any' :( + private callStatsAtEnd: any[]; + + // Perfect negotiation state: https://www.w3.org/TR/webrtc/#perfect-negotiation-example + private makingOffer = false; + private ignoreOffer: boolean; + + private responsePromiseChain?: Promise; + + // If candidates arrive before we've picked an opponent (which, in particular, + // will happen if the opponent sends candidates eagerly before the user answers + // the call) we buffer them up here so we can then add the ones from the party we pick + private remoteCandidateBuffer: Map = []; + + private remoteAssertedIdentity: AssertedIdentity; + + private remoteSDPStreamMetadata?: SDPStreamMetadata; + + + constructor( + private readonly handler: PeerCallHandler, + private localMedia: LocalMedia, + webRTC: WebRTC + ) { + this.peerConnection = webRTC.createPeerConnection(this); + } + + onIceConnectionStateChange(state: RTCIceConnectionState) {} + onLocalIceCandidate(candidate: RTCIceCandidate) {} + onIceGatheringStateChange(state: RTCIceGatheringState) {} + onRemoteTracksChanged(tracks: Track[]) {} + onDataChannelChanged(dataChannel: DataChannel | undefined) {} + onNegotiationNeeded() { + const message = { + offer: this.peerConnection.createOffer(), + sdp_stream_metadata: this.localMedia.getSDPMetadata(), + version: 1 + } + this.handler.sendSignallingMessage(CallSetupMessageType.Invite, message); + } + + setLocalMedia(localMedia: LocalMedia) { + this.localMedia = localMedia; + // TODO: send new metadata + } + + // request the type of incoming track + getPurposeForStreamId(streamId: string): StreamPurpose { + // TODO: should we return a promise here for the case where the metadata hasn't arrived yet? + const metaData = this.remoteSDPStreamMetadata[streamId]; + return metadata?.purpose as StreamPurpose ?? StreamPurpose.UserMedia; + } + + private setState(state: CallState): void { + const oldState = this.state; + this.state = state; + this.handler.emitUpdate(); + } + + handleIncomingSignallingMessage(type: CallSetupMessageType, content: Record, partyId: PartyId) { + switch (type) { + case CallSetupMessageType.Invite: + case CallSetupMessageType.Answer: + this.handleAnswer(content); + break; + case CallSetupMessageType.Candidates: + this.handleRemoteIceCandidates(content); + break; + case CallSetupMessageType.Hangup: + } + } + + private async handleAnswer(content: MCallAnswer, partyId: PartyId) { + // add buffered ice candidates to peerConnection + if (this.opponentPartyId !== undefined) { + return; + } + this.opponentPartyId = partyId; + const bufferedCandidates = this.remoteCandidateBuffer?.get(partyId); + if (bufferedCandidates) { + this.addIceCandidates(bufferedCandidates); + } + this.remoteCandidateBuffer = undefined; + + this.setState(CallState.Connecting); + + const sdpStreamMetadata = content[SDPStreamMetadataKey]; + if (sdpStreamMetadata) { + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); + } else { + logger.warn("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); + } + + try { + await this.peerConnection.setRemoteDescription(content.answer); + } catch (e) { + logger.debug("Failed to set remote description", e); + this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); + return; + } + + // If the answer we selected has a party_id, send a select_answer event + // We do this after setting the remote description since otherwise we'd block + // call setup on it + if (this.opponentPartyId !== null) { + try { + await this.sendVoipEvent(EventType.CallSelectAnswer, { + selected_party_id: this.opponentPartyId, + }); + } catch (err) { + // This isn't fatal, and will just mean that if another party has raced to answer + // the call, they won't know they got rejected, so we carry on & don't retry. + logger.warn("Failed to send select_answer event", err); + } + } + } + + private handleRemoteIceCandidates(content: Record) { + if (this.state === CallState.Ended) { + return; + } + const candidates = content.candidates; + if (!candidates) { + return; + } + if (this.opponentPartyId === undefined) { + if (!this.remoteCandidateBuffer) { + this.remoteCandidateBuffer = new Map(); + } + const bufferedCandidates = this.remoteCandidateBuffer.get(fromPartyId) || []; + bufferedCandidates.push(...candidates); + this.remoteCandidateBuffer.set(fromPartyId, bufferedCandidates); + } else { + this.addIceCandidates(candidates); + } + } + + private async addIceCandidates(candidates: RTCIceCandidate[]): Promise { + for (const candidate of candidates) { + if ( + (candidate.sdpMid === null || candidate.sdpMid === undefined) && + (candidate.sdpMLineIndex === null || candidate.sdpMLineIndex === undefined) + ) { + logger.debug("Ignoring remote ICE candidate with no sdpMid or sdpMLineIndex"); + continue; + } + logger.debug( + "Call " + this.callId + " got remote ICE " + candidate.sdpMid + " candidate: " + candidate.candidate, + ); + try { + await this.peerConnection.addIceCandidate(candidate); + } catch (err) { + if (!this.ignoreOffer) { + logger.info("Failed to add remote ICE candidate", err); + } + } + } + } +} + +/** + * Construct a new Matrix Call. + * @constructor + * @param {Object} opts Config options. + * @param {string} opts.roomId The room ID for this call. + * @param {Object} opts.webRtc The WebRTC globals from the browser. + * @param {boolean} opts.forceTURN whether relay through TURN should be forced. + * @param {Object} opts.URL The URL global. + * @param {Array} opts.turnServers Optional. A list of TURN servers. + * @param {MatrixClient} opts.client The Matrix Client instance to send events to. + */ +export class MatrixCall extends EventEmitter { + // should go into DirectCall/GroupCall class + public roomId: string; + public callId: string; + public invitee?: string; + public ourPartyId: string; + private opponentMember: RoomMember; + private opponentDeviceId: string; + private opponentSessionId: string; + public groupCallId: string; + private callLengthInterval: number; + private callLength = 0; + + + + public state = CallState.Fledgling; + public hangupParty: CallParty; + public hangupReason: string; + public direction: CallDirection; + public peerConn?: RTCPeerConnection; + + // A queue for candidates waiting to go out. + // We try to amalgamate candidates into a single candidate message where + // possible + private candidateSendQueue: Array = []; + private candidateSendTries = 0; + private sentEndOfCandidates = false; + + private inviteOrAnswerSent = false; + private waitForLocalAVStream: boolean; + private successor: MatrixCall; + private opponentVersion: number | string; + // The party ID of the other side: undefined if we haven't chosen a partner + // yet, null if we have but they didn't send a party ID. + private opponentPartyId: string; + private opponentCaps: CallCapabilities; + private inviteTimeout: number; + private iceDisconnectedTimeout: number; + + // The logic of when & if a call is on hold is nontrivial and explained in is*OnHold + // This flag represents whether we want the other party to be on hold + private remoteOnHold = false; + + // the stats for the call at the point it ended. We can't get these after we + // tear the call down, so we just grab a snapshot before we stop the call. + // The typescript definitions have this type as 'any' :( + private callStatsAtEnd: any[]; + + // Perfect negotiation state: https://www.w3.org/TR/webrtc/#perfect-negotiation-example + private makingOffer = false; + private ignoreOffer: boolean; + + private responsePromiseChain?: Promise; + + // If candidates arrive before we've picked an opponent (which, in particular, + // will happen if the opponent sends candidates eagerly before the user answers + // the call) we buffer them up here so we can then add the ones from the party we pick + private remoteCandidateBuffer: RTCIceCandidate[] = []; + + private remoteAssertedIdentity: AssertedIdentity; + + private remoteSDPStreamMetadata: SDPStreamMetadata; + + constructor(opts: CallOpts) { + super(); + this.roomId = opts.roomId; + this.invitee = opts.invitee; + this.client = opts.client; + this.forceTURN = opts.forceTURN; + this.ourPartyId = this.client.deviceId; + this.opponentDeviceId = opts.opponentDeviceId; + this.opponentSessionId = opts.opponentSessionId; + this.groupCallId = opts.groupCallId; + // Array of Objects with urls, username, credential keys + this.turnServers = opts.turnServers || []; + if (this.turnServers.length === 0 && this.client.isFallbackICEServerAllowed()) { + this.turnServers.push({ + urls: [FALLBACK_ICE_SERVER], + }); + } + for (const server of this.turnServers) { + utils.checkObjectHasKeys(server, ["urls"]); + } + this.callId = genCallID(); + } + + /** + * Place a voice call to this room. + * @throws If you have not specified a listener for 'error' events. + */ + public async placeVoiceCall(): Promise { + await this.placeCall(true, false); + } + + /** + * Place a video call to this room. + * @throws If you have not specified a listener for 'error' events. + */ + public async placeVideoCall(): Promise { + await this.placeCall(true, true); + } + + /** + * Create a datachannel using this call's peer connection. + * @param label A human readable label for this datachannel + * @param options An object providing configuration options for the data channel. + */ + public createDataChannel(label: string, options: RTCDataChannelInit) { + const dataChannel = this.peerConn.createDataChannel(label, options); + this.emit(CallEvent.DataChannel, dataChannel); + return dataChannel; + } + + public getOpponentMember(): RoomMember { + return this.opponentMember; + } + + public getOpponentSessionId(): string { + return this.opponentSessionId; + } + + public opponentCanBeTransferred(): boolean { + return Boolean(this.opponentCaps && this.opponentCaps["m.call.transferee"]); + } + + public opponentSupportsDTMF(): boolean { + return Boolean(this.opponentCaps && this.opponentCaps["m.call.dtmf"]); + } + + public getRemoteAssertedIdentity(): AssertedIdentity { + return this.remoteAssertedIdentity; + } + + public get type(): CallType { + return (this.hasLocalUserMediaVideoTrack || this.hasRemoteUserMediaVideoTrack) + ? CallType.Video + : CallType.Voice; + } + + public get hasLocalUserMediaVideoTrack(): boolean { + return this.localUsermediaStream?.getVideoTracks().length > 0; + } + + public get hasRemoteUserMediaVideoTrack(): boolean { + return this.getRemoteFeeds().some((feed) => { + return ( + feed.purpose === SDPStreamMetadataPurpose.Usermedia && + feed.stream.getVideoTracks().length > 0 + ); + }); + } + + public get hasLocalUserMediaAudioTrack(): boolean { + return this.localUsermediaStream?.getAudioTracks().length > 0; + } + + public get hasRemoteUserMediaAudioTrack(): boolean { + return this.getRemoteFeeds().some((feed) => { + return ( + feed.purpose === SDPStreamMetadataPurpose.Usermedia && + feed.stream.getAudioTracks().length > 0 + ); + }); + } + + public get localUsermediaFeed(): CallFeed { + return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); + } + + public get localScreensharingFeed(): CallFeed { + return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); + } + + public get localUsermediaStream(): MediaStream { + return this.localUsermediaFeed?.stream; + } + + public get localScreensharingStream(): MediaStream { + return this.localScreensharingFeed?.stream; + } + + public get remoteUsermediaFeed(): CallFeed { + return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); + } + + public get remoteScreensharingFeed(): CallFeed { + return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); + } + + public get remoteUsermediaStream(): MediaStream { + return this.remoteUsermediaFeed?.stream; + } + + public get remoteScreensharingStream(): MediaStream { + return this.remoteScreensharingFeed?.stream; + } + + private getFeedByStreamId(streamId: string): CallFeed { + return this.getFeeds().find((feed) => feed.stream.id === streamId); + } + + /** + * Returns an array of all CallFeeds + * @returns {Array} CallFeeds + */ + public getFeeds(): Array { + return this.feeds; + } + + /** + * Returns an array of all local CallFeeds + * @returns {Array} local CallFeeds + */ + public getLocalFeeds(): Array { + return this.feeds.filter((feed) => feed.isLocal()); + } + + /** + * Returns an array of all remote CallFeeds + * @returns {Array} remote CallFeeds + */ + public getRemoteFeeds(): Array { + return this.feeds.filter((feed) => !feed.isLocal()); + } + + /** + * Generates and returns localSDPStreamMetadata + * @returns {SDPStreamMetadata} localSDPStreamMetadata + */ + private getLocalSDPStreamMetadata(updateStreamIds = false): SDPStreamMetadata { + const metadata: SDPStreamMetadata = {}; + for (const localFeed of this.getLocalFeeds()) { + if (updateStreamIds) { + localFeed.sdpMetadataStreamId = localFeed.stream.id; + } + + metadata[localFeed.sdpMetadataStreamId] = { + purpose: localFeed.purpose, + audio_muted: localFeed.isAudioMuted(), + video_muted: localFeed.isVideoMuted(), + }; + } + return metadata; + } + + /** + * Returns true if there are no incoming feeds, + * otherwise returns false + * @returns {boolean} no incoming feeds + */ + public noIncomingFeeds(): boolean { + return !this.feeds.some((feed) => !feed.isLocal()); + } + + private pushRemoteFeed(stream: MediaStream): void { + // Fallback to old behavior if the other side doesn't support SDPStreamMetadata + if (!this.opponentSupportsSDPStreamMetadata()) { + this.pushRemoteFeedWithoutMetadata(stream); + return; + } + + const userId = this.getOpponentMember().userId; + const purpose = this.remoteSDPStreamMetadata[stream.id].purpose; + const audioMuted = this.remoteSDPStreamMetadata[stream.id].audio_muted; + const videoMuted = this.remoteSDPStreamMetadata[stream.id].video_muted; + + if (!purpose) { + logger.warn(`Ignoring stream with id ${stream.id} because we didn't get any metadata about it`); + return; + } + + // Try to find a feed with the same purpose as the new stream, + // if we find it replace the old stream with the new one + const existingFeed = this.getRemoteFeeds().find((feed) => feed.purpose === purpose); + if (existingFeed) { + existingFeed.setNewStream(stream); + } else { + this.feeds.push(new CallFeed({ + client: this.client, + roomId: this.roomId, + userId, + stream, + purpose, + audioMuted, + videoMuted, + })); + this.emit(CallEvent.FeedsChanged, this.feeds); + } + + logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}", purpose=${purpose})`); + } + + /** + * This method is used ONLY if the other client doesn't support sending SDPStreamMetadata + */ + private pushRemoteFeedWithoutMetadata(stream: MediaStream): void { + const userId = this.getOpponentMember().userId; + // We can guess the purpose here since the other client can only send one stream + const purpose = SDPStreamMetadataPurpose.Usermedia; + const oldRemoteStream = this.feeds.find((feed) => !feed.isLocal())?.stream; + + // Note that we check by ID and always set the remote stream: Chrome appears + // to make new stream objects when transceiver directionality is changed and the 'active' + // status of streams change - Dave + // If we already have a stream, check this stream has the same id + if (oldRemoteStream && stream.id !== oldRemoteStream.id) { + logger.warn(`Ignoring new stream ID ${stream.id}: we already have stream ID ${oldRemoteStream.id}`); + return; + } + + // Try to find a feed with the same stream id as the new stream, + // if we find it replace the old stream with the new one + const feed = this.getFeedByStreamId(stream.id); + if (feed) { + feed.setNewStream(stream); + } else { + this.feeds.push(new CallFeed({ + client: this.client, + roomId: this.roomId, + audioMuted: false, + videoMuted: false, + userId, + stream, + purpose, + })); + this.emit(CallEvent.FeedsChanged, this.feeds); + } + + logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}")`); + } + + private pushNewLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose, addToPeerConnection = true): void { + const userId = this.client.getUserId(); + + // TODO: Find out what is going on here + // why do we enable audio (and only audio) tracks here? -- matthew + setTracksEnabled(stream.getAudioTracks(), true); + + // We try to replace an existing feed if there already is one with the same purpose + const existingFeed = this.getLocalFeeds().find((feed) => feed.purpose === purpose); + if (existingFeed) { + existingFeed.setNewStream(stream); + } else { + this.pushLocalFeed( + new CallFeed({ + client: this.client, + roomId: this.roomId, + audioMuted: stream.getAudioTracks().length === 0, + videoMuted: stream.getVideoTracks().length === 0, + userId, + stream, + purpose, + }), + addToPeerConnection, + ); + this.emit(CallEvent.FeedsChanged, this.feeds); + } + } + + /** + * Pushes supplied feed to the call + * @param {CallFeed} callFeed to push + * @param {boolean} addToPeerConnection whether to add the tracks to the peer connection + */ + public pushLocalFeed(callFeed: CallFeed, addToPeerConnection = true): void { + if (this.feeds.some((feed) => callFeed.stream.id === feed.stream.id)) { + logger.info(`Ignoring duplicate local stream ${callFeed.stream.id} in call ${this.callId}`); + return; + } + + this.feeds.push(callFeed); + + if (addToPeerConnection) { + const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia ? + this.usermediaSenders : this.screensharingSenders; + // Empty the array + senderArray.splice(0, senderArray.length); + + for (const track of callFeed.stream.getTracks()) { + logger.info( + `Adding track (` + + `id="${track.id}", ` + + `kind="${track.kind}", ` + + `streamId="${callFeed.stream.id}", ` + + `streamPurpose="${callFeed.purpose}"` + + `) to peer connection`, + ); + senderArray.push(this.peerConn.addTrack(track, callFeed.stream)); + } + } + + logger.info( + `Pushed local stream `+ + `(id="${callFeed.stream.id}", `+ + `active="${callFeed.stream.active}", `+ + `purpose="${callFeed.purpose}")`, + ); + + this.emit(CallEvent.FeedsChanged, this.feeds); + } + + /** + * Removes local call feed from the call and its tracks from the peer + * connection + * @param callFeed to remove + */ + public removeLocalFeed(callFeed: CallFeed): void { + const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia + ? this.usermediaSenders + : this.screensharingSenders; + + for (const sender of senderArray) { + this.peerConn.removeTrack(sender); + } + + if (callFeed.purpose === SDPStreamMetadataPurpose.Screenshare) { + this.client.getMediaHandler().stopScreensharingStream(callFeed.stream); + } + + // Empty the array + senderArray.splice(0, senderArray.length); + this.deleteFeed(callFeed); + } + + private deleteAllFeeds(): void { + for (const feed of this.feeds) { + if (!feed.isLocal() || !this.groupCallId) { + feed.dispose(); + } + } + + this.feeds = []; + this.emit(CallEvent.FeedsChanged, this.feeds); + } + + private deleteFeedByStream(stream: MediaStream): void { + const feed = this.getFeedByStreamId(stream.id); + if (!feed) { + logger.warn(`Didn't find the feed with stream id ${stream.id} to delete`); + return; + } + this.deleteFeed(feed); + } + + private deleteFeed(feed: CallFeed): void { + feed.dispose(); + this.feeds.splice(this.feeds.indexOf(feed), 1); + this.emit(CallEvent.FeedsChanged, this.feeds); + } + + // The typescript definitions have this type as 'any' :( + public async getCurrentCallStats(): Promise { + if (this.callHasEnded()) { + return this.callStatsAtEnd; + } + + return this.collectCallStats(); + } + + private async collectCallStats(): Promise { + // This happens when the call fails before it starts. + // For example when we fail to get capture sources + if (!this.peerConn) return; + + const statsReport = await this.peerConn.getStats(); + const stats = []; + for (const item of statsReport) { + stats.push(item[1]); + } + + return stats; + } + + /** + * Configure this call from an invite event. Used by MatrixClient. + * @param {MatrixEvent} event The m.call.invite event + */ + public async initWithInvite(event: MatrixEvent): Promise { + const invite = event.getContent(); + this.direction = CallDirection.Inbound; + + // make sure we have valid turn creds. Unless something's gone wrong, it should + // poll and keep the credentials valid so this should be instant. + const haveTurnCreds = await this.client.checkTurnServers(); + if (!haveTurnCreds) { + logger.warn("Failed to get TURN credentials! Proceeding with call anyway..."); + } + + const sdpStreamMetadata = invite[SDPStreamMetadataKey]; + if (sdpStreamMetadata) { + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); + } else { + logger.debug("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); + } + + this.peerConn = this.createPeerConnection(); + // we must set the party ID before await-ing on anything: the call event + // handler will start giving us more call events (eg. candidates) so if + // we haven't set the party ID, we'll ignore them. + this.chooseOpponent(event); + try { + await this.peerConn.setRemoteDescription(invite.offer); + await this.addBufferedIceCandidates(); + } catch (e) { + logger.debug("Failed to set remote description", e); + this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); + return; + } + + const remoteStream = this.feeds.find((feed) => !feed.isLocal())?.stream; + + // According to previous comments in this file, firefox at some point did not + // add streams until media started arriving on them. Testing latest firefox + // (81 at time of writing), this is no longer a problem, so let's do it the correct way. + if (!remoteStream || remoteStream.getTracks().length === 0) { + logger.error("No remote stream or no tracks after setting remote description!"); + this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); + return; + } + + this.setState(CallState.Ringing); + + if (event.getLocalAge()) { + setTimeout(() => { + if (this.state == CallState.Ringing) { + logger.debug("Call invite has expired. Hanging up."); + this.hangupParty = CallParty.Remote; // effectively + this.setState(CallState.Ended); + this.stopAllMedia(); + if (this.peerConn.signalingState != 'closed') { + this.peerConn.close(); + } + this.emit(CallEvent.Hangup); + } + }, invite.lifetime - event.getLocalAge()); + } + } + + /** + * Configure this call from a hangup or reject event. Used by MatrixClient. + * @param {MatrixEvent} event The m.call.hangup event + */ + public initWithHangup(event: MatrixEvent): void { + // perverse as it may seem, sometimes we want to instantiate a call with a + // hangup message (because when getting the state of the room on load, events + // come in reverse order and we want to remember that a call has been hung up) + this.setState(CallState.Ended); + } + + private shouldAnswerWithMediaType( + wantedValue: boolean | undefined, valueOfTheOtherSide: boolean | undefined, type: "audio" | "video", + ): boolean { + if (wantedValue && !valueOfTheOtherSide) { + // TODO: Figure out how to do this + logger.warn(`Unable to answer with ${type} because the other side isn't sending it either.`); + return false; + } else if ( + !utils.isNullOrUndefined(wantedValue) && + wantedValue !== valueOfTheOtherSide && + !this.opponentSupportsSDPStreamMetadata() + ) { + logger.warn( + `Unable to answer with ${type}=${wantedValue} because the other side doesn't support it. ` + + `Answering with ${type}=${valueOfTheOtherSide}.`, + ); + return valueOfTheOtherSide; + } + return wantedValue ?? valueOfTheOtherSide; + } + + /** + * Answer a call. + */ + public async answer(audio?: boolean, video?: boolean): Promise { + if (this.inviteOrAnswerSent) return; + // TODO: Figure out how to do this + if (audio === false && video === false) throw new Error("You CANNOT answer a call without media"); + + if (!this.localUsermediaStream && !this.waitForLocalAVStream) { + const prevState = this.state; + const answerWithAudio = this.shouldAnswerWithMediaType(audio, this.hasRemoteUserMediaAudioTrack, "audio"); + const answerWithVideo = this.shouldAnswerWithMediaType(video, this.hasRemoteUserMediaVideoTrack, "video"); + + this.setState(CallState.WaitLocalMedia); + this.waitForLocalAVStream = true; + + try { + const stream = await this.client.getMediaHandler().getUserMediaStream( + answerWithAudio, answerWithVideo, + ); + this.waitForLocalAVStream = false; + const usermediaFeed = new CallFeed({ + client: this.client, + roomId: this.roomId, + userId: this.client.getUserId(), + stream, + purpose: SDPStreamMetadataPurpose.Usermedia, + audioMuted: stream.getAudioTracks().length === 0, + videoMuted: stream.getVideoTracks().length === 0, + }); + + const feeds = [usermediaFeed]; + + if (this.localScreensharingFeed) { + feeds.push(this.localScreensharingFeed); + } + + this.answerWithCallFeeds(feeds); + } catch (e) { + if (answerWithVideo) { + // Try to answer without video + logger.warn("Failed to getUserMedia(), trying to getUserMedia() without video"); + this.setState(prevState); + this.waitForLocalAVStream = false; + await this.answer(answerWithAudio, false); + } else { + this.getUserMediaFailed(e); + return; + } + } + } else if (this.waitForLocalAVStream) { + this.setState(CallState.WaitLocalMedia); + } + } + + public answerWithCallFeeds(callFeeds: CallFeed[]): void { + if (this.inviteOrAnswerSent) return; + + logger.debug(`Answering call ${this.callId}`); + + this.queueGotCallFeedsForAnswer(callFeeds); + } + + /** + * Replace this call with a new call, e.g. for glare resolution. Used by + * MatrixClient. + * @param {MatrixCall} newCall The new call. + */ + public replacedBy(newCall: MatrixCall): void { + if (this.state === CallState.WaitLocalMedia) { + logger.debug("Telling new call to wait for local media"); + newCall.waitForLocalAVStream = true; + } else if ([CallState.CreateOffer, CallState.InviteSent].includes(this.state)) { + if (newCall.direction === CallDirection.Outbound) { + newCall.queueGotCallFeedsForAnswer([]); + } else { + logger.debug("Handing local stream to new call"); + newCall.queueGotCallFeedsForAnswer(this.getLocalFeeds().map(feed => feed.clone())); + } + } + this.successor = newCall; + this.emit(CallEvent.Replaced, newCall); + this.hangup(CallErrorCode.Replaced, true); + } + + /** + * Hangup a call. + * @param {string} reason The reason why the call is being hung up. + * @param {boolean} suppressEvent True to suppress emitting an event. + */ + public hangup(reason: CallErrorCode, suppressEvent: boolean): void { + if (this.callHasEnded()) return; + + logger.debug("Ending call " + this.callId); + this.terminate(CallParty.Local, reason, !suppressEvent); + // We don't want to send hangup here if we didn't even get to sending an invite + if (this.state === CallState.WaitLocalMedia) return; + const content = {}; + // Don't send UserHangup reason to older clients + if ((this.opponentVersion && this.opponentVersion >= 1) || reason !== CallErrorCode.UserHangup) { + content["reason"] = reason; + } + this.sendVoipEvent(EventType.CallHangup, content); + } + + /** + * Reject a call + * This used to be done by calling hangup, but is a separate method and protocol + * event as of MSC2746. + */ + public reject(): void { + if (this.state !== CallState.Ringing) { + throw Error("Call must be in 'ringing' state to reject!"); + } + + if (this.opponentVersion < 1) { + logger.info( + `Opponent version is less than 1 (${this.opponentVersion}): sending hangup instead of reject`, + ); + this.hangup(CallErrorCode.UserHangup, true); + return; + } + + logger.debug("Rejecting call: " + this.callId); + this.terminate(CallParty.Local, CallErrorCode.UserHangup, true); + this.sendVoipEvent(EventType.CallReject, {}); + } + + /** + * Adds an audio and/or video track - upgrades the call + * @param {boolean} audio should add an audio track + * @param {boolean} video should add an video track + */ + private async upgradeCall( + audio: boolean, video: boolean, + ): Promise { + // We don't do call downgrades + if (!audio && !video) return; + if (!this.opponentSupportsSDPStreamMetadata()) return; + + try { + const upgradeAudio = audio && !this.hasLocalUserMediaAudioTrack; + const upgradeVideo = video && !this.hasLocalUserMediaVideoTrack; + logger.debug(`Upgrading call: audio?=${upgradeAudio} video?=${upgradeVideo}`); + + const stream = await this.client.getMediaHandler().getUserMediaStream(upgradeAudio, upgradeVideo); + if (upgradeAudio && upgradeVideo) { + if (this.hasLocalUserMediaAudioTrack) return; + if (this.hasLocalUserMediaVideoTrack) return; + + this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); + } else if (upgradeAudio) { + if (this.hasLocalUserMediaAudioTrack) return; + + const audioTrack = stream.getAudioTracks()[0]; + this.localUsermediaStream.addTrack(audioTrack); + this.peerConn.addTrack(audioTrack, this.localUsermediaStream); + } else if (upgradeVideo) { + if (this.hasLocalUserMediaVideoTrack) return; + + const videoTrack = stream.getVideoTracks()[0]; + this.localUsermediaStream.addTrack(videoTrack); + this.peerConn.addTrack(videoTrack, this.localUsermediaStream); + } + } catch (error) { + logger.error("Failed to upgrade the call", error); + this.emit(CallEvent.Error, + new CallError(CallErrorCode.NoUserMedia, "Failed to get camera access: ", error), + ); + } + } + + /** + * Returns true if this.remoteSDPStreamMetadata is defined, otherwise returns false + * @returns {boolean} can screenshare + */ + public opponentSupportsSDPStreamMetadata(): boolean { + return Boolean(this.remoteSDPStreamMetadata); + } + + /** + * If there is a screensharing stream returns true, otherwise returns false + * @returns {boolean} is screensharing + */ + public isScreensharing(): boolean { + return Boolean(this.localScreensharingStream); + } + + /** + * Starts/stops screensharing + * @param enabled the desired screensharing state + * @param {string} desktopCapturerSourceId optional id of the desktop capturer source to use + * @returns {boolean} new screensharing state + */ + public async setScreensharingEnabled( + enabled: boolean, desktopCapturerSourceId?: string, + ): Promise { + // Skip if there is nothing to do + if (enabled && this.isScreensharing()) { + logger.warn(`There is already a screensharing stream - there is nothing to do!`); + return true; + } else if (!enabled && !this.isScreensharing()) { + logger.warn(`There already isn't a screensharing stream - there is nothing to do!`); + return false; + } + + // Fallback to replaceTrack() + if (!this.opponentSupportsSDPStreamMetadata()) { + return await this.setScreensharingEnabledWithoutMetadataSupport(enabled, desktopCapturerSourceId); + } + + logger.debug(`Set screensharing enabled? ${enabled}`); + if (enabled) { + try { + const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId); + if (!stream) return false; + this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare); + return true; + } catch (err) { + logger.error("Failed to get screen-sharing stream:", err); + return false; + } + } else { + for (const sender of this.screensharingSenders) { + this.peerConn.removeTrack(sender); + } + this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream); + this.deleteFeedByStream(this.localScreensharingStream); + return false; + } + } + + /** + * Starts/stops screensharing + * Should be used ONLY if the opponent doesn't support SDPStreamMetadata + * @param enabled the desired screensharing state + * @param {string} desktopCapturerSourceId optional id of the desktop capturer source to use + * @returns {boolean} new screensharing state + */ + private async setScreensharingEnabledWithoutMetadataSupport( + enabled: boolean, desktopCapturerSourceId?: string, + ): Promise { + logger.debug(`Set screensharing enabled? ${enabled} using replaceTrack()`); + if (enabled) { + try { + const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId); + if (!stream) return false; + + const track = stream.getTracks().find((track) => { + return track.kind === "video"; + }); + const sender = this.usermediaSenders.find((sender) => { + return sender.track?.kind === "video"; + }); + sender.replaceTrack(track); + + this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare, false); + + return true; + } catch (err) { + logger.error("Failed to get screen-sharing stream:", err); + return false; + } + } else { + const track = this.localUsermediaStream.getTracks().find((track) => { + return track.kind === "video"; + }); + const sender = this.usermediaSenders.find((sender) => { + return sender.track?.kind === "video"; + }); + sender.replaceTrack(track); + + this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream); + this.deleteFeedByStream(this.localScreensharingStream); + + return false; + } + } + + /** + * Request a new local usermedia stream with the current device id. + */ + public async updateLocalUsermediaStream(stream: MediaStream) { + const callFeed = this.localUsermediaFeed; + callFeed.setNewStream(stream); + const micShouldBeMuted = callFeed.isAudioMuted() || this.remoteOnHold; + const vidShouldBeMuted = callFeed.isVideoMuted() || this.remoteOnHold; + setTracksEnabled(stream.getAudioTracks(), !micShouldBeMuted); + setTracksEnabled(stream.getVideoTracks(), !vidShouldBeMuted); + + const newSenders = []; + + for (const track of stream.getTracks()) { + const oldSender = this.usermediaSenders.find((sender) => { + return sender.track?.kind === track.kind; + }); + + let newSender: RTCRtpSender; + + try { + logger.info( + `Replacing track (` + + `id="${track.id}", ` + + `kind="${track.kind}", ` + + `streamId="${stream.id}", ` + + `streamPurpose="${callFeed.purpose}"` + + `) to peer connection`, + ); + await oldSender.replaceTrack(track); + newSender = oldSender; + } catch (error) { + logger.info( + `Adding track (` + + `id="${track.id}", ` + + `kind="${track.kind}", ` + + `streamId="${stream.id}", ` + + `streamPurpose="${callFeed.purpose}"` + + `) to peer connection`, + ); + newSender = this.peerConn.addTrack(track, stream); + } + + newSenders.push(newSender); + } + + this.usermediaSenders = newSenders; + } + + /** + * Set whether our outbound video should be muted or not. + * @param {boolean} muted True to mute the outbound video. + * @returns the new mute state + */ + public async setLocalVideoMuted(muted: boolean): Promise { + if (!await this.client.getMediaHandler().hasVideoDevice()) { + return this.isLocalVideoMuted(); + } + + if (!this.hasLocalUserMediaVideoTrack && !muted) { + await this.upgradeCall(false, true); + return this.isLocalVideoMuted(); + } + this.localUsermediaFeed?.setVideoMuted(muted); + this.updateMuteStatus(); + return this.isLocalVideoMuted(); + } + + /** + * Check if local video is muted. + * + * If there are multiple video tracks, all of the tracks need to be muted + * for this to return true. This means if there are no video tracks, this will + * return true. + * @return {Boolean} True if the local preview video is muted, else false + * (including if the call is not set up yet). + */ + public isLocalVideoMuted(): boolean { + return this.localUsermediaFeed?.isVideoMuted(); + } + + /** + * Set whether the microphone should be muted or not. + * @param {boolean} muted True to mute the mic. + * @returns the new mute state + */ + public async setMicrophoneMuted(muted: boolean): Promise { + if (!await this.client.getMediaHandler().hasAudioDevice()) { + return this.isMicrophoneMuted(); + } + + if (!this.hasLocalUserMediaAudioTrack && !muted) { + await this.upgradeCall(true, false); + return this.isMicrophoneMuted(); + } + this.localUsermediaFeed?.setAudioMuted(muted); + this.updateMuteStatus(); + return this.isMicrophoneMuted(); + } + + /** + * Check if the microphone is muted. + * + * If there are multiple audio tracks, all of the tracks need to be muted + * for this to return true. This means if there are no audio tracks, this will + * return true. + * @return {Boolean} True if the mic is muted, else false (including if the call + * is not set up yet). + */ + public isMicrophoneMuted(): boolean { + return this.localUsermediaFeed?.isAudioMuted(); + } + + /** + * @returns true if we have put the party on the other side of the call on hold + * (that is, we are signalling to them that we are not listening) + */ + public isRemoteOnHold(): boolean { + return this.remoteOnHold; + } + + public setRemoteOnHold(onHold: boolean): void { + if (this.isRemoteOnHold() === onHold) return; + this.remoteOnHold = onHold; + + for (const transceiver of this.peerConn.getTransceivers()) { + // We don't send hold music or anything so we're not actually + // sending anything, but sendrecv is fairly standard for hold and + // it makes it a lot easier to figure out who's put who on hold. + transceiver.direction = onHold ? 'sendonly' : 'sendrecv'; + } + this.updateMuteStatus(); + + this.emit(CallEvent.RemoteHoldUnhold, this.remoteOnHold); + } + + /** + * Indicates whether we are 'on hold' to the remote party (ie. if true, + * they cannot hear us). + * @returns true if the other party has put us on hold + */ + public isLocalOnHold(): boolean { + if (this.state !== CallState.Connected) return false; + + let callOnHold = true; + + // We consider a call to be on hold only if *all* the tracks are on hold + // (is this the right thing to do?) + for (const transceiver of this.peerConn.getTransceivers()) { + const trackOnHold = ['inactive', 'recvonly'].includes(transceiver.currentDirection); + + if (!trackOnHold) callOnHold = false; + } + + return callOnHold; + } + + /** + * Sends a DTMF digit to the other party + * @param digit The digit (nb. string - '#' and '*' are dtmf too) + */ + public sendDtmfDigit(digit: string): void { + for (const sender of this.peerConn.getSenders()) { + if (sender.track.kind === 'audio' && sender.dtmf) { + sender.dtmf.insertDTMF(digit); + return; + } + } + + throw new Error("Unable to find a track to send DTMF on"); + } + + private updateMuteStatus(): void { + this.sendVoipEvent(EventType.CallSDPStreamMetadataChangedPrefix, { + [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(), + }); + + const micShouldBeMuted = this.localUsermediaFeed?.isAudioMuted() || this.remoteOnHold; + const vidShouldBeMuted = this.localUsermediaFeed?.isVideoMuted() || this.remoteOnHold; + + setTracksEnabled(this.localUsermediaStream.getAudioTracks(), !micShouldBeMuted); + setTracksEnabled(this.localUsermediaStream.getVideoTracks(), !vidShouldBeMuted); + } + + private gotCallFeedsForInvite(callFeeds: CallFeed[], requestScreenshareFeed = false): void { + if (this.successor) { + this.successor.queueGotCallFeedsForAnswer(callFeeds); + return; + } + if (this.callHasEnded()) { + this.stopAllMedia(); + return; + } + + for (const feed of callFeeds) { + this.pushLocalFeed(feed); + } + + if (requestScreenshareFeed) { + this.peerConn.addTransceiver("video", { + direction: "recvonly", + }); + } + + this.setState(CallState.CreateOffer); + + logger.debug("gotUserMediaForInvite"); + // Now we wait for the negotiationneeded event + } + + private async sendAnswer(): Promise { + const answerContent = { + answer: { + sdp: this.peerConn.localDescription.sdp, + // type is now deprecated as of Matrix VoIP v1, but + // required to still be sent for backwards compat + type: this.peerConn.localDescription.type, + }, + [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(true), + } as MCallAnswer; + + answerContent.capabilities = { + 'm.call.transferee': this.client.supportsCallTransfer, + 'm.call.dtmf': false, + }; + + // We have just taken the local description from the peerConn which will + // contain all the local candidates added so far, so we can discard any candidates + // we had queued up because they'll be in the answer. + logger.info(`Discarding ${this.candidateSendQueue.length} candidates that will be sent in answer`); + this.candidateSendQueue = []; + + try { + await this.sendVoipEvent(EventType.CallAnswer, answerContent); + // If this isn't the first time we've tried to send the answer, + // we may have candidates queued up, so send them now. + this.inviteOrAnswerSent = true; + } catch (error) { + // We've failed to answer: back to the ringing state + this.setState(CallState.Ringing); + this.client.cancelPendingEvent(error.event); + + let code = CallErrorCode.SendAnswer; + let message = "Failed to send answer"; + if (error.name == 'UnknownDeviceError') { + code = CallErrorCode.UnknownDevices; + message = "Unknown devices present in the room"; + } + this.emit(CallEvent.Error, new CallError(code, message, error)); + throw error; + } + + // error handler re-throws so this won't happen on error, but + // we don't want the same error handling on the candidate queue + this.sendCandidateQueue(); + } + + private queueGotCallFeedsForAnswer(callFeeds: CallFeed[]): void { + // Ensure only one negotiate/answer event is being processed at a time. + if (this.responsePromiseChain) { + this.responsePromiseChain = + this.responsePromiseChain.then(() => this.gotCallFeedsForAnswer(callFeeds)); + } else { + this.responsePromiseChain = this.gotCallFeedsForAnswer(callFeeds); + } + } + + private async gotCallFeedsForAnswer(callFeeds: CallFeed[]): Promise { + if (this.callHasEnded()) return; + + this.waitForLocalAVStream = false; + + for (const feed of callFeeds) { + this.pushLocalFeed(feed); + } + + this.setState(CallState.CreateAnswer); + + let myAnswer; + try { + this.getRidOfRTXCodecs(); + myAnswer = await this.peerConn.createAnswer(); + } catch (err) { + logger.debug("Failed to create answer: ", err); + this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true); + return; + } + + try { + await this.peerConn.setLocalDescription(myAnswer); + this.setState(CallState.Connecting); + + // Allow a short time for initial candidates to be gathered + await new Promise(resolve => { + setTimeout(resolve, 200); + }); + + this.sendAnswer(); + } catch (err) { + logger.debug("Error setting local description!", err); + this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); + return; + } + + // HACK: Safari doesn't like it when we reuse MediaStreams. In most cases + // we can get around this by calling MediaStream.clone(), however inbound + // calls seem to still be broken unless we getUserMedia again and replace + // all MediaStreams using sender.replaceTrack + if (isSafari) { + await new Promise(resolve => { + setTimeout(resolve, 200); + }); + + if (this.state === CallState.Ended) { + return; + } + + const callFeed = this.localUsermediaFeed; + const stream = callFeed.stream; + + if (!stream.active) { + throw new Error(`Call ${this.callId} has an inactive stream ${ + stream.id} and its tracks cannot be replaced`); + } + + const newSenders = []; + + for (const track of this.localUsermediaStream.getTracks()) { + const oldSender = this.usermediaSenders.find((sender) => { + return sender.track?.kind === track.kind; + }); + + if (track.readyState === "ended") { + throw new Error(`Call ${this.callId} tried to replace track ${track.id} in the ended state`); + } + + let newSender: RTCRtpSender; + + try { + logger.info( + `Replacing track (` + + `id="${track.id}", ` + + `kind="${track.kind}", ` + + `streamId="${stream.id}", ` + + `streamPurpose="${callFeed.purpose}"` + + `) to peer connection`, + ); + await oldSender.replaceTrack(track); + newSender = oldSender; + } catch (error) { + logger.info( + `Adding track (` + + `id="${track.id}", ` + + `kind="${track.kind}", ` + + `streamId="${stream.id}", ` + + `streamPurpose="${callFeed.purpose}"` + + `) to peer connection`, + ); + newSender = this.peerConn.addTrack(track, stream); + } + + newSenders.push(newSender); + } + + this.usermediaSenders = newSenders; + } + } + + /** + * Internal + * @param {Object} event + */ + private gotLocalIceCandidate = (event: RTCPeerConnectionIceEvent): Promise => { + if (event.candidate) { + logger.debug( + "Call " + this.callId + " got local ICE " + event.candidate.sdpMid + " candidate: " + + event.candidate.candidate, + ); + + if (this.callHasEnded()) return; + + // As with the offer, note we need to make a copy of this object, not + // pass the original: that broke in Chrome ~m43. + if (event.candidate.candidate !== '' || !this.sentEndOfCandidates) { + this.queueCandidate(event.candidate); + + if (event.candidate.candidate === '') this.sentEndOfCandidates = true; + } + } + }; + + private onIceGatheringStateChange = (event: Event): void => { + logger.debug("ice gathering state changed to " + this.peerConn.iceGatheringState); + if (this.peerConn.iceGatheringState === 'complete' && !this.sentEndOfCandidates) { + // If we didn't get an empty-string candidate to signal the end of candidates, + // create one ourselves now gathering has finished. + // We cast because the interface lists all the properties as required but we + // only want to send 'candidate' + // XXX: We probably want to send either sdpMid or sdpMLineIndex, as it's not strictly + // correct to have a candidate that lacks both of these. We'd have to figure out what + // previous candidates had been sent with and copy them. + const c = { + candidate: '', + } as RTCIceCandidate; + this.queueCandidate(c); + this.sentEndOfCandidates = true; + } + }; + + public async onRemoteIceCandidatesReceived(ev: MatrixEvent): Promise { + if (this.callHasEnded()) { + //debuglog("Ignoring remote ICE candidate because call has ended"); + return; + } + + const content = ev.getContent(); + const candidates = content.candidates; + if (!candidates) { + logger.info("Ignoring candidates event with no candidates!"); + return; + } + + const fromPartyId = content.version === 0 ? null : content.party_id || null; + + if (this.opponentPartyId === undefined) { + // we haven't picked an opponent yet so save the candidates + logger.info(`Buffering ${candidates.length} candidates until we pick an opponent`); + const bufferedCandidates = this.remoteCandidateBuffer.get(fromPartyId) || []; + bufferedCandidates.push(...candidates); + this.remoteCandidateBuffer.set(fromPartyId, bufferedCandidates); + return; + } + + if (!this.partyIdMatches(content)) { + logger.info( + `Ignoring candidates from party ID ${content.party_id}: ` + + `we have chosen party ID ${this.opponentPartyId}`, + ); + + return; + } + + await this.addIceCandidates(candidates); + } + + /** + * Used by MatrixClient. + * @param {Object} msg + */ + public async onAnswerReceived(event: MatrixEvent): Promise { + const content = event.getContent(); + logger.debug(`Got answer for call ID ${this.callId} from party ID ${content.party_id}`); + + if (this.callHasEnded()) { + logger.debug(`Ignoring answer because call ID ${this.callId} has ended`); + return; + } + + if (this.opponentPartyId !== undefined) { + logger.info( + `Ignoring answer from party ID ${content.party_id}: ` + + `we already have an answer/reject from ${this.opponentPartyId}`, + ); + return; + } + + this.chooseOpponent(event); + await this.addBufferedIceCandidates(); + + this.setState(CallState.Connecting); + + const sdpStreamMetadata = content[SDPStreamMetadataKey]; + if (sdpStreamMetadata) { + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); + } else { + logger.warn("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); + } + + try { + await this.peerConn.setRemoteDescription(content.answer); + } catch (e) { + logger.debug("Failed to set remote description", e); + this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); + return; + } + + // If the answer we selected has a party_id, send a select_answer event + // We do this after setting the remote description since otherwise we'd block + // call setup on it + if (this.opponentPartyId !== null) { + try { + await this.sendVoipEvent(EventType.CallSelectAnswer, { + selected_party_id: this.opponentPartyId, + }); + } catch (err) { + // This isn't fatal, and will just mean that if another party has raced to answer + // the call, they won't know they got rejected, so we carry on & don't retry. + logger.warn("Failed to send select_answer event", err); + } + } + } + + public async onSelectAnswerReceived(event: MatrixEvent): Promise { + if (this.direction !== CallDirection.Inbound) { + logger.warn("Got select_answer for an outbound call: ignoring"); + return; + } + + const selectedPartyId = event.getContent().selected_party_id; + + if (selectedPartyId === undefined || selectedPartyId === null) { + logger.warn("Got nonsensical select_answer with null/undefined selected_party_id: ignoring"); + return; + } + + if (selectedPartyId !== this.ourPartyId) { + logger.info(`Got select_answer for party ID ${selectedPartyId}: we are party ID ${this.ourPartyId}.`); + // The other party has picked somebody else's answer + this.terminate(CallParty.Remote, CallErrorCode.AnsweredElsewhere, true); + } + } + + public async onNegotiateReceived(event: MatrixEvent): Promise { + const content = event.getContent(); + const description = content.description; + if (!description || !description.sdp || !description.type) { + logger.info("Ignoring invalid m.call.negotiate event"); + return; + } + // Politeness always follows the direction of the call: in a glare situation, + // we pick either the inbound or outbound call, so one side will always be + // inbound and one outbound + const polite = this.direction === CallDirection.Inbound; + + // Here we follow the perfect negotiation logic from + // https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation + const offerCollision = ( + (description.type === 'offer') && + (this.makingOffer || this.peerConn.signalingState !== 'stable') + ); + + this.ignoreOffer = !polite && offerCollision; + if (this.ignoreOffer) { + logger.info("Ignoring colliding negotiate event because we're impolite"); + return; + } + + const prevLocalOnHold = this.isLocalOnHold(); + + const sdpStreamMetadata = content[SDPStreamMetadataKey]; + if (sdpStreamMetadata) { + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); + } else { + logger.warn("Received negotiation event without SDPStreamMetadata!"); + } + + try { + await this.peerConn.setRemoteDescription(description); + + if (description.type === 'offer') { + this.getRidOfRTXCodecs(); + await this.peerConn.setLocalDescription(); + + this.sendVoipEvent(EventType.CallNegotiate, { + description: this.peerConn.localDescription, + [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(true), + }); + } + } catch (err) { + logger.warn("Failed to complete negotiation", err); + } + + const newLocalOnHold = this.isLocalOnHold(); + if (prevLocalOnHold !== newLocalOnHold) { + this.emit(CallEvent.LocalHoldUnhold, newLocalOnHold); + // also this one for backwards compat + this.emit(CallEvent.HoldUnhold, newLocalOnHold); + } + } + + private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { + this.remoteSDPStreamMetadata = utils.recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); + for (const feed of this.getRemoteFeeds()) { + const streamId = feed.stream.id; + feed.setAudioMuted(this.remoteSDPStreamMetadata[streamId]?.audio_muted); + feed.setVideoMuted(this.remoteSDPStreamMetadata[streamId]?.video_muted); + feed.purpose = this.remoteSDPStreamMetadata[streamId]?.purpose; + } + } + + public onSDPStreamMetadataChangedReceived(event: MatrixEvent): void { + const content = event.getContent(); + const metadata = content[SDPStreamMetadataKey]; + this.updateRemoteSDPStreamMetadata(metadata); + } + + public async onAssertedIdentityReceived(event: MatrixEvent): Promise { + const content = event.getContent(); + if (!content.asserted_identity) return; + + this.remoteAssertedIdentity = { + id: content.asserted_identity.id, + displayName: content.asserted_identity.display_name, + }; + this.emit(CallEvent.AssertedIdentityChanged); + } + + private callHasEnded(): boolean { + // This exists as workaround to typescript trying to be clever and erroring + // when putting if (this.state === CallState.Ended) return; twice in the same + // function, even though that function is async. + return this.state === CallState.Ended; + } + + private queueGotLocalOffer(): void { + // Ensure only one negotiate/answer event is being processed at a time. + if (this.responsePromiseChain) { + this.responsePromiseChain = + this.responsePromiseChain.then(() => this.wrappedGotLocalOffer()); + } else { + this.responsePromiseChain = this.wrappedGotLocalOffer(); + } + } + + private async wrappedGotLocalOffer(): Promise { + this.makingOffer = true; + try { + this.getRidOfRTXCodecs(); + await this.gotLocalOffer(); + } catch (e) { + this.getLocalOfferFailed(e); + return; + } finally { + this.makingOffer = false; + } + } + + private async gotLocalOffer(): Promise { + logger.debug("Setting local description"); + + if (this.callHasEnded()) { + logger.debug("Ignoring newly created offer on call ID " + this.callId + + " because the call has ended"); + return; + } + + try { + await this.peerConn.setLocalDescription(); + } catch (err) { + logger.debug("Error setting local description!", err); + this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); + return; + } + + if (this.peerConn.iceGatheringState === 'gathering') { + // Allow a short time for initial candidates to be gathered + await new Promise(resolve => { + setTimeout(resolve, 200); + }); + } + + if (this.callHasEnded()) return; + + const eventType = this.state === CallState.CreateOffer ? EventType.CallInvite : EventType.CallNegotiate; + + const content = { + lifetime: CALL_TIMEOUT_MS, + } as MCallInviteNegotiate; + + if (eventType === EventType.CallInvite && this.invitee) { + content.invitee = this.invitee; + } + + // clunky because TypeScript can't follow the types through if we use an expression as the key + if (this.state === CallState.CreateOffer) { + content.offer = this.peerConn.localDescription; + } else { + content.description = this.peerConn.localDescription; + } + + content.capabilities = { + 'm.call.transferee': this.client.supportsCallTransfer, + 'm.call.dtmf': false, + }; + + content[SDPStreamMetadataKey] = this.getLocalSDPStreamMetadata(true); + + // Get rid of any candidates waiting to be sent: they'll be included in the local + // description we just got and will send in the offer. + logger.info(`Discarding ${this.candidateSendQueue.length} candidates that will be sent in offer`); + this.candidateSendQueue = []; + + try { + await this.sendVoipEvent(eventType, content); + } catch (error) { + logger.error("Failed to send invite", error); + if (error.event) this.client.cancelPendingEvent(error.event); + + let code = CallErrorCode.SignallingFailed; + let message = "Signalling failed"; + if (this.state === CallState.CreateOffer) { + code = CallErrorCode.SendInvite; + message = "Failed to send invite"; + } + if (error.name == 'UnknownDeviceError') { + code = CallErrorCode.UnknownDevices; + message = "Unknown devices present in the room"; + } + + this.emit(CallEvent.Error, new CallError(code, message, error)); + this.terminate(CallParty.Local, code, false); + + // no need to carry on & send the candidate queue, but we also + // don't want to rethrow the error + return; + } + + this.sendCandidateQueue(); + if (this.state === CallState.CreateOffer) { + this.inviteOrAnswerSent = true; + this.setState(CallState.InviteSent); + this.inviteTimeout = setTimeout(() => { + this.inviteTimeout = null; + if (this.state === CallState.InviteSent) { + this.hangup(CallErrorCode.InviteTimeout, false); + } + }, CALL_TIMEOUT_MS); + } + } + + private getLocalOfferFailed = (err: Error): void => { + logger.error("Failed to get local offer", err); + + this.emit( + CallEvent.Error, + new CallError( + CallErrorCode.LocalOfferFailed, + "Failed to get local offer!", err, + ), + ); + this.terminate(CallParty.Local, CallErrorCode.LocalOfferFailed, false); + }; + + private getUserMediaFailed = (err: Error): void => { + if (this.successor) { + this.successor.getUserMediaFailed(err); + return; + } + + logger.warn("Failed to get user media - ending call", err); + + this.emit( + CallEvent.Error, + new CallError( + CallErrorCode.NoUserMedia, + "Couldn't start capturing media! Is your microphone set up and " + + "does this app have permission?", err, + ), + ); + this.terminate(CallParty.Local, CallErrorCode.NoUserMedia, false); + }; + + private onIceConnectionStateChanged = (): void => { + if (this.callHasEnded()) { + return; // because ICE can still complete as we're ending the call + } + logger.debug( + "Call ID " + this.callId + ": ICE connection state changed to: " + this.peerConn.iceConnectionState, + ); + // ideally we'd consider the call to be connected when we get media but + // chrome doesn't implement any of the 'onstarted' events yet + if (this.peerConn.iceConnectionState == 'connected') { + clearTimeout(this.iceDisconnectedTimeout); + this.setState(CallState.Connected); + + if (!this.callLengthInterval) { + this.callLengthInterval = setInterval(() => { + this.callLength++; + this.emit(CallEvent.LengthChanged, this.callLength); + }, 1000); + } + } else if (this.peerConn.iceConnectionState == 'failed') { + // Firefox for Android does not yet have support for restartIce() + if (this.peerConn.restartIce) { + this.peerConn.restartIce(); + } else { + this.hangup(CallErrorCode.IceFailed, false); + } + } else if (this.peerConn.iceConnectionState == 'disconnected') { + this.iceDisconnectedTimeout = setTimeout(() => { + this.hangup(CallErrorCode.IceFailed, false); + }, 30 * 1000); + } + }; + + private onSignallingStateChanged = (): void => { + logger.debug( + "call " + this.callId + ": Signalling state changed to: " + + this.peerConn.signalingState, + ); + }; + + private onTrack = (ev: RTCTrackEvent): void => { + if (ev.streams.length === 0) { + logger.warn(`Streamless ${ev.track.kind} found: ignoring.`); + return; + } + + const stream = ev.streams[0]; + this.pushRemoteFeed(stream); + stream.addEventListener("removetrack", () => { + logger.log(`Removing track streamId: ${stream.id}`); + this.deleteFeedByStream(stream); + }); + }; + + private onDataChannel = (ev: RTCDataChannelEvent): void => { + this.emit(CallEvent.DataChannel, ev.channel); + }; + + /** + * This method removes all video/rtx codecs from screensharing video + * transceivers. This is necessary since they can cause problems. Without + * this the following steps should produce an error: + * Chromium calls Firefox + * Firefox answers + * Firefox starts screen-sharing + * Chromium starts screen-sharing + * Call crashes for Chromium with: + * [96685:23:0518/162603.933321:ERROR:webrtc_video_engine.cc(3296)] RTX codec (PT=97) mapped to PT=96 which is not in the codec list. + * [96685:23:0518/162603.933377:ERROR:webrtc_video_engine.cc(1171)] GetChangedRecvParameters called without any video codecs. + * [96685:23:0518/162603.933430:ERROR:sdp_offer_answer.cc(4302)] Failed to set local video description recv parameters for m-section with mid='2'. (INVALID_PARAMETER) + */ + private getRidOfRTXCodecs(): void { + // RTCRtpReceiver.getCapabilities and RTCRtpSender.getCapabilities don't seem to be supported on FF + if (!RTCRtpReceiver.getCapabilities || !RTCRtpSender.getCapabilities) return; + + const recvCodecs = RTCRtpReceiver.getCapabilities("video").codecs; + const sendCodecs = RTCRtpSender.getCapabilities("video").codecs; + const codecs = [...sendCodecs, ...recvCodecs]; + + for (const codec of codecs) { + if (codec.mimeType === "video/rtx") { + const rtxCodecIndex = codecs.indexOf(codec); + codecs.splice(rtxCodecIndex, 1); + } + } + + for (const trans of this.peerConn.getTransceivers()) { + if ( + this.screensharingSenders.includes(trans.sender) && + ( + trans.sender.track?.kind === "video" || + trans.receiver.track?.kind === "video" + ) + ) { + trans.setCodecPreferences(codecs); + } + } + } + + private onNegotiationNeeded = async (): Promise => { + logger.info("Negotiation is needed!"); + + if (this.state !== CallState.CreateOffer && this.opponentVersion === 0) { + logger.info("Opponent does not support renegotiation: ignoring negotiationneeded event"); + return; + } + + this.queueGotLocalOffer(); + }; + + public onHangupReceived = (msg: MCallHangupReject): void => { + logger.debug("Hangup received for call ID " + this.callId); + + // party ID must match (our chosen partner hanging up the call) or be undefined (we haven't chosen + // a partner yet but we're treating the hangup as a reject as per VoIP v0) + if (this.partyIdMatches(msg) || this.state === CallState.Ringing) { + // default reason is user_hangup + this.terminate(CallParty.Remote, msg.reason || CallErrorCode.UserHangup, true); + } else { + logger.info(`Ignoring message from party ID ${msg.party_id}: our partner is ${this.opponentPartyId}`); + } + }; + + public onRejectReceived = (msg: MCallHangupReject): void => { + logger.debug("Reject received for call ID " + this.callId); + + // No need to check party_id for reject because if we'd received either + // an answer or reject, we wouldn't be in state InviteSent + + const shouldTerminate = ( + // reject events also end the call if it's ringing: it's another of + // our devices rejecting the call. + ([CallState.InviteSent, CallState.Ringing].includes(this.state)) || + // also if we're in the init state and it's an inbound call, since + // this means we just haven't entered the ringing state yet + this.state === CallState.Fledgling && this.direction === CallDirection.Inbound + ); + + if (shouldTerminate) { + this.terminate(CallParty.Remote, msg.reason || CallErrorCode.UserHangup, true); + } else { + logger.debug(`Call is in state: ${this.state}: ignoring reject`); + } + }; + + public onAnsweredElsewhere = (msg: MCallAnswer): void => { + logger.debug("Call ID " + this.callId + " answered elsewhere"); + this.terminate(CallParty.Remote, CallErrorCode.AnsweredElsewhere, true); + }; + + private setState(state: CallState): void { + const oldState = this.state; + this.state = state; + this.emit(CallEvent.State, state, oldState); + } + + /** + * Internal + * @param {string} eventType + * @param {Object} content + * @return {Promise} + */ + private sendVoipEvent(eventType: string, content: object): Promise { + const realContent = Object.assign({}, content, { + version: VOIP_PROTO_VERSION, + call_id: this.callId, + party_id: this.ourPartyId, + conf_id: this.groupCallId, + }); + + if (this.opponentDeviceId) { + this.emit(CallEvent.SendVoipEvent, { + type: "toDevice", + eventType, + userId: this.invitee || this.getOpponentMember().userId, + opponentDeviceId: this.opponentDeviceId, + content: { + ...realContent, + device_id: this.client.deviceId, + sender_session_id: this.client.getSessionId(), + dest_session_id: this.opponentSessionId, + }, + }); + + return this.client.sendToDevice(eventType, { + [this.invitee || this.getOpponentMember().userId]: { + [this.opponentDeviceId]: { + ...realContent, + device_id: this.client.deviceId, + sender_session_id: this.client.getSessionId(), + dest_session_id: this.opponentSessionId, + }, + }, + }); + } else { + this.emit(CallEvent.SendVoipEvent, { + type: "sendEvent", + eventType, + roomId: this.roomId, + content: realContent, + userId: this.invitee || this.getOpponentMember().userId, + }); + + return this.client.sendEvent(this.roomId, eventType, realContent); + } + } + + private queueCandidate(content: RTCIceCandidate): void { + // We partially de-trickle candidates by waiting for `delay` before sending them + // amalgamated, in order to avoid sending too many m.call.candidates events and hitting + // rate limits in Matrix. + // In practice, it'd be better to remove rate limits for m.call.* + + // N.B. this deliberately lets you queue and send blank candidates, which MSC2746 + // currently proposes as the way to indicate that candidate gathering is complete. + // This will hopefully be changed to an explicit rather than implicit notification + // shortly. + this.candidateSendQueue.push(content); + + // Don't send the ICE candidates yet if the call is in the ringing state: this + // means we tried to pick (ie. started generating candidates) and then failed to + // send the answer and went back to the ringing state. Queue up the candidates + // to send if we successfully send the answer. + // Equally don't send if we haven't yet sent the answer because we can send the + // first batch of candidates along with the answer + if (this.state === CallState.Ringing || !this.inviteOrAnswerSent) return; + + // MSC2746 recommends these values (can be quite long when calling because the + // callee will need a while to answer the call) + const delay = this.direction === CallDirection.Inbound ? 500 : 2000; + + if (this.candidateSendTries === 0) { + setTimeout(() => { + this.sendCandidateQueue(); + }, delay); + } + } + + /* + * Transfers this call to another user + */ + public async transfer(targetUserId: string): Promise { + // Fetch the target user's global profile info: their room avatar / displayname + // could be different in whatever room we share with them. + const profileInfo = await this.client.getProfileInfo(targetUserId); + + const replacementId = genCallID(); + + const body = { + replacement_id: genCallID(), + target_user: { + id: targetUserId, + display_name: profileInfo.displayname, + avatar_url: profileInfo.avatar_url, + }, + create_call: replacementId, + } as MCallReplacesEvent; + + await this.sendVoipEvent(EventType.CallReplaces, body); + + await this.terminate(CallParty.Local, CallErrorCode.Transfered, true); + } + + /* + * Transfers this call to the target call, effectively 'joining' the + * two calls (so the remote parties on each call are connected together). + */ + public async transferToCall(transferTargetCall?: MatrixCall): Promise { + const targetProfileInfo = await this.client.getProfileInfo(transferTargetCall.getOpponentMember().userId); + const transfereeProfileInfo = await this.client.getProfileInfo(this.getOpponentMember().userId); + + const newCallId = genCallID(); + + const bodyToTransferTarget = { + // the replacements on each side have their own ID, and it's distinct from the + // ID of the new call (but we can use the same function to generate it) + replacement_id: genCallID(), + target_user: { + id: this.getOpponentMember().userId, + display_name: transfereeProfileInfo.displayname, + avatar_url: transfereeProfileInfo.avatar_url, + }, + await_call: newCallId, + } as MCallReplacesEvent; + + await transferTargetCall.sendVoipEvent(EventType.CallReplaces, bodyToTransferTarget); + + const bodyToTransferee = { + replacement_id: genCallID(), + target_user: { + id: transferTargetCall.getOpponentMember().userId, + display_name: targetProfileInfo.displayname, + avatar_url: targetProfileInfo.avatar_url, + }, + create_call: newCallId, + } as MCallReplacesEvent; + + await this.sendVoipEvent(EventType.CallReplaces, bodyToTransferee); + + await this.terminate(CallParty.Local, CallErrorCode.Replaced, true); + await transferTargetCall.terminate(CallParty.Local, CallErrorCode.Transfered, true); + } + + private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean): Promise { + if (this.callHasEnded()) return; + + this.hangupParty = hangupParty; + this.hangupReason = hangupReason; + this.setState(CallState.Ended); + + if (this.inviteTimeout) { + clearTimeout(this.inviteTimeout); + this.inviteTimeout = null; + } + if (this.callLengthInterval) { + clearInterval(this.callLengthInterval); + this.callLengthInterval = null; + } + + this.callStatsAtEnd = await this.collectCallStats(); + + // Order is important here: first we stopAllMedia() and only then we can deleteAllFeeds() + this.stopAllMedia(); + this.deleteAllFeeds(); + + if (this.peerConn && this.peerConn.signalingState !== 'closed') { + this.peerConn.close(); + } + if (shouldEmit) { + this.emit(CallEvent.Hangup, this); + } + + this.client.callEventHandler.calls.delete(this.callId); + } + + private stopAllMedia(): void { + logger.debug(!this.groupCallId ? "Stopping all media" : "Stopping all media except local feeds" ); + for (const feed of this.feeds) { + if ( + feed.isLocal() && + feed.purpose === SDPStreamMetadataPurpose.Usermedia && + !this.groupCallId + ) { + this.client.getMediaHandler().stopUserMediaStream(feed.stream); + } else if ( + feed.isLocal() && + feed.purpose === SDPStreamMetadataPurpose.Screenshare && + !this.groupCallId + ) { + this.client.getMediaHandler().stopScreensharingStream(feed.stream); + } else if (!feed.isLocal() || !this.groupCallId) { + for (const track of feed.stream.getTracks()) { + track.stop(); + } + } + } + } + + private checkForErrorListener(): void { + if (this.listeners("error").length === 0) { + throw new Error( + "You MUST attach an error listener using call.on('error', function() {})", + ); + } + } + + private async sendCandidateQueue(): Promise { + if (this.candidateSendQueue.length === 0 || this.callHasEnded()) { + return; + } + + const candidates = this.candidateSendQueue; + this.candidateSendQueue = []; + ++this.candidateSendTries; + const content = { + candidates: candidates, + }; + logger.debug("Attempting to send " + candidates.length + " candidates"); + try { + await this.sendVoipEvent(EventType.CallCandidates, content); + // reset our retry count if we have successfully sent our candidates + // otherwise queueCandidate() will refuse to try to flush the queue + this.candidateSendTries = 0; + + // Try to send candidates again just in case we received more candidates while sending. + this.sendCandidateQueue(); + } catch (error) { + // don't retry this event: we'll send another one later as we might + // have more candidates by then. + if (error.event) this.client.cancelPendingEvent(error.event); + + // put all the candidates we failed to send back in the queue + this.candidateSendQueue.push(...candidates); + + if (this.candidateSendTries > 5) { + logger.debug( + "Failed to send candidates on attempt " + this.candidateSendTries + + ". Giving up on this call.", error, + ); + + const code = CallErrorCode.SignallingFailed; + const message = "Signalling failed"; + + this.emit(CallEvent.Error, new CallError(code, message, error)); + this.hangup(code, false); + + return; + } + + const delayMs = 500 * Math.pow(2, this.candidateSendTries); + ++this.candidateSendTries; + logger.debug("Failed to send candidates. Retrying in " + delayMs + "ms", error); + setTimeout(() => { + this.sendCandidateQueue(); + }, delayMs); + } + } + + /** + * Place a call to this room. + * @throws if you have not specified a listener for 'error' events. + * @throws if have passed audio=false. + */ + public async placeCall(audio: boolean, video: boolean): Promise { + if (!audio) { + throw new Error("You CANNOT start a call without audio"); + } + this.setState(CallState.WaitLocalMedia); + + try { + const stream = await this.client.getMediaHandler().getUserMediaStream(audio, video); + const callFeed = new CallFeed({ + client: this.client, + roomId: this.roomId, + userId: this.client.getUserId(), + stream, + purpose: SDPStreamMetadataPurpose.Usermedia, + audioMuted: stream.getAudioTracks().length === 0, + videoMuted: stream.getVideoTracks().length === 0, + }); + await this.placeCallWithCallFeeds([callFeed]); + } catch (e) { + this.getUserMediaFailed(e); + return; + } + } + + /** + * Place a call to this room with call feed. + * @param {CallFeed[]} callFeeds to use + * @throws if you have not specified a listener for 'error' events. + * @throws if have passed audio=false. + */ + public async placeCallWithCallFeeds(callFeeds: CallFeed[], requestScreenshareFeed = false): Promise { + this.checkForErrorListener(); + this.direction = CallDirection.Outbound; + + // XXX Find a better way to do this + this.client.callEventHandler.calls.set(this.callId, this); + + // make sure we have valid turn creds. Unless something's gone wrong, it should + // poll and keep the credentials valid so this should be instant. + const haveTurnCreds = await this.client.checkTurnServers(); + if (!haveTurnCreds) { + logger.warn("Failed to get TURN credentials! Proceeding with call anyway..."); + } + + // create the peer connection now so it can be gathering candidates while we get user + // media (assuming a candidate pool size is configured) + this.peerConn = this.createPeerConnection(); + this.gotCallFeedsForInvite(callFeeds, requestScreenshareFeed); + } + + private createPeerConnection(): RTCPeerConnection { + const pc = new window.RTCPeerConnection({ + iceTransportPolicy: this.forceTURN ? 'relay' : undefined, + iceServers: this.turnServers, + iceCandidatePoolSize: this.client.iceCandidatePoolSize, + }); + + // 'connectionstatechange' would be better, but firefox doesn't implement that. + pc.addEventListener('iceconnectionstatechange', this.onIceConnectionStateChanged); + pc.addEventListener('signalingstatechange', this.onSignallingStateChanged); + pc.addEventListener('icecandidate', this.gotLocalIceCandidate); + pc.addEventListener('icegatheringstatechange', this.onIceGatheringStateChange); + pc.addEventListener('track', this.onTrack); + pc.addEventListener('negotiationneeded', this.onNegotiationNeeded); + pc.addEventListener('datachannel', this.onDataChannel); + + return pc; + } + + private partyIdMatches(msg: MCallBase): boolean { + // They must either match or both be absent (in which case opponentPartyId will be null) + // Also we ignore party IDs on the invite/offer if the version is 0, so we must do the same + // here and use null if the version is 0 (woe betide any opponent sending messages in the + // same call with different versions) + const msgPartyId = msg.version === 0 ? null : msg.party_id || null; + return msgPartyId === this.opponentPartyId; + } + + // Commits to an opponent for the call + // ev: An invite or answer event + private chooseOpponent(ev: MatrixEvent): void { + // I choo-choo-choose you + const msg = ev.getContent(); + + logger.debug(`Choosing party ID ${msg.party_id} for call ID ${this.callId}`); + + this.opponentVersion = msg.version; + if (this.opponentVersion === 0) { + // set to null to indicate that we've chosen an opponent, but because + // they're v0 they have no party ID (even if they sent one, we're ignoring it) + this.opponentPartyId = null; + } else { + // set to their party ID, or if they're naughty and didn't send one despite + // not being v0, set it to null to indicate we picked an opponent with no + // party ID + this.opponentPartyId = msg.party_id || null; + } + this.opponentCaps = msg.capabilities || {} as CallCapabilities; + this.opponentMember = this.client.getRoom(this.roomId).getMember(ev.getSender()); + } + + private async addBufferedIceCandidates(): Promise { + const bufferedCandidates = this.remoteCandidateBuffer.get(this.opponentPartyId); + if (bufferedCandidates) { + logger.info(`Adding ${bufferedCandidates.length} buffered candidates for opponent ${this.opponentPartyId}`); + await this.addIceCandidates(bufferedCandidates); + } + this.remoteCandidateBuffer = null; + } + + private async addIceCandidates(candidates: RTCIceCandidate[]): Promise { + for (const candidate of candidates) { + if ( + (candidate.sdpMid === null || candidate.sdpMid === undefined) && + (candidate.sdpMLineIndex === null || candidate.sdpMLineIndex === undefined) + ) { + logger.debug("Ignoring remote ICE candidate with no sdpMid or sdpMLineIndex"); + continue; + } + logger.debug( + "Call " + this.callId + " got remote ICE " + candidate.sdpMid + " candidate: " + candidate.candidate, + ); + try { + await this.peerConn.addIceCandidate(candidate); + } catch (err) { + if (!this.ignoreOffer) { + logger.info("Failed to add remote ICE candidate", err); + } + } + } + } + + public get hasPeerConnection(): boolean { + return Boolean(this.peerConn); + } +} + +export function setTracksEnabled(tracks: Array, enabled: boolean): void { + for (let i = 0; i < tracks.length; i++) { + tracks[i].enabled = enabled; + } +} + +/** + * DEPRECATED + * Use client.createCall() + * + * Create a new Matrix call for the browser. + * @param {MatrixClient} client The client instance to use. + * @param {string} roomId The room the call is in. + * @param {Object?} options DEPRECATED optional options map. + * @param {boolean} options.forceTURN DEPRECATED whether relay through TURN should be + * forced. This option is deprecated - use opts.forceTURN when creating the matrix client + * since it's only possible to set this option on outbound calls. + * @return {MatrixCall} the call or null if the browser doesn't support calling. + */ +export function createNewMatrixCall(client: any, roomId: string, options?: CallOpts): MatrixCall { + // typeof prevents Node from erroring on an undefined reference + if (typeof(window) === 'undefined' || typeof(document) === 'undefined') { + // NB. We don't log here as apps try to create a call object as a test for + // whether calls are supported, so we shouldn't fill the logs up. + return null; + } + + // Firefox throws on so little as accessing the RTCPeerConnection when operating in + // a secure mode. There's some information at https://bugzilla.mozilla.org/show_bug.cgi?id=1542616 + // though the concern is that the browser throwing a SecurityError will brick the + // client creation process. + try { + const supported = Boolean( + window.RTCPeerConnection || window.RTCSessionDescription || + window.RTCIceCandidate || navigator.mediaDevices, + ); + if (!supported) { + // Adds a lot of noise to test runs, so disable logging there. + if (process.env.NODE_ENV !== "test") { + logger.error("WebRTC is not supported in this browser / environment"); + } + return null; + } + } catch (e) { + logger.error("Exception thrown when trying to access WebRTC", e); + return null; + } + + const optionsForceTURN = options ? options.forceTURN : false; + + const opts: CallOpts = { + client: client, + roomId: roomId, + invitee: options?.invitee, + turnServers: client.getTurnServers(), + // call level options + forceTURN: client.forceTURN || optionsForceTURN, + opponentDeviceId: options?.opponentDeviceId, + opponentSessionId: options?.opponentSessionId, + groupCallId: options?.groupCallId, + }; + const call = new MatrixCall(opts); + + client.reEmitter.reEmit(call, Object.values(CallEvent)); + + return call; +} + diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md new file mode 100644 index 00000000..33b408f6 --- /dev/null +++ b/src/matrix/calls/TODO.md @@ -0,0 +1,43 @@ +## Store ongoing calls + +Add store with all ongoing calls so when we quit and start again, we don't have to go through all the past calls to know which ones might still be ongoing. + + +## Notes + +we send m.call as state event in room + +we add m.call.participant for our own device + +we wait for other participants to add their user and device (in the sources) + +for each (userid, deviceid) + - if userId < ourUserId + - we setup a peer connection + - we wait for negotation event to get sdp + - we send an m.call.invite + - else + - wait for invite from other side + + +in some cases, we will actually send the invite to all devices (e.g. SFU), so +we probably still need to handle multiple anwsers? + +so we would send an invite to multiple devices and pick the one for which we +received the anwser first. between invite and anwser, we could already receive +ice candidates that we need to buffer. + +should a PeerCall only exist after we've received an answer? +Before that, we could have a PeerCallInvite + + + +updating the metadata: + +if we're renegotiating: use m.call.negotatie +if just muting: use m.call.sdp_stream_metadata_changed + + +party identification + - for 1:1 calls, we identify with a party_id + - for group calls, we identify with a device_id diff --git a/src/matrix/calls/group/Call.ts b/src/matrix/calls/group/Call.ts new file mode 100644 index 00000000..9abef197 --- /dev/null +++ b/src/matrix/calls/group/Call.ts @@ -0,0 +1,88 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {ObservableMap} from "../../../observable/map/ObservableMap"; + +function participantId(senderUserId: string, senderDeviceId: string | null) { + return JSON.stringify(senderUserId) + JSON.stringify(senderDeviceId); +} + +class Call { + private readonly participants: ObservableMap = new ObservableMap(); + private localMedia?: LocalMedia; + + constructor(private readonly ownUserId: string, private callEvent: StateEvent, private readonly room: Room, private readonly webRTC: WebRTC) { + + } + + get id(): string { return this.callEvent.state_key; } + + async participate(tracks: Track[]) { + this.localMedia = LocalMedia.fromTracks(tracks); + for (const [,participant] of this.participants) { + participant.setLocalMedia(this.localMedia.clone()); + } + // send m.call.member state event + + // send invite to all participants that are < my userId + for (const [,participant] of this.participants) { + if (participant.userId < this.ownUserId) { + participant.sendInvite(); + } + } + } + + updateCallEvent(callEvent: StateEvent) { + this.callEvent = callEvent; + } + + addParticipant(userId, source) { + const participantId = getParticipantId(userId, source.device_id); + const participant = this.participants.get(participantId); + if (participant) { + participant.updateSource(source); + } else { + participant.add(participantId, new Participant(userId, source.device_id, this.localMedia?.clone(), this.webRTC)); + } + } + + handleDeviceMessage(senderUserId: string, senderDeviceId: string, eventType: string, content: Record, log: ILogItem) { + const participantId = getParticipantId(senderUserId, senderDeviceId); + let peerCall = this.participants.get(participantId); + let hasDeviceInKey = true; + if (!peerCall) { + hasDeviceInKey = false; + peerCall = this.participants.get(getParticipantId(senderUserId, null)) + } + if (peerCall) { + peerCall.handleIncomingSignallingMessage(eventType, content, senderDeviceId); + if (!hasDeviceInKey && peerCall.opponentPartyId) { + this.participants.delete(getParticipantId(senderUserId, null)); + this.participants.add(getParticipantId(senderUserId, peerCall.opponentPartyId)); + } + } else { + // create peerCall + } + } + + get id(): string { + return this.callEvent.state_key; + } + + get isTerminated(): boolean { + return !!this.callEvent.content[CALL_TERMINATED]; + } +} diff --git a/src/matrix/calls/group/Participant.ts b/src/matrix/calls/group/Participant.ts new file mode 100644 index 00000000..5abdaf63 --- /dev/null +++ b/src/matrix/calls/group/Participant.ts @@ -0,0 +1,46 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +class Participant implements PeerCallHandler { + private peerCall?: PeerCall; + + constructor( + private readonly userId: string, + private readonly deviceId: string, + private localMedia: LocalMedia | undefined, + private readonly webRTC: WebRTC, + private readonly hsApi: HomeServerApi + ) {} + + sendInvite() { + this.peerCall = new PeerCall(this, this.webRTC); + this.peerCall.setLocalMedia(this.localMedia); + this.peerCall.sendOffer(); + } + + /** From PeerCallHandler + * @internal */ + override emitUpdate() { + + } + + /** From PeerCallHandler + * @internal */ + override onSendSignallingMessage() { + // TODO: this needs to be encrypted with olm first + this.hsApi.sendToDevice(type, {[this.userId]: {[this.deviceId ?? "*"]: content}}); + } +} diff --git a/src/platform/types/MediaDevices.ts b/src/platform/types/MediaDevices.ts index ec4895fa..3dda3059 100644 --- a/src/platform/types/MediaDevices.ts +++ b/src/platform/types/MediaDevices.ts @@ -32,6 +32,7 @@ export interface Track { get label(): string; get id(): string; get streamId(): string; + get settings(): MediaTrackSettings; get muted(): boolean; setMuted(muted: boolean): void; } diff --git a/src/platform/types/WebRTC.ts b/src/platform/types/WebRTC.ts index b34e4214..9a10ce5e 100644 --- a/src/platform/types/WebRTC.ts +++ b/src/platform/types/WebRTC.ts @@ -47,7 +47,8 @@ export interface PeerConnection { createOffer(): Promise; createAnswer(): Promise; setLocalDescription(description: RTCSessionDescriptionInit); - setRemoteDescription(description: RTCSessionDescriptionInit); + setRemoteDescription(description: RTCSessionDescriptionInit): Promise; + addIceCandidate(candidate: RTCIceCandidate): Promise; addTrack(track: Track): void; removeTrack(track: Track): boolean; replaceTrack(oldTrack: Track, newTrack: Track): Promise; diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index 08ccc636..7d5c8f4f 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -97,6 +97,7 @@ export class TrackWrapper implements Track { get id(): string { return this.track.id; } get streamId(): string { return this.stream.id; } get muted(): boolean { return this.track.muted; } + get settings(): MediaTrackSettings { return this.track.getSettings(); } setMuted(muted: boolean): void { this.track.enabled = !muted; diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 712e8cc6..36d5eb75 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -15,7 +15,7 @@ limitations under the License. */ import {TrackWrapper, wrapTrack} from "./MediaDevices"; -import {Track} from "../../types/MediaDevices"; +import {Track, TrackType} from "../../types/MediaDevices"; import {WebRTC, PeerConnectionHandler, DataChannel, PeerConnection, StreamPurpose} from "../../types/WebRTC"; const POLLING_INTERVAL = 200; // ms @@ -57,6 +57,10 @@ class DOMPeerConnection implements PeerConnection { this.peerConnection.setRemoteDescription(description); } + addIceCandidate(candidate: RTCIceCandidate): Promise { + return this.peerConnection.addIceCandidate(candidate); + } + addTrack(track: Track): void { if (!(track instanceof TrackWrapper)) { throw new Error("Not a TrackWrapper"); @@ -152,263 +156,10 @@ class DOMPeerConnection implements PeerConnection { let type: TrackType; if (track.kind === "video") { const purpose = this.handler.getPurposeForStreamId(stream.id); - type = purpose === StreamPurpose.Usermedia ? TrackType.Camera : TrackType.ScreenShare; + type = purpose === StreamPurpose.UserMedia ? TrackType.Camera : TrackType.ScreenShare; } else { type = TrackType.Microphone; } return wrapTrack(track, stream, type); } } - -export interface ICallFeedOpts { - client: MatrixClient; - roomId: string; - userId: string; - stream: MediaStream; - purpose: SDPStreamMetadataPurpose; - audioMuted: boolean; - videoMuted: boolean; -} - -export enum CallFeedEvent { - NewStream = "new_stream", - MuteStateChanged = "mute_state_changed", - VolumeChanged = "volume_changed", - Speaking = "speaking", -} - -export class CallFeed extends EventEmitter { - public stream: MediaStream; - public sdpMetadataStreamId: string; - public userId: string; - public purpose: SDPStreamMetadataPurpose; - public speakingVolumeSamples: number[]; - - private client: MatrixClient; - private roomId: string; - private audioMuted: boolean; - private videoMuted: boolean; - private measuringVolumeActivity = false; - private audioContext: AudioContext; - private analyser: AnalyserNode; - private frequencyBinCount: Float32Array; - private speakingThreshold = SPEAKING_THRESHOLD; - private speaking = false; - private volumeLooperTimeout: number; - - constructor(opts: ICallFeedOpts) { - super(); - - this.client = opts.client; - this.roomId = opts.roomId; - this.userId = opts.userId; - this.purpose = opts.purpose; - this.audioMuted = opts.audioMuted; - this.videoMuted = opts.videoMuted; - this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); - this.sdpMetadataStreamId = opts.stream.id; - - this.updateStream(null, opts.stream); - - if (this.hasAudioTrack) { - this.initVolumeMeasuring(); - } - } - - private get hasAudioTrack(): boolean { - return this.stream.getAudioTracks().length > 0; - } - - private updateStream(oldStream: MediaStream, newStream: MediaStream): void { - if (newStream === oldStream) return; - - if (oldStream) { - oldStream.removeEventListener("addtrack", this.onAddTrack); - this.measureVolumeActivity(false); - } - if (newStream) { - this.stream = newStream; - newStream.addEventListener("addtrack", this.onAddTrack); - - if (this.hasAudioTrack) { - this.initVolumeMeasuring(); - } else { - this.measureVolumeActivity(false); - } - } - - this.emit(CallFeedEvent.NewStream, this.stream); - } - - private initVolumeMeasuring(): void { - const AudioContext = window.AudioContext || window.webkitAudioContext; - if (!this.hasAudioTrack || !AudioContext) return; - - this.audioContext = new AudioContext(); - - this.analyser = this.audioContext.createAnalyser(); - this.analyser.fftSize = 512; - this.analyser.smoothingTimeConstant = 0.1; - - const mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.stream); - mediaStreamAudioSourceNode.connect(this.analyser); - - this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount); - } - - private onAddTrack = (): void => { - this.emit(CallFeedEvent.NewStream, this.stream); - }; - - /** - * Returns callRoom member - * @returns member of the callRoom - */ - public getMember(): RoomMember { - const callRoom = this.client.getRoom(this.roomId); - return callRoom.getMember(this.userId); - } - - /** - * Returns true if CallFeed is local, otherwise returns false - * @returns {boolean} is local? - */ - public isLocal(): boolean { - return this.userId === this.client.getUserId(); - } - - /** - * Returns true if audio is muted or if there are no audio - * tracks, otherwise returns false - * @returns {boolean} is audio muted? - */ - public isAudioMuted(): boolean { - return this.stream.getAudioTracks().length === 0 || this.audioMuted; - } - - /** - * Returns true video is muted or if there are no video - * tracks, otherwise returns false - * @returns {boolean} is video muted? - */ - public isVideoMuted(): boolean { - // We assume only one video track - return this.stream.getVideoTracks().length === 0 || this.videoMuted; - } - - public isSpeaking(): boolean { - return this.speaking; - } - - /** - * Replaces the current MediaStream with a new one. - * This method should be only used by MatrixCall. - * @param newStream new stream with which to replace the current one - */ - public setNewStream(newStream: MediaStream): void { - this.updateStream(this.stream, newStream); - } - - /** - * Set feed's internal audio mute state - * @param muted is the feed's audio muted? - */ - public setAudioMuted(muted: boolean): void { - this.audioMuted = muted; - this.speakingVolumeSamples.fill(-Infinity); - this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); - } - - /** - * Set feed's internal video mute state - * @param muted is the feed's video muted? - */ - public setVideoMuted(muted: boolean): void { - this.videoMuted = muted; - this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); - } - - /** - * Starts emitting volume_changed events where the emitter value is in decibels - * @param enabled emit volume changes - */ - public measureVolumeActivity(enabled: boolean): void { - if (enabled) { - if (!this.audioContext || !this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return; - - this.measuringVolumeActivity = true; - this.volumeLooper(); - } else { - this.measuringVolumeActivity = false; - this.speakingVolumeSamples.fill(-Infinity); - this.emit(CallFeedEvent.VolumeChanged, -Infinity); - } - } - - public setSpeakingThreshold(threshold: number) { - this.speakingThreshold = threshold; - } - - private volumeLooper = () => { - if (!this.analyser) return; - - if (!this.measuringVolumeActivity) return; - - this.analyser.getFloatFrequencyData(this.frequencyBinCount); - - let maxVolume = -Infinity; - for (let i = 0; i < this.frequencyBinCount.length; i++) { - if (this.frequencyBinCount[i] > maxVolume) { - maxVolume = this.frequencyBinCount[i]; - } - } - - this.speakingVolumeSamples.shift(); - this.speakingVolumeSamples.push(maxVolume); - - this.emit(CallFeedEvent.VolumeChanged, maxVolume); - - let newSpeaking = false; - - for (let i = 0; i < this.speakingVolumeSamples.length; i++) { - const volume = this.speakingVolumeSamples[i]; - - if (volume > this.speakingThreshold) { - newSpeaking = true; - break; - } - } - - if (this.speaking !== newSpeaking) { - this.speaking = newSpeaking; - this.emit(CallFeedEvent.Speaking, this.speaking); - } - - this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL); - }; - - public clone(): CallFeed { - const mediaHandler = this.client.getMediaHandler(); - const stream = this.stream.clone(); - - if (this.purpose === SDPStreamMetadataPurpose.Usermedia) { - mediaHandler.userMediaStreams.push(stream); - } else { - mediaHandler.screensharingStreams.push(stream); - } - - return new CallFeed({ - client: this.client, - roomId: this.roomId, - userId: this.userId, - stream, - purpose: this.purpose, - audioMuted: this.audioMuted, - videoMuted: this.videoMuted, - }); - } - - public dispose(): void { - clearTimeout(this.volumeLooperTimeout); - } -} From 98e1dcf79920b4b7d2064b4b8c33d2e521ceefb8 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Fri, 18 Feb 2022 16:38:10 +0100 Subject: [PATCH 005/435] WIP5 --- src/matrix/calls/PeerCall.ts | 2523 +++---------------------- src/matrix/calls/TODO.md | 23 + src/matrix/calls/group/Participant.ts | 10 +- src/platform/types/WebRTC.ts | 5 +- src/platform/web/dom/WebRTC.ts | 78 +- src/utils/AsyncQueue.ts | 52 + 6 files changed, 404 insertions(+), 2287 deletions(-) create mode 100644 src/utils/AsyncQueue.ts diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 8780258e..47dfaa3e 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -16,6 +16,7 @@ limitations under the License. import {ObservableMap} from "../../observable/map/ObservableMap"; +import {AsyncQueue} from "../../utils/AsyncQueue"; import type {Room} from "../room/Room"; import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; @@ -65,17 +66,6 @@ const GROUP_CALL_MEMBER_TYPE = "m.call.member"; // without MSC2746 that doesn't provide an id which device sent the message. type PartyId = string | null; -interface CallOpts { - roomId?: string; - invitee?: string; - client?: any; // Fix when client is TSified - forceTURN?: boolean; - turnServers?: Array; - opponentDeviceId?: string; - opponentSessionId?: string; - groupCallId?: string; -} - interface TurnServer { urls: Array; username?: string; @@ -115,6 +105,21 @@ export enum CallParty { Remote = 'remote', } +export enum EventType { + Invite = "m.call.invite", + Candidates = "m.call.candidates", + Answer = "m.call.answer", + Hangup = "m.call.hangup", + Reject = "m.call.reject", + SelectAnswer = "m.call.select_answer", + Negotiate = "m.call.negotiate", + SDPStreamMetadataChanged = "m.call.sdp_stream_metadata_changed", + SDPStreamMetadataChangedPrefix = "org.matrix.call.sdp_stream_metadata_changed", + Replaces = "m.call.replaces", + AssertedIdentity = "m.call.asserted_identity", + AssertedIdentityPrefix = "org.matrix.call.asserted_identity", +} + export enum CallEvent { Hangup = 'hangup', State = 'state', @@ -280,6 +285,8 @@ class LocalMedia { return new LocalMedia(cameraTrack, screenShareTrack, microphoneTrack as AudioTrack); } + get tracks(): Track[] { return []; } + getSDPMetadata(): any { const metadata = {}; const userMediaTrack = this.microphoneTrack ?? this.cameraTrack; @@ -299,14 +306,20 @@ class LocalMedia { } } -interface PeerCallHandler { - emitUpdate(); - sendSignallingMessage(type: string, content: Record); +export type InviteMessage = { + type: EventType.Invite, + content: { + version: number + } +} + +export interface PeerCallHandler { + emitUpdate(peerCall: PeerCall, params: any); + sendSignallingMessage(type: EventType, content: Record); } // when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already // do for sharing keys will be best as that already deals with room tracking. -type SendSignallingMessageCallback = (type: CallSetupMessageType, content: Record) => Promise; /** * Does WebRTC signalling for a single PeerConnection, and deals with WebRTC wrappers from platform * */ @@ -330,7 +343,6 @@ class PeerCall implements PeerConnectionHandler { private inviteOrAnswerSent = false; private waitForLocalAVStream: boolean; - private successor: MatrixCall; private opponentVersion: number | string; // The party ID of the other side: undefined if we haven't chosen a partner // yet, null if we have but they didn't send a party ID. @@ -352,17 +364,16 @@ class PeerCall implements PeerConnectionHandler { private makingOffer = false; private ignoreOffer: boolean; - private responsePromiseChain?: Promise; - // If candidates arrive before we've picked an opponent (which, in particular, // will happen if the opponent sends candidates eagerly before the user answers // the call) we buffer them up here so we can then add the ones from the party we pick - private remoteCandidateBuffer: Map = []; + private remoteCandidateBuffer: Map; private remoteAssertedIdentity: AssertedIdentity; private remoteSDPStreamMetadata?: SDPStreamMetadata; + private negotiationQueue: AsyncQueue; constructor( private readonly handler: PeerCallHandler, @@ -370,25 +381,233 @@ class PeerCall implements PeerConnectionHandler { webRTC: WebRTC ) { this.peerConnection = webRTC.createPeerConnection(this); + // TODO: should we use this to serialize all state changes? + this.negotiationQueue = new AsyncQueue(this.handleNegotiation, void); } + // PeerConnectionHandler method onIceConnectionStateChange(state: RTCIceConnectionState) {} + // PeerConnectionHandler method onLocalIceCandidate(candidate: RTCIceCandidate) {} + // PeerConnectionHandler method onIceGatheringStateChange(state: RTCIceGatheringState) {} + // PeerConnectionHandler method onRemoteTracksChanged(tracks: Track[]) {} + // PeerConnectionHandler method onDataChannelChanged(dataChannel: DataChannel | undefined) {} + // PeerConnectionHandler method onNegotiationNeeded() { + // trigger handleNegotiation + this.negotiationQueue.push(void); + } + + // calls are serialized and deduplicated by negotiationQueue + private handleNegotiation = async (): Promise => { + const offer = await this.peerConnection.createOffer(); + this.peerConnection.setLocalDescription(offer); + // need to queue this const message = { - offer: this.peerConnection.createOffer(), + offer, sdp_stream_metadata: this.localMedia.getSDPMetadata(), version: 1 } - this.handler.sendSignallingMessage(CallSetupMessageType.Invite, message); + if (this.state === CallState.Fledgling) { + const sendPromise = this.handler.sendSignallingMessage(EventType.Invite, message); + this.setState(CallState.InviteSent); + } else { + await this.handler.sendSignallingMessage(EventType.Negotiate, message); + } + }; + + async sendInvite(localMediaPromise: Promise): Promise { + if (this.state !== CallState.Fledgling) { + return; + } + this.setState(CallState.WaitLocalMedia); + this.localMedia = await localMediaPromise; + // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called + for (const t of this.localMedia.tracks) { + this.peerConnection.addTrack(t); + } + await this.waitForState(CallState.Ended, CallState.InviteSent); } - setLocalMedia(localMedia: LocalMedia) { - this.localMedia = localMedia; - // TODO: send new metadata + async sendAnswer(localMediaPromise: Promise): Promise { + if (this.callHasEnded()) return; + + if (this.state !== CallState.Ringing) { + return; + } + + this.setState(CallState.WaitLocalMedia); + this.waitForLocalAVStream = true; + this.localMedia = await localMediaPromise; + this.waitForLocalAVStream = false; + + // enqueue the following + + // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called + for (const t of this.localMedia.tracks) { + this.peerConnection.addTrack(t); + } + + this.setState(CallState.CreateAnswer); + + let myAnswer; + try { + myAnswer = await this.peerConn.createAnswer(); + } catch (err) { + logger.debug("Failed to create answer: ", err); + this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true); + return; + } + + try { + await this.peerConn.setLocalDescription(myAnswer); + this.setState(CallState.Connecting); + + // Allow a short time for initial candidates to be gathered + await new Promise(resolve => { + setTimeout(resolve, 200); + }); + // inlined sendAnswer + const answerContent = { + answer: { + sdp: this.peerConn.localDescription.sdp, + // type is now deprecated as of Matrix VoIP v1, but + // required to still be sent for backwards compat + type: this.peerConn.localDescription.type, + }, + [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(true), + } as MCallAnswer; + + answerContent.capabilities = { + 'm.call.transferee': this.client.supportsCallTransfer, + 'm.call.dtmf': false, + }; + + // We have just taken the local description from the peerConn which will + // contain all the local candidates added so far, so we can discard any candidates + // we had queued up because they'll be in the answer. + logger.info(`Discarding ${this.candidateSendQueue.length} candidates that will be sent in answer`); + this.candidateSendQueue = []; + + try { + await this.sendVoipEvent(EventType.CallAnswer, answerContent); + // If this isn't the first time we've tried to send the answer, + // we may have candidates queued up, so send them now. + this.inviteOrAnswerSent = true; + } catch (error) { + // We've failed to answer: back to the ringing state + this.setState(CallState.Ringing); + this.client.cancelPendingEvent(error.event); + + let code = CallErrorCode.SendAnswer; + let message = "Failed to send answer"; + if (error.name == 'UnknownDeviceError') { + code = CallErrorCode.UnknownDevices; + message = "Unknown devices present in the room"; + } + this.emit(CallEvent.Error, new CallError(code, message, error)); + throw error; + } + + // error handler re-throws so this won't happen on error, but + // we don't want the same error handling on the candidate queue + this.sendCandidateQueue(); + } catch (err) { + logger.debug("Error setting local description!", err); + this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); + return; + } + } + + async updateLocalMedia(localMediaPromise: Promise) { + const oldMedia = this.localMedia; + this.localMedia = await localMediaPromise; + + const applyTrack = (selectTrack: (media: LocalMedia) => Track | undefined) => { + const oldTrack = selectTrack(oldMedia); + const newTrack = selectTrack(this.localMedia); + if (oldTrack && newTrack) { + this.peerConnection.replaceTrack(oldTrack, newTrack); + } else if (oldTrack) { + this.peerConnection.removeTrack(oldTrack); + } else if (newTrack) { + this.peerConnection.addTrack(newTrack); + } + }; + + // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called + applyTrack(m => m.microphoneTrack); + applyTrack(m => m.cameraTrack); + applyTrack(m => m.screenShareTrack); + } + + + /** + * Replace this call with a new call, e.g. for glare resolution. Used by + * MatrixClient. + * @param {MatrixCall} newCall The new call. + */ + public replacedBy(newCall: MatrixCall): void { + if (this.state === CallState.WaitLocalMedia) { + logger.debug("Telling new call to wait for local media"); + newCall.waitForLocalAVStream = true; + } else if ([CallState.CreateOffer, CallState.InviteSent].includes(this.state)) { + if (newCall.direction === CallDirection.Outbound) { + newCall.queueGotCallFeedsForAnswer([]); + } else { + logger.debug("Handing local stream to new call"); + newCall.queueGotCallFeedsForAnswer(this.getLocalFeeds().map(feed => feed.clone())); + } + } + this.successor = newCall; + this.emit(CallEvent.Replaced, newCall); + this.hangup(CallErrorCode.Replaced, true); + } + + /** + * Hangup a call. + * @param {string} reason The reason why the call is being hung up. + * @param {boolean} suppressEvent True to suppress emitting an event. + */ + public hangup(reason: CallErrorCode, suppressEvent: boolean): void { + if (this.callHasEnded()) return; + + logger.debug("Ending call " + this.callId); + this.terminate(CallParty.Local, reason, !suppressEvent); + // We don't want to send hangup here if we didn't even get to sending an invite + if (this.state === CallState.WaitLocalMedia) return; + const content = {}; + // Don't send UserHangup reason to older clients + if ((this.opponentVersion && this.opponentVersion >= 1) || reason !== CallErrorCode.UserHangup) { + content["reason"] = reason; + } + this.sendVoipEvent(EventType.CallHangup, content); + } + + /** + * Reject a call + * This used to be done by calling hangup, but is a separate method and protocol + * event as of MSC2746. + */ + public reject(): void { + if (this.state !== CallState.Ringing) { + throw Error("Call must be in 'ringing' state to reject!"); + } + + if (this.opponentVersion < 1) { + logger.info( + `Opponent version is less than 1 (${this.opponentVersion}): sending hangup instead of reject`, + ); + this.hangup(CallErrorCode.UserHangup, true); + return; + } + + logger.debug("Rejecting call: " + this.callId); + this.terminate(CallParty.Local, CallErrorCode.UserHangup, true); + this.sendVoipEvent(EventType.CallReject, {}); } // request the type of incoming track @@ -402,6 +621,11 @@ class PeerCall implements PeerConnectionHandler { const oldState = this.state; this.state = state; this.handler.emitUpdate(); + if (this.inviteDeferred) { + if (this.state === CallState.InviteSent) { + this.inviteDeferred.resolve(); + } + } } handleIncomingSignallingMessage(type: CallSetupMessageType, content: Record, partyId: PartyId) { @@ -504,2254 +728,3 @@ class PeerCall implements PeerConnectionHandler { } } } - -/** - * Construct a new Matrix Call. - * @constructor - * @param {Object} opts Config options. - * @param {string} opts.roomId The room ID for this call. - * @param {Object} opts.webRtc The WebRTC globals from the browser. - * @param {boolean} opts.forceTURN whether relay through TURN should be forced. - * @param {Object} opts.URL The URL global. - * @param {Array} opts.turnServers Optional. A list of TURN servers. - * @param {MatrixClient} opts.client The Matrix Client instance to send events to. - */ -export class MatrixCall extends EventEmitter { - // should go into DirectCall/GroupCall class - public roomId: string; - public callId: string; - public invitee?: string; - public ourPartyId: string; - private opponentMember: RoomMember; - private opponentDeviceId: string; - private opponentSessionId: string; - public groupCallId: string; - private callLengthInterval: number; - private callLength = 0; - - - - public state = CallState.Fledgling; - public hangupParty: CallParty; - public hangupReason: string; - public direction: CallDirection; - public peerConn?: RTCPeerConnection; - - // A queue for candidates waiting to go out. - // We try to amalgamate candidates into a single candidate message where - // possible - private candidateSendQueue: Array = []; - private candidateSendTries = 0; - private sentEndOfCandidates = false; - - private inviteOrAnswerSent = false; - private waitForLocalAVStream: boolean; - private successor: MatrixCall; - private opponentVersion: number | string; - // The party ID of the other side: undefined if we haven't chosen a partner - // yet, null if we have but they didn't send a party ID. - private opponentPartyId: string; - private opponentCaps: CallCapabilities; - private inviteTimeout: number; - private iceDisconnectedTimeout: number; - - // The logic of when & if a call is on hold is nontrivial and explained in is*OnHold - // This flag represents whether we want the other party to be on hold - private remoteOnHold = false; - - // the stats for the call at the point it ended. We can't get these after we - // tear the call down, so we just grab a snapshot before we stop the call. - // The typescript definitions have this type as 'any' :( - private callStatsAtEnd: any[]; - - // Perfect negotiation state: https://www.w3.org/TR/webrtc/#perfect-negotiation-example - private makingOffer = false; - private ignoreOffer: boolean; - - private responsePromiseChain?: Promise; - - // If candidates arrive before we've picked an opponent (which, in particular, - // will happen if the opponent sends candidates eagerly before the user answers - // the call) we buffer them up here so we can then add the ones from the party we pick - private remoteCandidateBuffer: RTCIceCandidate[] = []; - - private remoteAssertedIdentity: AssertedIdentity; - - private remoteSDPStreamMetadata: SDPStreamMetadata; - - constructor(opts: CallOpts) { - super(); - this.roomId = opts.roomId; - this.invitee = opts.invitee; - this.client = opts.client; - this.forceTURN = opts.forceTURN; - this.ourPartyId = this.client.deviceId; - this.opponentDeviceId = opts.opponentDeviceId; - this.opponentSessionId = opts.opponentSessionId; - this.groupCallId = opts.groupCallId; - // Array of Objects with urls, username, credential keys - this.turnServers = opts.turnServers || []; - if (this.turnServers.length === 0 && this.client.isFallbackICEServerAllowed()) { - this.turnServers.push({ - urls: [FALLBACK_ICE_SERVER], - }); - } - for (const server of this.turnServers) { - utils.checkObjectHasKeys(server, ["urls"]); - } - this.callId = genCallID(); - } - - /** - * Place a voice call to this room. - * @throws If you have not specified a listener for 'error' events. - */ - public async placeVoiceCall(): Promise { - await this.placeCall(true, false); - } - - /** - * Place a video call to this room. - * @throws If you have not specified a listener for 'error' events. - */ - public async placeVideoCall(): Promise { - await this.placeCall(true, true); - } - - /** - * Create a datachannel using this call's peer connection. - * @param label A human readable label for this datachannel - * @param options An object providing configuration options for the data channel. - */ - public createDataChannel(label: string, options: RTCDataChannelInit) { - const dataChannel = this.peerConn.createDataChannel(label, options); - this.emit(CallEvent.DataChannel, dataChannel); - return dataChannel; - } - - public getOpponentMember(): RoomMember { - return this.opponentMember; - } - - public getOpponentSessionId(): string { - return this.opponentSessionId; - } - - public opponentCanBeTransferred(): boolean { - return Boolean(this.opponentCaps && this.opponentCaps["m.call.transferee"]); - } - - public opponentSupportsDTMF(): boolean { - return Boolean(this.opponentCaps && this.opponentCaps["m.call.dtmf"]); - } - - public getRemoteAssertedIdentity(): AssertedIdentity { - return this.remoteAssertedIdentity; - } - - public get type(): CallType { - return (this.hasLocalUserMediaVideoTrack || this.hasRemoteUserMediaVideoTrack) - ? CallType.Video - : CallType.Voice; - } - - public get hasLocalUserMediaVideoTrack(): boolean { - return this.localUsermediaStream?.getVideoTracks().length > 0; - } - - public get hasRemoteUserMediaVideoTrack(): boolean { - return this.getRemoteFeeds().some((feed) => { - return ( - feed.purpose === SDPStreamMetadataPurpose.Usermedia && - feed.stream.getVideoTracks().length > 0 - ); - }); - } - - public get hasLocalUserMediaAudioTrack(): boolean { - return this.localUsermediaStream?.getAudioTracks().length > 0; - } - - public get hasRemoteUserMediaAudioTrack(): boolean { - return this.getRemoteFeeds().some((feed) => { - return ( - feed.purpose === SDPStreamMetadataPurpose.Usermedia && - feed.stream.getAudioTracks().length > 0 - ); - }); - } - - public get localUsermediaFeed(): CallFeed { - return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); - } - - public get localScreensharingFeed(): CallFeed { - return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); - } - - public get localUsermediaStream(): MediaStream { - return this.localUsermediaFeed?.stream; - } - - public get localScreensharingStream(): MediaStream { - return this.localScreensharingFeed?.stream; - } - - public get remoteUsermediaFeed(): CallFeed { - return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); - } - - public get remoteScreensharingFeed(): CallFeed { - return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); - } - - public get remoteUsermediaStream(): MediaStream { - return this.remoteUsermediaFeed?.stream; - } - - public get remoteScreensharingStream(): MediaStream { - return this.remoteScreensharingFeed?.stream; - } - - private getFeedByStreamId(streamId: string): CallFeed { - return this.getFeeds().find((feed) => feed.stream.id === streamId); - } - - /** - * Returns an array of all CallFeeds - * @returns {Array} CallFeeds - */ - public getFeeds(): Array { - return this.feeds; - } - - /** - * Returns an array of all local CallFeeds - * @returns {Array} local CallFeeds - */ - public getLocalFeeds(): Array { - return this.feeds.filter((feed) => feed.isLocal()); - } - - /** - * Returns an array of all remote CallFeeds - * @returns {Array} remote CallFeeds - */ - public getRemoteFeeds(): Array { - return this.feeds.filter((feed) => !feed.isLocal()); - } - - /** - * Generates and returns localSDPStreamMetadata - * @returns {SDPStreamMetadata} localSDPStreamMetadata - */ - private getLocalSDPStreamMetadata(updateStreamIds = false): SDPStreamMetadata { - const metadata: SDPStreamMetadata = {}; - for (const localFeed of this.getLocalFeeds()) { - if (updateStreamIds) { - localFeed.sdpMetadataStreamId = localFeed.stream.id; - } - - metadata[localFeed.sdpMetadataStreamId] = { - purpose: localFeed.purpose, - audio_muted: localFeed.isAudioMuted(), - video_muted: localFeed.isVideoMuted(), - }; - } - return metadata; - } - - /** - * Returns true if there are no incoming feeds, - * otherwise returns false - * @returns {boolean} no incoming feeds - */ - public noIncomingFeeds(): boolean { - return !this.feeds.some((feed) => !feed.isLocal()); - } - - private pushRemoteFeed(stream: MediaStream): void { - // Fallback to old behavior if the other side doesn't support SDPStreamMetadata - if (!this.opponentSupportsSDPStreamMetadata()) { - this.pushRemoteFeedWithoutMetadata(stream); - return; - } - - const userId = this.getOpponentMember().userId; - const purpose = this.remoteSDPStreamMetadata[stream.id].purpose; - const audioMuted = this.remoteSDPStreamMetadata[stream.id].audio_muted; - const videoMuted = this.remoteSDPStreamMetadata[stream.id].video_muted; - - if (!purpose) { - logger.warn(`Ignoring stream with id ${stream.id} because we didn't get any metadata about it`); - return; - } - - // Try to find a feed with the same purpose as the new stream, - // if we find it replace the old stream with the new one - const existingFeed = this.getRemoteFeeds().find((feed) => feed.purpose === purpose); - if (existingFeed) { - existingFeed.setNewStream(stream); - } else { - this.feeds.push(new CallFeed({ - client: this.client, - roomId: this.roomId, - userId, - stream, - purpose, - audioMuted, - videoMuted, - })); - this.emit(CallEvent.FeedsChanged, this.feeds); - } - - logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}", purpose=${purpose})`); - } - - /** - * This method is used ONLY if the other client doesn't support sending SDPStreamMetadata - */ - private pushRemoteFeedWithoutMetadata(stream: MediaStream): void { - const userId = this.getOpponentMember().userId; - // We can guess the purpose here since the other client can only send one stream - const purpose = SDPStreamMetadataPurpose.Usermedia; - const oldRemoteStream = this.feeds.find((feed) => !feed.isLocal())?.stream; - - // Note that we check by ID and always set the remote stream: Chrome appears - // to make new stream objects when transceiver directionality is changed and the 'active' - // status of streams change - Dave - // If we already have a stream, check this stream has the same id - if (oldRemoteStream && stream.id !== oldRemoteStream.id) { - logger.warn(`Ignoring new stream ID ${stream.id}: we already have stream ID ${oldRemoteStream.id}`); - return; - } - - // Try to find a feed with the same stream id as the new stream, - // if we find it replace the old stream with the new one - const feed = this.getFeedByStreamId(stream.id); - if (feed) { - feed.setNewStream(stream); - } else { - this.feeds.push(new CallFeed({ - client: this.client, - roomId: this.roomId, - audioMuted: false, - videoMuted: false, - userId, - stream, - purpose, - })); - this.emit(CallEvent.FeedsChanged, this.feeds); - } - - logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}")`); - } - - private pushNewLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose, addToPeerConnection = true): void { - const userId = this.client.getUserId(); - - // TODO: Find out what is going on here - // why do we enable audio (and only audio) tracks here? -- matthew - setTracksEnabled(stream.getAudioTracks(), true); - - // We try to replace an existing feed if there already is one with the same purpose - const existingFeed = this.getLocalFeeds().find((feed) => feed.purpose === purpose); - if (existingFeed) { - existingFeed.setNewStream(stream); - } else { - this.pushLocalFeed( - new CallFeed({ - client: this.client, - roomId: this.roomId, - audioMuted: stream.getAudioTracks().length === 0, - videoMuted: stream.getVideoTracks().length === 0, - userId, - stream, - purpose, - }), - addToPeerConnection, - ); - this.emit(CallEvent.FeedsChanged, this.feeds); - } - } - - /** - * Pushes supplied feed to the call - * @param {CallFeed} callFeed to push - * @param {boolean} addToPeerConnection whether to add the tracks to the peer connection - */ - public pushLocalFeed(callFeed: CallFeed, addToPeerConnection = true): void { - if (this.feeds.some((feed) => callFeed.stream.id === feed.stream.id)) { - logger.info(`Ignoring duplicate local stream ${callFeed.stream.id} in call ${this.callId}`); - return; - } - - this.feeds.push(callFeed); - - if (addToPeerConnection) { - const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia ? - this.usermediaSenders : this.screensharingSenders; - // Empty the array - senderArray.splice(0, senderArray.length); - - for (const track of callFeed.stream.getTracks()) { - logger.info( - `Adding track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${callFeed.stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - senderArray.push(this.peerConn.addTrack(track, callFeed.stream)); - } - } - - logger.info( - `Pushed local stream `+ - `(id="${callFeed.stream.id}", `+ - `active="${callFeed.stream.active}", `+ - `purpose="${callFeed.purpose}")`, - ); - - this.emit(CallEvent.FeedsChanged, this.feeds); - } - - /** - * Removes local call feed from the call and its tracks from the peer - * connection - * @param callFeed to remove - */ - public removeLocalFeed(callFeed: CallFeed): void { - const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia - ? this.usermediaSenders - : this.screensharingSenders; - - for (const sender of senderArray) { - this.peerConn.removeTrack(sender); - } - - if (callFeed.purpose === SDPStreamMetadataPurpose.Screenshare) { - this.client.getMediaHandler().stopScreensharingStream(callFeed.stream); - } - - // Empty the array - senderArray.splice(0, senderArray.length); - this.deleteFeed(callFeed); - } - - private deleteAllFeeds(): void { - for (const feed of this.feeds) { - if (!feed.isLocal() || !this.groupCallId) { - feed.dispose(); - } - } - - this.feeds = []; - this.emit(CallEvent.FeedsChanged, this.feeds); - } - - private deleteFeedByStream(stream: MediaStream): void { - const feed = this.getFeedByStreamId(stream.id); - if (!feed) { - logger.warn(`Didn't find the feed with stream id ${stream.id} to delete`); - return; - } - this.deleteFeed(feed); - } - - private deleteFeed(feed: CallFeed): void { - feed.dispose(); - this.feeds.splice(this.feeds.indexOf(feed), 1); - this.emit(CallEvent.FeedsChanged, this.feeds); - } - - // The typescript definitions have this type as 'any' :( - public async getCurrentCallStats(): Promise { - if (this.callHasEnded()) { - return this.callStatsAtEnd; - } - - return this.collectCallStats(); - } - - private async collectCallStats(): Promise { - // This happens when the call fails before it starts. - // For example when we fail to get capture sources - if (!this.peerConn) return; - - const statsReport = await this.peerConn.getStats(); - const stats = []; - for (const item of statsReport) { - stats.push(item[1]); - } - - return stats; - } - - /** - * Configure this call from an invite event. Used by MatrixClient. - * @param {MatrixEvent} event The m.call.invite event - */ - public async initWithInvite(event: MatrixEvent): Promise { - const invite = event.getContent(); - this.direction = CallDirection.Inbound; - - // make sure we have valid turn creds. Unless something's gone wrong, it should - // poll and keep the credentials valid so this should be instant. - const haveTurnCreds = await this.client.checkTurnServers(); - if (!haveTurnCreds) { - logger.warn("Failed to get TURN credentials! Proceeding with call anyway..."); - } - - const sdpStreamMetadata = invite[SDPStreamMetadataKey]; - if (sdpStreamMetadata) { - this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); - } else { - logger.debug("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); - } - - this.peerConn = this.createPeerConnection(); - // we must set the party ID before await-ing on anything: the call event - // handler will start giving us more call events (eg. candidates) so if - // we haven't set the party ID, we'll ignore them. - this.chooseOpponent(event); - try { - await this.peerConn.setRemoteDescription(invite.offer); - await this.addBufferedIceCandidates(); - } catch (e) { - logger.debug("Failed to set remote description", e); - this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); - return; - } - - const remoteStream = this.feeds.find((feed) => !feed.isLocal())?.stream; - - // According to previous comments in this file, firefox at some point did not - // add streams until media started arriving on them. Testing latest firefox - // (81 at time of writing), this is no longer a problem, so let's do it the correct way. - if (!remoteStream || remoteStream.getTracks().length === 0) { - logger.error("No remote stream or no tracks after setting remote description!"); - this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); - return; - } - - this.setState(CallState.Ringing); - - if (event.getLocalAge()) { - setTimeout(() => { - if (this.state == CallState.Ringing) { - logger.debug("Call invite has expired. Hanging up."); - this.hangupParty = CallParty.Remote; // effectively - this.setState(CallState.Ended); - this.stopAllMedia(); - if (this.peerConn.signalingState != 'closed') { - this.peerConn.close(); - } - this.emit(CallEvent.Hangup); - } - }, invite.lifetime - event.getLocalAge()); - } - } - - /** - * Configure this call from a hangup or reject event. Used by MatrixClient. - * @param {MatrixEvent} event The m.call.hangup event - */ - public initWithHangup(event: MatrixEvent): void { - // perverse as it may seem, sometimes we want to instantiate a call with a - // hangup message (because when getting the state of the room on load, events - // come in reverse order and we want to remember that a call has been hung up) - this.setState(CallState.Ended); - } - - private shouldAnswerWithMediaType( - wantedValue: boolean | undefined, valueOfTheOtherSide: boolean | undefined, type: "audio" | "video", - ): boolean { - if (wantedValue && !valueOfTheOtherSide) { - // TODO: Figure out how to do this - logger.warn(`Unable to answer with ${type} because the other side isn't sending it either.`); - return false; - } else if ( - !utils.isNullOrUndefined(wantedValue) && - wantedValue !== valueOfTheOtherSide && - !this.opponentSupportsSDPStreamMetadata() - ) { - logger.warn( - `Unable to answer with ${type}=${wantedValue} because the other side doesn't support it. ` + - `Answering with ${type}=${valueOfTheOtherSide}.`, - ); - return valueOfTheOtherSide; - } - return wantedValue ?? valueOfTheOtherSide; - } - - /** - * Answer a call. - */ - public async answer(audio?: boolean, video?: boolean): Promise { - if (this.inviteOrAnswerSent) return; - // TODO: Figure out how to do this - if (audio === false && video === false) throw new Error("You CANNOT answer a call without media"); - - if (!this.localUsermediaStream && !this.waitForLocalAVStream) { - const prevState = this.state; - const answerWithAudio = this.shouldAnswerWithMediaType(audio, this.hasRemoteUserMediaAudioTrack, "audio"); - const answerWithVideo = this.shouldAnswerWithMediaType(video, this.hasRemoteUserMediaVideoTrack, "video"); - - this.setState(CallState.WaitLocalMedia); - this.waitForLocalAVStream = true; - - try { - const stream = await this.client.getMediaHandler().getUserMediaStream( - answerWithAudio, answerWithVideo, - ); - this.waitForLocalAVStream = false; - const usermediaFeed = new CallFeed({ - client: this.client, - roomId: this.roomId, - userId: this.client.getUserId(), - stream, - purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: stream.getAudioTracks().length === 0, - videoMuted: stream.getVideoTracks().length === 0, - }); - - const feeds = [usermediaFeed]; - - if (this.localScreensharingFeed) { - feeds.push(this.localScreensharingFeed); - } - - this.answerWithCallFeeds(feeds); - } catch (e) { - if (answerWithVideo) { - // Try to answer without video - logger.warn("Failed to getUserMedia(), trying to getUserMedia() without video"); - this.setState(prevState); - this.waitForLocalAVStream = false; - await this.answer(answerWithAudio, false); - } else { - this.getUserMediaFailed(e); - return; - } - } - } else if (this.waitForLocalAVStream) { - this.setState(CallState.WaitLocalMedia); - } - } - - public answerWithCallFeeds(callFeeds: CallFeed[]): void { - if (this.inviteOrAnswerSent) return; - - logger.debug(`Answering call ${this.callId}`); - - this.queueGotCallFeedsForAnswer(callFeeds); - } - - /** - * Replace this call with a new call, e.g. for glare resolution. Used by - * MatrixClient. - * @param {MatrixCall} newCall The new call. - */ - public replacedBy(newCall: MatrixCall): void { - if (this.state === CallState.WaitLocalMedia) { - logger.debug("Telling new call to wait for local media"); - newCall.waitForLocalAVStream = true; - } else if ([CallState.CreateOffer, CallState.InviteSent].includes(this.state)) { - if (newCall.direction === CallDirection.Outbound) { - newCall.queueGotCallFeedsForAnswer([]); - } else { - logger.debug("Handing local stream to new call"); - newCall.queueGotCallFeedsForAnswer(this.getLocalFeeds().map(feed => feed.clone())); - } - } - this.successor = newCall; - this.emit(CallEvent.Replaced, newCall); - this.hangup(CallErrorCode.Replaced, true); - } - - /** - * Hangup a call. - * @param {string} reason The reason why the call is being hung up. - * @param {boolean} suppressEvent True to suppress emitting an event. - */ - public hangup(reason: CallErrorCode, suppressEvent: boolean): void { - if (this.callHasEnded()) return; - - logger.debug("Ending call " + this.callId); - this.terminate(CallParty.Local, reason, !suppressEvent); - // We don't want to send hangup here if we didn't even get to sending an invite - if (this.state === CallState.WaitLocalMedia) return; - const content = {}; - // Don't send UserHangup reason to older clients - if ((this.opponentVersion && this.opponentVersion >= 1) || reason !== CallErrorCode.UserHangup) { - content["reason"] = reason; - } - this.sendVoipEvent(EventType.CallHangup, content); - } - - /** - * Reject a call - * This used to be done by calling hangup, but is a separate method and protocol - * event as of MSC2746. - */ - public reject(): void { - if (this.state !== CallState.Ringing) { - throw Error("Call must be in 'ringing' state to reject!"); - } - - if (this.opponentVersion < 1) { - logger.info( - `Opponent version is less than 1 (${this.opponentVersion}): sending hangup instead of reject`, - ); - this.hangup(CallErrorCode.UserHangup, true); - return; - } - - logger.debug("Rejecting call: " + this.callId); - this.terminate(CallParty.Local, CallErrorCode.UserHangup, true); - this.sendVoipEvent(EventType.CallReject, {}); - } - - /** - * Adds an audio and/or video track - upgrades the call - * @param {boolean} audio should add an audio track - * @param {boolean} video should add an video track - */ - private async upgradeCall( - audio: boolean, video: boolean, - ): Promise { - // We don't do call downgrades - if (!audio && !video) return; - if (!this.opponentSupportsSDPStreamMetadata()) return; - - try { - const upgradeAudio = audio && !this.hasLocalUserMediaAudioTrack; - const upgradeVideo = video && !this.hasLocalUserMediaVideoTrack; - logger.debug(`Upgrading call: audio?=${upgradeAudio} video?=${upgradeVideo}`); - - const stream = await this.client.getMediaHandler().getUserMediaStream(upgradeAudio, upgradeVideo); - if (upgradeAudio && upgradeVideo) { - if (this.hasLocalUserMediaAudioTrack) return; - if (this.hasLocalUserMediaVideoTrack) return; - - this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); - } else if (upgradeAudio) { - if (this.hasLocalUserMediaAudioTrack) return; - - const audioTrack = stream.getAudioTracks()[0]; - this.localUsermediaStream.addTrack(audioTrack); - this.peerConn.addTrack(audioTrack, this.localUsermediaStream); - } else if (upgradeVideo) { - if (this.hasLocalUserMediaVideoTrack) return; - - const videoTrack = stream.getVideoTracks()[0]; - this.localUsermediaStream.addTrack(videoTrack); - this.peerConn.addTrack(videoTrack, this.localUsermediaStream); - } - } catch (error) { - logger.error("Failed to upgrade the call", error); - this.emit(CallEvent.Error, - new CallError(CallErrorCode.NoUserMedia, "Failed to get camera access: ", error), - ); - } - } - - /** - * Returns true if this.remoteSDPStreamMetadata is defined, otherwise returns false - * @returns {boolean} can screenshare - */ - public opponentSupportsSDPStreamMetadata(): boolean { - return Boolean(this.remoteSDPStreamMetadata); - } - - /** - * If there is a screensharing stream returns true, otherwise returns false - * @returns {boolean} is screensharing - */ - public isScreensharing(): boolean { - return Boolean(this.localScreensharingStream); - } - - /** - * Starts/stops screensharing - * @param enabled the desired screensharing state - * @param {string} desktopCapturerSourceId optional id of the desktop capturer source to use - * @returns {boolean} new screensharing state - */ - public async setScreensharingEnabled( - enabled: boolean, desktopCapturerSourceId?: string, - ): Promise { - // Skip if there is nothing to do - if (enabled && this.isScreensharing()) { - logger.warn(`There is already a screensharing stream - there is nothing to do!`); - return true; - } else if (!enabled && !this.isScreensharing()) { - logger.warn(`There already isn't a screensharing stream - there is nothing to do!`); - return false; - } - - // Fallback to replaceTrack() - if (!this.opponentSupportsSDPStreamMetadata()) { - return await this.setScreensharingEnabledWithoutMetadataSupport(enabled, desktopCapturerSourceId); - } - - logger.debug(`Set screensharing enabled? ${enabled}`); - if (enabled) { - try { - const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId); - if (!stream) return false; - this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare); - return true; - } catch (err) { - logger.error("Failed to get screen-sharing stream:", err); - return false; - } - } else { - for (const sender of this.screensharingSenders) { - this.peerConn.removeTrack(sender); - } - this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream); - this.deleteFeedByStream(this.localScreensharingStream); - return false; - } - } - - /** - * Starts/stops screensharing - * Should be used ONLY if the opponent doesn't support SDPStreamMetadata - * @param enabled the desired screensharing state - * @param {string} desktopCapturerSourceId optional id of the desktop capturer source to use - * @returns {boolean} new screensharing state - */ - private async setScreensharingEnabledWithoutMetadataSupport( - enabled: boolean, desktopCapturerSourceId?: string, - ): Promise { - logger.debug(`Set screensharing enabled? ${enabled} using replaceTrack()`); - if (enabled) { - try { - const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId); - if (!stream) return false; - - const track = stream.getTracks().find((track) => { - return track.kind === "video"; - }); - const sender = this.usermediaSenders.find((sender) => { - return sender.track?.kind === "video"; - }); - sender.replaceTrack(track); - - this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare, false); - - return true; - } catch (err) { - logger.error("Failed to get screen-sharing stream:", err); - return false; - } - } else { - const track = this.localUsermediaStream.getTracks().find((track) => { - return track.kind === "video"; - }); - const sender = this.usermediaSenders.find((sender) => { - return sender.track?.kind === "video"; - }); - sender.replaceTrack(track); - - this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream); - this.deleteFeedByStream(this.localScreensharingStream); - - return false; - } - } - - /** - * Request a new local usermedia stream with the current device id. - */ - public async updateLocalUsermediaStream(stream: MediaStream) { - const callFeed = this.localUsermediaFeed; - callFeed.setNewStream(stream); - const micShouldBeMuted = callFeed.isAudioMuted() || this.remoteOnHold; - const vidShouldBeMuted = callFeed.isVideoMuted() || this.remoteOnHold; - setTracksEnabled(stream.getAudioTracks(), !micShouldBeMuted); - setTracksEnabled(stream.getVideoTracks(), !vidShouldBeMuted); - - const newSenders = []; - - for (const track of stream.getTracks()) { - const oldSender = this.usermediaSenders.find((sender) => { - return sender.track?.kind === track.kind; - }); - - let newSender: RTCRtpSender; - - try { - logger.info( - `Replacing track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - await oldSender.replaceTrack(track); - newSender = oldSender; - } catch (error) { - logger.info( - `Adding track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - newSender = this.peerConn.addTrack(track, stream); - } - - newSenders.push(newSender); - } - - this.usermediaSenders = newSenders; - } - - /** - * Set whether our outbound video should be muted or not. - * @param {boolean} muted True to mute the outbound video. - * @returns the new mute state - */ - public async setLocalVideoMuted(muted: boolean): Promise { - if (!await this.client.getMediaHandler().hasVideoDevice()) { - return this.isLocalVideoMuted(); - } - - if (!this.hasLocalUserMediaVideoTrack && !muted) { - await this.upgradeCall(false, true); - return this.isLocalVideoMuted(); - } - this.localUsermediaFeed?.setVideoMuted(muted); - this.updateMuteStatus(); - return this.isLocalVideoMuted(); - } - - /** - * Check if local video is muted. - * - * If there are multiple video tracks, all of the tracks need to be muted - * for this to return true. This means if there are no video tracks, this will - * return true. - * @return {Boolean} True if the local preview video is muted, else false - * (including if the call is not set up yet). - */ - public isLocalVideoMuted(): boolean { - return this.localUsermediaFeed?.isVideoMuted(); - } - - /** - * Set whether the microphone should be muted or not. - * @param {boolean} muted True to mute the mic. - * @returns the new mute state - */ - public async setMicrophoneMuted(muted: boolean): Promise { - if (!await this.client.getMediaHandler().hasAudioDevice()) { - return this.isMicrophoneMuted(); - } - - if (!this.hasLocalUserMediaAudioTrack && !muted) { - await this.upgradeCall(true, false); - return this.isMicrophoneMuted(); - } - this.localUsermediaFeed?.setAudioMuted(muted); - this.updateMuteStatus(); - return this.isMicrophoneMuted(); - } - - /** - * Check if the microphone is muted. - * - * If there are multiple audio tracks, all of the tracks need to be muted - * for this to return true. This means if there are no audio tracks, this will - * return true. - * @return {Boolean} True if the mic is muted, else false (including if the call - * is not set up yet). - */ - public isMicrophoneMuted(): boolean { - return this.localUsermediaFeed?.isAudioMuted(); - } - - /** - * @returns true if we have put the party on the other side of the call on hold - * (that is, we are signalling to them that we are not listening) - */ - public isRemoteOnHold(): boolean { - return this.remoteOnHold; - } - - public setRemoteOnHold(onHold: boolean): void { - if (this.isRemoteOnHold() === onHold) return; - this.remoteOnHold = onHold; - - for (const transceiver of this.peerConn.getTransceivers()) { - // We don't send hold music or anything so we're not actually - // sending anything, but sendrecv is fairly standard for hold and - // it makes it a lot easier to figure out who's put who on hold. - transceiver.direction = onHold ? 'sendonly' : 'sendrecv'; - } - this.updateMuteStatus(); - - this.emit(CallEvent.RemoteHoldUnhold, this.remoteOnHold); - } - - /** - * Indicates whether we are 'on hold' to the remote party (ie. if true, - * they cannot hear us). - * @returns true if the other party has put us on hold - */ - public isLocalOnHold(): boolean { - if (this.state !== CallState.Connected) return false; - - let callOnHold = true; - - // We consider a call to be on hold only if *all* the tracks are on hold - // (is this the right thing to do?) - for (const transceiver of this.peerConn.getTransceivers()) { - const trackOnHold = ['inactive', 'recvonly'].includes(transceiver.currentDirection); - - if (!trackOnHold) callOnHold = false; - } - - return callOnHold; - } - - /** - * Sends a DTMF digit to the other party - * @param digit The digit (nb. string - '#' and '*' are dtmf too) - */ - public sendDtmfDigit(digit: string): void { - for (const sender of this.peerConn.getSenders()) { - if (sender.track.kind === 'audio' && sender.dtmf) { - sender.dtmf.insertDTMF(digit); - return; - } - } - - throw new Error("Unable to find a track to send DTMF on"); - } - - private updateMuteStatus(): void { - this.sendVoipEvent(EventType.CallSDPStreamMetadataChangedPrefix, { - [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(), - }); - - const micShouldBeMuted = this.localUsermediaFeed?.isAudioMuted() || this.remoteOnHold; - const vidShouldBeMuted = this.localUsermediaFeed?.isVideoMuted() || this.remoteOnHold; - - setTracksEnabled(this.localUsermediaStream.getAudioTracks(), !micShouldBeMuted); - setTracksEnabled(this.localUsermediaStream.getVideoTracks(), !vidShouldBeMuted); - } - - private gotCallFeedsForInvite(callFeeds: CallFeed[], requestScreenshareFeed = false): void { - if (this.successor) { - this.successor.queueGotCallFeedsForAnswer(callFeeds); - return; - } - if (this.callHasEnded()) { - this.stopAllMedia(); - return; - } - - for (const feed of callFeeds) { - this.pushLocalFeed(feed); - } - - if (requestScreenshareFeed) { - this.peerConn.addTransceiver("video", { - direction: "recvonly", - }); - } - - this.setState(CallState.CreateOffer); - - logger.debug("gotUserMediaForInvite"); - // Now we wait for the negotiationneeded event - } - - private async sendAnswer(): Promise { - const answerContent = { - answer: { - sdp: this.peerConn.localDescription.sdp, - // type is now deprecated as of Matrix VoIP v1, but - // required to still be sent for backwards compat - type: this.peerConn.localDescription.type, - }, - [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(true), - } as MCallAnswer; - - answerContent.capabilities = { - 'm.call.transferee': this.client.supportsCallTransfer, - 'm.call.dtmf': false, - }; - - // We have just taken the local description from the peerConn which will - // contain all the local candidates added so far, so we can discard any candidates - // we had queued up because they'll be in the answer. - logger.info(`Discarding ${this.candidateSendQueue.length} candidates that will be sent in answer`); - this.candidateSendQueue = []; - - try { - await this.sendVoipEvent(EventType.CallAnswer, answerContent); - // If this isn't the first time we've tried to send the answer, - // we may have candidates queued up, so send them now. - this.inviteOrAnswerSent = true; - } catch (error) { - // We've failed to answer: back to the ringing state - this.setState(CallState.Ringing); - this.client.cancelPendingEvent(error.event); - - let code = CallErrorCode.SendAnswer; - let message = "Failed to send answer"; - if (error.name == 'UnknownDeviceError') { - code = CallErrorCode.UnknownDevices; - message = "Unknown devices present in the room"; - } - this.emit(CallEvent.Error, new CallError(code, message, error)); - throw error; - } - - // error handler re-throws so this won't happen on error, but - // we don't want the same error handling on the candidate queue - this.sendCandidateQueue(); - } - - private queueGotCallFeedsForAnswer(callFeeds: CallFeed[]): void { - // Ensure only one negotiate/answer event is being processed at a time. - if (this.responsePromiseChain) { - this.responsePromiseChain = - this.responsePromiseChain.then(() => this.gotCallFeedsForAnswer(callFeeds)); - } else { - this.responsePromiseChain = this.gotCallFeedsForAnswer(callFeeds); - } - } - - private async gotCallFeedsForAnswer(callFeeds: CallFeed[]): Promise { - if (this.callHasEnded()) return; - - this.waitForLocalAVStream = false; - - for (const feed of callFeeds) { - this.pushLocalFeed(feed); - } - - this.setState(CallState.CreateAnswer); - - let myAnswer; - try { - this.getRidOfRTXCodecs(); - myAnswer = await this.peerConn.createAnswer(); - } catch (err) { - logger.debug("Failed to create answer: ", err); - this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true); - return; - } - - try { - await this.peerConn.setLocalDescription(myAnswer); - this.setState(CallState.Connecting); - - // Allow a short time for initial candidates to be gathered - await new Promise(resolve => { - setTimeout(resolve, 200); - }); - - this.sendAnswer(); - } catch (err) { - logger.debug("Error setting local description!", err); - this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); - return; - } - - // HACK: Safari doesn't like it when we reuse MediaStreams. In most cases - // we can get around this by calling MediaStream.clone(), however inbound - // calls seem to still be broken unless we getUserMedia again and replace - // all MediaStreams using sender.replaceTrack - if (isSafari) { - await new Promise(resolve => { - setTimeout(resolve, 200); - }); - - if (this.state === CallState.Ended) { - return; - } - - const callFeed = this.localUsermediaFeed; - const stream = callFeed.stream; - - if (!stream.active) { - throw new Error(`Call ${this.callId} has an inactive stream ${ - stream.id} and its tracks cannot be replaced`); - } - - const newSenders = []; - - for (const track of this.localUsermediaStream.getTracks()) { - const oldSender = this.usermediaSenders.find((sender) => { - return sender.track?.kind === track.kind; - }); - - if (track.readyState === "ended") { - throw new Error(`Call ${this.callId} tried to replace track ${track.id} in the ended state`); - } - - let newSender: RTCRtpSender; - - try { - logger.info( - `Replacing track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - await oldSender.replaceTrack(track); - newSender = oldSender; - } catch (error) { - logger.info( - `Adding track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - newSender = this.peerConn.addTrack(track, stream); - } - - newSenders.push(newSender); - } - - this.usermediaSenders = newSenders; - } - } - - /** - * Internal - * @param {Object} event - */ - private gotLocalIceCandidate = (event: RTCPeerConnectionIceEvent): Promise => { - if (event.candidate) { - logger.debug( - "Call " + this.callId + " got local ICE " + event.candidate.sdpMid + " candidate: " + - event.candidate.candidate, - ); - - if (this.callHasEnded()) return; - - // As with the offer, note we need to make a copy of this object, not - // pass the original: that broke in Chrome ~m43. - if (event.candidate.candidate !== '' || !this.sentEndOfCandidates) { - this.queueCandidate(event.candidate); - - if (event.candidate.candidate === '') this.sentEndOfCandidates = true; - } - } - }; - - private onIceGatheringStateChange = (event: Event): void => { - logger.debug("ice gathering state changed to " + this.peerConn.iceGatheringState); - if (this.peerConn.iceGatheringState === 'complete' && !this.sentEndOfCandidates) { - // If we didn't get an empty-string candidate to signal the end of candidates, - // create one ourselves now gathering has finished. - // We cast because the interface lists all the properties as required but we - // only want to send 'candidate' - // XXX: We probably want to send either sdpMid or sdpMLineIndex, as it's not strictly - // correct to have a candidate that lacks both of these. We'd have to figure out what - // previous candidates had been sent with and copy them. - const c = { - candidate: '', - } as RTCIceCandidate; - this.queueCandidate(c); - this.sentEndOfCandidates = true; - } - }; - - public async onRemoteIceCandidatesReceived(ev: MatrixEvent): Promise { - if (this.callHasEnded()) { - //debuglog("Ignoring remote ICE candidate because call has ended"); - return; - } - - const content = ev.getContent(); - const candidates = content.candidates; - if (!candidates) { - logger.info("Ignoring candidates event with no candidates!"); - return; - } - - const fromPartyId = content.version === 0 ? null : content.party_id || null; - - if (this.opponentPartyId === undefined) { - // we haven't picked an opponent yet so save the candidates - logger.info(`Buffering ${candidates.length} candidates until we pick an opponent`); - const bufferedCandidates = this.remoteCandidateBuffer.get(fromPartyId) || []; - bufferedCandidates.push(...candidates); - this.remoteCandidateBuffer.set(fromPartyId, bufferedCandidates); - return; - } - - if (!this.partyIdMatches(content)) { - logger.info( - `Ignoring candidates from party ID ${content.party_id}: ` + - `we have chosen party ID ${this.opponentPartyId}`, - ); - - return; - } - - await this.addIceCandidates(candidates); - } - - /** - * Used by MatrixClient. - * @param {Object} msg - */ - public async onAnswerReceived(event: MatrixEvent): Promise { - const content = event.getContent(); - logger.debug(`Got answer for call ID ${this.callId} from party ID ${content.party_id}`); - - if (this.callHasEnded()) { - logger.debug(`Ignoring answer because call ID ${this.callId} has ended`); - return; - } - - if (this.opponentPartyId !== undefined) { - logger.info( - `Ignoring answer from party ID ${content.party_id}: ` + - `we already have an answer/reject from ${this.opponentPartyId}`, - ); - return; - } - - this.chooseOpponent(event); - await this.addBufferedIceCandidates(); - - this.setState(CallState.Connecting); - - const sdpStreamMetadata = content[SDPStreamMetadataKey]; - if (sdpStreamMetadata) { - this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); - } else { - logger.warn("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); - } - - try { - await this.peerConn.setRemoteDescription(content.answer); - } catch (e) { - logger.debug("Failed to set remote description", e); - this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); - return; - } - - // If the answer we selected has a party_id, send a select_answer event - // We do this after setting the remote description since otherwise we'd block - // call setup on it - if (this.opponentPartyId !== null) { - try { - await this.sendVoipEvent(EventType.CallSelectAnswer, { - selected_party_id: this.opponentPartyId, - }); - } catch (err) { - // This isn't fatal, and will just mean that if another party has raced to answer - // the call, they won't know they got rejected, so we carry on & don't retry. - logger.warn("Failed to send select_answer event", err); - } - } - } - - public async onSelectAnswerReceived(event: MatrixEvent): Promise { - if (this.direction !== CallDirection.Inbound) { - logger.warn("Got select_answer for an outbound call: ignoring"); - return; - } - - const selectedPartyId = event.getContent().selected_party_id; - - if (selectedPartyId === undefined || selectedPartyId === null) { - logger.warn("Got nonsensical select_answer with null/undefined selected_party_id: ignoring"); - return; - } - - if (selectedPartyId !== this.ourPartyId) { - logger.info(`Got select_answer for party ID ${selectedPartyId}: we are party ID ${this.ourPartyId}.`); - // The other party has picked somebody else's answer - this.terminate(CallParty.Remote, CallErrorCode.AnsweredElsewhere, true); - } - } - - public async onNegotiateReceived(event: MatrixEvent): Promise { - const content = event.getContent(); - const description = content.description; - if (!description || !description.sdp || !description.type) { - logger.info("Ignoring invalid m.call.negotiate event"); - return; - } - // Politeness always follows the direction of the call: in a glare situation, - // we pick either the inbound or outbound call, so one side will always be - // inbound and one outbound - const polite = this.direction === CallDirection.Inbound; - - // Here we follow the perfect negotiation logic from - // https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation - const offerCollision = ( - (description.type === 'offer') && - (this.makingOffer || this.peerConn.signalingState !== 'stable') - ); - - this.ignoreOffer = !polite && offerCollision; - if (this.ignoreOffer) { - logger.info("Ignoring colliding negotiate event because we're impolite"); - return; - } - - const prevLocalOnHold = this.isLocalOnHold(); - - const sdpStreamMetadata = content[SDPStreamMetadataKey]; - if (sdpStreamMetadata) { - this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); - } else { - logger.warn("Received negotiation event without SDPStreamMetadata!"); - } - - try { - await this.peerConn.setRemoteDescription(description); - - if (description.type === 'offer') { - this.getRidOfRTXCodecs(); - await this.peerConn.setLocalDescription(); - - this.sendVoipEvent(EventType.CallNegotiate, { - description: this.peerConn.localDescription, - [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(true), - }); - } - } catch (err) { - logger.warn("Failed to complete negotiation", err); - } - - const newLocalOnHold = this.isLocalOnHold(); - if (prevLocalOnHold !== newLocalOnHold) { - this.emit(CallEvent.LocalHoldUnhold, newLocalOnHold); - // also this one for backwards compat - this.emit(CallEvent.HoldUnhold, newLocalOnHold); - } - } - - private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { - this.remoteSDPStreamMetadata = utils.recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); - for (const feed of this.getRemoteFeeds()) { - const streamId = feed.stream.id; - feed.setAudioMuted(this.remoteSDPStreamMetadata[streamId]?.audio_muted); - feed.setVideoMuted(this.remoteSDPStreamMetadata[streamId]?.video_muted); - feed.purpose = this.remoteSDPStreamMetadata[streamId]?.purpose; - } - } - - public onSDPStreamMetadataChangedReceived(event: MatrixEvent): void { - const content = event.getContent(); - const metadata = content[SDPStreamMetadataKey]; - this.updateRemoteSDPStreamMetadata(metadata); - } - - public async onAssertedIdentityReceived(event: MatrixEvent): Promise { - const content = event.getContent(); - if (!content.asserted_identity) return; - - this.remoteAssertedIdentity = { - id: content.asserted_identity.id, - displayName: content.asserted_identity.display_name, - }; - this.emit(CallEvent.AssertedIdentityChanged); - } - - private callHasEnded(): boolean { - // This exists as workaround to typescript trying to be clever and erroring - // when putting if (this.state === CallState.Ended) return; twice in the same - // function, even though that function is async. - return this.state === CallState.Ended; - } - - private queueGotLocalOffer(): void { - // Ensure only one negotiate/answer event is being processed at a time. - if (this.responsePromiseChain) { - this.responsePromiseChain = - this.responsePromiseChain.then(() => this.wrappedGotLocalOffer()); - } else { - this.responsePromiseChain = this.wrappedGotLocalOffer(); - } - } - - private async wrappedGotLocalOffer(): Promise { - this.makingOffer = true; - try { - this.getRidOfRTXCodecs(); - await this.gotLocalOffer(); - } catch (e) { - this.getLocalOfferFailed(e); - return; - } finally { - this.makingOffer = false; - } - } - - private async gotLocalOffer(): Promise { - logger.debug("Setting local description"); - - if (this.callHasEnded()) { - logger.debug("Ignoring newly created offer on call ID " + this.callId + - " because the call has ended"); - return; - } - - try { - await this.peerConn.setLocalDescription(); - } catch (err) { - logger.debug("Error setting local description!", err); - this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); - return; - } - - if (this.peerConn.iceGatheringState === 'gathering') { - // Allow a short time for initial candidates to be gathered - await new Promise(resolve => { - setTimeout(resolve, 200); - }); - } - - if (this.callHasEnded()) return; - - const eventType = this.state === CallState.CreateOffer ? EventType.CallInvite : EventType.CallNegotiate; - - const content = { - lifetime: CALL_TIMEOUT_MS, - } as MCallInviteNegotiate; - - if (eventType === EventType.CallInvite && this.invitee) { - content.invitee = this.invitee; - } - - // clunky because TypeScript can't follow the types through if we use an expression as the key - if (this.state === CallState.CreateOffer) { - content.offer = this.peerConn.localDescription; - } else { - content.description = this.peerConn.localDescription; - } - - content.capabilities = { - 'm.call.transferee': this.client.supportsCallTransfer, - 'm.call.dtmf': false, - }; - - content[SDPStreamMetadataKey] = this.getLocalSDPStreamMetadata(true); - - // Get rid of any candidates waiting to be sent: they'll be included in the local - // description we just got and will send in the offer. - logger.info(`Discarding ${this.candidateSendQueue.length} candidates that will be sent in offer`); - this.candidateSendQueue = []; - - try { - await this.sendVoipEvent(eventType, content); - } catch (error) { - logger.error("Failed to send invite", error); - if (error.event) this.client.cancelPendingEvent(error.event); - - let code = CallErrorCode.SignallingFailed; - let message = "Signalling failed"; - if (this.state === CallState.CreateOffer) { - code = CallErrorCode.SendInvite; - message = "Failed to send invite"; - } - if (error.name == 'UnknownDeviceError') { - code = CallErrorCode.UnknownDevices; - message = "Unknown devices present in the room"; - } - - this.emit(CallEvent.Error, new CallError(code, message, error)); - this.terminate(CallParty.Local, code, false); - - // no need to carry on & send the candidate queue, but we also - // don't want to rethrow the error - return; - } - - this.sendCandidateQueue(); - if (this.state === CallState.CreateOffer) { - this.inviteOrAnswerSent = true; - this.setState(CallState.InviteSent); - this.inviteTimeout = setTimeout(() => { - this.inviteTimeout = null; - if (this.state === CallState.InviteSent) { - this.hangup(CallErrorCode.InviteTimeout, false); - } - }, CALL_TIMEOUT_MS); - } - } - - private getLocalOfferFailed = (err: Error): void => { - logger.error("Failed to get local offer", err); - - this.emit( - CallEvent.Error, - new CallError( - CallErrorCode.LocalOfferFailed, - "Failed to get local offer!", err, - ), - ); - this.terminate(CallParty.Local, CallErrorCode.LocalOfferFailed, false); - }; - - private getUserMediaFailed = (err: Error): void => { - if (this.successor) { - this.successor.getUserMediaFailed(err); - return; - } - - logger.warn("Failed to get user media - ending call", err); - - this.emit( - CallEvent.Error, - new CallError( - CallErrorCode.NoUserMedia, - "Couldn't start capturing media! Is your microphone set up and " + - "does this app have permission?", err, - ), - ); - this.terminate(CallParty.Local, CallErrorCode.NoUserMedia, false); - }; - - private onIceConnectionStateChanged = (): void => { - if (this.callHasEnded()) { - return; // because ICE can still complete as we're ending the call - } - logger.debug( - "Call ID " + this.callId + ": ICE connection state changed to: " + this.peerConn.iceConnectionState, - ); - // ideally we'd consider the call to be connected when we get media but - // chrome doesn't implement any of the 'onstarted' events yet - if (this.peerConn.iceConnectionState == 'connected') { - clearTimeout(this.iceDisconnectedTimeout); - this.setState(CallState.Connected); - - if (!this.callLengthInterval) { - this.callLengthInterval = setInterval(() => { - this.callLength++; - this.emit(CallEvent.LengthChanged, this.callLength); - }, 1000); - } - } else if (this.peerConn.iceConnectionState == 'failed') { - // Firefox for Android does not yet have support for restartIce() - if (this.peerConn.restartIce) { - this.peerConn.restartIce(); - } else { - this.hangup(CallErrorCode.IceFailed, false); - } - } else if (this.peerConn.iceConnectionState == 'disconnected') { - this.iceDisconnectedTimeout = setTimeout(() => { - this.hangup(CallErrorCode.IceFailed, false); - }, 30 * 1000); - } - }; - - private onSignallingStateChanged = (): void => { - logger.debug( - "call " + this.callId + ": Signalling state changed to: " + - this.peerConn.signalingState, - ); - }; - - private onTrack = (ev: RTCTrackEvent): void => { - if (ev.streams.length === 0) { - logger.warn(`Streamless ${ev.track.kind} found: ignoring.`); - return; - } - - const stream = ev.streams[0]; - this.pushRemoteFeed(stream); - stream.addEventListener("removetrack", () => { - logger.log(`Removing track streamId: ${stream.id}`); - this.deleteFeedByStream(stream); - }); - }; - - private onDataChannel = (ev: RTCDataChannelEvent): void => { - this.emit(CallEvent.DataChannel, ev.channel); - }; - - /** - * This method removes all video/rtx codecs from screensharing video - * transceivers. This is necessary since they can cause problems. Without - * this the following steps should produce an error: - * Chromium calls Firefox - * Firefox answers - * Firefox starts screen-sharing - * Chromium starts screen-sharing - * Call crashes for Chromium with: - * [96685:23:0518/162603.933321:ERROR:webrtc_video_engine.cc(3296)] RTX codec (PT=97) mapped to PT=96 which is not in the codec list. - * [96685:23:0518/162603.933377:ERROR:webrtc_video_engine.cc(1171)] GetChangedRecvParameters called without any video codecs. - * [96685:23:0518/162603.933430:ERROR:sdp_offer_answer.cc(4302)] Failed to set local video description recv parameters for m-section with mid='2'. (INVALID_PARAMETER) - */ - private getRidOfRTXCodecs(): void { - // RTCRtpReceiver.getCapabilities and RTCRtpSender.getCapabilities don't seem to be supported on FF - if (!RTCRtpReceiver.getCapabilities || !RTCRtpSender.getCapabilities) return; - - const recvCodecs = RTCRtpReceiver.getCapabilities("video").codecs; - const sendCodecs = RTCRtpSender.getCapabilities("video").codecs; - const codecs = [...sendCodecs, ...recvCodecs]; - - for (const codec of codecs) { - if (codec.mimeType === "video/rtx") { - const rtxCodecIndex = codecs.indexOf(codec); - codecs.splice(rtxCodecIndex, 1); - } - } - - for (const trans of this.peerConn.getTransceivers()) { - if ( - this.screensharingSenders.includes(trans.sender) && - ( - trans.sender.track?.kind === "video" || - trans.receiver.track?.kind === "video" - ) - ) { - trans.setCodecPreferences(codecs); - } - } - } - - private onNegotiationNeeded = async (): Promise => { - logger.info("Negotiation is needed!"); - - if (this.state !== CallState.CreateOffer && this.opponentVersion === 0) { - logger.info("Opponent does not support renegotiation: ignoring negotiationneeded event"); - return; - } - - this.queueGotLocalOffer(); - }; - - public onHangupReceived = (msg: MCallHangupReject): void => { - logger.debug("Hangup received for call ID " + this.callId); - - // party ID must match (our chosen partner hanging up the call) or be undefined (we haven't chosen - // a partner yet but we're treating the hangup as a reject as per VoIP v0) - if (this.partyIdMatches(msg) || this.state === CallState.Ringing) { - // default reason is user_hangup - this.terminate(CallParty.Remote, msg.reason || CallErrorCode.UserHangup, true); - } else { - logger.info(`Ignoring message from party ID ${msg.party_id}: our partner is ${this.opponentPartyId}`); - } - }; - - public onRejectReceived = (msg: MCallHangupReject): void => { - logger.debug("Reject received for call ID " + this.callId); - - // No need to check party_id for reject because if we'd received either - // an answer or reject, we wouldn't be in state InviteSent - - const shouldTerminate = ( - // reject events also end the call if it's ringing: it's another of - // our devices rejecting the call. - ([CallState.InviteSent, CallState.Ringing].includes(this.state)) || - // also if we're in the init state and it's an inbound call, since - // this means we just haven't entered the ringing state yet - this.state === CallState.Fledgling && this.direction === CallDirection.Inbound - ); - - if (shouldTerminate) { - this.terminate(CallParty.Remote, msg.reason || CallErrorCode.UserHangup, true); - } else { - logger.debug(`Call is in state: ${this.state}: ignoring reject`); - } - }; - - public onAnsweredElsewhere = (msg: MCallAnswer): void => { - logger.debug("Call ID " + this.callId + " answered elsewhere"); - this.terminate(CallParty.Remote, CallErrorCode.AnsweredElsewhere, true); - }; - - private setState(state: CallState): void { - const oldState = this.state; - this.state = state; - this.emit(CallEvent.State, state, oldState); - } - - /** - * Internal - * @param {string} eventType - * @param {Object} content - * @return {Promise} - */ - private sendVoipEvent(eventType: string, content: object): Promise { - const realContent = Object.assign({}, content, { - version: VOIP_PROTO_VERSION, - call_id: this.callId, - party_id: this.ourPartyId, - conf_id: this.groupCallId, - }); - - if (this.opponentDeviceId) { - this.emit(CallEvent.SendVoipEvent, { - type: "toDevice", - eventType, - userId: this.invitee || this.getOpponentMember().userId, - opponentDeviceId: this.opponentDeviceId, - content: { - ...realContent, - device_id: this.client.deviceId, - sender_session_id: this.client.getSessionId(), - dest_session_id: this.opponentSessionId, - }, - }); - - return this.client.sendToDevice(eventType, { - [this.invitee || this.getOpponentMember().userId]: { - [this.opponentDeviceId]: { - ...realContent, - device_id: this.client.deviceId, - sender_session_id: this.client.getSessionId(), - dest_session_id: this.opponentSessionId, - }, - }, - }); - } else { - this.emit(CallEvent.SendVoipEvent, { - type: "sendEvent", - eventType, - roomId: this.roomId, - content: realContent, - userId: this.invitee || this.getOpponentMember().userId, - }); - - return this.client.sendEvent(this.roomId, eventType, realContent); - } - } - - private queueCandidate(content: RTCIceCandidate): void { - // We partially de-trickle candidates by waiting for `delay` before sending them - // amalgamated, in order to avoid sending too many m.call.candidates events and hitting - // rate limits in Matrix. - // In practice, it'd be better to remove rate limits for m.call.* - - // N.B. this deliberately lets you queue and send blank candidates, which MSC2746 - // currently proposes as the way to indicate that candidate gathering is complete. - // This will hopefully be changed to an explicit rather than implicit notification - // shortly. - this.candidateSendQueue.push(content); - - // Don't send the ICE candidates yet if the call is in the ringing state: this - // means we tried to pick (ie. started generating candidates) and then failed to - // send the answer and went back to the ringing state. Queue up the candidates - // to send if we successfully send the answer. - // Equally don't send if we haven't yet sent the answer because we can send the - // first batch of candidates along with the answer - if (this.state === CallState.Ringing || !this.inviteOrAnswerSent) return; - - // MSC2746 recommends these values (can be quite long when calling because the - // callee will need a while to answer the call) - const delay = this.direction === CallDirection.Inbound ? 500 : 2000; - - if (this.candidateSendTries === 0) { - setTimeout(() => { - this.sendCandidateQueue(); - }, delay); - } - } - - /* - * Transfers this call to another user - */ - public async transfer(targetUserId: string): Promise { - // Fetch the target user's global profile info: their room avatar / displayname - // could be different in whatever room we share with them. - const profileInfo = await this.client.getProfileInfo(targetUserId); - - const replacementId = genCallID(); - - const body = { - replacement_id: genCallID(), - target_user: { - id: targetUserId, - display_name: profileInfo.displayname, - avatar_url: profileInfo.avatar_url, - }, - create_call: replacementId, - } as MCallReplacesEvent; - - await this.sendVoipEvent(EventType.CallReplaces, body); - - await this.terminate(CallParty.Local, CallErrorCode.Transfered, true); - } - - /* - * Transfers this call to the target call, effectively 'joining' the - * two calls (so the remote parties on each call are connected together). - */ - public async transferToCall(transferTargetCall?: MatrixCall): Promise { - const targetProfileInfo = await this.client.getProfileInfo(transferTargetCall.getOpponentMember().userId); - const transfereeProfileInfo = await this.client.getProfileInfo(this.getOpponentMember().userId); - - const newCallId = genCallID(); - - const bodyToTransferTarget = { - // the replacements on each side have their own ID, and it's distinct from the - // ID of the new call (but we can use the same function to generate it) - replacement_id: genCallID(), - target_user: { - id: this.getOpponentMember().userId, - display_name: transfereeProfileInfo.displayname, - avatar_url: transfereeProfileInfo.avatar_url, - }, - await_call: newCallId, - } as MCallReplacesEvent; - - await transferTargetCall.sendVoipEvent(EventType.CallReplaces, bodyToTransferTarget); - - const bodyToTransferee = { - replacement_id: genCallID(), - target_user: { - id: transferTargetCall.getOpponentMember().userId, - display_name: targetProfileInfo.displayname, - avatar_url: targetProfileInfo.avatar_url, - }, - create_call: newCallId, - } as MCallReplacesEvent; - - await this.sendVoipEvent(EventType.CallReplaces, bodyToTransferee); - - await this.terminate(CallParty.Local, CallErrorCode.Replaced, true); - await transferTargetCall.terminate(CallParty.Local, CallErrorCode.Transfered, true); - } - - private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean): Promise { - if (this.callHasEnded()) return; - - this.hangupParty = hangupParty; - this.hangupReason = hangupReason; - this.setState(CallState.Ended); - - if (this.inviteTimeout) { - clearTimeout(this.inviteTimeout); - this.inviteTimeout = null; - } - if (this.callLengthInterval) { - clearInterval(this.callLengthInterval); - this.callLengthInterval = null; - } - - this.callStatsAtEnd = await this.collectCallStats(); - - // Order is important here: first we stopAllMedia() and only then we can deleteAllFeeds() - this.stopAllMedia(); - this.deleteAllFeeds(); - - if (this.peerConn && this.peerConn.signalingState !== 'closed') { - this.peerConn.close(); - } - if (shouldEmit) { - this.emit(CallEvent.Hangup, this); - } - - this.client.callEventHandler.calls.delete(this.callId); - } - - private stopAllMedia(): void { - logger.debug(!this.groupCallId ? "Stopping all media" : "Stopping all media except local feeds" ); - for (const feed of this.feeds) { - if ( - feed.isLocal() && - feed.purpose === SDPStreamMetadataPurpose.Usermedia && - !this.groupCallId - ) { - this.client.getMediaHandler().stopUserMediaStream(feed.stream); - } else if ( - feed.isLocal() && - feed.purpose === SDPStreamMetadataPurpose.Screenshare && - !this.groupCallId - ) { - this.client.getMediaHandler().stopScreensharingStream(feed.stream); - } else if (!feed.isLocal() || !this.groupCallId) { - for (const track of feed.stream.getTracks()) { - track.stop(); - } - } - } - } - - private checkForErrorListener(): void { - if (this.listeners("error").length === 0) { - throw new Error( - "You MUST attach an error listener using call.on('error', function() {})", - ); - } - } - - private async sendCandidateQueue(): Promise { - if (this.candidateSendQueue.length === 0 || this.callHasEnded()) { - return; - } - - const candidates = this.candidateSendQueue; - this.candidateSendQueue = []; - ++this.candidateSendTries; - const content = { - candidates: candidates, - }; - logger.debug("Attempting to send " + candidates.length + " candidates"); - try { - await this.sendVoipEvent(EventType.CallCandidates, content); - // reset our retry count if we have successfully sent our candidates - // otherwise queueCandidate() will refuse to try to flush the queue - this.candidateSendTries = 0; - - // Try to send candidates again just in case we received more candidates while sending. - this.sendCandidateQueue(); - } catch (error) { - // don't retry this event: we'll send another one later as we might - // have more candidates by then. - if (error.event) this.client.cancelPendingEvent(error.event); - - // put all the candidates we failed to send back in the queue - this.candidateSendQueue.push(...candidates); - - if (this.candidateSendTries > 5) { - logger.debug( - "Failed to send candidates on attempt " + this.candidateSendTries + - ". Giving up on this call.", error, - ); - - const code = CallErrorCode.SignallingFailed; - const message = "Signalling failed"; - - this.emit(CallEvent.Error, new CallError(code, message, error)); - this.hangup(code, false); - - return; - } - - const delayMs = 500 * Math.pow(2, this.candidateSendTries); - ++this.candidateSendTries; - logger.debug("Failed to send candidates. Retrying in " + delayMs + "ms", error); - setTimeout(() => { - this.sendCandidateQueue(); - }, delayMs); - } - } - - /** - * Place a call to this room. - * @throws if you have not specified a listener for 'error' events. - * @throws if have passed audio=false. - */ - public async placeCall(audio: boolean, video: boolean): Promise { - if (!audio) { - throw new Error("You CANNOT start a call without audio"); - } - this.setState(CallState.WaitLocalMedia); - - try { - const stream = await this.client.getMediaHandler().getUserMediaStream(audio, video); - const callFeed = new CallFeed({ - client: this.client, - roomId: this.roomId, - userId: this.client.getUserId(), - stream, - purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: stream.getAudioTracks().length === 0, - videoMuted: stream.getVideoTracks().length === 0, - }); - await this.placeCallWithCallFeeds([callFeed]); - } catch (e) { - this.getUserMediaFailed(e); - return; - } - } - - /** - * Place a call to this room with call feed. - * @param {CallFeed[]} callFeeds to use - * @throws if you have not specified a listener for 'error' events. - * @throws if have passed audio=false. - */ - public async placeCallWithCallFeeds(callFeeds: CallFeed[], requestScreenshareFeed = false): Promise { - this.checkForErrorListener(); - this.direction = CallDirection.Outbound; - - // XXX Find a better way to do this - this.client.callEventHandler.calls.set(this.callId, this); - - // make sure we have valid turn creds. Unless something's gone wrong, it should - // poll and keep the credentials valid so this should be instant. - const haveTurnCreds = await this.client.checkTurnServers(); - if (!haveTurnCreds) { - logger.warn("Failed to get TURN credentials! Proceeding with call anyway..."); - } - - // create the peer connection now so it can be gathering candidates while we get user - // media (assuming a candidate pool size is configured) - this.peerConn = this.createPeerConnection(); - this.gotCallFeedsForInvite(callFeeds, requestScreenshareFeed); - } - - private createPeerConnection(): RTCPeerConnection { - const pc = new window.RTCPeerConnection({ - iceTransportPolicy: this.forceTURN ? 'relay' : undefined, - iceServers: this.turnServers, - iceCandidatePoolSize: this.client.iceCandidatePoolSize, - }); - - // 'connectionstatechange' would be better, but firefox doesn't implement that. - pc.addEventListener('iceconnectionstatechange', this.onIceConnectionStateChanged); - pc.addEventListener('signalingstatechange', this.onSignallingStateChanged); - pc.addEventListener('icecandidate', this.gotLocalIceCandidate); - pc.addEventListener('icegatheringstatechange', this.onIceGatheringStateChange); - pc.addEventListener('track', this.onTrack); - pc.addEventListener('negotiationneeded', this.onNegotiationNeeded); - pc.addEventListener('datachannel', this.onDataChannel); - - return pc; - } - - private partyIdMatches(msg: MCallBase): boolean { - // They must either match or both be absent (in which case opponentPartyId will be null) - // Also we ignore party IDs on the invite/offer if the version is 0, so we must do the same - // here and use null if the version is 0 (woe betide any opponent sending messages in the - // same call with different versions) - const msgPartyId = msg.version === 0 ? null : msg.party_id || null; - return msgPartyId === this.opponentPartyId; - } - - // Commits to an opponent for the call - // ev: An invite or answer event - private chooseOpponent(ev: MatrixEvent): void { - // I choo-choo-choose you - const msg = ev.getContent(); - - logger.debug(`Choosing party ID ${msg.party_id} for call ID ${this.callId}`); - - this.opponentVersion = msg.version; - if (this.opponentVersion === 0) { - // set to null to indicate that we've chosen an opponent, but because - // they're v0 they have no party ID (even if they sent one, we're ignoring it) - this.opponentPartyId = null; - } else { - // set to their party ID, or if they're naughty and didn't send one despite - // not being v0, set it to null to indicate we picked an opponent with no - // party ID - this.opponentPartyId = msg.party_id || null; - } - this.opponentCaps = msg.capabilities || {} as CallCapabilities; - this.opponentMember = this.client.getRoom(this.roomId).getMember(ev.getSender()); - } - - private async addBufferedIceCandidates(): Promise { - const bufferedCandidates = this.remoteCandidateBuffer.get(this.opponentPartyId); - if (bufferedCandidates) { - logger.info(`Adding ${bufferedCandidates.length} buffered candidates for opponent ${this.opponentPartyId}`); - await this.addIceCandidates(bufferedCandidates); - } - this.remoteCandidateBuffer = null; - } - - private async addIceCandidates(candidates: RTCIceCandidate[]): Promise { - for (const candidate of candidates) { - if ( - (candidate.sdpMid === null || candidate.sdpMid === undefined) && - (candidate.sdpMLineIndex === null || candidate.sdpMLineIndex === undefined) - ) { - logger.debug("Ignoring remote ICE candidate with no sdpMid or sdpMLineIndex"); - continue; - } - logger.debug( - "Call " + this.callId + " got remote ICE " + candidate.sdpMid + " candidate: " + candidate.candidate, - ); - try { - await this.peerConn.addIceCandidate(candidate); - } catch (err) { - if (!this.ignoreOffer) { - logger.info("Failed to add remote ICE candidate", err); - } - } - } - } - - public get hasPeerConnection(): boolean { - return Boolean(this.peerConn); - } -} - -export function setTracksEnabled(tracks: Array, enabled: boolean): void { - for (let i = 0; i < tracks.length; i++) { - tracks[i].enabled = enabled; - } -} - -/** - * DEPRECATED - * Use client.createCall() - * - * Create a new Matrix call for the browser. - * @param {MatrixClient} client The client instance to use. - * @param {string} roomId The room the call is in. - * @param {Object?} options DEPRECATED optional options map. - * @param {boolean} options.forceTURN DEPRECATED whether relay through TURN should be - * forced. This option is deprecated - use opts.forceTURN when creating the matrix client - * since it's only possible to set this option on outbound calls. - * @return {MatrixCall} the call or null if the browser doesn't support calling. - */ -export function createNewMatrixCall(client: any, roomId: string, options?: CallOpts): MatrixCall { - // typeof prevents Node from erroring on an undefined reference - if (typeof(window) === 'undefined' || typeof(document) === 'undefined') { - // NB. We don't log here as apps try to create a call object as a test for - // whether calls are supported, so we shouldn't fill the logs up. - return null; - } - - // Firefox throws on so little as accessing the RTCPeerConnection when operating in - // a secure mode. There's some information at https://bugzilla.mozilla.org/show_bug.cgi?id=1542616 - // though the concern is that the browser throwing a SecurityError will brick the - // client creation process. - try { - const supported = Boolean( - window.RTCPeerConnection || window.RTCSessionDescription || - window.RTCIceCandidate || navigator.mediaDevices, - ); - if (!supported) { - // Adds a lot of noise to test runs, so disable logging there. - if (process.env.NODE_ENV !== "test") { - logger.error("WebRTC is not supported in this browser / environment"); - } - return null; - } - } catch (e) { - logger.error("Exception thrown when trying to access WebRTC", e); - return null; - } - - const optionsForceTURN = options ? options.forceTURN : false; - - const opts: CallOpts = { - client: client, - roomId: roomId, - invitee: options?.invitee, - turnServers: client.getTurnServers(), - // call level options - forceTURN: client.forceTURN || optionsForceTURN, - opponentDeviceId: options?.opponentDeviceId, - opponentSessionId: options?.opponentSessionId, - groupCallId: options?.groupCallId, - }; - const call = new MatrixCall(opts); - - client.reEmitter.reEmit(call, Object.values(CallEvent)); - - return call; -} - diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index 33b408f6..a220984e 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -1,3 +1,26 @@ +## TODO + - PeerCall + - send invite + - find out if we need to do something different when renegotation is triggered (a subsequent onnegotiationneeded event) whether + we sent the invite/offer or answer. e.g. do we always do createOffer/setLocalDescription and then send it over a matrix negotiation event? even if we before called createAnswer. + - handle receiving offer and send anwser + - handle sending ice candidates + - handle ice candidates finished (iceGatheringState === 'complete') + - handle receiving ice candidates + - handle sending renegotiation + - handle receiving renegotiation + - reject call + - hangup call + - handle muting tracks + - handle remote track being muted + - handle adding/removing tracks to an ongoing call + - handle sdp metadata + - Participant + - handle glare + - encrypt to_device message with olm + - batch outgoing to_device messages in one request to homeserver for operations that will send out an event to all participants (e.g. mute) + - find out if we should start muted or not? + ## Store ongoing calls Add store with all ongoing calls so when we quit and start again, we don't have to go through all the past calls to know which ones might still be ongoing. diff --git a/src/matrix/calls/group/Participant.ts b/src/matrix/calls/group/Participant.ts index 5abdaf63..26747e56 100644 --- a/src/matrix/calls/group/Participant.ts +++ b/src/matrix/calls/group/Participant.ts @@ -14,6 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */ +import {EventType} from "../PeerCall"; +import type {PeerCallHandler} from "../PeerCall"; + class Participant implements PeerCallHandler { private peerCall?: PeerCall; @@ -27,19 +30,18 @@ class Participant implements PeerCallHandler { sendInvite() { this.peerCall = new PeerCall(this, this.webRTC); - this.peerCall.setLocalMedia(this.localMedia); - this.peerCall.sendOffer(); + this.peerCall.call(this.localMedia); } /** From PeerCallHandler * @internal */ - override emitUpdate() { + emitUpdate(params: any) { } /** From PeerCallHandler * @internal */ - override onSendSignallingMessage() { + onSendSignallingMessage(type: EventType, content: Record) { // TODO: this needs to be encrypted with olm first this.hsApi.sendToDevice(type, {[this.userId]: {[this.deviceId ?? "*"]: content}}); } diff --git a/src/platform/types/WebRTC.ts b/src/platform/types/WebRTC.ts index 9a10ce5e..aff74f62 100644 --- a/src/platform/types/WebRTC.ts +++ b/src/platform/types/WebRTC.ts @@ -33,7 +33,7 @@ export interface PeerConnectionHandler { onDataChannelChanged(dataChannel: DataChannel | undefined); onNegotiationNeeded(); // request the type of incoming stream - getPurposeForStreamId(trackId: string): StreamPurpose; + getPurposeForStreamId(streamId: string): StreamPurpose; } // does it make sense to wrap this? export interface DataChannel { @@ -46,11 +46,12 @@ export interface PeerConnection { get dataChannel(): DataChannel | undefined; createOffer(): Promise; createAnswer(): Promise; - setLocalDescription(description: RTCSessionDescriptionInit); + setLocalDescription(description?: RTCSessionDescriptionInit): Promise; setRemoteDescription(description: RTCSessionDescriptionInit): Promise; addIceCandidate(candidate: RTCIceCandidate): Promise; addTrack(track: Track): void; removeTrack(track: Track): boolean; replaceTrack(oldTrack: Track, newTrack: Track): Promise; createDataChannel(): DataChannel; + dispose(): void; } diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 36d5eb75..917f42af 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -49,12 +49,12 @@ class DOMPeerConnection implements PeerConnection { return this.peerConnection.createAnswer(); } - setLocalDescription(description: RTCSessionDescriptionInit) { - this.peerConnection.setLocalDescription(description); + setLocalDescription(description?: RTCSessionDescriptionInit): Promise { + return this.peerConnection.setLocalDescription(description); } - setRemoteDescription(description: RTCSessionDescriptionInit) { - this.peerConnection.setRemoteDescription(description); + setRemoteDescription(description: RTCSessionDescriptionInit): Promise { + return this.peerConnection.setRemoteDescription(description); } addIceCandidate(candidate: RTCIceCandidate): Promise { @@ -66,6 +66,9 @@ class DOMPeerConnection implements PeerConnection { throw new Error("Not a TrackWrapper"); } this.peerConnection.addTrack(track.track, track.stream); + if (track.type === TrackType.ScreenShare) { + this.getRidOfRTXCodecs(track); + } } removeTrack(track: Track): boolean { @@ -87,6 +90,9 @@ class DOMPeerConnection implements PeerConnection { const sender = this.peerConnection.getSenders().find(s => s.track === oldTrack.track); if (sender) { await sender.replaceTrack(newTrack.track); + if (newTrack.type === TrackType.ScreenShare) { + this.getRidOfRTXCodecs(newTrack); + } return true; } return false; @@ -106,6 +112,17 @@ class DOMPeerConnection implements PeerConnection { pc.addEventListener('datachannel', this); } + private deregisterHandler() { + const pc = this.peerConnection; + pc.removeEventListener('negotiationneeded', this); + pc.removeEventListener('icecandidate', this); + pc.removeEventListener('iceconnectionstatechange', this); + pc.removeEventListener('icegatheringstatechange', this); + pc.removeEventListener('signalingstatechange', this); + pc.removeEventListener('track', this); + pc.removeEventListener('datachannel', this); + } + /** @internal */ handleEvent(evt: Event) { switch (evt.type) { @@ -129,6 +146,10 @@ class DOMPeerConnection implements PeerConnection { } } + dispose(): void { + this.deregisterHandler(); + } + private handleLocalIceCandidate(event: RTCPeerConnectionIceEvent) { if (event.candidate) { this.handler.onLocalIceCandidate(event.candidate); @@ -145,10 +166,15 @@ class DOMPeerConnection implements PeerConnection { } private handleRemoteTrack(evt: RTCTrackEvent) { + // the tracks on the new stream (with their stream) const updatedTracks = evt.streams.flatMap(stream => stream.getTracks().map(track => {return {stream, track};})); - const withoutRemovedTracks = this._remoteTracks.filter(t => !updatedTracks.some(ut => t.track == ut.track)); - const addedTracks = updatedTracks.filter(ut => !this._remoteTracks.some(t => t.track === ut.track)); + // of the tracks we already know about, filter the ones that aren't in the new stream + const withoutRemovedTracks = this._remoteTracks.filter(t => !updatedTracks.some(ut => t.track.id === ut.track.id)); + // of the new tracks, filter the ones that we didn't already knew about + const addedTracks = updatedTracks.filter(ut => !this._remoteTracks.some(t => t.track.id === ut.track.id)); + // wrap them const wrappedAddedTracks = addedTracks.map(t => this.wrapRemoteTrack(t.track, t.stream)); + // and concat the tracks for other streams with the added tracks this._remoteTracks = withoutRemovedTracks.concat(...wrappedAddedTracks); this.handler.onRemoteTracksChanged(this.remoteTracks); } @@ -162,4 +188,44 @@ class DOMPeerConnection implements PeerConnection { } return wrapTrack(track, stream, type); } + + /** + * This method removes all video/rtx codecs from screensharing video + * transceivers. This is necessary since they can cause problems. Without + * this the following steps should produce an error: + * Chromium calls Firefox + * Firefox answers + * Firefox starts screen-sharing + * Chromium starts screen-sharing + * Call crashes for Chromium with: + * [96685:23:0518/162603.933321:ERROR:webrtc_video_engine.cc(3296)] RTX codec (PT=97) mapped to PT=96 which is not in the codec list. + * [96685:23:0518/162603.933377:ERROR:webrtc_video_engine.cc(1171)] GetChangedRecvParameters called without any video codecs. + * [96685:23:0518/162603.933430:ERROR:sdp_offer_answer.cc(4302)] Failed to set local video description recv parameters for m-section with mid='2'. (INVALID_PARAMETER) + */ + private getRidOfRTXCodecs(screensharingTrack: TrackWrapper): void { + // RTCRtpReceiver.getCapabilities and RTCRtpSender.getCapabilities don't seem to be supported on FF + if (!RTCRtpReceiver.getCapabilities || !RTCRtpSender.getCapabilities) return; + + const recvCodecs = RTCRtpReceiver.getCapabilities("video")?.codecs ?? []; + const sendCodecs = RTCRtpSender.getCapabilities("video")?.codecs ?? []; + const codecs = [...sendCodecs, ...recvCodecs]; + + for (const codec of codecs) { + if (codec.mimeType === "video/rtx") { + const rtxCodecIndex = codecs.indexOf(codec); + codecs.splice(rtxCodecIndex, 1); + } + } + + for (const trans of this.peerConnection.getTransceivers()) { + if (trans.sender.track === screensharingTrack.track && + ( + trans.sender.track?.kind === "video" || + trans.receiver.track?.kind === "video" + ) + ) { + trans.setCodecPreferences(codecs); + } + } + } } diff --git a/src/utils/AsyncQueue.ts b/src/utils/AsyncQueue.ts new file mode 100644 index 00000000..0686314c --- /dev/null +++ b/src/utils/AsyncQueue.ts @@ -0,0 +1,52 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +export class AsyncQueue { + private isRunning = false; + private queue: T[] = []; + private error?: Error; + + constructor( + private readonly reducer: (v: V, t: T) => Promise, + private value: V, + private readonly contains: (t: T, queue: T[]) => boolean = (t, queue) => queue.includes(t) + ) {} + + push(t: T) { + if (this.contains(t, this.queue)) { + return; + } + this.queue.push(t); + this.runLoopIfNeeded(); + } + + private async runLoopIfNeeded() { + if (this.isRunning || this.error) { + return; + } + this.isRunning = true; + try { + let item: T | undefined; + while (item = this.queue.shift()) { + this.value = await this.reducer(this.value, item); + } + } catch (err) { + this.error = err; + } finally { + this.isRunning = false; + } + } +} From 179c7e74b56511fafaf9972568d2bb7928be4889 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Fri, 25 Feb 2022 16:54:00 +0100 Subject: [PATCH 006/435] WIP6 --- src/matrix/calls/CallHandler.ts | 2 + src/matrix/calls/LocalMedia.ts | 57 ++ src/matrix/calls/PeerCall.ts | 822 ++++++++++----------------- src/matrix/calls/TODO.md | 72 ++- src/platform/types/WebRTC.ts | 13 +- src/platform/web/dom/MediaDevices.ts | 7 +- src/platform/web/dom/WebRTC.ts | 32 +- src/utils/recursivelyAssign.ts | 39 ++ 8 files changed, 487 insertions(+), 557 deletions(-) create mode 100644 src/matrix/calls/LocalMedia.ts create mode 100644 src/utils/recursivelyAssign.ts diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index a31ff666..4a2027f3 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -44,6 +44,8 @@ export class GroupCallHandler { } + // TODO: check and poll turn server credentials here + handleRoomState(room: Room, events: StateEvent[], log: ILogItem) { // first update call events for (const event of events) { diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts new file mode 100644 index 00000000..2429ff2c --- /dev/null +++ b/src/matrix/calls/LocalMedia.ts @@ -0,0 +1,57 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {StreamPurpose} from "../../platform/types/WebRTC"; +import {Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; +import {SDPStreamMetadata} from "./callEventTypes"; + +export class LocalMedia { + constructor( + public readonly cameraTrack?: Track, + public readonly screenShareTrack?: Track, + public readonly microphoneTrack?: AudioTrack + ) {} + + withTracks(tracks: Track[]) { + const cameraTrack = tracks.find(t => t.type === TrackType.Camera) ?? this.cameraTrack; + const screenShareTrack = tracks.find(t => t.type === TrackType.ScreenShare) ?? this.screenShareTrack; + const microphoneTrack = tracks.find(t => t.type === TrackType.Microphone) ?? this.microphoneTrack; + if (cameraTrack && microphoneTrack && cameraTrack.streamId !== microphoneTrack.streamId) { + throw new Error("The camera and audio track should have the same stream id"); + } + return new LocalMedia(cameraTrack, screenShareTrack, microphoneTrack as AudioTrack); + } + + get tracks(): Track[] { return []; } + + getSDPMetadata(): SDPStreamMetadata { + const metadata = {}; + const userMediaTrack = this.microphoneTrack ?? this.cameraTrack; + if (userMediaTrack) { + metadata[userMediaTrack.streamId] = { + purpose: StreamPurpose.UserMedia, + audio_muted: this.microphoneTrack?.muted ?? false, + video_muted: this.cameraTrack?.muted ?? false, + }; + } + if (this.screenShareTrack) { + metadata[this.screenShareTrack.streamId] = { + purpose: StreamPurpose.ScreenShare + }; + } + return metadata; + } +} diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 47dfaa3e..17e49afd 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -15,7 +15,7 @@ limitations under the License. */ import {ObservableMap} from "../../observable/map/ObservableMap"; - +import {recursivelyAssign} from "../../utils/recursivelyAssign"; import {AsyncQueue} from "../../utils/AsyncQueue"; import type {Room} from "../room/Room"; import type {StateEvent} from "../storage/types"; @@ -23,8 +23,284 @@ import type {ILogItem} from "../../logging/types"; import {WebRTC, PeerConnection, PeerConnectionHandler, StreamPurpose} from "../../platform/types/WebRTC"; import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; +import type {LocalMedia} from "./LocalMedia"; -import { randomString } from '../randomstring'; +// when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already +// do for sharing keys will be best as that already deals with room tracking. +/** + * Does WebRTC signalling for a single PeerConnection, and deals with WebRTC wrappers from platform + * */ +/** Implements a call between two peers with the signalling state keeping, while still delegating the signalling message sending. Used by GroupCall.*/ +class PeerCall { + private readonly peerConnection: PeerConnection; + private state = CallState.Fledgling; + private direction: CallDirection; + // A queue for candidates waiting to go out. + // We try to amalgamate candidates into a single candidate message where + // possible + private candidateSendQueue: Array = []; + // If candidates arrive before we've picked an opponent (which, in particular, + // will happen if the opponent sends candidates eagerly before the user answers + // the call) we buffer them up here so we can then add the ones from the party we pick + private remoteCandidateBuffer? = new Map(); + + private logger: any; + private remoteSDPStreamMetadata?: SDPStreamMetadata; + private responsePromiseChain?: Promise; + private opponentPartyId?: PartyId; + + constructor( + private readonly handler: PeerCallHandler, + private localMedia: LocalMedia, + webRTC: WebRTC + ) { + const outer = this; + this.peerConnection = webRTC.createPeerConnection({ + onIceConnectionStateChange(state: RTCIceConnectionState) {}, + onLocalIceCandidate(candidate: RTCIceCandidate) {}, + onIceGatheringStateChange(state: RTCIceGatheringState) {}, + onRemoteTracksChanged(tracks: Track[]) {}, + onDataChannelChanged(dataChannel: DataChannel | undefined) {}, + onNegotiationNeeded() { + const promiseCreator = () => outer.handleNegotiation(); + outer.responsePromiseChain = outer.responsePromiseChain?.then(promiseCreator) ?? promiseCreator(); + }, + getPurposeForStreamId(streamId: string): SDPStreamMetadataPurpose { + return outer.remoteSDPStreamMetadata?.[streamId]?.purpose ?? SDPStreamMetadataPurpose.Usermedia; + } + }); + this.logger = { + debug(...args) { console.log.apply(console, ["WebRTC debug:", ...args])}, + log(...args) { console.log.apply(console, ["WebRTC log:", ...args])}, + warn(...args) { console.log.apply(console, ["WebRTC warn:", ...args])}, + error(...args) { console.error.apply(console, ["WebRTC error:", ...args])}, + } + } + + handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId) { + switch (message.type) { + case EventType.Invite: + this.handleInvite(message.content); + break; + case EventType.Answer: + this.handleAnswer(message.content); + break; + case EventType.Candidates: + this.handleRemoteIceCandidates(message.content); + break; + case EventType.Hangup: + } + } + + async call(localMediaPromise: Promise): Promise { + if (this.state !== CallState.Fledgling) { + return; + } + this.direction = CallDirection.Outbound; + this.setState(CallState.WaitLocalMedia); + try { + this.localMedia = await localMediaPromise; + } catch (err) { + this.setState(CallState.Ended); + return; + } + this.setState(CallState.CreateOffer); + // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called + for (const t of this.localMedia.tracks) { + this.peerConnection.addTrack(t); + } + await this.waitForState(CallState.InviteSent); + } + + async answer() { + + } + + async hangup() { + + } + + async updateLocalMedia(localMediaPromise: Promise) { + const oldMedia = this.localMedia; + this.localMedia = await localMediaPromise; + + const applyTrack = (selectTrack: (media: LocalMedia) => Track | undefined) => { + const oldTrack = selectTrack(oldMedia); + const newTrack = selectTrack(this.localMedia); + if (oldTrack && newTrack) { + this.peerConnection.replaceTrack(oldTrack, newTrack); + } else if (oldTrack) { + this.peerConnection.removeTrack(oldTrack); + } else if (newTrack) { + this.peerConnection.addTrack(newTrack); + } + }; + + // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called + applyTrack(m => m.microphoneTrack); + applyTrack(m => m.cameraTrack); + applyTrack(m => m.screenShareTrack); + } + + // calls are serialized and deduplicated by negotiationQueue + private handleNegotiation = async (): Promise => { + try { + await this.peerConnection.setLocalDescription(); + } catch (err) { + this.logger.debug(`Call ${this.callId} Error setting local description!`, err); + this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); + return; + } + + if (this.peerConnection.iceGatheringState === 'gathering') { + // Allow a short time for initial candidates to be gathered + await new Promise(resolve => setTimeout(resolve, 200)); + } + + if (this.state === CallState.Ended) { + return; + } + + const offer = this.peerConnection.localDescription!; + // Get rid of any candidates waiting to be sent: they'll be included in the local + // description we just got and will send in the offer. + this.logger.info(`Call ${this.callId} Discarding ${ + this.candidateSendQueue.length} candidates that will be sent in offer`); + this.candidateSendQueue = []; + + // need to queue this + const content = { + offer, + [SDPStreamMetadataKey]: this.localMedia.getSDPMetadata(), + version: 1, + lifetime: CALL_TIMEOUT_MS + }; + if (this.state === CallState.CreateOffer) { + await this.handler.sendSignallingMessage({type: EventType.Invite, content}); + this.setState(CallState.InviteSent); + } + }; + + private async handleInvite(content: InviteContent, partyId: PartyId): Promise { + if (this.state !== CallState.Fledgling || this.opponentPartyId !== undefined) { + // TODO: hangup or ignore? + return; + } + + // we must set the party ID before await-ing on anything: the call event + // handler will start giving us more call events (eg. candidates) so if + // we haven't set the party ID, we'll ignore them. + this.opponentPartyId = partyId; + this.direction = CallDirection.Inbound; + + const sdpStreamMetadata = content[SDPStreamMetadataKey]; + if (sdpStreamMetadata) { + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); + } else { + this.logger.debug(`Call ${ + this.callId} did not get any SDPStreamMetadata! Can not send/receive multiple streams`); + } + + try { + await this.peerConnection.setRemoteDescription(content.offer); + await this.addBufferedIceCandidates(); + } catch (e) { + this.logger.debug(`Call ${this.callId} failed to set remote description`, e); + this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); + return; + } + + // According to previous comments in this file, firefox at some point did not + // add streams until media started arriving on them. Testing latest firefox + // (81 at time of writing), this is no longer a problem, so let's do it the correct way. + if (this.peerConnection.remoteTracks.length === 0) { + this.logger.error(`Call ${this.callId} no remote stream or no tracks after setting remote description!`); + this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); + return; + } + + this.setState(CallState.Ringing); + + setTimeout(() => { + if (this.state == CallState.Ringing) { + this.logger.debug(`Call ${this.callId} invite has expired. Hanging up.`); + this.hangupParty = CallParty.Remote; // effectively + this.setState(CallState.Ended); + this.stopAllMedia(); + if (this.peerConnection.signalingState != 'closed') { + this.peerConnection.close(); + } + this.emit(CallEvent.Hangup); + } + }, content.lifetime ?? CALL_TIMEOUT_MS /* - event.getLocalAge() */ ); + } + + private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { + this.remoteSDPStreamMetadata = recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); + // will rerequest stream purpose for all tracks and set track.type accordingly + this.peerConnection.notifyStreamPurposeChanged(); + for (const track of this.peerConnection.remoteTracks) { + const streamMetaData = this.remoteSDPStreamMetadata?.[track.streamId]; + if (streamMetaData) { + if (track.type === TrackType.Microphone) { + track.setMuted(streamMetaData.audio_muted); + } else { // Camera or ScreenShare + track.setMuted(streamMetaData.video_muted); + } + } + } + } + + + private async addBufferedIceCandidates(): Promise { + const bufferedCandidates = this.remoteCandidateBuffer!.get(this.opponentPartyId!); + if (bufferedCandidates) { + this.logger.info(`Call ${this.callId} Adding ${ + bufferedCandidates.length} buffered candidates for opponent ${this.opponentPartyId}`); + await this.addIceCandidates(bufferedCandidates); + } + this.remoteCandidateBuffer = undefined; + } + + private async addIceCandidates(candidates: RTCIceCandidate[]): Promise { + for (const candidate of candidates) { + if ( + (candidate.sdpMid === null || candidate.sdpMid === undefined) && + (candidate.sdpMLineIndex === null || candidate.sdpMLineIndex === undefined) + ) { + this.logger.debug(`Call ${this.callId} ignoring remote ICE candidate with no sdpMid or sdpMLineIndex`); + continue; + } + this.logger.debug(`Call ${this.callId} got remote ICE ${candidate.sdpMid} candidate: ${candidate.candidate}`); + try { + await this.peerConnection.addIceCandidate(candidate); + } catch (err) { + if (!this.ignoreOffer) { + this.logger.info(`Call ${this.callId} failed to add remote ICE candidate`, err); + } + } + } + } + + + private setState(state: CallState): void { + const oldState = this.state; + this.state = state; + this.handler.emitUpdate(); + } + + private waitForState(state: CallState): Promise { + + } + + private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean): Promise { + + } +} + + + +//import { randomString } from '../randomstring'; import { MCallReplacesEvent, MCallAnswer, @@ -41,41 +317,13 @@ import { MCallHangupReject, } from './callEventTypes'; - -const GROUP_CALL_TYPE = "m.call"; -const GROUP_CALL_MEMBER_TYPE = "m.call.member"; - - -/** - * Fires whenever an error occurs when call.js encounters an issue with setting up the call. - *

- * The error given will have a code equal to either `MatrixCall.ERR_LOCAL_OFFER_FAILED` or - * `MatrixCall.ERR_NO_USER_MEDIA`. `ERR_LOCAL_OFFER_FAILED` is emitted when the local client - * fails to create an offer. `ERR_NO_USER_MEDIA` is emitted when the user has denied access - * to their audio/video hardware. - * - * @event module:webrtc/call~MatrixCall#"error" - * @param {Error} err The error raised by MatrixCall. - * @example - * matrixCall.on("error", function(err){ - * console.error(err.code, err); - * }); - */ - // null is used as a special value meaning that the we're in a legacy 1:1 call // without MSC2746 that doesn't provide an id which device sent the message. type PartyId = string | null; -interface TurnServer { - urls: Array; - username?: string; - password?: string; - ttl?: number; -} - -interface AssertedIdentity { - id: string; - displayName: string; +export enum CallParty { + Local = 'local', + Remote = 'remote', } export enum CallState { @@ -90,21 +338,11 @@ export enum CallState { Ended = 'ended', } -export enum CallType { - Voice = 'voice', - Video = 'video', -} - export enum CallDirection { Inbound = 'inbound', Outbound = 'outbound', } -export enum CallParty { - Local = 'local', - Remote = 'remote', -} - export enum EventType { Invite = "m.call.invite", Candidates = "m.call.candidates", @@ -120,30 +358,6 @@ export enum EventType { AssertedIdentityPrefix = "org.matrix.call.asserted_identity", } -export enum CallEvent { - Hangup = 'hangup', - State = 'state', - Error = 'error', - Replaced = 'replaced', - - // The value of isLocalOnHold() has changed - LocalHoldUnhold = 'local_hold_unhold', - // The value of isRemoteOnHold() has changed - RemoteHoldUnhold = 'remote_hold_unhold', - // backwards compat alias for LocalHoldUnhold: remove in a major version bump - HoldUnhold = 'hold_unhold', - // Feeds have changed - FeedsChanged = 'feeds_changed', - - AssertedIdentityChanged = 'asserted_identity_changed', - - LengthChanged = 'length_changed', - - DataChannel = 'datachannel', - - SendVoipEvent = "send_voip_event", -} - export enum CallErrorCode { /** The user chose to end the call */ UserHangup = 'user_hangup', @@ -254,477 +468,21 @@ export class CallError extends Error { } } -export function genCallID(): string { - return Date.now().toString() + randomString(16); -} - -enum CallSetupMessageType { - Invite = "m.call.invite", - Answer = "m.call.answer", - Candidates = "m.call.candidates", - Hangup = "m.call.hangup", -} - -const CALL_ID = "m.call_id"; -const CALL_TERMINATED = "m.terminated"; - -class LocalMedia { - constructor( - public readonly cameraTrack?: Track, - public readonly screenShareTrack?: Track, - public readonly microphoneTrack?: AudioTrack - ) {} - - withTracks(tracks: Track[]) { - const cameraTrack = tracks.find(t => t.type === TrackType.Camera) ?? this.cameraTrack; - const screenShareTrack = tracks.find(t => t.type === TrackType.ScreenShare) ?? this.screenShareTrack; - const microphoneTrack = tracks.find(t => t.type === TrackType.Microphone) ?? this.microphoneTrack; - if (cameraTrack && microphoneTrack && cameraTrack.streamId !== microphoneTrack.streamId) { - throw new Error("The camera and audio track should have the same stream id"); - } - return new LocalMedia(cameraTrack, screenShareTrack, microphoneTrack as AudioTrack); - } - - get tracks(): Track[] { return []; } - - getSDPMetadata(): any { - const metadata = {}; - const userMediaTrack = this.microphoneTrack ?? this.cameraTrack; - if (userMediaTrack) { - metadata[userMediaTrack.streamId] = { - purpose: StreamPurpose.UserMedia, - audio_muted: this.microphoneTrack?.muted ?? false, - video_muted: this.cameraTrack?.muted ?? false, - }; - } - if (this.screenShareTrack) { - metadata[this.screenShareTrack.streamId] = { - purpose: StreamPurpose.ScreenShare - }; - } - return metadata; - } +type InviteContent = { + offer: RTCSessionDescriptionInit, + [SDPStreamMetadataKey]: SDPStreamMetadata, + version?: number, + lifetime?: number } export type InviteMessage = { type: EventType.Invite, - content: { - version: number - } + content: InviteContent } +export type SignallingMessage = InviteMessage; + export interface PeerCallHandler { emitUpdate(peerCall: PeerCall, params: any); - sendSignallingMessage(type: EventType, content: Record); -} - -// when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already -// do for sharing keys will be best as that already deals with room tracking. -/** - * Does WebRTC signalling for a single PeerConnection, and deals with WebRTC wrappers from platform - * */ -/** Implements a call between two peers with the signalling state keeping, while still delegating the signalling message sending. Used by GroupCall.*/ -class PeerCall implements PeerConnectionHandler { - private readonly peerConnection: PeerConnection; - - - public state = CallState.Fledgling; - public hangupParty: CallParty; - public hangupReason: string; - public direction: CallDirection; - public peerConn?: RTCPeerConnection; - - // A queue for candidates waiting to go out. - // We try to amalgamate candidates into a single candidate message where - // possible - private candidateSendQueue: Array = []; - private candidateSendTries = 0; - private sentEndOfCandidates = false; - - private inviteOrAnswerSent = false; - private waitForLocalAVStream: boolean; - private opponentVersion: number | string; - // The party ID of the other side: undefined if we haven't chosen a partner - // yet, null if we have but they didn't send a party ID. - private opponentPartyId: PartyId; - private opponentCaps: CallCapabilities; - private inviteTimeout: number; - private iceDisconnectedTimeout: number; - - // The logic of when & if a call is on hold is nontrivial and explained in is*OnHold - // This flag represents whether we want the other party to be on hold - private remoteOnHold = false; - - // the stats for the call at the point it ended. We can't get these after we - // tear the call down, so we just grab a snapshot before we stop the call. - // The typescript definitions have this type as 'any' :( - private callStatsAtEnd: any[]; - - // Perfect negotiation state: https://www.w3.org/TR/webrtc/#perfect-negotiation-example - private makingOffer = false; - private ignoreOffer: boolean; - - // If candidates arrive before we've picked an opponent (which, in particular, - // will happen if the opponent sends candidates eagerly before the user answers - // the call) we buffer them up here so we can then add the ones from the party we pick - private remoteCandidateBuffer: Map; - - private remoteAssertedIdentity: AssertedIdentity; - - private remoteSDPStreamMetadata?: SDPStreamMetadata; - - private negotiationQueue: AsyncQueue; - - constructor( - private readonly handler: PeerCallHandler, - private localMedia: LocalMedia, - webRTC: WebRTC - ) { - this.peerConnection = webRTC.createPeerConnection(this); - // TODO: should we use this to serialize all state changes? - this.negotiationQueue = new AsyncQueue(this.handleNegotiation, void); - } - - // PeerConnectionHandler method - onIceConnectionStateChange(state: RTCIceConnectionState) {} - // PeerConnectionHandler method - onLocalIceCandidate(candidate: RTCIceCandidate) {} - // PeerConnectionHandler method - onIceGatheringStateChange(state: RTCIceGatheringState) {} - // PeerConnectionHandler method - onRemoteTracksChanged(tracks: Track[]) {} - // PeerConnectionHandler method - onDataChannelChanged(dataChannel: DataChannel | undefined) {} - // PeerConnectionHandler method - onNegotiationNeeded() { - // trigger handleNegotiation - this.negotiationQueue.push(void); - } - - // calls are serialized and deduplicated by negotiationQueue - private handleNegotiation = async (): Promise => { - const offer = await this.peerConnection.createOffer(); - this.peerConnection.setLocalDescription(offer); - // need to queue this - const message = { - offer, - sdp_stream_metadata: this.localMedia.getSDPMetadata(), - version: 1 - } - if (this.state === CallState.Fledgling) { - const sendPromise = this.handler.sendSignallingMessage(EventType.Invite, message); - this.setState(CallState.InviteSent); - } else { - await this.handler.sendSignallingMessage(EventType.Negotiate, message); - } - }; - - async sendInvite(localMediaPromise: Promise): Promise { - if (this.state !== CallState.Fledgling) { - return; - } - this.setState(CallState.WaitLocalMedia); - this.localMedia = await localMediaPromise; - // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called - for (const t of this.localMedia.tracks) { - this.peerConnection.addTrack(t); - } - await this.waitForState(CallState.Ended, CallState.InviteSent); - } - - async sendAnswer(localMediaPromise: Promise): Promise { - if (this.callHasEnded()) return; - - if (this.state !== CallState.Ringing) { - return; - } - - this.setState(CallState.WaitLocalMedia); - this.waitForLocalAVStream = true; - this.localMedia = await localMediaPromise; - this.waitForLocalAVStream = false; - - // enqueue the following - - // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called - for (const t of this.localMedia.tracks) { - this.peerConnection.addTrack(t); - } - - this.setState(CallState.CreateAnswer); - - let myAnswer; - try { - myAnswer = await this.peerConn.createAnswer(); - } catch (err) { - logger.debug("Failed to create answer: ", err); - this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true); - return; - } - - try { - await this.peerConn.setLocalDescription(myAnswer); - this.setState(CallState.Connecting); - - // Allow a short time for initial candidates to be gathered - await new Promise(resolve => { - setTimeout(resolve, 200); - }); - // inlined sendAnswer - const answerContent = { - answer: { - sdp: this.peerConn.localDescription.sdp, - // type is now deprecated as of Matrix VoIP v1, but - // required to still be sent for backwards compat - type: this.peerConn.localDescription.type, - }, - [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(true), - } as MCallAnswer; - - answerContent.capabilities = { - 'm.call.transferee': this.client.supportsCallTransfer, - 'm.call.dtmf': false, - }; - - // We have just taken the local description from the peerConn which will - // contain all the local candidates added so far, so we can discard any candidates - // we had queued up because they'll be in the answer. - logger.info(`Discarding ${this.candidateSendQueue.length} candidates that will be sent in answer`); - this.candidateSendQueue = []; - - try { - await this.sendVoipEvent(EventType.CallAnswer, answerContent); - // If this isn't the first time we've tried to send the answer, - // we may have candidates queued up, so send them now. - this.inviteOrAnswerSent = true; - } catch (error) { - // We've failed to answer: back to the ringing state - this.setState(CallState.Ringing); - this.client.cancelPendingEvent(error.event); - - let code = CallErrorCode.SendAnswer; - let message = "Failed to send answer"; - if (error.name == 'UnknownDeviceError') { - code = CallErrorCode.UnknownDevices; - message = "Unknown devices present in the room"; - } - this.emit(CallEvent.Error, new CallError(code, message, error)); - throw error; - } - - // error handler re-throws so this won't happen on error, but - // we don't want the same error handling on the candidate queue - this.sendCandidateQueue(); - } catch (err) { - logger.debug("Error setting local description!", err); - this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); - return; - } - } - - async updateLocalMedia(localMediaPromise: Promise) { - const oldMedia = this.localMedia; - this.localMedia = await localMediaPromise; - - const applyTrack = (selectTrack: (media: LocalMedia) => Track | undefined) => { - const oldTrack = selectTrack(oldMedia); - const newTrack = selectTrack(this.localMedia); - if (oldTrack && newTrack) { - this.peerConnection.replaceTrack(oldTrack, newTrack); - } else if (oldTrack) { - this.peerConnection.removeTrack(oldTrack); - } else if (newTrack) { - this.peerConnection.addTrack(newTrack); - } - }; - - // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called - applyTrack(m => m.microphoneTrack); - applyTrack(m => m.cameraTrack); - applyTrack(m => m.screenShareTrack); - } - - - /** - * Replace this call with a new call, e.g. for glare resolution. Used by - * MatrixClient. - * @param {MatrixCall} newCall The new call. - */ - public replacedBy(newCall: MatrixCall): void { - if (this.state === CallState.WaitLocalMedia) { - logger.debug("Telling new call to wait for local media"); - newCall.waitForLocalAVStream = true; - } else if ([CallState.CreateOffer, CallState.InviteSent].includes(this.state)) { - if (newCall.direction === CallDirection.Outbound) { - newCall.queueGotCallFeedsForAnswer([]); - } else { - logger.debug("Handing local stream to new call"); - newCall.queueGotCallFeedsForAnswer(this.getLocalFeeds().map(feed => feed.clone())); - } - } - this.successor = newCall; - this.emit(CallEvent.Replaced, newCall); - this.hangup(CallErrorCode.Replaced, true); - } - - /** - * Hangup a call. - * @param {string} reason The reason why the call is being hung up. - * @param {boolean} suppressEvent True to suppress emitting an event. - */ - public hangup(reason: CallErrorCode, suppressEvent: boolean): void { - if (this.callHasEnded()) return; - - logger.debug("Ending call " + this.callId); - this.terminate(CallParty.Local, reason, !suppressEvent); - // We don't want to send hangup here if we didn't even get to sending an invite - if (this.state === CallState.WaitLocalMedia) return; - const content = {}; - // Don't send UserHangup reason to older clients - if ((this.opponentVersion && this.opponentVersion >= 1) || reason !== CallErrorCode.UserHangup) { - content["reason"] = reason; - } - this.sendVoipEvent(EventType.CallHangup, content); - } - - /** - * Reject a call - * This used to be done by calling hangup, but is a separate method and protocol - * event as of MSC2746. - */ - public reject(): void { - if (this.state !== CallState.Ringing) { - throw Error("Call must be in 'ringing' state to reject!"); - } - - if (this.opponentVersion < 1) { - logger.info( - `Opponent version is less than 1 (${this.opponentVersion}): sending hangup instead of reject`, - ); - this.hangup(CallErrorCode.UserHangup, true); - return; - } - - logger.debug("Rejecting call: " + this.callId); - this.terminate(CallParty.Local, CallErrorCode.UserHangup, true); - this.sendVoipEvent(EventType.CallReject, {}); - } - - // request the type of incoming track - getPurposeForStreamId(streamId: string): StreamPurpose { - // TODO: should we return a promise here for the case where the metadata hasn't arrived yet? - const metaData = this.remoteSDPStreamMetadata[streamId]; - return metadata?.purpose as StreamPurpose ?? StreamPurpose.UserMedia; - } - - private setState(state: CallState): void { - const oldState = this.state; - this.state = state; - this.handler.emitUpdate(); - if (this.inviteDeferred) { - if (this.state === CallState.InviteSent) { - this.inviteDeferred.resolve(); - } - } - } - - handleIncomingSignallingMessage(type: CallSetupMessageType, content: Record, partyId: PartyId) { - switch (type) { - case CallSetupMessageType.Invite: - case CallSetupMessageType.Answer: - this.handleAnswer(content); - break; - case CallSetupMessageType.Candidates: - this.handleRemoteIceCandidates(content); - break; - case CallSetupMessageType.Hangup: - } - } - - private async handleAnswer(content: MCallAnswer, partyId: PartyId) { - // add buffered ice candidates to peerConnection - if (this.opponentPartyId !== undefined) { - return; - } - this.opponentPartyId = partyId; - const bufferedCandidates = this.remoteCandidateBuffer?.get(partyId); - if (bufferedCandidates) { - this.addIceCandidates(bufferedCandidates); - } - this.remoteCandidateBuffer = undefined; - - this.setState(CallState.Connecting); - - const sdpStreamMetadata = content[SDPStreamMetadataKey]; - if (sdpStreamMetadata) { - this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); - } else { - logger.warn("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); - } - - try { - await this.peerConnection.setRemoteDescription(content.answer); - } catch (e) { - logger.debug("Failed to set remote description", e); - this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); - return; - } - - // If the answer we selected has a party_id, send a select_answer event - // We do this after setting the remote description since otherwise we'd block - // call setup on it - if (this.opponentPartyId !== null) { - try { - await this.sendVoipEvent(EventType.CallSelectAnswer, { - selected_party_id: this.opponentPartyId, - }); - } catch (err) { - // This isn't fatal, and will just mean that if another party has raced to answer - // the call, they won't know they got rejected, so we carry on & don't retry. - logger.warn("Failed to send select_answer event", err); - } - } - } - - private handleRemoteIceCandidates(content: Record) { - if (this.state === CallState.Ended) { - return; - } - const candidates = content.candidates; - if (!candidates) { - return; - } - if (this.opponentPartyId === undefined) { - if (!this.remoteCandidateBuffer) { - this.remoteCandidateBuffer = new Map(); - } - const bufferedCandidates = this.remoteCandidateBuffer.get(fromPartyId) || []; - bufferedCandidates.push(...candidates); - this.remoteCandidateBuffer.set(fromPartyId, bufferedCandidates); - } else { - this.addIceCandidates(candidates); - } - } - - private async addIceCandidates(candidates: RTCIceCandidate[]): Promise { - for (const candidate of candidates) { - if ( - (candidate.sdpMid === null || candidate.sdpMid === undefined) && - (candidate.sdpMLineIndex === null || candidate.sdpMLineIndex === undefined) - ) { - logger.debug("Ignoring remote ICE candidate with no sdpMid or sdpMLineIndex"); - continue; - } - logger.debug( - "Call " + this.callId + " got remote ICE " + candidate.sdpMid + " candidate: " + candidate.candidate, - ); - try { - await this.peerConnection.addIceCandidate(candidate); - } catch (err) { - if (!this.ignoreOffer) { - logger.info("Failed to add remote ICE candidate", err); - } - } - } - } + sendSignallingMessage(message: InviteMessage); } diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index a220984e..397c9d38 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -1,8 +1,19 @@ + - relevant MSCs next to spec: + - https://github.com/matrix-org/matrix-doc/pull/2746 Improved Signalling for 1:1 VoIP + - https://github.com/matrix-org/matrix-doc/pull/2747 Transferring VoIP Calls + - https://github.com/matrix-org/matrix-doc/pull/3077 Support for multi-stream VoIP + - https://github.com/matrix-org/matrix-doc/pull/3086 Asserted identity on VoIP calls + - https://github.com/matrix-org/matrix-doc/pull/3291 Muting in VoIP calls + - https://github.com/matrix-org/matrix-doc/pull/3401 Native Group VoIP Signalling + ## TODO - PeerCall - send invite - - find out if we need to do something different when renegotation is triggered (a subsequent onnegotiationneeded event) whether - we sent the invite/offer or answer. e.g. do we always do createOffer/setLocalDescription and then send it over a matrix negotiation event? even if we before called createAnswer. + - implement terminate + - implement waitForState + + - find out if we need to do something different when renegotation is triggered (a subsequent onnegotiationneeded event) whether + we sent the invite/offer or answer. e.g. do we always do createOffer/setLocalDescription and then send it over a matrix negotiation event? even if we before called createAnswer. - handle receiving offer and send anwser - handle sending ice candidates - handle ice candidates finished (iceGatheringState === 'complete') @@ -36,12 +47,29 @@ we wait for other participants to add their user and device (in the sources) for each (userid, deviceid) - if userId < ourUserId - - we setup a peer connection + - get local media + - we setup a peer connection + - add local tracks - we wait for negotation event to get sdp + - peerConn.createOffer + - peerConn.setLocalDescription - we send an m.call.invite - else - wait for invite from other side +on local ice candidate: + - if we haven't ... sent invite yet? or received answer? buffer candidate + - otherwise send candidate (without buffering?) + +on incoming call: + - ring, offer to answer + +answering incoming call + - get local media + - peerConn.setRemoteDescription + - add local tracks to peerConn + - peerConn.createAnswer() + - peerConn.setLocalDescription in some cases, we will actually send the invite to all devices (e.g. SFU), so we probably still need to handle multiple anwsers? @@ -50,9 +78,6 @@ so we would send an invite to multiple devices and pick the one for which we received the anwser first. between invite and anwser, we could already receive ice candidates that we need to buffer. -should a PeerCall only exist after we've received an answer? -Before that, we could have a PeerCallInvite - updating the metadata: @@ -64,3 +89,38 @@ if just muting: use m.call.sdp_stream_metadata_changed party identification - for 1:1 calls, we identify with a party_id - for group calls, we identify with a device_id + + + + +## TODO + +Build basic version of PeerCall +Build basic version of GroupCall +Make it possible to olm encrypt the messages +Do work needed for state events + - receiving (almost done?) + - sending +Expose call objects +Write view model +write view + +## Calls questions\ + - how do we handle glare between group calls (e.g. different state events with different call ids?) + - Split up DOM part into platform code? What abstractions to choose? + Does it make sense to come up with our own API very similar to DOM api? + - what code do we copy over vs what do we implement ourselves? + - MatrixCall: perhaps we can copy it over and modify it to our needs? Seems to have a lot of edge cases implemented. + - what is partyId about? + - CallFeed: I need better understand where it is used. It's basically a wrapper around a MediaStream with volume detection. Could it make sense to put this in platform for example? + + - which parts of MSC2746 are still relevant for group calls? + - which parts of MSC2747 are still relevant for group calls? it seems mostly orthogonal? + - SOLVED: how does switching channels work? This was only enabled by MSC 2746 + - you do getUserMedia()/getDisplayMedia() to get the stream(s) + - you call removeTrack/addTrack on the peerConnection + - you receive a negotiationneeded event + - you call createOffer + - you send m.call.negotiate + - SOLVED: wrt to MSC2746, is the screen share track and the audio track (and video track) part of the same stream? or do screen share tracks need to go in a different stream? it sounds incompatible with the MSC2746 requirement. + - SOLVED: how does muting work? MediaStreamTrack.enabled diff --git a/src/platform/types/WebRTC.ts b/src/platform/types/WebRTC.ts index aff74f62..8b224dfe 100644 --- a/src/platform/types/WebRTC.ts +++ b/src/platform/types/WebRTC.ts @@ -15,11 +15,7 @@ limitations under the License. */ import {Track, TrackType} from "./MediaDevices"; - -export enum StreamPurpose { - UserMedia = "m.usermedia", - ScreenShare = "m.screenshare" -} +import {SDPStreamMetadataPurpose} from "../../matrix/calls/callEventTypes"; export interface WebRTC { createPeerConnection(handler: PeerConnectionHandler): PeerConnection; @@ -33,7 +29,7 @@ export interface PeerConnectionHandler { onDataChannelChanged(dataChannel: DataChannel | undefined); onNegotiationNeeded(); // request the type of incoming stream - getPurposeForStreamId(streamId: string): StreamPurpose; + getPurposeForStreamId(streamId: string): SDPStreamMetadataPurpose; } // does it make sense to wrap this? export interface DataChannel { @@ -42,8 +38,11 @@ export interface DataChannel { } export interface PeerConnection { - get remoteTracks(): Track[] | undefined; + notifyStreamPurposeChanged(): void; + get remoteTracks(): Track[]; get dataChannel(): DataChannel | undefined; + get iceGatheringState(): RTCIceGatheringState; + get localDescription(): RTCSessionDescription | undefined; createOffer(): Promise; createAnswer(): Promise; setLocalDescription(description?: RTCSessionDescriptionInit): Promise; diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index 7d5c8f4f..7af15168 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -90,9 +90,10 @@ export class TrackWrapper implements Track { constructor( public readonly track: MediaStreamTrack, public readonly stream: MediaStream, - public readonly type: TrackType + private _type: TrackType, ) {} + get type(): TrackType { return this._type; } get label(): string { return this.track.label; } get id(): string { return this.track.id; } get streamId(): string { return this.stream.id; } @@ -102,6 +103,10 @@ export class TrackWrapper implements Track { setMuted(muted: boolean): void { this.track.enabled = !muted; } + + setType(type: TrackType): void { + this._type = type; + } } export class AudioTrackWrapper extends TrackWrapper { diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 917f42af..98c77ff9 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -16,7 +16,8 @@ limitations under the License. import {TrackWrapper, wrapTrack} from "./MediaDevices"; import {Track, TrackType} from "../../types/MediaDevices"; -import {WebRTC, PeerConnectionHandler, DataChannel, PeerConnection, StreamPurpose} from "../../types/WebRTC"; +import {WebRTC, PeerConnectionHandler, DataChannel, PeerConnection} from "../../types/WebRTC"; +import {SDPStreamMetadataPurpose} from "../../../matrix/calls/callEventTypes"; const POLLING_INTERVAL = 200; // ms export const SPEAKING_THRESHOLD = -60; // dB @@ -25,8 +26,8 @@ const SPEAKING_SAMPLE_COUNT = 8; // samples class DOMPeerConnection implements PeerConnection { private readonly peerConnection: RTCPeerConnection; private readonly handler: PeerConnectionHandler; - private dataChannelWrapper?: DOMDataChannel; - private _remoteTracks: TrackWrapper[]; + //private dataChannelWrapper?: DOMDataChannel; + private _remoteTracks: TrackWrapper[] = []; constructor(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize = 0) { this.handler = handler; @@ -39,7 +40,9 @@ class DOMPeerConnection implements PeerConnection { } get remoteTracks(): Track[] { return this._remoteTracks; } - get dataChannel(): DataChannel | undefined { return this.dataChannelWrapper; } + get dataChannel(): DataChannel | undefined { return undefined; } + get iceGatheringState(): RTCIceGatheringState { return this.peerConnection.iceGatheringState; } + get localDescription(): RTCSessionDescription | undefined { return this.peerConnection.localDescription ?? undefined; } createOffer(): Promise { return this.peerConnection.createOffer(); @@ -97,6 +100,14 @@ class DOMPeerConnection implements PeerConnection { } return false; } + + notifyStreamPurposeChanged(): void { + for (const track of this.remoteTracks) { + const wrapper = track as TrackWrapper; + wrapper.setType(this.getRemoteTrackType(wrapper.track, wrapper.streamId)); + } + } + createDataChannel(): DataChannel { return new DataChannel(this.peerConnection.createDataChannel()); } @@ -173,20 +184,19 @@ class DOMPeerConnection implements PeerConnection { // of the new tracks, filter the ones that we didn't already knew about const addedTracks = updatedTracks.filter(ut => !this._remoteTracks.some(t => t.track.id === ut.track.id)); // wrap them - const wrappedAddedTracks = addedTracks.map(t => this.wrapRemoteTrack(t.track, t.stream)); + const wrappedAddedTracks = addedTracks.map(t => wrapTrack(t.track, t.stream, this.getRemoteTrackType(t.track, t.stream.id))); // and concat the tracks for other streams with the added tracks this._remoteTracks = withoutRemovedTracks.concat(...wrappedAddedTracks); this.handler.onRemoteTracksChanged(this.remoteTracks); } - private wrapRemoteTrack(track: MediaStreamTrack, stream: MediaStream): TrackWrapper { - let type: TrackType; + + private getRemoteTrackType(track: MediaStreamTrack, streamId: string): TrackType { if (track.kind === "video") { - const purpose = this.handler.getPurposeForStreamId(stream.id); - type = purpose === StreamPurpose.UserMedia ? TrackType.Camera : TrackType.ScreenShare; + const purpose = this.handler.getPurposeForStreamId(streamId); + return purpose === SDPStreamMetadataPurpose.Usermedia ? TrackType.Camera : TrackType.ScreenShare; } else { - type = TrackType.Microphone; + return TrackType.Microphone; } - return wrapTrack(track, stream, type); } /** diff --git a/src/utils/recursivelyAssign.ts b/src/utils/recursivelyAssign.ts new file mode 100644 index 00000000..adf5f2ef --- /dev/null +++ b/src/utils/recursivelyAssign.ts @@ -0,0 +1,39 @@ +/* +Copyright 2015, 2016 OpenMarket Ltd +Copyright 2019 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + + +/** + * This function is similar to Object.assign() but it assigns recursively and + * allows you to ignore nullish values from the source + * + * @param {Object} target + * @param {Object} source + * @returns the target object + */ +export function recursivelyAssign(target: Object, source: Object, ignoreNullish = false): any { + for (const [sourceKey, sourceValue] of Object.entries(source)) { + if (target[sourceKey] instanceof Object && sourceValue) { + recursivelyAssign(target[sourceKey], sourceValue); + continue; + } + if ((sourceValue !== null && sourceValue !== undefined) || !ignoreNullish) { + target[sourceKey] = sourceValue; + continue; + } + } + return target; +} \ No newline at end of file From 98b77fc7615ac3bed972e0ae3dc34e0eb18fddba Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Tue, 1 Mar 2022 15:36:31 +0100 Subject: [PATCH 007/435] WIP7 --- src/matrix/calls/PeerCall.ts | 174 ++++++++++++++++++++++++++- src/platform/types/MediaDevices.ts | 1 + src/platform/web/dom/MediaDevices.ts | 4 + 3 files changed, 173 insertions(+), 6 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 17e49afd..60d89e45 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -21,7 +21,7 @@ import type {Room} from "../room/Room"; import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; -import {WebRTC, PeerConnection, PeerConnectionHandler, StreamPurpose} from "../../platform/types/WebRTC"; +import {WebRTC, PeerConnection, PeerConnectionHandler} from "../../platform/types/WebRTC"; import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; import type {LocalMedia} from "./LocalMedia"; @@ -48,6 +48,7 @@ class PeerCall { private remoteSDPStreamMetadata?: SDPStreamMetadata; private responsePromiseChain?: Promise; private opponentPartyId?: PartyId; + private hangupParty: CallParty; constructor( private readonly handler: PeerCallHandler, @@ -80,7 +81,7 @@ class PeerCall { handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId) { switch (message.type) { case EventType.Invite: - this.handleInvite(message.content); + this.handleInvite(message.content, partyId); break; case EventType.Answer: this.handleAnswer(message.content); @@ -112,15 +113,52 @@ class PeerCall { await this.waitForState(CallState.InviteSent); } - async answer() { + async answer(localMediaPromise: Promise): Promise { + if (this.state !== CallState.Ringing) { + return; + } + this.setState(CallState.WaitLocalMedia); + try { + this.localMedia = await localMediaPromise; + } catch (err) { + this.setState(CallState.Ended); + return; + } + this.setState(CallState.CreateAnswer); + for (const t of this.localMedia.tracks) { + this.peerConnection.addTrack(t); + } + let myAnswer: RTCSessionDescriptionInit; + try { + myAnswer = await this.peerConnection.createAnswer(); + } catch (err) { + this.logger.debug(`Call ${this.callId} Failed to create answer: `, err); + this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true); + return; + } + + try { + await this.peerConnection.setLocalDescription(myAnswer); + this.setState(CallState.Connecting); + + } catch (err) { + this.logger.debug(`Call ${this.callId} Error setting local description!`, err); + this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); + return; + } + // Allow a short time for initial candidates to be gathered + await new Promise(resolve => { + setTimeout(resolve, 200); + }); + this.sendAnswer(); } async hangup() { } - async updateLocalMedia(localMediaPromise: Promise) { + async setMedia(localMediaPromise: Promise) { const oldMedia = this.localMedia; this.localMedia = await localMediaPromise; @@ -179,6 +217,16 @@ class PeerCall { await this.handler.sendSignallingMessage({type: EventType.Invite, content}); this.setState(CallState.InviteSent); } + this.sendCandidateQueue(); + + if (this.state === CallState.CreateOffer) { + this.inviteTimeout = setTimeout(() => { + this.inviteTimeout = null; + if (this.state === CallState.InviteSent) { + this.hangup(CallErrorCode.InviteTimeout); + } + }, CALL_TIMEOUT_MS); + } }; private async handleInvite(content: InviteContent, partyId: PartyId): Promise { @@ -202,6 +250,7 @@ class PeerCall { } try { + // Q: Why do we set the remote description before accepting the call? To start creating ICE candidates? await this.peerConnection.setRemoteDescription(content.offer); await this.addBufferedIceCandidates(); } catch (e) { @@ -235,6 +284,88 @@ class PeerCall { }, content.lifetime ?? CALL_TIMEOUT_MS /* - event.getLocalAge() */ ); } + private async sendAnswer(): Promise { + const answerMessage: AnswerMessage = { + type: EventType.Answer, + content: { + answer: { + sdp: this.peerConnection.localDescription!.sdp, + type: this.peerConnection.localDescription!.type, + }, + [SDPStreamMetadataKey]: this.localMedia.getSDPMetadata(), + } + }; + + // We have just taken the local description from the peerConn which will + // contain all the local candidates added so far, so we can discard any candidates + // we had queued up because they'll be in the answer. + this.logger.info(`Call ${this.callId} Discarding ${ + this.candidateSendQueue.length} candidates that will be sent in answer`); + this.candidateSendQueue = []; + + try { + await this.handler.sendSignallingMessage(answerMessage); + } catch (error) { + this.terminate(CallParty.Local, CallErrorCode.SendAnswer, false); + throw error; + } + + // error handler re-throws so this won't happen on error, but + // we don't want the same error handling on the candidate queue + this.sendCandidateQueue(); + } + + private queueCandidate(content: RTCIceCandidate): void { + // We partially de-trickle candidates by waiting for `delay` before sending them + // amalgamated, in order to avoid sending too many m.call.candidates events and hitting + // rate limits in Matrix. + // In practice, it'd be better to remove rate limits for m.call.* + + // N.B. this deliberately lets you queue and send blank candidates, which MSC2746 + // currently proposes as the way to indicate that candidate gathering is complete. + // This will hopefully be changed to an explicit rather than implicit notification + // shortly. + this.candidateSendQueue.push(content); + + // Don't send the ICE candidates yet if the call is in the ringing state + if (this.state === CallState.Ringing) return; + + // MSC2746 recommends these values (can be quite long when calling because the + // callee will need a while to answer the call) + const delay = this.direction === CallDirection.Inbound ? 500 : 2000; + + setTimeout(() => { + this.sendCandidateQueue(); + }, delay); + } + + + private async sendCandidateQueue(): Promise { + if (this.candidateSendQueue.length === 0 || this.state === CallState.Ended) { + return; + } + + const candidates = this.candidateSendQueue; + this.candidateSendQueue = []; + const candidatesMessage: CandidatesMessage = { + type: EventType.Candidates, + content: { + candidates: candidates, + } + }; + this.logger.debug(`Call ${this.callId} attempting to send ${candidates.length} candidates`); + try { + await this.handler.sendSignallingMessage(candidatesMessage); + // Try to send candidates again just in case we received more candidates while sending. + this.sendCandidateQueue(); + } catch (error) { + // don't retry this event: we'll send another one later as we might + // have more candidates by then. + // put all the candidates we failed to send back in the queue + this.terminate(CallParty.Local, CallErrorCode.SignallingFailed, false); + } + } + private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { this.remoteSDPStreamMetadata = recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); // will rerequest stream purpose for all tracks and set track.type accordingly @@ -296,6 +427,12 @@ class PeerCall { private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean): Promise { } + + private stopAllMedia(): void { + for (const track of this.localMedia.tracks) { + track.stop(); + } + } } @@ -480,9 +617,34 @@ export type InviteMessage = { content: InviteContent } -export type SignallingMessage = InviteMessage; +type AnwserContent = { + answer: { + sdp: string, + // type is now deprecated as of Matrix VoIP v1, but + // required to still be sent for backwards compat + type: RTCSdpType, + }, + [SDPStreamMetadataKey]: SDPStreamMetadata, +} + +export type AnswerMessage = { + type: EventType.Answer, + content: AnwserContent +} + +type CandidatesContent = { + candidates: RTCIceCandidate[] +} + +export type CandidatesMessage = { + type: EventType.Candidates, + content: CandidatesContent +} + + +export type SignallingMessage = InviteMessage | AnswerMessage | CandidatesMessage; export interface PeerCallHandler { emitUpdate(peerCall: PeerCall, params: any); - sendSignallingMessage(message: InviteMessage); + sendSignallingMessage(message: SignallingMessage); } diff --git a/src/platform/types/MediaDevices.ts b/src/platform/types/MediaDevices.ts index 3dda3059..8bf608ce 100644 --- a/src/platform/types/MediaDevices.ts +++ b/src/platform/types/MediaDevices.ts @@ -35,6 +35,7 @@ export interface Track { get settings(): MediaTrackSettings; get muted(): boolean; setMuted(muted: boolean): void; + stop(): void; } export interface AudioTrack extends Track { diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index 7af15168..e1991a1c 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -107,6 +107,10 @@ export class TrackWrapper implements Track { setType(type: TrackType): void { this._type = type; } + + stop() { + this.track.stop(); + } } export class AudioTrackWrapper extends TrackWrapper { From 25b01480737e6935ecef8d59509c9d40e88394e1 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Tue, 1 Mar 2022 17:01:47 +0100 Subject: [PATCH 008/435] WIP8 --- src/matrix/calls/PeerCall.ts | 30 +++++++++++++++--------------- src/platform/types/types.ts | 8 ++++++++ 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 60d89e45..75b9cc2d 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -21,6 +21,7 @@ import type {Room} from "../room/Room"; import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; +import type {TimeoutCreator, Timeout} from "../../platform/types/types"; import {WebRTC, PeerConnection, PeerConnectionHandler} from "../../platform/types/WebRTC"; import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; import type {LocalMedia} from "./LocalMedia"; @@ -49,10 +50,12 @@ class PeerCall { private responsePromiseChain?: Promise; private opponentPartyId?: PartyId; private hangupParty: CallParty; + private hangupTimeout?: Timeout; constructor( private readonly handler: PeerCallHandler, private localMedia: LocalMedia, + private readonly createTimeout: TimeoutCreator, webRTC: WebRTC ) { const outer = this; @@ -84,7 +87,7 @@ class PeerCall { this.handleInvite(message.content, partyId); break; case EventType.Answer: - this.handleAnswer(message.content); + this.handleAnswer(message.content, partyId); break; case EventType.Candidates: this.handleRemoteIceCandidates(message.content); @@ -148,9 +151,7 @@ class PeerCall { return; } // Allow a short time for initial candidates to be gathered - await new Promise(resolve => { - setTimeout(resolve, 200); - }); + await this.createTimeout(200).elapsed(); this.sendAnswer(); } @@ -180,7 +181,7 @@ class PeerCall { applyTrack(m => m.screenShareTrack); } - // calls are serialized and deduplicated by negotiationQueue + // calls are serialized and deduplicated by responsePromiseChain private handleNegotiation = async (): Promise => { try { await this.peerConnection.setLocalDescription(); @@ -192,7 +193,7 @@ class PeerCall { if (this.peerConnection.iceGatheringState === 'gathering') { // Allow a short time for initial candidates to be gathered - await new Promise(resolve => setTimeout(resolve, 200)); + await this.createTimeout(200).elapsed(); } if (this.state === CallState.Ended) { @@ -220,12 +221,12 @@ class PeerCall { this.sendCandidateQueue(); if (this.state === CallState.CreateOffer) { - this.inviteTimeout = setTimeout(() => { - this.inviteTimeout = null; - if (this.state === CallState.InviteSent) { - this.hangup(CallErrorCode.InviteTimeout); - } - }, CALL_TIMEOUT_MS); + this.hangupTimeout = this.createTimeout(CALL_TIMEOUT_MS); + await this.hangupTimeout.elapsed(); + // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between + if (this.state === CallState.InviteSent) { + this.hangup(CallErrorCode.InviteTimeout); + } } }; @@ -333,10 +334,9 @@ class PeerCall { // MSC2746 recommends these values (can be quite long when calling because the // callee will need a while to answer the call) const delay = this.direction === CallDirection.Inbound ? 500 : 2000; - - setTimeout(() => { + this.createTimeout(delay).elapsed().then(() => { this.sendCandidateQueue(); - }, delay); + }); } diff --git a/src/platform/types/types.ts b/src/platform/types/types.ts index da8ec8e7..c59278b9 100644 --- a/src/platform/types/types.ts +++ b/src/platform/types/types.ts @@ -43,3 +43,11 @@ export type File = { readonly name: string; readonly blob: IBlobHandle; } + +export interface Timeout { + elapsed(): Promise; + abort(): void; + dispose(): void; +}; + +export type TimeoutCreator = (timeout: number) => Timeout; From ecf7eab3ee3ebfb5ae31b0fca85dc2990c2bb349 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 2 Mar 2022 13:53:22 +0100 Subject: [PATCH 009/435] WIP8 - implement PeerCall.handleAnswer and other things --- src/matrix/calls/PeerCall.ts | 118 +++++++++++++++++++++++++-------- src/matrix/calls/TODO.md | 2 +- src/platform/types/WebRTC.ts | 2 + src/platform/web/dom/WebRTC.ts | 5 ++ 4 files changed, 99 insertions(+), 28 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 75b9cc2d..a08263f6 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -17,6 +17,7 @@ limitations under the License. import {ObservableMap} from "../../observable/map/ObservableMap"; import {recursivelyAssign} from "../../utils/recursivelyAssign"; import {AsyncQueue} from "../../utils/AsyncQueue"; +import {Disposables, Disposable} from "../../utils/Disposables"; import type {Room} from "../room/Room"; import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; @@ -50,7 +51,8 @@ class PeerCall { private responsePromiseChain?: Promise; private opponentPartyId?: PartyId; private hangupParty: CallParty; - private hangupTimeout?: Timeout; + private disposables = new Disposables(); + private statePromiseMap = new Map void, promise: Promise}>(); constructor( private readonly handler: PeerCallHandler, @@ -144,14 +146,13 @@ class PeerCall { try { await this.peerConnection.setLocalDescription(myAnswer); this.setState(CallState.Connecting); - } catch (err) { this.logger.debug(`Call ${this.callId} Error setting local description!`, err); this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); return; } // Allow a short time for initial candidates to be gathered - await this.createTimeout(200).elapsed(); + await this.delay(200); this.sendAnswer(); } @@ -193,7 +194,7 @@ class PeerCall { if (this.peerConnection.iceGatheringState === 'gathering') { // Allow a short time for initial candidates to be gathered - await this.createTimeout(200).elapsed(); + await this.delay(200); } if (this.state === CallState.Ended) { @@ -221,8 +222,7 @@ class PeerCall { this.sendCandidateQueue(); if (this.state === CallState.CreateOffer) { - this.hangupTimeout = this.createTimeout(CALL_TIMEOUT_MS); - await this.hangupTimeout.elapsed(); + await this.delay(CALL_TIMEOUT_MS); // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between if (this.state === CallState.InviteSent) { this.hangup(CallErrorCode.InviteTimeout); @@ -271,18 +271,55 @@ class PeerCall { this.setState(CallState.Ringing); - setTimeout(() => { - if (this.state == CallState.Ringing) { - this.logger.debug(`Call ${this.callId} invite has expired. Hanging up.`); - this.hangupParty = CallParty.Remote; // effectively - this.setState(CallState.Ended); - this.stopAllMedia(); - if (this.peerConnection.signalingState != 'closed') { - this.peerConnection.close(); - } - this.emit(CallEvent.Hangup); + await this.delay(content.lifetime ?? CALL_TIMEOUT_MS); + // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between + if (this.state === CallState.Ringing) { + this.logger.debug(`Call ${this.callId} invite has expired. Hanging up.`); + this.hangupParty = CallParty.Remote; // effectively + this.setState(CallState.Ended); + this.stopAllMedia(); + if (this.peerConnection.signalingState != 'closed') { + this.peerConnection.close(); } - }, content.lifetime ?? CALL_TIMEOUT_MS /* - event.getLocalAge() */ ); + } + } + + private async handleAnswer(content: AnwserContent, partyId: PartyId): Promise { + this.logger.debug(`Got answer for call ID ${this.callId} from party ID ${content.party_id}`); + + if (this.state === CallState.Ended) { + this.logger.debug(`Ignoring answer because call ID ${this.callId} has ended`); + return; + } + + if (this.opponentPartyId !== undefined) { + this.logger.info( + `Call ${this.callId} ` + + `Ignoring answer from party ID ${content.party_id}: ` + + `we already have an answer/reject from ${this.opponentPartyId}`, + ); + return; + } + + this.opponentPartyId = partyId; + await this.addBufferedIceCandidates(); + + this.setState(CallState.Connecting); + + const sdpStreamMetadata = content[SDPStreamMetadataKey]; + if (sdpStreamMetadata) { + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); + } else { + this.logger.warn(`Call ${this.callId} Did not get any SDPStreamMetadata! Can not send/receive multiple streams`); + } + + try { + await this.peerConnection.setRemoteDescription(content.answer); + } catch (e) { + this.logger.debug(`Call ${this.callId} Failed to set remote description`, e); + this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); + return; + } } private async sendAnswer(): Promise { @@ -333,8 +370,7 @@ class PeerCall { // MSC2746 recommends these values (can be quite long when calling because the // callee will need a while to answer the call) - const delay = this.direction === CallDirection.Inbound ? 500 : 2000; - this.createTimeout(delay).elapsed().then(() => { + this.delay(this.direction === CallDirection.Inbound ? 500 : 2000).then(() => { this.sendCandidateQueue(); }); } @@ -384,13 +420,15 @@ class PeerCall { private async addBufferedIceCandidates(): Promise { - const bufferedCandidates = this.remoteCandidateBuffer!.get(this.opponentPartyId!); - if (bufferedCandidates) { - this.logger.info(`Call ${this.callId} Adding ${ - bufferedCandidates.length} buffered candidates for opponent ${this.opponentPartyId}`); - await this.addIceCandidates(bufferedCandidates); + if (this.remoteCandidateBuffer && this.opponentPartyId) { + const bufferedCandidates = this.remoteCandidateBuffer.get(this.opponentPartyId); + if (bufferedCandidates) { + this.logger.info(`Call ${this.callId} Adding ${ + bufferedCandidates.length} buffered candidates for opponent ${this.opponentPartyId}`); + await this.addIceCandidates(bufferedCandidates); + } + this.remoteCandidateBuffer = undefined; } - this.remoteCandidateBuffer = undefined; } private async addIceCandidates(candidates: RTCIceCandidate[]): Promise { @@ -417,11 +455,25 @@ class PeerCall { private setState(state: CallState): void { const oldState = this.state; this.state = state; - this.handler.emitUpdate(); + let deferred = this.statePromiseMap.get(state); + if (deferred) { + deferred.resolve(); + this.statePromiseMap.delete(state); + } + this.handler.emitUpdate(this, undefined); } private waitForState(state: CallState): Promise { - + let deferred = this.statePromiseMap.get(state); + if (!deferred) { + let resolve; + const promise = new Promise(r => { + resolve = r; + }); + deferred = {resolve, promise}; + this.statePromiseMap.set(state, deferred); + } + return deferred.promise; } private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean): Promise { @@ -433,6 +485,18 @@ class PeerCall { track.stop(); } } + + private async delay(timeoutMs: number): Promise { + // Allow a short time for initial candidates to be gathered + const timeout = this.disposables.track(this.createTimeout(timeoutMs)); + await timeout.elapsed(); + this.disposables.untrack(timeout); + } + + public dispose(): void { + this.disposables.dispose(); + // TODO: dispose peerConnection? + } } diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index 397c9d38..69268b1d 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -105,7 +105,7 @@ Expose call objects Write view model write view -## Calls questions\ +## Calls questions - how do we handle glare between group calls (e.g. different state events with different call ids?) - Split up DOM part into platform code? What abstractions to choose? Does it make sense to come up with our own API very similar to DOM api? diff --git a/src/platform/types/WebRTC.ts b/src/platform/types/WebRTC.ts index 8b224dfe..df8133ee 100644 --- a/src/platform/types/WebRTC.ts +++ b/src/platform/types/WebRTC.ts @@ -42,6 +42,7 @@ export interface PeerConnection { get remoteTracks(): Track[]; get dataChannel(): DataChannel | undefined; get iceGatheringState(): RTCIceGatheringState; + get signalingState(): RTCSignalingState; get localDescription(): RTCSessionDescription | undefined; createOffer(): Promise; createAnswer(): Promise; @@ -53,4 +54,5 @@ export interface PeerConnection { replaceTrack(oldTrack: Track, newTrack: Track): Promise; createDataChannel(): DataChannel; dispose(): void; + close(): void; } diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 98c77ff9..08c0d96d 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -43,6 +43,7 @@ class DOMPeerConnection implements PeerConnection { get dataChannel(): DataChannel | undefined { return undefined; } get iceGatheringState(): RTCIceGatheringState { return this.peerConnection.iceGatheringState; } get localDescription(): RTCSessionDescription | undefined { return this.peerConnection.localDescription ?? undefined; } + get signalingState(): RTCSignalingState { return this.peerConnection.signalingState; } createOffer(): Promise { return this.peerConnection.createOffer(); @@ -64,6 +65,10 @@ class DOMPeerConnection implements PeerConnection { return this.peerConnection.addIceCandidate(candidate); } + close(): void { + return this.peerConnection.close(); + } + addTrack(track: Track): void { if (!(track instanceof TrackWrapper)) { throw new Error("Not a TrackWrapper"); From 6fe90e60db68007359b88ed5673ba085b455feb4 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Mon, 7 Mar 2022 10:15:54 +0100 Subject: [PATCH 010/435] WIP9 --- src/matrix/DeviceMessageHandler.js | 6 ++- src/matrix/calls/CallHandler.ts | 5 +-- src/matrix/calls/PeerCall.ts | 22 ++++++++--- src/matrix/calls/TODO.md | 61 ++++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 9 deletions(-) diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index c6bce31e..20f4abd3 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -53,10 +53,14 @@ export class DeviceMessageHandler { } const newRoomKeys = this._megolmDecryption.roomKeysFromDeviceMessages(olmDecryptChanges.results, log); const callMessages = olmDecryptChanges.results.filter(dr => this._callHandler.handlesDeviceMessageEventType(dr.event?.type)); + // load devices by sender key await Promise.all(callMessages.map(async dr => { dr.setDevice(await this._getDevice(dr.senderCurve25519Key, txn)); - this._callHandler.handleDeviceMessage(dr.device.userId, dr.device.deviceId, dr.event.type, dr.event.content, log); })); + // TODO: pass this in the prep and run it in afterSync or afterSyncComplete (as callHandler can send events as well)? + for (const dr of callMessages) { + this._callHandler.handleDeviceMessage(dr.device.userId, dr.device.deviceId, dr.event.type, dr.event.content, log); + } // TODO: somehow include rooms that received a call to_device message in the sync state? // or have updates flow through event emitter? // well, we don't really need to update the room other then when a call starts or stops diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 4a2027f3..1dad9ce8 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -109,8 +109,7 @@ class GroupParticipant implements PeerCallHandler { sendInvite() { this.peerCall = new PeerCall(this, this.webRTC); - this.peerCall.setLocalMedia(this.localMedia); - this.peerCall.sendOffer(); + this.peerCall.call(this.localMedia); } /** From PeerCallHandler @@ -140,7 +139,7 @@ class GroupCall { async participate(tracks: Track[]) { this.localMedia = LocalMedia.fromTracks(tracks); for (const [,participant] of this.participants) { - participant.setLocalMedia(this.localMedia.clone()); + participant.setMedia(this.localMedia.clone()); } // send m.call.member state event diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index a08263f6..a02de99f 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -86,13 +86,23 @@ class PeerCall { handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId) { switch (message.type) { case EventType.Invite: - this.handleInvite(message.content, partyId); + // determining whether or not an incoming invite glares + // with an instance of PeerCall is different for group calls + // and 1:1 calls, so done outside of this class. + // If you pass an event for another call id in here it will assume it glares. + + //const newCallId = message.content.call_id; + //if (this.id && newCallId !== this.id) { + // this.handleInviteGlare(message.content); + //} else { + this.handleInvite(message.content, partyId); + //} break; case EventType.Answer: this.handleAnswer(message.content, partyId); break; case EventType.Candidates: - this.handleRemoteIceCandidates(message.content); + this.handleRemoteIceCandidates(message.content, partyId); break; case EventType.Hangup: } @@ -184,6 +194,8 @@ class PeerCall { // calls are serialized and deduplicated by responsePromiseChain private handleNegotiation = async (): Promise => { + // TODO: does this make sense to have this state if we're already connected? + this.setState(CallState.MakingOffer) try { await this.peerConnection.setLocalDescription(); } catch (err) { @@ -221,7 +233,7 @@ class PeerCall { } this.sendCandidateQueue(); - if (this.state === CallState.CreateOffer) { + if (this.state === CallState.InviteSent) { await this.delay(CALL_TIMEOUT_MS); // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between if (this.state === CallState.InviteSent) { @@ -495,7 +507,7 @@ class PeerCall { public dispose(): void { this.disposables.dispose(); - // TODO: dispose peerConnection? + this.peerConnection.dispose(); } } @@ -529,9 +541,9 @@ export enum CallParty { export enum CallState { Fledgling = 'fledgling', - InviteSent = 'invite_sent', WaitLocalMedia = 'wait_local_media', CreateOffer = 'create_offer', + InviteSent = 'invite_sent', CreateAnswer = 'create_answer', Connecting = 'connecting', Connected = 'connected', diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index 69268b1d..da2b1ad6 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -124,3 +124,64 @@ write view - you send m.call.negotiate - SOLVED: wrt to MSC2746, is the screen share track and the audio track (and video track) part of the same stream? or do screen share tracks need to go in a different stream? it sounds incompatible with the MSC2746 requirement. - SOLVED: how does muting work? MediaStreamTrack.enabled + - SOLVED: so, what's the difference between the call_id and the conf_id in group call events? + - call_id is the specific 1:1 call, conf_id is the thing in the m.call state event key + - so a group call has a conf_id with MxN peer calls, each having their call_id. + +I think we need to synchronize the negotiation needed because we don't use a CallState to guard it... + + + +## Thursday 3-3 notes + +we probably best keep the perfect negotiation flags, as they are needed for both starting the call AND renegotiation? if only for the former, it would make sense as it is a step in setting up the call, but if the call is ongoing, does it make sense to have a MakingOffer state? it actually looks like they are only needed for renegotiation! for call setup we compare the call_ids. What does that mean for these flags? + + +List state transitions + +FROM CALLER FROM CALLEE + +Fledgling Fledgling + V calling `call()` V handleInvite +WaitLocalMedia Ringing + V media promise resolves V answer() +CreateOffer WaitLocalMedia + V add tracks V media promise resolves + V wait for negotionneeded events CreateAnswer + V setLocalDescription() V + V send invite events +InviteSent + V receive anwser, setRemoteDescription() | + \__________________________________________________/ + V + Connecting + V receive ice candidates and + iceConnectionState becomes 'connected' + Connected + V hangup for some reason + Ended + +## From callee + +Fledgling +Ringing +WaitLocalMedia +CreateAnswer +Connecting +Connected +Ended + +Fledgling +WaitLocalMedia +CreateOffer +InviteSent +CreateAnswer +Connecting +Connected +Ringing +Ended + +so if we don't want to bother with having two call objects, we can make the existing call hangup his old call_id? That way we keep the old peerConnection. + + +when glare, won't we drop both calls? No: https://github.com/matrix-org/matrix-spec-proposals/pull/2746#discussion_r819388754 From 60da85d64102146cf912fc30356e3d0203f31d51 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 9 Mar 2022 11:29:39 +0100 Subject: [PATCH 011/435] WIP10 --- package.json | 2 +- src/matrix/DeviceMessageHandler.js | 2 +- src/matrix/calls/Call.ts | 2493 ----------------- src/matrix/calls/CallFeed.ts | 274 -- src/matrix/calls/CallHandler.ts | 113 +- src/matrix/calls/PeerCall.ts | 415 +-- src/matrix/calls/TODO.md | 49 +- src/matrix/calls/callEventTypes.ts | 41 +- .../calls/group/{Call.ts => GroupCall.ts} | 21 +- src/matrix/calls/group/Participant.ts | 49 +- src/platform/web/dom/MediaDevices.ts | 6 +- src/platform/web/dom/WebRTC.ts | 2 +- yarn.lock | 6 +- 13 files changed, 347 insertions(+), 3126 deletions(-) delete mode 100644 src/matrix/calls/Call.ts delete mode 100644 src/matrix/calls/CallFeed.ts rename src/matrix/calls/group/{Call.ts => GroupCall.ts} (83%) diff --git a/package.json b/package.json index 12c73994..e356347b 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,7 @@ "postcss-flexbugs-fixes": "^5.0.2", "regenerator-runtime": "^0.13.7", "text-encoding": "^0.7.0", - "typescript": "^4.3.5", + "typescript": "^4.4", "vite": "^2.6.14", "xxhashjs": "^0.2.2" }, diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index 20f4abd3..470559a9 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -59,7 +59,7 @@ export class DeviceMessageHandler { })); // TODO: pass this in the prep and run it in afterSync or afterSyncComplete (as callHandler can send events as well)? for (const dr of callMessages) { - this._callHandler.handleDeviceMessage(dr.device.userId, dr.device.deviceId, dr.event.type, dr.event.content, log); + this._callHandler.handleDeviceMessage(dr.device.userId, dr.device.deviceId, dr.event, log); } // TODO: somehow include rooms that received a call to_device message in the sync state? // or have updates flow through event emitter? diff --git a/src/matrix/calls/Call.ts b/src/matrix/calls/Call.ts deleted file mode 100644 index d6ac4612..00000000 --- a/src/matrix/calls/Call.ts +++ /dev/null @@ -1,2493 +0,0 @@ -/* -Copyright 2015, 2016 OpenMarket Ltd -Copyright 2017 New Vector Ltd -Copyright 2019, 2020 The Matrix.org Foundation C.I.C. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { - MCallReplacesEvent, - MCallAnswer, - MCallInviteNegotiate, - CallCapabilities, - SDPStreamMetadataPurpose, - SDPStreamMetadata, - SDPStreamMetadataKey, - MCallSDPStreamMetadataChanged, - MCallSelectAnswer, - MCAllAssertedIdentity, - MCallCandidates, - MCallBase, - MCallHangupReject, -} from './callEventTypes'; -import { CallFeed } from './CallFeed'; - -// events: hangup, error(err), replaced(call), state(state, oldState) - -/** - * Fires whenever an error occurs when call.js encounters an issue with setting up the call. - *

- * The error given will have a code equal to either `MatrixCall.ERR_LOCAL_OFFER_FAILED` or - * `MatrixCall.ERR_NO_USER_MEDIA`. `ERR_LOCAL_OFFER_FAILED` is emitted when the local client - * fails to create an offer. `ERR_NO_USER_MEDIA` is emitted when the user has denied access - * to their audio/video hardware. - * - * @event module:webrtc/call~MatrixCall#"error" - * @param {Error} err The error raised by MatrixCall. - * @example - * matrixCall.on("error", function(err){ - * console.error(err.code, err); - * }); - */ - -interface CallOpts { - roomId?: string; - invitee?: string; - client?: any; // Fix when client is TSified - forceTURN?: boolean; - turnServers?: Array; - opponentDeviceId?: string; - opponentSessionId?: string; - groupCallId?: string; -} - -interface TurnServer { - urls: Array; - username?: string; - password?: string; - ttl?: number; -} - -interface AssertedIdentity { - id: string; - displayName: string; -} - -export enum CallState { - Fledgling = 'fledgling', - InviteSent = 'invite_sent', - WaitLocalMedia = 'wait_local_media', - CreateOffer = 'create_offer', - CreateAnswer = 'create_answer', - Connecting = 'connecting', - Connected = 'connected', - Ringing = 'ringing', - Ended = 'ended', -} - -export enum CallType { - Voice = 'voice', - Video = 'video', -} - -export enum CallDirection { - Inbound = 'inbound', - Outbound = 'outbound', -} - -export enum CallParty { - Local = 'local', - Remote = 'remote', -} - -export enum CallEvent { - Hangup = 'hangup', - State = 'state', - Error = 'error', - Replaced = 'replaced', - - // The value of isLocalOnHold() has changed - LocalHoldUnhold = 'local_hold_unhold', - // The value of isRemoteOnHold() has changed - RemoteHoldUnhold = 'remote_hold_unhold', - // backwards compat alias for LocalHoldUnhold: remove in a major version bump - HoldUnhold = 'hold_unhold', - // Feeds have changed - FeedsChanged = 'feeds_changed', - - AssertedIdentityChanged = 'asserted_identity_changed', - - LengthChanged = 'length_changed', - - DataChannel = 'datachannel', - - SendVoipEvent = "send_voip_event", -} - -export enum CallErrorCode { - /** The user chose to end the call */ - UserHangup = 'user_hangup', - - /** An error code when the local client failed to create an offer. */ - LocalOfferFailed = 'local_offer_failed', - /** - * An error code when there is no local mic/camera to use. This may be because - * the hardware isn't plugged in, or the user has explicitly denied access. - */ - NoUserMedia = 'no_user_media', - - /** - * Error code used when a call event failed to send - * because unknown devices were present in the room - */ - UnknownDevices = 'unknown_devices', - - /** - * Error code used when we fail to send the invite - * for some reason other than there being unknown devices - */ - SendInvite = 'send_invite', - - /** - * An answer could not be created - */ - CreateAnswer = 'create_answer', - - /** - * Error code used when we fail to send the answer - * for some reason other than there being unknown devices - */ - SendAnswer = 'send_answer', - - /** - * The session description from the other side could not be set - */ - SetRemoteDescription = 'set_remote_description', - - /** - * The session description from this side could not be set - */ - SetLocalDescription = 'set_local_description', - - /** - * A different device answered the call - */ - AnsweredElsewhere = 'answered_elsewhere', - - /** - * No media connection could be established to the other party - */ - IceFailed = 'ice_failed', - - /** - * The invite timed out whilst waiting for an answer - */ - InviteTimeout = 'invite_timeout', - - /** - * The call was replaced by another call - */ - Replaced = 'replaced', - - /** - * Signalling for the call could not be sent (other than the initial invite) - */ - SignallingFailed = 'signalling_timeout', - - /** - * The remote party is busy - */ - UserBusy = 'user_busy', - - /** - * We transferred the call off to somewhere else - */ - Transfered = 'transferred', - - /** - * A call from the same user was found with a new session id - */ - NewSession = 'new_session', -} - -/** - * The version field that we set in m.call.* events - */ -const VOIP_PROTO_VERSION = 1; - -/** The fallback ICE server to use for STUN or TURN protocols. */ -const FALLBACK_ICE_SERVER = 'stun:turn.matrix.org'; - -/** The length of time a call can be ringing for. */ -const CALL_TIMEOUT_MS = 60000; - -const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); - -export class CallError extends Error { - code: string; - - constructor(code: CallErrorCode, msg: string, err: Error) { - // Still don't think there's any way to have proper nested errors - super(msg + ": " + err); - - this.code = code; - } -} - -export function genCallID(): string { - return Date.now().toString() + randomString(16); -} - -/** - * Construct a new Matrix Call. - * @constructor - * @param {Object} opts Config options. - * @param {string} opts.roomId The room ID for this call. - * @param {Object} opts.webRtc The WebRTC globals from the browser. - * @param {boolean} opts.forceTURN whether relay through TURN should be forced. - * @param {Object} opts.URL The URL global. - * @param {Array} opts.turnServers Optional. A list of TURN servers. - * @param {MatrixClient} opts.client The Matrix Client instance to send events to. - */ -export class MatrixCall extends EventEmitter { - public roomId: string; - public callId: string; - public invitee?: string; - public state = CallState.Fledgling; - public hangupParty: CallParty; - public hangupReason: string; - public direction: CallDirection; - public ourPartyId: string; - public peerConn?: RTCPeerConnection; - - private client: MatrixClient; - private forceTURN: boolean; - private turnServers: Array; - // A queue for candidates waiting to go out. - // We try to amalgamate candidates into a single candidate message where - // possible - private candidateSendQueue: Array = []; - private candidateSendTries = 0; - private sentEndOfCandidates = false; - private feeds: Array = []; - private usermediaSenders: Array = []; - private screensharingSenders: Array = []; - private inviteOrAnswerSent = false; - private waitForLocalAVStream: boolean; - private successor: MatrixCall; - private opponentMember: RoomMember; - private opponentVersion: number | string; - // The party ID of the other side: undefined if we haven't chosen a partner - // yet, null if we have but they didn't send a party ID. - private opponentPartyId: string; - private opponentCaps: CallCapabilities; - private inviteTimeout: number; - private iceDisconnectedTimeout: number; - - // The logic of when & if a call is on hold is nontrivial and explained in is*OnHold - // This flag represents whether we want the other party to be on hold - private remoteOnHold = false; - - // the stats for the call at the point it ended. We can't get these after we - // tear the call down, so we just grab a snapshot before we stop the call. - // The typescript definitions have this type as 'any' :( - private callStatsAtEnd: any[]; - - // Perfect negotiation state: https://www.w3.org/TR/webrtc/#perfect-negotiation-example - private makingOffer = false; - private ignoreOffer: boolean; - - private responsePromiseChain?: Promise; - - // If candidates arrive before we've picked an opponent (which, in particular, - // will happen if the opponent sends candidates eagerly before the user answers - // the call) we buffer them up here so we can then add the ones from the party we pick - private remoteCandidateBuffer = new Map(); - - private remoteAssertedIdentity: AssertedIdentity; - - private remoteSDPStreamMetadata: SDPStreamMetadata; - - private callLengthInterval: number; - private callLength = 0; - - private opponentDeviceId: string; - private opponentSessionId: string; - public groupCallId: string; - - constructor(opts: CallOpts) { - super(); - this.roomId = opts.roomId; - this.invitee = opts.invitee; - this.client = opts.client; - this.forceTURN = opts.forceTURN; - this.ourPartyId = this.client.deviceId; - this.opponentDeviceId = opts.opponentDeviceId; - this.opponentSessionId = opts.opponentSessionId; - this.groupCallId = opts.groupCallId; - // Array of Objects with urls, username, credential keys - this.turnServers = opts.turnServers || []; - if (this.turnServers.length === 0 && this.client.isFallbackICEServerAllowed()) { - this.turnServers.push({ - urls: [FALLBACK_ICE_SERVER], - }); - } - for (const server of this.turnServers) { - utils.checkObjectHasKeys(server, ["urls"]); - } - this.callId = genCallID(); - } - - /** - * Place a voice call to this room. - * @throws If you have not specified a listener for 'error' events. - */ - public async placeVoiceCall(): Promise { - await this.placeCall(true, false); - } - - /** - * Place a video call to this room. - * @throws If you have not specified a listener for 'error' events. - */ - public async placeVideoCall(): Promise { - await this.placeCall(true, true); - } - - /** - * Create a datachannel using this call's peer connection. - * @param label A human readable label for this datachannel - * @param options An object providing configuration options for the data channel. - */ - public createDataChannel(label: string, options: RTCDataChannelInit) { - const dataChannel = this.peerConn.createDataChannel(label, options); - this.emit(CallEvent.DataChannel, dataChannel); - return dataChannel; - } - - public getOpponentMember(): RoomMember { - return this.opponentMember; - } - - public getOpponentSessionId(): string { - return this.opponentSessionId; - } - - public opponentCanBeTransferred(): boolean { - return Boolean(this.opponentCaps && this.opponentCaps["m.call.transferee"]); - } - - public opponentSupportsDTMF(): boolean { - return Boolean(this.opponentCaps && this.opponentCaps["m.call.dtmf"]); - } - - public getRemoteAssertedIdentity(): AssertedIdentity { - return this.remoteAssertedIdentity; - } - - public get type(): CallType { - return (this.hasLocalUserMediaVideoTrack || this.hasRemoteUserMediaVideoTrack) - ? CallType.Video - : CallType.Voice; - } - - public get hasLocalUserMediaVideoTrack(): boolean { - return this.localUsermediaStream?.getVideoTracks().length > 0; - } - - public get hasRemoteUserMediaVideoTrack(): boolean { - return this.getRemoteFeeds().some((feed) => { - return ( - feed.purpose === SDPStreamMetadataPurpose.Usermedia && - feed.stream.getVideoTracks().length > 0 - ); - }); - } - - public get hasLocalUserMediaAudioTrack(): boolean { - return this.localUsermediaStream?.getAudioTracks().length > 0; - } - - public get hasRemoteUserMediaAudioTrack(): boolean { - return this.getRemoteFeeds().some((feed) => { - return ( - feed.purpose === SDPStreamMetadataPurpose.Usermedia && - feed.stream.getAudioTracks().length > 0 - ); - }); - } - - public get localUsermediaFeed(): CallFeed { - return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); - } - - public get localScreensharingFeed(): CallFeed { - return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); - } - - public get localUsermediaStream(): MediaStream { - return this.localUsermediaFeed?.stream; - } - - public get localScreensharingStream(): MediaStream { - return this.localScreensharingFeed?.stream; - } - - public get remoteUsermediaFeed(): CallFeed { - return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); - } - - public get remoteScreensharingFeed(): CallFeed { - return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); - } - - public get remoteUsermediaStream(): MediaStream { - return this.remoteUsermediaFeed?.stream; - } - - public get remoteScreensharingStream(): MediaStream { - return this.remoteScreensharingFeed?.stream; - } - - private getFeedByStreamId(streamId: string): CallFeed { - return this.getFeeds().find((feed) => feed.stream.id === streamId); - } - - /** - * Returns an array of all CallFeeds - * @returns {Array} CallFeeds - */ - public getFeeds(): Array { - return this.feeds; - } - - /** - * Returns an array of all local CallFeeds - * @returns {Array} local CallFeeds - */ - public getLocalFeeds(): Array { - return this.feeds.filter((feed) => feed.isLocal()); - } - - /** - * Returns an array of all remote CallFeeds - * @returns {Array} remote CallFeeds - */ - public getRemoteFeeds(): Array { - return this.feeds.filter((feed) => !feed.isLocal()); - } - - /** - * Generates and returns localSDPStreamMetadata - * @returns {SDPStreamMetadata} localSDPStreamMetadata - */ - private getLocalSDPStreamMetadata(updateStreamIds = false): SDPStreamMetadata { - const metadata: SDPStreamMetadata = {}; - for (const localFeed of this.getLocalFeeds()) { - if (updateStreamIds) { - localFeed.sdpMetadataStreamId = localFeed.stream.id; - } - - metadata[localFeed.sdpMetadataStreamId] = { - purpose: localFeed.purpose, - audio_muted: localFeed.isAudioMuted(), - video_muted: localFeed.isVideoMuted(), - }; - } - return metadata; - } - - /** - * Returns true if there are no incoming feeds, - * otherwise returns false - * @returns {boolean} no incoming feeds - */ - public noIncomingFeeds(): boolean { - return !this.feeds.some((feed) => !feed.isLocal()); - } - - private pushRemoteFeed(stream: MediaStream): void { - // Fallback to old behavior if the other side doesn't support SDPStreamMetadata - if (!this.opponentSupportsSDPStreamMetadata()) { - this.pushRemoteFeedWithoutMetadata(stream); - return; - } - - const userId = this.getOpponentMember().userId; - const purpose = this.remoteSDPStreamMetadata[stream.id].purpose; - const audioMuted = this.remoteSDPStreamMetadata[stream.id].audio_muted; - const videoMuted = this.remoteSDPStreamMetadata[stream.id].video_muted; - - if (!purpose) { - logger.warn(`Ignoring stream with id ${stream.id} because we didn't get any metadata about it`); - return; - } - - // Try to find a feed with the same purpose as the new stream, - // if we find it replace the old stream with the new one - const existingFeed = this.getRemoteFeeds().find((feed) => feed.purpose === purpose); - if (existingFeed) { - existingFeed.setNewStream(stream); - } else { - this.feeds.push(new CallFeed({ - client: this.client, - roomId: this.roomId, - userId, - stream, - purpose, - audioMuted, - videoMuted, - })); - this.emit(CallEvent.FeedsChanged, this.feeds); - } - - logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}", purpose=${purpose})`); - } - - /** - * This method is used ONLY if the other client doesn't support sending SDPStreamMetadata - */ - private pushRemoteFeedWithoutMetadata(stream: MediaStream): void { - const userId = this.getOpponentMember().userId; - // We can guess the purpose here since the other client can only send one stream - const purpose = SDPStreamMetadataPurpose.Usermedia; - const oldRemoteStream = this.feeds.find((feed) => !feed.isLocal())?.stream; - - // Note that we check by ID and always set the remote stream: Chrome appears - // to make new stream objects when transceiver directionality is changed and the 'active' - // status of streams change - Dave - // If we already have a stream, check this stream has the same id - if (oldRemoteStream && stream.id !== oldRemoteStream.id) { - logger.warn(`Ignoring new stream ID ${stream.id}: we already have stream ID ${oldRemoteStream.id}`); - return; - } - - // Try to find a feed with the same stream id as the new stream, - // if we find it replace the old stream with the new one - const feed = this.getFeedByStreamId(stream.id); - if (feed) { - feed.setNewStream(stream); - } else { - this.feeds.push(new CallFeed({ - client: this.client, - roomId: this.roomId, - audioMuted: false, - videoMuted: false, - userId, - stream, - purpose, - })); - this.emit(CallEvent.FeedsChanged, this.feeds); - } - - logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}")`); - } - - private pushNewLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose, addToPeerConnection = true): void { - const userId = this.client.getUserId(); - - // TODO: Find out what is going on here - // why do we enable audio (and only audio) tracks here? -- matthew - setTracksEnabled(stream.getAudioTracks(), true); - - // We try to replace an existing feed if there already is one with the same purpose - const existingFeed = this.getLocalFeeds().find((feed) => feed.purpose === purpose); - if (existingFeed) { - existingFeed.setNewStream(stream); - } else { - this.pushLocalFeed( - new CallFeed({ - client: this.client, - roomId: this.roomId, - audioMuted: stream.getAudioTracks().length === 0, - videoMuted: stream.getVideoTracks().length === 0, - userId, - stream, - purpose, - }), - addToPeerConnection, - ); - this.emit(CallEvent.FeedsChanged, this.feeds); - } - } - - /** - * Pushes supplied feed to the call - * @param {CallFeed} callFeed to push - * @param {boolean} addToPeerConnection whether to add the tracks to the peer connection - */ - public pushLocalFeed(callFeed: CallFeed, addToPeerConnection = true): void { - if (this.feeds.some((feed) => callFeed.stream.id === feed.stream.id)) { - logger.info(`Ignoring duplicate local stream ${callFeed.stream.id} in call ${this.callId}`); - return; - } - - this.feeds.push(callFeed); - - if (addToPeerConnection) { - const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia ? - this.usermediaSenders : this.screensharingSenders; - // Empty the array - senderArray.splice(0, senderArray.length); - - for (const track of callFeed.stream.getTracks()) { - logger.info( - `Adding track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${callFeed.stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - senderArray.push(this.peerConn.addTrack(track, callFeed.stream)); - } - } - - logger.info( - `Pushed local stream `+ - `(id="${callFeed.stream.id}", `+ - `active="${callFeed.stream.active}", `+ - `purpose="${callFeed.purpose}")`, - ); - - this.emit(CallEvent.FeedsChanged, this.feeds); - } - - /** - * Removes local call feed from the call and its tracks from the peer - * connection - * @param callFeed to remove - */ - public removeLocalFeed(callFeed: CallFeed): void { - const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia - ? this.usermediaSenders - : this.screensharingSenders; - - for (const sender of senderArray) { - this.peerConn.removeTrack(sender); - } - - if (callFeed.purpose === SDPStreamMetadataPurpose.Screenshare) { - this.client.getMediaHandler().stopScreensharingStream(callFeed.stream); - } - - // Empty the array - senderArray.splice(0, senderArray.length); - this.deleteFeed(callFeed); - } - - private deleteAllFeeds(): void { - for (const feed of this.feeds) { - if (!feed.isLocal() || !this.groupCallId) { - feed.dispose(); - } - } - - this.feeds = []; - this.emit(CallEvent.FeedsChanged, this.feeds); - } - - private deleteFeedByStream(stream: MediaStream): void { - const feed = this.getFeedByStreamId(stream.id); - if (!feed) { - logger.warn(`Didn't find the feed with stream id ${stream.id} to delete`); - return; - } - this.deleteFeed(feed); - } - - private deleteFeed(feed: CallFeed): void { - feed.dispose(); - this.feeds.splice(this.feeds.indexOf(feed), 1); - this.emit(CallEvent.FeedsChanged, this.feeds); - } - - // The typescript definitions have this type as 'any' :( - public async getCurrentCallStats(): Promise { - if (this.callHasEnded()) { - return this.callStatsAtEnd; - } - - return this.collectCallStats(); - } - - private async collectCallStats(): Promise { - // This happens when the call fails before it starts. - // For example when we fail to get capture sources - if (!this.peerConn) return; - - const statsReport = await this.peerConn.getStats(); - const stats = []; - for (const item of statsReport) { - stats.push(item[1]); - } - - return stats; - } - - /** - * Configure this call from an invite event. Used by MatrixClient. - * @param {MatrixEvent} event The m.call.invite event - */ - public async initWithInvite(event: MatrixEvent): Promise { - const invite = event.getContent(); - this.direction = CallDirection.Inbound; - - // make sure we have valid turn creds. Unless something's gone wrong, it should - // poll and keep the credentials valid so this should be instant. - const haveTurnCreds = await this.client.checkTurnServers(); - if (!haveTurnCreds) { - logger.warn("Failed to get TURN credentials! Proceeding with call anyway..."); - } - - const sdpStreamMetadata = invite[SDPStreamMetadataKey]; - if (sdpStreamMetadata) { - this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); - } else { - logger.debug("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); - } - - this.peerConn = this.createPeerConnection(); - // we must set the party ID before await-ing on anything: the call event - // handler will start giving us more call events (eg. candidates) so if - // we haven't set the party ID, we'll ignore them. - this.chooseOpponent(event); - try { - await this.peerConn.setRemoteDescription(invite.offer); - await this.addBufferedIceCandidates(); - } catch (e) { - logger.debug("Failed to set remote description", e); - this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); - return; - } - - const remoteStream = this.feeds.find((feed) => !feed.isLocal())?.stream; - - // According to previous comments in this file, firefox at some point did not - // add streams until media started arriving on them. Testing latest firefox - // (81 at time of writing), this is no longer a problem, so let's do it the correct way. - if (!remoteStream || remoteStream.getTracks().length === 0) { - logger.error("No remote stream or no tracks after setting remote description!"); - this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); - return; - } - - this.setState(CallState.Ringing); - - if (event.getLocalAge()) { - setTimeout(() => { - if (this.state == CallState.Ringing) { - logger.debug("Call invite has expired. Hanging up."); - this.hangupParty = CallParty.Remote; // effectively - this.setState(CallState.Ended); - this.stopAllMedia(); - if (this.peerConn.signalingState != 'closed') { - this.peerConn.close(); - } - this.emit(CallEvent.Hangup); - } - }, invite.lifetime - event.getLocalAge()); - } - } - - /** - * Configure this call from a hangup or reject event. Used by MatrixClient. - * @param {MatrixEvent} event The m.call.hangup event - */ - public initWithHangup(event: MatrixEvent): void { - // perverse as it may seem, sometimes we want to instantiate a call with a - // hangup message (because when getting the state of the room on load, events - // come in reverse order and we want to remember that a call has been hung up) - this.setState(CallState.Ended); - } - - private shouldAnswerWithMediaType( - wantedValue: boolean | undefined, valueOfTheOtherSide: boolean | undefined, type: "audio" | "video", - ): boolean { - if (wantedValue && !valueOfTheOtherSide) { - // TODO: Figure out how to do this - logger.warn(`Unable to answer with ${type} because the other side isn't sending it either.`); - return false; - } else if ( - !utils.isNullOrUndefined(wantedValue) && - wantedValue !== valueOfTheOtherSide && - !this.opponentSupportsSDPStreamMetadata() - ) { - logger.warn( - `Unable to answer with ${type}=${wantedValue} because the other side doesn't support it. ` + - `Answering with ${type}=${valueOfTheOtherSide}.`, - ); - return valueOfTheOtherSide; - } - return wantedValue ?? valueOfTheOtherSide; - } - - /** - * Answer a call. - */ - public async answer(audio?: boolean, video?: boolean): Promise { - if (this.inviteOrAnswerSent) return; - // TODO: Figure out how to do this - if (audio === false && video === false) throw new Error("You CANNOT answer a call without media"); - - if (!this.localUsermediaStream && !this.waitForLocalAVStream) { - const prevState = this.state; - const answerWithAudio = this.shouldAnswerWithMediaType(audio, this.hasRemoteUserMediaAudioTrack, "audio"); - const answerWithVideo = this.shouldAnswerWithMediaType(video, this.hasRemoteUserMediaVideoTrack, "video"); - - this.setState(CallState.WaitLocalMedia); - this.waitForLocalAVStream = true; - - try { - const stream = await this.client.getMediaHandler().getUserMediaStream( - answerWithAudio, answerWithVideo, - ); - this.waitForLocalAVStream = false; - const usermediaFeed = new CallFeed({ - client: this.client, - roomId: this.roomId, - userId: this.client.getUserId(), - stream, - purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: stream.getAudioTracks().length === 0, - videoMuted: stream.getVideoTracks().length === 0, - }); - - const feeds = [usermediaFeed]; - - if (this.localScreensharingFeed) { - feeds.push(this.localScreensharingFeed); - } - - this.answerWithCallFeeds(feeds); - } catch (e) { - if (answerWithVideo) { - // Try to answer without video - logger.warn("Failed to getUserMedia(), trying to getUserMedia() without video"); - this.setState(prevState); - this.waitForLocalAVStream = false; - await this.answer(answerWithAudio, false); - } else { - this.getUserMediaFailed(e); - return; - } - } - } else if (this.waitForLocalAVStream) { - this.setState(CallState.WaitLocalMedia); - } - } - - public answerWithCallFeeds(callFeeds: CallFeed[]): void { - if (this.inviteOrAnswerSent) return; - - logger.debug(`Answering call ${this.callId}`); - - this.queueGotCallFeedsForAnswer(callFeeds); - } - - /** - * Replace this call with a new call, e.g. for glare resolution. Used by - * MatrixClient. - * @param {MatrixCall} newCall The new call. - */ - public replacedBy(newCall: MatrixCall): void { - if (this.state === CallState.WaitLocalMedia) { - logger.debug("Telling new call to wait for local media"); - newCall.waitForLocalAVStream = true; - } else if ([CallState.CreateOffer, CallState.InviteSent].includes(this.state)) { - if (newCall.direction === CallDirection.Outbound) { - newCall.queueGotCallFeedsForAnswer([]); - } else { - logger.debug("Handing local stream to new call"); - newCall.queueGotCallFeedsForAnswer(this.getLocalFeeds().map(feed => feed.clone())); - } - } - this.successor = newCall; - this.emit(CallEvent.Replaced, newCall); - this.hangup(CallErrorCode.Replaced, true); - } - - /** - * Hangup a call. - * @param {string} reason The reason why the call is being hung up. - * @param {boolean} suppressEvent True to suppress emitting an event. - */ - public hangup(reason: CallErrorCode, suppressEvent: boolean): void { - if (this.callHasEnded()) return; - - logger.debug("Ending call " + this.callId); - this.terminate(CallParty.Local, reason, !suppressEvent); - // We don't want to send hangup here if we didn't even get to sending an invite - if (this.state === CallState.WaitLocalMedia) return; - const content = {}; - // Don't send UserHangup reason to older clients - if ((this.opponentVersion && this.opponentVersion >= 1) || reason !== CallErrorCode.UserHangup) { - content["reason"] = reason; - } - this.sendVoipEvent(EventType.CallHangup, content); - } - - /** - * Reject a call - * This used to be done by calling hangup, but is a separate method and protocol - * event as of MSC2746. - */ - public reject(): void { - if (this.state !== CallState.Ringing) { - throw Error("Call must be in 'ringing' state to reject!"); - } - - if (this.opponentVersion < 1) { - logger.info( - `Opponent version is less than 1 (${this.opponentVersion}): sending hangup instead of reject`, - ); - this.hangup(CallErrorCode.UserHangup, true); - return; - } - - logger.debug("Rejecting call: " + this.callId); - this.terminate(CallParty.Local, CallErrorCode.UserHangup, true); - this.sendVoipEvent(EventType.CallReject, {}); - } - - /** - * Adds an audio and/or video track - upgrades the call - * @param {boolean} audio should add an audio track - * @param {boolean} video should add an video track - */ - private async upgradeCall( - audio: boolean, video: boolean, - ): Promise { - // We don't do call downgrades - if (!audio && !video) return; - if (!this.opponentSupportsSDPStreamMetadata()) return; - - try { - const upgradeAudio = audio && !this.hasLocalUserMediaAudioTrack; - const upgradeVideo = video && !this.hasLocalUserMediaVideoTrack; - logger.debug(`Upgrading call: audio?=${upgradeAudio} video?=${upgradeVideo}`); - - const stream = await this.client.getMediaHandler().getUserMediaStream(upgradeAudio, upgradeVideo); - if (upgradeAudio && upgradeVideo) { - if (this.hasLocalUserMediaAudioTrack) return; - if (this.hasLocalUserMediaVideoTrack) return; - - this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); - } else if (upgradeAudio) { - if (this.hasLocalUserMediaAudioTrack) return; - - const audioTrack = stream.getAudioTracks()[0]; - this.localUsermediaStream.addTrack(audioTrack); - this.peerConn.addTrack(audioTrack, this.localUsermediaStream); - } else if (upgradeVideo) { - if (this.hasLocalUserMediaVideoTrack) return; - - const videoTrack = stream.getVideoTracks()[0]; - this.localUsermediaStream.addTrack(videoTrack); - this.peerConn.addTrack(videoTrack, this.localUsermediaStream); - } - } catch (error) { - logger.error("Failed to upgrade the call", error); - this.emit(CallEvent.Error, - new CallError(CallErrorCode.NoUserMedia, "Failed to get camera access: ", error), - ); - } - } - - /** - * Returns true if this.remoteSDPStreamMetadata is defined, otherwise returns false - * @returns {boolean} can screenshare - */ - public opponentSupportsSDPStreamMetadata(): boolean { - return Boolean(this.remoteSDPStreamMetadata); - } - - /** - * If there is a screensharing stream returns true, otherwise returns false - * @returns {boolean} is screensharing - */ - public isScreensharing(): boolean { - return Boolean(this.localScreensharingStream); - } - - /** - * Starts/stops screensharing - * @param enabled the desired screensharing state - * @param {string} desktopCapturerSourceId optional id of the desktop capturer source to use - * @returns {boolean} new screensharing state - */ - public async setScreensharingEnabled( - enabled: boolean, desktopCapturerSourceId?: string, - ): Promise { - // Skip if there is nothing to do - if (enabled && this.isScreensharing()) { - logger.warn(`There is already a screensharing stream - there is nothing to do!`); - return true; - } else if (!enabled && !this.isScreensharing()) { - logger.warn(`There already isn't a screensharing stream - there is nothing to do!`); - return false; - } - - // Fallback to replaceTrack() - if (!this.opponentSupportsSDPStreamMetadata()) { - return await this.setScreensharingEnabledWithoutMetadataSupport(enabled, desktopCapturerSourceId); - } - - logger.debug(`Set screensharing enabled? ${enabled}`); - if (enabled) { - try { - const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId); - if (!stream) return false; - this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare); - return true; - } catch (err) { - logger.error("Failed to get screen-sharing stream:", err); - return false; - } - } else { - for (const sender of this.screensharingSenders) { - this.peerConn.removeTrack(sender); - } - this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream); - this.deleteFeedByStream(this.localScreensharingStream); - return false; - } - } - - /** - * Starts/stops screensharing - * Should be used ONLY if the opponent doesn't support SDPStreamMetadata - * @param enabled the desired screensharing state - * @param {string} desktopCapturerSourceId optional id of the desktop capturer source to use - * @returns {boolean} new screensharing state - */ - private async setScreensharingEnabledWithoutMetadataSupport( - enabled: boolean, desktopCapturerSourceId?: string, - ): Promise { - logger.debug(`Set screensharing enabled? ${enabled} using replaceTrack()`); - if (enabled) { - try { - const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId); - if (!stream) return false; - - const track = stream.getTracks().find((track) => { - return track.kind === "video"; - }); - const sender = this.usermediaSenders.find((sender) => { - return sender.track?.kind === "video"; - }); - sender.replaceTrack(track); - - this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare, false); - - return true; - } catch (err) { - logger.error("Failed to get screen-sharing stream:", err); - return false; - } - } else { - const track = this.localUsermediaStream.getTracks().find((track) => { - return track.kind === "video"; - }); - const sender = this.usermediaSenders.find((sender) => { - return sender.track?.kind === "video"; - }); - sender.replaceTrack(track); - - this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream); - this.deleteFeedByStream(this.localScreensharingStream); - - return false; - } - } - - /** - * Request a new local usermedia stream with the current device id. - */ - public async updateLocalUsermediaStream(stream: MediaStream) { - const callFeed = this.localUsermediaFeed; - callFeed.setNewStream(stream); - const micShouldBeMuted = callFeed.isAudioMuted() || this.remoteOnHold; - const vidShouldBeMuted = callFeed.isVideoMuted() || this.remoteOnHold; - setTracksEnabled(stream.getAudioTracks(), !micShouldBeMuted); - setTracksEnabled(stream.getVideoTracks(), !vidShouldBeMuted); - - const newSenders = []; - - for (const track of stream.getTracks()) { - const oldSender = this.usermediaSenders.find((sender) => { - return sender.track?.kind === track.kind; - }); - - let newSender: RTCRtpSender; - - try { - logger.info( - `Replacing track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - await oldSender.replaceTrack(track); - newSender = oldSender; - } catch (error) { - logger.info( - `Adding track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - newSender = this.peerConn.addTrack(track, stream); - } - - newSenders.push(newSender); - } - - this.usermediaSenders = newSenders; - } - - /** - * Set whether our outbound video should be muted or not. - * @param {boolean} muted True to mute the outbound video. - * @returns the new mute state - */ - public async setLocalVideoMuted(muted: boolean): Promise { - if (!await this.client.getMediaHandler().hasVideoDevice()) { - return this.isLocalVideoMuted(); - } - - if (!this.hasLocalUserMediaVideoTrack && !muted) { - await this.upgradeCall(false, true); - return this.isLocalVideoMuted(); - } - this.localUsermediaFeed?.setVideoMuted(muted); - this.updateMuteStatus(); - return this.isLocalVideoMuted(); - } - - /** - * Check if local video is muted. - * - * If there are multiple video tracks, all of the tracks need to be muted - * for this to return true. This means if there are no video tracks, this will - * return true. - * @return {Boolean} True if the local preview video is muted, else false - * (including if the call is not set up yet). - */ - public isLocalVideoMuted(): boolean { - return this.localUsermediaFeed?.isVideoMuted(); - } - - /** - * Set whether the microphone should be muted or not. - * @param {boolean} muted True to mute the mic. - * @returns the new mute state - */ - public async setMicrophoneMuted(muted: boolean): Promise { - if (!await this.client.getMediaHandler().hasAudioDevice()) { - return this.isMicrophoneMuted(); - } - - if (!this.hasLocalUserMediaAudioTrack && !muted) { - await this.upgradeCall(true, false); - return this.isMicrophoneMuted(); - } - this.localUsermediaFeed?.setAudioMuted(muted); - this.updateMuteStatus(); - return this.isMicrophoneMuted(); - } - - /** - * Check if the microphone is muted. - * - * If there are multiple audio tracks, all of the tracks need to be muted - * for this to return true. This means if there are no audio tracks, this will - * return true. - * @return {Boolean} True if the mic is muted, else false (including if the call - * is not set up yet). - */ - public isMicrophoneMuted(): boolean { - return this.localUsermediaFeed?.isAudioMuted(); - } - - /** - * @returns true if we have put the party on the other side of the call on hold - * (that is, we are signalling to them that we are not listening) - */ - public isRemoteOnHold(): boolean { - return this.remoteOnHold; - } - - public setRemoteOnHold(onHold: boolean): void { - if (this.isRemoteOnHold() === onHold) return; - this.remoteOnHold = onHold; - - for (const transceiver of this.peerConn.getTransceivers()) { - // We don't send hold music or anything so we're not actually - // sending anything, but sendrecv is fairly standard for hold and - // it makes it a lot easier to figure out who's put who on hold. - transceiver.direction = onHold ? 'sendonly' : 'sendrecv'; - } - this.updateMuteStatus(); - - this.emit(CallEvent.RemoteHoldUnhold, this.remoteOnHold); - } - - /** - * Indicates whether we are 'on hold' to the remote party (ie. if true, - * they cannot hear us). - * @returns true if the other party has put us on hold - */ - public isLocalOnHold(): boolean { - if (this.state !== CallState.Connected) return false; - - let callOnHold = true; - - // We consider a call to be on hold only if *all* the tracks are on hold - // (is this the right thing to do?) - for (const transceiver of this.peerConn.getTransceivers()) { - const trackOnHold = ['inactive', 'recvonly'].includes(transceiver.currentDirection); - - if (!trackOnHold) callOnHold = false; - } - - return callOnHold; - } - - /** - * Sends a DTMF digit to the other party - * @param digit The digit (nb. string - '#' and '*' are dtmf too) - */ - public sendDtmfDigit(digit: string): void { - for (const sender of this.peerConn.getSenders()) { - if (sender.track.kind === 'audio' && sender.dtmf) { - sender.dtmf.insertDTMF(digit); - return; - } - } - - throw new Error("Unable to find a track to send DTMF on"); - } - - private updateMuteStatus(): void { - this.sendVoipEvent(EventType.CallSDPStreamMetadataChangedPrefix, { - [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(), - }); - - const micShouldBeMuted = this.localUsermediaFeed?.isAudioMuted() || this.remoteOnHold; - const vidShouldBeMuted = this.localUsermediaFeed?.isVideoMuted() || this.remoteOnHold; - - setTracksEnabled(this.localUsermediaStream.getAudioTracks(), !micShouldBeMuted); - setTracksEnabled(this.localUsermediaStream.getVideoTracks(), !vidShouldBeMuted); - } - - private gotCallFeedsForInvite(callFeeds: CallFeed[], requestScreenshareFeed = false): void { - if (this.successor) { - this.successor.queueGotCallFeedsForAnswer(callFeeds); - return; - } - if (this.callHasEnded()) { - this.stopAllMedia(); - return; - } - - for (const feed of callFeeds) { - this.pushLocalFeed(feed); - } - - if (requestScreenshareFeed) { - this.peerConn.addTransceiver("video", { - direction: "recvonly", - }); - } - - this.setState(CallState.CreateOffer); - - logger.debug("gotUserMediaForInvite"); - // Now we wait for the negotiationneeded event - } - - private async sendAnswer(): Promise { - const answerContent = { - answer: { - sdp: this.peerConn.localDescription.sdp, - // type is now deprecated as of Matrix VoIP v1, but - // required to still be sent for backwards compat - type: this.peerConn.localDescription.type, - }, - [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(true), - } as MCallAnswer; - - answerContent.capabilities = { - 'm.call.transferee': this.client.supportsCallTransfer, - 'm.call.dtmf': false, - }; - - // We have just taken the local description from the peerConn which will - // contain all the local candidates added so far, so we can discard any candidates - // we had queued up because they'll be in the answer. - logger.info(`Discarding ${this.candidateSendQueue.length} candidates that will be sent in answer`); - this.candidateSendQueue = []; - - try { - await this.sendVoipEvent(EventType.CallAnswer, answerContent); - // If this isn't the first time we've tried to send the answer, - // we may have candidates queued up, so send them now. - this.inviteOrAnswerSent = true; - } catch (error) { - // We've failed to answer: back to the ringing state - this.setState(CallState.Ringing); - this.client.cancelPendingEvent(error.event); - - let code = CallErrorCode.SendAnswer; - let message = "Failed to send answer"; - if (error.name == 'UnknownDeviceError') { - code = CallErrorCode.UnknownDevices; - message = "Unknown devices present in the room"; - } - this.emit(CallEvent.Error, new CallError(code, message, error)); - throw error; - } - - // error handler re-throws so this won't happen on error, but - // we don't want the same error handling on the candidate queue - this.sendCandidateQueue(); - } - - private queueGotCallFeedsForAnswer(callFeeds: CallFeed[]): void { - // Ensure only one negotiate/answer event is being processed at a time. - if (this.responsePromiseChain) { - this.responsePromiseChain = - this.responsePromiseChain.then(() => this.gotCallFeedsForAnswer(callFeeds)); - } else { - this.responsePromiseChain = this.gotCallFeedsForAnswer(callFeeds); - } - } - - private async gotCallFeedsForAnswer(callFeeds: CallFeed[]): Promise { - if (this.callHasEnded()) return; - - this.waitForLocalAVStream = false; - - for (const feed of callFeeds) { - this.pushLocalFeed(feed); - } - - this.setState(CallState.CreateAnswer); - - let myAnswer; - try { - this.getRidOfRTXCodecs(); - myAnswer = await this.peerConn.createAnswer(); - } catch (err) { - logger.debug("Failed to create answer: ", err); - this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true); - return; - } - - try { - await this.peerConn.setLocalDescription(myAnswer); - this.setState(CallState.Connecting); - - // Allow a short time for initial candidates to be gathered - await new Promise(resolve => { - setTimeout(resolve, 200); - }); - - this.sendAnswer(); - } catch (err) { - logger.debug("Error setting local description!", err); - this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); - return; - } - - // HACK: Safari doesn't like it when we reuse MediaStreams. In most cases - // we can get around this by calling MediaStream.clone(), however inbound - // calls seem to still be broken unless we getUserMedia again and replace - // all MediaStreams using sender.replaceTrack - if (isSafari) { - await new Promise(resolve => { - setTimeout(resolve, 200); - }); - - if (this.state === CallState.Ended) { - return; - } - - const callFeed = this.localUsermediaFeed; - const stream = callFeed.stream; - - if (!stream.active) { - throw new Error(`Call ${this.callId} has an inactive stream ${ - stream.id} and its tracks cannot be replaced`); - } - - const newSenders = []; - - for (const track of this.localUsermediaStream.getTracks()) { - const oldSender = this.usermediaSenders.find((sender) => { - return sender.track?.kind === track.kind; - }); - - if (track.readyState === "ended") { - throw new Error(`Call ${this.callId} tried to replace track ${track.id} in the ended state`); - } - - let newSender: RTCRtpSender; - - try { - logger.info( - `Replacing track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - await oldSender.replaceTrack(track); - newSender = oldSender; - } catch (error) { - logger.info( - `Adding track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - newSender = this.peerConn.addTrack(track, stream); - } - - newSenders.push(newSender); - } - - this.usermediaSenders = newSenders; - } - } - - /** - * Internal - * @param {Object} event - */ - private gotLocalIceCandidate = (event: RTCPeerConnectionIceEvent): Promise => { - if (event.candidate) { - logger.debug( - "Call " + this.callId + " got local ICE " + event.candidate.sdpMid + " candidate: " + - event.candidate.candidate, - ); - - if (this.callHasEnded()) return; - - // As with the offer, note we need to make a copy of this object, not - // pass the original: that broke in Chrome ~m43. - if (event.candidate.candidate !== '' || !this.sentEndOfCandidates) { - this.queueCandidate(event.candidate); - - if (event.candidate.candidate === '') this.sentEndOfCandidates = true; - } - } - }; - - private onIceGatheringStateChange = (event: Event): void => { - logger.debug("ice gathering state changed to " + this.peerConn.iceGatheringState); - if (this.peerConn.iceGatheringState === 'complete' && !this.sentEndOfCandidates) { - // If we didn't get an empty-string candidate to signal the end of candidates, - // create one ourselves now gathering has finished. - // We cast because the interface lists all the properties as required but we - // only want to send 'candidate' - // XXX: We probably want to send either sdpMid or sdpMLineIndex, as it's not strictly - // correct to have a candidate that lacks both of these. We'd have to figure out what - // previous candidates had been sent with and copy them. - const c = { - candidate: '', - } as RTCIceCandidate; - this.queueCandidate(c); - this.sentEndOfCandidates = true; - } - }; - - public async onRemoteIceCandidatesReceived(ev: MatrixEvent): Promise { - if (this.callHasEnded()) { - //debuglog("Ignoring remote ICE candidate because call has ended"); - return; - } - - const content = ev.getContent(); - const candidates = content.candidates; - if (!candidates) { - logger.info("Ignoring candidates event with no candidates!"); - return; - } - - const fromPartyId = content.version === 0 ? null : content.party_id || null; - - if (this.opponentPartyId === undefined) { - // we haven't picked an opponent yet so save the candidates - logger.info(`Buffering ${candidates.length} candidates until we pick an opponent`); - const bufferedCandidates = this.remoteCandidateBuffer.get(fromPartyId) || []; - bufferedCandidates.push(...candidates); - this.remoteCandidateBuffer.set(fromPartyId, bufferedCandidates); - return; - } - - if (!this.partyIdMatches(content)) { - logger.info( - `Ignoring candidates from party ID ${content.party_id}: ` + - `we have chosen party ID ${this.opponentPartyId}`, - ); - - return; - } - - await this.addIceCandidates(candidates); - } - - /** - * Used by MatrixClient. - * @param {Object} msg - */ - public async onAnswerReceived(event: MatrixEvent): Promise { - const content = event.getContent(); - logger.debug(`Got answer for call ID ${this.callId} from party ID ${content.party_id}`); - - if (this.callHasEnded()) { - logger.debug(`Ignoring answer because call ID ${this.callId} has ended`); - return; - } - - if (this.opponentPartyId !== undefined) { - logger.info( - `Ignoring answer from party ID ${content.party_id}: ` + - `we already have an answer/reject from ${this.opponentPartyId}`, - ); - return; - } - - this.chooseOpponent(event); - await this.addBufferedIceCandidates(); - - this.setState(CallState.Connecting); - - const sdpStreamMetadata = content[SDPStreamMetadataKey]; - if (sdpStreamMetadata) { - this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); - } else { - logger.warn("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); - } - - try { - await this.peerConn.setRemoteDescription(content.answer); - } catch (e) { - logger.debug("Failed to set remote description", e); - this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); - return; - } - - // If the answer we selected has a party_id, send a select_answer event - // We do this after setting the remote description since otherwise we'd block - // call setup on it - if (this.opponentPartyId !== null) { - try { - await this.sendVoipEvent(EventType.CallSelectAnswer, { - selected_party_id: this.opponentPartyId, - }); - } catch (err) { - // This isn't fatal, and will just mean that if another party has raced to answer - // the call, they won't know they got rejected, so we carry on & don't retry. - logger.warn("Failed to send select_answer event", err); - } - } - } - - public async onSelectAnswerReceived(event: MatrixEvent): Promise { - if (this.direction !== CallDirection.Inbound) { - logger.warn("Got select_answer for an outbound call: ignoring"); - return; - } - - const selectedPartyId = event.getContent().selected_party_id; - - if (selectedPartyId === undefined || selectedPartyId === null) { - logger.warn("Got nonsensical select_answer with null/undefined selected_party_id: ignoring"); - return; - } - - if (selectedPartyId !== this.ourPartyId) { - logger.info(`Got select_answer for party ID ${selectedPartyId}: we are party ID ${this.ourPartyId}.`); - // The other party has picked somebody else's answer - this.terminate(CallParty.Remote, CallErrorCode.AnsweredElsewhere, true); - } - } - - public async onNegotiateReceived(event: MatrixEvent): Promise { - const content = event.getContent(); - const description = content.description; - if (!description || !description.sdp || !description.type) { - logger.info("Ignoring invalid m.call.negotiate event"); - return; - } - // Politeness always follows the direction of the call: in a glare situation, - // we pick either the inbound or outbound call, so one side will always be - // inbound and one outbound - const polite = this.direction === CallDirection.Inbound; - - // Here we follow the perfect negotiation logic from - // https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation - const offerCollision = ( - (description.type === 'offer') && - (this.makingOffer || this.peerConn.signalingState !== 'stable') - ); - - this.ignoreOffer = !polite && offerCollision; - if (this.ignoreOffer) { - logger.info("Ignoring colliding negotiate event because we're impolite"); - return; - } - - const prevLocalOnHold = this.isLocalOnHold(); - - const sdpStreamMetadata = content[SDPStreamMetadataKey]; - if (sdpStreamMetadata) { - this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); - } else { - logger.warn("Received negotiation event without SDPStreamMetadata!"); - } - - try { - await this.peerConn.setRemoteDescription(description); - - if (description.type === 'offer') { - this.getRidOfRTXCodecs(); - await this.peerConn.setLocalDescription(); - - this.sendVoipEvent(EventType.CallNegotiate, { - description: this.peerConn.localDescription, - [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(true), - }); - } - } catch (err) { - logger.warn("Failed to complete negotiation", err); - } - - const newLocalOnHold = this.isLocalOnHold(); - if (prevLocalOnHold !== newLocalOnHold) { - this.emit(CallEvent.LocalHoldUnhold, newLocalOnHold); - // also this one for backwards compat - this.emit(CallEvent.HoldUnhold, newLocalOnHold); - } - } - - private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { - this.remoteSDPStreamMetadata = utils.recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); - for (const feed of this.getRemoteFeeds()) { - const streamId = feed.stream.id; - feed.setAudioMuted(this.remoteSDPStreamMetadata[streamId]?.audio_muted); - feed.setVideoMuted(this.remoteSDPStreamMetadata[streamId]?.video_muted); - feed.purpose = this.remoteSDPStreamMetadata[streamId]?.purpose; - } - } - - public onSDPStreamMetadataChangedReceived(event: MatrixEvent): void { - const content = event.getContent(); - const metadata = content[SDPStreamMetadataKey]; - this.updateRemoteSDPStreamMetadata(metadata); - } - - public async onAssertedIdentityReceived(event: MatrixEvent): Promise { - const content = event.getContent(); - if (!content.asserted_identity) return; - - this.remoteAssertedIdentity = { - id: content.asserted_identity.id, - displayName: content.asserted_identity.display_name, - }; - this.emit(CallEvent.AssertedIdentityChanged); - } - - private callHasEnded(): boolean { - // This exists as workaround to typescript trying to be clever and erroring - // when putting if (this.state === CallState.Ended) return; twice in the same - // function, even though that function is async. - return this.state === CallState.Ended; - } - - private queueGotLocalOffer(): void { - // Ensure only one negotiate/answer event is being processed at a time. - if (this.responsePromiseChain) { - this.responsePromiseChain = - this.responsePromiseChain.then(() => this.wrappedGotLocalOffer()); - } else { - this.responsePromiseChain = this.wrappedGotLocalOffer(); - } - } - - private async wrappedGotLocalOffer(): Promise { - this.makingOffer = true; - try { - this.getRidOfRTXCodecs(); - await this.gotLocalOffer(); - } catch (e) { - this.getLocalOfferFailed(e); - return; - } finally { - this.makingOffer = false; - } - } - - private async gotLocalOffer(): Promise { - logger.debug("Setting local description"); - - if (this.callHasEnded()) { - logger.debug("Ignoring newly created offer on call ID " + this.callId + - " because the call has ended"); - return; - } - - try { - await this.peerConn.setLocalDescription(); - } catch (err) { - logger.debug("Error setting local description!", err); - this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); - return; - } - - if (this.peerConn.iceGatheringState === 'gathering') { - // Allow a short time for initial candidates to be gathered - await new Promise(resolve => { - setTimeout(resolve, 200); - }); - } - - if (this.callHasEnded()) return; - - const eventType = this.state === CallState.CreateOffer ? EventType.CallInvite : EventType.CallNegotiate; - - const content = { - lifetime: CALL_TIMEOUT_MS, - } as MCallInviteNegotiate; - - if (eventType === EventType.CallInvite && this.invitee) { - content.invitee = this.invitee; - } - - // clunky because TypeScript can't follow the types through if we use an expression as the key - if (this.state === CallState.CreateOffer) { - content.offer = this.peerConn.localDescription; - } else { - content.description = this.peerConn.localDescription; - } - - content.capabilities = { - 'm.call.transferee': this.client.supportsCallTransfer, - 'm.call.dtmf': false, - }; - - content[SDPStreamMetadataKey] = this.getLocalSDPStreamMetadata(true); - - // Get rid of any candidates waiting to be sent: they'll be included in the local - // description we just got and will send in the offer. - logger.info(`Discarding ${this.candidateSendQueue.length} candidates that will be sent in offer`); - this.candidateSendQueue = []; - - try { - await this.sendVoipEvent(eventType, content); - } catch (error) { - logger.error("Failed to send invite", error); - if (error.event) this.client.cancelPendingEvent(error.event); - - let code = CallErrorCode.SignallingFailed; - let message = "Signalling failed"; - if (this.state === CallState.CreateOffer) { - code = CallErrorCode.SendInvite; - message = "Failed to send invite"; - } - if (error.name == 'UnknownDeviceError') { - code = CallErrorCode.UnknownDevices; - message = "Unknown devices present in the room"; - } - - this.emit(CallEvent.Error, new CallError(code, message, error)); - this.terminate(CallParty.Local, code, false); - - // no need to carry on & send the candidate queue, but we also - // don't want to rethrow the error - return; - } - - this.sendCandidateQueue(); - if (this.state === CallState.CreateOffer) { - this.inviteOrAnswerSent = true; - this.setState(CallState.InviteSent); - this.inviteTimeout = setTimeout(() => { - this.inviteTimeout = null; - if (this.state === CallState.InviteSent) { - this.hangup(CallErrorCode.InviteTimeout, false); - } - }, CALL_TIMEOUT_MS); - } - } - - private getLocalOfferFailed = (err: Error): void => { - logger.error("Failed to get local offer", err); - - this.emit( - CallEvent.Error, - new CallError( - CallErrorCode.LocalOfferFailed, - "Failed to get local offer!", err, - ), - ); - this.terminate(CallParty.Local, CallErrorCode.LocalOfferFailed, false); - }; - - private getUserMediaFailed = (err: Error): void => { - if (this.successor) { - this.successor.getUserMediaFailed(err); - return; - } - - logger.warn("Failed to get user media - ending call", err); - - this.emit( - CallEvent.Error, - new CallError( - CallErrorCode.NoUserMedia, - "Couldn't start capturing media! Is your microphone set up and " + - "does this app have permission?", err, - ), - ); - this.terminate(CallParty.Local, CallErrorCode.NoUserMedia, false); - }; - - private onIceConnectionStateChanged = (): void => { - if (this.callHasEnded()) { - return; // because ICE can still complete as we're ending the call - } - logger.debug( - "Call ID " + this.callId + ": ICE connection state changed to: " + this.peerConn.iceConnectionState, - ); - // ideally we'd consider the call to be connected when we get media but - // chrome doesn't implement any of the 'onstarted' events yet - if (this.peerConn.iceConnectionState == 'connected') { - clearTimeout(this.iceDisconnectedTimeout); - this.setState(CallState.Connected); - - if (!this.callLengthInterval) { - this.callLengthInterval = setInterval(() => { - this.callLength++; - this.emit(CallEvent.LengthChanged, this.callLength); - }, 1000); - } - } else if (this.peerConn.iceConnectionState == 'failed') { - // Firefox for Android does not yet have support for restartIce() - if (this.peerConn.restartIce) { - this.peerConn.restartIce(); - } else { - this.hangup(CallErrorCode.IceFailed, false); - } - } else if (this.peerConn.iceConnectionState == 'disconnected') { - this.iceDisconnectedTimeout = setTimeout(() => { - this.hangup(CallErrorCode.IceFailed, false); - }, 30 * 1000); - } - }; - - private onSignallingStateChanged = (): void => { - logger.debug( - "call " + this.callId + ": Signalling state changed to: " + - this.peerConn.signalingState, - ); - }; - - private onTrack = (ev: RTCTrackEvent): void => { - if (ev.streams.length === 0) { - logger.warn(`Streamless ${ev.track.kind} found: ignoring.`); - return; - } - - const stream = ev.streams[0]; - this.pushRemoteFeed(stream); - stream.addEventListener("removetrack", () => { - logger.log(`Removing track streamId: ${stream.id}`); - this.deleteFeedByStream(stream); - }); - }; - - private onDataChannel = (ev: RTCDataChannelEvent): void => { - this.emit(CallEvent.DataChannel, ev.channel); - }; - - /** - * This method removes all video/rtx codecs from screensharing video - * transceivers. This is necessary since they can cause problems. Without - * this the following steps should produce an error: - * Chromium calls Firefox - * Firefox answers - * Firefox starts screen-sharing - * Chromium starts screen-sharing - * Call crashes for Chromium with: - * [96685:23:0518/162603.933321:ERROR:webrtc_video_engine.cc(3296)] RTX codec (PT=97) mapped to PT=96 which is not in the codec list. - * [96685:23:0518/162603.933377:ERROR:webrtc_video_engine.cc(1171)] GetChangedRecvParameters called without any video codecs. - * [96685:23:0518/162603.933430:ERROR:sdp_offer_answer.cc(4302)] Failed to set local video description recv parameters for m-section with mid='2'. (INVALID_PARAMETER) - */ - private getRidOfRTXCodecs(): void { - // RTCRtpReceiver.getCapabilities and RTCRtpSender.getCapabilities don't seem to be supported on FF - if (!RTCRtpReceiver.getCapabilities || !RTCRtpSender.getCapabilities) return; - - const recvCodecs = RTCRtpReceiver.getCapabilities("video").codecs; - const sendCodecs = RTCRtpSender.getCapabilities("video").codecs; - const codecs = [...sendCodecs, ...recvCodecs]; - - for (const codec of codecs) { - if (codec.mimeType === "video/rtx") { - const rtxCodecIndex = codecs.indexOf(codec); - codecs.splice(rtxCodecIndex, 1); - } - } - - for (const trans of this.peerConn.getTransceivers()) { - if ( - this.screensharingSenders.includes(trans.sender) && - ( - trans.sender.track?.kind === "video" || - trans.receiver.track?.kind === "video" - ) - ) { - trans.setCodecPreferences(codecs); - } - } - } - - private onNegotiationNeeded = async (): Promise => { - logger.info("Negotiation is needed!"); - - if (this.state !== CallState.CreateOffer && this.opponentVersion === 0) { - logger.info("Opponent does not support renegotiation: ignoring negotiationneeded event"); - return; - } - - this.queueGotLocalOffer(); - }; - - public onHangupReceived = (msg: MCallHangupReject): void => { - logger.debug("Hangup received for call ID " + this.callId); - - // party ID must match (our chosen partner hanging up the call) or be undefined (we haven't chosen - // a partner yet but we're treating the hangup as a reject as per VoIP v0) - if (this.partyIdMatches(msg) || this.state === CallState.Ringing) { - // default reason is user_hangup - this.terminate(CallParty.Remote, msg.reason || CallErrorCode.UserHangup, true); - } else { - logger.info(`Ignoring message from party ID ${msg.party_id}: our partner is ${this.opponentPartyId}`); - } - }; - - public onRejectReceived = (msg: MCallHangupReject): void => { - logger.debug("Reject received for call ID " + this.callId); - - // No need to check party_id for reject because if we'd received either - // an answer or reject, we wouldn't be in state InviteSent - - const shouldTerminate = ( - // reject events also end the call if it's ringing: it's another of - // our devices rejecting the call. - ([CallState.InviteSent, CallState.Ringing].includes(this.state)) || - // also if we're in the init state and it's an inbound call, since - // this means we just haven't entered the ringing state yet - this.state === CallState.Fledgling && this.direction === CallDirection.Inbound - ); - - if (shouldTerminate) { - this.terminate(CallParty.Remote, msg.reason || CallErrorCode.UserHangup, true); - } else { - logger.debug(`Call is in state: ${this.state}: ignoring reject`); - } - }; - - public onAnsweredElsewhere = (msg: MCallAnswer): void => { - logger.debug("Call ID " + this.callId + " answered elsewhere"); - this.terminate(CallParty.Remote, CallErrorCode.AnsweredElsewhere, true); - }; - - private setState(state: CallState): void { - const oldState = this.state; - this.state = state; - this.emit(CallEvent.State, state, oldState); - } - - /** - * Internal - * @param {string} eventType - * @param {Object} content - * @return {Promise} - */ - private sendVoipEvent(eventType: string, content: object): Promise { - const realContent = Object.assign({}, content, { - version: VOIP_PROTO_VERSION, - call_id: this.callId, - party_id: this.ourPartyId, - conf_id: this.groupCallId, - }); - - if (this.opponentDeviceId) { - this.emit(CallEvent.SendVoipEvent, { - type: "toDevice", - eventType, - userId: this.invitee || this.getOpponentMember().userId, - opponentDeviceId: this.opponentDeviceId, - content: { - ...realContent, - device_id: this.client.deviceId, - sender_session_id: this.client.getSessionId(), - dest_session_id: this.opponentSessionId, - }, - }); - - return this.client.sendToDevice(eventType, { - [this.invitee || this.getOpponentMember().userId]: { - [this.opponentDeviceId]: { - ...realContent, - device_id: this.client.deviceId, - sender_session_id: this.client.getSessionId(), - dest_session_id: this.opponentSessionId, - }, - }, - }); - } else { - this.emit(CallEvent.SendVoipEvent, { - type: "sendEvent", - eventType, - roomId: this.roomId, - content: realContent, - userId: this.invitee || this.getOpponentMember().userId, - }); - - return this.client.sendEvent(this.roomId, eventType, realContent); - } - } - - private queueCandidate(content: RTCIceCandidate): void { - // We partially de-trickle candidates by waiting for `delay` before sending them - // amalgamated, in order to avoid sending too many m.call.candidates events and hitting - // rate limits in Matrix. - // In practice, it'd be better to remove rate limits for m.call.* - - // N.B. this deliberately lets you queue and send blank candidates, which MSC2746 - // currently proposes as the way to indicate that candidate gathering is complete. - // This will hopefully be changed to an explicit rather than implicit notification - // shortly. - this.candidateSendQueue.push(content); - - // Don't send the ICE candidates yet if the call is in the ringing state: this - // means we tried to pick (ie. started generating candidates) and then failed to - // send the answer and went back to the ringing state. Queue up the candidates - // to send if we successfully send the answer. - // Equally don't send if we haven't yet sent the answer because we can send the - // first batch of candidates along with the answer - if (this.state === CallState.Ringing || !this.inviteOrAnswerSent) return; - - // MSC2746 recommends these values (can be quite long when calling because the - // callee will need a while to answer the call) - const delay = this.direction === CallDirection.Inbound ? 500 : 2000; - - if (this.candidateSendTries === 0) { - setTimeout(() => { - this.sendCandidateQueue(); - }, delay); - } - } - - /* - * Transfers this call to another user - */ - public async transfer(targetUserId: string): Promise { - // Fetch the target user's global profile info: their room avatar / displayname - // could be different in whatever room we share with them. - const profileInfo = await this.client.getProfileInfo(targetUserId); - - const replacementId = genCallID(); - - const body = { - replacement_id: genCallID(), - target_user: { - id: targetUserId, - display_name: profileInfo.displayname, - avatar_url: profileInfo.avatar_url, - }, - create_call: replacementId, - } as MCallReplacesEvent; - - await this.sendVoipEvent(EventType.CallReplaces, body); - - await this.terminate(CallParty.Local, CallErrorCode.Transfered, true); - } - - /* - * Transfers this call to the target call, effectively 'joining' the - * two calls (so the remote parties on each call are connected together). - */ - public async transferToCall(transferTargetCall?: MatrixCall): Promise { - const targetProfileInfo = await this.client.getProfileInfo(transferTargetCall.getOpponentMember().userId); - const transfereeProfileInfo = await this.client.getProfileInfo(this.getOpponentMember().userId); - - const newCallId = genCallID(); - - const bodyToTransferTarget = { - // the replacements on each side have their own ID, and it's distinct from the - // ID of the new call (but we can use the same function to generate it) - replacement_id: genCallID(), - target_user: { - id: this.getOpponentMember().userId, - display_name: transfereeProfileInfo.displayname, - avatar_url: transfereeProfileInfo.avatar_url, - }, - await_call: newCallId, - } as MCallReplacesEvent; - - await transferTargetCall.sendVoipEvent(EventType.CallReplaces, bodyToTransferTarget); - - const bodyToTransferee = { - replacement_id: genCallID(), - target_user: { - id: transferTargetCall.getOpponentMember().userId, - display_name: targetProfileInfo.displayname, - avatar_url: targetProfileInfo.avatar_url, - }, - create_call: newCallId, - } as MCallReplacesEvent; - - await this.sendVoipEvent(EventType.CallReplaces, bodyToTransferee); - - await this.terminate(CallParty.Local, CallErrorCode.Replaced, true); - await transferTargetCall.terminate(CallParty.Local, CallErrorCode.Transfered, true); - } - - private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean): Promise { - if (this.callHasEnded()) return; - - this.hangupParty = hangupParty; - this.hangupReason = hangupReason; - this.setState(CallState.Ended); - - if (this.inviteTimeout) { - clearTimeout(this.inviteTimeout); - this.inviteTimeout = null; - } - if (this.callLengthInterval) { - clearInterval(this.callLengthInterval); - this.callLengthInterval = null; - } - - this.callStatsAtEnd = await this.collectCallStats(); - - // Order is important here: first we stopAllMedia() and only then we can deleteAllFeeds() - this.stopAllMedia(); - this.deleteAllFeeds(); - - if (this.peerConn && this.peerConn.signalingState !== 'closed') { - this.peerConn.close(); - } - if (shouldEmit) { - this.emit(CallEvent.Hangup, this); - } - - this.client.callEventHandler.calls.delete(this.callId); - } - - private stopAllMedia(): void { - logger.debug(!this.groupCallId ? "Stopping all media" : "Stopping all media except local feeds" ); - for (const feed of this.feeds) { - if ( - feed.isLocal() && - feed.purpose === SDPStreamMetadataPurpose.Usermedia && - !this.groupCallId - ) { - this.client.getMediaHandler().stopUserMediaStream(feed.stream); - } else if ( - feed.isLocal() && - feed.purpose === SDPStreamMetadataPurpose.Screenshare && - !this.groupCallId - ) { - this.client.getMediaHandler().stopScreensharingStream(feed.stream); - } else if (!feed.isLocal() || !this.groupCallId) { - for (const track of feed.stream.getTracks()) { - track.stop(); - } - } - } - } - - private checkForErrorListener(): void { - if (this.listeners("error").length === 0) { - throw new Error( - "You MUST attach an error listener using call.on('error', function() {})", - ); - } - } - - private async sendCandidateQueue(): Promise { - if (this.candidateSendQueue.length === 0 || this.callHasEnded()) { - return; - } - - const candidates = this.candidateSendQueue; - this.candidateSendQueue = []; - ++this.candidateSendTries; - const content = { - candidates: candidates, - }; - logger.debug("Attempting to send " + candidates.length + " candidates"); - try { - await this.sendVoipEvent(EventType.CallCandidates, content); - // reset our retry count if we have successfully sent our candidates - // otherwise queueCandidate() will refuse to try to flush the queue - this.candidateSendTries = 0; - - // Try to send candidates again just in case we received more candidates while sending. - this.sendCandidateQueue(); - } catch (error) { - // don't retry this event: we'll send another one later as we might - // have more candidates by then. - if (error.event) this.client.cancelPendingEvent(error.event); - - // put all the candidates we failed to send back in the queue - this.candidateSendQueue.push(...candidates); - - if (this.candidateSendTries > 5) { - logger.debug( - "Failed to send candidates on attempt " + this.candidateSendTries + - ". Giving up on this call.", error, - ); - - const code = CallErrorCode.SignallingFailed; - const message = "Signalling failed"; - - this.emit(CallEvent.Error, new CallError(code, message, error)); - this.hangup(code, false); - - return; - } - - const delayMs = 500 * Math.pow(2, this.candidateSendTries); - ++this.candidateSendTries; - logger.debug("Failed to send candidates. Retrying in " + delayMs + "ms", error); - setTimeout(() => { - this.sendCandidateQueue(); - }, delayMs); - } - } - - /** - * Place a call to this room. - * @throws if you have not specified a listener for 'error' events. - * @throws if have passed audio=false. - */ - public async placeCall(audio: boolean, video: boolean): Promise { - if (!audio) { - throw new Error("You CANNOT start a call without audio"); - } - this.setState(CallState.WaitLocalMedia); - - try { - const stream = await this.client.getMediaHandler().getUserMediaStream(audio, video); - const callFeed = new CallFeed({ - client: this.client, - roomId: this.roomId, - userId: this.client.getUserId(), - stream, - purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: stream.getAudioTracks().length === 0, - videoMuted: stream.getVideoTracks().length === 0, - }); - await this.placeCallWithCallFeeds([callFeed]); - } catch (e) { - this.getUserMediaFailed(e); - return; - } - } - - /** - * Place a call to this room with call feed. - * @param {CallFeed[]} callFeeds to use - * @throws if you have not specified a listener for 'error' events. - * @throws if have passed audio=false. - */ - public async placeCallWithCallFeeds(callFeeds: CallFeed[], requestScreenshareFeed = false): Promise { - this.checkForErrorListener(); - this.direction = CallDirection.Outbound; - - // XXX Find a better way to do this - this.client.callEventHandler.calls.set(this.callId, this); - - // make sure we have valid turn creds. Unless something's gone wrong, it should - // poll and keep the credentials valid so this should be instant. - const haveTurnCreds = await this.client.checkTurnServers(); - if (!haveTurnCreds) { - logger.warn("Failed to get TURN credentials! Proceeding with call anyway..."); - } - - // create the peer connection now so it can be gathering candidates while we get user - // media (assuming a candidate pool size is configured) - this.peerConn = this.createPeerConnection(); - this.gotCallFeedsForInvite(callFeeds, requestScreenshareFeed); - } - - private createPeerConnection(): RTCPeerConnection { - const pc = new window.RTCPeerConnection({ - iceTransportPolicy: this.forceTURN ? 'relay' : undefined, - iceServers: this.turnServers, - iceCandidatePoolSize: this.client.iceCandidatePoolSize, - }); - - // 'connectionstatechange' would be better, but firefox doesn't implement that. - pc.addEventListener('iceconnectionstatechange', this.onIceConnectionStateChanged); - pc.addEventListener('signalingstatechange', this.onSignallingStateChanged); - pc.addEventListener('icecandidate', this.gotLocalIceCandidate); - pc.addEventListener('icegatheringstatechange', this.onIceGatheringStateChange); - pc.addEventListener('track', this.onTrack); - pc.addEventListener('negotiationneeded', this.onNegotiationNeeded); - pc.addEventListener('datachannel', this.onDataChannel); - - return pc; - } - - private partyIdMatches(msg: MCallBase): boolean { - // They must either match or both be absent (in which case opponentPartyId will be null) - // Also we ignore party IDs on the invite/offer if the version is 0, so we must do the same - // here and use null if the version is 0 (woe betide any opponent sending messages in the - // same call with different versions) - const msgPartyId = msg.version === 0 ? null : msg.party_id || null; - return msgPartyId === this.opponentPartyId; - } - - // Commits to an opponent for the call - // ev: An invite or answer event - private chooseOpponent(ev: MatrixEvent): void { - // I choo-choo-choose you - const msg = ev.getContent(); - - logger.debug(`Choosing party ID ${msg.party_id} for call ID ${this.callId}`); - - this.opponentVersion = msg.version; - if (this.opponentVersion === 0) { - // set to null to indicate that we've chosen an opponent, but because - // they're v0 they have no party ID (even if they sent one, we're ignoring it) - this.opponentPartyId = null; - } else { - // set to their party ID, or if they're naughty and didn't send one despite - // not being v0, set it to null to indicate we picked an opponent with no - // party ID - this.opponentPartyId = msg.party_id || null; - } - this.opponentCaps = msg.capabilities || {} as CallCapabilities; - this.opponentMember = this.client.getRoom(this.roomId).getMember(ev.getSender()); - } - - private async addBufferedIceCandidates(): Promise { - const bufferedCandidates = this.remoteCandidateBuffer.get(this.opponentPartyId); - if (bufferedCandidates) { - logger.info(`Adding ${bufferedCandidates.length} buffered candidates for opponent ${this.opponentPartyId}`); - await this.addIceCandidates(bufferedCandidates); - } - this.remoteCandidateBuffer = null; - } - - private async addIceCandidates(candidates: RTCIceCandidate[]): Promise { - for (const candidate of candidates) { - if ( - (candidate.sdpMid === null || candidate.sdpMid === undefined) && - (candidate.sdpMLineIndex === null || candidate.sdpMLineIndex === undefined) - ) { - logger.debug("Ignoring remote ICE candidate with no sdpMid or sdpMLineIndex"); - continue; - } - logger.debug( - "Call " + this.callId + " got remote ICE " + candidate.sdpMid + " candidate: " + candidate.candidate, - ); - try { - await this.peerConn.addIceCandidate(candidate); - } catch (err) { - if (!this.ignoreOffer) { - logger.info("Failed to add remote ICE candidate", err); - } - } - } - } - - public get hasPeerConnection(): boolean { - return Boolean(this.peerConn); - } -} - -export function setTracksEnabled(tracks: Array, enabled: boolean): void { - for (let i = 0; i < tracks.length; i++) { - tracks[i].enabled = enabled; - } -} - -/** - * DEPRECATED - * Use client.createCall() - * - * Create a new Matrix call for the browser. - * @param {MatrixClient} client The client instance to use. - * @param {string} roomId The room the call is in. - * @param {Object?} options DEPRECATED optional options map. - * @param {boolean} options.forceTURN DEPRECATED whether relay through TURN should be - * forced. This option is deprecated - use opts.forceTURN when creating the matrix client - * since it's only possible to set this option on outbound calls. - * @return {MatrixCall} the call or null if the browser doesn't support calling. - */ -export function createNewMatrixCall(client: any, roomId: string, options?: CallOpts): MatrixCall { - // typeof prevents Node from erroring on an undefined reference - if (typeof(window) === 'undefined' || typeof(document) === 'undefined') { - // NB. We don't log here as apps try to create a call object as a test for - // whether calls are supported, so we shouldn't fill the logs up. - return null; - } - - // Firefox throws on so little as accessing the RTCPeerConnection when operating in - // a secure mode. There's some information at https://bugzilla.mozilla.org/show_bug.cgi?id=1542616 - // though the concern is that the browser throwing a SecurityError will brick the - // client creation process. - try { - const supported = Boolean( - window.RTCPeerConnection || window.RTCSessionDescription || - window.RTCIceCandidate || navigator.mediaDevices, - ); - if (!supported) { - // Adds a lot of noise to test runs, so disable logging there. - if (process.env.NODE_ENV !== "test") { - logger.error("WebRTC is not supported in this browser / environment"); - } - return null; - } - } catch (e) { - logger.error("Exception thrown when trying to access WebRTC", e); - return null; - } - - const optionsForceTURN = options ? options.forceTURN : false; - - const opts: CallOpts = { - client: client, - roomId: roomId, - invitee: options?.invitee, - turnServers: client.getTurnServers(), - // call level options - forceTURN: client.forceTURN || optionsForceTURN, - opponentDeviceId: options?.opponentDeviceId, - opponentSessionId: options?.opponentSessionId, - groupCallId: options?.groupCallId, - }; - const call = new MatrixCall(opts); - - client.reEmitter.reEmit(call, Object.values(CallEvent)); - - return call; -} diff --git a/src/matrix/calls/CallFeed.ts b/src/matrix/calls/CallFeed.ts deleted file mode 100644 index c8cc8662..00000000 --- a/src/matrix/calls/CallFeed.ts +++ /dev/null @@ -1,274 +0,0 @@ -/* -Copyright 2021 Šimon Brandner - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { SDPStreamMetadataPurpose } from "./callEventTypes"; - -const POLLING_INTERVAL = 200; // ms -export const SPEAKING_THRESHOLD = -60; // dB -const SPEAKING_SAMPLE_COUNT = 8; // samples - -export interface ICallFeedOpts { - client: MatrixClient; - roomId: string; - userId: string; - stream: MediaStream; - purpose: SDPStreamMetadataPurpose; - audioMuted: boolean; - videoMuted: boolean; -} - -export enum CallFeedEvent { - NewStream = "new_stream", - MuteStateChanged = "mute_state_changed", - VolumeChanged = "volume_changed", - Speaking = "speaking", -} - -export class CallFeed extends EventEmitter { - public stream: MediaStream; - public sdpMetadataStreamId: string; - public userId: string; - public purpose: SDPStreamMetadataPurpose; - public speakingVolumeSamples: number[]; - - private client: MatrixClient; - private roomId: string; - private audioMuted: boolean; - private videoMuted: boolean; - private measuringVolumeActivity = false; - private audioContext: AudioContext; - private analyser: AnalyserNode; - private frequencyBinCount: Float32Array; - private speakingThreshold = SPEAKING_THRESHOLD; - private speaking = false; - private volumeLooperTimeout: number; - - constructor(opts: ICallFeedOpts) { - super(); - - this.client = opts.client; - this.roomId = opts.roomId; - this.userId = opts.userId; - this.purpose = opts.purpose; - this.audioMuted = opts.audioMuted; - this.videoMuted = opts.videoMuted; - this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); - this.sdpMetadataStreamId = opts.stream.id; - - this.updateStream(null, opts.stream); - - if (this.hasAudioTrack) { - this.initVolumeMeasuring(); - } - } - - private get hasAudioTrack(): boolean { - return this.stream.getAudioTracks().length > 0; - } - - private updateStream(oldStream: MediaStream, newStream: MediaStream): void { - if (newStream === oldStream) return; - - if (oldStream) { - oldStream.removeEventListener("addtrack", this.onAddTrack); - this.measureVolumeActivity(false); - } - if (newStream) { - this.stream = newStream; - newStream.addEventListener("addtrack", this.onAddTrack); - - if (this.hasAudioTrack) { - this.initVolumeMeasuring(); - } else { - this.measureVolumeActivity(false); - } - } - - this.emit(CallFeedEvent.NewStream, this.stream); - } - - private initVolumeMeasuring(): void { - const AudioContext = window.AudioContext || window.webkitAudioContext; - if (!this.hasAudioTrack || !AudioContext) return; - - this.audioContext = new AudioContext(); - - this.analyser = this.audioContext.createAnalyser(); - this.analyser.fftSize = 512; - this.analyser.smoothingTimeConstant = 0.1; - - const mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.stream); - mediaStreamAudioSourceNode.connect(this.analyser); - - this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount); - } - - private onAddTrack = (): void => { - this.emit(CallFeedEvent.NewStream, this.stream); - }; - - /** - * Returns callRoom member - * @returns member of the callRoom - */ - public getMember(): RoomMember { - const callRoom = this.client.getRoom(this.roomId); - return callRoom.getMember(this.userId); - } - - /** - * Returns true if CallFeed is local, otherwise returns false - * @returns {boolean} is local? - */ - public isLocal(): boolean { - return this.userId === this.client.getUserId(); - } - - /** - * Returns true if audio is muted or if there are no audio - * tracks, otherwise returns false - * @returns {boolean} is audio muted? - */ - public isAudioMuted(): boolean { - return this.stream.getAudioTracks().length === 0 || this.audioMuted; - } - - /** - * Returns true video is muted or if there are no video - * tracks, otherwise returns false - * @returns {boolean} is video muted? - */ - public isVideoMuted(): boolean { - // We assume only one video track - return this.stream.getVideoTracks().length === 0 || this.videoMuted; - } - - public isSpeaking(): boolean { - return this.speaking; - } - - /** - * Replaces the current MediaStream with a new one. - * This method should be only used by MatrixCall. - * @param newStream new stream with which to replace the current one - */ - public setNewStream(newStream: MediaStream): void { - this.updateStream(this.stream, newStream); - } - - /** - * Set feed's internal audio mute state - * @param muted is the feed's audio muted? - */ - public setAudioMuted(muted: boolean): void { - this.audioMuted = muted; - this.speakingVolumeSamples.fill(-Infinity); - this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); - } - - /** - * Set feed's internal video mute state - * @param muted is the feed's video muted? - */ - public setVideoMuted(muted: boolean): void { - this.videoMuted = muted; - this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); - } - - /** - * Starts emitting volume_changed events where the emitter value is in decibels - * @param enabled emit volume changes - */ - public measureVolumeActivity(enabled: boolean): void { - if (enabled) { - if (!this.audioContext || !this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return; - - this.measuringVolumeActivity = true; - this.volumeLooper(); - } else { - this.measuringVolumeActivity = false; - this.speakingVolumeSamples.fill(-Infinity); - this.emit(CallFeedEvent.VolumeChanged, -Infinity); - } - } - - public setSpeakingThreshold(threshold: number) { - this.speakingThreshold = threshold; - } - - private volumeLooper = () => { - if (!this.analyser) return; - - if (!this.measuringVolumeActivity) return; - - this.analyser.getFloatFrequencyData(this.frequencyBinCount); - - let maxVolume = -Infinity; - for (let i = 0; i < this.frequencyBinCount.length; i++) { - if (this.frequencyBinCount[i] > maxVolume) { - maxVolume = this.frequencyBinCount[i]; - } - } - - this.speakingVolumeSamples.shift(); - this.speakingVolumeSamples.push(maxVolume); - - this.emit(CallFeedEvent.VolumeChanged, maxVolume); - - let newSpeaking = false; - - for (let i = 0; i < this.speakingVolumeSamples.length; i++) { - const volume = this.speakingVolumeSamples[i]; - - if (volume > this.speakingThreshold) { - newSpeaking = true; - break; - } - } - - if (this.speaking !== newSpeaking) { - this.speaking = newSpeaking; - this.emit(CallFeedEvent.Speaking, this.speaking); - } - - this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL); - }; - - public clone(): CallFeed { - const mediaHandler = this.client.getMediaHandler(); - const stream = this.stream.clone(); - - if (this.purpose === SDPStreamMetadataPurpose.Usermedia) { - mediaHandler.userMediaStreams.push(stream); - } else { - mediaHandler.screensharingStreams.push(stream); - } - - return new CallFeed({ - client: this.client, - roomId: this.roomId, - userId: this.userId, - stream, - purpose: this.purpose, - audioMuted: this.audioMuted, - videoMuted: this.videoMuted, - }); - } - - public dispose(): void { - clearTimeout(this.volumeLooperTimeout); - } -} diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 1dad9ce8..d7d673ed 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -22,6 +22,8 @@ import type {ILogItem} from "../../logging/types"; import {WebRTC, PeerConnection, PeerConnectionHandler, StreamPurpose} from "../../platform/types/WebRTC"; import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; +import type {SignallingMessage} from "./PeerCall"; +import type {MGroupCallBase} from "./callEventTypes"; const GROUP_CALL_TYPE = "m.call"; const GROUP_CALL_MEMBER_TYPE = "m.call.member"; @@ -33,7 +35,7 @@ enum CallSetupMessageType { Hangup = "m.call.hangup", } -const CALL_ID = "m.call_id"; +const CONF_ID = "conf_id"; const CALL_TERMINATED = "m.terminated"; export class GroupCallHandler { @@ -69,7 +71,7 @@ export class GroupCallHandler { const participant = event.state_key; const sources = event.content["m.sources"]; for (const source of sources) { - const call = this.calls.get(source[CALL_ID]); + const call = this.calls.get(source[CONF_ID]); if (call && !call.isTerminated) { call.addParticipant(participant, source); } @@ -85,110 +87,9 @@ export class GroupCallHandler { eventType === CallSetupMessageType.Hangup; } - handleDeviceMessage(senderUserId: string, senderDeviceId: string, eventType: string, content: Record, log: ILogItem) { - const callId = content[CALL_ID]; - const call = this.calls.get(callId); - call?.handleDeviceMessage(senderUserId, senderDeviceId, eventType, content, log); + handleDeviceMessage(senderUserId: string, senderDeviceId: string, event: SignallingMessage, log: ILogItem) { + const call = this.calls.get(event.content.conf_id); + call?.handleDeviceMessage(senderUserId, senderDeviceId, event, log); } } -function participantId(senderUserId: string, senderDeviceId: string | null) { - return JSON.stringify(senderUserId) + JSON.stringify(senderDeviceId); -} - -class GroupParticipant implements PeerCallHandler { - private peerCall?: PeerCall; - - constructor( - private readonly userId: string, - private readonly deviceId: string, - private localMedia: LocalMedia | undefined, - private readonly webRTC: WebRTC, - private readonly hsApi: HomeServerApi - ) {} - - sendInvite() { - this.peerCall = new PeerCall(this, this.webRTC); - this.peerCall.call(this.localMedia); - } - - /** From PeerCallHandler - * @internal */ - override emitUpdate() { - - } - - /** From PeerCallHandler - * @internal */ - override onSendSignallingMessage() { - // TODO: this needs to be encrypted with olm first - this.hsApi.sendToDevice(type, {[this.userId]: {[this.deviceId ?? "*"]: content}}); - } -} - -class GroupCall { - private readonly participants: ObservableMap = new ObservableMap(); - private localMedia?: LocalMedia; - - constructor(private readonly ownUserId: string, private callEvent: StateEvent, private readonly room: Room, private readonly webRTC: WebRTC) { - - } - - get id(): string { return this.callEvent.state_key; } - - async participate(tracks: Track[]) { - this.localMedia = LocalMedia.fromTracks(tracks); - for (const [,participant] of this.participants) { - participant.setMedia(this.localMedia.clone()); - } - // send m.call.member state event - - // send invite to all participants that are < my userId - for (const [,participant] of this.participants) { - if (participant.userId < this.ownUserId) { - participant.sendInvite(); - } - } - } - - updateCallEvent(callEvent: StateEvent) { - this.callEvent = callEvent; - } - - addParticipant(userId, source) { - const participantId = getParticipantId(userId, source.device_id); - const participant = this.participants.get(participantId); - if (participant) { - participant.updateSource(source); - } else { - participant.add(participantId, new GroupParticipant(userId, source.device_id, this.localMedia?.clone(), this.webRTC)); - } - } - - handleDeviceMessage(senderUserId: string, senderDeviceId: string, eventType: string, content: Record, log: ILogItem) { - const participantId = getParticipantId(senderUserId, senderDeviceId); - let peerCall = this.participants.get(participantId); - let hasDeviceInKey = true; - if (!peerCall) { - hasDeviceInKey = false; - peerCall = this.participants.get(getParticipantId(senderUserId, null)) - } - if (peerCall) { - peerCall.handleIncomingSignallingMessage(eventType, content, senderDeviceId); - if (!hasDeviceInKey && peerCall.opponentPartyId) { - this.participants.delete(getParticipantId(senderUserId, null)); - this.participants.add(getParticipantId(senderUserId, peerCall.opponentPartyId)); - } - } else { - // create peerCall - } - } - - get id(): string { - return this.callEvent.state_key; - } - - get isTerminated(): boolean { - return !!this.callEvent.content[CALL_TERMINATED]; - } -} diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index a02de99f..5040c805 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -17,26 +17,41 @@ limitations under the License. import {ObservableMap} from "../../observable/map/ObservableMap"; import {recursivelyAssign} from "../../utils/recursivelyAssign"; import {AsyncQueue} from "../../utils/AsyncQueue"; -import {Disposables, Disposable} from "../../utils/Disposables"; +import {Disposables, IDisposable} from "../../utils/Disposables"; import type {Room} from "../room/Room"; import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; import type {TimeoutCreator, Timeout} from "../../platform/types/types"; -import {WebRTC, PeerConnection, PeerConnectionHandler} from "../../platform/types/WebRTC"; +import {WebRTC, PeerConnection, PeerConnectionHandler, DataChannel} from "../../platform/types/WebRTC"; import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; import type {LocalMedia} from "./LocalMedia"; +import { + SDPStreamMetadataKey, + SDPStreamMetadataPurpose +} from "./callEventTypes"; +import type { + MCallBase, + MCallInvite, + MCallAnswer, + MCallSDPStreamMetadataChanged, + MCallCandidates, + MCallHangupReject, + SDPStreamMetadata, +} from "./callEventTypes"; + // when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already // do for sharing keys will be best as that already deals with room tracking. /** * Does WebRTC signalling for a single PeerConnection, and deals with WebRTC wrappers from platform * */ /** Implements a call between two peers with the signalling state keeping, while still delegating the signalling message sending. Used by GroupCall.*/ -class PeerCall { +export class PeerCall implements IDisposable { private readonly peerConnection: PeerConnection; private state = CallState.Fledgling; private direction: CallDirection; + private localMedia?: LocalMedia; // A queue for candidates waiting to go out. // We try to amalgamate candidates into a single candidate message where // possible @@ -54,9 +69,13 @@ class PeerCall { private disposables = new Disposables(); private statePromiseMap = new Map void, promise: Promise}>(); + // perfect negotiation flags + private makingOffer: boolean = false; + private ignoreOffer: boolean = false; + constructor( + private callId: string, // generated or from invite private readonly handler: PeerCallHandler, - private localMedia: LocalMedia, private readonly createTimeout: TimeoutCreator, webRTC: WebRTC ) { @@ -83,29 +102,8 @@ class PeerCall { } } - handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId) { - switch (message.type) { - case EventType.Invite: - // determining whether or not an incoming invite glares - // with an instance of PeerCall is different for group calls - // and 1:1 calls, so done outside of this class. - // If you pass an event for another call id in here it will assume it glares. - - //const newCallId = message.content.call_id; - //if (this.id && newCallId !== this.id) { - // this.handleInviteGlare(message.content); - //} else { - this.handleInvite(message.content, partyId); - //} - break; - case EventType.Answer: - this.handleAnswer(message.content, partyId); - break; - case EventType.Candidates: - this.handleRemoteIceCandidates(message.content, partyId); - break; - case EventType.Hangup: - } + get remoteTracks(): Track[] { + return this.peerConnection.remoteTracks; } async call(localMediaPromise: Promise): Promise { @@ -125,7 +123,9 @@ class PeerCall { for (const t of this.localMedia.tracks) { this.peerConnection.addTrack(t); } - await this.waitForState(CallState.InviteSent); + // TODO: in case of glare, we would not go to InviteSent if we haven't started sending yet + // but we would go straight to CreateAnswer, so also need to wait for that state + await this.waitForState([CallState.InviteSent, CallState.CreateAnswer]); } async answer(localMediaPromise: Promise): Promise { @@ -166,15 +166,11 @@ class PeerCall { this.sendAnswer(); } - async hangup() { - - } - async setMedia(localMediaPromise: Promise) { const oldMedia = this.localMedia; this.localMedia = await localMediaPromise; - const applyTrack = (selectTrack: (media: LocalMedia) => Track | undefined) => { + const applyTrack = (selectTrack: (media: LocalMedia | undefined) => Track | undefined) => { const oldTrack = selectTrack(oldMedia); const newTrack = selectTrack(this.localMedia); if (oldTrack && newTrack) { @@ -187,50 +183,101 @@ class PeerCall { }; // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called - applyTrack(m => m.microphoneTrack); - applyTrack(m => m.cameraTrack); - applyTrack(m => m.screenShareTrack); + applyTrack(m => m?.microphoneTrack); + applyTrack(m => m?.cameraTrack); + applyTrack(m => m?.screenShareTrack); + } + + async reject() { + + } + + async hangup(errorCode: CallErrorCode) { + } + + async handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId): Promise { + switch (message.type) { + case EventType.Invite: + if (this.callId !== message.content.call_id) { + await this.handleInviteGlare(message.content, partyId); + } else { + await this.handleFirstInvite(message.content, partyId); + } + break; + case EventType.Answer: + await this.handleAnswer(message.content, partyId); + break; + //case EventType.Candidates: + // await this.handleRemoteIceCandidates(message.content, partyId); + // break; + case EventType.Hangup: + default: + throw new Error(`Unknown event type for call: ${message.type}`); + } + } + + private sendHangupWithCallId(callId: string, reason?: CallErrorCode): Promise { + const content = { + call_id: callId, + version: 1, + }; + if (reason) { + content["reason"] = reason; + } + return this.handler.sendSignallingMessage({ + type: EventType.Hangup, + content + }); } // calls are serialized and deduplicated by responsePromiseChain private handleNegotiation = async (): Promise => { - // TODO: does this make sense to have this state if we're already connected? - this.setState(CallState.MakingOffer) + this.makingOffer = true; try { - await this.peerConnection.setLocalDescription(); - } catch (err) { - this.logger.debug(`Call ${this.callId} Error setting local description!`, err); - this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); - return; + try { + await this.peerConnection.setLocalDescription(); + } catch (err) { + this.logger.debug(`Call ${this.callId} Error setting local description!`, err); + this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); + return; + } + + if (this.peerConnection.iceGatheringState === 'gathering') { + // Allow a short time for initial candidates to be gathered + await this.delay(200); + } + + if (this.state === CallState.Ended) { + return; + } + + const offer = this.peerConnection.localDescription!; + // Get rid of any candidates waiting to be sent: they'll be included in the local + // description we just got and will send in the offer. + this.logger.info(`Call ${this.callId} Discarding ${ + this.candidateSendQueue.length} candidates that will be sent in offer`); + this.candidateSendQueue = []; + + // need to queue this + const content = { + call_id: this.callId, + offer, + [SDPStreamMetadataKey]: this.localMedia!.getSDPMetadata(), + version: 1, + lifetime: CALL_TIMEOUT_MS + }; + if (this.state === CallState.CreateOffer) { + await this.handler.sendSignallingMessage({type: EventType.Invite, content}); + this.setState(CallState.InviteSent); + } else if (this.state === CallState.Connected || this.state === CallState.Connecting) { + // send Negotiate message + //await this.handler.sendSignallingMessage({type: EventType.Invite, content}); + //this.setState(CallState.InviteSent); + } + } finally { + this.makingOffer = false; } - if (this.peerConnection.iceGatheringState === 'gathering') { - // Allow a short time for initial candidates to be gathered - await this.delay(200); - } - - if (this.state === CallState.Ended) { - return; - } - - const offer = this.peerConnection.localDescription!; - // Get rid of any candidates waiting to be sent: they'll be included in the local - // description we just got and will send in the offer. - this.logger.info(`Call ${this.callId} Discarding ${ - this.candidateSendQueue.length} candidates that will be sent in offer`); - this.candidateSendQueue = []; - - // need to queue this - const content = { - offer, - [SDPStreamMetadataKey]: this.localMedia.getSDPMetadata(), - version: 1, - lifetime: CALL_TIMEOUT_MS - }; - if (this.state === CallState.CreateOffer) { - await this.handler.sendSignallingMessage({type: EventType.Invite, content}); - this.setState(CallState.InviteSent); - } this.sendCandidateQueue(); if (this.state === CallState.InviteSent) { @@ -242,11 +289,47 @@ class PeerCall { } }; - private async handleInvite(content: InviteContent, partyId: PartyId): Promise { + private async handleInviteGlare(content: MCallInvite, partyId: PartyId): Promise { + // this is only called when the ids are different + const newCallId = content.call_id; + if (this.callId! > newCallId) { + this.logger.log( + "Glare detected: answering incoming call " + newCallId + + " and canceling outgoing call " + this.callId, + ); + + /* + first, we should set CallDirection + we should anser the call + */ + + // TODO: review states to be unambigous, WaitLocalMedia for sending offer or answer? + // How do we interrupt `call()`? well, perhaps we need to not just await InviteSent but also CreateAnswer? + if (this.state === CallState.Fledgling || this.state === CallState.CreateOffer || this.state === CallState.WaitLocalMedia) { + + } else { + await this.sendHangupWithCallId(this.callId, CallErrorCode.Replaced); + } + await this.handleInvite(content, partyId); + await this.answer(Promise.resolve(this.localMedia!)); + } else { + this.logger.log( + "Glare detected: rejecting incoming call " + newCallId + + " and keeping outgoing call " + this.callId, + ); + await this.sendHangupWithCallId(newCallId, CallErrorCode.Replaced); + } + } + + private async handleFirstInvite(content: MCallInvite, partyId: PartyId): Promise { if (this.state !== CallState.Fledgling || this.opponentPartyId !== undefined) { // TODO: hangup or ignore? return; } + await this.handleInvite(content, partyId); + } + + private async handleInvite(content: MCallInvite, partyId: PartyId): Promise { // we must set the party ID before await-ing on anything: the call event // handler will start giving us more call events (eg. candidates) so if @@ -296,8 +379,8 @@ class PeerCall { } } - private async handleAnswer(content: AnwserContent, partyId: PartyId): Promise { - this.logger.debug(`Got answer for call ID ${this.callId} from party ID ${content.party_id}`); + private async handleAnswer(content: MCallAnswer, partyId: PartyId): Promise { + this.logger.debug(`Got answer for call ID ${this.callId} from party ID ${partyId}`); if (this.state === CallState.Ended) { this.logger.debug(`Ignoring answer because call ID ${this.callId} has ended`); @@ -307,7 +390,7 @@ class PeerCall { if (this.opponentPartyId !== undefined) { this.logger.info( `Call ${this.callId} ` + - `Ignoring answer from party ID ${content.party_id}: ` + + `Ignoring answer from party ID ${partyId}: ` + `we already have an answer/reject from ${this.opponentPartyId}`, ); return; @@ -334,16 +417,66 @@ class PeerCall { } } + // private async onNegotiateReceived(event: MatrixEvent): Promise { + // const content = event.getContent(); + // const description = content.description; + // if (!description || !description.sdp || !description.type) { + // this.logger.info(`Call ${this.callId} Ignoring invalid m.call.negotiate event`); + // return; + // } + // // Politeness always follows the direction of the call: in a glare situation, + // // we pick either the inbound or outbound call, so one side will always be + // // inbound and one outbound + // const polite = this.direction === CallDirection.Inbound; + + // // Here we follow the perfect negotiation logic from + // // https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation + // const offerCollision = ( + // (description.type === 'offer') && + // (this.makingOffer || this.peerConnection.signalingState !== 'stable') + // ); + + // this.ignoreOffer = !polite && offerCollision; + // if (this.ignoreOffer) { + // this.logger.info(`Call ${this.callId} Ignoring colliding negotiate event because we're impolite`); + // return; + // } + + // const sdpStreamMetadata = content[SDPStreamMetadataKey]; + // if (sdpStreamMetadata) { + // this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); + // } else { + // this.logger.warn(`Call ${this.callId} Received negotiation event without SDPStreamMetadata!`); + // } + + // try { + // await this.peerConnection.setRemoteDescription(description); + + // if (description.type === 'offer') { + // await this.peerConnection.setLocalDescription(); + // await this.handler.sendSignallingMessage({ + // type: EventType.CallNegotiate, + // content: { + // description: this.peerConnection.localDescription!, + // [SDPStreamMetadataKey]: this.localMedia.getSDPMetadata(), + // } + // }); + // } + // } catch (err) { + // this.logger.warn(`Call ${this.callId} Failed to complete negotiation`, err); + // } + // } + private async sendAnswer(): Promise { - const answerMessage: AnswerMessage = { - type: EventType.Answer, - content: { - answer: { - sdp: this.peerConnection.localDescription!.sdp, - type: this.peerConnection.localDescription!.type, - }, - [SDPStreamMetadataKey]: this.localMedia.getSDPMetadata(), - } + const localDescription = this.peerConnection.localDescription!; + const answerContent: MCallAnswer = { + call_id: this.callId, + version: 1, + answer: { + sdp: localDescription.sdp, + type: localDescription.type, + }, + [SDPStreamMetadataKey]: this.localMedia!.getSDPMetadata(), }; // We have just taken the local description from the peerConn which will @@ -354,7 +487,7 @@ class PeerCall { this.candidateSendQueue = []; try { - await this.handler.sendSignallingMessage(answerMessage); + await this.handler.sendSignallingMessage({type: EventType.Answer, content: answerContent}); } catch (error) { this.terminate(CallParty.Local, CallErrorCode.SendAnswer, false); throw error; @@ -387,7 +520,6 @@ class PeerCall { }); } - private async sendCandidateQueue(): Promise { if (this.candidateSendQueue.length === 0 || this.state === CallState.Ended) { return; @@ -395,15 +527,16 @@ class PeerCall { const candidates = this.candidateSendQueue; this.candidateSendQueue = []; - const candidatesMessage: CandidatesMessage = { - type: EventType.Candidates, - content: { - candidates: candidates, - } - }; this.logger.debug(`Call ${this.callId} attempting to send ${candidates.length} candidates`); try { - await this.handler.sendSignallingMessage(candidatesMessage); + await this.handler.sendSignallingMessage({ + type: EventType.Candidates, + content: { + call_id: this.callId, + version: 1, + candidates + } + }); // Try to send candidates again just in case we received more candidates while sending. this.sendCandidateQueue(); } catch (error) { @@ -430,7 +563,6 @@ class PeerCall { } } - private async addBufferedIceCandidates(): Promise { if (this.remoteCandidateBuffer && this.opponentPartyId) { const bufferedCandidates = this.remoteCandidateBuffer.get(this.opponentPartyId); @@ -463,7 +595,6 @@ class PeerCall { } } - private setState(state: CallState): void { const oldState = this.state; this.state = state; @@ -475,17 +606,20 @@ class PeerCall { this.handler.emitUpdate(this, undefined); } - private waitForState(state: CallState): Promise { - let deferred = this.statePromiseMap.get(state); - if (!deferred) { - let resolve; - const promise = new Promise(r => { - resolve = r; - }); - deferred = {resolve, promise}; - this.statePromiseMap.set(state, deferred); - } - return deferred.promise; + private waitForState(states: CallState[]): Promise { + // TODO: rework this, do we need to clean up the promises? + return Promise.race(states.map(state => { + let deferred = this.statePromiseMap.get(state); + if (!deferred) { + let resolve; + const promise = new Promise(r => { + resolve = r; + }); + deferred = {resolve, promise}; + this.statePromiseMap.set(state, deferred); + } + return deferred.promise; + })); } private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean): Promise { @@ -493,8 +627,10 @@ class PeerCall { } private stopAllMedia(): void { - for (const track of this.localMedia.tracks) { - track.stop(); + if (this.localMedia) { + for (const track of this.localMedia.tracks) { + track.stop(); + } } } @@ -514,21 +650,6 @@ class PeerCall { //import { randomString } from '../randomstring'; -import { - MCallReplacesEvent, - MCallAnswer, - MCallInviteNegotiate, - CallCapabilities, - SDPStreamMetadataPurpose, - SDPStreamMetadata, - SDPStreamMetadataKey, - MCallSDPStreamMetadataChanged, - MCallSelectAnswer, - MCAllAssertedIdentity, - MCallCandidates, - MCallBase, - MCallHangupReject, -} from './callEventTypes'; // null is used as a special value meaning that the we're in a legacy 1:1 call // without MSC2746 that doesn't provide an id which device sent the message. @@ -681,46 +802,18 @@ export class CallError extends Error { } } -type InviteContent = { - offer: RTCSessionDescriptionInit, - [SDPStreamMetadataKey]: SDPStreamMetadata, - version?: number, - lifetime?: number -} - -export type InviteMessage = { - type: EventType.Invite, - content: InviteContent -} - -type AnwserContent = { - answer: { - sdp: string, - // type is now deprecated as of Matrix VoIP v1, but - // required to still be sent for backwards compat - type: RTCSdpType, - }, - [SDPStreamMetadataKey]: SDPStreamMetadata, -} - -export type AnswerMessage = { - type: EventType.Answer, - content: AnwserContent -} - -type CandidatesContent = { - candidates: RTCIceCandidate[] -} - -export type CandidatesMessage = { - type: EventType.Candidates, - content: CandidatesContent -} - - -export type SignallingMessage = InviteMessage | AnswerMessage | CandidatesMessage; +export type SignallingMessage = + {type: EventType.Invite, content: MCallInvite} | + {type: EventType.Answer, content: MCallAnswer} | + {type: EventType.SDPStreamMetadataChanged | EventType.SDPStreamMetadataChangedPrefix, content: MCallSDPStreamMetadataChanged} | + {type: EventType.Candidates, content: MCallCandidates} | + {type: EventType.Hangup | EventType.Reject, content: MCallHangupReject}; export interface PeerCallHandler { emitUpdate(peerCall: PeerCall, params: any); - sendSignallingMessage(message: SignallingMessage); + sendSignallingMessage(message: SignallingMessage); +} + +export function tests() { + } diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index da2b1ad6..a8fbeafd 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -130,57 +130,34 @@ write view I think we need to synchronize the negotiation needed because we don't use a CallState to guard it... - - ## Thursday 3-3 notes we probably best keep the perfect negotiation flags, as they are needed for both starting the call AND renegotiation? if only for the former, it would make sense as it is a step in setting up the call, but if the call is ongoing, does it make sense to have a MakingOffer state? it actually looks like they are only needed for renegotiation! for call setup we compare the call_ids. What does that mean for these flags? -List state transitions +## Peer call state transitions FROM CALLER FROM CALLEE Fledgling Fledgling - V calling `call()` V handleInvite + V `call()` V `handleInvite()`: setRemoteDescription(event.offer), add buffered candidates WaitLocalMedia Ringing - V media promise resolves V answer() -CreateOffer WaitLocalMedia - V add tracks V media promise resolves - V wait for negotionneeded events CreateAnswer - V setLocalDescription() V - V send invite events -InviteSent - V receive anwser, setRemoteDescription() | - \__________________________________________________/ + V media promise resolves V `answer()` + V add local tracks WaitLocalMedia +CreateOffer V media promise resolves + V wait for negotionneeded events V add local tracks + V setLocalDescription() CreateAnswer + V send invite event V setLocalDescription(createAnswer()) +InviteSent | + V receive anwser, setRemoteDescription() | + \___________________________________________________/ V Connecting - V receive ice candidates and - iceConnectionState becomes 'connected' + V receive ice candidates and iceConnectionState becomes 'connected' Connected - V hangup for some reason + V `hangup()` or some terminate condition Ended -## From callee - -Fledgling -Ringing -WaitLocalMedia -CreateAnswer -Connecting -Connected -Ended - -Fledgling -WaitLocalMedia -CreateOffer -InviteSent -CreateAnswer -Connecting -Connected -Ringing -Ended - so if we don't want to bother with having two call objects, we can make the existing call hangup his old call_id? That way we keep the old peerConnection. diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index cd0fbb9f..aa1bc079 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -1,11 +1,14 @@ // allow non-camelcase as these are events type that go onto the wire /* eslint-disable camelcase */ -import { CallErrorCode } from "./Call"; - // TODO: Change to "sdp_stream_metadata" when MSC3077 is merged export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata"; +export interface SessionDescription { + sdp?: string; + type: RTCSdpType +} + export enum SDPStreamMetadataPurpose { Usermedia = "m.usermedia", Screenshare = "m.screenshare", @@ -32,40 +35,36 @@ export interface CallReplacesTarget { avatar_url: string; } -export interface MCallBase { +export type MCallBase = { call_id: string; version: string | number; - party_id?: string; - sender_session_id?: string; - dest_session_id?: string; } -export interface MCallAnswer extends MCallBase { - answer: RTCSessionDescription; +export type MGroupCallBase = MCallBase & { + conf_id: string; +} + +export type MCallAnswer = Base & { + answer: SessionDescription; capabilities?: CallCapabilities; [SDPStreamMetadataKey]: SDPStreamMetadata; } -export interface MCallSelectAnswer extends MCallBase { +export type MCallSelectAnswer = Base & { selected_party_id: string; } -export interface MCallInviteNegotiate extends MCallBase { - offer: RTCSessionDescription; - description: RTCSessionDescription; +export type MCallInvite = Base & { + offer: SessionDescription; lifetime: number; - capabilities?: CallCapabilities; - invitee?: string; - sender_session_id?: string; - dest_session_id?: string; [SDPStreamMetadataKey]: SDPStreamMetadata; } -export interface MCallSDPStreamMetadataChanged extends MCallBase { +export type MCallSDPStreamMetadataChanged = Base & { [SDPStreamMetadataKey]: SDPStreamMetadata; } -export interface MCallReplacesEvent extends MCallBase { +export type MCallReplacesEvent = Base & { replacement_id: string; target_user: CallReplacesTarget; create_call: string; @@ -73,7 +72,7 @@ export interface MCallReplacesEvent extends MCallBase { target_room: string; } -export interface MCAllAssertedIdentity extends MCallBase { +export type MCAllAssertedIdentity = Base & { asserted_identity: { id: string; display_name: string; @@ -81,11 +80,11 @@ export interface MCAllAssertedIdentity extends MCallBase { }; } -export interface MCallCandidates extends MCallBase { +export type MCallCandidates = Base & { candidates: RTCIceCandidate[]; } -export interface MCallHangupReject extends MCallBase { +export type MCallHangupReject = Base & { reason?: CallErrorCode; } diff --git a/src/matrix/calls/group/Call.ts b/src/matrix/calls/group/GroupCall.ts similarity index 83% rename from src/matrix/calls/group/Call.ts rename to src/matrix/calls/group/GroupCall.ts index 9abef197..e05f572d 100644 --- a/src/matrix/calls/group/Call.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -15,14 +15,17 @@ limitations under the License. */ import {ObservableMap} from "../../../observable/map/ObservableMap"; +import {Participant} from "./Participant"; +import {LocalMedia} from "../LocalMedia"; +import type {Track} from "../../../platform/types/MediaDevices"; -function participantId(senderUserId: string, senderDeviceId: string | null) { +function getParticipantId(senderUserId: string, senderDeviceId: string | null) { return JSON.stringify(senderUserId) + JSON.stringify(senderDeviceId); } -class Call { +export class GroupCall { private readonly participants: ObservableMap = new ObservableMap(); - private localMedia?: LocalMedia; + private localMedia?: Promise; constructor(private readonly ownUserId: string, private callEvent: StateEvent, private readonly room: Room, private readonly webRTC: WebRTC) { @@ -30,17 +33,17 @@ class Call { get id(): string { return this.callEvent.state_key; } - async participate(tracks: Track[]) { - this.localMedia = LocalMedia.fromTracks(tracks); + async participate(tracks: Promise) { + this.localMedia = tracks.then(tracks => LocalMedia.fromTracks(tracks)); for (const [,participant] of this.participants) { - participant.setLocalMedia(this.localMedia.clone()); + participant.setLocalMedia(this.localMedia.then(localMedia => localMedia.clone())); } // send m.call.member state event // send invite to all participants that are < my userId for (const [,participant] of this.participants) { if (participant.userId < this.ownUserId) { - participant.sendInvite(); + participant.call(); } } } @@ -78,10 +81,6 @@ class Call { } } - get id(): string { - return this.callEvent.state_key; - } - get isTerminated(): boolean { return !!this.callEvent.content[CALL_TERMINATED]; } diff --git a/src/matrix/calls/group/Participant.ts b/src/matrix/calls/group/Participant.ts index 26747e56..2b873aa0 100644 --- a/src/matrix/calls/group/Participant.ts +++ b/src/matrix/calls/group/Participant.ts @@ -14,35 +14,54 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {EventType} from "../PeerCall"; +import {EventType, PeerCall, SignallingMessage} from "../PeerCall"; +import {makeTxnId} from "../../common"; + import type {PeerCallHandler} from "../PeerCall"; +import type {LocalMedia} from "../LocalMedia"; +import type {HomeServerApi} from "../../net/HomeServerApi"; +import type {Track} from "../../../platform/types/MediaDevices"; +import type {MCallBase, MGroupCallBase} from "../callEventTypes"; +import type {GroupCall} from "./GroupCall"; +import type {RoomMember} from "../../room/members/RoomMember"; -class Participant implements PeerCallHandler { - private peerCall?: PeerCall; - +export class Participant implements PeerCallHandler { constructor( - private readonly userId: string, - private readonly deviceId: string, - private localMedia: LocalMedia | undefined, - private readonly webRTC: WebRTC, - private readonly hsApi: HomeServerApi + public readonly member: RoomMember, + private readonly deviceId: string | undefined, + private readonly peerCall: PeerCall, + private readonly hsApi: HomeServerApi, + private readonly groupCall: GroupCall ) {} - sendInvite() { - this.peerCall = new PeerCall(this, this.webRTC); - this.peerCall.call(this.localMedia); + /* @internal */ + call(localMedia: Promise) { + this.peerCall.call(localMedia); + } + + get remoteTracks(): Track[] { + return this.peerCall.remoteTracks; } /** From PeerCallHandler * @internal */ emitUpdate(params: any) { - + this.groupCall.emitParticipantUpdate(this, params); } /** From PeerCallHandler * @internal */ - onSendSignallingMessage(type: EventType, content: Record) { + async sendSignallingMessage(message: SignallingMessage) { + const groupMessage = message as SignallingMessage; + groupMessage.content.conf_id = this.groupCall.id; // TODO: this needs to be encrypted with olm first - this.hsApi.sendToDevice(type, {[this.userId]: {[this.deviceId ?? "*"]: content}}); + + const request = this.hsApi.sendToDevice( + groupMessage.type, + {[this.member.userId]: { + [this.deviceId ?? "*"]: groupMessage.content + } + }, makeTxnId()); + await request.response(); } } diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index e1991a1c..445ff22d 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -144,7 +144,7 @@ export class AudioTrackWrapper extends TrackWrapper { } else { this.measuringVolumeActivity = false; this.speakingVolumeSamples.fill(-Infinity); - this.emit(CallFeedEvent.VolumeChanged, -Infinity); + // this.emit(CallFeedEvent.VolumeChanged, -Infinity); } } @@ -186,7 +186,7 @@ export class AudioTrackWrapper extends TrackWrapper { this.speakingVolumeSamples.shift(); this.speakingVolumeSamples.push(maxVolume); - this.emit(CallFeedEvent.VolumeChanged, maxVolume); + // this.emit(CallFeedEvent.VolumeChanged, maxVolume); let newSpeaking = false; @@ -201,7 +201,7 @@ export class AudioTrackWrapper extends TrackWrapper { if (this.speaking !== newSpeaking) { this.speaking = newSpeaking; - this.emit(CallFeedEvent.Speaking, this.speaking); + // this.emit(CallFeedEvent.Speaking, this.speaking); } this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL) as unknown as number; diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 08c0d96d..0025dfec 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -114,7 +114,7 @@ class DOMPeerConnection implements PeerConnection { } createDataChannel(): DataChannel { - return new DataChannel(this.peerConnection.createDataChannel()); + return undefined as any;// new DataChannel(this.peerConnection.createDataChannel()); } private registerHandler() { diff --git a/yarn.lock b/yarn.lock index 7bcefdd4..0eb74b87 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1486,9 +1486,9 @@ type-fest@^0.20.2: integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ== typescript@^4.3.5: - version "4.3.5" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.3.5.tgz#4d1c37cc16e893973c45a06886b7113234f119f4" - integrity sha512-DqQgihaQ9cUrskJo9kIyW/+g0Vxsk8cDtZ52a3NGh0YNTfpUSArXSohyUGnvbPazEPLu398C0UxmKSOrPumUzA== + version "4.6.2" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.6.2.tgz#fe12d2727b708f4eef40f51598b3398baa9611d4" + integrity sha512-HM/hFigTBHZhLXshn9sN37H085+hQGeJHJ/X7LpBWLID/fbc2acUMfU+lGD98X81sKP+pFa9f0DZmCwB9GnbAg== typeson-registry@^1.0.0-alpha.20: version "1.0.0-alpha.39" From 4bedd4737b59e4455642e83c21f553c828933cc5 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 9 Mar 2022 18:53:51 +0100 Subject: [PATCH 012/435] WIP11 --- src/matrix/calls/CallHandler.ts | 105 ++++++++++++++++---------- src/matrix/calls/PeerCall.ts | 33 +++------ src/matrix/calls/callEventTypes.ts | 111 +++++++++++++++++++++++++++- src/matrix/calls/group/GroupCall.ts | 50 ++++++------- src/matrix/room/Room.js | 3 + 5 files changed, 213 insertions(+), 89 deletions(-) diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index d7d673ed..d84be9e3 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -17,33 +17,34 @@ limitations under the License. import {ObservableMap} from "../../observable/map/ObservableMap"; import type {Room} from "../room/Room"; +import type {MemberChange} from "../room/members/RoomMember"; import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; +import type {Platform} from "../../platform/web/Platform"; import {WebRTC, PeerConnection, PeerConnectionHandler, StreamPurpose} from "../../platform/types/WebRTC"; import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; -import type {SignallingMessage} from "./PeerCall"; -import type {MGroupCallBase} from "./callEventTypes"; +import {handlesEventType, PeerCall, PeerCallHandler} from "./PeerCall"; +import {EventType} from "./callEventTypes"; +import type {SignallingMessage, MGroupCallBase} from "./callEventTypes"; +import type {GroupCall} from "./group/GroupCall"; const GROUP_CALL_TYPE = "m.call"; const GROUP_CALL_MEMBER_TYPE = "m.call.member"; - -enum CallSetupMessageType { - Invite = "m.call.invite", - Answer = "m.call.answer", - Candidates = "m.call.candidates", - Hangup = "m.call.hangup", -} - -const CONF_ID = "conf_id"; const CALL_TERMINATED = "m.terminated"; export class GroupCallHandler { + + private createPeerCall: (callId: string, handler: PeerCallHandler) => PeerCall; // group calls by call id public readonly calls: ObservableMap = new ObservableMap(); + // map of userId to set of conf_id's they are in + private memberToCallIds: Map> = new Map(); - constructor() { - + constructor(hsApi: HomeServerApi, platform: Platform, ownUserId: string, ownDeviceId: string) { + this.createPeerCall = (callId: string, handler: PeerCallHandler) => { + return new PeerCall(callId, handler, platform.createTimeout, platform.webRTC); + } } // TODO: check and poll turn server credentials here @@ -51,43 +52,69 @@ export class GroupCallHandler { handleRoomState(room: Room, events: StateEvent[], log: ILogItem) { // first update call events for (const event of events) { - if (event.type === GROUP_CALL_TYPE) { - const callId = event.state_key; - let call = this.calls.get(callId); - if (call) { - call.updateCallEvent(event); - if (call.isTerminated) { - this.calls.remove(call.id); - } - } else { - call = new GroupCall(event, room); - this.calls.set(call.id, call); - } + if (event.type === EventType.GroupCall) { + this.handleCallEvent(event); } } // then update participants for (const event of events) { - if (event.type === GROUP_CALL_MEMBER_TYPE) { - const participant = event.state_key; - const sources = event.content["m.sources"]; - for (const source of sources) { - const call = this.calls.get(source[CONF_ID]); - if (call && !call.isTerminated) { - call.addParticipant(participant, source); - } - } + if (event.type === EventType.GroupCallMember) { + this.handleCallMemberEvent(event); } } } - handlesDeviceMessageEventType(eventType: string | undefined): boolean { - return eventType === CallSetupMessageType.Invite || - eventType === CallSetupMessageType.Candidates || - eventType === CallSetupMessageType.Answer || - eventType === CallSetupMessageType.Hangup; + updateRoomMembers(room: Room, memberChanges: Map) { + + } + + private handleCallEvent(event: StateEvent) { + const callId = event.state_key; + let call = this.calls.get(callId); + if (call) { + call.updateCallEvent(event); + if (call.isTerminated) { + this.calls.remove(call.id); + } + } else { + call = new GroupCall(event, room, this.createPeerCall); + this.calls.set(call.id, call); + } + } + + private handleCallMemberEvent(event: StateEvent) { + const participant = event.state_key; + const calls = event.content["m.calls"] ?? []; + const newCallIdsMemberOf = new Set(calls.map(call => { + const callId = call["m.call_id"]; + const groupCall = this.calls.get(callId); + // TODO: also check the participant when receiving the m.call event + groupCall?.addParticipant(participant, call); + return callId; + })); + let previousCallIdsMemberOf = this.memberToCallIds.get(participant); + // remove user as participant of any calls not present anymore + if (previousCallIdsMemberOf) { + for (const previousCallId of previousCallIdsMemberOf) { + if (!newCallIdsMemberOf.has(previousCallId)) { + const groupCall = this.calls.get(previousCallId); + groupCall?.removeParticipant(participant); + } + } + } + if (newCallIdsMemberOf.size === 0) { + this.memberToCallIds.delete(participant); + } else { + this.memberToCallIds.set(participant, newCallIdsMemberOf); + } + } + + handlesDeviceMessageEventType(eventType: string): boolean { + return handlesEventType(eventType); } handleDeviceMessage(senderUserId: string, senderDeviceId: string, event: SignallingMessage, log: ILogItem) { + // TODO: buffer messages for calls we haven't received the state event for yet? const call = this.calls.get(event.content.conf_id); call?.handleDeviceMessage(senderUserId, senderDeviceId, event, log); } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 5040c805..f06b291d 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -29,7 +29,8 @@ import type {LocalMedia} from "./LocalMedia"; import { SDPStreamMetadataKey, - SDPStreamMetadataPurpose + SDPStreamMetadataPurpose, + EventType, } from "./callEventTypes"; import type { MCallBase, @@ -39,6 +40,7 @@ import type { MCallCandidates, MCallHangupReject, SDPStreamMetadata, + SignallingMessage } from "./callEventTypes"; // when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already @@ -677,21 +679,6 @@ export enum CallDirection { Outbound = 'outbound', } -export enum EventType { - Invite = "m.call.invite", - Candidates = "m.call.candidates", - Answer = "m.call.answer", - Hangup = "m.call.hangup", - Reject = "m.call.reject", - SelectAnswer = "m.call.select_answer", - Negotiate = "m.call.negotiate", - SDPStreamMetadataChanged = "m.call.sdp_stream_metadata_changed", - SDPStreamMetadataChangedPrefix = "org.matrix.call.sdp_stream_metadata_changed", - Replaces = "m.call.replaces", - AssertedIdentity = "m.call.asserted_identity", - AssertedIdentityPrefix = "org.matrix.call.asserted_identity", -} - export enum CallErrorCode { /** The user chose to end the call */ UserHangup = 'user_hangup', @@ -802,18 +789,18 @@ export class CallError extends Error { } } -export type SignallingMessage = - {type: EventType.Invite, content: MCallInvite} | - {type: EventType.Answer, content: MCallAnswer} | - {type: EventType.SDPStreamMetadataChanged | EventType.SDPStreamMetadataChangedPrefix, content: MCallSDPStreamMetadataChanged} | - {type: EventType.Candidates, content: MCallCandidates} | - {type: EventType.Hangup | EventType.Reject, content: MCallHangupReject}; - export interface PeerCallHandler { emitUpdate(peerCall: PeerCall, params: any); sendSignallingMessage(message: SignallingMessage); } +export function handlesEventType(eventType: string): boolean { + return eventType === EventType.Invite || + eventType === EventType.Candidates || + eventType === EventType.Answer || + eventType === EventType.Hangup; +} + export function tests() { } diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index aa1bc079..0e9eb8f8 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -1,6 +1,24 @@ // allow non-camelcase as these are events type that go onto the wire /* eslint-disable camelcase */ + +export enum EventType { + GroupCall = "m.call", + GroupCallMember = "m.call.member", + Invite = "m.call.invite", + Candidates = "m.call.candidates", + Answer = "m.call.answer", + Hangup = "m.call.hangup", + Reject = "m.call.reject", + SelectAnswer = "m.call.select_answer", + Negotiate = "m.call.negotiate", + SDPStreamMetadataChanged = "m.call.sdp_stream_metadata_changed", + SDPStreamMetadataChangedPrefix = "org.matrix.call.sdp_stream_metadata_changed", + Replaces = "m.call.replaces", + AssertedIdentity = "m.call.asserted_identity", + AssertedIdentityPrefix = "org.matrix.call.asserted_identity", +} + // TODO: Change to "sdp_stream_metadata" when MSC3077 is merged export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata"; @@ -88,4 +106,95 @@ export type MCallHangupReject = Base & { reason?: CallErrorCode; } -/* eslint-enable camelcase */ +export enum CallErrorCode { + /** The user chose to end the call */ + UserHangup = 'user_hangup', + + /** An error code when the local client failed to create an offer. */ + LocalOfferFailed = 'local_offer_failed', + /** + * An error code when there is no local mic/camera to use. This may be because + * the hardware isn't plugged in, or the user has explicitly denied access. + */ + NoUserMedia = 'no_user_media', + + /** + * Error code used when a call event failed to send + * because unknown devices were present in the room + */ + UnknownDevices = 'unknown_devices', + + /** + * Error code used when we fail to send the invite + * for some reason other than there being unknown devices + */ + SendInvite = 'send_invite', + + /** + * An answer could not be created + */ + CreateAnswer = 'create_answer', + + /** + * Error code used when we fail to send the answer + * for some reason other than there being unknown devices + */ + SendAnswer = 'send_answer', + + /** + * The session description from the other side could not be set + */ + SetRemoteDescription = 'set_remote_description', + + /** + * The session description from this side could not be set + */ + SetLocalDescription = 'set_local_description', + + /** + * A different device answered the call + */ + AnsweredElsewhere = 'answered_elsewhere', + + /** + * No media connection could be established to the other party + */ + IceFailed = 'ice_failed', + + /** + * The invite timed out whilst waiting for an answer + */ + InviteTimeout = 'invite_timeout', + + /** + * The call was replaced by another call + */ + Replaced = 'replaced', + + /** + * Signalling for the call could not be sent (other than the initial invite) + */ + SignallingFailed = 'signalling_timeout', + + /** + * The remote party is busy + */ + UserBusy = 'user_busy', + + /** + * We transferred the call off to somewhere else + */ + Transfered = 'transferred', + + /** + * A call from the same user was found with a new session id + */ + NewSession = 'new_session', +} + +export type SignallingMessage = + {type: EventType.Invite, content: MCallInvite} | + {type: EventType.Answer, content: MCallAnswer} | + {type: EventType.SDPStreamMetadataChanged | EventType.SDPStreamMetadataChangedPrefix, content: MCallSDPStreamMetadataChanged} | + {type: EventType.Candidates, content: MCallCandidates} | + {type: EventType.Hangup | EventType.Reject, content: MCallHangupReject}; diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index e05f572d..dce50846 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -18,16 +18,21 @@ import {ObservableMap} from "../../../observable/map/ObservableMap"; import {Participant} from "./Participant"; import {LocalMedia} from "../LocalMedia"; import type {Track} from "../../../platform/types/MediaDevices"; - -function getParticipantId(senderUserId: string, senderDeviceId: string | null) { - return JSON.stringify(senderUserId) + JSON.stringify(senderDeviceId); -} +import type {SignallingMessage, MGroupCallBase} from "../callEventTypes"; +import type {Room} from "../../room/Room"; +import type {StateEvent} from "../../storage/types"; +import type {Platform} from "../../../platform/web/Platform"; export class GroupCall { private readonly participants: ObservableMap = new ObservableMap(); private localMedia?: Promise; - constructor(private readonly ownUserId: string, private callEvent: StateEvent, private readonly room: Room, private readonly webRTC: WebRTC) { + constructor( + private readonly ownUserId: string, + private callEvent: StateEvent, + private readonly room: Room, + private readonly platform: Platform + ) { } @@ -52,32 +57,25 @@ export class GroupCall { this.callEvent = callEvent; } - addParticipant(userId, source) { - const participantId = getParticipantId(userId, source.device_id); - const participant = this.participants.get(participantId); + addParticipant(userId, memberCallInfo) { + let participant = this.participants.get(userId); if (participant) { - participant.updateSource(source); + participant.updateCallInfo(memberCallInfo); } else { - participant.add(participantId, new Participant(userId, source.device_id, this.localMedia?.clone(), this.webRTC)); + participant = new Participant(userId, source.device_id, this.localMedia?.clone(), this.webRTC); + participant.updateCallInfo(memberCallInfo); + this.participants.add(userId, participant); } } - handleDeviceMessage(senderUserId: string, senderDeviceId: string, eventType: string, content: Record, log: ILogItem) { - const participantId = getParticipantId(senderUserId, senderDeviceId); - let peerCall = this.participants.get(participantId); - let hasDeviceInKey = true; - if (!peerCall) { - hasDeviceInKey = false; - peerCall = this.participants.get(getParticipantId(senderUserId, null)) - } - if (peerCall) { - peerCall.handleIncomingSignallingMessage(eventType, content, senderDeviceId); - if (!hasDeviceInKey && peerCall.opponentPartyId) { - this.participants.delete(getParticipantId(senderUserId, null)); - this.participants.add(getParticipantId(senderUserId, peerCall.opponentPartyId)); - } - } else { - // create peerCall + removeParticipant(userId) { + + } + + handleDeviceMessage(userId: string, senderDeviceId: string, message: SignallingMessage, log: ILogItem) { + let participant = this.participants.get(userId); + if (participant) { + participant.handleIncomingSignallingMessage(message, senderDeviceId); } } diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js index b9ec82a3..d0fcbc84 100644 --- a/src/matrix/room/Room.js +++ b/src/matrix/room/Room.js @@ -218,6 +218,9 @@ export class Room extends BaseRoom { if (this._memberList) { this._memberList.afterSync(memberChanges); } + if (this._callHandler) { + this._callHandler.updateRoomMembers(this, memberChanges); + } if (this._observedMembers) { this._updateObservedMembers(memberChanges); } From 6da4a4209c95598da1daf451a82f6aa0649d8f62 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 10 Mar 2022 14:53:31 +0100 Subject: [PATCH 013/435] WIP: work on group calling code --- src/matrix/DeviceMessageHandler.js | 2 +- src/matrix/Session.js | 14 ++- src/matrix/calls/CallHandler.ts | 126 ++++++++++++++------------ src/matrix/calls/PeerCall.ts | 79 ++++++++-------- src/matrix/calls/group/GroupCall.ts | 90 +++++++++++------- src/matrix/calls/group/Member.ts | 112 +++++++++++++++++++++++ src/matrix/calls/group/Participant.ts | 67 -------------- src/matrix/common.js | 6 +- src/matrix/e2ee/olm/Encryption.ts | 2 +- src/matrix/net/HomeServerApi.ts | 4 + src/platform/types/MediaDevices.ts | 1 + 11 files changed, 301 insertions(+), 202 deletions(-) create mode 100644 src/matrix/calls/group/Member.ts delete mode 100644 src/matrix/calls/group/Participant.ts diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index 470559a9..91ef82f6 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -59,7 +59,7 @@ export class DeviceMessageHandler { })); // TODO: pass this in the prep and run it in afterSync or afterSyncComplete (as callHandler can send events as well)? for (const dr of callMessages) { - this._callHandler.handleDeviceMessage(dr.device.userId, dr.device.deviceId, dr.event, log); + this._callHandler.handleDeviceMessage(dr.event, dr.device.userId, dr.device.deviceId, log); } // TODO: somehow include rooms that received a call to_device message in the sync state? // or have updates flow through event emitter? diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 3d9b13c8..94fb5dee 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -73,6 +73,19 @@ export class Session { }; this._roomsBeingCreated = new ObservableMap(); this._user = new User(sessionInfo.userId); + this._callHandler = new CallHandler({ + createTimeout: this._platform.clock.createTimeout, + hsApi: this._hsApi, + encryptDeviceMessage: async (roomId, message, log) => { + if (!this._deviceTracker || !this._olmEncryption) { + throw new Error("encryption is not enabled"); + } + await this._deviceTracker.trackRoom(roomId, log); + const devices = await this._deviceTracker.devicesForTrackedRoom(roomId, this._hsApi, log); + const encryptedMessage = await this._olmEncryption.encrypt(message.type, message.content, devices, this._hsApi, log); + return encryptedMessage; + } + }); this._deviceMessageHandler = new DeviceMessageHandler({storage, callHandler: this._callHandler}); this._olm = olm; this._olmUtil = null; @@ -100,7 +113,6 @@ export class Session { this._createRoomEncryption = this._createRoomEncryption.bind(this); this._forgetArchivedRoom = this._forgetArchivedRoom.bind(this); this.needsKeyBackup = new ObservableValue(false); - this._callHandler = new CallHandler(this._platform, this._hsApi); } get fingerprintKey() { diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index d84be9e3..0e70fb5a 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -15,48 +15,53 @@ limitations under the License. */ import {ObservableMap} from "../../observable/map/ObservableMap"; +import {WebRTC, PeerConnection, PeerConnectionHandler} from "../../platform/types/WebRTC"; +import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; +import {handlesEventType} from "./PeerCall"; +import {EventType} from "./callEventTypes"; +import {GroupCall} from "./group/GroupCall"; import type {Room} from "../room/Room"; import type {MemberChange} from "../room/members/RoomMember"; import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; import type {Platform} from "../../platform/web/Platform"; - -import {WebRTC, PeerConnection, PeerConnectionHandler, StreamPurpose} from "../../platform/types/WebRTC"; -import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; -import {handlesEventType, PeerCall, PeerCallHandler} from "./PeerCall"; -import {EventType} from "./callEventTypes"; +import type {BaseObservableMap} from "../../observable/map/BaseObservableMap"; import type {SignallingMessage, MGroupCallBase} from "./callEventTypes"; -import type {GroupCall} from "./group/GroupCall"; +import type {Options as GroupCallOptions} from "./group/GroupCall"; const GROUP_CALL_TYPE = "m.call"; const GROUP_CALL_MEMBER_TYPE = "m.call.member"; const CALL_TERMINATED = "m.terminated"; -export class GroupCallHandler { +export type Options = Omit; - private createPeerCall: (callId: string, handler: PeerCallHandler) => PeerCall; +export class GroupCallHandler { // group calls by call id - public readonly calls: ObservableMap = new ObservableMap(); + private readonly _calls: ObservableMap = new ObservableMap(); // map of userId to set of conf_id's they are in private memberToCallIds: Map> = new Map(); + private groupCallOptions: GroupCallOptions; - constructor(hsApi: HomeServerApi, platform: Platform, ownUserId: string, ownDeviceId: string) { - this.createPeerCall = (callId: string, handler: PeerCallHandler) => { - return new PeerCall(callId, handler, platform.createTimeout, platform.webRTC); - } + constructor(private readonly options: Options) { + this.groupCallOptions = Object.assign({}, this.options, { + emitUpdate: (groupCall, params) => this._calls.update(groupCall.id, params) + }); } + get calls(): BaseObservableMap { return this._calls; } + // TODO: check and poll turn server credentials here + /** @internal */ handleRoomState(room: Room, events: StateEvent[], log: ILogItem) { // first update call events for (const event of events) { if (event.type === EventType.GroupCall) { - this.handleCallEvent(event); + this.handleCallEvent(event, room); } } - // then update participants + // then update members for (const event of events) { if (event.type === EventType.GroupCallMember) { this.handleCallMemberEvent(event); @@ -64,59 +69,62 @@ export class GroupCallHandler { } } + /** @internal */ updateRoomMembers(room: Room, memberChanges: Map) { } - private handleCallEvent(event: StateEvent) { - const callId = event.state_key; - let call = this.calls.get(callId); - if (call) { - call.updateCallEvent(event); - if (call.isTerminated) { - this.calls.remove(call.id); - } - } else { - call = new GroupCall(event, room, this.createPeerCall); - this.calls.set(call.id, call); - } - } - - private handleCallMemberEvent(event: StateEvent) { - const participant = event.state_key; - const calls = event.content["m.calls"] ?? []; - const newCallIdsMemberOf = new Set(calls.map(call => { - const callId = call["m.call_id"]; - const groupCall = this.calls.get(callId); - // TODO: also check the participant when receiving the m.call event - groupCall?.addParticipant(participant, call); - return callId; - })); - let previousCallIdsMemberOf = this.memberToCallIds.get(participant); - // remove user as participant of any calls not present anymore - if (previousCallIdsMemberOf) { - for (const previousCallId of previousCallIdsMemberOf) { - if (!newCallIdsMemberOf.has(previousCallId)) { - const groupCall = this.calls.get(previousCallId); - groupCall?.removeParticipant(participant); - } - } - } - if (newCallIdsMemberOf.size === 0) { - this.memberToCallIds.delete(participant); - } else { - this.memberToCallIds.set(participant, newCallIdsMemberOf); - } - } - + /** @internal */ handlesDeviceMessageEventType(eventType: string): boolean { return handlesEventType(eventType); } - handleDeviceMessage(senderUserId: string, senderDeviceId: string, event: SignallingMessage, log: ILogItem) { + /** @internal */ + handleDeviceMessage(message: SignallingMessage, userId: string, deviceId: string, log: ILogItem) { // TODO: buffer messages for calls we haven't received the state event for yet? - const call = this.calls.get(event.content.conf_id); - call?.handleDeviceMessage(senderUserId, senderDeviceId, event, log); + const call = this._calls.get(message.content.conf_id); + call?.handleDeviceMessage(message, userId, deviceId, log); + } + + private handleCallEvent(event: StateEvent, room: Room) { + const callId = event.state_key; + let call = this._calls.get(callId); + if (call) { + call.updateCallEvent(event); + if (call.isTerminated) { + this._calls.remove(call.id); + } + } else { + call = new GroupCall(event, room, this.groupCallOptions); + this._calls.set(call.id, call); + } + } + + private handleCallMemberEvent(event: StateEvent) { + const userId = event.state_key; + const calls = event.content["m.calls"] ?? []; + const newCallIdsMemberOf = new Set(calls.map(call => { + const callId = call["m.call_id"]; + const groupCall = this._calls.get(callId); + // TODO: also check the member when receiving the m.call event + groupCall?.addMember(userId, call); + return callId; + })); + let previousCallIdsMemberOf = this.memberToCallIds.get(userId); + // remove user as member of any calls not present anymore + if (previousCallIdsMemberOf) { + for (const previousCallId of previousCallIdsMemberOf) { + if (!newCallIdsMemberOf.has(previousCallId)) { + const groupCall = this._calls.get(previousCallId); + groupCall?.removeMember(userId); + } + } + } + if (newCallIdsMemberOf.size === 0) { + this.memberToCallIds.delete(userId); + } else { + this.memberToCallIds.set(userId, newCallIdsMemberOf); + } } } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index f06b291d..ed8351ac 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -43,6 +43,13 @@ import type { SignallingMessage } from "./callEventTypes"; +export type Options = { + webRTC: WebRTC, + createTimeout: TimeoutCreator, + emitUpdate: (peerCall: PeerCall, params: any) => void; + sendSignallingMessage: (message: SignallingMessage, log: ILogItem) => Promise; +}; + // when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already // do for sharing keys will be best as that already deals with room tracking. /** @@ -51,7 +58,7 @@ import type { /** Implements a call between two peers with the signalling state keeping, while still delegating the signalling message sending. Used by GroupCall.*/ export class PeerCall implements IDisposable { private readonly peerConnection: PeerConnection; - private state = CallState.Fledgling; + private _state = CallState.Fledgling; private direction: CallDirection; private localMedia?: LocalMedia; // A queue for candidates waiting to go out. @@ -74,15 +81,12 @@ export class PeerCall implements IDisposable { // perfect negotiation flags private makingOffer: boolean = false; private ignoreOffer: boolean = false; - constructor( private callId: string, // generated or from invite - private readonly handler: PeerCallHandler, - private readonly createTimeout: TimeoutCreator, - webRTC: WebRTC + private readonly options: Options ) { const outer = this; - this.peerConnection = webRTC.createPeerConnection({ + this.peerConnection = options.webRTC.createPeerConnection({ onIceConnectionStateChange(state: RTCIceConnectionState) {}, onLocalIceCandidate(candidate: RTCIceCandidate) {}, onIceGatheringStateChange(state: RTCIceGatheringState) {}, @@ -104,12 +108,14 @@ export class PeerCall implements IDisposable { } } + get state(): CallState { return this._state; } + get remoteTracks(): Track[] { return this.peerConnection.remoteTracks; } async call(localMediaPromise: Promise): Promise { - if (this.state !== CallState.Fledgling) { + if (this._state !== CallState.Fledgling) { return; } this.direction = CallDirection.Outbound; @@ -131,7 +137,7 @@ export class PeerCall implements IDisposable { } async answer(localMediaPromise: Promise): Promise { - if (this.state !== CallState.Ringing) { + if (this._state !== CallState.Ringing) { return; } this.setState(CallState.WaitLocalMedia); @@ -197,7 +203,7 @@ export class PeerCall implements IDisposable { async hangup(errorCode: CallErrorCode) { } - async handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId): Promise { + async handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId, log: ILogItem): Promise { switch (message.type) { case EventType.Invite: if (this.callId !== message.content.call_id) { @@ -226,10 +232,10 @@ export class PeerCall implements IDisposable { if (reason) { content["reason"] = reason; } - return this.handler.sendSignallingMessage({ + return this.options.sendSignallingMessage({ type: EventType.Hangup, content - }); + }, undefined); } // calls are serialized and deduplicated by responsePromiseChain @@ -249,7 +255,7 @@ export class PeerCall implements IDisposable { await this.delay(200); } - if (this.state === CallState.Ended) { + if (this._state === CallState.Ended) { return; } @@ -268,12 +274,12 @@ export class PeerCall implements IDisposable { version: 1, lifetime: CALL_TIMEOUT_MS }; - if (this.state === CallState.CreateOffer) { - await this.handler.sendSignallingMessage({type: EventType.Invite, content}); + if (this._state === CallState.CreateOffer) { + await this.options.sendSignallingMessage({type: EventType.Invite, content}); this.setState(CallState.InviteSent); - } else if (this.state === CallState.Connected || this.state === CallState.Connecting) { + } else if (this._state === CallState.Connected || this._state === CallState.Connecting) { // send Negotiate message - //await this.handler.sendSignallingMessage({type: EventType.Invite, content}); + //await this.options.sendSignallingMessage({type: EventType.Invite, content}); //this.setState(CallState.InviteSent); } } finally { @@ -282,10 +288,10 @@ export class PeerCall implements IDisposable { this.sendCandidateQueue(); - if (this.state === CallState.InviteSent) { + if (this._state === CallState.InviteSent) { await this.delay(CALL_TIMEOUT_MS); // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between - if (this.state === CallState.InviteSent) { + if (this._state === CallState.InviteSent) { this.hangup(CallErrorCode.InviteTimeout); } } @@ -307,7 +313,7 @@ export class PeerCall implements IDisposable { // TODO: review states to be unambigous, WaitLocalMedia for sending offer or answer? // How do we interrupt `call()`? well, perhaps we need to not just await InviteSent but also CreateAnswer? - if (this.state === CallState.Fledgling || this.state === CallState.CreateOffer || this.state === CallState.WaitLocalMedia) { + if (this._state === CallState.Fledgling || this._state === CallState.CreateOffer || this._state === CallState.WaitLocalMedia) { } else { await this.sendHangupWithCallId(this.callId, CallErrorCode.Replaced); @@ -324,7 +330,7 @@ export class PeerCall implements IDisposable { } private async handleFirstInvite(content: MCallInvite, partyId: PartyId): Promise { - if (this.state !== CallState.Fledgling || this.opponentPartyId !== undefined) { + if (this._state !== CallState.Fledgling || this.opponentPartyId !== undefined) { // TODO: hangup or ignore? return; } @@ -370,7 +376,7 @@ export class PeerCall implements IDisposable { await this.delay(content.lifetime ?? CALL_TIMEOUT_MS); // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between - if (this.state === CallState.Ringing) { + if (this._state === CallState.Ringing) { this.logger.debug(`Call ${this.callId} invite has expired. Hanging up.`); this.hangupParty = CallParty.Remote; // effectively this.setState(CallState.Ended); @@ -384,7 +390,7 @@ export class PeerCall implements IDisposable { private async handleAnswer(content: MCallAnswer, partyId: PartyId): Promise { this.logger.debug(`Got answer for call ID ${this.callId} from party ID ${partyId}`); - if (this.state === CallState.Ended) { + if (this._state === CallState.Ended) { this.logger.debug(`Ignoring answer because call ID ${this.callId} has ended`); return; } @@ -456,7 +462,7 @@ export class PeerCall implements IDisposable { // if (description.type === 'offer') { // await this.peerConnection.setLocalDescription(); - // await this.handler.sendSignallingMessage({ + // await this.options.sendSignallingMessage({ // type: EventType.CallNegotiate, // content: { // description: this.peerConnection.localDescription!, @@ -471,7 +477,7 @@ export class PeerCall implements IDisposable { private async sendAnswer(): Promise { const localDescription = this.peerConnection.localDescription!; - const answerContent: MCallAnswer = { + const answerContent: MCallAnswer = { call_id: this.callId, version: 1, answer: { @@ -489,7 +495,7 @@ export class PeerCall implements IDisposable { this.candidateSendQueue = []; try { - await this.handler.sendSignallingMessage({type: EventType.Answer, content: answerContent}); + await this.options.sendSignallingMessage({type: EventType.Answer, content: answerContent}, undefined); } catch (error) { this.terminate(CallParty.Local, CallErrorCode.SendAnswer, false); throw error; @@ -513,7 +519,7 @@ export class PeerCall implements IDisposable { this.candidateSendQueue.push(content); // Don't send the ICE candidates yet if the call is in the ringing state - if (this.state === CallState.Ringing) return; + if (this._state === CallState.Ringing) return; // MSC2746 recommends these values (can be quite long when calling because the // callee will need a while to answer the call) @@ -523,7 +529,7 @@ export class PeerCall implements IDisposable { } private async sendCandidateQueue(): Promise { - if (this.candidateSendQueue.length === 0 || this.state === CallState.Ended) { + if (this.candidateSendQueue.length === 0 || this._state === CallState.Ended) { return; } @@ -531,14 +537,14 @@ export class PeerCall implements IDisposable { this.candidateSendQueue = []; this.logger.debug(`Call ${this.callId} attempting to send ${candidates.length} candidates`); try { - await this.handler.sendSignallingMessage({ + await this.options.sendSignallingMessage({ type: EventType.Candidates, content: { call_id: this.callId, version: 1, candidates - } - }); + }, + }, undefined); // Try to send candidates again just in case we received more candidates while sending. this.sendCandidateQueue(); } catch (error) { @@ -598,14 +604,14 @@ export class PeerCall implements IDisposable { } private setState(state: CallState): void { - const oldState = this.state; - this.state = state; + const oldState = this._state; + this._state = state; let deferred = this.statePromiseMap.get(state); if (deferred) { deferred.resolve(); this.statePromiseMap.delete(state); } - this.handler.emitUpdate(this, undefined); + this.options.emitUpdate(this, undefined); } private waitForState(states: CallState[]): Promise { @@ -638,7 +644,7 @@ export class PeerCall implements IDisposable { private async delay(timeoutMs: number): Promise { // Allow a short time for initial candidates to be gathered - const timeout = this.disposables.track(this.createTimeout(timeoutMs)); + const timeout = this.disposables.track(this.options.createTimeout(timeoutMs)); await timeout.elapsed(); this.disposables.untrack(timeout); } @@ -789,11 +795,6 @@ export class CallError extends Error { } } -export interface PeerCallHandler { - emitUpdate(peerCall: PeerCall, params: any); - sendSignallingMessage(message: SignallingMessage); -} - export function handlesEventType(eventType: string): boolean { return eventType === EventType.Invite || eventType === EventType.Candidates || diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index dce50846..7266645d 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -15,71 +15,95 @@ limitations under the License. */ import {ObservableMap} from "../../../observable/map/ObservableMap"; -import {Participant} from "./Participant"; +import {Member} from "./Member"; import {LocalMedia} from "../LocalMedia"; +import {RoomMember} from "../../room/members/RoomMember"; +import type {Options as MemberOptions} from "./Member"; +import type {BaseObservableMap} from "../../../observable/map/BaseObservableMap"; import type {Track} from "../../../platform/types/MediaDevices"; import type {SignallingMessage, MGroupCallBase} from "../callEventTypes"; import type {Room} from "../../room/Room"; import type {StateEvent} from "../../storage/types"; import type {Platform} from "../../../platform/web/Platform"; +import type {EncryptedMessage} from "../../e2ee/olm/Encryption"; +import type {ILogItem} from "../../../logging/types"; + +export type Options = Omit & { + emitUpdate: (call: GroupCall, params?: any) => void; + encryptDeviceMessage: (roomId: string, message: SignallingMessage, log: ILogItem) => Promise, +}; export class GroupCall { - private readonly participants: ObservableMap = new ObservableMap(); + private readonly _members: ObservableMap = new ObservableMap(); private localMedia?: Promise; + private _memberOptions: MemberOptions; constructor( - private readonly ownUserId: string, private callEvent: StateEvent, private readonly room: Room, - private readonly platform: Platform + private readonly options: Options ) { - + this._memberOptions = Object.assign({ + confId: this.id, + emitUpdate: member => this._members.update(member.member.userId, member), + encryptDeviceMessage: (message: SignallingMessage, log) => { + return this.options.encryptDeviceMessage(this.room.id, message, log); + } + }, options); } + get members(): BaseObservableMap { return this._members; } + get id(): string { return this.callEvent.state_key; } - async participate(tracks: Promise) { - this.localMedia = tracks.then(tracks => LocalMedia.fromTracks(tracks)); - for (const [,participant] of this.participants) { - participant.setLocalMedia(this.localMedia.then(localMedia => localMedia.clone())); - } - // send m.call.member state event + get isTerminated(): boolean { + return this.callEvent.content["m.terminated"] === true; + } - // send invite to all participants that are < my userId - for (const [,participant] of this.participants) { - if (participant.userId < this.ownUserId) { - participant.call(); - } + async join(tracks: Promise) { + this.localMedia = tracks.then(tracks => LocalMedia.fromTracks(tracks)); + // send m.call.member state event + const request = this.options.hsApi.sendState(this.room.id, "m.call.member", this.options.ownUserId, { + + }); + await request.response(); + // send invite to all members that are < my userId + for (const [,member] of this._members) { + member.connect(this.localMedia); } } + /** @internal */ updateCallEvent(callEvent: StateEvent) { this.callEvent = callEvent; + // TODO: emit update } - addParticipant(userId, memberCallInfo) { - let participant = this.participants.get(userId); - if (participant) { - participant.updateCallInfo(memberCallInfo); + /** @internal */ + addMember(userId, memberCallInfo) { + let member = this._members.get(userId); + if (member) { + member.updateCallInfo(memberCallInfo); } else { - participant = new Participant(userId, source.device_id, this.localMedia?.clone(), this.webRTC); - participant.updateCallInfo(memberCallInfo); - this.participants.add(userId, participant); + member = new Member(RoomMember.fromUserId(this.room.id, userId, "join"), this._memberOptions); + member.updateCallInfo(memberCallInfo); + this._members.add(userId, member); } } - removeParticipant(userId) { - + /** @internal */ + removeMember(userId) { + this._members.remove(userId); } - handleDeviceMessage(userId: string, senderDeviceId: string, message: SignallingMessage, log: ILogItem) { - let participant = this.participants.get(userId); - if (participant) { - participant.handleIncomingSignallingMessage(message, senderDeviceId); + /** @internal */ + handleDeviceMessage(message: SignallingMessage, userId: string, deviceId: string, log: ILogItem) { + // TODO: return if we are not membering to the call + let member = this._members.get(userId); + if (member) { + member.handleDeviceMessage(message, deviceId, log); + } else { + // we haven't received the m.call.member yet for this caller. buffer the device messages or create the member/call anyway? } } - - get isTerminated(): boolean { - return !!this.callEvent.content[CALL_TERMINATED]; - } } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts new file mode 100644 index 00000000..c6791568 --- /dev/null +++ b/src/matrix/calls/group/Member.ts @@ -0,0 +1,112 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {PeerCall, CallState} from "../PeerCall"; +import {makeTxnId, makeId} from "../../common"; +import {EventType} from "../callEventTypes"; + +import type {Options as PeerCallOptions} from "../PeerCall"; +import type {LocalMedia} from "../LocalMedia"; +import type {HomeServerApi} from "../../net/HomeServerApi"; +import type {Track} from "../../../platform/types/MediaDevices"; +import type {MCallBase, MGroupCallBase, SignallingMessage} from "../callEventTypes"; +import type {GroupCall} from "./GroupCall"; +import type {RoomMember} from "../../room/members/RoomMember"; +import type {EncryptedMessage} from "../../e2ee/olm/Encryption"; +import type {ILogItem} from "../../../logging/types"; + +export type Options = Omit & { + confId: string, + ownUserId: string, + hsApi: HomeServerApi, + encryptDeviceMessage: (message: SignallingMessage, log: ILogItem) => Promise, + emitUpdate: (participant: Member, params?: any) => void, +} + +export class Member { + private peerCall?: PeerCall; + private localMedia?: Promise; + + constructor( + public readonly member: RoomMember, + private readonly options: Options + ) {} + + get remoteTracks(): Track[] { + return this.peerCall?.remoteTracks ?? []; + } + + get isConnected(): boolean { + return this.peerCall?.state === CallState.Connected; + } + + /* @internal */ + connect(localMedia: Promise) { + this.localMedia = localMedia; + // otherwise wait for it to connect + if (this.member.userId < this.options.ownUserId) { + this.peerCall = this._createPeerCall(makeId("c")); + this.peerCall.call(localMedia); + } + } + + /** @internal */ + updateCallInfo(memberCallInfo) { + + } + + /** @internal */ + emitUpdate = (peerCall: PeerCall, params: any) => { + if (peerCall.state === CallState.Ringing) { + peerCall.answer(this.localMedia!); + } + this.options.emitUpdate(this, params); + } + + /** From PeerCallHandler + * @internal */ + sendSignallingMessage = async (message: SignallingMessage, log: ILogItem) => { + const groupMessage = message as SignallingMessage; + groupMessage.content.conf_id = this.options.confId; + const encryptedMessage = await this.options.encryptDeviceMessage(groupMessage, log); + const request = this.options.hsApi.sendToDevice( + "m.room.encrypted", + {[this.member.userId]: { + ["*"]: encryptedMessage.content + } + }, makeTxnId(), {log}); + await request.response(); + } + + /** @internal */ + handleDeviceMessage(message: SignallingMessage, deviceId: string, log: ILogItem) { + if (message.type === EventType.Invite && !this.peerCall) { + this.peerCall = this._createPeerCall(message.content.call_id); + } + if (this.peerCall) { + this.peerCall.handleIncomingSignallingMessage(message, deviceId, log); + } else { + // TODO: need to buffer events until invite comes? + } + } + + private _createPeerCall(callId: string): PeerCall { + return new PeerCall(callId, Object.assign({}, this.options, { + emitUpdate: this.emitUpdate, + sendSignallingMessage: this.sendSignallingMessage + })); + } +} diff --git a/src/matrix/calls/group/Participant.ts b/src/matrix/calls/group/Participant.ts deleted file mode 100644 index 2b873aa0..00000000 --- a/src/matrix/calls/group/Participant.ts +++ /dev/null @@ -1,67 +0,0 @@ -/* -Copyright 2022 The Matrix.org Foundation C.I.C. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import {EventType, PeerCall, SignallingMessage} from "../PeerCall"; -import {makeTxnId} from "../../common"; - -import type {PeerCallHandler} from "../PeerCall"; -import type {LocalMedia} from "../LocalMedia"; -import type {HomeServerApi} from "../../net/HomeServerApi"; -import type {Track} from "../../../platform/types/MediaDevices"; -import type {MCallBase, MGroupCallBase} from "../callEventTypes"; -import type {GroupCall} from "./GroupCall"; -import type {RoomMember} from "../../room/members/RoomMember"; - -export class Participant implements PeerCallHandler { - constructor( - public readonly member: RoomMember, - private readonly deviceId: string | undefined, - private readonly peerCall: PeerCall, - private readonly hsApi: HomeServerApi, - private readonly groupCall: GroupCall - ) {} - - /* @internal */ - call(localMedia: Promise) { - this.peerCall.call(localMedia); - } - - get remoteTracks(): Track[] { - return this.peerCall.remoteTracks; - } - - /** From PeerCallHandler - * @internal */ - emitUpdate(params: any) { - this.groupCall.emitParticipantUpdate(this, params); - } - - /** From PeerCallHandler - * @internal */ - async sendSignallingMessage(message: SignallingMessage) { - const groupMessage = message as SignallingMessage; - groupMessage.content.conf_id = this.groupCall.id; - // TODO: this needs to be encrypted with olm first - - const request = this.hsApi.sendToDevice( - groupMessage.type, - {[this.member.userId]: { - [this.deviceId ?? "*"]: groupMessage.content - } - }, makeTxnId()); - await request.response(); - } -} diff --git a/src/matrix/common.js b/src/matrix/common.js index ba7876ed..5919ad9c 100644 --- a/src/matrix/common.js +++ b/src/matrix/common.js @@ -16,9 +16,13 @@ limitations under the License. */ export function makeTxnId() { + return makeId("t"); +} + +export function makeId(prefix) { const n = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER); const str = n.toString(16); - return "t" + "0".repeat(14 - str.length) + str; + return prefix + "0".repeat(14 - str.length) + str; } export function isTxnId(txnId) { diff --git a/src/matrix/e2ee/olm/Encryption.ts b/src/matrix/e2ee/olm/Encryption.ts index 9b754272..dcc9f0b1 100644 --- a/src/matrix/e2ee/olm/Encryption.ts +++ b/src/matrix/e2ee/olm/Encryption.ts @@ -311,7 +311,7 @@ class EncryptionTarget { } } -class EncryptedMessage { +export class EncryptedMessage { constructor( public readonly content: OlmEncryptedMessageContent, public readonly device: DeviceIdentity diff --git a/src/matrix/net/HomeServerApi.ts b/src/matrix/net/HomeServerApi.ts index e9902ef8..30406c34 100644 --- a/src/matrix/net/HomeServerApi.ts +++ b/src/matrix/net/HomeServerApi.ts @@ -159,6 +159,10 @@ export class HomeServerApi { state(roomId: string, eventType: string, stateKey: string, options?: BaseRequestOptions): IHomeServerRequest { return this._get(`/rooms/${encodeURIComponent(roomId)}/state/${encodeURIComponent(eventType)}/${encodeURIComponent(stateKey)}`, {}, undefined, options); } + + sendState(roomId: string, eventType: string, stateKey: string, content: Record, options?: BaseRequestOptions): IHomeServerRequest { + return this._put(`/rooms/${encodeURIComponent(roomId)}/state/${encodeURIComponent(eventType)}/${encodeURIComponent(stateKey)}`, {}, content, options); + } getLoginFlows(): IHomeServerRequest { return this._unauthedRequest("GET", this._url("/login")); diff --git a/src/platform/types/MediaDevices.ts b/src/platform/types/MediaDevices.ts index 8bf608ce..ed9015bf 100644 --- a/src/platform/types/MediaDevices.ts +++ b/src/platform/types/MediaDevices.ts @@ -17,6 +17,7 @@ limitations under the License. export interface MediaDevices { // filter out audiooutput enumerate(): Promise; + // to assign to a video element, we downcast to WrappedTrack and use the stream property. getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise; getScreenShareTrack(): Promise; } From b2ac4bc291b869935468e724e35de22445e0c670 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 10 Mar 2022 17:39:29 +0100 Subject: [PATCH 014/435] WIP13 --- src/matrix/calls/TODO.md | 5 ++- src/matrix/calls/group/GroupCall.ts | 56 +++++++++++++++++++++++++---- src/matrix/calls/group/Member.ts | 2 +- 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index a8fbeafd..ad8fe185 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -96,12 +96,15 @@ party identification ## TODO Build basic version of PeerCall + - add candidates code Build basic version of GroupCall -Make it possible to olm encrypt the messages + - add state, block invalid actions +DONE: Make it possible to olm encrypt the messages Do work needed for state events - receiving (almost done?) - sending Expose call objects +expose volume events from audiotrack to group call Write view model write view diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 7266645d..9e80b0ff 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -27,17 +27,31 @@ import type {StateEvent} from "../../storage/types"; import type {Platform} from "../../../platform/web/Platform"; import type {EncryptedMessage} from "../../e2ee/olm/Encryption"; import type {ILogItem} from "../../../logging/types"; +import type {Storage} from "../../storage/idb/Storage"; + +export enum GroupCallState { + LocalCallFeedUninitialized = "local_call_feed_uninitialized", + InitializingLocalCallFeed = "initializing_local_call_feed", + LocalCallFeedInitialized = "local_call_feed_initialized", + Joining = "entering", + Joined = "entered", + Ended = "ended", +} export type Options = Omit & { emitUpdate: (call: GroupCall, params?: any) => void; encryptDeviceMessage: (roomId: string, message: SignallingMessage, log: ILogItem) => Promise, + storage: Storage, + ownDeviceId: string }; export class GroupCall { private readonly _members: ObservableMap = new ObservableMap(); private localMedia?: Promise; private _memberOptions: MemberOptions; - + private _state: GroupCallState = GroupCallState.LocalCallFeedInitialized; + + // TODO: keep connected state and deal constructor( private callEvent: StateEvent, private readonly room: Room, @@ -52,6 +66,10 @@ export class GroupCall { }, options); } + static async create(roomId: string, options: Options): Promise { + + } + get members(): BaseObservableMap { return this._members; } get id(): string { return this.callEvent.state_key; } @@ -60,12 +78,11 @@ export class GroupCall { return this.callEvent.content["m.terminated"] === true; } - async join(tracks: Promise) { - this.localMedia = tracks.then(tracks => LocalMedia.fromTracks(tracks)); + async join(localMedia: Promise) { + this.localMedia = localMedia; + const memberContent = await this._createOrUpdateOwnMemberStateContent(); // send m.call.member state event - const request = this.options.hsApi.sendState(this.room.id, "m.call.member", this.options.ownUserId, { - - }); + const request = this.options.hsApi.sendState(this.room.id, "m.call.member", this.options.ownUserId, memberContent); await request.response(); // send invite to all members that are < my userId for (const [,member] of this._members) { @@ -106,4 +123,31 @@ export class GroupCall { // we haven't received the m.call.member yet for this caller. buffer the device messages or create the member/call anyway? } } + + private async _createOrUpdateOwnMemberStateContent() { + const {storage} = this.options; + const txn = await storage.readTxn([storage.storeNames.roomState]); + const stateEvent = await txn.roomState.get(this.room.id, "m.call.member", this.options.ownUserId); + const stateContent = stateEvent?.event?.content ?? { + ["m.calls"]: [] + }; + const callsInfo = stateContent["m.calls"]; + let callInfo = callsInfo.find(c => c["m.call_id"] === this.id); + if (!callInfo) { + callInfo = { + ["m.call_id"]: this.id, + ["m.devices"]: [] + }; + callsInfo.push(callInfo); + } + const devicesInfo = callInfo["m.devices"]; + let deviceInfo = devicesInfo.find(d => d["device_id"] === this.options.ownDeviceId); + if (!deviceInfo) { + deviceInfo = { + ["device_id"]: this.options.ownDeviceId + }; + devicesInfo.push(deviceInfo); + } + return stateContent; + } } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index c6791568..cf020f37 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -65,7 +65,7 @@ export class Member { /** @internal */ updateCallInfo(memberCallInfo) { - + // m.calls object from the m.call.member event } /** @internal */ From b213a45c5c759ad90d3fc7adde489dcbf2ae5c09 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 11 Mar 2022 14:40:37 +0100 Subject: [PATCH 015/435] WIP: work on group call state transitions --- src/matrix/calls/CallHandler.ts | 26 ++++++- src/matrix/calls/PeerCall.ts | 5 +- src/matrix/calls/TODO.md | 8 +- src/matrix/calls/group/GroupCall.ts | 116 ++++++++++++++++++++-------- src/matrix/calls/group/Member.ts | 14 ++-- 5 files changed, 119 insertions(+), 50 deletions(-) diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 0e70fb5a..a0cd8473 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -21,6 +21,7 @@ import {handlesEventType} from "./PeerCall"; import {EventType} from "./callEventTypes"; import {GroupCall} from "./group/GroupCall"; +import type {LocalMedia} from "./LocalMedia"; import type {Room} from "../room/Room"; import type {MemberChange} from "../room/members/RoomMember"; import type {StateEvent} from "../storage/types"; @@ -49,6 +50,22 @@ export class GroupCallHandler { }); } + async createCall(roomId: string, localMedia: LocalMedia, name: string): Promise { + const call = new GroupCall(undefined, undefined, roomId, this.groupCallOptions); + this._calls.set(call.id, call); + try { + await call.create(localMedia, name); + } catch (err) { + if (err.name === "ConnectionError") { + // if we're offline, give up and remove the call again + this._calls.remove(call.id); + } + throw err; + } + await call.join(localMedia); + return call; + } + get calls(): BaseObservableMap { return this._calls; } // TODO: check and poll turn server credentials here @@ -58,7 +75,7 @@ export class GroupCallHandler { // first update call events for (const event of events) { if (event.type === EventType.GroupCall) { - this.handleCallEvent(event, room); + this.handleCallEvent(event, room.id); } } // then update members @@ -71,7 +88,8 @@ export class GroupCallHandler { /** @internal */ updateRoomMembers(room: Room, memberChanges: Map) { - + // TODO: also have map for roomId to calls, so we can easily update members + // we will also need this to get the call for a room } /** @internal */ @@ -86,7 +104,7 @@ export class GroupCallHandler { call?.handleDeviceMessage(message, userId, deviceId, log); } - private handleCallEvent(event: StateEvent, room: Room) { + private handleCallEvent(event: StateEvent, roomId: string) { const callId = event.state_key; let call = this._calls.get(callId); if (call) { @@ -95,7 +113,7 @@ export class GroupCallHandler { this._calls.remove(call.id); } } else { - call = new GroupCall(event, room, this.groupCallOptions); + call = new GroupCall(event.state_key, event.content, roomId, this.groupCallOptions); this._calls.set(call.id, call); } } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index ed8351ac..270e0fa4 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -127,12 +127,11 @@ export class PeerCall implements IDisposable { return; } this.setState(CallState.CreateOffer); - // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called for (const t of this.localMedia.tracks) { this.peerConnection.addTrack(t); } - // TODO: in case of glare, we would not go to InviteSent if we haven't started sending yet - // but we would go straight to CreateAnswer, so also need to wait for that state + // after adding the local tracks, and wait for handleNegotiation to be called, + // or invite glare where we give up our invite and answer instead await this.waitForState([CallState.InviteSent, CallState.CreateAnswer]); } diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index ad8fe185..83d706f6 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -97,12 +97,12 @@ party identification Build basic version of PeerCall - add candidates code -Build basic version of GroupCall - - add state, block invalid actions +DONE: Build basic version of GroupCall + - DONE: add state, block invalid actions DONE: Make it possible to olm encrypt the messages Do work needed for state events - - receiving (almost done?) - - sending + - DONEish: receiving (almost done?) + - DONEish: sending Expose call objects expose volume events from audiotrack to group call Write view model diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 9e80b0ff..6c26c995 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -18,6 +18,8 @@ import {ObservableMap} from "../../../observable/map/ObservableMap"; import {Member} from "./Member"; import {LocalMedia} from "../LocalMedia"; import {RoomMember} from "../../room/members/RoomMember"; +import {makeId} from "../../common"; + import type {Options as MemberOptions} from "./Member"; import type {BaseObservableMap} from "../../../observable/map/BaseObservableMap"; import type {Track} from "../../../platform/types/MediaDevices"; @@ -30,12 +32,11 @@ import type {ILogItem} from "../../../logging/types"; import type {Storage} from "../../storage/idb/Storage"; export enum GroupCallState { - LocalCallFeedUninitialized = "local_call_feed_uninitialized", - InitializingLocalCallFeed = "initializing_local_call_feed", - LocalCallFeedInitialized = "local_call_feed_initialized", - Joining = "entering", - Joined = "entered", - Ended = "ended", + Fledgling = "fledgling", + Creating = "creating", + Created = "created", + Joining = "joining", + Joined = "joined", } export type Options = Omit & { @@ -46,70 +47,112 @@ export type Options = Omit = new ObservableMap(); - private localMedia?: Promise; + private _localMedia?: LocalMedia; private _memberOptions: MemberOptions; - private _state: GroupCallState = GroupCallState.LocalCallFeedInitialized; - - // TODO: keep connected state and deal + private _state: GroupCallState; + constructor( - private callEvent: StateEvent, - private readonly room: Room, + id: string | undefined, + private callContent: Record | undefined, + private readonly roomId: string, private readonly options: Options ) { + this.id = id ?? makeId("conf-"); + this._state = id ? GroupCallState.Created : GroupCallState.Fledgling; this._memberOptions = Object.assign({ confId: this.id, emitUpdate: member => this._members.update(member.member.userId, member), encryptDeviceMessage: (message: SignallingMessage, log) => { - return this.options.encryptDeviceMessage(this.room.id, message, log); + return this.options.encryptDeviceMessage(this.roomId, message, log); } }, options); } - static async create(roomId: string, options: Options): Promise { - - } - + get localMedia(): LocalMedia | undefined { return this._localMedia; } get members(): BaseObservableMap { return this._members; } - get id(): string { return this.callEvent.state_key; } - get isTerminated(): boolean { - return this.callEvent.content["m.terminated"] === true; + return this.callContent?.["m.terminated"] === true; } - async join(localMedia: Promise) { - this.localMedia = localMedia; - const memberContent = await this._createOrUpdateOwnMemberStateContent(); + async join(localMedia: LocalMedia) { + if (this._state !== GroupCallState.Created) { + return; + } + this._state = GroupCallState.Joining; + this._localMedia = localMedia; + const memberContent = await this._joinCallMemberContent(); // send m.call.member state event - const request = this.options.hsApi.sendState(this.room.id, "m.call.member", this.options.ownUserId, memberContent); + const request = this.options.hsApi.sendState(this.roomId, "m.call.member", this.options.ownUserId, memberContent); await request.response(); // send invite to all members that are < my userId for (const [,member] of this._members) { - member.connect(this.localMedia); + member.connect(this._localMedia); + } + } + + async leave() { + const memberContent = await this._leaveCallMemberContent(); + // send m.call.member state event + if (memberContent) { + const request = this.options.hsApi.sendState(this.roomId, "m.call.member", this.options.ownUserId, memberContent); + await request.response(); } } /** @internal */ - updateCallEvent(callEvent: StateEvent) { - this.callEvent = callEvent; - // TODO: emit update + async create(localMedia: LocalMedia, name: string) { + if (this._state !== GroupCallState.Fledgling) { + return; + } + this._state = GroupCallState.Creating; + this.callContent = { + "m.type": localMedia.cameraTrack ? "m.video" : "m.voice", + "m.name": name, + "m.intent": "m.ring" + }; + const request = this.options.hsApi.sendState(this.roomId, "m.call", this.id, this.callContent); + await request.response(); + } + + /** @internal */ + updateCallEvent(callContent: Record) { + this.callContent = callContent; + if (this._state === GroupCallState.Creating) { + this._state = GroupCallState.Created; + } } /** @internal */ addMember(userId, memberCallInfo) { + if (userId === this.options.ownUserId) { + if (this._state === GroupCallState.Joining) { + this._state = GroupCallState.Joined; + } + return; + } let member = this._members.get(userId); if (member) { member.updateCallInfo(memberCallInfo); } else { - member = new Member(RoomMember.fromUserId(this.room.id, userId, "join"), this._memberOptions); - member.updateCallInfo(memberCallInfo); + member = new Member(RoomMember.fromUserId(this.roomId, userId, "join"), memberCallInfo, this._memberOptions); this._members.add(userId, member); + if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { + member.connect(this._localMedia!); + } } } /** @internal */ removeMember(userId) { + if (userId === this.options.ownUserId) { + if (this._state === GroupCallState.Joined) { + this._state = GroupCallState.Created; + } + return; + } this._members.remove(userId); } @@ -124,10 +167,10 @@ export class GroupCall { } } - private async _createOrUpdateOwnMemberStateContent() { + private async _joinCallMemberContent() { const {storage} = this.options; const txn = await storage.readTxn([storage.storeNames.roomState]); - const stateEvent = await txn.roomState.get(this.room.id, "m.call.member", this.options.ownUserId); + const stateEvent = await txn.roomState.get(this.roomId, "m.call.member", this.options.ownUserId); const stateContent = stateEvent?.event?.content ?? { ["m.calls"]: [] }; @@ -150,4 +193,13 @@ export class GroupCall { } return stateContent; } + + private async _leaveCallMemberContent(): Promise | undefined> { + const {storage} = this.options; + const txn = await storage.readTxn([storage.storeNames.roomState]); + const stateEvent = await txn.roomState.get(this.roomId, "m.call.member", this.options.ownUserId); + const callsInfo = stateEvent?.event?.content?.["m.calls"]; + callsInfo?.filter(c => c["m.call_id"] === this.id); + return stateEvent?.event.content; + } } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index cf020f37..0c574bb0 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -38,10 +38,11 @@ export type Options = Omit; + private localMedia?: LocalMedia; constructor( public readonly member: RoomMember, + private memberCallInfo: Record, private readonly options: Options ) {} @@ -53,13 +54,13 @@ export class Member { return this.peerCall?.state === CallState.Connected; } - /* @internal */ - connect(localMedia: Promise) { + /** @internal */ + connect(localMedia: LocalMedia) { this.localMedia = localMedia; // otherwise wait for it to connect if (this.member.userId < this.options.ownUserId) { this.peerCall = this._createPeerCall(makeId("c")); - this.peerCall.call(localMedia); + this.peerCall.call(Promise.resolve(localMedia.clone())); } } @@ -71,13 +72,12 @@ export class Member { /** @internal */ emitUpdate = (peerCall: PeerCall, params: any) => { if (peerCall.state === CallState.Ringing) { - peerCall.answer(this.localMedia!); + peerCall.answer(Promise.resolve(this.localMedia!)); } this.options.emitUpdate(this, params); } - /** From PeerCallHandler - * @internal */ + /** @internal */ sendSignallingMessage = async (message: SignallingMessage, log: ILogItem) => { const groupMessage = message as SignallingMessage; groupMessage.content.conf_id = this.options.confId; From 3c160c8a37c2c429f913ae2dc0cf3999b527c53a Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 11 Mar 2022 16:35:32 +0100 Subject: [PATCH 016/435] handle remote ice candidates --- src/matrix/calls/PeerCall.ts | 48 +++++++++++++++++++++++++++++++----- src/matrix/calls/TODO.md | 1 + 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 270e0fa4..fa56601e 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -105,7 +105,7 @@ export class PeerCall implements IDisposable { log(...args) { console.log.apply(console, ["WebRTC log:", ...args])}, warn(...args) { console.log.apply(console, ["WebRTC warn:", ...args])}, error(...args) { console.error.apply(console, ["WebRTC error:", ...args])}, - } + }; } get state(): CallState { return this._state; } @@ -214,9 +214,9 @@ export class PeerCall implements IDisposable { case EventType.Answer: await this.handleAnswer(message.content, partyId); break; - //case EventType.Candidates: - // await this.handleRemoteIceCandidates(message.content, partyId); - // break; + case EventType.Candidates: + await this.handleRemoteIceCandidates(message.content, partyId); + break; case EventType.Hangup: default: throw new Error(`Unknown event type for call: ${message.type}`); @@ -296,7 +296,7 @@ export class PeerCall implements IDisposable { } }; - private async handleInviteGlare(content: MCallInvite, partyId: PartyId): Promise { + private async handleInviteGlare(content: MCallInvite, partyId: PartyId): Promise { // this is only called when the ids are different const newCallId = content.call_id; if (this.callId! > newCallId) { @@ -386,7 +386,7 @@ export class PeerCall implements IDisposable { } } - private async handleAnswer(content: MCallAnswer, partyId: PartyId): Promise { + private async handleAnswer(content: MCallAnswer, partyId: PartyId): Promise { this.logger.debug(`Got answer for call ID ${this.callId} from party ID ${partyId}`); if (this._state === CallState.Ended) { @@ -424,6 +424,42 @@ export class PeerCall implements IDisposable { } } + async handleRemoteIceCandidates(content: MCallCandidates, partyId) { + if (this.state === CallState.Ended) { + //debuglog("Ignoring remote ICE candidate because call has ended"); + return; + } + + const candidates = content.candidates; + if (!candidates) { + this.logger.info(`Call ${this.callId} Ignoring candidates event with no candidates!`); + return; + } + + const fromPartyId = content.version === 0 ? null : partyId || null; + + if (this.opponentPartyId === undefined) { + // we haven't picked an opponent yet so save the candidates + this.logger.info(`Call ${this.callId} Buffering ${candidates.length} candidates until we pick an opponent`); + const bufferedCandidates = this.remoteCandidateBuffer!.get(fromPartyId) || []; + bufferedCandidates.push(...candidates); + this.remoteCandidateBuffer!.set(fromPartyId, bufferedCandidates); + return; + } + + if (this.opponentPartyId !== partyId) { + this.logger.info( + `Call ${this.callId} `+ + `Ignoring candidates from party ID ${partyId}: ` + + `we have chosen party ID ${this.opponentPartyId}`, + ); + + return; + } + + await this.addIceCandidates(candidates); + } + // private async onNegotiateReceived(event: MatrixEvent): Promise { // const content = event.getContent(); // const description = content.description; diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index 83d706f6..23306c56 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -103,6 +103,7 @@ DONE: Make it possible to olm encrypt the messages Do work needed for state events - DONEish: receiving (almost done?) - DONEish: sending +logging Expose call objects expose volume events from audiotrack to group call Write view model From f6744926850f7e483f07b7e275a675aa2bb1915d Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 11 Mar 2022 16:56:21 +0100 Subject: [PATCH 017/435] remove local media promises (handle them outside of call code) + glare --- src/matrix/calls/PeerCall.ts | 41 +++++++++----------------------- src/matrix/calls/TODO.md | 8 +++---- src/matrix/calls/group/Member.ts | 4 ++-- 3 files changed, 17 insertions(+), 36 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index fa56601e..8725756f 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -114,18 +114,12 @@ export class PeerCall implements IDisposable { return this.peerConnection.remoteTracks; } - async call(localMediaPromise: Promise): Promise { + async call(localMedia: LocalMedia): Promise { if (this._state !== CallState.Fledgling) { return; } + this.localMedia = localMedia; this.direction = CallDirection.Outbound; - this.setState(CallState.WaitLocalMedia); - try { - this.localMedia = await localMediaPromise; - } catch (err) { - this.setState(CallState.Ended); - return; - } this.setState(CallState.CreateOffer); for (const t of this.localMedia.tracks) { this.peerConnection.addTrack(t); @@ -135,17 +129,11 @@ export class PeerCall implements IDisposable { await this.waitForState([CallState.InviteSent, CallState.CreateAnswer]); } - async answer(localMediaPromise: Promise): Promise { + async answer(localMedia: LocalMedia): Promise { if (this._state !== CallState.Ringing) { return; } - this.setState(CallState.WaitLocalMedia); - try { - this.localMedia = await localMediaPromise; - } catch (err) { - this.setState(CallState.Ended); - return; - } + this.localMedia = localMedia; this.setState(CallState.CreateAnswer); for (const t of this.localMedia.tracks) { this.peerConnection.addTrack(t); @@ -274,7 +262,7 @@ export class PeerCall implements IDisposable { lifetime: CALL_TIMEOUT_MS }; if (this._state === CallState.CreateOffer) { - await this.options.sendSignallingMessage({type: EventType.Invite, content}); + await this.options.sendSignallingMessage({type: EventType.Invite, content}, undefined); this.setState(CallState.InviteSent); } else if (this._state === CallState.Connected || this._state === CallState.Connecting) { // send Negotiate message @@ -304,21 +292,15 @@ export class PeerCall implements IDisposable { "Glare detected: answering incoming call " + newCallId + " and canceling outgoing call " + this.callId, ); - - /* - first, we should set CallDirection - we should anser the call - */ - - // TODO: review states to be unambigous, WaitLocalMedia for sending offer or answer? // How do we interrupt `call()`? well, perhaps we need to not just await InviteSent but also CreateAnswer? - if (this._state === CallState.Fledgling || this._state === CallState.CreateOffer || this._state === CallState.WaitLocalMedia) { - + if (this._state === CallState.Fledgling || this._state === CallState.CreateOffer) { + // TODO: don't send invite! } else { await this.sendHangupWithCallId(this.callId, CallErrorCode.Replaced); } await this.handleInvite(content, partyId); - await this.answer(Promise.resolve(this.localMedia!)); + // TODO: need to skip state check + await this.answer(this.localMedia!); } else { this.logger.log( "Glare detected: rejecting incoming call " + newCallId + @@ -328,7 +310,7 @@ export class PeerCall implements IDisposable { } } - private async handleFirstInvite(content: MCallInvite, partyId: PartyId): Promise { + private async handleFirstInvite(content: MCallInvite, partyId: PartyId): Promise { if (this._state !== CallState.Fledgling || this.opponentPartyId !== undefined) { // TODO: hangup or ignore? return; @@ -336,7 +318,7 @@ export class PeerCall implements IDisposable { await this.handleInvite(content, partyId); } - private async handleInvite(content: MCallInvite, partyId: PartyId): Promise { + private async handleInvite(content: MCallInvite, partyId: PartyId): Promise { // we must set the party ID before await-ing on anything: the call event // handler will start giving us more call events (eg. candidates) so if @@ -705,7 +687,6 @@ export enum CallParty { export enum CallState { Fledgling = 'fledgling', - WaitLocalMedia = 'wait_local_media', CreateOffer = 'create_offer', InviteSent = 'invite_sent', CreateAnswer = 'create_answer', diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index 23306c56..a07da60e 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -145,10 +145,10 @@ FROM CALLER FROM CALLEE Fledgling Fledgling V `call()` V `handleInvite()`: setRemoteDescription(event.offer), add buffered candidates -WaitLocalMedia Ringing - V media promise resolves V `answer()` - V add local tracks WaitLocalMedia -CreateOffer V media promise resolves + V Ringing + V V `answer()` +CreateOffer V + V add local tracks V V wait for negotionneeded events V add local tracks V setLocalDescription() CreateAnswer V send invite event V setLocalDescription(createAnswer()) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 0c574bb0..bd1613cd 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -60,7 +60,7 @@ export class Member { // otherwise wait for it to connect if (this.member.userId < this.options.ownUserId) { this.peerCall = this._createPeerCall(makeId("c")); - this.peerCall.call(Promise.resolve(localMedia.clone())); + this.peerCall.call(localMedia); } } @@ -72,7 +72,7 @@ export class Member { /** @internal */ emitUpdate = (peerCall: PeerCall, params: any) => { if (peerCall.state === CallState.Ringing) { - peerCall.answer(Promise.resolve(this.localMedia!)); + peerCall.answer(this.localMedia!); } this.options.emitUpdate(this, params); } From 1bccbbfa082ea2e7a0f3157fd54d773612ef31c7 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 16 Mar 2022 19:10:51 +0100 Subject: [PATCH 018/435] fix typescript errors --- src/matrix/calls/LocalMedia.ts | 6 +- src/matrix/calls/PeerCall.ts | 101 +++++++++++++++++++++++++++++---- 2 files changed, 92 insertions(+), 15 deletions(-) diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 2429ff2c..4f5d3b7a 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {StreamPurpose} from "../../platform/types/WebRTC"; +import {SDPStreamMetadataPurpose} from "./callEventTypes"; import {Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; import {SDPStreamMetadata} from "./callEventTypes"; @@ -42,14 +42,14 @@ export class LocalMedia { const userMediaTrack = this.microphoneTrack ?? this.cameraTrack; if (userMediaTrack) { metadata[userMediaTrack.streamId] = { - purpose: StreamPurpose.UserMedia, + purpose: SDPStreamMetadataPurpose.Usermedia, audio_muted: this.microphoneTrack?.muted ?? false, video_muted: this.cameraTrack?.muted ?? false, }; } if (this.screenShareTrack) { metadata[this.screenShareTrack.streamId] = { - purpose: StreamPurpose.ScreenShare + purpose: SDPStreamMetadataPurpose.Screenshare }; } return metadata; diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 8725756f..a4ea8cca 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -21,6 +21,7 @@ import {Disposables, IDisposable} from "../../utils/Disposables"; import type {Room} from "../room/Room"; import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; +import {Instance as logger} from "../../logging/NullLogger"; import type {TimeoutCreator, Timeout} from "../../platform/types/types"; import {WebRTC, PeerConnection, PeerConnectionHandler, DataChannel} from "../../platform/types/WebRTC"; @@ -81,16 +82,28 @@ export class PeerCall implements IDisposable { // perfect negotiation flags private makingOffer: boolean = false; private ignoreOffer: boolean = false; + + private sentEndOfCandidates: boolean = false; + private iceDisconnectedTimeout?: Timeout; + constructor( - private callId: string, // generated or from invite + private callId: string, private readonly options: Options ) { const outer = this; this.peerConnection = options.webRTC.createPeerConnection({ - onIceConnectionStateChange(state: RTCIceConnectionState) {}, - onLocalIceCandidate(candidate: RTCIceCandidate) {}, - onIceGatheringStateChange(state: RTCIceGatheringState) {}, - onRemoteTracksChanged(tracks: Track[]) {}, + onIceConnectionStateChange(state: RTCIceConnectionState) { + outer.onIceConnectionStateChange(state); + }, + onLocalIceCandidate(candidate: RTCIceCandidate) { + outer.handleLocalIceCandidate(candidate); + }, + onIceGatheringStateChange(state: RTCIceGatheringState) { + outer.handleIceGatheringState(state); + }, + onRemoteTracksChanged(tracks: Track[]) { + outer.options.emitUpdate(outer, undefined); + }, onDataChannelChanged(dataChannel: DataChannel | undefined) {}, onNegotiationNeeded() { const promiseCreator = () => outer.handleNegotiation(); @@ -158,7 +171,7 @@ export class PeerCall implements IDisposable { } // Allow a short time for initial candidates to be gathered await this.delay(200); - this.sendAnswer(); + await this.sendAnswer(); } async setMedia(localMediaPromise: Promise) { @@ -187,7 +200,11 @@ export class PeerCall implements IDisposable { } - async hangup(errorCode: CallErrorCode) { + async hangup(errorCode: CallErrorCode): Promise { + if (this._state !== CallState.Ended) { + this._state = CallState.Ended; + await this.sendHangupWithCallId(this.callId, errorCode); + } } async handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId, log: ILogItem): Promise { @@ -222,7 +239,7 @@ export class PeerCall implements IDisposable { return this.options.sendSignallingMessage({ type: EventType.Hangup, content - }, undefined); + }, logger.item); } // calls are serialized and deduplicated by responsePromiseChain @@ -262,7 +279,7 @@ export class PeerCall implements IDisposable { lifetime: CALL_TIMEOUT_MS }; if (this._state === CallState.CreateOffer) { - await this.options.sendSignallingMessage({type: EventType.Invite, content}, undefined); + await this.options.sendSignallingMessage({type: EventType.Invite, content}, logger.item); this.setState(CallState.InviteSent); } else if (this._state === CallState.Connected || this._state === CallState.Connecting) { // send Negotiate message @@ -406,7 +423,42 @@ export class PeerCall implements IDisposable { } } - async handleRemoteIceCandidates(content: MCallCandidates, partyId) { + private handleIceGatheringState(state: RTCIceGatheringState) { + this.logger.debug(`Call ${this.callId} ice gathering state changed to ${state}`); + if (state === 'complete' && !this.sentEndOfCandidates) { + // If we didn't get an empty-string candidate to signal the end of candidates, + // create one ourselves now gathering has finished. + // We cast because the interface lists all the properties as required but we + // only want to send 'candidate' + // XXX: We probably want to send either sdpMid or sdpMLineIndex, as it's not strictly + // correct to have a candidate that lacks both of these. We'd have to figure out what + // previous candidates had been sent with and copy them. + const c = { + candidate: '', + } as RTCIceCandidate; + this.queueCandidate(c); + this.sentEndOfCandidates = true; + } + } + + private handleLocalIceCandidate(candidate: RTCIceCandidate) { + this.logger.debug( + "Call " + this.callId + " got local ICE " + candidate.sdpMid + " candidate: " + + candidate.candidate, + ); + + if (this._state === CallState.Ended) return; + + // As with the offer, note we need to make a copy of this object, not + // pass the original: that broke in Chrome ~m43. + if (candidate.candidate !== '' || !this.sentEndOfCandidates) { + this.queueCandidate(candidate); + + if (candidate.candidate === '') this.sentEndOfCandidates = true; + } + } + + private async handleRemoteIceCandidates(content: MCallCandidates, partyId) { if (this.state === CallState.Ended) { //debuglog("Ignoring remote ICE candidate because call has ended"); return; @@ -512,7 +564,7 @@ export class PeerCall implements IDisposable { this.candidateSendQueue = []; try { - await this.options.sendSignallingMessage({type: EventType.Answer, content: answerContent}, undefined); + await this.options.sendSignallingMessage({type: EventType.Answer, content: answerContent}, logger.item); } catch (error) { this.terminate(CallParty.Local, CallErrorCode.SendAnswer, false); throw error; @@ -561,7 +613,7 @@ export class PeerCall implements IDisposable { version: 1, candidates }, - }, undefined); + }, logger.item); // Try to send candidates again just in case we received more candidates while sending. this.sendCandidateQueue(); } catch (error) { @@ -620,6 +672,31 @@ export class PeerCall implements IDisposable { } } + private onIceConnectionStateChange = (state: RTCIceConnectionState): void => { + if (this._state === CallState.Ended) { + return; // because ICE can still complete as we're ending the call + } + this.logger.debug( + "Call ID " + this.callId + ": ICE connection state changed to: " + state, + ); + // ideally we'd consider the call to be connected when we get media but + // chrome doesn't implement any of the 'onstarted' events yet + if (state == 'connected') { + this.iceDisconnectedTimeout?.abort(); + this.iceDisconnectedTimeout = undefined; + this.setState(CallState.Connected); + } else if (state == 'failed') { + this.iceDisconnectedTimeout?.abort(); + this.iceDisconnectedTimeout = undefined; + this.hangup(CallErrorCode.IceFailed); + } else if (state == 'disconnected') { + this.iceDisconnectedTimeout = this.options.createTimeout(30 * 1000); + this.iceDisconnectedTimeout.elapsed().then(() => { + this.hangup(CallErrorCode.IceFailed); + }, () => { /* ignore AbortError */ }); + } + }; + private setState(state: CallState): void { const oldState = this._state; this._state = state; From 07bc0a2376d2e910521071f9448d20f11f4849fe Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 17 Mar 2022 11:30:40 +0100 Subject: [PATCH 019/435] move observable values each in their own file --- src/domain/navigation/Navigation.js | 3 +- src/domain/session/RoomGridViewModel.js | 2 +- src/domain/session/RoomViewModelObservable.js | 2 +- .../room/timeline/ReactionsViewModel.js | 2 +- .../session/settings/KeyBackupViewModel.js | 5 +- src/lib.ts | 8 +- src/matrix/Client.js | 2 +- src/matrix/Session.js | 16 +- src/matrix/Sync.js | 2 +- src/matrix/e2ee/megolm/keybackup/KeyBackup.ts | 2 +- src/matrix/net/Reconnector.ts | 2 +- src/matrix/room/BaseRoom.js | 2 +- src/matrix/room/ObservedEventMap.js | 2 +- src/mocks/Clock.js | 2 +- src/observable/ObservableValue.ts | 248 ------------------ src/observable/value/BaseObservableValue.ts | 83 ++++++ .../value/FlatMapObservableValue.ts | 109 ++++++++ src/observable/value/ObservableValue.ts | 82 ++++++ src/observable/value/PickMapObservable.ts | 89 +++++++ .../value/RetainedObservableValue.ts | 31 +++ src/platform/web/dom/History.js | 2 +- src/platform/web/dom/OnlineStatus.js | 2 +- src/utils/AbortableOperation.ts | 3 +- 23 files changed, 429 insertions(+), 272 deletions(-) delete mode 100644 src/observable/ObservableValue.ts create mode 100644 src/observable/value/BaseObservableValue.ts create mode 100644 src/observable/value/FlatMapObservableValue.ts create mode 100644 src/observable/value/ObservableValue.ts create mode 100644 src/observable/value/PickMapObservable.ts create mode 100644 src/observable/value/RetainedObservableValue.ts diff --git a/src/domain/navigation/Navigation.js b/src/domain/navigation/Navigation.js index 340ae0d5..c2b2b54c 100644 --- a/src/domain/navigation/Navigation.js +++ b/src/domain/navigation/Navigation.js @@ -14,7 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {BaseObservableValue, ObservableValue} from "../../observable/ObservableValue"; +import {ObservableValue} from "../../observable/value/ObservableValue"; +import {BaseObservableValue} from "../../observable/value/BaseObservableValue"; export class Navigation { constructor(allowsChild) { diff --git a/src/domain/session/RoomGridViewModel.js b/src/domain/session/RoomGridViewModel.js index a7d19054..5d42f0f6 100644 --- a/src/domain/session/RoomGridViewModel.js +++ b/src/domain/session/RoomGridViewModel.js @@ -186,7 +186,7 @@ export class RoomGridViewModel extends ViewModel { } import {createNavigation} from "../navigation/index.js"; -import {ObservableValue} from "../../observable/ObservableValue"; +import {ObservableValue} from "../../observable/value/ObservableValue"; export function tests() { class RoomVMMock { diff --git a/src/domain/session/RoomViewModelObservable.js b/src/domain/session/RoomViewModelObservable.js index 52833332..86e2bb5d 100644 --- a/src/domain/session/RoomViewModelObservable.js +++ b/src/domain/session/RoomViewModelObservable.js @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {ObservableValue} from "../../observable/ObservableValue"; +import {ObservableValue} from "../../observable/value/ObservableValue"; import {RoomStatus} from "../../matrix/room/common"; /** diff --git a/src/domain/session/room/timeline/ReactionsViewModel.js b/src/domain/session/room/timeline/ReactionsViewModel.js index 4f366af0..214ec17f 100644 --- a/src/domain/session/room/timeline/ReactionsViewModel.js +++ b/src/domain/session/room/timeline/ReactionsViewModel.js @@ -189,7 +189,7 @@ import {HomeServer as MockHomeServer} from "../../../../mocks/HomeServer.js"; // other imports import {BaseMessageTile} from "./tiles/BaseMessageTile.js"; import {MappedList} from "../../../../observable/list/MappedList"; -import {ObservableValue} from "../../../../observable/ObservableValue"; +import {ObservableValue} from "../../../../observable/value/ObservableValue"; import {PowerLevels} from "../../../../matrix/room/PowerLevels.js"; export function tests() { diff --git a/src/domain/session/settings/KeyBackupViewModel.js b/src/domain/session/settings/KeyBackupViewModel.js index 243b0d7c..b4ee9a0e 100644 --- a/src/domain/session/settings/KeyBackupViewModel.js +++ b/src/domain/session/settings/KeyBackupViewModel.js @@ -17,6 +17,7 @@ limitations under the License. import {ViewModel} from "../../ViewModel"; import {KeyType} from "../../../matrix/ssss/index"; import {createEnum} from "../../../utils/enum"; +import {FlatMapObservableValue} from "../../../observable/value/FlatMapObservableValue"; export const Status = createEnum("Enabled", "SetupKey", "SetupPhrase", "Pending", "NewVersionAvailable"); export const BackupWriteStatus = createEnum("Writing", "Stopped", "Done", "Pending"); @@ -29,8 +30,8 @@ export class KeyBackupViewModel extends ViewModel { this._isBusy = false; this._dehydratedDeviceId = undefined; this._status = undefined; - this._backupOperation = this._session.keyBackup.flatMap(keyBackup => keyBackup.operationInProgress); - this._progress = this._backupOperation.flatMap(op => op.progress); + this._backupOperation = new FlatMapObservableValue(this._session.keyBackup, keyBackup => keyBackup.operationInProgress); + this._progress = new FlatMapObservableValue(this._backupOperation, op => op.progress); this.track(this._backupOperation.subscribe(() => { // see if needsNewKey might be set this._reevaluateStatus(); diff --git a/src/lib.ts b/src/lib.ts index a0ada84f..cb939949 100644 --- a/src/lib.ts +++ b/src/lib.ts @@ -46,8 +46,6 @@ export { ConcatList, ObservableMap } from "./observable/index"; -export { - BaseObservableValue, - ObservableValue, - RetainedObservableValue -} from "./observable/ObservableValue"; +export {BaseObservableValue} from "./observable/value/BaseObservableValue"; +export {ObservableValue} from "./observable/value/ObservableValue"; +export {RetainedObservableValue} from "./observable/value/RetainedObservableValue"; diff --git a/src/matrix/Client.js b/src/matrix/Client.js index b24c1ec9..83861cb7 100644 --- a/src/matrix/Client.js +++ b/src/matrix/Client.js @@ -18,7 +18,7 @@ limitations under the License. import {createEnum} from "../utils/enum"; import {lookupHomeserver} from "./well-known.js"; import {AbortableOperation} from "../utils/AbortableOperation"; -import {ObservableValue} from "../observable/ObservableValue"; +import {ObservableValue} from "../observable/value/ObservableValue"; import {HomeServerApi} from "./net/HomeServerApi"; import {Reconnector, ConnectionStatus} from "./net/Reconnector"; import {ExponentialRetryDelay} from "./net/ExponentialRetryDelay"; diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 94fb5dee..69cd9ee0 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -45,7 +45,8 @@ import { keyFromDehydratedDeviceKey as createSSSSKeyFromDehydratedDeviceKey } from "./ssss/index"; import {SecretStorage} from "./ssss/SecretStorage"; -import {ObservableValue, RetainedObservableValue} from "../observable/ObservableValue"; +import {ObservableValue} from "../observable/value/ObservableValue"; +import {RetainedObservableValue} from "../observable/value/RetainedObservableValue"; const PICKLE_KEY = "DEFAULT_KEY"; const PUSHER_KEY = "pusher"; @@ -997,9 +998,18 @@ export function tests() { return { "session data is not modified until after sync": async (assert) => { - const session = new Session({storage: createStorageMock({ + const storage = createStorageMock({ sync: {token: "a", filterId: 5} - }), sessionInfo: {userId: ""}}); + }); + const session = new Session({ + storage, + sessionInfo: {userId: ""}, + platform: { + clock: { + createTimeout: () => undefined + } + } + }); await session.load(); let syncSet = false; const syncTxn = { diff --git a/src/matrix/Sync.js b/src/matrix/Sync.js index 3574213e..8e880def 100644 --- a/src/matrix/Sync.js +++ b/src/matrix/Sync.js @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {ObservableValue} from "../observable/ObservableValue"; +import {ObservableValue} from "../observable/value/ObservableValue"; import {createEnum} from "../utils/enum"; const INCREMENTAL_TIMEOUT = 30000; diff --git a/src/matrix/e2ee/megolm/keybackup/KeyBackup.ts b/src/matrix/e2ee/megolm/keybackup/KeyBackup.ts index 43631552..3da1c704 100644 --- a/src/matrix/e2ee/megolm/keybackup/KeyBackup.ts +++ b/src/matrix/e2ee/megolm/keybackup/KeyBackup.ts @@ -19,7 +19,7 @@ import {StoredRoomKey, keyFromBackup} from "../decryption/RoomKey"; import {MEGOLM_ALGORITHM} from "../../common"; import * as Curve25519 from "./Curve25519"; import {AbortableOperation} from "../../../../utils/AbortableOperation"; -import {ObservableValue} from "../../../../observable/ObservableValue"; +import {ObservableValue} from "../../../../observable/value/ObservableValue"; import {SetAbortableFn} from "../../../../utils/AbortableOperation"; import type {BackupInfo, SessionData, SessionKeyInfo, SessionInfo, KeyBackupPayload} from "./types"; diff --git a/src/matrix/net/Reconnector.ts b/src/matrix/net/Reconnector.ts index bc54ab73..a3739425 100644 --- a/src/matrix/net/Reconnector.ts +++ b/src/matrix/net/Reconnector.ts @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {ObservableValue} from "../../observable/ObservableValue"; +import {ObservableValue} from "../../observable/value/ObservableValue"; import type {ExponentialRetryDelay} from "./ExponentialRetryDelay"; import type {TimeMeasure} from "../../platform/web/dom/Clock.js"; import type {OnlineStatus} from "../../platform/web/dom/OnlineStatus.js"; diff --git a/src/matrix/room/BaseRoom.js b/src/matrix/room/BaseRoom.js index dda3e2e5..cc65a320 100644 --- a/src/matrix/room/BaseRoom.js +++ b/src/matrix/room/BaseRoom.js @@ -29,7 +29,7 @@ import {ObservedEventMap} from "./ObservedEventMap.js"; import {DecryptionSource} from "../e2ee/common.js"; import {ensureLogItem} from "../../logging/utils"; import {PowerLevels} from "./PowerLevels.js"; -import {RetainedObservableValue} from "../../observable/ObservableValue"; +import {RetainedObservableValue} from "../../observable/value/RetainedObservableValue"; import {TimelineReader} from "./timeline/persistence/TimelineReader"; const EVENT_ENCRYPTED_TYPE = "m.room.encrypted"; diff --git a/src/matrix/room/ObservedEventMap.js b/src/matrix/room/ObservedEventMap.js index 6b20f85e..8ee1bca8 100644 --- a/src/matrix/room/ObservedEventMap.js +++ b/src/matrix/room/ObservedEventMap.js @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {BaseObservableValue} from "../../observable/ObservableValue"; +import {BaseObservableValue} from "../../observable/value/BaseObservableValue"; export class ObservedEventMap { constructor(notifyEmpty) { diff --git a/src/mocks/Clock.js b/src/mocks/Clock.js index 440c4cb4..b9d5457d 100644 --- a/src/mocks/Clock.js +++ b/src/mocks/Clock.js @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {ObservableValue} from "../observable/ObservableValue"; +import {ObservableValue} from "../observable/value/ObservableValue"; class Timeout { constructor(elapsed, ms) { diff --git a/src/observable/ObservableValue.ts b/src/observable/ObservableValue.ts deleted file mode 100644 index ad0a226d..00000000 --- a/src/observable/ObservableValue.ts +++ /dev/null @@ -1,248 +0,0 @@ -/* -Copyright 2020 Bruno Windels - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import {AbortError} from "../utils/error"; -import {BaseObservable} from "./BaseObservable"; -import type {SubscriptionHandle} from "./BaseObservable"; - -// like an EventEmitter, but doesn't have an event type -export abstract class BaseObservableValue extends BaseObservable<(value: T) => void> { - emit(argument: T) { - for (const h of this._handlers) { - h(argument); - } - } - - abstract get(): T; - - waitFor(predicate: (value: T) => boolean): IWaitHandle { - if (predicate(this.get())) { - return new ResolvedWaitForHandle(Promise.resolve(this.get())); - } else { - return new WaitForHandle(this, predicate); - } - } - - flatMap(mapper: (value: T) => (BaseObservableValue | undefined)): BaseObservableValue { - return new FlatMapObservableValue(this, mapper); - } -} - -interface IWaitHandle { - promise: Promise; - dispose(): void; -} - -class WaitForHandle implements IWaitHandle { - private _promise: Promise - private _reject: ((reason?: any) => void) | null; - private _subscription: (() => void) | null; - - constructor(observable: BaseObservableValue, predicate: (value: T) => boolean) { - this._promise = new Promise((resolve, reject) => { - this._reject = reject; - this._subscription = observable.subscribe(v => { - if (predicate(v)) { - this._reject = null; - resolve(v); - this.dispose(); - } - }); - }); - } - - get promise(): Promise { - return this._promise; - } - - dispose() { - if (this._subscription) { - this._subscription(); - this._subscription = null; - } - if (this._reject) { - this._reject(new AbortError()); - this._reject = null; - } - } -} - -class ResolvedWaitForHandle implements IWaitHandle { - constructor(public promise: Promise) {} - dispose() {} -} - -export class ObservableValue extends BaseObservableValue { - private _value: T; - - constructor(initialValue: T) { - super(); - this._value = initialValue; - } - - get(): T { - return this._value; - } - - set(value: T): void { - if (value !== this._value) { - this._value = value; - this.emit(this._value); - } - } -} - -export class RetainedObservableValue extends ObservableValue { - private _freeCallback: () => void; - - constructor(initialValue: T, freeCallback: () => void) { - super(initialValue); - this._freeCallback = freeCallback; - } - - onUnsubscribeLast() { - super.onUnsubscribeLast(); - this._freeCallback(); - } -} - -export class FlatMapObservableValue extends BaseObservableValue { - private sourceSubscription?: SubscriptionHandle; - private targetSubscription?: SubscriptionHandle; - - constructor( - private readonly source: BaseObservableValue

, - private readonly mapper: (value: P) => (BaseObservableValue | undefined) - ) { - super(); - } - - onUnsubscribeLast() { - super.onUnsubscribeLast(); - this.sourceSubscription = this.sourceSubscription!(); - if (this.targetSubscription) { - this.targetSubscription = this.targetSubscription(); - } - } - - onSubscribeFirst() { - super.onSubscribeFirst(); - this.sourceSubscription = this.source.subscribe(() => { - this.updateTargetSubscription(); - this.emit(this.get()); - }); - this.updateTargetSubscription(); - } - - private updateTargetSubscription() { - const sourceValue = this.source.get(); - if (sourceValue) { - const target = this.mapper(sourceValue); - if (target) { - if (!this.targetSubscription) { - this.targetSubscription = target.subscribe(() => this.emit(this.get())); - } - return; - } - } - // if no sourceValue or target - if (this.targetSubscription) { - this.targetSubscription = this.targetSubscription(); - } - } - - get(): C | undefined { - const sourceValue = this.source.get(); - if (!sourceValue) { - return undefined; - } - const mapped = this.mapper(sourceValue); - return mapped?.get(); - } -} - -export function tests() { - return { - "set emits an update": assert => { - const a = new ObservableValue(0); - let fired = false; - const subscription = a.subscribe(v => { - fired = true; - assert.strictEqual(v, 5); - }); - a.set(5); - assert(fired); - subscription(); - }, - "set doesn't emit if value hasn't changed": assert => { - const a = new ObservableValue(5); - let fired = false; - const subscription = a.subscribe(() => { - fired = true; - }); - a.set(5); - a.set(5); - assert(!fired); - subscription(); - }, - "waitFor promise resolves on matching update": async assert => { - const a = new ObservableValue(5); - const handle = a.waitFor(v => v === 6); - Promise.resolve().then(() => { - a.set(6); - }); - await handle.promise; - assert.strictEqual(a.get(), 6); - }, - "waitFor promise rejects when disposed": async assert => { - const a = new ObservableValue(0); - const handle = a.waitFor(() => false); - Promise.resolve().then(() => { - handle.dispose(); - }); - await assert.rejects(handle.promise, AbortError); - }, - "flatMap.get": assert => { - const a = new ObservableValue}>(undefined); - const countProxy = a.flatMap(a => a!.count); - assert.strictEqual(countProxy.get(), undefined); - const count = new ObservableValue(0); - a.set({count}); - assert.strictEqual(countProxy.get(), 0); - }, - "flatMap update from source": assert => { - const a = new ObservableValue}>(undefined); - const updates: (number | undefined)[] = []; - a.flatMap(a => a!.count).subscribe(count => { - updates.push(count); - }); - const count = new ObservableValue(0); - a.set({count}); - assert.deepEqual(updates, [0]); - }, - "flatMap update from target": assert => { - const a = new ObservableValue}>(undefined); - const updates: (number | undefined)[] = []; - a.flatMap(a => a!.count).subscribe(count => { - updates.push(count); - }); - const count = new ObservableValue(0); - a.set({count}); - count.set(5); - assert.deepEqual(updates, [0, 5]); - } - } -} diff --git a/src/observable/value/BaseObservableValue.ts b/src/observable/value/BaseObservableValue.ts new file mode 100644 index 00000000..85437262 --- /dev/null +++ b/src/observable/value/BaseObservableValue.ts @@ -0,0 +1,83 @@ +/* +Copyright 2020 Bruno Windels + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {AbortError} from "../../utils/error"; +import {BaseObservable} from "../BaseObservable"; +import type {SubscriptionHandle} from "../BaseObservable"; +import {FlatMapObservableValue} from "./FlatMapObservableValue"; + +// like an EventEmitter, but doesn't have an event type +export abstract class BaseObservableValue extends BaseObservable<(value: T) => void> { + emit(argument: T) { + for (const h of this._handlers) { + h(argument); + } + } + + abstract get(): T; + + waitFor(predicate: (value: T) => boolean): IWaitHandle { + if (predicate(this.get())) { + return new ResolvedWaitForHandle(Promise.resolve(this.get())); + } else { + return new WaitForHandle(this, predicate); + } + } +} + +interface IWaitHandle { + promise: Promise; + dispose(): void; +} + +class WaitForHandle implements IWaitHandle { + private _promise: Promise + private _reject: ((reason?: any) => void) | null; + private _subscription: (() => void) | null; + + constructor(observable: BaseObservableValue, predicate: (value: T) => boolean) { + this._promise = new Promise((resolve, reject) => { + this._reject = reject; + this._subscription = observable.subscribe(v => { + if (predicate(v)) { + this._reject = null; + resolve(v); + this.dispose(); + } + }); + }); + } + + get promise(): Promise { + return this._promise; + } + + dispose() { + if (this._subscription) { + this._subscription(); + this._subscription = null; + } + if (this._reject) { + this._reject(new AbortError()); + this._reject = null; + } + } +} + +class ResolvedWaitForHandle implements IWaitHandle { + constructor(public promise: Promise) {} + dispose() {} +} diff --git a/src/observable/value/FlatMapObservableValue.ts b/src/observable/value/FlatMapObservableValue.ts new file mode 100644 index 00000000..9dff07a6 --- /dev/null +++ b/src/observable/value/FlatMapObservableValue.ts @@ -0,0 +1,109 @@ +/* +Copyright 2020 Bruno Windels + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {BaseObservableValue} from "./BaseObservableValue"; +import {SubscriptionHandle} from "../BaseObservable"; + +export class FlatMapObservableValue extends BaseObservableValue { + private sourceSubscription?: SubscriptionHandle; + private targetSubscription?: SubscriptionHandle; + + constructor( + private readonly source: BaseObservableValue

, + private readonly mapper: (value: P) => (BaseObservableValue | undefined) + ) { + super(); + } + + onUnsubscribeLast() { + super.onUnsubscribeLast(); + this.sourceSubscription = this.sourceSubscription!(); + if (this.targetSubscription) { + this.targetSubscription = this.targetSubscription(); + } + } + + onSubscribeFirst() { + super.onSubscribeFirst(); + this.sourceSubscription = this.source.subscribe(() => { + this.updateTargetSubscription(); + this.emit(this.get()); + }); + this.updateTargetSubscription(); + } + + private updateTargetSubscription() { + const sourceValue = this.source.get(); + if (sourceValue) { + const target = this.mapper(sourceValue); + if (target) { + if (!this.targetSubscription) { + this.targetSubscription = target.subscribe(() => this.emit(this.get())); + } + return; + } + } + // if no sourceValue or target + if (this.targetSubscription) { + this.targetSubscription = this.targetSubscription(); + } + } + + get(): C | undefined { + const sourceValue = this.source.get(); + if (!sourceValue) { + return undefined; + } + const mapped = this.mapper(sourceValue); + return mapped?.get(); + } +} + +import {ObservableValue} from "./ObservableValue"; + +export function tests() { + return { + "flatMap.get": assert => { + const a = new ObservableValue}>(undefined); + const countProxy = new FlatMapObservableValue(a, a => a!.count); + assert.strictEqual(countProxy.get(), undefined); + const count = new ObservableValue(0); + a.set({count}); + assert.strictEqual(countProxy.get(), 0); + }, + "flatMap update from source": assert => { + const a = new ObservableValue}>(undefined); + const updates: (number | undefined)[] = []; + new FlatMapObservableValue(a, a => a!.count).subscribe(count => { + updates.push(count); + }); + const count = new ObservableValue(0); + a.set({count}); + assert.deepEqual(updates, [0]); + }, + "flatMap update from target": assert => { + const a = new ObservableValue}>(undefined); + const updates: (number | undefined)[] = []; + new FlatMapObservableValue(a, a => a!.count).subscribe(count => { + updates.push(count); + }); + const count = new ObservableValue(0); + a.set({count}); + count.set(5); + assert.deepEqual(updates, [0, 5]); + } + } +} diff --git a/src/observable/value/ObservableValue.ts b/src/observable/value/ObservableValue.ts new file mode 100644 index 00000000..d75a0d76 --- /dev/null +++ b/src/observable/value/ObservableValue.ts @@ -0,0 +1,82 @@ +/* +Copyright 2020 Bruno Windels + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {AbortError} from "../../utils/error"; +import {BaseObservableValue} from "./BaseObservableValue"; + +export class ObservableValue extends BaseObservableValue { + private _value: T; + + constructor(initialValue: T) { + super(); + this._value = initialValue; + } + + get(): T { + return this._value; + } + + set(value: T): void { + if (value !== this._value) { + this._value = value; + this.emit(this._value); + } + } +} + +export function tests() { + return { + "set emits an update": assert => { + const a = new ObservableValue(0); + let fired = false; + const subscription = a.subscribe(v => { + fired = true; + assert.strictEqual(v, 5); + }); + a.set(5); + assert(fired); + subscription(); + }, + "set doesn't emit if value hasn't changed": assert => { + const a = new ObservableValue(5); + let fired = false; + const subscription = a.subscribe(() => { + fired = true; + }); + a.set(5); + a.set(5); + assert(!fired); + subscription(); + }, + "waitFor promise resolves on matching update": async assert => { + const a = new ObservableValue(5); + const handle = a.waitFor(v => v === 6); + Promise.resolve().then(() => { + a.set(6); + }); + await handle.promise; + assert.strictEqual(a.get(), 6); + }, + "waitFor promise rejects when disposed": async assert => { + const a = new ObservableValue(0); + const handle = a.waitFor(() => false); + Promise.resolve().then(() => { + handle.dispose(); + }); + await assert.rejects(handle.promise, AbortError); + } + } +} diff --git a/src/observable/value/PickMapObservable.ts b/src/observable/value/PickMapObservable.ts new file mode 100644 index 00000000..835cdf90 --- /dev/null +++ b/src/observable/value/PickMapObservable.ts @@ -0,0 +1,89 @@ +/* +Copyright 2020 Bruno Windels + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {BaseObservableValue} from "./BaseObservableValue"; +import {BaseObservableMap, IMapObserver} from "../map/BaseObservableMap"; +import {SubscriptionHandle} from "../BaseObservable"; + +function pickLowestKey(currentKey: K, newKey: K): boolean { + return newKey < currentKey; +} + +export class PickMapObservable implements IMapObserver extends BaseObservableValue { + + private key?: K; + private mapSubscription?: SubscriptionHandle; + + constructor( + private readonly map: BaseObservableMap, + private readonly pickKey: (currentKey: K, newKey: K) => boolean = pickLowestKey + ) { + super(); + } + + private trySetKey(newKey: K): boolean { + if (this.key === undefined || this.pickKey(this.key, newKey)) { + this.key = newKey; + return true; + } + return false; + } + + onReset(): void { + this.key = undefined; + this.emit(this.get()); + } + + onAdd(key: K, value:V): void { + if (this.trySetKey(key)) { + this.emit(this.get()); + } + } + + onUpdate(key: K, value: V, params: any): void {} + + onRemove(key: K, value: V): void { + if (key === this.key) { + this.key = undefined; + let changed = false; + for (const [key] of this.map) { + changed = this.trySetKey(key) || changed; + } + if (changed) { + this.emit(this.get()); + } + } + } + + onSubscribeFirst(): void { + this.mapSubscription = this.map.subscribe(this); + for (const [key] of this.map) { + this.trySetKey(key); + } + } + + onUnsubscribeLast(): void { + this.mapSubscription(); + this.key = undefined; + } + + get(): V | undefined { + if (this.key !== undefined) { + return this.map.get(this.key); + } + return undefined; + } +} diff --git a/src/observable/value/RetainedObservableValue.ts b/src/observable/value/RetainedObservableValue.ts new file mode 100644 index 00000000..edfb6c15 --- /dev/null +++ b/src/observable/value/RetainedObservableValue.ts @@ -0,0 +1,31 @@ +/* +Copyright 2020 Bruno Windels + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {ObservableValue} from "./ObservableValue"; + +export class RetainedObservableValue extends ObservableValue { + private _freeCallback: () => void; + + constructor(initialValue: T, freeCallback: () => void) { + super(initialValue); + this._freeCallback = freeCallback; + } + + onUnsubscribeLast() { + super.onUnsubscribeLast(); + this._freeCallback(); + } +} diff --git a/src/platform/web/dom/History.js b/src/platform/web/dom/History.js index d51974bb..96576626 100644 --- a/src/platform/web/dom/History.js +++ b/src/platform/web/dom/History.js @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {BaseObservableValue} from "../../../observable/ObservableValue"; +import {BaseObservableValue} from "../../../observable/value/BaseObservableValue"; export class History extends BaseObservableValue { handleEvent(event) { diff --git a/src/platform/web/dom/OnlineStatus.js b/src/platform/web/dom/OnlineStatus.js index 48e4e912..fd114603 100644 --- a/src/platform/web/dom/OnlineStatus.js +++ b/src/platform/web/dom/OnlineStatus.js @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {BaseObservableValue} from "../../../observable/ObservableValue"; +import {BaseObservableValue} from "../../../observable/value/BaseObservableValue"; export class OnlineStatus extends BaseObservableValue { constructor() { diff --git a/src/utils/AbortableOperation.ts b/src/utils/AbortableOperation.ts index fba71a8c..b3f663bd 100644 --- a/src/utils/AbortableOperation.ts +++ b/src/utils/AbortableOperation.ts @@ -14,7 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {BaseObservableValue, ObservableValue} from "../observable/ObservableValue"; +import {BaseObservableValue} from "../observable/value/BaseObservableValue"; +import {ObservableValue} from "../observable/value/ObservableValue"; export interface IAbortable { abort(); From 6daae797e5e38589d8cc215e7d3f3abc8313c033 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 17 Mar 2022 11:31:12 +0100 Subject: [PATCH 020/435] fix some ts/lint errors --- src/matrix/DeviceMessageHandler.js | 2 +- src/matrix/Session.js | 7 +++++++ src/matrix/calls/CallHandler.ts | 2 +- src/matrix/calls/PeerCall.ts | 2 +- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index 91ef82f6..80fd1592 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -16,7 +16,7 @@ limitations under the License. import {OLM_ALGORITHM} from "./e2ee/common.js"; import {countBy, groupBy} from "../utils/groupBy"; -import {LRUCache} from "../../utils/LRUCache"; +import {LRUCache} from "../utils/LRUCache"; export class DeviceMessageHandler { constructor({storage, callHandler}) { diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 69cd9ee0..dbc0b17e 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -47,6 +47,7 @@ import { import {SecretStorage} from "./ssss/SecretStorage"; import {ObservableValue} from "../observable/value/ObservableValue"; import {RetainedObservableValue} from "../observable/value/RetainedObservableValue"; +import {CallHandler} from "./calls/CallHandler"; const PICKLE_KEY = "DEFAULT_KEY"; const PUSHER_KEY = "pusher"; @@ -81,6 +82,8 @@ export class Session { if (!this._deviceTracker || !this._olmEncryption) { throw new Error("encryption is not enabled"); } + // TODO: just get the devices we're sending the message to, not all the room devices + // although we probably already fetched all devices to send messages in the likely e2ee room await this._deviceTracker.trackRoom(roomId, log); const devices = await this._deviceTracker.devicesForTrackedRoom(roomId, this._hsApi, log); const encryptedMessage = await this._olmEncryption.encrypt(message.type, message.content, devices, this._hsApi, log); @@ -132,6 +135,10 @@ export class Session { return this._sessionInfo.userId; } + get callHandler() { + return this._callHandler; + } + // called once this._e2eeAccount is assigned _setupEncryption() { // TODO: this should all go in a wrapper in e2ee/ that is bootstrapped by passing in the account diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index a0cd8473..a36b17f6 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -37,7 +37,7 @@ const CALL_TERMINATED = "m.terminated"; export type Options = Omit; -export class GroupCallHandler { +export class CallHandler { // group calls by call id private readonly _calls: ObservableMap = new ObservableMap(); // map of userId to set of conf_id's they are in diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index a4ea8cca..9a702ebd 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -875,7 +875,7 @@ const FALLBACK_ICE_SERVER = 'stun:turn.matrix.org'; /** The length of time a call can be ringing for. */ const CALL_TIMEOUT_MS = 60000; -const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); +//const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); export class CallError extends Error { code: string; From e482e3aeef0af29171418592b30d0b2df2150b40 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 17 Mar 2022 11:31:36 +0100 Subject: [PATCH 021/435] expose mediaDevices and webRTC from platform --- src/platform/web/Platform.js | 4 ++++ src/platform/web/dom/MediaDevices.ts | 2 +- src/platform/web/dom/WebRTC.ts | 6 ++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/platform/web/Platform.js b/src/platform/web/Platform.js index 984bc45c..7426b138 100644 --- a/src/platform/web/Platform.js +++ b/src/platform/web/Platform.js @@ -38,6 +38,8 @@ import {downloadInIframe} from "./dom/download.js"; import {Disposables} from "../../utils/Disposables"; import {parseHTML} from "./parsehtml.js"; import {handleAvatarError} from "./ui/avatar"; +import {MediaDevicesWrapper} from "./dom/MediaDevices"; +import {DOMWebRTC} from "./dom/WebRTC"; function addScript(src) { return new Promise(function (resolve, reject) { @@ -163,6 +165,8 @@ export class Platform { this._disposables = new Disposables(); this._olmPromise = undefined; this._workerPromise = undefined; + this.mediaDevices = new MediaDevicesWrapper(navigator.mediaDevices); + this.webRTC = new DOMWebRTC(); } _createLogger(isDevelopment) { diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index 445ff22d..22f3d634 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -21,7 +21,7 @@ const POLLING_INTERVAL = 200; // ms export const SPEAKING_THRESHOLD = -60; // dB const SPEAKING_SAMPLE_COUNT = 8; // samples -class MediaDevicesWrapper implements IMediaDevices { +export class MediaDevicesWrapper implements IMediaDevices { constructor(private readonly mediaDevices: MediaDevices) {} enumerate(): Promise { diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 0025dfec..39a6f9d2 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -23,6 +23,12 @@ const POLLING_INTERVAL = 200; // ms export const SPEAKING_THRESHOLD = -60; // dB const SPEAKING_SAMPLE_COUNT = 8; // samples +export class DOMWebRTC implements WebRTC { + createPeerConnection(handler: PeerConnectionHandler): PeerConnection { + return new DOMPeerConnection(handler, false, []); + } +} + class DOMPeerConnection implements PeerConnection { private readonly peerConnection: RTCPeerConnection; private readonly handler: PeerConnectionHandler; From e760b8e5567425d016e88fe64ebf4b04aa66871a Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 17 Mar 2022 13:04:14 +0100 Subject: [PATCH 022/435] basic view model setup --- src/domain/session/SessionViewModel.js | 2 +- src/domain/session/room/CallViewModel.ts | 34 +++++++++++++++++++ src/domain/session/room/RoomViewModel.js | 29 +++++++++++++++- src/matrix/calls/group/GroupCall.ts | 6 +++- ...bservable.ts => PickMapObservableValue.ts} | 12 +++---- src/platform/web/ui/session/room/RoomView.js | 4 +++ 6 files changed, 78 insertions(+), 9 deletions(-) create mode 100644 src/domain/session/room/CallViewModel.ts rename src/observable/value/{PickMapObservable.ts => PickMapObservableValue.ts} (87%) diff --git a/src/domain/session/SessionViewModel.js b/src/domain/session/SessionViewModel.js index a67df3a7..0c9dddaa 100644 --- a/src/domain/session/SessionViewModel.js +++ b/src/domain/session/SessionViewModel.js @@ -174,7 +174,7 @@ export class SessionViewModel extends ViewModel { _createRoomViewModelInstance(roomId) { const room = this._client.session.rooms.get(roomId); if (room) { - const roomVM = new RoomViewModel(this.childOptions({room})); + const roomVM = new RoomViewModel(this.childOptions({room, session: this._client.session})); roomVM.load(); return roomVM; } diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts new file mode 100644 index 00000000..52f04d32 --- /dev/null +++ b/src/domain/session/room/CallViewModel.ts @@ -0,0 +1,34 @@ +/* +Copyright 2020 Bruno Windels +Copyright 2020 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {ViewModel} from "../../ViewModel"; +import type {GroupCall} from "../../../matrix/calls/group/GroupCall"; + +export class CallViewModel extends ViewModel { + + private call: GroupCall; + + constructor(options) { + super(options); + const {call} = options; + this.call = call; + } + + get name(): string { + return this.call.name; + } +} diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 71060728..4540fdb1 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -17,15 +17,18 @@ limitations under the License. import {TimelineViewModel} from "./timeline/TimelineViewModel.js"; import {ComposerViewModel} from "./ComposerViewModel.js" +import {CallViewModel} from "./CallViewModel" +import {PickMapObservableValue} from "../../../observable/value/PickMapObservableValue"; import {avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../avatar"; import {tilesCreator} from "./timeline/tilesCreator.js"; import {ViewModel} from "../../ViewModel"; import {imageToInfo} from "../common.js"; +import {LocalMedia} from "../../../matrix/calls/LocalMedia"; export class RoomViewModel extends ViewModel { constructor(options) { super(options); - const {room} = options; + const {room, session} = options; this._room = room; this._timelineVM = null; this._tilesCreator = null; @@ -40,6 +43,19 @@ export class RoomViewModel extends ViewModel { } this._clearUnreadTimout = null; this._closeUrl = this.urlCreator.urlUntilSegment("session"); + // pick call for this room with lowest key + this._callObservable = new PickMapObservableValue(session.callHandler.calls.filterValues(c => c.roomId === this.roomId)); + this._callViewModel = undefined; + this.track(this._callObservable.subscribe(call => { + this._callViewModel = this.disposeTracked(this._callViewModel); + if (call) { + this._callViewModel = new CallViewModel(this.childOptions({call})); + } + this.emitChange("callViewModel"); + })); + if (this._callObservable.get()) { + this._callViewModel = new CallViewModel(this.childOptions({call: this._callObservable.get()})); + } } async load() { @@ -308,6 +324,10 @@ export class RoomViewModel extends ViewModel { return this._composerVM; } + get callViewModel() { + return this._callViewModel; + } + openDetailsPanel() { let path = this.navigation.path.until("room"); path = path.with(this.navigation.segment("right-panel", true)); @@ -320,6 +340,13 @@ export class RoomViewModel extends ViewModel { this._composerVM.setReplyingTo(entry); } } + + async startCall() { + const mediaTracks = await this.platform.mediaDevices.getMediaTracks(true, true); + const localMedia = LocalMedia.fromTracks(mediaTracks); + // this will set the callViewModel above as a call will be added to callHandler.calls + await this.session.callHandler.createCall(this.roomId, localMedia, "A call " + Math.round(this.platform.random() * 100)); + } } function videoToInfo(video) { diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 6c26c995..8ea4528a 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -56,7 +56,7 @@ export class GroupCall { constructor( id: string | undefined, private callContent: Record | undefined, - private readonly roomId: string, + public readonly roomId: string, private readonly options: Options ) { this.id = id ?? makeId("conf-"); @@ -77,6 +77,10 @@ export class GroupCall { return this.callContent?.["m.terminated"] === true; } + get name(): string { + return this.callContent?.["m.name"]; + } + async join(localMedia: LocalMedia) { if (this._state !== GroupCallState.Created) { return; diff --git a/src/observable/value/PickMapObservable.ts b/src/observable/value/PickMapObservableValue.ts similarity index 87% rename from src/observable/value/PickMapObservable.ts rename to src/observable/value/PickMapObservableValue.ts index 835cdf90..a20ec04e 100644 --- a/src/observable/value/PickMapObservable.ts +++ b/src/observable/value/PickMapObservableValue.ts @@ -22,7 +22,7 @@ function pickLowestKey(currentKey: K, newKey: K): boolean { return newKey < currentKey; } -export class PickMapObservable implements IMapObserver extends BaseObservableValue { +export class PickMapObservableValue extends BaseObservableValue implements IMapObserver{ private key?: K; private mapSubscription?: SubscriptionHandle; @@ -34,7 +34,7 @@ export class PickMapObservable implements IMapObserver extends BaseO super(); } - private trySetKey(newKey: K): boolean { + private updateKey(newKey: K): boolean { if (this.key === undefined || this.pickKey(this.key, newKey)) { this.key = newKey; return true; @@ -48,7 +48,7 @@ export class PickMapObservable implements IMapObserver extends BaseO } onAdd(key: K, value:V): void { - if (this.trySetKey(key)) { + if (this.updateKey(key)) { this.emit(this.get()); } } @@ -60,7 +60,7 @@ export class PickMapObservable implements IMapObserver extends BaseO this.key = undefined; let changed = false; for (const [key] of this.map) { - changed = this.trySetKey(key) || changed; + changed = this.updateKey(key) || changed; } if (changed) { this.emit(this.get()); @@ -71,12 +71,12 @@ export class PickMapObservable implements IMapObserver extends BaseO onSubscribeFirst(): void { this.mapSubscription = this.map.subscribe(this); for (const [key] of this.map) { - this.trySetKey(key); + this.updateKey(key); } } onUnsubscribeLast(): void { - this.mapSubscription(); + this.mapSubscription!(); this.key = undefined; } diff --git a/src/platform/web/ui/session/room/RoomView.js b/src/platform/web/ui/session/room/RoomView.js index c172766a..541cc4d8 100644 --- a/src/platform/web/ui/session/room/RoomView.js +++ b/src/platform/web/ui/session/room/RoomView.js @@ -52,6 +52,9 @@ export class RoomView extends TemplateView { ]), t.div({className: "RoomView_body"}, [ t.div({className: "RoomView_error"}, vm => vm.error), + t.map(vm => vm.callViewModel, (callViewModel, t) => { + return t.p(["A call is in progress", callViewModel => callViewModel.name]) + }), t.mapView(vm => vm.timelineViewModel, timelineViewModel => { return timelineViewModel ? new TimelineView(timelineViewModel) : @@ -69,6 +72,7 @@ export class RoomView extends TemplateView { const vm = this.value; const options = []; options.push(Menu.option(vm.i18n`Room details`, () => vm.openDetailsPanel())) + options.push(Menu.option(vm.i18n`Start call`, () => vm.startCall())) if (vm.canLeave) { options.push(Menu.option(vm.i18n`Leave room`, () => this._confirmToLeaveRoom()).setDestructive()); } From 4be82cd4722db5ee2e5bfd5b675732adc5df288b Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 17 Mar 2022 13:07:55 +0100 Subject: [PATCH 023/435] WIP on UI --- src/domain/session/room/CallViewModel.ts | 37 +++++++++++++---- src/domain/session/room/RoomViewModel.js | 9 +++- src/platform/web/ui/session/room/CallView.ts | 43 ++++++++++++++++++++ src/platform/web/ui/session/room/RoomView.js | 5 ++- 4 files changed, 81 insertions(+), 13 deletions(-) create mode 100644 src/platform/web/ui/session/room/CallView.ts diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 52f04d32..0c5f28d2 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -1,6 +1,5 @@ /* -Copyright 2020 Bruno Windels -Copyright 2020 The Matrix.org Foundation C.I.C. +Copyright 2022 The Matrix.org Foundation C.I.C. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -15,20 +14,40 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {ViewModel} from "../../ViewModel"; +import {ViewModel, Options as BaseOptions} from "../../ViewModel"; import type {GroupCall} from "../../../matrix/calls/group/GroupCall"; +import type {Member} from "../../../matrix/calls/group/Member"; +import type {BaseObservableList} from "../../../observable/list/BaseObservableList"; +import type {Track} from "../../../platform/types/MediaDevices"; -export class CallViewModel extends ViewModel { +type Options = BaseOptions & {call: GroupCall}; - private call: GroupCall; +export class CallViewModel extends ViewModel { + + public readonly memberViewModels: BaseObservableList; - constructor(options) { + constructor(options: Options) { super(options); - const {call} = options; - this.call = call; + this.memberViewModels = this.getOption("call").members + .mapValues(member => new CallMemberViewModel(this.childOptions({member}))) + .sortValues((a, b) => { + + }); } get name(): string { - return this.call.name; + return this.getOption("call").name; + } + + get localTracks(): Track[] { + return this.getOption("call").localMedia?.tracks ?? []; + } +} + +type MemberOptions = BaseOptions & {member: Member}; + +class CallMemberViewModel extends ViewModel { + get tracks(): Track[] { + return this.getOption("member").remoteTracks; } } diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 4540fdb1..f79be3c0 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -43,6 +43,10 @@ export class RoomViewModel extends ViewModel { } this._clearUnreadTimout = null; this._closeUrl = this.urlCreator.urlUntilSegment("session"); + this._setupCallViewModel(); + } + + _setupCallViewModel() { // pick call for this room with lowest key this._callObservable = new PickMapObservableValue(session.callHandler.calls.filterValues(c => c.roomId === this.roomId)); this._callViewModel = undefined; @@ -53,8 +57,9 @@ export class RoomViewModel extends ViewModel { } this.emitChange("callViewModel"); })); - if (this._callObservable.get()) { - this._callViewModel = new CallViewModel(this.childOptions({call: this._callObservable.get()})); + const call = this._callObservable.get(); + if (call) { + this._callViewModel = new CallViewModel(this.childOptions({call})); } } diff --git a/src/platform/web/ui/session/room/CallView.ts b/src/platform/web/ui/session/room/CallView.ts new file mode 100644 index 00000000..2c10f82f --- /dev/null +++ b/src/platform/web/ui/session/room/CallView.ts @@ -0,0 +1,43 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {TemplateView, TemplateBuilder} from "../../general/TemplateView"; +import {Track, TrackType} from "../../../../types/MediaDevices"; +import type {TrackWrapper} from "../../../dom/MediaDevices"; +import type {CallViewModel} from "../../../../../domain/session/room/CallViewModel"; + +function bindVideoTracks(t: TemplateBuilder, video: HTMLVideoElement, propSelector: (vm: T) => Track[]) { + t.mapSideEffect(propSelector, tracks => { + if (tracks.length) { + video.srcObject = (tracks[0] as TrackWrapper).stream; + } + }); +} + +export class CallView extends TemplateView { + render(t: TemplateBuilder, vm: CallViewModel): HTMLElement { + return t.div({class: "CallView"}, [ + t.div({class: "CallView_me"}, bindVideoTracks(t, t.video(), vm => vm.localTracks)), + t.view(new ListView(vm.memberViewModels, vm => new MemberView(vm))) + ]); + } +} + +class MemberView extends TemplateView { + render(t, vm) { + return bindVideoTracks(t, t.video(), vm => vm.tracks); + } +} diff --git a/src/platform/web/ui/session/room/RoomView.js b/src/platform/web/ui/session/room/RoomView.js index 541cc4d8..4d010a7c 100644 --- a/src/platform/web/ui/session/room/RoomView.js +++ b/src/platform/web/ui/session/room/RoomView.js @@ -23,6 +23,7 @@ import {TimelineLoadingView} from "./TimelineLoadingView.js"; import {MessageComposer} from "./MessageComposer.js"; import {RoomArchivedView} from "./RoomArchivedView.js"; import {AvatarView} from "../../AvatarView.js"; +import {CallView} from "./CallView"; export class RoomView extends TemplateView { constructor(options) { @@ -52,8 +53,8 @@ export class RoomView extends TemplateView { ]), t.div({className: "RoomView_body"}, [ t.div({className: "RoomView_error"}, vm => vm.error), - t.map(vm => vm.callViewModel, (callViewModel, t) => { - return t.p(["A call is in progress", callViewModel => callViewModel.name]) + t.mapView(vm => vm.callViewModel, callViewModel => { + return new CallView(callViewModel); }), t.mapView(vm => vm.timelineViewModel, timelineViewModel => { return timelineViewModel ? From cad2aa760d73556d862429e87492fdc23fd79593 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 21 Mar 2022 16:30:13 +0100 Subject: [PATCH 024/435] some fixes --- src/domain/session/room/CallViewModel.ts | 20 ++++++++++++++++---- src/domain/session/room/RoomViewModel.js | 8 +++++--- src/matrix/calls/CallHandler.ts | 1 + src/matrix/calls/LocalMedia.ts | 8 +++++++- src/matrix/room/Room.js | 15 ++++++--------- src/platform/web/ui/session/room/CallView.ts | 10 +++++++--- src/platform/web/ui/session/room/RoomView.js | 4 +--- 7 files changed, 43 insertions(+), 23 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 0c5f28d2..95d9b188 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -30,24 +30,36 @@ export class CallViewModel extends ViewModel { super(options); this.memberViewModels = this.getOption("call").members .mapValues(member => new CallMemberViewModel(this.childOptions({member}))) - .sortValues((a, b) => { - - }); + .sortValues((a, b) => a.compare(b)); } get name(): string { return this.getOption("call").name; } + get id(): string { + return this.getOption("call").id; + } + get localTracks(): Track[] { + console.log("localTracks", this.getOption("call").localMedia); return this.getOption("call").localMedia?.tracks ?? []; } } type MemberOptions = BaseOptions & {member: Member}; -class CallMemberViewModel extends ViewModel { +export class CallMemberViewModel extends ViewModel { get tracks(): Track[] { return this.getOption("member").remoteTracks; } + + compare(other: CallMemberViewModel): number { + const myUserId = this.getOption("member").member.userId; + const otherUserId = other.getOption("member").member.userId; + if(myUserId === otherUserId) { + return 0; + } + return myUserId < otherUserId ? -1 : 1; + } } diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index f79be3c0..1d4bca42 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -48,7 +48,7 @@ export class RoomViewModel extends ViewModel { _setupCallViewModel() { // pick call for this room with lowest key - this._callObservable = new PickMapObservableValue(session.callHandler.calls.filterValues(c => c.roomId === this.roomId)); + this._callObservable = new PickMapObservableValue(this.getOption("session").callHandler.calls.filterValues(c => c.roomId === this._room.id)); this._callViewModel = undefined; this.track(this._callObservable.subscribe(call => { this._callViewModel = this.disposeTracked(this._callViewModel); @@ -347,10 +347,12 @@ export class RoomViewModel extends ViewModel { } async startCall() { + const session = this.getOption("session"); const mediaTracks = await this.platform.mediaDevices.getMediaTracks(true, true); - const localMedia = LocalMedia.fromTracks(mediaTracks); + const localMedia = new LocalMedia().withTracks(mediaTracks); + console.log("localMedia", localMedia.tracks); // this will set the callViewModel above as a call will be added to callHandler.calls - await this.session.callHandler.createCall(this.roomId, localMedia, "A call " + Math.round(this.platform.random() * 100)); + await session.callHandler.createCall(this._room.id, localMedia, "A call " + Math.round(this.platform.random() * 100)); } } diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index a36b17f6..a9b59f4f 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -52,6 +52,7 @@ export class CallHandler { async createCall(roomId: string, localMedia: LocalMedia, name: string): Promise { const call = new GroupCall(undefined, undefined, roomId, this.groupCallOptions); + console.log("created call with id", call.id); this._calls.set(call.id, call); try { await call.create(localMedia, name); diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 4f5d3b7a..1fd4aad6 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -35,7 +35,13 @@ export class LocalMedia { return new LocalMedia(cameraTrack, screenShareTrack, microphoneTrack as AudioTrack); } - get tracks(): Track[] { return []; } + get tracks(): Track[] { + const tracks: Track[] = []; + if (this.cameraTrack) { tracks.push(this.cameraTrack); } + if (this.screenShareTrack) { tracks.push(this.screenShareTrack); } + if (this.microphoneTrack) { tracks.push(this.microphoneTrack); } + return tracks; + } getSDPMetadata(): SDPStreamMetadata { const metadata = {}; diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js index d0fcbc84..ca11cdd8 100644 --- a/src/matrix/room/Room.js +++ b/src/matrix/room/Room.js @@ -93,7 +93,7 @@ export class Room extends BaseRoom { } } - this._updateCallHandler(roomResponse); + this._updateCallHandler(roomResponse, log); return { roomEncryption, @@ -448,20 +448,17 @@ export class Room extends BaseRoom { return this._sendQueue.pendingEvents; } - _updateCallHandler(roomResponse) { + _updateCallHandler(roomResponse, log) { if (this._callHandler) { const stateEvents = roomResponse.state?.events; if (stateEvents) { - for (const e of stateEvents) { - this._callHandler.handleRoomState(this, e); - } + this._callHandler.handleRoomState(this, stateEvents, log); } let timelineEvents = roomResponse.timeline?.events; if (timelineEvents) { - for (const e of timelineEvents) { - if (typeof e.state_key === "string") { - this._callHandler.handleRoomState(this, e); - } + const timelineStateEvents = timelineEvents.filter(e => typeof e.state_key === "string"); + if (timelineEvents.length !== 0) { + this._callHandler.handleRoomState(this, timelineStateEvents, log); } } } diff --git a/src/platform/web/ui/session/room/CallView.ts b/src/platform/web/ui/session/room/CallView.ts index 2c10f82f..29d4eee9 100644 --- a/src/platform/web/ui/session/room/CallView.ts +++ b/src/platform/web/ui/session/room/CallView.ts @@ -15,29 +15,33 @@ limitations under the License. */ import {TemplateView, TemplateBuilder} from "../../general/TemplateView"; +import {ListView} from "../../general/ListView"; import {Track, TrackType} from "../../../../types/MediaDevices"; import type {TrackWrapper} from "../../../dom/MediaDevices"; -import type {CallViewModel} from "../../../../../domain/session/room/CallViewModel"; +import type {CallViewModel, CallMemberViewModel} from "../../../../../domain/session/room/CallViewModel"; function bindVideoTracks(t: TemplateBuilder, video: HTMLVideoElement, propSelector: (vm: T) => Track[]) { t.mapSideEffect(propSelector, tracks => { + console.log("tracks", tracks); if (tracks.length) { video.srcObject = (tracks[0] as TrackWrapper).stream; } }); + return video; } export class CallView extends TemplateView { render(t: TemplateBuilder, vm: CallViewModel): HTMLElement { return t.div({class: "CallView"}, [ + t.p(`Call ${vm.name} (${vm.id})`), t.div({class: "CallView_me"}, bindVideoTracks(t, t.video(), vm => vm.localTracks)), - t.view(new ListView(vm.memberViewModels, vm => new MemberView(vm))) + t.view(new ListView({list: vm.memberViewModels}, vm => new MemberView(vm))) ]); } } class MemberView extends TemplateView { - render(t, vm) { + render(t: TemplateBuilder, vm: CallMemberViewModel) { return bindVideoTracks(t, t.video(), vm => vm.tracks); } } diff --git a/src/platform/web/ui/session/room/RoomView.js b/src/platform/web/ui/session/room/RoomView.js index 4d010a7c..2936adf3 100644 --- a/src/platform/web/ui/session/room/RoomView.js +++ b/src/platform/web/ui/session/room/RoomView.js @@ -53,9 +53,7 @@ export class RoomView extends TemplateView { ]), t.div({className: "RoomView_body"}, [ t.div({className: "RoomView_error"}, vm => vm.error), - t.mapView(vm => vm.callViewModel, callViewModel => { - return new CallView(callViewModel); - }), + t.mapView(vm => vm.callViewModel, callViewModel => callViewModel ? new CallView(callViewModel) : null), t.mapView(vm => vm.timelineViewModel, timelineViewModel => { return timelineViewModel ? new TimelineView(timelineViewModel) : From 9efd191f4eccb5d00b06ee2c4804294c6d5168d2 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 22 Mar 2022 20:29:31 +0100 Subject: [PATCH 025/435] some more fixes --- src/domain/session/room/CallViewModel.ts | 1 - src/domain/session/room/RoomViewModel.js | 17 +++++++++++------ src/matrix/Session.js | 6 +++++- src/matrix/calls/CallHandler.ts | 3 +++ src/matrix/calls/group/GroupCall.ts | 5 ++++- src/observable/value/PickMapObservableValue.ts | 4 +++- src/platform/web/ui/session/room/CallView.ts | 6 +++--- 7 files changed, 29 insertions(+), 13 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 95d9b188..7fcabf02 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -42,7 +42,6 @@ export class CallViewModel extends ViewModel { } get localTracks(): Track[] { - console.log("localTracks", this.getOption("call").localMedia); return this.getOption("call").localMedia?.tracks ?? []; } } diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 1d4bca42..81ca58d2 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -347,12 +347,17 @@ export class RoomViewModel extends ViewModel { } async startCall() { - const session = this.getOption("session"); - const mediaTracks = await this.platform.mediaDevices.getMediaTracks(true, true); - const localMedia = new LocalMedia().withTracks(mediaTracks); - console.log("localMedia", localMedia.tracks); - // this will set the callViewModel above as a call will be added to callHandler.calls - await session.callHandler.createCall(this._room.id, localMedia, "A call " + Math.round(this.platform.random() * 100)); + try { + const session = this.getOption("session"); + const mediaTracks = await this.platform.mediaDevices.getMediaTracks(true, true); + const localMedia = new LocalMedia().withTracks(mediaTracks); + console.log("localMedia", localMedia.tracks); + // this will set the callViewModel above as a call will be added to callHandler.calls + await session.callHandler.createCall(this._room.id, localMedia, "A call " + Math.round(this.platform.random() * 100)); + } catch (err) { + console.error(err.stack); + alert(err.message); + } } } diff --git a/src/matrix/Session.js b/src/matrix/Session.js index dbc0b17e..e604e068 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -88,7 +88,11 @@ export class Session { const devices = await this._deviceTracker.devicesForTrackedRoom(roomId, this._hsApi, log); const encryptedMessage = await this._olmEncryption.encrypt(message.type, message.content, devices, this._hsApi, log); return encryptedMessage; - } + }, + storage: this._storage, + webRTC: this._platform.webRTC, + ownDeviceId: sessionInfo.deviceId, + ownUserId: sessionInfo.userId, }); this._deviceMessageHandler = new DeviceMessageHandler({storage, callHandler: this._callHandler}); this._olm = olm; diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index a9b59f4f..269f6020 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -63,7 +63,9 @@ export class CallHandler { } throw err; } + console.log("joining call I just created"); await call.join(localMedia); + console.log("joined!"); return call; } @@ -73,6 +75,7 @@ export class CallHandler { /** @internal */ handleRoomState(room: Room, events: StateEvent[], log: ILogItem) { + console.log("handling room state"); // first update call events for (const event of events) { if (event.type === EventType.GroupCall) { diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 8ea4528a..d22135d1 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -49,7 +49,7 @@ export type Options = Omit = new ObservableMap(); - private _localMedia?: LocalMedia; + private _localMedia?: LocalMedia = undefined; private _memberOptions: MemberOptions; private _state: GroupCallState; @@ -87,10 +87,12 @@ export class GroupCall { } this._state = GroupCallState.Joining; this._localMedia = localMedia; + this.options.emitUpdate(this); const memberContent = await this._joinCallMemberContent(); // send m.call.member state event const request = this.options.hsApi.sendState(this.roomId, "m.call.member", this.options.ownUserId, memberContent); await request.response(); + this.options.emitUpdate(this); // send invite to all members that are < my userId for (const [,member] of this._members) { member.connect(this._localMedia); @@ -119,6 +121,7 @@ export class GroupCall { }; const request = this.options.hsApi.sendState(this.roomId, "m.call", this.id, this.callContent); await request.response(); + this._state = GroupCallState.Created; } /** @internal */ diff --git a/src/observable/value/PickMapObservableValue.ts b/src/observable/value/PickMapObservableValue.ts index a20ec04e..67f9d562 100644 --- a/src/observable/value/PickMapObservableValue.ts +++ b/src/observable/value/PickMapObservableValue.ts @@ -53,7 +53,9 @@ export class PickMapObservableValue extends BaseObservableValue(t: TemplateBuilder, video: HTMLVideoElement, prop export class CallView extends TemplateView { render(t: TemplateBuilder, vm: CallViewModel): HTMLElement { return t.div({class: "CallView"}, [ - t.p(`Call ${vm.name} (${vm.id})`), - t.div({class: "CallView_me"}, bindVideoTracks(t, t.video(), vm => vm.localTracks)), + t.p(["Call ", vm => vm.name, vm => ` (${vm.id})`]), + t.div({class: "CallView_me"}, bindVideoTracks(t, t.video({autoplay: true}), vm => vm.localTracks)), t.view(new ListView({list: vm.memberViewModels}, vm => new MemberView(vm))) ]); } @@ -42,6 +42,6 @@ export class CallView extends TemplateView { class MemberView extends TemplateView { render(t: TemplateBuilder, vm: CallMemberViewModel) { - return bindVideoTracks(t, t.video(), vm => vm.tracks); + return bindVideoTracks(t, t.video({autoplay: true}), vm => vm.tracks); } } From 0a37fd561e3c713faf4639d3413918d5bd111d06 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 23 Mar 2022 12:23:10 +0100 Subject: [PATCH 026/435] just enough view code to join a call --- src/domain/session/room/RoomViewModel.js | 7 +-- .../session/room/timeline/tiles/CallTile.js | 48 +++++++++++++++++++ .../session/room/timeline/tilesCreator.js | 3 ++ src/matrix/DeviceMessageHandler.js | 1 + src/matrix/Session.js | 7 +-- src/matrix/Sync.js | 1 + src/matrix/calls/PeerCall.ts | 1 + src/matrix/calls/group/GroupCall.ts | 15 ++++-- src/matrix/calls/group/Member.ts | 14 +++--- src/matrix/common.js | 17 +++++++ src/matrix/e2ee/RoomEncryption.js | 14 ++---- src/platform/web/ui/session/room/common.ts | 3 ++ .../ui/session/room/timeline/CallTileView.ts | 38 +++++++++++++++ 13 files changed, 141 insertions(+), 28 deletions(-) create mode 100644 src/domain/session/room/timeline/tiles/CallTile.js create mode 100644 src/platform/web/ui/session/room/timeline/CallTileView.ts diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 81ca58d2..2b47673f 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -48,7 +48,8 @@ export class RoomViewModel extends ViewModel { _setupCallViewModel() { // pick call for this room with lowest key - this._callObservable = new PickMapObservableValue(this.getOption("session").callHandler.calls.filterValues(c => c.roomId === this._room.id)); + const calls = this.getOption("session").callHandler.calls; + this._callObservable = new PickMapObservableValue(calls.filterValues(c => c.roomId === this._room.id && c.hasJoined)); this._callViewModel = undefined; this.track(this._callObservable.subscribe(call => { this._callViewModel = this.disposeTracked(this._callViewModel); @@ -68,6 +69,7 @@ export class RoomViewModel extends ViewModel { try { const timeline = await this._room.openTimeline(); this._tilesCreator = tilesCreator(this.childOptions({ + session: this.getOption("session"), roomVM: this, timeline, })); @@ -349,9 +351,8 @@ export class RoomViewModel extends ViewModel { async startCall() { try { const session = this.getOption("session"); - const mediaTracks = await this.platform.mediaDevices.getMediaTracks(true, true); + const mediaTracks = await this.platform.mediaDevices.getMediaTracks(false, true); const localMedia = new LocalMedia().withTracks(mediaTracks); - console.log("localMedia", localMedia.tracks); // this will set the callViewModel above as a call will be added to callHandler.calls await session.callHandler.createCall(this._room.id, localMedia, "A call " + Math.round(this.platform.random() * 100)); } catch (err) { diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js new file mode 100644 index 00000000..7129bb75 --- /dev/null +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -0,0 +1,48 @@ +/* +Copyright 2020 Bruno Windels + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {SimpleTile} from "./SimpleTile.js"; +import {LocalMedia} from "../../../../../matrix/calls/LocalMedia"; + +// TODO: timeline entries for state events with the same state key and type +// should also update previous entries in the timeline, so we can update the name of the call, whether it is terminated, etc ... + +// alternatively, we could just subscribe to the GroupCall and spontanously emit an update when it updates + +export class CallTile extends SimpleTile { + + get shape() { + return "call"; + } + + get name() { + return this._entry.content["m.name"]; + } + + get _call() { + const calls = this.getOption("session").callHandler.calls; + return calls.get(this._entry.stateKey); + } + + async join() { + const call = this._call; + if (call) { + const mediaTracks = await this.platform.mediaDevices.getMediaTracks(false, true); + const localMedia = new LocalMedia().withTracks(mediaTracks); + await call.join(localMedia); + } + } +} diff --git a/src/domain/session/room/timeline/tilesCreator.js b/src/domain/session/room/timeline/tilesCreator.js index dc9a850e..659a5e76 100644 --- a/src/domain/session/room/timeline/tilesCreator.js +++ b/src/domain/session/room/timeline/tilesCreator.js @@ -26,6 +26,7 @@ import {RoomMemberTile} from "./tiles/RoomMemberTile.js"; import {EncryptedEventTile} from "./tiles/EncryptedEventTile.js"; import {EncryptionEnabledTile} from "./tiles/EncryptionEnabledTile.js"; import {MissingAttachmentTile} from "./tiles/MissingAttachmentTile.js"; +import {CallTile} from "./tiles/CallTile.js"; export function tilesCreator(baseOptions) { const tilesCreator = function tilesCreator(entry, emitUpdate) { @@ -71,6 +72,8 @@ export function tilesCreator(baseOptions) { return new EncryptedEventTile(options); case "m.room.encryption": return new EncryptionEnabledTile(options); + case "m.call": + return entry.stateKey ? new CallTile(options) : null; default: // unknown type not rendered return null; diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index 80fd1592..11c10750 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -86,6 +86,7 @@ export class DeviceMessageHandler { this._senderDeviceCache.set(device); } } + console.log("incoming device message", senderKey, device, this._senderDeviceCache); return device; } } diff --git a/src/matrix/Session.js b/src/matrix/Session.js index e604e068..27689013 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -78,14 +78,15 @@ export class Session { this._callHandler = new CallHandler({ createTimeout: this._platform.clock.createTimeout, hsApi: this._hsApi, - encryptDeviceMessage: async (roomId, message, log) => { + encryptDeviceMessage: async (roomId, userId, message, log) => { if (!this._deviceTracker || !this._olmEncryption) { throw new Error("encryption is not enabled"); } // TODO: just get the devices we're sending the message to, not all the room devices // although we probably already fetched all devices to send messages in the likely e2ee room - await this._deviceTracker.trackRoom(roomId, log); - const devices = await this._deviceTracker.devicesForTrackedRoom(roomId, this._hsApi, log); + await this._deviceTracker.trackRoom(this.rooms.get(roomId), log); + const devices = await this._deviceTracker.devicesForRoomMembers(roomId, [userId], this._hsApi, log); + console.log("devices", devices); const encryptedMessage = await this._olmEncryption.encrypt(message.type, message.content, devices, this._hsApi, log); return encryptedMessage; }, diff --git a/src/matrix/Sync.js b/src/matrix/Sync.js index 8e880def..b4ea702f 100644 --- a/src/matrix/Sync.js +++ b/src/matrix/Sync.js @@ -224,6 +224,7 @@ export class Sync { _openPrepareSyncTxn() { const storeNames = this._storage.storeNames; return this._storage.readTxn([ + storeNames.deviceIdentities, // to read device from olm messages storeNames.olmSessions, storeNames.inboundGroupSessions, // to read fragments when loading sync writer when rejoining archived room diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 9a702ebd..319fde3a 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -114,6 +114,7 @@ export class PeerCall implements IDisposable { } }); this.logger = { + info(...args) { console.info.apply(console, ["WebRTC debug:", ...args])}, debug(...args) { console.log.apply(console, ["WebRTC debug:", ...args])}, log(...args) { console.log.apply(console, ["WebRTC log:", ...args])}, warn(...args) { console.log.apply(console, ["WebRTC warn:", ...args])}, diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index d22135d1..901eb3b3 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -41,7 +41,7 @@ export enum GroupCallState { export type Options = Omit & { emitUpdate: (call: GroupCall, params?: any) => void; - encryptDeviceMessage: (roomId: string, message: SignallingMessage, log: ILogItem) => Promise, + encryptDeviceMessage: (roomId: string, userId: string, message: SignallingMessage, log: ILogItem) => Promise, storage: Storage, ownDeviceId: string }; @@ -61,13 +61,13 @@ export class GroupCall { ) { this.id = id ?? makeId("conf-"); this._state = id ? GroupCallState.Created : GroupCallState.Fledgling; - this._memberOptions = Object.assign({ + this._memberOptions = Object.assign({}, options, { confId: this.id, emitUpdate: member => this._members.update(member.member.userId, member), - encryptDeviceMessage: (message: SignallingMessage, log) => { - return this.options.encryptDeviceMessage(this.roomId, message, log); + encryptDeviceMessage: (userId: string, message: SignallingMessage, log) => { + return this.options.encryptDeviceMessage(this.roomId, userId, message, log); } - }, options); + }); } get localMedia(): LocalMedia | undefined { return this._localMedia; } @@ -99,6 +99,10 @@ export class GroupCall { } } + get hasJoined() { + return this._state === GroupCallState.Joining || this._state === GroupCallState.Joined; + } + async leave() { const memberContent = await this._leaveCallMemberContent(); // send m.call.member state event @@ -165,6 +169,7 @@ export class GroupCall { /** @internal */ handleDeviceMessage(message: SignallingMessage, userId: string, deviceId: string, log: ILogItem) { + console.log("incoming to_device call signalling message from", userId, deviceId, message); // TODO: return if we are not membering to the call let member = this._members.get(userId); if (member) { diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index bd1613cd..0a80bbef 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -17,6 +17,7 @@ limitations under the License. import {PeerCall, CallState} from "../PeerCall"; import {makeTxnId, makeId} from "../../common"; import {EventType} from "../callEventTypes"; +import {formatToDeviceMessagesPayload} from "../../common"; import type {Options as PeerCallOptions} from "../PeerCall"; import type {LocalMedia} from "../LocalMedia"; @@ -32,7 +33,7 @@ export type Options = Omit, log: ILogItem) => Promise, + encryptDeviceMessage: (userId: string, message: SignallingMessage, log: ILogItem) => Promise, emitUpdate: (participant: Member, params?: any) => void, } @@ -81,13 +82,14 @@ export class Member { sendSignallingMessage = async (message: SignallingMessage, log: ILogItem) => { const groupMessage = message as SignallingMessage; groupMessage.content.conf_id = this.options.confId; - const encryptedMessage = await this.options.encryptDeviceMessage(groupMessage, log); + const encryptedMessages = await this.options.encryptDeviceMessage(this.member.userId, groupMessage, log); + const payload = formatToDeviceMessagesPayload(encryptedMessages); const request = this.options.hsApi.sendToDevice( "m.room.encrypted", - {[this.member.userId]: { - ["*"]: encryptedMessage.content - } - }, makeTxnId(), {log}); + payload, + makeTxnId(), + {log} + ); await request.response(); } diff --git a/src/matrix/common.js b/src/matrix/common.js index 5919ad9c..7cd72ae1 100644 --- a/src/matrix/common.js +++ b/src/matrix/common.js @@ -15,6 +15,9 @@ See the License for the specific language governing permissions and limitations under the License. */ +import {groupBy} from "../utils/groupBy"; + + export function makeTxnId() { return makeId("t"); } @@ -29,6 +32,20 @@ export function isTxnId(txnId) { return txnId.startsWith("t") && txnId.length === 15; } +export function formatToDeviceMessagesPayload(messages) { + const messagesByUser = groupBy(messages, message => message.device.userId); + const payload = { + messages: Array.from(messagesByUser.entries()).reduce((userMap, [userId, messages]) => { + userMap[userId] = messages.reduce((deviceMap, message) => { + deviceMap[message.device.deviceId] = message.content; + return deviceMap; + }, {}); + return userMap; + }, {}) + }; + return payload; +} + export function tests() { return { "isTxnId succeeds on result of makeTxnId": assert => { diff --git a/src/matrix/e2ee/RoomEncryption.js b/src/matrix/e2ee/RoomEncryption.js index 80f57507..cb0dd333 100644 --- a/src/matrix/e2ee/RoomEncryption.js +++ b/src/matrix/e2ee/RoomEncryption.js @@ -18,7 +18,7 @@ import {MEGOLM_ALGORITHM, DecryptionSource} from "./common.js"; import {groupEventsBySession} from "./megolm/decryption/utils"; import {mergeMap} from "../../utils/mergeMap"; import {groupBy} from "../../utils/groupBy"; -import {makeTxnId} from "../common.js"; +import {makeTxnId, formatToDeviceMessagesPayload} from "../common.js"; const ENCRYPTED_TYPE = "m.room.encrypted"; // how often ensureMessageKeyIsShared can check if it needs to @@ -386,6 +386,7 @@ export class RoomEncryption { await writeTxn.complete(); } + // TODO: make this use _sendMessagesToDevices async _sendSharedMessageToDevices(type, message, devices, hsApi, log) { const devicesByUser = groupBy(devices, device => device.userId); const payload = { @@ -403,16 +404,7 @@ export class RoomEncryption { async _sendMessagesToDevices(type, messages, hsApi, log) { log.set("messages", messages.length); - const messagesByUser = groupBy(messages, message => message.device.userId); - const payload = { - messages: Array.from(messagesByUser.entries()).reduce((userMap, [userId, messages]) => { - userMap[userId] = messages.reduce((deviceMap, message) => { - deviceMap[message.device.deviceId] = message.content; - return deviceMap; - }, {}); - return userMap; - }, {}) - }; + const payload = formatToDeviceMessagesPayload(messages); const txnId = makeTxnId(); await hsApi.sendToDevice(type, payload, txnId, {log}).response(); } diff --git a/src/platform/web/ui/session/room/common.ts b/src/platform/web/ui/session/room/common.ts index 5048211a..a2732ff4 100644 --- a/src/platform/web/ui/session/room/common.ts +++ b/src/platform/web/ui/session/room/common.ts @@ -24,6 +24,7 @@ import {AnnouncementView} from "./timeline/AnnouncementView.js"; import {RedactedView} from "./timeline/RedactedView.js"; import {SimpleTile} from "../../../../../domain/session/room/timeline/tiles/SimpleTile.js"; import {GapView} from "./timeline/GapView.js"; +import {CallTileView} from "./timeline/CallTileView"; export type TileView = GapView | AnnouncementView | TextMessageView | ImageView | VideoView | FileView | LocationView | MissingAttachmentView | RedactedView; @@ -51,5 +52,7 @@ export function viewClassForEntry(entry: SimpleTile): TileViewConstructor | unde return MissingAttachmentView; case "redacted": return RedactedView; + case "call": + return CallTileView; } } diff --git a/src/platform/web/ui/session/room/timeline/CallTileView.ts b/src/platform/web/ui/session/room/timeline/CallTileView.ts new file mode 100644 index 00000000..dfb04228 --- /dev/null +++ b/src/platform/web/ui/session/room/timeline/CallTileView.ts @@ -0,0 +1,38 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {TemplateView} from "../../../general/TemplateView"; +import type {CallTile} from "../../../../../../domain/session/room/timeline/tiles/CallTile"; + +export class CallTileView extends TemplateView { + render(t, vm) { + return t.li( + {className: "AnnouncementView"}, + t.div([ + "Call ", + vm => vm.name, + t.button({className: "CallTileView_join"}, "Join") + ]) + ); + } + + /* This is called by the parent ListView, which just has 1 listener for the whole list */ + onClick(evt) { + if (evt.target.className === "CallTileView_join") { + this.value.join(); + } + } +} From a0a07355d47c336883eab7b00b06a9033701259b Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 24 Mar 2022 13:52:19 +0100 Subject: [PATCH 027/435] more improvements, make hangup work --- src/domain/session/room/CallViewModel.ts | 26 +++++-- src/domain/session/room/RoomViewModel.js | 9 ++- .../room/timeline/tiles/BaseMessageTile.js | 8 --- .../session/room/timeline/tiles/CallTile.js | 59 +++++++++++++-- .../session/room/timeline/tiles/SimpleTile.js | 8 +++ .../session/room/timeline/tilesCreator.js | 4 +- src/matrix/Session.js | 1 - src/matrix/calls/CallHandler.ts | 8 +-- src/matrix/calls/LocalMedia.ts | 6 ++ src/matrix/calls/PeerCall.ts | 4 ++ src/matrix/calls/group/GroupCall.ts | 72 +++++++++++++++---- src/matrix/calls/group/Member.ts | 8 +++ src/matrix/room/Room.js | 2 +- .../value/PickMapObservableValue.ts | 8 +-- src/platform/web/ui/session/room/CallView.ts | 7 +- .../ui/session/room/timeline/CallTileView.ts | 8 ++- 16 files changed, 182 insertions(+), 56 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 7fcabf02..30b18bc1 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -33,16 +33,26 @@ export class CallViewModel extends ViewModel { .sortValues((a, b) => a.compare(b)); } + private get call(): GroupCall { + return this.getOption("call"); + } + get name(): string { - return this.getOption("call").name; + return this.call.name; } get id(): string { - return this.getOption("call").id; + return this.call.id; } get localTracks(): Track[] { - return this.getOption("call").localMedia?.tracks ?? []; + return this.call.localMedia?.tracks ?? []; + } + + leave() { + if (this.call.hasJoined) { + this.call.leave(); + } } } @@ -50,12 +60,16 @@ type MemberOptions = BaseOptions & {member: Member}; export class CallMemberViewModel extends ViewModel { get tracks(): Track[] { - return this.getOption("member").remoteTracks; + return this.member.remoteTracks; + } + + private get member(): Member { + return this.getOption("member"); } compare(other: CallMemberViewModel): number { - const myUserId = this.getOption("member").member.userId; - const otherUserId = other.getOption("member").member.userId; + const myUserId = this.member.member.userId; + const otherUserId = other.member.member.userId; if(myUserId === otherUserId) { return 0; } diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 2b47673f..6dd8c8bd 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -49,12 +49,17 @@ export class RoomViewModel extends ViewModel { _setupCallViewModel() { // pick call for this room with lowest key const calls = this.getOption("session").callHandler.calls; - this._callObservable = new PickMapObservableValue(calls.filterValues(c => c.roomId === this._room.id && c.hasJoined)); + this._callObservable = new PickMapObservableValue(calls.filterValues(c => { + return c.roomId === this._room.id && c.hasJoined; + })); this._callViewModel = undefined; this.track(this._callObservable.subscribe(call => { + if (call && this._callViewModel && call.id === this._callViewModel.id) { + return; + } this._callViewModel = this.disposeTracked(this._callViewModel); if (call) { - this._callViewModel = new CallViewModel(this.childOptions({call})); + this._callViewModel = this.track(new CallViewModel(this.childOptions({call}))); } this.emitChange("callViewModel"); })); diff --git a/src/domain/session/room/timeline/tiles/BaseMessageTile.js b/src/domain/session/room/timeline/tiles/BaseMessageTile.js index 3385a587..c2041a2c 100644 --- a/src/domain/session/room/timeline/tiles/BaseMessageTile.js +++ b/src/domain/session/room/timeline/tiles/BaseMessageTile.js @@ -49,14 +49,6 @@ export class BaseMessageTile extends SimpleTile { return `https://matrix.to/#/${encodeURIComponent(this.sender)}`; } - get displayName() { - return this._entry.displayName || this.sender; - } - - get sender() { - return this._entry.sender; - } - get memberPanelLink() { return `${this.urlCreator.urlUntilSegment("room")}/member/${this.sender}`; } diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index 7129bb75..3e3918b6 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -23,6 +23,27 @@ import {LocalMedia} from "../../../../../matrix/calls/LocalMedia"; // alternatively, we could just subscribe to the GroupCall and spontanously emit an update when it updates export class CallTile extends SimpleTile { + + constructor(options) { + super(options); + const calls = this.getOption("session").callHandler.calls; + this._call = calls.get(this._entry.stateKey); + this._callSubscription = undefined; + if (this._call) { + this._callSubscription = this._call.disposableOn("change", () => { + // unsubscribe when terminated + if (this._call.isTerminated) { + this._callSubscription = this._callSubscription(); + this._call = undefined; + } + this.emitChange(); + }); + } + } + + get confId() { + return this._entry.stateKey; + } get shape() { return "call"; @@ -32,17 +53,43 @@ export class CallTile extends SimpleTile { return this._entry.content["m.name"]; } - get _call() { - const calls = this.getOption("session").callHandler.calls; - return calls.get(this._entry.stateKey); + get canJoin() { + return this._call && !this._call.hasJoined; + } + + get canLeave() { + return this._call && this._call.hasJoined; + } + + get label() { + if (this._call) { + if (this._call.hasJoined) { + return `Ongoing call (${this.name}, ${this.confId})`; + } else { + return `${this.displayName} started a call (${this.name}, ${this.confId})`; + } + } else { + return `Call finished, started by ${this.displayName} (${this.name}, ${this.confId})`; + } } async join() { - const call = this._call; - if (call) { + if (this.canJoin) { const mediaTracks = await this.platform.mediaDevices.getMediaTracks(false, true); const localMedia = new LocalMedia().withTracks(mediaTracks); - await call.join(localMedia); + await this._call.join(localMedia); + } + } + + async leave() { + if (this.canLeave) { + this._call.leave(); + } + } + + dispose() { + if (this._callSubscription) { + this._callSubscription = this._callSubscription(); } } } diff --git a/src/domain/session/room/timeline/tiles/SimpleTile.js b/src/domain/session/room/timeline/tiles/SimpleTile.js index af2b0e12..d70a0a37 100644 --- a/src/domain/session/room/timeline/tiles/SimpleTile.js +++ b/src/domain/session/room/timeline/tiles/SimpleTile.js @@ -154,4 +154,12 @@ export class SimpleTile extends ViewModel { get _ownMember() { return this._options.timeline.me; } + + get displayName() { + return this._entry.displayName || this.sender; + } + + get sender() { + return this._entry.sender; + } } diff --git a/src/domain/session/room/timeline/tilesCreator.js b/src/domain/session/room/timeline/tilesCreator.js index 659a5e76..f35f6536 100644 --- a/src/domain/session/room/timeline/tilesCreator.js +++ b/src/domain/session/room/timeline/tilesCreator.js @@ -73,7 +73,9 @@ export function tilesCreator(baseOptions) { case "m.room.encryption": return new EncryptionEnabledTile(options); case "m.call": - return entry.stateKey ? new CallTile(options) : null; + // if prevContent is present, it's an update to a call event, which we don't render + // as the original event is updated through the call object which receive state event updates + return entry.stateKey && !entry.prevContent ? new CallTile(options) : null; default: // unknown type not rendered return null; diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 27689013..56822c70 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -86,7 +86,6 @@ export class Session { // although we probably already fetched all devices to send messages in the likely e2ee room await this._deviceTracker.trackRoom(this.rooms.get(roomId), log); const devices = await this._deviceTracker.devicesForRoomMembers(roomId, [userId], this._hsApi, log); - console.log("devices", devices); const encryptedMessage = await this._olmEncryption.encrypt(message.type, message.content, devices, this._hsApi, log); return encryptedMessage; }, diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 269f6020..7b8a7f16 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -112,7 +112,7 @@ export class CallHandler { const callId = event.state_key; let call = this._calls.get(callId); if (call) { - call.updateCallEvent(event); + call.updateCallEvent(event.content); if (call.isTerminated) { this._calls.remove(call.id); } @@ -125,13 +125,13 @@ export class CallHandler { private handleCallMemberEvent(event: StateEvent) { const userId = event.state_key; const calls = event.content["m.calls"] ?? []; - const newCallIdsMemberOf = new Set(calls.map(call => { + for (const call of calls) { const callId = call["m.call_id"]; const groupCall = this._calls.get(callId); // TODO: also check the member when receiving the m.call event groupCall?.addMember(userId, call); - return callId; - })); + }; + const newCallIdsMemberOf = new Set(calls.map(call => call["m.call_id"])); let previousCallIdsMemberOf = this.memberToCallIds.get(userId); // remove user as member of any calls not present anymore if (previousCallIdsMemberOf) { diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 1fd4aad6..b148dd92 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -60,4 +60,10 @@ export class LocalMedia { } return metadata; } + + dispose() { + this.cameraTrack?.stop(); + this.microphoneTrack?.stop(); + this.screenShareTrack?.stop(); + } } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 319fde3a..c6c6fca3 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -748,6 +748,10 @@ export class PeerCall implements IDisposable { this.disposables.dispose(); this.peerConnection.dispose(); } + + public close(): void { + this.peerConnection.close(); + } } diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 901eb3b3..a3148518 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -19,6 +19,7 @@ import {Member} from "./Member"; import {LocalMedia} from "../LocalMedia"; import {RoomMember} from "../../room/members/RoomMember"; import {makeId} from "../../common"; +import {EventEmitter} from "../../../utils/EventEmitter"; import type {Options as MemberOptions} from "./Member"; import type {BaseObservableMap} from "../../../observable/map/BaseObservableMap"; @@ -31,6 +32,9 @@ import type {EncryptedMessage} from "../../e2ee/olm/Encryption"; import type {ILogItem} from "../../../logging/types"; import type {Storage} from "../../storage/idb/Storage"; +const CALL_TYPE = "m.call"; +const CALL_MEMBER_TYPE = "m.call.member"; + export enum GroupCallState { Fledgling = "fledgling", Creating = "creating", @@ -46,7 +50,7 @@ export type Options = Omit { public readonly id: string; private readonly _members: ObservableMap = new ObservableMap(); private _localMedia?: LocalMedia = undefined; @@ -59,6 +63,7 @@ export class GroupCall { public readonly roomId: string, private readonly options: Options ) { + super(); this.id = id ?? makeId("conf-"); this._state = id ? GroupCallState.Created : GroupCallState.Fledgling; this._memberOptions = Object.assign({}, options, { @@ -87,12 +92,12 @@ export class GroupCall { } this._state = GroupCallState.Joining; this._localMedia = localMedia; - this.options.emitUpdate(this); - const memberContent = await this._joinCallMemberContent(); + this.emitChange(); + const memberContent = await this._createJoinPayload(); // send m.call.member state event - const request = this.options.hsApi.sendState(this.roomId, "m.call.member", this.options.ownUserId, memberContent); + const request = this.options.hsApi.sendState(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId, memberContent); await request.response(); - this.options.emitUpdate(this); + this.emitChange(); // send invite to all members that are < my userId for (const [,member] of this._members) { member.connect(this._localMedia); @@ -107,25 +112,41 @@ export class GroupCall { const memberContent = await this._leaveCallMemberContent(); // send m.call.member state event if (memberContent) { - const request = this.options.hsApi.sendState(this.roomId, "m.call.member", this.options.ownUserId, memberContent); + const request = this.options.hsApi.sendState(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId, memberContent); await request.response(); + // our own user isn't included in members, so not in the count + if (this._members.size === 0) { + this.terminate(); + } } } + async terminate() { + if (this._state === GroupCallState.Fledgling) { + return; + } + const request = this.options.hsApi.sendState(this.roomId, CALL_TYPE, this.id, Object.assign({}, this.callContent, { + "m.terminated": true + })); + await request.response(); + } + /** @internal */ async create(localMedia: LocalMedia, name: string) { if (this._state !== GroupCallState.Fledgling) { return; } this._state = GroupCallState.Creating; + this.emitChange(); this.callContent = { "m.type": localMedia.cameraTrack ? "m.video" : "m.voice", "m.name": name, "m.intent": "m.ring" }; - const request = this.options.hsApi.sendState(this.roomId, "m.call", this.id, this.callContent); + const request = this.options.hsApi.sendState(this.roomId, CALL_TYPE, this.id, this.callContent); await request.response(); this._state = GroupCallState.Created; + this.emitChange(); } /** @internal */ @@ -134,6 +155,7 @@ export class GroupCall { if (this._state === GroupCallState.Creating) { this._state = GroupCallState.Created; } + this.emitChange(); } /** @internal */ @@ -141,6 +163,7 @@ export class GroupCall { if (userId === this.options.ownUserId) { if (this._state === GroupCallState.Joining) { this._state = GroupCallState.Joined; + this.emitChange(); } return; } @@ -160,11 +183,21 @@ export class GroupCall { removeMember(userId) { if (userId === this.options.ownUserId) { if (this._state === GroupCallState.Joined) { + this._localMedia?.dispose(); + this._localMedia = undefined; + for (const [,member] of this._members) { + member.disconnect(); + } this._state = GroupCallState.Created; } - return; + } else { + const member = this._members.get(userId); + if (member) { + this._members.remove(userId); + member.disconnect(); + } } - this._members.remove(userId); + this.emitChange(); } /** @internal */ @@ -179,10 +212,10 @@ export class GroupCall { } } - private async _joinCallMemberContent() { + private async _createJoinPayload() { const {storage} = this.options; const txn = await storage.readTxn([storage.storeNames.roomState]); - const stateEvent = await txn.roomState.get(this.roomId, "m.call.member", this.options.ownUserId); + const stateEvent = await txn.roomState.get(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId); const stateContent = stateEvent?.event?.content ?? { ["m.calls"]: [] }; @@ -209,9 +242,18 @@ export class GroupCall { private async _leaveCallMemberContent(): Promise | undefined> { const {storage} = this.options; const txn = await storage.readTxn([storage.storeNames.roomState]); - const stateEvent = await txn.roomState.get(this.roomId, "m.call.member", this.options.ownUserId); - const callsInfo = stateEvent?.event?.content?.["m.calls"]; - callsInfo?.filter(c => c["m.call_id"] === this.id); - return stateEvent?.event.content; + const stateEvent = await txn.roomState.get(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId); + if (stateEvent) { + const content = stateEvent.event.content; + const callsInfo = content["m.calls"]; + content["m.calls"] = callsInfo?.filter(c => c["m.call_id"] !== this.id); + return content; + + } + } + + protected emitChange() { + this.emit("change"); + this.options.emitUpdate(this); } } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 0a80bbef..dedde429 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -65,6 +65,14 @@ export class Member { } } + /** @internal */ + disconnect() { + this.peerCall?.close(); + this.peerCall?.dispose(); + this.peerCall = undefined; + this.localMedia = undefined; + } + /** @internal */ updateCallInfo(memberCallInfo) { // m.calls object from the m.call.member event diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js index ca11cdd8..ff1926b4 100644 --- a/src/matrix/room/Room.js +++ b/src/matrix/room/Room.js @@ -451,7 +451,7 @@ export class Room extends BaseRoom { _updateCallHandler(roomResponse, log) { if (this._callHandler) { const stateEvents = roomResponse.state?.events; - if (stateEvents) { + if (stateEvents?.length) { this._callHandler.handleRoomState(this, stateEvents, log); } let timelineEvents = roomResponse.timeline?.events; diff --git a/src/observable/value/PickMapObservableValue.ts b/src/observable/value/PickMapObservableValue.ts index 67f9d562..b493d841 100644 --- a/src/observable/value/PickMapObservableValue.ts +++ b/src/observable/value/PickMapObservableValue.ts @@ -60,13 +60,11 @@ export class PickMapObservableValue extends BaseObservableValue(t: TemplateBuilder, video: HTMLVideoElement, propSelector: (vm: T) => Track[]) { t.mapSideEffect(propSelector, tracks => { - console.log("tracks", tracks); if (tracks.length) { video.srcObject = (tracks[0] as TrackWrapper).stream; } @@ -33,8 +32,8 @@ function bindVideoTracks(t: TemplateBuilder, video: HTMLVideoElement, prop export class CallView extends TemplateView { render(t: TemplateBuilder, vm: CallViewModel): HTMLElement { return t.div({class: "CallView"}, [ - t.p(["Call ", vm => vm.name, vm => ` (${vm.id})`]), - t.div({class: "CallView_me"}, bindVideoTracks(t, t.video({autoplay: true}), vm => vm.localTracks)), + t.p(vm => `Call ${vm.name} (${vm.id})`), + t.div({class: "CallView_me"}, bindVideoTracks(t, t.video({autoplay: true, width: 240}), vm => vm.localTracks)), t.view(new ListView({list: vm.memberViewModels}, vm => new MemberView(vm))) ]); } @@ -42,6 +41,6 @@ export class CallView extends TemplateView { class MemberView extends TemplateView { render(t: TemplateBuilder, vm: CallMemberViewModel) { - return bindVideoTracks(t, t.video({autoplay: true}), vm => vm.tracks); + return bindVideoTracks(t, t.video({autoplay: true, width: 360}), vm => vm.tracks); } } diff --git a/src/platform/web/ui/session/room/timeline/CallTileView.ts b/src/platform/web/ui/session/room/timeline/CallTileView.ts index dfb04228..e0ca00bd 100644 --- a/src/platform/web/ui/session/room/timeline/CallTileView.ts +++ b/src/platform/web/ui/session/room/timeline/CallTileView.ts @@ -22,9 +22,9 @@ export class CallTileView extends TemplateView { return t.li( {className: "AnnouncementView"}, t.div([ - "Call ", - vm => vm.name, - t.button({className: "CallTileView_join"}, "Join") + vm => vm.label, + t.button({className: "CallTileView_join", hidden: vm => !vm.canJoin}, "Join"), + t.button({className: "CallTileView_leave", hidden: vm => !vm.canLeave}, "Leave") ]) ); } @@ -33,6 +33,8 @@ export class CallTileView extends TemplateView { onClick(evt) { if (evt.target.className === "CallTileView_join") { this.value.join(); + } else if (evt.target.className === "CallTileView_leave") { + this.value.leave(); } } } From eaf92b382bfe5155f19f256bd7b9f437d92b8d20 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 25 Mar 2022 14:43:02 +0100 Subject: [PATCH 028/435] add structured logging to call code --- src/logging/BaseLogger.ts | 23 +- src/logging/LogItem.ts | 9 +- src/logging/NullLogger.ts | 9 +- src/logging/types.ts | 13 + src/matrix/DeviceMessageHandler.js | 1 - src/matrix/Session.js | 7 +- src/matrix/calls/CallHandler.ts | 28 +- src/matrix/calls/PeerCall.ts | 479 +++++++++++++++------------- src/matrix/calls/group/GroupCall.ts | 242 +++++++------- src/matrix/calls/group/Member.ts | 18 +- 10 files changed, 478 insertions(+), 351 deletions(-) diff --git a/src/logging/BaseLogger.ts b/src/logging/BaseLogger.ts index e32b9f0f..21643c48 100644 --- a/src/logging/BaseLogger.ts +++ b/src/logging/BaseLogger.ts @@ -36,6 +36,15 @@ export abstract class BaseLogger implements ILogger { this._persistItem(item, undefined, false); } + /** Prefer `run()` or `log()` above this method; only use it if you have a long-running operation + * *without* a single call stack that should be logged into one sub-tree. + * You need to call `finish()` on the returned item or it will stay open until the app unloads. */ + child(labelOrValues: LabelOrValues, logLevel: LogLevel = LogLevel.Info, filterCreator?: FilterCreator): ILogItem { + const item = new DeferredPersistRootLogItem(labelOrValues, logLevel, this, filterCreator); + this._openItems.add(item); + return item; + } + /** if item is a log item, wrap the callback in a child of it, otherwise start a new root log item. */ wrapOrRun(item: ILogItem | undefined, labelOrValues: LabelOrValues, callback: LogCallback, logLevel?: LogLevel, filterCreator?: FilterCreator): T { if (item) { @@ -127,7 +136,7 @@ export abstract class BaseLogger implements ILogger { _finishOpenItems() { for (const openItem of this._openItems) { - openItem.finish(); + openItem.forceFinish(); try { // for now, serialize with an all-permitting filter // as the createFilter function would get a distorted image anyway @@ -158,3 +167,15 @@ export abstract class BaseLogger implements ILogger { return Math.round(this._platform.random() * Number.MAX_SAFE_INTEGER); } } + +class DeferredPersistRootLogItem extends LogItem { + finish() { + super.finish(); + (this._logger as BaseLogger)._persistItem(this, undefined, false); + } + + forceFinish() { + super.finish(); + /// no need to persist when force-finishing as _finishOpenItems above will do it + } +} diff --git a/src/logging/LogItem.ts b/src/logging/LogItem.ts index b47b69c1..216cc6bb 100644 --- a/src/logging/LogItem.ts +++ b/src/logging/LogItem.ts @@ -25,7 +25,7 @@ export class LogItem implements ILogItem { public error?: Error; public end?: number; private _values: LogItemValues; - private _logger: BaseLogger; + protected _logger: BaseLogger; private _filterCreator?: FilterCreator; private _children?: Array; @@ -221,6 +221,11 @@ export class LogItem implements ILogItem { } } + /** @internal */ + forceFinish(): void { + this.finish(); + } + // expose log level without needing import everywhere get level(): typeof LogLevel { return LogLevel; @@ -235,7 +240,7 @@ export class LogItem implements ILogItem { child(labelOrValues: LabelOrValues, logLevel?: LogLevel, filterCreator?: FilterCreator): LogItem { if (this.end) { - console.trace("log item is finished, additional logs will likely not be recorded"); + console.trace(`log item ${this.values.l} finished, additional log ${JSON.stringify(labelOrValues)} will likely not be recorded`); } if (!logLevel) { logLevel = this.logLevel || LogLevel.Info; diff --git a/src/logging/NullLogger.ts b/src/logging/NullLogger.ts index 21c3d349..adc2b843 100644 --- a/src/logging/NullLogger.ts +++ b/src/logging/NullLogger.ts @@ -23,6 +23,10 @@ export class NullLogger implements ILogger { log(): void {} + child(): ILogItem { + return this.item; + } + run(_, callback: LogCallback): T { return callback(this.item); } @@ -50,13 +54,13 @@ export class NullLogger implements ILogger { } export class NullLogItem implements ILogItem { - public readonly logger: NullLogger; + public readonly logger: ILogger; public readonly logLevel: LogLevel; public children?: Array; public values: LogItemValues; public error?: Error; - constructor(logger: NullLogger) { + constructor(logger: ILogger) { this.logger = logger; } @@ -99,6 +103,7 @@ export class NullLogItem implements ILogItem { } finish(): void {} + forceFinish(): void {} serialize(): undefined { return undefined; diff --git a/src/logging/types.ts b/src/logging/types.ts index bf9861a5..8e35dbf3 100644 --- a/src/logging/types.ts +++ b/src/logging/types.ts @@ -51,11 +51,24 @@ export interface ILogItem { catch(err: Error): Error; serialize(filter: LogFilter, parentStartTime: number | undefined, forced: boolean): ISerializedItem | undefined; finish(): void; + forceFinish(): void; child(labelOrValues: LabelOrValues, logLevel?: LogLevel, filterCreator?: FilterCreator): ILogItem; } +/* +extend both ILogger and ILogItem from this interface, but need to rename ILogger.run => wrap then. Or both to `span`? + +export interface ILogItemCreator { + child(labelOrValues: LabelOrValues, logLevel?: LogLevel, filterCreator?: FilterCreator): ILogItem; + refDetached(logItem: ILogItem, logLevel?: LogLevel): void; + log(labelOrValues: LabelOrValues, logLevel?: LogLevel): ILogItem; + wrap(labelOrValues: LabelOrValues, callback: LogCallback, logLevel?: LogLevel, filterCreator?: FilterCreator): T; + get level(): typeof LogLevel; +} +*/ export interface ILogger { log(labelOrValues: LabelOrValues, logLevel?: LogLevel): void; + child(labelOrValues: LabelOrValues, logLevel?: LogLevel, filterCreator?: FilterCreator): ILogItem; wrapOrRun(item: ILogItem | undefined, labelOrValues: LabelOrValues, callback: LogCallback, logLevel?: LogLevel, filterCreator?: FilterCreator): T; runDetached(labelOrValues: LabelOrValues, callback: LogCallback, logLevel?: LogLevel, filterCreator?: FilterCreator): ILogItem; run(labelOrValues: LabelOrValues, callback: LogCallback, logLevel?: LogLevel, filterCreator?: FilterCreator): T; diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index 11c10750..80fd1592 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -86,7 +86,6 @@ export class DeviceMessageHandler { this._senderDeviceCache.set(device); } } - console.log("incoming device message", senderKey, device, this._senderDeviceCache); return device; } } diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 56822c70..a1e1cc28 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -84,8 +84,10 @@ export class Session { } // TODO: just get the devices we're sending the message to, not all the room devices // although we probably already fetched all devices to send messages in the likely e2ee room - await this._deviceTracker.trackRoom(this.rooms.get(roomId), log); - const devices = await this._deviceTracker.devicesForRoomMembers(roomId, [userId], this._hsApi, log); + const devices = await log.wrap("get device keys", async log => { + await this._deviceTracker.trackRoom(this.rooms.get(roomId), log); + return this._deviceTracker.devicesForRoomMembers(roomId, [userId], this._hsApi, log); + }); const encryptedMessage = await this._olmEncryption.encrypt(message.type, message.content, devices, this._hsApi, log); return encryptedMessage; }, @@ -93,6 +95,7 @@ export class Session { webRTC: this._platform.webRTC, ownDeviceId: sessionInfo.deviceId, ownUserId: sessionInfo.userId, + logger: this._platform.logger, }); this._deviceMessageHandler = new DeviceMessageHandler({storage, callHandler: this._callHandler}); this._olm = olm; diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 7b8a7f16..ef617cd2 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -25,7 +25,7 @@ import type {LocalMedia} from "./LocalMedia"; import type {Room} from "../room/Room"; import type {MemberChange} from "../room/members/RoomMember"; import type {StateEvent} from "../storage/types"; -import type {ILogItem} from "../../logging/types"; +import type {ILogItem, ILogger} from "../../logging/types"; import type {Platform} from "../../platform/web/Platform"; import type {BaseObservableMap} from "../../observable/map/BaseObservableMap"; import type {SignallingMessage, MGroupCallBase} from "./callEventTypes"; @@ -35,7 +35,9 @@ const GROUP_CALL_TYPE = "m.call"; const GROUP_CALL_MEMBER_TYPE = "m.call.member"; const CALL_TERMINATED = "m.terminated"; -export type Options = Omit; +export type Options = Omit & { + logger: ILogger +}; export class CallHandler { // group calls by call id @@ -51,7 +53,8 @@ export class CallHandler { } async createCall(roomId: string, localMedia: LocalMedia, name: string): Promise { - const call = new GroupCall(undefined, undefined, roomId, this.groupCallOptions); + const logItem = this.options.logger.child({l: "call", incoming: false}); + const call = new GroupCall(undefined, undefined, roomId, this.groupCallOptions, logItem); console.log("created call with id", call.id); this._calls.set(call.id, call); try { @@ -59,6 +62,7 @@ export class CallHandler { } catch (err) { if (err.name === "ConnectionError") { // if we're offline, give up and remove the call again + call.dispose(); this._calls.remove(call.id); } throw err; @@ -79,13 +83,13 @@ export class CallHandler { // first update call events for (const event of events) { if (event.type === EventType.GroupCall) { - this.handleCallEvent(event, room.id); + this.handleCallEvent(event, room.id, log); } } // then update members for (const event of events) { if (event.type === EventType.GroupCallMember) { - this.handleCallMemberEvent(event); + this.handleCallMemberEvent(event, log); } } } @@ -108,28 +112,30 @@ export class CallHandler { call?.handleDeviceMessage(message, userId, deviceId, log); } - private handleCallEvent(event: StateEvent, roomId: string) { + private handleCallEvent(event: StateEvent, roomId: string, log: ILogItem) { const callId = event.state_key; let call = this._calls.get(callId); if (call) { - call.updateCallEvent(event.content); + call.updateCallEvent(event.content, log); if (call.isTerminated) { + call.dispose(); this._calls.remove(call.id); } } else { - call = new GroupCall(event.state_key, event.content, roomId, this.groupCallOptions); + const logItem = this.options.logger.child({l: "call", incoming: true}); + call = new GroupCall(event.state_key, event.content, roomId, this.groupCallOptions, logItem); this._calls.set(call.id, call); } } - private handleCallMemberEvent(event: StateEvent) { + private handleCallMemberEvent(event: StateEvent, log: ILogItem) { const userId = event.state_key; const calls = event.content["m.calls"] ?? []; for (const call of calls) { const callId = call["m.call_id"]; const groupCall = this._calls.get(callId); // TODO: also check the member when receiving the m.call event - groupCall?.addMember(userId, call); + groupCall?.addMember(userId, call, log); }; const newCallIdsMemberOf = new Set(calls.map(call => call["m.call_id"])); let previousCallIdsMemberOf = this.memberToCallIds.get(userId); @@ -138,7 +144,7 @@ export class CallHandler { for (const previousCallId of previousCallIdsMemberOf) { if (!newCallIdsMemberOf.has(previousCallId)) { const groupCall = this._calls.get(previousCallId); - groupCall?.removeMember(userId); + groupCall?.removeMember(userId, log); } } } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index c6c6fca3..c4355e63 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -21,7 +21,6 @@ import {Disposables, IDisposable} from "../../utils/Disposables"; import type {Room} from "../room/Room"; import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; -import {Instance as logger} from "../../logging/NullLogger"; import type {TimeoutCreator, Timeout} from "../../platform/types/types"; import {WebRTC, PeerConnection, PeerConnectionHandler, DataChannel} from "../../platform/types/WebRTC"; @@ -69,9 +68,8 @@ export class PeerCall implements IDisposable { // If candidates arrive before we've picked an opponent (which, in particular, // will happen if the opponent sends candidates eagerly before the user answers // the call) we buffer them up here so we can then add the ones from the party we pick - private remoteCandidateBuffer? = new Map(); + private remoteCandidateBuffer? = new Map(); - private logger: any; private remoteSDPStreamMetadata?: SDPStreamMetadata; private responsePromiseChain?: Promise; private opponentPartyId?: PartyId; @@ -88,38 +86,44 @@ export class PeerCall implements IDisposable { constructor( private callId: string, - private readonly options: Options + private readonly options: Options, + private readonly logItem: ILogItem, ) { const outer = this; this.peerConnection = options.webRTC.createPeerConnection({ onIceConnectionStateChange(state: RTCIceConnectionState) { - outer.onIceConnectionStateChange(state); + outer.logItem.wrap({l: "onIceConnectionStateChange", status: state}, log => { + outer.onIceConnectionStateChange(state, log); + }); }, onLocalIceCandidate(candidate: RTCIceCandidate) { - outer.handleLocalIceCandidate(candidate); + outer.logItem.wrap("onLocalIceCandidate", log => { + outer.handleLocalIceCandidate(candidate, log); + }); }, onIceGatheringStateChange(state: RTCIceGatheringState) { - outer.handleIceGatheringState(state); + outer.logItem.wrap({l: "onIceGatheringStateChange", status: state}, log => { + outer.handleIceGatheringState(state, log); + }); }, onRemoteTracksChanged(tracks: Track[]) { - outer.options.emitUpdate(outer, undefined); + outer.logItem.wrap("onRemoteTracksChanged", log => { + outer.options.emitUpdate(outer, undefined); + }); }, onDataChannelChanged(dataChannel: DataChannel | undefined) {}, onNegotiationNeeded() { - const promiseCreator = () => outer.handleNegotiation(); + const log = outer.logItem.child("onNegotiationNeeded"); + const promiseCreator = async () => { + await outer.handleNegotiation(log); + log.finish(); + }; outer.responsePromiseChain = outer.responsePromiseChain?.then(promiseCreator) ?? promiseCreator(); }, getPurposeForStreamId(streamId: string): SDPStreamMetadataPurpose { return outer.remoteSDPStreamMetadata?.[streamId]?.purpose ?? SDPStreamMetadataPurpose.Usermedia; } }); - this.logger = { - info(...args) { console.info.apply(console, ["WebRTC debug:", ...args])}, - debug(...args) { console.log.apply(console, ["WebRTC debug:", ...args])}, - log(...args) { console.log.apply(console, ["WebRTC log:", ...args])}, - warn(...args) { console.log.apply(console, ["WebRTC warn:", ...args])}, - error(...args) { console.error.apply(console, ["WebRTC error:", ...args])}, - }; } get state(): CallState { return this._state; } @@ -128,108 +132,127 @@ export class PeerCall implements IDisposable { return this.peerConnection.remoteTracks; } - async call(localMedia: LocalMedia): Promise { - if (this._state !== CallState.Fledgling) { - return; - } - this.localMedia = localMedia; - this.direction = CallDirection.Outbound; - this.setState(CallState.CreateOffer); - for (const t of this.localMedia.tracks) { - this.peerConnection.addTrack(t); - } - // after adding the local tracks, and wait for handleNegotiation to be called, - // or invite glare where we give up our invite and answer instead - await this.waitForState([CallState.InviteSent, CallState.CreateAnswer]); - } - - async answer(localMedia: LocalMedia): Promise { - if (this._state !== CallState.Ringing) { - return; - } - this.localMedia = localMedia; - this.setState(CallState.CreateAnswer); - for (const t of this.localMedia.tracks) { - this.peerConnection.addTrack(t); - } - - let myAnswer: RTCSessionDescriptionInit; - try { - myAnswer = await this.peerConnection.createAnswer(); - } catch (err) { - this.logger.debug(`Call ${this.callId} Failed to create answer: `, err); - this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true); - return; - } - - try { - await this.peerConnection.setLocalDescription(myAnswer); - this.setState(CallState.Connecting); - } catch (err) { - this.logger.debug(`Call ${this.callId} Error setting local description!`, err); - this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); - return; - } - // Allow a short time for initial candidates to be gathered - await this.delay(200); - await this.sendAnswer(); - } - - async setMedia(localMediaPromise: Promise) { - const oldMedia = this.localMedia; - this.localMedia = await localMediaPromise; - - const applyTrack = (selectTrack: (media: LocalMedia | undefined) => Track | undefined) => { - const oldTrack = selectTrack(oldMedia); - const newTrack = selectTrack(this.localMedia); - if (oldTrack && newTrack) { - this.peerConnection.replaceTrack(oldTrack, newTrack); - } else if (oldTrack) { - this.peerConnection.removeTrack(oldTrack); - } else if (newTrack) { - this.peerConnection.addTrack(newTrack); + call(localMedia: LocalMedia): Promise { + return this.logItem.wrap("call", async log => { + if (this._state !== CallState.Fledgling) { + return; } - }; + this.localMedia = localMedia; + this.direction = CallDirection.Outbound; + this.setState(CallState.CreateOffer); + for (const t of this.localMedia.tracks) { + this.peerConnection.addTrack(t); + } + // after adding the local tracks, and wait for handleNegotiation to be called, + // or invite glare where we give up our invite and answer instead + await this.waitForState([CallState.InviteSent, CallState.CreateAnswer]); + }); + } - // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called - applyTrack(m => m?.microphoneTrack); - applyTrack(m => m?.cameraTrack); - applyTrack(m => m?.screenShareTrack); + answer(localMedia: LocalMedia): Promise { + return this.logItem.wrap("answer", async log => { + if (this._state !== CallState.Ringing) { + return; + } + this.localMedia = localMedia; + this.setState(CallState.CreateAnswer); + for (const t of this.localMedia.tracks) { + this.peerConnection.addTrack(t); + } + + let myAnswer: RTCSessionDescriptionInit; + try { + myAnswer = await this.peerConnection.createAnswer(); + } catch (err) { + await log.wrap(`Failed to create answer`, log => { + log.catch(err); + this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true, log); + }); + return; + } + + try { + await this.peerConnection.setLocalDescription(myAnswer); + this.setState(CallState.Connecting); + } catch (err) { + await log.wrap(`Error setting local description!`, log => { + log.catch(err); + this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true, log); + }); + return; + } + // Allow a short time for initial candidates to be gathered + try { await this.delay(200); } + catch (err) { return; } + await this.sendAnswer(log); + }); + } + + setMedia(localMediaPromise: Promise): Promise { + return this.logItem.wrap("setMedia", async log => { + const oldMedia = this.localMedia; + this.localMedia = await localMediaPromise; + + const applyTrack = (selectTrack: (media: LocalMedia | undefined) => Track | undefined) => { + const oldTrack = selectTrack(oldMedia); + const newTrack = selectTrack(this.localMedia); + if (oldTrack && newTrack) { + this.peerConnection.replaceTrack(oldTrack, newTrack); + } else if (oldTrack) { + this.peerConnection.removeTrack(oldTrack); + } else if (newTrack) { + this.peerConnection.addTrack(newTrack); + } + }; + + // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called + applyTrack(m => m?.microphoneTrack); + applyTrack(m => m?.cameraTrack); + applyTrack(m => m?.screenShareTrack); + }); } async reject() { } - async hangup(errorCode: CallErrorCode): Promise { + hangup(errorCode: CallErrorCode): Promise { + return this.logItem.wrap("hangup", log => { + return this._hangup(errorCode, log); + }); + } + + private async _hangup(errorCode: CallErrorCode, log: ILogItem): Promise { if (this._state !== CallState.Ended) { this._state = CallState.Ended; - await this.sendHangupWithCallId(this.callId, errorCode); + await this.sendHangupWithCallId(this.callId, errorCode, log); } } - async handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId, log: ILogItem): Promise { - switch (message.type) { - case EventType.Invite: - if (this.callId !== message.content.call_id) { - await this.handleInviteGlare(message.content, partyId); - } else { - await this.handleFirstInvite(message.content, partyId); - } - break; - case EventType.Answer: - await this.handleAnswer(message.content, partyId); - break; - case EventType.Candidates: - await this.handleRemoteIceCandidates(message.content, partyId); - break; - case EventType.Hangup: - default: - throw new Error(`Unknown event type for call: ${message.type}`); - } + handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId): Promise { + return this.logItem.wrap({l: "receive", id: message.type, partyId}, async log => { + switch (message.type) { + case EventType.Invite: + if (this.callId !== message.content.call_id) { + await this.handleInviteGlare(message.content, partyId, log); + } else { + await this.handleFirstInvite(message.content, partyId, log); + } + break; + case EventType.Answer: + await this.handleAnswer(message.content, partyId, log); + break; + case EventType.Candidates: + await this.handleRemoteIceCandidates(message.content, partyId, log); + break; + case EventType.Hangup: + default: + throw new Error(`Unknown event type for call: ${message.type}`); + } + }); } - private sendHangupWithCallId(callId: string, reason?: CallErrorCode): Promise { + private sendHangupWithCallId(callId: string, reason: CallErrorCode | undefined, log: ILogItem): Promise { const content = { call_id: callId, version: 1, @@ -237,27 +260,28 @@ export class PeerCall implements IDisposable { if (reason) { content["reason"] = reason; } - return this.options.sendSignallingMessage({ + return this.sendSignallingMessage({ type: EventType.Hangup, content - }, logger.item); + }, log); } // calls are serialized and deduplicated by responsePromiseChain - private handleNegotiation = async (): Promise => { + private handleNegotiation = async (log: ILogItem): Promise => { this.makingOffer = true; try { try { await this.peerConnection.setLocalDescription(); } catch (err) { - this.logger.debug(`Call ${this.callId} Error setting local description!`, err); - this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); + log.log(`Error setting local description!`).catch(err); + this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true, log); return; } if (this.peerConnection.iceGatheringState === 'gathering') { // Allow a short time for initial candidates to be gathered - await this.delay(200); + try { await this.delay(200); } + catch (err) { return; } } if (this._state === CallState.Ended) { @@ -267,7 +291,7 @@ export class PeerCall implements IDisposable { const offer = this.peerConnection.localDescription!; // Get rid of any candidates waiting to be sent: they'll be included in the local // description we just got and will send in the offer. - this.logger.info(`Call ${this.callId} Discarding ${ + log.log(`Discarding ${ this.candidateSendQueue.length} candidates that will be sent in offer`); this.candidateSendQueue = []; @@ -280,63 +304,64 @@ export class PeerCall implements IDisposable { lifetime: CALL_TIMEOUT_MS }; if (this._state === CallState.CreateOffer) { - await this.options.sendSignallingMessage({type: EventType.Invite, content}, logger.item); + await this.sendSignallingMessage({type: EventType.Invite, content}, log); this.setState(CallState.InviteSent); } else if (this._state === CallState.Connected || this._state === CallState.Connecting) { // send Negotiate message - //await this.options.sendSignallingMessage({type: EventType.Invite, content}); + //await this.sendSignallingMessage({type: EventType.Invite, content}); //this.setState(CallState.InviteSent); } } finally { this.makingOffer = false; } - this.sendCandidateQueue(); + this.sendCandidateQueue(log); if (this._state === CallState.InviteSent) { - await this.delay(CALL_TIMEOUT_MS); + try { await this.delay(CALL_TIMEOUT_MS); } + catch (err) { return; } // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between if (this._state === CallState.InviteSent) { - this.hangup(CallErrorCode.InviteTimeout); + this._hangup(CallErrorCode.InviteTimeout, log); } } }; - private async handleInviteGlare(content: MCallInvite, partyId: PartyId): Promise { + private async handleInviteGlare(content: MCallInvite, partyId: PartyId, log: ILogItem): Promise { // this is only called when the ids are different const newCallId = content.call_id; if (this.callId! > newCallId) { - this.logger.log( + log.log( "Glare detected: answering incoming call " + newCallId + - " and canceling outgoing call " + this.callId, + " and canceling outgoing call ", ); // How do we interrupt `call()`? well, perhaps we need to not just await InviteSent but also CreateAnswer? if (this._state === CallState.Fledgling || this._state === CallState.CreateOffer) { // TODO: don't send invite! } else { - await this.sendHangupWithCallId(this.callId, CallErrorCode.Replaced); + await this.sendHangupWithCallId(this.callId, CallErrorCode.Replaced, log); } - await this.handleInvite(content, partyId); + await this.handleInvite(content, partyId, log); // TODO: need to skip state check await this.answer(this.localMedia!); } else { - this.logger.log( + log.log( "Glare detected: rejecting incoming call " + newCallId + - " and keeping outgoing call " + this.callId, + " and keeping outgoing call ", ); - await this.sendHangupWithCallId(newCallId, CallErrorCode.Replaced); + await this.sendHangupWithCallId(newCallId, CallErrorCode.Replaced, log); } } - private async handleFirstInvite(content: MCallInvite, partyId: PartyId): Promise { + private async handleFirstInvite(content: MCallInvite, partyId: PartyId, log: ILogItem): Promise { if (this._state !== CallState.Fledgling || this.opponentPartyId !== undefined) { // TODO: hangup or ignore? return; } - await this.handleInvite(content, partyId); + await this.handleInvite(content, partyId, log); } - private async handleInvite(content: MCallInvite, partyId: PartyId): Promise { + private async handleInvite(content: MCallInvite, partyId: PartyId, log: ILogItem): Promise { // we must set the party ID before await-ing on anything: the call event // handler will start giving us more call events (eg. candidates) so if @@ -348,17 +373,18 @@ export class PeerCall implements IDisposable { if (sdpStreamMetadata) { this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); } else { - this.logger.debug(`Call ${ - this.callId} did not get any SDPStreamMetadata! Can not send/receive multiple streams`); + log.log(`Call did not get any SDPStreamMetadata! Can not send/receive multiple streams`); } try { // Q: Why do we set the remote description before accepting the call? To start creating ICE candidates? await this.peerConnection.setRemoteDescription(content.offer); - await this.addBufferedIceCandidates(); + await this.addBufferedIceCandidates(log); } catch (e) { - this.logger.debug(`Call ${this.callId} failed to set remote description`, e); - this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); + await log.wrap(`Call failed to set remote description`, async log => { + log.catch(e); + return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false, log); + }); return; } @@ -366,17 +392,19 @@ export class PeerCall implements IDisposable { // add streams until media started arriving on them. Testing latest firefox // (81 at time of writing), this is no longer a problem, so let's do it the correct way. if (this.peerConnection.remoteTracks.length === 0) { - this.logger.error(`Call ${this.callId} no remote stream or no tracks after setting remote description!`); - this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); + await log.wrap(`Call no remote stream or no tracks after setting remote description!`, async log => { + return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false, log); + }); return; } this.setState(CallState.Ringing); - await this.delay(content.lifetime ?? CALL_TIMEOUT_MS); + try { await this.delay(content.lifetime ?? CALL_TIMEOUT_MS); } + catch (err) { return; } // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between if (this._state === CallState.Ringing) { - this.logger.debug(`Call ${this.callId} invite has expired. Hanging up.`); + log.log(`Invite has expired. Hanging up.`); this.hangupParty = CallParty.Remote; // effectively this.setState(CallState.Ended); this.stopAllMedia(); @@ -386,25 +414,19 @@ export class PeerCall implements IDisposable { } } - private async handleAnswer(content: MCallAnswer, partyId: PartyId): Promise { - this.logger.debug(`Got answer for call ID ${this.callId} from party ID ${partyId}`); - + private async handleAnswer(content: MCallAnswer, partyId: PartyId, log: ILogItem): Promise { if (this._state === CallState.Ended) { - this.logger.debug(`Ignoring answer because call ID ${this.callId} has ended`); + log.log(`Ignoring answer because call has ended`); return; } if (this.opponentPartyId !== undefined) { - this.logger.info( - `Call ${this.callId} ` + - `Ignoring answer from party ID ${partyId}: ` + - `we already have an answer/reject from ${this.opponentPartyId}`, - ); + log.log(`Ignoring answer: we already have an answer/reject from ${this.opponentPartyId}`); return; } this.opponentPartyId = partyId; - await this.addBufferedIceCandidates(); + await this.addBufferedIceCandidates(log); this.setState(CallState.Connecting); @@ -412,20 +434,22 @@ export class PeerCall implements IDisposable { if (sdpStreamMetadata) { this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); } else { - this.logger.warn(`Call ${this.callId} Did not get any SDPStreamMetadata! Can not send/receive multiple streams`); + log.log(`Did not get any SDPStreamMetadata! Can not send/receive multiple streams`); } try { await this.peerConnection.setRemoteDescription(content.answer); } catch (e) { - this.logger.debug(`Call ${this.callId} Failed to set remote description`, e); - this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); + await log.wrap(`Failed to set remote description`, log => { + log.catch(e); + this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false, log); + }); return; } } - private handleIceGatheringState(state: RTCIceGatheringState) { - this.logger.debug(`Call ${this.callId} ice gathering state changed to ${state}`); + private handleIceGatheringState(state: RTCIceGatheringState, log: ILogItem) { + log.set("state", state); if (state === 'complete' && !this.sentEndOfCandidates) { // If we didn't get an empty-string candidate to signal the end of candidates, // create one ourselves now gathering has finished. @@ -437,37 +461,37 @@ export class PeerCall implements IDisposable { const c = { candidate: '', } as RTCIceCandidate; - this.queueCandidate(c); + this.queueCandidate(c, log); this.sentEndOfCandidates = true; } } - private handleLocalIceCandidate(candidate: RTCIceCandidate) { - this.logger.debug( - "Call " + this.callId + " got local ICE " + candidate.sdpMid + " candidate: " + - candidate.candidate, - ); - - if (this._state === CallState.Ended) return; + private handleLocalIceCandidate(candidate: RTCIceCandidate, log: ILogItem) { + log.set("sdpMid", candidate.sdpMid); + log.set("candidate", candidate.candidate); + if (this._state === CallState.Ended) { + return; + } // As with the offer, note we need to make a copy of this object, not // pass the original: that broke in Chrome ~m43. if (candidate.candidate !== '' || !this.sentEndOfCandidates) { - this.queueCandidate(candidate); - - if (candidate.candidate === '') this.sentEndOfCandidates = true; + this.queueCandidate(candidate, log); + if (candidate.candidate === '') { + this.sentEndOfCandidates = true; + } } } - private async handleRemoteIceCandidates(content: MCallCandidates, partyId) { + private async handleRemoteIceCandidates(content: MCallCandidates, partyId: PartyId, log: ILogItem) { if (this.state === CallState.Ended) { - //debuglog("Ignoring remote ICE candidate because call has ended"); + log.log("Ignoring remote ICE candidate because call has ended"); return; } const candidates = content.candidates; if (!candidates) { - this.logger.info(`Call ${this.callId} Ignoring candidates event with no candidates!`); + log.log(`Ignoring candidates event with no candidates!`); return; } @@ -475,7 +499,7 @@ export class PeerCall implements IDisposable { if (this.opponentPartyId === undefined) { // we haven't picked an opponent yet so save the candidates - this.logger.info(`Call ${this.callId} Buffering ${candidates.length} candidates until we pick an opponent`); + log.log(`Buffering ${candidates.length} candidates until we pick an opponent`); const bufferedCandidates = this.remoteCandidateBuffer!.get(fromPartyId) || []; bufferedCandidates.push(...candidates); this.remoteCandidateBuffer!.set(fromPartyId, bufferedCandidates); @@ -483,8 +507,7 @@ export class PeerCall implements IDisposable { } if (this.opponentPartyId !== partyId) { - this.logger.info( - `Call ${this.callId} `+ + log.log( `Ignoring candidates from party ID ${partyId}: ` + `we have chosen party ID ${this.opponentPartyId}`, ); @@ -492,14 +515,14 @@ export class PeerCall implements IDisposable { return; } - await this.addIceCandidates(candidates); + await this.addIceCandidates(candidates, log); } // private async onNegotiateReceived(event: MatrixEvent): Promise { // const content = event.getContent(); // const description = content.description; // if (!description || !description.sdp || !description.type) { - // this.logger.info(`Call ${this.callId} Ignoring invalid m.call.negotiate event`); + // this.logger.info(`Ignoring invalid m.call.negotiate event`); // return; // } // // Politeness always follows the direction of the call: in a glare situation, @@ -516,7 +539,7 @@ export class PeerCall implements IDisposable { // this.ignoreOffer = !polite && offerCollision; // if (this.ignoreOffer) { - // this.logger.info(`Call ${this.callId} Ignoring colliding negotiate event because we're impolite`); + // this.logger.info(`Ignoring colliding negotiate event because we're impolite`); // return; // } @@ -524,7 +547,7 @@ export class PeerCall implements IDisposable { // if (sdpStreamMetadata) { // this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); // } else { - // this.logger.warn(`Call ${this.callId} Received negotiation event without SDPStreamMetadata!`); + // this.logger.warn(`Received negotiation event without SDPStreamMetadata!`); // } // try { @@ -532,7 +555,7 @@ export class PeerCall implements IDisposable { // if (description.type === 'offer') { // await this.peerConnection.setLocalDescription(); - // await this.options.sendSignallingMessage({ + // await this.sendSignallingMessage({ // type: EventType.CallNegotiate, // content: { // description: this.peerConnection.localDescription!, @@ -541,11 +564,11 @@ export class PeerCall implements IDisposable { // }); // } // } catch (err) { - // this.logger.warn(`Call ${this.callId} Failed to complete negotiation`, err); + // this.logger.warn(`Failed to complete negotiation`, err); // } // } - private async sendAnswer(): Promise { + private async sendAnswer(log: ILogItem): Promise { const localDescription = this.peerConnection.localDescription!; const answerContent: MCallAnswer = { call_id: this.callId, @@ -560,23 +583,23 @@ export class PeerCall implements IDisposable { // We have just taken the local description from the peerConn which will // contain all the local candidates added so far, so we can discard any candidates // we had queued up because they'll be in the answer. - this.logger.info(`Call ${this.callId} Discarding ${ + log.log(`Discarding ${ this.candidateSendQueue.length} candidates that will be sent in answer`); this.candidateSendQueue = []; try { - await this.options.sendSignallingMessage({type: EventType.Answer, content: answerContent}, logger.item); + await this.sendSignallingMessage({type: EventType.Answer, content: answerContent}, log); } catch (error) { - this.terminate(CallParty.Local, CallErrorCode.SendAnswer, false); + this.terminate(CallParty.Local, CallErrorCode.SendAnswer, false, log); throw error; } // error handler re-throws so this won't happen on error, but // we don't want the same error handling on the candidate queue - this.sendCandidateQueue(); + this.sendCandidateQueue(log); } - private queueCandidate(content: RTCIceCandidate): void { + private queueCandidate(content: RTCIceCandidate, log: ILogItem): void { // We partially de-trickle candidates by waiting for `delay` before sending them // amalgamated, in order to avoid sending too many m.call.candidates events and hitting // rate limits in Matrix. @@ -593,36 +616,48 @@ export class PeerCall implements IDisposable { // MSC2746 recommends these values (can be quite long when calling because the // callee will need a while to answer the call) - this.delay(this.direction === CallDirection.Inbound ? 500 : 2000).then(() => { - this.sendCandidateQueue(); - }); + const sendLogItem = this.logItem.child("wait to send candidates"); + log.refDetached(sendLogItem); + this.delay(this.direction === CallDirection.Inbound ? 500 : 2000) + .then(() => { + return this.sendCandidateQueue(sendLogItem); + }, err => {}) // swallow delay AbortError + .finally(() => { + sendLogItem.finish(); + }); } - private async sendCandidateQueue(): Promise { - if (this.candidateSendQueue.length === 0 || this._state === CallState.Ended) { - return; - } + private async sendCandidateQueue(log: ILogItem): Promise { + return log.wrap("send candidates queue", async log => { + log.set("queueLength", this.candidateSendQueue.length); - const candidates = this.candidateSendQueue; - this.candidateSendQueue = []; - this.logger.debug(`Call ${this.callId} attempting to send ${candidates.length} candidates`); - try { - await this.options.sendSignallingMessage({ - type: EventType.Candidates, - content: { - call_id: this.callId, - version: 1, - candidates - }, - }, logger.item); - // Try to send candidates again just in case we received more candidates while sending. - this.sendCandidateQueue(); - } catch (error) { - // don't retry this event: we'll send another one later as we might - // have more candidates by then. - // put all the candidates we failed to send back in the queue - this.terminate(CallParty.Local, CallErrorCode.SignallingFailed, false); - } + if (this.candidateSendQueue.length === 0 || this._state === CallState.Ended) { + return; + } + + const candidates = this.candidateSendQueue; + this.candidateSendQueue = []; + try { + await this.sendSignallingMessage({ + type: EventType.Candidates, + content: { + call_id: this.callId, + version: 1, + candidates + }, + }, log); + // Try to send candidates again just in case we received more candidates while sending. + this.sendCandidateQueue(log); + } catch (error) { + log.catch(error); + // don't retry this event: we'll send another one later as we might + // have more candidates by then. + // put all the candidates we failed to send back in the queue + + // TODO: terminate doesn't seem to vibe with the comment above? + this.terminate(CallParty.Local, CallErrorCode.SignallingFailed, false, log); + } + }); } private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { @@ -641,44 +676,44 @@ export class PeerCall implements IDisposable { } } - private async addBufferedIceCandidates(): Promise { + private async addBufferedIceCandidates(log: ILogItem): Promise { if (this.remoteCandidateBuffer && this.opponentPartyId) { const bufferedCandidates = this.remoteCandidateBuffer.get(this.opponentPartyId); if (bufferedCandidates) { - this.logger.info(`Call ${this.callId} Adding ${ + log.log(`Adding ${ bufferedCandidates.length} buffered candidates for opponent ${this.opponentPartyId}`); - await this.addIceCandidates(bufferedCandidates); + await this.addIceCandidates(bufferedCandidates, log); } this.remoteCandidateBuffer = undefined; } } - private async addIceCandidates(candidates: RTCIceCandidate[]): Promise { + private async addIceCandidates(candidates: RTCIceCandidate[], log: ILogItem): Promise { for (const candidate of candidates) { if ( (candidate.sdpMid === null || candidate.sdpMid === undefined) && (candidate.sdpMLineIndex === null || candidate.sdpMLineIndex === undefined) ) { - this.logger.debug(`Call ${this.callId} ignoring remote ICE candidate with no sdpMid or sdpMLineIndex`); + log.log(`Ignoring remote ICE candidate with no sdpMid or sdpMLineIndex`); continue; } - this.logger.debug(`Call ${this.callId} got remote ICE ${candidate.sdpMid} candidate: ${candidate.candidate}`); + log.log(`Got remote ICE ${candidate.sdpMid} candidate: ${candidate.candidate}`); try { await this.peerConnection.addIceCandidate(candidate); } catch (err) { if (!this.ignoreOffer) { - this.logger.info(`Call ${this.callId} failed to add remote ICE candidate`, err); + log.log(`Failed to add remote ICE candidate`, err); } } } } - private onIceConnectionStateChange = (state: RTCIceConnectionState): void => { + private onIceConnectionStateChange = (state: RTCIceConnectionState, log: ILogItem): void => { if (this._state === CallState.Ended) { return; // because ICE can still complete as we're ending the call } - this.logger.debug( - "Call ID " + this.callId + ": ICE connection state changed to: " + state, + log.log( + "ICE connection state changed to: " + state, ); // ideally we'd consider the call to be connected when we get media but // chrome doesn't implement any of the 'onstarted' events yet @@ -689,11 +724,11 @@ export class PeerCall implements IDisposable { } else if (state == 'failed') { this.iceDisconnectedTimeout?.abort(); this.iceDisconnectedTimeout = undefined; - this.hangup(CallErrorCode.IceFailed); + this._hangup(CallErrorCode.IceFailed, log); } else if (state == 'disconnected') { this.iceDisconnectedTimeout = this.options.createTimeout(30 * 1000); this.iceDisconnectedTimeout.elapsed().then(() => { - this.hangup(CallErrorCode.IceFailed); + this._hangup(CallErrorCode.IceFailed, log); }, () => { /* ignore AbortError */ }); } }; @@ -725,7 +760,7 @@ export class PeerCall implements IDisposable { })); } - private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean): Promise { + private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean, log: ILogItem): Promise { } @@ -744,6 +779,12 @@ export class PeerCall implements IDisposable { this.disposables.untrack(timeout); } + private sendSignallingMessage(message: SignallingMessage, log: ILogItem) { + return log.wrap({l: "send", id: message.type}, async log => { + return this.options.sendSignallingMessage(message, log); + }); + } + public dispose(): void { this.disposables.dispose(); this.peerConnection.dispose(); diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index a3148518..86c93ba3 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -61,10 +61,12 @@ export class GroupCall extends EventEmitter<{change: never}> { id: string | undefined, private callContent: Record | undefined, public readonly roomId: string, - private readonly options: Options + private readonly options: Options, + private readonly logItem: ILogItem, ) { super(); this.id = id ?? makeId("conf-"); + logItem.set("id", this.id); this._state = id ? GroupCallState.Created : GroupCallState.Fledgling; this._memberOptions = Object.assign({}, options, { confId: this.id, @@ -86,132 +88,158 @@ export class GroupCall extends EventEmitter<{change: never}> { return this.callContent?.["m.name"]; } - async join(localMedia: LocalMedia) { - if (this._state !== GroupCallState.Created) { - return; - } - this._state = GroupCallState.Joining; - this._localMedia = localMedia; - this.emitChange(); - const memberContent = await this._createJoinPayload(); - // send m.call.member state event - const request = this.options.hsApi.sendState(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId, memberContent); - await request.response(); - this.emitChange(); - // send invite to all members that are < my userId - for (const [,member] of this._members) { - member.connect(this._localMedia); - } + join(localMedia: LocalMedia): Promise { + return this.logItem.wrap("join", async log => { + if (this._state !== GroupCallState.Created) { + return; + } + this._state = GroupCallState.Joining; + this._localMedia = localMedia; + this.emitChange(); + const memberContent = await this._createJoinPayload(); + // send m.call.member state event + const request = this.options.hsApi.sendState(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId, memberContent, {log}); + await request.response(); + this.emitChange(); + // send invite to all members that are < my userId + for (const [,member] of this._members) { + member.connect(this._localMedia); + } + }); } get hasJoined() { return this._state === GroupCallState.Joining || this._state === GroupCallState.Joined; } - async leave() { - const memberContent = await this._leaveCallMemberContent(); - // send m.call.member state event - if (memberContent) { - const request = this.options.hsApi.sendState(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId, memberContent); - await request.response(); - // our own user isn't included in members, so not in the count - if (this._members.size === 0) { - this.terminate(); - } - } - } - - async terminate() { - if (this._state === GroupCallState.Fledgling) { - return; - } - const request = this.options.hsApi.sendState(this.roomId, CALL_TYPE, this.id, Object.assign({}, this.callContent, { - "m.terminated": true - })); - await request.response(); - } - - /** @internal */ - async create(localMedia: LocalMedia, name: string) { - if (this._state !== GroupCallState.Fledgling) { - return; - } - this._state = GroupCallState.Creating; - this.emitChange(); - this.callContent = { - "m.type": localMedia.cameraTrack ? "m.video" : "m.voice", - "m.name": name, - "m.intent": "m.ring" - }; - const request = this.options.hsApi.sendState(this.roomId, CALL_TYPE, this.id, this.callContent); - await request.response(); - this._state = GroupCallState.Created; - this.emitChange(); - } - - /** @internal */ - updateCallEvent(callContent: Record) { - this.callContent = callContent; - if (this._state === GroupCallState.Creating) { - this._state = GroupCallState.Created; - } - this.emitChange(); - } - - /** @internal */ - addMember(userId, memberCallInfo) { - if (userId === this.options.ownUserId) { - if (this._state === GroupCallState.Joining) { - this._state = GroupCallState.Joined; - this.emitChange(); - } - return; - } - let member = this._members.get(userId); - if (member) { - member.updateCallInfo(memberCallInfo); - } else { - member = new Member(RoomMember.fromUserId(this.roomId, userId, "join"), memberCallInfo, this._memberOptions); - this._members.add(userId, member); - if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { - member.connect(this._localMedia!); - } - } - } - - /** @internal */ - removeMember(userId) { - if (userId === this.options.ownUserId) { - if (this._state === GroupCallState.Joined) { - this._localMedia?.dispose(); - this._localMedia = undefined; - for (const [,member] of this._members) { - member.disconnect(); + leave(): Promise { + return this.logItem.wrap("leave", async log => { + const memberContent = await this._leaveCallMemberContent(); + // send m.call.member state event + if (memberContent) { + const request = this.options.hsApi.sendState(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId, memberContent, {log}); + await request.response(); + // our own user isn't included in members, so not in the count + if (this._members.size === 0) { + await this.terminate(); } + } + }); + } + + terminate(): Promise { + return this.logItem.wrap("terminate", async log => { + if (this._state === GroupCallState.Fledgling) { + return; + } + const request = this.options.hsApi.sendState(this.roomId, CALL_TYPE, this.id, Object.assign({}, this.callContent, { + "m.terminated": true + }), {log}); + await request.response(); + }); + } + + /** @internal */ + create(localMedia: LocalMedia, name: string): Promise { + return this.logItem.wrap("create", async log => { + if (this._state !== GroupCallState.Fledgling) { + return; + } + this._state = GroupCallState.Creating; + this.emitChange(); + this.callContent = { + "m.type": localMedia.cameraTrack ? "m.video" : "m.voice", + "m.name": name, + "m.intent": "m.ring" + }; + const request = this.options.hsApi.sendState(this.roomId, CALL_TYPE, this.id, this.callContent, {log}); + await request.response(); + this._state = GroupCallState.Created; + this.emitChange(); + }); + } + + /** @internal */ + updateCallEvent(callContent: Record, syncLog: ILogItem) { + this.logItem.wrap("updateCallEvent", log => { + syncLog.refDetached(log); + this.callContent = callContent; + if (this._state === GroupCallState.Creating) { this._state = GroupCallState.Created; } - } else { - const member = this._members.get(userId); - if (member) { - this._members.remove(userId); - member.disconnect(); - } - } - this.emitChange(); + log.set("status", this._state); + this.emitChange(); + }); } /** @internal */ - handleDeviceMessage(message: SignallingMessage, userId: string, deviceId: string, log: ILogItem) { - console.log("incoming to_device call signalling message from", userId, deviceId, message); + addMember(userId: string, memberCallInfo, syncLog: ILogItem) { + this.logItem.wrap({l: "addMember", id: userId}, log => { + syncLog.refDetached(log); + + if (userId === this.options.ownUserId) { + if (this._state === GroupCallState.Joining) { + this._state = GroupCallState.Joined; + this.emitChange(); + } + return; + } + let member = this._members.get(userId); + if (member) { + member.updateCallInfo(memberCallInfo); + } else { + const logItem = this.logItem.child("member"); + member = new Member(RoomMember.fromUserId(this.roomId, userId, "join"), memberCallInfo, this._memberOptions, logItem); + this._members.add(userId, member); + if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { + member.connect(this._localMedia!); + } + } + }); + } + + /** @internal */ + removeMember(userId: string, syncLog: ILogItem) { + this.logItem.wrap({l: "removeMember", id: userId}, log => { + syncLog.refDetached(log); + if (userId === this.options.ownUserId) { + if (this._state === GroupCallState.Joined) { + this._localMedia?.dispose(); + this._localMedia = undefined; + for (const [,member] of this._members) { + member.disconnect(); + } + this._state = GroupCallState.Created; + } + } else { + const member = this._members.get(userId); + if (member) { + this._members.remove(userId); + member.disconnect(); + } + } + this.emitChange(); + }); + } + + /** @internal */ + handleDeviceMessage(message: SignallingMessage, userId: string, deviceId: string, syncLog: ILogItem) { // TODO: return if we are not membering to the call let member = this._members.get(userId); if (member) { - member.handleDeviceMessage(message, deviceId, log); + member.handleDeviceMessage(message, deviceId, syncLog); } else { + const item = this.logItem.log({l: "could not find member for signalling message", userId, deviceId}); + syncLog.refDetached(item); // we haven't received the m.call.member yet for this caller. buffer the device messages or create the member/call anyway? } } + /** @internal */ + dispose() { + this.logItem.finish(); + } + private async _createJoinPayload() { const {storage} = this.options; const txn = await storage.readTxn([storage.storeNames.roomState]); diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index dedde429..a2dc43de 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -44,8 +44,11 @@ export class Member { constructor( public readonly member: RoomMember, private memberCallInfo: Record, - private readonly options: Options - ) {} + private readonly options: Options, + private readonly logItem: ILogItem, + ) { + logItem.set("id", member.userId); + } get remoteTracks(): Track[] { return this.peerCall?.remoteTracks ?? []; @@ -57,6 +60,7 @@ export class Member { /** @internal */ connect(localMedia: LocalMedia) { + this.logItem.log("connect"); this.localMedia = localMedia; // otherwise wait for it to connect if (this.member.userId < this.options.ownUserId) { @@ -71,6 +75,7 @@ export class Member { this.peerCall?.dispose(); this.peerCall = undefined; this.localMedia = undefined; + this.logItem.log("disconnect"); } /** @internal */ @@ -87,7 +92,7 @@ export class Member { } /** @internal */ - sendSignallingMessage = async (message: SignallingMessage, log: ILogItem) => { + sendSignallingMessage = async (message: SignallingMessage, log: ILogItem): Promise => { const groupMessage = message as SignallingMessage; groupMessage.content.conf_id = this.options.confId; const encryptedMessages = await this.options.encryptDeviceMessage(this.member.userId, groupMessage, log); @@ -102,12 +107,13 @@ export class Member { } /** @internal */ - handleDeviceMessage(message: SignallingMessage, deviceId: string, log: ILogItem) { + handleDeviceMessage(message: SignallingMessage, deviceId: string, syncLog: ILogItem) { + syncLog.refDetached(this.logItem); if (message.type === EventType.Invite && !this.peerCall) { this.peerCall = this._createPeerCall(message.content.call_id); } if (this.peerCall) { - this.peerCall.handleIncomingSignallingMessage(message, deviceId, log); + this.peerCall.handleIncomingSignallingMessage(message, deviceId); } else { // TODO: need to buffer events until invite comes? } @@ -117,6 +123,6 @@ export class Member { return new PeerCall(callId, Object.assign({}, this.options, { emitUpdate: this.emitUpdate, sendSignallingMessage: this.sendSignallingMessage - })); + }), this.logItem); } } From 4bf171def9ce111421f8ab5900a9aa0b1d58c6a3 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 25 Mar 2022 14:43:22 +0100 Subject: [PATCH 029/435] small fixes --- scripts/logviewer/main.js | 2 +- src/domain/session/SessionViewModel.js | 2 +- src/observable/value/PickMapObservableValue.ts | 2 +- src/platform/web/ui/session/room/common.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/logviewer/main.js b/scripts/logviewer/main.js index b6883667..dce82fb8 100644 --- a/scripts/logviewer/main.js +++ b/scripts/logviewer/main.js @@ -395,4 +395,4 @@ document.getElementById("showAll").addEventListener("click", () => { for (const node of document.querySelectorAll(".hidden")) { node.classList.remove("hidden"); } -}); \ No newline at end of file +}); diff --git a/src/domain/session/SessionViewModel.js b/src/domain/session/SessionViewModel.js index 0c9dddaa..4e5930b1 100644 --- a/src/domain/session/SessionViewModel.js +++ b/src/domain/session/SessionViewModel.js @@ -191,7 +191,7 @@ export class SessionViewModel extends ViewModel { async _createArchivedRoomViewModel(roomId) { const room = await this._client.session.loadArchivedRoom(roomId); if (room) { - const roomVM = new RoomViewModel(this.childOptions({room})); + const roomVM = new RoomViewModel(this.childOptions({room, session: this._client.session})); roomVM.load(); return roomVM; } diff --git a/src/observable/value/PickMapObservableValue.ts b/src/observable/value/PickMapObservableValue.ts index b493d841..0b30fcfd 100644 --- a/src/observable/value/PickMapObservableValue.ts +++ b/src/observable/value/PickMapObservableValue.ts @@ -62,7 +62,7 @@ export class PickMapObservableValue extends BaseObservableValue Date: Fri, 25 Mar 2022 16:53:12 +0100 Subject: [PATCH 030/435] log state changes in PeerCall --- src/matrix/calls/PeerCall.ts | 61 ++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index c4355e63..975b1dff 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -113,10 +113,10 @@ export class PeerCall implements IDisposable { }, onDataChannelChanged(dataChannel: DataChannel | undefined) {}, onNegotiationNeeded() { - const log = outer.logItem.child("onNegotiationNeeded"); - const promiseCreator = async () => { - await outer.handleNegotiation(log); - log.finish(); + const promiseCreator = () => { + return outer.logItem.wrap("onNegotiationNeeded", log => { + return outer.handleNegotiation(log); + }); }; outer.responsePromiseChain = outer.responsePromiseChain?.then(promiseCreator) ?? promiseCreator(); }, @@ -139,7 +139,7 @@ export class PeerCall implements IDisposable { } this.localMedia = localMedia; this.direction = CallDirection.Outbound; - this.setState(CallState.CreateOffer); + this.setState(CallState.CreateOffer, log); for (const t of this.localMedia.tracks) { this.peerConnection.addTrack(t); } @@ -155,7 +155,7 @@ export class PeerCall implements IDisposable { return; } this.localMedia = localMedia; - this.setState(CallState.CreateAnswer); + this.setState(CallState.CreateAnswer, log); for (const t of this.localMedia.tracks) { this.peerConnection.addTrack(t); } @@ -173,7 +173,7 @@ export class PeerCall implements IDisposable { try { await this.peerConnection.setLocalDescription(myAnswer); - this.setState(CallState.Connecting); + this.setState(CallState.Connecting, log); } catch (err) { await log.wrap(`Error setting local description!`, log => { log.catch(err); @@ -305,7 +305,7 @@ export class PeerCall implements IDisposable { }; if (this._state === CallState.CreateOffer) { await this.sendSignallingMessage({type: EventType.Invite, content}, log); - this.setState(CallState.InviteSent); + this.setState(CallState.InviteSent, log); } else if (this._state === CallState.Connected || this._state === CallState.Connecting) { // send Negotiate message //await this.sendSignallingMessage({type: EventType.Invite, content}); @@ -317,14 +317,16 @@ export class PeerCall implements IDisposable { this.sendCandidateQueue(log); - if (this._state === CallState.InviteSent) { - try { await this.delay(CALL_TIMEOUT_MS); } - catch (err) { return; } - // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between + await log.wrap("invite timeout", async log => { if (this._state === CallState.InviteSent) { - this._hangup(CallErrorCode.InviteTimeout, log); + try { await this.delay(CALL_TIMEOUT_MS); } + catch (err) { return; } + // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between + if (this._state === CallState.InviteSent) { + this._hangup(CallErrorCode.InviteTimeout, log); + } } - } + }); }; private async handleInviteGlare(content: MCallInvite, partyId: PartyId, log: ILogItem): Promise { @@ -398,7 +400,7 @@ export class PeerCall implements IDisposable { return; } - this.setState(CallState.Ringing); + this.setState(CallState.Ringing, log); try { await this.delay(content.lifetime ?? CALL_TIMEOUT_MS); } catch (err) { return; } @@ -406,7 +408,7 @@ export class PeerCall implements IDisposable { if (this._state === CallState.Ringing) { log.log(`Invite has expired. Hanging up.`); this.hangupParty = CallParty.Remote; // effectively - this.setState(CallState.Ended); + this.setState(CallState.Ended, log); this.stopAllMedia(); if (this.peerConnection.signalingState != 'closed') { this.peerConnection.close(); @@ -428,7 +430,7 @@ export class PeerCall implements IDisposable { this.opponentPartyId = partyId; await this.addBufferedIceCandidates(log); - this.setState(CallState.Connecting); + this.setState(CallState.Connecting, log); const sdpStreamMetadata = content[SDPStreamMetadataKey]; if (sdpStreamMetadata) { @@ -449,7 +451,6 @@ export class PeerCall implements IDisposable { } private handleIceGatheringState(state: RTCIceGatheringState, log: ILogItem) { - log.set("state", state); if (state === 'complete' && !this.sentEndOfCandidates) { // If we didn't get an empty-string candidate to signal the end of candidates, // create one ourselves now gathering has finished. @@ -712,15 +713,12 @@ export class PeerCall implements IDisposable { if (this._state === CallState.Ended) { return; // because ICE can still complete as we're ending the call } - log.log( - "ICE connection state changed to: " + state, - ); // ideally we'd consider the call to be connected when we get media but // chrome doesn't implement any of the 'onstarted' events yet if (state == 'connected') { this.iceDisconnectedTimeout?.abort(); this.iceDisconnectedTimeout = undefined; - this.setState(CallState.Connected); + this.setState(CallState.Connected, log); } else if (state == 'failed') { this.iceDisconnectedTimeout?.abort(); this.iceDisconnectedTimeout = undefined; @@ -733,15 +731,18 @@ export class PeerCall implements IDisposable { } }; - private setState(state: CallState): void { - const oldState = this._state; - this._state = state; - let deferred = this.statePromiseMap.get(state); - if (deferred) { - deferred.resolve(); - this.statePromiseMap.delete(state); + private setState(state: CallState, log: ILogItem): void { + if (state !== this._state) { + log.log({l: "change state", status: state, oldState: this._state}); + const oldState = this._state; + this._state = state; + let deferred = this.statePromiseMap.get(state); + if (deferred) { + deferred.resolve(); + this.statePromiseMap.delete(state); + } + this.options.emitUpdate(this, undefined); } - this.options.emitUpdate(this, undefined); } private waitForState(states: CallState[]): Promise { From ba45178e04d625e3b909aad7b50d747f84fe08bb Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 29 Mar 2022 12:00:54 +0200 Subject: [PATCH 031/435] implement terminate and hangup (currently unused) --- src/matrix/calls/PeerCall.ts | 45 ++++++++++++++++++++++---------- src/matrix/calls/group/Member.ts | 11 ++++---- 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 975b1dff..3e23b3a6 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -166,7 +166,7 @@ export class PeerCall implements IDisposable { } catch (err) { await log.wrap(`Failed to create answer`, log => { log.catch(err); - this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true, log); + this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, log); }); return; } @@ -177,7 +177,7 @@ export class PeerCall implements IDisposable { } catch (err) { await log.wrap(`Error setting local description!`, log => { log.catch(err); - this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true, log); + this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, log); }); return; } @@ -212,6 +212,7 @@ export class PeerCall implements IDisposable { }); } + /** group calls would handle reject at the group call level, not at the peer call level */ async reject() { } @@ -223,10 +224,11 @@ export class PeerCall implements IDisposable { } private async _hangup(errorCode: CallErrorCode, log: ILogItem): Promise { - if (this._state !== CallState.Ended) { - this._state = CallState.Ended; - await this.sendHangupWithCallId(this.callId, errorCode, log); + if (this._state === CallState.Ended) { + return; } + this.terminate(CallParty.Local, errorCode, log); + await this.sendHangupWithCallId(this.callId, errorCode, log); } handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId): Promise { @@ -257,6 +259,7 @@ export class PeerCall implements IDisposable { call_id: callId, version: 1, }; + // TODO: Don't send UserHangup reason to older clients if (reason) { content["reason"] = reason; } @@ -274,7 +277,7 @@ export class PeerCall implements IDisposable { await this.peerConnection.setLocalDescription(); } catch (err) { log.log(`Error setting local description!`).catch(err); - this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true, log); + this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, log); return; } @@ -385,7 +388,7 @@ export class PeerCall implements IDisposable { } catch (e) { await log.wrap(`Call failed to set remote description`, async log => { log.catch(e); - return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false, log); + return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); }); return; } @@ -395,7 +398,7 @@ export class PeerCall implements IDisposable { // (81 at time of writing), this is no longer a problem, so let's do it the correct way. if (this.peerConnection.remoteTracks.length === 0) { await log.wrap(`Call no remote stream or no tracks after setting remote description!`, async log => { - return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false, log); + return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); }); return; } @@ -444,7 +447,7 @@ export class PeerCall implements IDisposable { } catch (e) { await log.wrap(`Failed to set remote description`, log => { log.catch(e); - this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false, log); + this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); }); return; } @@ -591,7 +594,7 @@ export class PeerCall implements IDisposable { try { await this.sendSignallingMessage({type: EventType.Answer, content: answerContent}, log); } catch (error) { - this.terminate(CallParty.Local, CallErrorCode.SendAnswer, false, log); + this.terminate(CallParty.Local, CallErrorCode.SendAnswer, log); throw error; } @@ -656,7 +659,7 @@ export class PeerCall implements IDisposable { // put all the candidates we failed to send back in the queue // TODO: terminate doesn't seem to vibe with the comment above? - this.terminate(CallParty.Local, CallErrorCode.SignallingFailed, false, log); + this.terminate(CallParty.Local, CallErrorCode.SignallingFailed, log); } }); } @@ -761,8 +764,19 @@ export class PeerCall implements IDisposable { })); } - private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean, log: ILogItem): Promise { + private terminate(hangupParty: CallParty, hangupReason: CallErrorCode, log: ILogItem): void { + if (this._state === CallState.Ended) { + return; + } + this.hangupParty = hangupParty; + // this.hangupReason = hangupReason; + this.setState(CallState.Ended, log); + this.stopAllMedia(); + + if (this.peerConnection && this.peerConnection.signalingState !== 'closed') { + this.peerConnection.close(); + } } private stopAllMedia(): void { @@ -791,8 +805,11 @@ export class PeerCall implements IDisposable { this.peerConnection.dispose(); } - public close(): void { - this.peerConnection.close(); + public close(reason: CallErrorCode | undefined, log: ILogItem): void { + if (reason === undefined) { + reason = CallErrorCode.UserHangup; + } + this.terminate(CallParty.Local, reason, log); } } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index a2dc43de..89e0db4c 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -71,11 +71,12 @@ export class Member { /** @internal */ disconnect() { - this.peerCall?.close(); - this.peerCall?.dispose(); - this.peerCall = undefined; - this.localMedia = undefined; - this.logItem.log("disconnect"); + this.logItem.wrap("disconnect", log => { + this.peerCall?.close(undefined, log); + this.peerCall?.dispose(); + this.peerCall = undefined; + this.localMedia = undefined; + }); } /** @internal */ From c54ffd4fc3ef67c1fe14d7ac0b72583248933022 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 29 Mar 2022 17:13:33 +0200 Subject: [PATCH 032/435] support multiple devices in call per user --- src/matrix/calls/CallHandler.ts | 6 +- src/matrix/calls/LocalMedia.ts | 5 ++ src/matrix/calls/TODO.md | 1 + src/matrix/calls/callEventTypes.ts | 13 ++++ src/matrix/calls/group/GroupCall.ts | 92 +++++++++++++++++++++-------- src/matrix/calls/group/Member.ts | 38 ++++++++---- src/observable/map/ObservableMap.ts | 6 +- 7 files changed, 117 insertions(+), 44 deletions(-) diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index ef617cd2..678f4f44 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -55,7 +55,6 @@ export class CallHandler { async createCall(roomId: string, localMedia: LocalMedia, name: string): Promise { const logItem = this.options.logger.child({l: "call", incoming: false}); const call = new GroupCall(undefined, undefined, roomId, this.groupCallOptions, logItem); - console.log("created call with id", call.id); this._calls.set(call.id, call); try { await call.create(localMedia, name); @@ -67,9 +66,7 @@ export class CallHandler { } throw err; } - console.log("joining call I just created"); await call.join(localMedia); - console.log("joined!"); return call; } @@ -79,7 +76,6 @@ export class CallHandler { /** @internal */ handleRoomState(room: Room, events: StateEvent[], log: ILogItem) { - console.log("handling room state"); // first update call events for (const event of events) { if (event.type === EventType.GroupCall) { @@ -135,7 +131,7 @@ export class CallHandler { const callId = call["m.call_id"]; const groupCall = this._calls.get(callId); // TODO: also check the member when receiving the m.call event - groupCall?.addMember(userId, call, log); + groupCall?.updateMember(userId, call, log); }; const newCallIdsMemberOf = new Set(calls.map(call => call["m.call_id"])); let previousCallIdsMemberOf = this.memberToCallIds.get(userId); diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index b148dd92..b64bdee5 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -61,6 +61,11 @@ export class LocalMedia { return metadata; } + clone() { + // TODO: implement + return this; + } + dispose() { this.cameraTrack?.stop(); this.microphoneTrack?.stop(); diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index a07da60e..4faf4f4e 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -108,6 +108,7 @@ Expose call objects expose volume events from audiotrack to group call Write view model write view + - handle glare edge-cases (not yet sent): https://spec.matrix.org/latest/client-server-api/#glare ## Calls questions - how do we handle glare between group calls (e.g. different state events with different call ids?) diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index 0e9eb8f8..4416087b 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -22,6 +22,19 @@ export enum EventType { // TODO: Change to "sdp_stream_metadata" when MSC3077 is merged export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata"; +export interface CallDeviceMembership { + device_id: string +} + +export interface CallMembership { + ["m.call_id"]: string, + ["m.devices"]: CallDeviceMembership[] +} + +export interface CallMemberContent { + ["m.calls"]: CallMembership[]; +} + export interface SessionDescription { sdp?: string; type: RTCSdpType diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 86c93ba3..3ef6a5ff 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -24,7 +24,7 @@ import {EventEmitter} from "../../../utils/EventEmitter"; import type {Options as MemberOptions} from "./Member"; import type {BaseObservableMap} from "../../../observable/map/BaseObservableMap"; import type {Track} from "../../../platform/types/MediaDevices"; -import type {SignallingMessage, MGroupCallBase} from "../callEventTypes"; +import type {SignallingMessage, MGroupCallBase, CallMembership} from "../callEventTypes"; import type {Room} from "../../room/Room"; import type {StateEvent} from "../../storage/types"; import type {Platform} from "../../../platform/web/Platform"; @@ -43,11 +43,22 @@ export enum GroupCallState { Joined = "joined", } +function getMemberKey(userId: string, deviceId: string) { + return JSON.stringify(userId)+`,`+JSON.stringify(deviceId); +} + +function memberKeyIsForUser(key: string, userId: string) { + return key.startsWith(JSON.stringify(userId)+`,`); +} + +function getDeviceFromMemberKey(key: string): string { + return JSON.parse(`[${key}]`)[1]; +} + export type Options = Omit & { emitUpdate: (call: GroupCall, params?: any) => void; encryptDeviceMessage: (roomId: string, userId: string, message: SignallingMessage, log: ILogItem) => Promise, storage: Storage, - ownDeviceId: string }; export class GroupCall extends EventEmitter<{change: never}> { @@ -70,7 +81,7 @@ export class GroupCall extends EventEmitter<{change: never}> { this._state = id ? GroupCallState.Created : GroupCallState.Fledgling; this._memberOptions = Object.assign({}, options, { confId: this.id, - emitUpdate: member => this._members.update(member.member.userId, member), + emitUpdate: member => this._members.update(getMemberKey(member.userId, member.deviceId), member), encryptDeviceMessage: (userId: string, message: SignallingMessage, log) => { return this.options.encryptDeviceMessage(this.roomId, userId, message, log); } @@ -173,26 +184,42 @@ export class GroupCall extends EventEmitter<{change: never}> { } /** @internal */ - addMember(userId: string, memberCallInfo, syncLog: ILogItem) { - this.logItem.wrap({l: "addMember", id: userId}, log => { + updateMember(userId: string, callMembership: CallMembership, syncLog: ILogItem) { + this.logItem.wrap({l: "updateMember", id: userId}, log => { syncLog.refDetached(log); - - if (userId === this.options.ownUserId) { - if (this._state === GroupCallState.Joining) { - this._state = GroupCallState.Joined; - this.emitChange(); + const devices = callMembership["m.devices"]; + const previousDeviceIds = this.getDeviceIdsForUserId(userId); + for (const device of devices) { + const deviceId = device.device_id; + const memberKey = getMemberKey(userId, deviceId); + if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) { + if (this._state === GroupCallState.Joining) { + this._state = GroupCallState.Joined; + this.emitChange(); + } + return; + } + let member = this._members.get(memberKey); + if (member) { + member.updateCallInfo(device); + } else { + const logItem = this.logItem.child("member"); + member = new Member( + RoomMember.fromUserId(this.roomId, userId, "join"), + device, this._memberOptions, logItem + ); + this._members.add(memberKey, member); + if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { + member.connect(this._localMedia!.clone()); + } } - return; } - let member = this._members.get(userId); - if (member) { - member.updateCallInfo(memberCallInfo); - } else { - const logItem = this.logItem.child("member"); - member = new Member(RoomMember.fromUserId(this.roomId, userId, "join"), memberCallInfo, this._memberOptions, logItem); - this._members.add(userId, member); - if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { - member.connect(this._localMedia!); + + const newDeviceIds = new Set(devices.map(call => call.device_id)); + // remove user as member of any calls not present anymore + for (const previousDeviceId of previousDeviceIds) { + if (!newDeviceIds.has(previousDeviceId)) { + this.removeMemberDevice(userId, previousDeviceId, syncLog); } } }); @@ -200,9 +227,24 @@ export class GroupCall extends EventEmitter<{change: never}> { /** @internal */ removeMember(userId: string, syncLog: ILogItem) { - this.logItem.wrap({l: "removeMember", id: userId}, log => { + const deviceIds = this.getDeviceIdsForUserId(userId); + for (const deviceId of deviceIds) { + this.removeMemberDevice(userId, deviceId, syncLog); + } + } + + private getDeviceIdsForUserId(userId: string): string[] { + return Array.from(this._members.keys()) + .filter(key => memberKeyIsForUser(key, userId)) + .map(key => getDeviceFromMemberKey(key)); + } + + /** @internal */ + private removeMemberDevice(userId: string, deviceId: string, syncLog: ILogItem) { + const memberKey = getMemberKey(userId, deviceId); + this.logItem.wrap({l: "removeMemberDevice", id: memberKey}, log => { syncLog.refDetached(log); - if (userId === this.options.ownUserId) { + if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) { if (this._state === GroupCallState.Joined) { this._localMedia?.dispose(); this._localMedia = undefined; @@ -212,9 +254,9 @@ export class GroupCall extends EventEmitter<{change: never}> { this._state = GroupCallState.Created; } } else { - const member = this._members.get(userId); + const member = this._members.get(memberKey); if (member) { - this._members.remove(userId); + this._members.remove(memberKey); member.disconnect(); } } @@ -225,7 +267,7 @@ export class GroupCall extends EventEmitter<{change: never}> { /** @internal */ handleDeviceMessage(message: SignallingMessage, userId: string, deviceId: string, syncLog: ILogItem) { // TODO: return if we are not membering to the call - let member = this._members.get(userId); + let member = this._members.get(getMemberKey(userId, deviceId)); if (member) { member.handleDeviceMessage(message, deviceId, syncLog); } else { diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 89e0db4c..d31837f4 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -23,7 +23,7 @@ import type {Options as PeerCallOptions} from "../PeerCall"; import type {LocalMedia} from "../LocalMedia"; import type {HomeServerApi} from "../../net/HomeServerApi"; import type {Track} from "../../../platform/types/MediaDevices"; -import type {MCallBase, MGroupCallBase, SignallingMessage} from "../callEventTypes"; +import type {MCallBase, MGroupCallBase, SignallingMessage, CallDeviceMembership} from "../callEventTypes"; import type {GroupCall} from "./GroupCall"; import type {RoomMember} from "../../room/members/RoomMember"; import type {EncryptedMessage} from "../../e2ee/olm/Encryption"; @@ -32,6 +32,7 @@ import type {ILogItem} from "../../../logging/types"; export type Options = Omit & { confId: string, ownUserId: string, + ownDeviceId: string, hsApi: HomeServerApi, encryptDeviceMessage: (userId: string, message: SignallingMessage, log: ILogItem) => Promise, emitUpdate: (participant: Member, params?: any) => void, @@ -43,7 +44,7 @@ export class Member { constructor( public readonly member: RoomMember, - private memberCallInfo: Record, + private callDeviceMembership: CallDeviceMembership, private readonly options: Options, private readonly logItem: ILogItem, ) { @@ -58,15 +59,30 @@ export class Member { return this.peerCall?.state === CallState.Connected; } + get userId(): string { + return this.member.userId; + } + + get deviceId(): string { + return this.callDeviceMembership.device_id; + } + /** @internal */ connect(localMedia: LocalMedia) { - this.logItem.log("connect"); - this.localMedia = localMedia; - // otherwise wait for it to connect - if (this.member.userId < this.options.ownUserId) { - this.peerCall = this._createPeerCall(makeId("c")); - this.peerCall.call(localMedia); - } + this.logItem.wrap("connect", () => { + this.localMedia = localMedia; + // otherwise wait for it to connect + let shouldInitiateCall; + if (this.member.userId === this.options.ownUserId) { + shouldInitiateCall = this.deviceId < this.options.ownDeviceId; + } else { + shouldInitiateCall = this.member.userId < this.options.ownUserId; + } + if (shouldInitiateCall) { + this.peerCall = this._createPeerCall(makeId("c")); + this.peerCall.call(localMedia); + } + }); } /** @internal */ @@ -80,8 +96,8 @@ export class Member { } /** @internal */ - updateCallInfo(memberCallInfo) { - // m.calls object from the m.call.member event + updateCallInfo(callDeviceMembership: CallDeviceMembership) { + this.callDeviceMembership = callDeviceMembership; } /** @internal */ diff --git a/src/observable/map/ObservableMap.ts b/src/observable/map/ObservableMap.ts index d604ab0a..79662e29 100644 --- a/src/observable/map/ObservableMap.ts +++ b/src/observable/map/ObservableMap.ts @@ -80,15 +80,15 @@ export class ObservableMap extends BaseObservableMap { return this._values.size; } - [Symbol.iterator](): Iterator<[K, V]> { + [Symbol.iterator](): IterableIterator<[K, V]> { return this._values.entries(); } - values(): Iterator { + values(): IterableIterator { return this._values.values(); } - keys(): Iterator { + keys(): IterableIterator { return this._values.keys(); } } From d7360e774103feae4225c7be1afb67760e7a1489 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 30 Mar 2022 15:18:46 +0200 Subject: [PATCH 033/435] fix multiple device support --- src/matrix/DeviceMessageHandler.js | 6 +- src/matrix/calls/CallHandler.ts | 4 +- src/matrix/calls/PeerCall.ts | 14 +-- src/matrix/calls/group/GroupCall.ts | 133 +++++++++++++++++---------- src/matrix/calls/group/Member.ts | 4 +- src/platform/types/MediaDevices.ts | 1 + src/platform/web/dom/MediaDevices.ts | 4 + 7 files changed, 103 insertions(+), 63 deletions(-) diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index 80fd1592..ee10716e 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -59,7 +59,11 @@ export class DeviceMessageHandler { })); // TODO: pass this in the prep and run it in afterSync or afterSyncComplete (as callHandler can send events as well)? for (const dr of callMessages) { - this._callHandler.handleDeviceMessage(dr.event, dr.device.userId, dr.device.deviceId, log); + if (dr.device) { + this._callHandler.handleDeviceMessage(dr.event, dr.device.userId, dr.device.deviceId, log); + } else { + console.error("could not deliver message because don't have device for sender key", dr.event); + } } // TODO: somehow include rooms that received a call to_device message in the sync state? // or have updates flow through event emitter? diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 678f4f44..89730391 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -131,7 +131,7 @@ export class CallHandler { const callId = call["m.call_id"]; const groupCall = this._calls.get(callId); // TODO: also check the member when receiving the m.call event - groupCall?.updateMember(userId, call, log); + groupCall?.updateMembership(userId, call, log); }; const newCallIdsMemberOf = new Set(calls.map(call => call["m.call_id"])); let previousCallIdsMemberOf = this.memberToCallIds.get(userId); @@ -140,7 +140,7 @@ export class CallHandler { for (const previousCallId of previousCallIdsMemberOf) { if (!newCallIdsMemberOf.has(previousCallId)) { const groupCall = this._calls.get(previousCallId); - groupCall?.removeMember(userId, log); + groupCall?.removeMembership(userId, log); } } } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 3e23b3a6..b85cee61 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -412,7 +412,8 @@ export class PeerCall implements IDisposable { log.log(`Invite has expired. Hanging up.`); this.hangupParty = CallParty.Remote; // effectively this.setState(CallState.Ended, log); - this.stopAllMedia(); + //this.localMedia?.dispose(); + //this.localMedia = undefined; if (this.peerConnection.signalingState != 'closed') { this.peerConnection.close(); } @@ -772,21 +773,14 @@ export class PeerCall implements IDisposable { this.hangupParty = hangupParty; // this.hangupReason = hangupReason; this.setState(CallState.Ended, log); - this.stopAllMedia(); + //this.localMedia?.dispose(); + //this.localMedia = undefined; if (this.peerConnection && this.peerConnection.signalingState !== 'closed') { this.peerConnection.close(); } } - private stopAllMedia(): void { - if (this.localMedia) { - for (const track of this.localMedia.tracks) { - track.stop(); - } - } - } - private async delay(timeoutMs: number): Promise { // Allow a short time for initial candidates to be gathered const timeout = this.disposables.track(this.options.createTimeout(timeoutMs)); diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 3ef6a5ff..98e3381d 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -95,10 +95,18 @@ export class GroupCall extends EventEmitter<{change: never}> { return this.callContent?.["m.terminated"] === true; } + get isRinging(): boolean { + return this._state === GroupCallState.Created && this.intent === "m.ring" && !this.isMember(this.options.ownUserId); + } + get name(): string { return this.callContent?.["m.name"]; } + get intent(): string { + return this.callContent?.["m.intent"]; + } + join(localMedia: LocalMedia): Promise { return this.logItem.wrap("join", async log => { if (this._state !== GroupCallState.Created) { @@ -134,6 +142,8 @@ export class GroupCall extends EventEmitter<{change: never}> { if (this._members.size === 0) { await this.terminate(); } + } else { + log.set("already_left", true); } }); } @@ -184,7 +194,7 @@ export class GroupCall extends EventEmitter<{change: never}> { } /** @internal */ - updateMember(userId: string, callMembership: CallMembership, syncLog: ILogItem) { + updateMembership(userId: string, callMembership: CallMembership, syncLog: ILogItem) { this.logItem.wrap({l: "updateMember", id: userId}, log => { syncLog.refDetached(log); const devices = callMembership["m.devices"]; @@ -192,45 +202,62 @@ export class GroupCall extends EventEmitter<{change: never}> { for (const device of devices) { const deviceId = device.device_id; const memberKey = getMemberKey(userId, deviceId); - if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) { - if (this._state === GroupCallState.Joining) { - this._state = GroupCallState.Joined; - this.emitChange(); + log.wrap({l: "update device member", id: memberKey}, log => { + if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) { + if (this._state === GroupCallState.Joining) { + log.set("update_own", true); + this._state = GroupCallState.Joined; + this.emitChange(); + } + } else { + let member = this._members.get(memberKey); + if (member) { + log.set("update", true); + member!.updateCallInfo(device); + } else { + const logItem = this.logItem.child({l: "member", id: memberKey}); + log.set("add", true); + log.refDetached(logItem); + member = new Member( + RoomMember.fromUserId(this.roomId, userId, "join"), + device, this._memberOptions, logItem + ); + this._members.add(memberKey, member); + if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { + member.connect(this._localMedia!.clone()); + } + } } - return; - } - let member = this._members.get(memberKey); - if (member) { - member.updateCallInfo(device); - } else { - const logItem = this.logItem.child("member"); - member = new Member( - RoomMember.fromUserId(this.roomId, userId, "join"), - device, this._memberOptions, logItem - ); - this._members.add(memberKey, member); - if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { - member.connect(this._localMedia!.clone()); - } - } + }); } const newDeviceIds = new Set(devices.map(call => call.device_id)); // remove user as member of any calls not present anymore for (const previousDeviceId of previousDeviceIds) { if (!newDeviceIds.has(previousDeviceId)) { - this.removeMemberDevice(userId, previousDeviceId, syncLog); + log.wrap({l: "remove device member", id: getMemberKey(userId, previousDeviceId)}, log => { + this.removeMemberDevice(userId, previousDeviceId, log); + }); } } + if (userId === this.options.ownUserId && !newDeviceIds.has(this.options.ownDeviceId)) { + this.removeOwnDevice(log); + } }); } /** @internal */ - removeMember(userId: string, syncLog: ILogItem) { + removeMembership(userId: string, syncLog: ILogItem) { const deviceIds = this.getDeviceIdsForUserId(userId); - for (const deviceId of deviceIds) { - this.removeMemberDevice(userId, deviceId, syncLog); - } + this.logItem.wrap("removeMember", log => { + syncLog.refDetached(log); + for (const deviceId of deviceIds) { + this.removeMemberDevice(userId, deviceId, log); + } + if (userId === this.options.ownUserId) { + this.removeOwnDevice(log); + } + }); } private getDeviceIdsForUserId(userId: string): string[] { @@ -239,26 +266,32 @@ export class GroupCall extends EventEmitter<{change: never}> { .map(key => getDeviceFromMemberKey(key)); } + private isMember(userId: string): boolean { + return Array.from(this._members.keys()).some(key => memberKeyIsForUser(key, userId)); + } + + private removeOwnDevice(log: ILogItem) { + if (this._state === GroupCallState.Joined) { + log.set("leave_own", true); + this._localMedia?.dispose(); + this._localMedia = undefined; + for (const [,member] of this._members) { + member.disconnect(); + } + this._state = GroupCallState.Created; + this.emitChange(); + } + } + /** @internal */ - private removeMemberDevice(userId: string, deviceId: string, syncLog: ILogItem) { + private removeMemberDevice(userId: string, deviceId: string, log: ILogItem) { const memberKey = getMemberKey(userId, deviceId); - this.logItem.wrap({l: "removeMemberDevice", id: memberKey}, log => { - syncLog.refDetached(log); - if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) { - if (this._state === GroupCallState.Joined) { - this._localMedia?.dispose(); - this._localMedia = undefined; - for (const [,member] of this._members) { - member.disconnect(); - } - this._state = GroupCallState.Created; - } - } else { - const member = this._members.get(memberKey); - if (member) { - this._members.remove(memberKey); - member.disconnect(); - } + log.wrap({l: "removeMemberDevice", id: memberKey}, log => { + const member = this._members.get(memberKey); + if (member) { + log.set("leave", true); + this._members.remove(memberKey); + member.disconnect(); } this.emitChange(); }); @@ -315,9 +348,15 @@ export class GroupCall extends EventEmitter<{change: never}> { const stateEvent = await txn.roomState.get(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId); if (stateEvent) { const content = stateEvent.event.content; - const callsInfo = content["m.calls"]; - content["m.calls"] = callsInfo?.filter(c => c["m.call_id"] !== this.id); - return content; + const callInfo = content["m.calls"]?.find(c => c["m.call_id"] === this.id); + if (callInfo) { + const devicesInfo = callInfo["m.devices"]; + const deviceIndex = devicesInfo.findIndex(d => d["device_id"] === this.options.ownDeviceId); + if (deviceIndex !== -1) { + devicesInfo.splice(deviceIndex, 1); + return content; + } + } } } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index d31837f4..cd26fba2 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -47,9 +47,7 @@ export class Member { private callDeviceMembership: CallDeviceMembership, private readonly options: Options, private readonly logItem: ILogItem, - ) { - logItem.set("id", member.userId); - } + ) {} get remoteTracks(): Track[] { return this.peerCall?.remoteTracks ?? []; diff --git a/src/platform/types/MediaDevices.ts b/src/platform/types/MediaDevices.ts index ed9015bf..db267871 100644 --- a/src/platform/types/MediaDevices.ts +++ b/src/platform/types/MediaDevices.ts @@ -37,6 +37,7 @@ export interface Track { get muted(): boolean; setMuted(muted: boolean): void; stop(): void; + clone(): Track; } export interface AudioTrack extends Track { diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index 22f3d634..49fd7fa5 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -111,6 +111,10 @@ export class TrackWrapper implements Track { stop() { this.track.stop(); } + + clone() { + return this.track.clone(); + } } export class AudioTrackWrapper extends TrackWrapper { From 42b470b06b649d0223d19b97e27650405effaf08 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 30 Mar 2022 15:19:07 +0200 Subject: [PATCH 034/435] helper to print open items with console logger --- src/logging/ConsoleLogger.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/logging/ConsoleLogger.ts b/src/logging/ConsoleLogger.ts index f48c72b2..1643e48a 100644 --- a/src/logging/ConsoleLogger.ts +++ b/src/logging/ConsoleLogger.ts @@ -25,6 +25,12 @@ export class ConsoleLogger extends BaseLogger { async export(): Promise { return undefined; } + + printOpenItems(): void { + for (const item of this._openItems) { + this._persistItem(item); + } + } } const excludedKeysFromTable = ["l", "id"]; From 1ad5db73a933314c18db0ab519f69a1123cfb323 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 6 Apr 2022 18:11:06 +0200 Subject: [PATCH 035/435] some logviewer improvement to help debug call signalling --- scripts/logviewer/main.js | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/scripts/logviewer/main.js b/scripts/logviewer/main.js index dce82fb8..4c6b3792 100644 --- a/scripts/logviewer/main.js +++ b/scripts/logviewer/main.js @@ -22,6 +22,7 @@ const main = document.querySelector("main"); let selectedItemNode; let rootItem; let itemByRef; +let itemsRefFrom; const logLevels = [undefined, "All", "Debug", "Detail", "Info", "Warn", "Error", "Fatal", "Off"]; @@ -49,6 +50,7 @@ window.addEventListener("hashchange", () => { const id = window.location.hash.substr(1); const itemNode = document.getElementById(id); if (itemNode && itemNode.closest("main")) { + ensureParentsExpanded(itemNode); selectNode(itemNode); itemNode.scrollIntoView({behavior: "smooth", block: "nearest"}); } @@ -70,6 +72,14 @@ function selectNode(itemNode) { showItemDetails(item, parent, selectedItemNode); } +function ensureParentsExpanded(itemNode) { + let li = itemNode.parentElement.parentElement; + while (li.tagName === "LI") { + li.classList.add("expanded"); + li = li.parentElement.parentElement; + } +} + function stringifyItemValue(value) { if (typeof value === "object" && value !== null) { return JSON.stringify(value, undefined, 2); @@ -102,6 +112,11 @@ function showItemDetails(item, parent, itemNode) { } else { valueNode = `unknown ref ${value}`; } + } else if (key === "refId") { + const refSources = itemsRefFrom.get(value) ?? []; + valueNode = t.div([t.p([`${value}`, t.br(),`Found these references:`]),t.ul(refSources.map(item => { + return t.li(t.a({href: `#${item.id}`}, itemCaption(item))); + }))]); } else { valueNode = stringifyItemValue(value); } @@ -153,7 +168,8 @@ async function loadFile() { logs.items.sort((a, b) => itemStart(a) - itemStart(b)); rootItem = {c: logs.items}; itemByRef = new Map(); - preprocessRecursively(rootItem, null, itemByRef, []); + itemsRefFrom = new Map(); + preprocessRecursively(rootItem, null, itemByRef, itemsRefFrom, []); const fragment = logs.items.reduce((fragment, item, i, items) => { const prevItem = i === 0 ? null : items[i - 1]; @@ -167,18 +183,26 @@ async function loadFile() { } // TODO: make this use processRecursively -function preprocessRecursively(item, parentElement, refsMap, path) { +function preprocessRecursively(item, parentElement, refsMap, refsFromMap, path) { item.s = (parentElement?.s || 0) + item.s; if (itemRefSource(item)) { refsMap.set(itemRefSource(item), item); } + if (itemRef(item)) { + let refs = refsFromMap.get(itemRef(item)); + if (!refs) { + refs = []; + refsFromMap.set(itemRef(item), refs); + } + refs.push(item); + } if (itemChildren(item)) { for (let i = 0; i < itemChildren(item).length; i += 1) { // do it in advance for a child as we don't want to do it for the rootItem const child = itemChildren(item)[i]; const childPath = path.concat(i); child.id = childPath.join("/"); - preprocessRecursively(child, item, refsMap, childPath); + preprocessRecursively(child, item, refsMap, refsFromMap, childPath); } } } From 2852834ce34cdbda85a1a13d22ad4ff03daa81a1 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 7 Apr 2022 10:32:23 +0200 Subject: [PATCH 036/435] persist calls so they can be quickly loaded after a restart also use event prefixes compatible with Element Call/MSC --- src/domain/session/SessionViewModel.js | 2 + .../session/room/timeline/tilesCreator.js | 2 +- src/matrix/Session.js | 2 +- src/matrix/Sync.js | 1 + src/matrix/calls/CallHandler.ts | 99 ++++++++++++++++--- src/matrix/calls/callEventTypes.ts | 12 ++- src/matrix/calls/group/GroupCall.ts | 41 ++++---- src/matrix/room/Room.js | 9 +- src/matrix/storage/common.ts | 1 + src/matrix/storage/idb/Transaction.ts | 5 + src/matrix/storage/idb/schema.ts | 8 +- src/matrix/storage/idb/stores/CallStore.ts | 83 ++++++++++++++++ .../storage/idb/stores/RoomStateStore.ts | 12 ++- 13 files changed, 229 insertions(+), 48 deletions(-) create mode 100644 src/matrix/storage/idb/stores/CallStore.ts diff --git a/src/domain/session/SessionViewModel.js b/src/domain/session/SessionViewModel.js index 4e5930b1..1242edf4 100644 --- a/src/domain/session/SessionViewModel.js +++ b/src/domain/session/SessionViewModel.js @@ -99,6 +99,8 @@ export class SessionViewModel extends ViewModel { start() { this._sessionStatusViewModel.start(); + //this._client.session.callHandler.loadCalls("m.prompt"); + this._client.session.callHandler.loadCalls("m.ring"); } get activeMiddleViewModel() { diff --git a/src/domain/session/room/timeline/tilesCreator.js b/src/domain/session/room/timeline/tilesCreator.js index f35f6536..ad562df0 100644 --- a/src/domain/session/room/timeline/tilesCreator.js +++ b/src/domain/session/room/timeline/tilesCreator.js @@ -72,7 +72,7 @@ export function tilesCreator(baseOptions) { return new EncryptedEventTile(options); case "m.room.encryption": return new EncryptionEnabledTile(options); - case "m.call": + case "org.matrix.msc3401.call": // if prevContent is present, it's an update to a call event, which we don't render // as the original event is updated through the call object which receive state event updates return entry.stateKey && !entry.prevContent ? new CallTile(options) : null; diff --git a/src/matrix/Session.js b/src/matrix/Session.js index a1e1cc28..aa7dbf7b 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -76,7 +76,7 @@ export class Session { this._roomsBeingCreated = new ObservableMap(); this._user = new User(sessionInfo.userId); this._callHandler = new CallHandler({ - createTimeout: this._platform.clock.createTimeout, + clock: this._platform.clock, hsApi: this._hsApi, encryptDeviceMessage: async (roomId, userId, message, log) => { if (!this._deviceTracker || !this._olmEncryption) { diff --git a/src/matrix/Sync.js b/src/matrix/Sync.js index b4ea702f..4f907563 100644 --- a/src/matrix/Sync.js +++ b/src/matrix/Sync.js @@ -344,6 +344,7 @@ export class Sync { // to decrypt and store new room keys storeNames.olmSessions, storeNames.inboundGroupSessions, + storeNames.calls, ]); } diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 89730391..7cd60208 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -18,8 +18,9 @@ import {ObservableMap} from "../../observable/map/ObservableMap"; import {WebRTC, PeerConnection, PeerConnectionHandler} from "../../platform/types/WebRTC"; import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; import {handlesEventType} from "./PeerCall"; -import {EventType} from "./callEventTypes"; +import {EventType, CallIntent} from "./callEventTypes"; import {GroupCall} from "./group/GroupCall"; +import {makeId} from "../common"; import type {LocalMedia} from "./LocalMedia"; import type {Room} from "../room/Room"; @@ -30,13 +31,17 @@ import type {Platform} from "../../platform/web/Platform"; import type {BaseObservableMap} from "../../observable/map/BaseObservableMap"; import type {SignallingMessage, MGroupCallBase} from "./callEventTypes"; import type {Options as GroupCallOptions} from "./group/GroupCall"; +import type {Transaction} from "../storage/idb/Transaction"; +import type {CallEntry} from "../storage/idb/stores/CallStore"; +import type {Clock} from "../../platform/web/dom/Clock"; const GROUP_CALL_TYPE = "m.call"; const GROUP_CALL_MEMBER_TYPE = "m.call.member"; const CALL_TERMINATED = "m.terminated"; -export type Options = Omit & { - logger: ILogger +export type Options = Omit & { + logger: ILogger, + clock: Clock }; export class CallHandler { @@ -48,25 +53,86 @@ export class CallHandler { constructor(private readonly options: Options) { this.groupCallOptions = Object.assign({}, this.options, { - emitUpdate: (groupCall, params) => this._calls.update(groupCall.id, params) + emitUpdate: (groupCall, params) => this._calls.update(groupCall.id, params), + createTimeout: this.options.clock.createTimeout, + }); + } + + async loadCalls(intent: CallIntent = CallIntent.Ring) { + const txn = await this._getLoadTxn(); + const callEntries = await txn.calls.getByIntent(intent); + this._loadCallEntries(callEntries, txn); + } + + async loadCallsForRoom(intent: CallIntent, roomId: string) { + const txn = await this._getLoadTxn(); + const callEntries = await txn.calls.getByIntentAndRoom(intent, roomId); + this._loadCallEntries(callEntries, txn); + } + + private async _getLoadTxn(): Promise { + const names = this.options.storage.storeNames; + const txn = await this.options.storage.readTxn([ + names.calls, + names.roomState + ]); + return txn; + } + + private async _loadCallEntries(callEntries: CallEntry[], txn: Transaction): Promise { + return this.options.logger.run("loading calls", async log => { + log.set("entries", callEntries.length); + await Promise.all(callEntries.map(async callEntry => { + if (this._calls.get(callEntry.callId)) { + return; + } + const event = await txn.roomState.get(callEntry.roomId, EventType.GroupCall, callEntry.callId); + if (event) { + const logItem = this.options.logger.child({l: "call", loaded: true}); + const call = new GroupCall(event.event.state_key, false, event.event.content, event.roomId, this.groupCallOptions, logItem); + this._calls.set(call.id, call); + } + })); + const roomIds = Array.from(new Set(callEntries.map(e => e.roomId))); + await Promise.all(roomIds.map(async roomId => { + const ownCallsMemberEvent = await txn.roomState.get(roomId, EventType.GroupCallMember, this.options.ownUserId); + if (ownCallsMemberEvent) { + this.handleCallMemberEvent(ownCallsMemberEvent.event, log); + } + // TODO: we should be loading the other members as well at some point + })); + log.set("newSize", this._calls.size); }); } async createCall(roomId: string, localMedia: LocalMedia, name: string): Promise { const logItem = this.options.logger.child({l: "call", incoming: false}); - const call = new GroupCall(undefined, undefined, roomId, this.groupCallOptions, logItem); + const call = new GroupCall(makeId("conf-"), true, { + "m.name": name, + "m.intent": CallIntent.Ring + }, roomId, this.groupCallOptions, logItem); this._calls.set(call.id, call); + try { - await call.create(localMedia, name); + await call.create(localMedia); + await call.join(localMedia); + // store call info so it will ring again when reopening the app + const txn = await this.options.storage.readWriteTxn([this.options.storage.storeNames.calls]); + txn.calls.add({ + intent: call.intent, + callId: call.id, + timestamp: this.options.clock.now(), + roomId: roomId + }); + await txn.complete(); } catch (err) { - if (err.name === "ConnectionError") { + //if (err.name === "ConnectionError") { // if we're offline, give up and remove the call again call.dispose(); this._calls.remove(call.id); - } + //} throw err; } - await call.join(localMedia); return call; } @@ -75,11 +141,11 @@ export class CallHandler { // TODO: check and poll turn server credentials here /** @internal */ - handleRoomState(room: Room, events: StateEvent[], log: ILogItem) { + handleRoomState(room: Room, events: StateEvent[], txn: Transaction, log: ILogItem) { // first update call events for (const event of events) { if (event.type === EventType.GroupCall) { - this.handleCallEvent(event, room.id, log); + this.handleCallEvent(event, room.id, txn, log); } } // then update members @@ -108,7 +174,7 @@ export class CallHandler { call?.handleDeviceMessage(message, userId, deviceId, log); } - private handleCallEvent(event: StateEvent, roomId: string, log: ILogItem) { + private handleCallEvent(event: StateEvent, roomId: string, txn: Transaction, log: ILogItem) { const callId = event.state_key; let call = this._calls.get(callId); if (call) { @@ -116,11 +182,18 @@ export class CallHandler { if (call.isTerminated) { call.dispose(); this._calls.remove(call.id); + txn.calls.remove(call.intent, roomId, call.id); } } else { const logItem = this.options.logger.child({l: "call", incoming: true}); - call = new GroupCall(event.state_key, event.content, roomId, this.groupCallOptions, logItem); + call = new GroupCall(event.state_key, false, event.content, roomId, this.groupCallOptions, logItem); this._calls.set(call.id, call); + txn.calls.add({ + intent: call.intent, + callId: call.id, + timestamp: event.origin_server_ts, + roomId: roomId + }); } } diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index 4416087b..a32a7739 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -1,10 +1,10 @@ // allow non-camelcase as these are events type that go onto the wire /* eslint-disable camelcase */ - +import type {StateEvent} from "../storage/types"; export enum EventType { - GroupCall = "m.call", - GroupCallMember = "m.call.member", + GroupCall = "org.matrix.msc3401.call", + GroupCallMember = "org.matrix.msc3401.call.member", Invite = "m.call.invite", Candidates = "m.call.candidates", Answer = "m.call.answer", @@ -211,3 +211,9 @@ export type SignallingMessage = {type: EventType.SDPStreamMetadataChanged | EventType.SDPStreamMetadataChangedPrefix, content: MCallSDPStreamMetadataChanged} | {type: EventType.Candidates, content: MCallCandidates} | {type: EventType.Hangup | EventType.Reject, content: MCallHangupReject}; + +export enum CallIntent { + Ring = "m.ring", + Prompt = "m.prompt", + Room = "m.room", +}; diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 98e3381d..3a28c9a7 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -18,8 +18,8 @@ import {ObservableMap} from "../../../observable/map/ObservableMap"; import {Member} from "./Member"; import {LocalMedia} from "../LocalMedia"; import {RoomMember} from "../../room/members/RoomMember"; -import {makeId} from "../../common"; import {EventEmitter} from "../../../utils/EventEmitter"; +import {EventType, CallIntent} from "../callEventTypes"; import type {Options as MemberOptions} from "./Member"; import type {BaseObservableMap} from "../../../observable/map/BaseObservableMap"; @@ -32,9 +32,6 @@ import type {EncryptedMessage} from "../../e2ee/olm/Encryption"; import type {ILogItem} from "../../../logging/types"; import type {Storage} from "../../storage/idb/Storage"; -const CALL_TYPE = "m.call"; -const CALL_MEMBER_TYPE = "m.call.member"; - export enum GroupCallState { Fledgling = "fledgling", Creating = "creating", @@ -62,23 +59,22 @@ export type Options = Omit { - public readonly id: string; private readonly _members: ObservableMap = new ObservableMap(); private _localMedia?: LocalMedia = undefined; private _memberOptions: MemberOptions; private _state: GroupCallState; constructor( - id: string | undefined, - private callContent: Record | undefined, + public readonly id: string, + newCall: boolean, + private callContent: Record, public readonly roomId: string, private readonly options: Options, private readonly logItem: ILogItem, ) { super(); - this.id = id ?? makeId("conf-"); logItem.set("id", this.id); - this._state = id ? GroupCallState.Created : GroupCallState.Fledgling; + this._state = newCall ? GroupCallState.Fledgling : GroupCallState.Created; this._memberOptions = Object.assign({}, options, { confId: this.id, emitUpdate: member => this._members.update(getMemberKey(member.userId, member.deviceId), member), @@ -103,7 +99,7 @@ export class GroupCall extends EventEmitter<{change: never}> { return this.callContent?.["m.name"]; } - get intent(): string { + get intent(): CallIntent { return this.callContent?.["m.intent"]; } @@ -117,7 +113,7 @@ export class GroupCall extends EventEmitter<{change: never}> { this.emitChange(); const memberContent = await this._createJoinPayload(); // send m.call.member state event - const request = this.options.hsApi.sendState(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId, memberContent, {log}); + const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCallMember, this.options.ownUserId, memberContent, {log}); await request.response(); this.emitChange(); // send invite to all members that are < my userId @@ -136,10 +132,10 @@ export class GroupCall extends EventEmitter<{change: never}> { const memberContent = await this._leaveCallMemberContent(); // send m.call.member state event if (memberContent) { - const request = this.options.hsApi.sendState(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId, memberContent, {log}); + const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCallMember, this.options.ownUserId, memberContent, {log}); await request.response(); // our own user isn't included in members, so not in the count - if (this._members.size === 0) { + if (this.intent === CallIntent.Ring && this._members.size === 0) { await this.terminate(); } } else { @@ -153,7 +149,7 @@ export class GroupCall extends EventEmitter<{change: never}> { if (this._state === GroupCallState.Fledgling) { return; } - const request = this.options.hsApi.sendState(this.roomId, CALL_TYPE, this.id, Object.assign({}, this.callContent, { + const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCall, this.id, Object.assign({}, this.callContent, { "m.terminated": true }), {log}); await request.response(); @@ -161,19 +157,17 @@ export class GroupCall extends EventEmitter<{change: never}> { } /** @internal */ - create(localMedia: LocalMedia, name: string): Promise { + create(localMedia: LocalMedia): Promise { return this.logItem.wrap("create", async log => { if (this._state !== GroupCallState.Fledgling) { return; } this._state = GroupCallState.Creating; this.emitChange(); - this.callContent = { + this.callContent = Object.assign({ "m.type": localMedia.cameraTrack ? "m.video" : "m.voice", - "m.name": name, - "m.intent": "m.ring" - }; - const request = this.options.hsApi.sendState(this.roomId, CALL_TYPE, this.id, this.callContent, {log}); + }, this.callContent); + const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCall, this.id, this.callContent!, {log}); await request.response(); this._state = GroupCallState.Created; this.emitChange(); @@ -318,7 +312,7 @@ export class GroupCall extends EventEmitter<{change: never}> { private async _createJoinPayload() { const {storage} = this.options; const txn = await storage.readTxn([storage.storeNames.roomState]); - const stateEvent = await txn.roomState.get(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId); + const stateEvent = await txn.roomState.get(this.roomId, EventType.GroupCallMember, this.options.ownUserId); const stateContent = stateEvent?.event?.content ?? { ["m.calls"]: [] }; @@ -335,7 +329,8 @@ export class GroupCall extends EventEmitter<{change: never}> { let deviceInfo = devicesInfo.find(d => d["device_id"] === this.options.ownDeviceId); if (!deviceInfo) { deviceInfo = { - ["device_id"]: this.options.ownDeviceId + ["device_id"]: this.options.ownDeviceId, + feeds: [{purpose: "m.usermedia"}] }; devicesInfo.push(deviceInfo); } @@ -345,7 +340,7 @@ export class GroupCall extends EventEmitter<{change: never}> { private async _leaveCallMemberContent(): Promise | undefined> { const {storage} = this.options; const txn = await storage.readTxn([storage.storeNames.roomState]); - const stateEvent = await txn.roomState.get(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId); + const stateEvent = await txn.roomState.get(this.roomId, EventType.GroupCallMember, this.options.ownUserId); if (stateEvent) { const content = stateEvent.event.content; const callInfo = content["m.calls"]?.find(c => c["m.call_id"] === this.id); diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js index ff1926b4..34f35af8 100644 --- a/src/matrix/room/Room.js +++ b/src/matrix/room/Room.js @@ -93,8 +93,6 @@ export class Room extends BaseRoom { } } - this._updateCallHandler(roomResponse, log); - return { roomEncryption, summaryChanges, @@ -181,6 +179,7 @@ export class Room extends BaseRoom { removedPendingEvents = await this._sendQueue.removeRemoteEchos(roomResponse.timeline.events, txn, log); } const powerLevelsEvent = this._getPowerLevelsEvent(roomResponse); + this._updateCallHandler(roomResponse, txn, log); return { summaryChanges, roomEncryption, @@ -448,17 +447,17 @@ export class Room extends BaseRoom { return this._sendQueue.pendingEvents; } - _updateCallHandler(roomResponse, log) { + _updateCallHandler(roomResponse, txn, log) { if (this._callHandler) { const stateEvents = roomResponse.state?.events; if (stateEvents?.length) { - this._callHandler.handleRoomState(this, stateEvents, log); + this._callHandler.handleRoomState(this, stateEvents, txn, log); } let timelineEvents = roomResponse.timeline?.events; if (timelineEvents) { const timelineStateEvents = timelineEvents.filter(e => typeof e.state_key === "string"); if (timelineEvents.length !== 0) { - this._callHandler.handleRoomState(this, timelineStateEvents, log); + this._callHandler.handleRoomState(this, timelineStateEvents, txn, log); } } } diff --git a/src/matrix/storage/common.ts b/src/matrix/storage/common.ts index 23bb0d31..e1e34917 100644 --- a/src/matrix/storage/common.ts +++ b/src/matrix/storage/common.ts @@ -33,6 +33,7 @@ export enum StoreNames { groupSessionDecryptions = "groupSessionDecryptions", operations = "operations", accountData = "accountData", + calls = "calls" } export const STORE_NAMES: Readonly = Object.values(StoreNames); diff --git a/src/matrix/storage/idb/Transaction.ts b/src/matrix/storage/idb/Transaction.ts index 80894105..7a8de420 100644 --- a/src/matrix/storage/idb/Transaction.ts +++ b/src/matrix/storage/idb/Transaction.ts @@ -36,6 +36,7 @@ import {OutboundGroupSessionStore} from "./stores/OutboundGroupSessionStore"; import {GroupSessionDecryptionStore} from "./stores/GroupSessionDecryptionStore"; import {OperationStore} from "./stores/OperationStore"; import {AccountDataStore} from "./stores/AccountDataStore"; +import {CallStore} from "./stores/CallStore"; import type {ILogger, ILogItem} from "../../../logging/types"; export type IDBKey = IDBValidKey | IDBKeyRange; @@ -167,6 +168,10 @@ export class Transaction { get accountData(): AccountDataStore { return this._store(StoreNames.accountData, idbStore => new AccountDataStore(idbStore)); } + + get calls(): CallStore { + return this._store(StoreNames.calls, idbStore => new CallStore(idbStore)); + } async complete(log?: ILogItem): Promise { try { diff --git a/src/matrix/storage/idb/schema.ts b/src/matrix/storage/idb/schema.ts index 7819130e..4461ae15 100644 --- a/src/matrix/storage/idb/schema.ts +++ b/src/matrix/storage/idb/schema.ts @@ -34,7 +34,8 @@ export const schema: MigrationFunc[] = [ backupAndRestoreE2EEAccountToLocalStorage, clearAllStores, addInboundSessionBackupIndex, - migrateBackupStatus + migrateBackupStatus, + createCallStore ]; // TODO: how to deal with git merge conflicts of this array? @@ -309,3 +310,8 @@ async function migrateBackupStatus(db: IDBDatabase, txn: IDBTransaction, localSt log.set("countWithoutSession", countWithoutSession); log.set("countWithSession", countWithSession); } + +//v17 create calls store +function createCallStore(db: IDBDatabase) : void { + db.createObjectStore("calls", {keyPath: "key"}); +} diff --git a/src/matrix/storage/idb/stores/CallStore.ts b/src/matrix/storage/idb/stores/CallStore.ts new file mode 100644 index 00000000..566bcc40 --- /dev/null +++ b/src/matrix/storage/idb/stores/CallStore.ts @@ -0,0 +1,83 @@ +/* +Copyright 2020 Bruno Windels +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {Store} from "../Store"; +import {StateEvent} from "../../types"; +import {MIN_UNICODE, MAX_UNICODE} from "./common"; + +function encodeKey(intent: string, roomId: string, callId: string) { + return `${intent}|${roomId}|${callId}`; +} + +function decodeStorageEntry(storageEntry: CallStorageEntry): CallEntry { + const [intent, roomId, callId] = storageEntry.key.split("|"); + return {intent, roomId, callId, timestamp: storageEntry.timestamp}; +} + +export interface CallEntry { + intent: string; + roomId: string; + callId: string; + timestamp: number; +} + +type CallStorageEntry = { + key: string; + timestamp: number; +} + +export class CallStore { + private _callStore: Store; + + constructor(idbStore: Store) { + this._callStore = idbStore; + } + + async getByIntent(intent: string): Promise { + const range = this._callStore.IDBKeyRange.bound( + encodeKey(intent, MIN_UNICODE, MIN_UNICODE), + encodeKey(intent, MAX_UNICODE, MAX_UNICODE), + true, + true + ); + const storageEntries = await this._callStore.selectAll(range); + return storageEntries.map(e => decodeStorageEntry(e)); + } + + async getByIntentAndRoom(intent: string, roomId: string): Promise { + const range = this._callStore.IDBKeyRange.bound( + encodeKey(intent, roomId, MIN_UNICODE), + encodeKey(intent, roomId, MAX_UNICODE), + true, + true + ); + const storageEntries = await this._callStore.selectAll(range); + return storageEntries.map(e => decodeStorageEntry(e)); + } + + add(entry: CallEntry) { + const storageEntry: CallStorageEntry = { + key: encodeKey(entry.intent, entry.roomId, entry.callId), + timestamp: entry.timestamp + }; + this._callStore.add(storageEntry); + } + + remove(intent: string, roomId: string, callId: string): void { + this._callStore.delete(encodeKey(intent, roomId, callId)); + } +} diff --git a/src/matrix/storage/idb/stores/RoomStateStore.ts b/src/matrix/storage/idb/stores/RoomStateStore.ts index d2bf811d..99315e9e 100644 --- a/src/matrix/storage/idb/stores/RoomStateStore.ts +++ b/src/matrix/storage/idb/stores/RoomStateStore.ts @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {MAX_UNICODE} from "./common"; +import {MIN_UNICODE, MAX_UNICODE} from "./common"; import {Store} from "../Store"; import {StateEvent} from "../../types"; @@ -41,6 +41,16 @@ export class RoomStateStore { return this._roomStateStore.get(key); } + getAllForType(roomId: string, type: string): Promise { + const range = this._roomStateStore.IDBKeyRange.bound( + encodeKey(roomId, type, MIN_UNICODE), + encodeKey(roomId, type, MAX_UNICODE), + true, + true + ); + return this._roomStateStore.selectAll(range); + } + set(roomId: string, event: StateEvent): void { const key = encodeKey(roomId, event.type, event.state_key); const entry = {roomId, event, key}; From ad1cceac86be15dfe8b442ecc1fd6c3670d5cf44 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 7 Apr 2022 10:33:12 +0200 Subject: [PATCH 037/435] fix error thrown during request when response code is not used --- src/matrix/net/RequestScheduler.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/matrix/net/RequestScheduler.ts b/src/matrix/net/RequestScheduler.ts index dc5c501b..c6e546a1 100644 --- a/src/matrix/net/RequestScheduler.ts +++ b/src/matrix/net/RequestScheduler.ts @@ -27,8 +27,8 @@ class Request implements IHomeServerRequest { public readonly args: any[]; private responseResolve: (result: any) => void; public responseReject: (error: Error) => void; - private responseCodeResolve: (result: any) => void; - private responseCodeReject: (result: any) => void; + private responseCodeResolve?: (result: any) => void; + private responseCodeReject?: (result: any) => void; private _requestResult?: IHomeServerRequest; private readonly _responsePromise: Promise; private _responseCodePromise: Promise; @@ -73,7 +73,7 @@ class Request implements IHomeServerRequest { const response = await this._requestResult?.response(); this.responseResolve(response); const responseCode = await this._requestResult?.responseCode(); - this.responseCodeResolve(responseCode); + this.responseCodeResolve?.(responseCode); } get requestResult() { From fe6e7b09b568fa608a397a963b24b2e73b19d044 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 7 Apr 2022 16:50:16 +0200 Subject: [PATCH 038/435] don't encrypt to_device messages for now --- src/matrix/DeviceMessageHandler.js | 38 ++++++++++++++++++++---------- src/matrix/calls/group/Member.ts | 18 +++++++++++--- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index ee10716e..c33236aa 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -38,6 +38,7 @@ export class DeviceMessageHandler { async prepareSync(toDeviceEvents, lock, txn, log) { log.set("messageTypes", countBy(toDeviceEvents, e => e.type)); + this._handleUnencryptedCallEvents(toDeviceEvents, log); const encryptedEvents = toDeviceEvents.filter(e => e.type === "m.room.encrypted"); if (!this._olmDecryption) { log.log("can't decrypt, encryption not enabled", log.level.Warn); @@ -52,19 +53,21 @@ export class DeviceMessageHandler { log.child("decrypt_error").catch(err); } const newRoomKeys = this._megolmDecryption.roomKeysFromDeviceMessages(olmDecryptChanges.results, log); - const callMessages = olmDecryptChanges.results.filter(dr => this._callHandler.handlesDeviceMessageEventType(dr.event?.type)); - // load devices by sender key - await Promise.all(callMessages.map(async dr => { - dr.setDevice(await this._getDevice(dr.senderCurve25519Key, txn)); - })); - // TODO: pass this in the prep and run it in afterSync or afterSyncComplete (as callHandler can send events as well)? - for (const dr of callMessages) { - if (dr.device) { - this._callHandler.handleDeviceMessage(dr.event, dr.device.userId, dr.device.deviceId, log); - } else { - console.error("could not deliver message because don't have device for sender key", dr.event); - } - } + + // const callMessages = olmDecryptChanges.results.filter(dr => this._callHandler.handlesDeviceMessageEventType(dr.event?.type)); + // // load devices by sender key + // await Promise.all(callMessages.map(async dr => { + // dr.setDevice(await this._getDevice(dr.senderCurve25519Key, txn)); + // })); + // // TODO: pass this in the prep and run it in afterSync or afterSyncComplete (as callHandler can send events as well)? + // for (const dr of callMessages) { + // if (dr.device) { + // this._callHandler.handleDeviceMessage(dr.event, dr.device.userId, dr.device.deviceId, log); + // } else { + // console.error("could not deliver message because don't have device for sender key", dr.event); + // } + // } + // TODO: somehow include rooms that received a call to_device message in the sync state? // or have updates flow through event emitter? // well, we don't really need to update the room other then when a call starts or stops @@ -73,6 +76,15 @@ export class DeviceMessageHandler { } } + _handleUnencryptedCallEvents(toDeviceEvents, log) { + const callMessages = toDeviceEvents.filter(e => this._callHandler.handlesDeviceMessageEventType(e.type)); + for (const event of callMessages) { + const userId = event.sender; + const deviceId = event.content.device_id; + this._callHandler.handleDeviceMessage(event, userId, deviceId, log); + } + } + /** check that prep is not undefined before calling this */ async writeSync(prep, txn) { // write olm changes diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index cd26fba2..fe2de0af 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -110,10 +110,22 @@ export class Member { sendSignallingMessage = async (message: SignallingMessage, log: ILogItem): Promise => { const groupMessage = message as SignallingMessage; groupMessage.content.conf_id = this.options.confId; - const encryptedMessages = await this.options.encryptDeviceMessage(this.member.userId, groupMessage, log); - const payload = formatToDeviceMessagesPayload(encryptedMessages); + groupMessage.content.device_id = this.options.ownDeviceId; + groupMessage.content.party_id = this.options.ownDeviceId; + groupMessage.content.sender_session_id = this.options.sessionId; + groupMessage.content.dest_session_id = this.destSessionId!; + // const encryptedMessages = await this.options.encryptDeviceMessage(this.member.userId, groupMessage, log); + // const payload = formatToDeviceMessagesPayload(encryptedMessages); + const payload = { + messages: { + [this.member.userId]: { + ['*']: groupMessage.content + } + } + }; const request = this.options.hsApi.sendToDevice( - "m.room.encrypted", + message.type, + //"m.room.encrypted", payload, makeTxnId(), {log} From 64728003870e948ccf6e2d8ea11a70a1acc0d7d0 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 7 Apr 2022 16:53:37 +0200 Subject: [PATCH 039/435] impl session id so EC does not ignore our messages --- src/matrix/calls/CallHandler.ts | 2 ++ src/matrix/calls/PeerCall.ts | 5 +++++ src/matrix/calls/callEventTypes.ts | 5 +++++ src/matrix/calls/group/Member.ts | 8 ++++++++ 4 files changed, 20 insertions(+) diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 7cd60208..f60c1fe6 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -50,11 +50,13 @@ export class CallHandler { // map of userId to set of conf_id's they are in private memberToCallIds: Map> = new Map(); private groupCallOptions: GroupCallOptions; + private sessionId = makeId("s"); constructor(private readonly options: Options) { this.groupCallOptions = Object.assign({}, this.options, { emitUpdate: (groupCall, params) => this._calls.update(groupCall.id, params), createTimeout: this.options.clock.createTimeout, + sessionId: this.sessionId }); } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index b85cee61..1b869f47 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -61,6 +61,7 @@ export class PeerCall implements IDisposable { private _state = CallState.Fledgling; private direction: CallDirection; private localMedia?: LocalMedia; + private seq: number = 0; // A queue for candidates waiting to go out. // We try to amalgamate candidates into a single candidate message where // possible @@ -258,6 +259,7 @@ export class PeerCall implements IDisposable { const content = { call_id: callId, version: 1, + seq: this.seq++, }; // TODO: Don't send UserHangup reason to older clients if (reason) { @@ -304,6 +306,7 @@ export class PeerCall implements IDisposable { offer, [SDPStreamMetadataKey]: this.localMedia!.getSDPMetadata(), version: 1, + seq: this.seq++, lifetime: CALL_TIMEOUT_MS }; if (this._state === CallState.CreateOffer) { @@ -578,6 +581,7 @@ export class PeerCall implements IDisposable { const answerContent: MCallAnswer = { call_id: this.callId, version: 1, + seq: this.seq++, answer: { sdp: localDescription.sdp, type: localDescription.type, @@ -648,6 +652,7 @@ export class PeerCall implements IDisposable { content: { call_id: this.callId, version: 1, + seq: this.seq++, candidates }, }, log); diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index a32a7739..9189911f 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -69,10 +69,15 @@ export interface CallReplacesTarget { export type MCallBase = { call_id: string; version: string | number; + seq: number; } export type MGroupCallBase = MCallBase & { conf_id: string; + device_id: string; + sender_session_id: string; + dest_session_id: string; + party_id: string; // Should not need this? } export type MCallAnswer = Base & { diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index fe2de0af..7404d647 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -33,6 +33,7 @@ export type Options = Omit, log: ILogItem) => Promise, emitUpdate: (participant: Member, params?: any) => void, @@ -41,6 +42,7 @@ export type Options = Omit Date: Thu, 7 Apr 2022 16:53:57 +0200 Subject: [PATCH 040/435] fix who initiates call, needs to be lower, not higher --- src/matrix/calls/group/Member.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 7404d647..e3b1545b 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -73,10 +73,11 @@ export class Member { this.localMedia = localMedia; // otherwise wait for it to connect let shouldInitiateCall; + // the lexicographically lower side initiates the call if (this.member.userId === this.options.ownUserId) { - shouldInitiateCall = this.deviceId < this.options.ownDeviceId; + shouldInitiateCall = this.deviceId > this.options.ownDeviceId; } else { - shouldInitiateCall = this.member.userId < this.options.ownUserId; + shouldInitiateCall = this.member.userId > this.options.ownUserId; } if (shouldInitiateCall) { this.peerCall = this._createPeerCall(makeId("c")); From 1dc46127c3f45ba38369ac0ec40118a90ce4be0c Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 7 Apr 2022 16:54:24 +0200 Subject: [PATCH 041/435] no need to throw here --- src/matrix/calls/PeerCall.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 1b869f47..232aa612 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -250,7 +250,8 @@ export class PeerCall implements IDisposable { break; case EventType.Hangup: default: - throw new Error(`Unknown event type for call: ${message.type}`); + log.log(`Unknown event type for call: ${message.type}`); + break; } }); } From bade40acc61ada815ed2a5fd63e0cc9df2f97bae Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 7 Apr 2022 16:54:36 +0200 Subject: [PATCH 042/435] log track length --- src/matrix/calls/PeerCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 232aa612..3dbb74fc 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -108,7 +108,7 @@ export class PeerCall implements IDisposable { }); }, onRemoteTracksChanged(tracks: Track[]) { - outer.logItem.wrap("onRemoteTracksChanged", log => { + outer.logItem.wrap({l: "onRemoteTracksChanged", length: tracks.length}, log => { outer.options.emitUpdate(outer, undefined); }); }, From b133f58f7a3b7fa401a7294a0928a8800a75fa40 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 7 Apr 2022 16:54:47 +0200 Subject: [PATCH 043/435] don't throw here for now, although it is probably a sign of why the tracks disappear --- src/matrix/calls/PeerCall.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 3dbb74fc..7338cf2c 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -400,12 +400,12 @@ export class PeerCall implements IDisposable { // According to previous comments in this file, firefox at some point did not // add streams until media started arriving on them. Testing latest firefox // (81 at time of writing), this is no longer a problem, so let's do it the correct way. - if (this.peerConnection.remoteTracks.length === 0) { - await log.wrap(`Call no remote stream or no tracks after setting remote description!`, async log => { - return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); - }); - return; - } + // if (this.peerConnection.remoteTracks.length === 0) { + // await log.wrap(`Call no remote stream or no tracks after setting remote description!`, async log => { + // return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); + // }); + // return; + // } this.setState(CallState.Ringing, log); From a78ae52a54c52b3c73cb4016180b5544861bac51 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 7 Apr 2022 16:55:10 +0200 Subject: [PATCH 044/435] to test with EC, also load prompt calls at startup --- src/domain/session/SessionViewModel.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/domain/session/SessionViewModel.js b/src/domain/session/SessionViewModel.js index 1242edf4..715e51c5 100644 --- a/src/domain/session/SessionViewModel.js +++ b/src/domain/session/SessionViewModel.js @@ -99,8 +99,9 @@ export class SessionViewModel extends ViewModel { start() { this._sessionStatusViewModel.start(); - //this._client.session.callHandler.loadCalls("m.prompt"); this._client.session.callHandler.loadCalls("m.ring"); + // TODO: only do this when opening the room + this._client.session.callHandler.loadCalls("m.prompt"); } get activeMiddleViewModel() { From ad140d5af1f1809274990b9649cb5c87bef0ea5d Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 7 Apr 2022 16:55:26 +0200 Subject: [PATCH 045/435] only show video feed when connected --- src/domain/session/room/CallViewModel.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 30b18bc1..72353e4e 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -29,6 +29,7 @@ export class CallViewModel extends ViewModel { constructor(options: Options) { super(options); this.memberViewModels = this.getOption("call").members + .filterValues(member => member.isConnected) .mapValues(member => new CallMemberViewModel(this.childOptions({member}))) .sortValues((a, b) => a.compare(b)); } From 8a06663023019003b23c839bbb80ef5f64fe329a Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 7 Apr 2022 16:55:41 +0200 Subject: [PATCH 046/435] load all call members for now at startup later on we can be smarter and load then once you interact with the call --- src/matrix/calls/CallHandler.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index f60c1fe6..43fbcdb7 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -97,9 +97,13 @@ export class CallHandler { })); const roomIds = Array.from(new Set(callEntries.map(e => e.roomId))); await Promise.all(roomIds.map(async roomId => { - const ownCallsMemberEvent = await txn.roomState.get(roomId, EventType.GroupCallMember, this.options.ownUserId); - if (ownCallsMemberEvent) { - this.handleCallMemberEvent(ownCallsMemberEvent.event, log); + // const ownCallsMemberEvent = await txn.roomState.get(roomId, EventType.GroupCallMember, this.options.ownUserId); + // if (ownCallsMemberEvent) { + // this.handleCallMemberEvent(ownCallsMemberEvent.event, log); + // } + const callsMemberEvents = await txn.roomState.getAllForType(roomId, EventType.GroupCallMember); + for (const entry of callsMemberEvents) { + this.handleCallMemberEvent(entry.event, log); } // TODO: we should be loading the other members as well at some point })); From 156f5b78bf132e4bb3c2197e5e7ee034c239c646 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 11 Apr 2022 12:36:02 +0200 Subject: [PATCH 047/435] use session_id from member event to set dest_session_id so our invite event isn't ignored by EC --- src/matrix/calls/callEventTypes.ts | 3 ++- src/matrix/calls/group/Member.ts | 8 +------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index 9189911f..4726175c 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -23,7 +23,8 @@ export enum EventType { export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata"; export interface CallDeviceMembership { - device_id: string + device_id: string, + session_id: string } export interface CallMembership { diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index e3b1545b..2929e26c 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -42,7 +42,6 @@ export type Options = Omit Date: Mon, 11 Apr 2022 12:37:05 +0200 Subject: [PATCH 048/435] remove unused constants --- src/matrix/calls/CallHandler.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 43fbcdb7..d658121e 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -35,10 +35,6 @@ import type {Transaction} from "../storage/idb/Transaction"; import type {CallEntry} from "../storage/idb/stores/CallStore"; import type {Clock} from "../../platform/web/dom/Clock"; -const GROUP_CALL_TYPE = "m.call"; -const GROUP_CALL_MEMBER_TYPE = "m.call.member"; -const CALL_TERMINATED = "m.terminated"; - export type Options = Omit & { logger: ILogger, clock: Clock From 302d4bc02d426f2cebc8e3091e5ac7a1354163b9 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 11 Apr 2022 13:39:18 +0200 Subject: [PATCH 049/435] use session id from member event, and also send it for other party --- src/matrix/calls/group/GroupCall.ts | 1 + src/matrix/calls/group/Member.ts | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 3a28c9a7..886cb53a 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -330,6 +330,7 @@ export class GroupCall extends EventEmitter<{change: never}> { if (!deviceInfo) { deviceInfo = { ["device_id"]: this.options.ownDeviceId, + ["session_id"]: this.options.sessionId, feeds: [{purpose: "m.usermedia"}] }; devicesInfo.push(deviceInfo); diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 2929e26c..4086d060 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -138,6 +138,11 @@ export class Member { /** @internal */ handleDeviceMessage(message: SignallingMessage, deviceId: string, syncLog: ILogItem) { syncLog.refDetached(this.logItem); + const destSessionId = message.content.dest_session_id; + if (destSessionId !== this.options.sessionId) { + this.logItem.log({l: "ignoring to_device event with wrong session_id", destSessionId, type: message.type}); + return; + } if (message.type === EventType.Invite && !this.peerCall) { this.peerCall = this._createPeerCall(message.content.call_id); } From 81530608313fd39b5cd95f475322bdbc325e2fbd Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 11 Apr 2022 13:39:40 +0200 Subject: [PATCH 050/435] only send to target device, not all user devices --- src/matrix/calls/group/Member.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 4086d060..05d64b92 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -121,7 +121,7 @@ export class Member { const payload = { messages: { [this.member.userId]: { - ['*']: groupMessage.content + [this.callDeviceMembership.device_id]: groupMessage.content } } }; From 8e82aad86be656250b732458fa3a560659332fd9 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 11 Apr 2022 14:54:53 +0200 Subject: [PATCH 051/435] fix logic error that made tracks disappear on the second track event --- src/platform/web/dom/WebRTC.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 39a6f9d2..8c840ddc 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -188,10 +188,11 @@ class DOMPeerConnection implements PeerConnection { } private handleRemoteTrack(evt: RTCTrackEvent) { + // TODO: unit test this code somehow // the tracks on the new stream (with their stream) const updatedTracks = evt.streams.flatMap(stream => stream.getTracks().map(track => {return {stream, track};})); // of the tracks we already know about, filter the ones that aren't in the new stream - const withoutRemovedTracks = this._remoteTracks.filter(t => !updatedTracks.some(ut => t.track.id === ut.track.id)); + const withoutRemovedTracks = this._remoteTracks.filter(t => updatedTracks.some(ut => t.track.id === ut.track.id)); // of the new tracks, filter the ones that we didn't already knew about const addedTracks = updatedTracks.filter(ut => !this._remoteTracks.some(t => t.track.id === ut.track.id)); // wrap them From c02e1de00194aaff3f9efdd9e592ebc54dc2d681 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 11 Apr 2022 14:55:14 +0200 Subject: [PATCH 052/435] log when renegotiation would be triggered --- src/matrix/calls/PeerCall.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 7338cf2c..ed10eebb 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -314,6 +314,7 @@ export class PeerCall implements IDisposable { await this.sendSignallingMessage({type: EventType.Invite, content}, log); this.setState(CallState.InviteSent, log); } else if (this._state === CallState.Connected || this._state === CallState.Connecting) { + log.log("would send renegotiation now but not implemented"); // send Negotiate message //await this.sendSignallingMessage({type: EventType.Invite, content}); //this.setState(CallState.InviteSent); From b84c90891ca90038c05e0e65186cb2bd4369ed78 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 11 Apr 2022 15:53:34 +0200 Subject: [PATCH 053/435] add very early datachannel support --- src/matrix/calls/LocalMedia.ts | 9 +++++++-- src/matrix/calls/PeerCall.ts | 18 ++++++++++++++++-- src/matrix/calls/group/Member.ts | 4 ++++ src/platform/types/WebRTC.ts | 10 ++-------- src/platform/web/dom/WebRTC.ts | 6 +++--- 5 files changed, 32 insertions(+), 15 deletions(-) diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index b64bdee5..fd91c61c 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -22,7 +22,8 @@ export class LocalMedia { constructor( public readonly cameraTrack?: Track, public readonly screenShareTrack?: Track, - public readonly microphoneTrack?: AudioTrack + public readonly microphoneTrack?: AudioTrack, + public readonly dataChannelOptions?: RTCDataChannelInit, ) {} withTracks(tracks: Track[]) { @@ -32,7 +33,11 @@ export class LocalMedia { if (cameraTrack && microphoneTrack && cameraTrack.streamId !== microphoneTrack.streamId) { throw new Error("The camera and audio track should have the same stream id"); } - return new LocalMedia(cameraTrack, screenShareTrack, microphoneTrack as AudioTrack); + return new LocalMedia(cameraTrack, screenShareTrack, microphoneTrack as AudioTrack, this.dataChannelOptions); + } + + withDataChannel(options: RTCDataChannelInit): LocalMedia { + return new LocalMedia(this.cameraTrack, this.screenShareTrack, this.microphoneTrack as AudioTrack, options); } get tracks(): Track[] { diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index ed10eebb..7e4dbb4a 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -85,6 +85,8 @@ export class PeerCall implements IDisposable { private sentEndOfCandidates: boolean = false; private iceDisconnectedTimeout?: Timeout; + private _dataChannel?: any; + constructor( private callId: string, private readonly options: Options, @@ -112,7 +114,12 @@ export class PeerCall implements IDisposable { outer.options.emitUpdate(outer, undefined); }); }, - onDataChannelChanged(dataChannel: DataChannel | undefined) {}, + onRemoteDataChannel(dataChannel: any | undefined) { + outer.logItem.wrap("onRemoteDataChannel", log => { + outer._dataChannel = dataChannel; + outer.options.emitUpdate(outer, undefined); + }); + }, onNegotiationNeeded() { const promiseCreator = () => { return outer.logItem.wrap("onNegotiationNeeded", log => { @@ -127,6 +134,8 @@ export class PeerCall implements IDisposable { }); } + get dataChannel(): any | undefined { return this._dataChannel; } + get state(): CallState { return this._state; } get remoteTracks(): Track[] { @@ -144,6 +153,9 @@ export class PeerCall implements IDisposable { for (const t of this.localMedia.tracks) { this.peerConnection.addTrack(t); } + if (this.localMedia.dataChannelOptions) { + this._dataChannel = this.peerConnection.createDataChannel(this.localMedia.dataChannelOptions); + } // after adding the local tracks, and wait for handleNegotiation to be called, // or invite glare where we give up our invite and answer instead await this.waitForState([CallState.InviteSent, CallState.CreateAnswer]); @@ -160,7 +172,9 @@ export class PeerCall implements IDisposable { for (const t of this.localMedia.tracks) { this.peerConnection.addTrack(t); } - + if (this.localMedia.dataChannelOptions) { + this._dataChannel = this.peerConnection.createDataChannel(this.localMedia.dataChannelOptions); + } let myAnswer: RTCSessionDescriptionInit; try { myAnswer = await this.peerConnection.createAnswer(); diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 05d64b92..7d849929 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -66,6 +66,10 @@ export class Member { return this.callDeviceMembership.device_id; } + get dataChannel(): any | undefined { + return this.peerCall?.dataChannel; + } + /** @internal */ connect(localMedia: LocalMedia) { this.logItem.wrap("connect", () => { diff --git a/src/platform/types/WebRTC.ts b/src/platform/types/WebRTC.ts index df8133ee..130cb7cb 100644 --- a/src/platform/types/WebRTC.ts +++ b/src/platform/types/WebRTC.ts @@ -26,21 +26,15 @@ export interface PeerConnectionHandler { onLocalIceCandidate(candidate: RTCIceCandidate); onIceGatheringStateChange(state: RTCIceGatheringState); onRemoteTracksChanged(tracks: Track[]); - onDataChannelChanged(dataChannel: DataChannel | undefined); + onRemoteDataChannel(dataChannel: any | undefined); onNegotiationNeeded(); // request the type of incoming stream getPurposeForStreamId(streamId: string): SDPStreamMetadataPurpose; } -// does it make sense to wrap this? -export interface DataChannel { - close(); - send(); -} export interface PeerConnection { notifyStreamPurposeChanged(): void; get remoteTracks(): Track[]; - get dataChannel(): DataChannel | undefined; get iceGatheringState(): RTCIceGatheringState; get signalingState(): RTCSignalingState; get localDescription(): RTCSessionDescription | undefined; @@ -52,7 +46,7 @@ export interface PeerConnection { addTrack(track: Track): void; removeTrack(track: Track): boolean; replaceTrack(oldTrack: Track, newTrack: Track): Promise; - createDataChannel(): DataChannel; + createDataChannel(options: RTCDataChannelInit): any; dispose(): void; close(): void; } diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 8c840ddc..78977eb8 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -46,7 +46,6 @@ class DOMPeerConnection implements PeerConnection { } get remoteTracks(): Track[] { return this._remoteTracks; } - get dataChannel(): DataChannel | undefined { return undefined; } get iceGatheringState(): RTCIceGatheringState { return this.peerConnection.iceGatheringState; } get localDescription(): RTCSessionDescription | undefined { return this.peerConnection.localDescription ?? undefined; } get signalingState(): RTCSignalingState { return this.peerConnection.signalingState; } @@ -119,8 +118,8 @@ class DOMPeerConnection implements PeerConnection { } } - createDataChannel(): DataChannel { - return undefined as any;// new DataChannel(this.peerConnection.createDataChannel()); + createDataChannel(options: RTCDataChannelInit): any { + return this.peerConnection.createDataChannel("channel", options); } private registerHandler() { @@ -164,6 +163,7 @@ class DOMPeerConnection implements PeerConnection { this.handler.onNegotiationNeeded(); break; case "datachannel": + this.handler.onRemoteDataChannel((evt as RTCDataChannelEvent).channel); break; } } From 9be64730b6ecfd34bc62454e4c3e53f21d18e676 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 11 Apr 2022 15:54:06 +0200 Subject: [PATCH 054/435] don't automatically join a call we create --- src/domain/session/room/RoomViewModel.js | 3 ++- src/matrix/calls/CallHandler.ts | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 6dd8c8bd..87b4aa9d 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -359,7 +359,8 @@ export class RoomViewModel extends ViewModel { const mediaTracks = await this.platform.mediaDevices.getMediaTracks(false, true); const localMedia = new LocalMedia().withTracks(mediaTracks); // this will set the callViewModel above as a call will be added to callHandler.calls - await session.callHandler.createCall(this._room.id, localMedia, "A call " + Math.round(this.platform.random() * 100)); + const call = await session.callHandler.createCall(this._room.id, localMedia, "A call " + Math.round(this.platform.random() * 100)); + await call.join(localMedia); } catch (err) { console.error(err.stack); alert(err.message); diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index d658121e..31829396 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -117,7 +117,6 @@ export class CallHandler { try { await call.create(localMedia); - await call.join(localMedia); // store call info so it will ring again when reopening the app const txn = await this.options.storage.readWriteTxn([this.options.storage.storeNames.calls]); txn.calls.add({ From 387bad73b01686aed3280719db20a856f16bcb61 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 11 Apr 2022 15:54:20 +0200 Subject: [PATCH 055/435] remove debug alert --- src/domain/session/room/RoomViewModel.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 87b4aa9d..0683a0f2 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -363,7 +363,6 @@ export class RoomViewModel extends ViewModel { await call.join(localMedia); } catch (err) { console.error(err.stack); - alert(err.message); } } } From e0efbaeb4e79ae014ae16b24f89fe92dfcc61687 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 11 Apr 2022 15:54:31 +0200 Subject: [PATCH 056/435] show start time in console logger --- src/logging/ConsoleLogger.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/logging/ConsoleLogger.ts b/src/logging/ConsoleLogger.ts index 1643e48a..1d8232f9 100644 --- a/src/logging/ConsoleLogger.ts +++ b/src/logging/ConsoleLogger.ts @@ -45,7 +45,7 @@ function filterValues(values: LogItemValues): LogItemValues | null { } function printToConsole(item: LogItem): void { - const label = `${itemCaption(item)} (${item.duration}ms)`; + const label = `${itemCaption(item)} (@${item.start}ms, duration: ${item.duration}ms)`; const filteredValues = filterValues(item.values); const shouldGroup = item.children || filteredValues; if (shouldGroup) { From c99fc2ad706a0e481546078363430c51f0abca40 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 11 Apr 2022 15:54:41 +0200 Subject: [PATCH 057/435] use deviceId getter in Member --- src/matrix/calls/group/Member.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 7d849929..3eb5f17f 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -125,7 +125,7 @@ export class Member { const payload = { messages: { [this.member.userId]: { - [this.callDeviceMembership.device_id]: groupMessage.content + [this.deviceId]: groupMessage.content } } }; From 5cacdcfee0d3c9a6d3664cd5e86dead7d8cef96f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 11 Apr 2022 15:55:02 +0200 Subject: [PATCH 058/435] Add leave button to call view --- src/platform/web/ui/session/room/CallView.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/platform/web/ui/session/room/CallView.ts b/src/platform/web/ui/session/room/CallView.ts index ae7b4138..f8386116 100644 --- a/src/platform/web/ui/session/room/CallView.ts +++ b/src/platform/web/ui/session/room/CallView.ts @@ -34,7 +34,10 @@ export class CallView extends TemplateView { return t.div({class: "CallView"}, [ t.p(vm => `Call ${vm.name} (${vm.id})`), t.div({class: "CallView_me"}, bindVideoTracks(t, t.video({autoplay: true, width: 240}), vm => vm.localTracks)), - t.view(new ListView({list: vm.memberViewModels}, vm => new MemberView(vm))) + t.view(new ListView({list: vm.memberViewModels}, vm => new MemberView(vm))), + t.div({class: "buttons"}, [ + t.button({onClick: () => vm.leave()}, "Leave") + ]) ]); } } From 517e796e902a27d5b5ae81367480589f0c066ff5 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 11 Apr 2022 15:56:31 +0200 Subject: [PATCH 059/435] remove obsolete import --- src/matrix/calls/PeerCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 7e4dbb4a..87b96139 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -23,7 +23,7 @@ import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; import type {TimeoutCreator, Timeout} from "../../platform/types/types"; -import {WebRTC, PeerConnection, PeerConnectionHandler, DataChannel} from "../../platform/types/WebRTC"; +import {WebRTC, PeerConnection, PeerConnectionHandler} from "../../platform/types/WebRTC"; import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; import type {LocalMedia} from "./LocalMedia"; From a710f394eb2c923f3752f42ea613f652f44ab9eb Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 11 Apr 2022 15:57:23 +0200 Subject: [PATCH 060/435] fix lint warning --- src/domain/session/room/RoomViewModel.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 0683a0f2..c12ed3b8 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -28,7 +28,7 @@ import {LocalMedia} from "../../../matrix/calls/LocalMedia"; export class RoomViewModel extends ViewModel { constructor(options) { super(options); - const {room, session} = options; + const {room} = options; this._room = room; this._timelineVM = null; this._tilesCreator = null; From fd5b2aa7bb0ff31ff84e614ca07c823b9862dd59 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 11 Apr 2022 16:29:46 +0200 Subject: [PATCH 061/435] only create datachannel on side that sends invite --- src/matrix/calls/PeerCall.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 87b96139..fc37807d 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -172,9 +172,6 @@ export class PeerCall implements IDisposable { for (const t of this.localMedia.tracks) { this.peerConnection.addTrack(t); } - if (this.localMedia.dataChannelOptions) { - this._dataChannel = this.peerConnection.createDataChannel(this.localMedia.dataChannelOptions); - } let myAnswer: RTCSessionDescriptionInit; try { myAnswer = await this.peerConnection.createAnswer(); From 797cb23cc7adb06111d4a3e3e1c8bdd7628aaa09 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 12 Apr 2022 14:02:13 +0200 Subject: [PATCH 062/435] implement receiving hangup, and retry on connection failure --- src/matrix/calls/PeerCall.ts | 16 +++++++++++++++- src/matrix/calls/group/Member.ts | 24 ++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index fc37807d..fa1063ec 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -86,6 +86,7 @@ export class PeerCall implements IDisposable { private iceDisconnectedTimeout?: Timeout; private _dataChannel?: any; + private _hangupReason?: CallErrorCode; constructor( private callId: string, @@ -138,6 +139,8 @@ export class PeerCall implements IDisposable { get state(): CallState { return this._state; } + get hangupReason(): CallErrorCode | undefined { return this._hangupReason; } + get remoteTracks(): Track[] { return this.peerConnection.remoteTracks; } @@ -374,6 +377,17 @@ export class PeerCall implements IDisposable { } } + private handleHangupReceived(content: MCallHangupReject, log: ILogItem) { + // party ID must match (our chosen partner hanging up the call) or be undefined (we haven't chosen + // a partner yet but we're treating the hangup as a reject as per VoIP v0) + // if (this.state === CallState.Ringing) { + // default reason is user_hangup + this.terminate(CallParty.Remote, content.reason || CallErrorCode.UserHangup, log); + // } else { + // log.set("ignored", true); + // } + }; + private async handleFirstInvite(content: MCallInvite, partyId: PartyId, log: ILogItem): Promise { if (this._state !== CallState.Fledgling || this.opponentPartyId !== undefined) { // TODO: hangup or ignore? @@ -789,7 +803,7 @@ export class PeerCall implements IDisposable { } this.hangupParty = hangupParty; - // this.hangupReason = hangupReason; + this._hangupReason = hangupReason; this.setState(CallState.Ended, log); //this.localMedia?.dispose(); //this.localMedia = undefined; diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 3eb5f17f..0ac7df99 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -16,7 +16,7 @@ limitations under the License. import {PeerCall, CallState} from "../PeerCall"; import {makeTxnId, makeId} from "../../common"; -import {EventType} from "../callEventTypes"; +import {EventType, CallErrorCode} from "../callEventTypes"; import {formatToDeviceMessagesPayload} from "../../common"; import type {Options as PeerCallOptions} from "../PeerCall"; @@ -39,9 +39,19 @@ export type Options = Omit void, } +const errorCodesWithoutRetry = [ + CallErrorCode.UserHangup, + CallErrorCode.AnsweredElsewhere, + CallErrorCode.Replaced, + CallErrorCode.UserBusy, + CallErrorCode.Transfered, + CallErrorCode.NewSession +]; + export class Member { private peerCall?: PeerCall; private localMedia?: LocalMedia; + private retryCount: number = 0; constructor( public readonly member: RoomMember, @@ -109,6 +119,17 @@ export class Member { if (peerCall.state === CallState.Ringing) { peerCall.answer(this.localMedia!); } + else if (peerCall.state === CallState.Ended) { + const hangupReason = peerCall.hangupReason; + peerCall.dispose(); + this.peerCall = undefined; + if (hangupReason && !errorCodesWithoutRetry.includes(hangupReason)) { + this.retryCount += 1; + if (this.retryCount <= 3) { + this.connect(this.localMedia!); + } + } + } this.options.emitUpdate(this, params); } @@ -151,7 +172,6 @@ export class Member { this.peerCall = this._createPeerCall(message.content.call_id); } if (this.peerCall) { - const prevState = this.peerCall.state; this.peerCall.handleIncomingSignallingMessage(message, deviceId); } else { // TODO: need to buffer events until invite comes? From 2635adb2325009c7a51af1e53b42e0cdff3c52ee Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 12 Apr 2022 14:02:38 +0200 Subject: [PATCH 063/435] hardcode turn server for now --- src/matrix/Session.js | 4 ++++ src/matrix/calls/PeerCall.ts | 4 +++- src/platform/types/WebRTC.ts | 2 +- src/platform/web/dom/WebRTC.ts | 6 +++--- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index aa7dbf7b..bd361434 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -96,6 +96,10 @@ export class Session { ownDeviceId: sessionInfo.deviceId, ownUserId: sessionInfo.userId, logger: this._platform.logger, + turnServers: [{ + urls: ["stun:turn.matrix.org"], + }], + forceTURN: false, }); this._deviceMessageHandler = new DeviceMessageHandler({storage, callHandler: this._callHandler}); this._olm = olm; diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index fa1063ec..8559982c 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -45,6 +45,8 @@ import type { export type Options = { webRTC: WebRTC, + forceTURN: boolean, + turnServers: RTCIceServer[], createTimeout: TimeoutCreator, emitUpdate: (peerCall: PeerCall, params: any) => void; sendSignallingMessage: (message: SignallingMessage, log: ILogItem) => Promise; @@ -132,7 +134,7 @@ export class PeerCall implements IDisposable { getPurposeForStreamId(streamId: string): SDPStreamMetadataPurpose { return outer.remoteSDPStreamMetadata?.[streamId]?.purpose ?? SDPStreamMetadataPurpose.Usermedia; } - }); + }, this.options.forceTURN, this.options.turnServers, 0); } get dataChannel(): any | undefined { return this._dataChannel; } diff --git a/src/platform/types/WebRTC.ts b/src/platform/types/WebRTC.ts index 130cb7cb..a1a7b7cd 100644 --- a/src/platform/types/WebRTC.ts +++ b/src/platform/types/WebRTC.ts @@ -18,7 +18,7 @@ import {Track, TrackType} from "./MediaDevices"; import {SDPStreamMetadataPurpose} from "../../matrix/calls/callEventTypes"; export interface WebRTC { - createPeerConnection(handler: PeerConnectionHandler): PeerConnection; + createPeerConnection(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize): PeerConnection; } export interface PeerConnectionHandler { diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 78977eb8..861a91f1 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -24,8 +24,8 @@ export const SPEAKING_THRESHOLD = -60; // dB const SPEAKING_SAMPLE_COUNT = 8; // samples export class DOMWebRTC implements WebRTC { - createPeerConnection(handler: PeerConnectionHandler): PeerConnection { - return new DOMPeerConnection(handler, false, []); + createPeerConnection(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize): PeerConnection { + return new DOMPeerConnection(handler, forceTURN, turnServers, iceCandidatePoolSize); } } @@ -35,7 +35,7 @@ class DOMPeerConnection implements PeerConnection { //private dataChannelWrapper?: DOMDataChannel; private _remoteTracks: TrackWrapper[] = []; - constructor(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize = 0) { + constructor(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize) { this.handler = handler; this.peerConnection = new RTCPeerConnection({ iceTransportPolicy: forceTURN ? 'relay' : undefined, From 0e9307608b3ad04bc2687fa4c1766a1eda12865e Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 12 Apr 2022 14:02:57 +0200 Subject: [PATCH 064/435] update TODO --- src/matrix/calls/TODO.md | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index 4faf4f4e..ae35447b 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -6,7 +6,25 @@ - https://github.com/matrix-org/matrix-doc/pull/3291 Muting in VoIP calls - https://github.com/matrix-org/matrix-doc/pull/3401 Native Group VoIP Signalling + ## TODO + - DONE: implement receiving hangup + - making logging better + - implement renegotiation + - implement to_device messages arriving before m.call(.member) state event + - implement muting tracks with m.call.sdp_stream_metadata_changed + - implement cloning the localMedia so it works in safari? + - DONE: implement 3 retries per peer + - reeable crypto & implement fetching olm keys before sending encrypted signalling message + - local echo for join/leave buttons? + - make UI pretsy + - figure out video layout + - figure out nav structure + - batch outgoing to_device messages in one request to homeserver for operations that will send out an event to all participants (e.g. mute) + - don't load all members when loading calls to know whether they are ringing and joined by ourself + - only load our own member once, then have a way to load additional members on a call. + +## TODO (old) - PeerCall - send invite - implement terminate @@ -34,7 +52,7 @@ ## Store ongoing calls -Add store with all ongoing calls so when we quit and start again, we don't have to go through all the past calls to know which ones might still be ongoing. +DONE: Add store with all ongoing calls so when we quit and start again, we don't have to go through all the past calls to know which ones might still be ongoing. ## Notes @@ -104,10 +122,10 @@ Do work needed for state events - DONEish: receiving (almost done?) - DONEish: sending logging -Expose call objects -expose volume events from audiotrack to group call -Write view model -write view +DONE: Expose call objects + expose volume events from audiotrack to group call +DONE: Write view model +DONE: write view - handle glare edge-cases (not yet sent): https://spec.matrix.org/latest/client-server-api/#glare ## Calls questions From 36dc463d2325c857493072c2a69d681589e22684 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 12 Apr 2022 21:20:15 +0200 Subject: [PATCH 065/435] update TODO --- src/matrix/calls/TODO.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index ae35447b..8c041fcc 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -11,6 +11,8 @@ - DONE: implement receiving hangup - making logging better - implement renegotiation + - finish session id support + - call peers are essentially identified by (userid, deviceid, sessionid). If see a new session id, we first disconnect from the current member so we're ready to connect with a clean slate again (in a member event, also in to_device? no harm I suppose, given olm encryption ensures you can't spoof the deviceid). - implement to_device messages arriving before m.call(.member) state event - implement muting tracks with m.call.sdp_stream_metadata_changed - implement cloning the localMedia so it works in safari? @@ -23,7 +25,7 @@ - batch outgoing to_device messages in one request to homeserver for operations that will send out an event to all participants (e.g. mute) - don't load all members when loading calls to know whether they are ringing and joined by ourself - only load our own member once, then have a way to load additional members on a call. - + - see if we remove partyId entirely, it is only used for detecting remote echo which is not an issue for group calls? see https://github.com/matrix-org/matrix-spec-proposals/blob/dbkr/msc2746/proposals/2746-reliable-voip.md#add-party_id-to-all-voip-events ## TODO (old) - PeerCall - send invite From 2d4301fe5a5c451100f68e275f7a6fe90b0f7ba2 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 12 Apr 2022 21:20:24 +0200 Subject: [PATCH 066/435] WIP: expose streams, senders and receivers --- .../session/room/timeline/tiles/CallTile.js | 1 + src/matrix/calls/LocalMedia.ts | 44 +-- src/matrix/calls/PeerCall.ts | 4 +- src/platform/types/MediaDevices.ts | 31 +- src/platform/types/WebRTC.ts | 42 ++- src/platform/web/dom/MediaDevices.ts | 79 +++-- src/platform/web/dom/WebRTC.ts | 299 ++++++++++++------ 7 files changed, 308 insertions(+), 192 deletions(-) diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index 32807958..2d2d9dab 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -75,6 +75,7 @@ export class CallTile extends SimpleTile { async join() { if (this.canJoin) { const mediaTracks = await this.platform.mediaDevices.getMediaTracks(false, true); + // const screenShareTrack = await this.platform.mediaDevices.getScreenShareTrack(); const localMedia = new LocalMedia().withTracks(mediaTracks); await this._call.join(localMedia); } diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index fd91c61c..6eb7a225 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -15,37 +15,26 @@ limitations under the License. */ import {SDPStreamMetadataPurpose} from "./callEventTypes"; -import {Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; +import {Stream} from "../../platform/types/MediaDevices"; import {SDPStreamMetadata} from "./callEventTypes"; export class LocalMedia { constructor( - public readonly cameraTrack?: Track, - public readonly screenShareTrack?: Track, - public readonly microphoneTrack?: AudioTrack, + public readonly userMedia?: Stream, + public readonly screenShare?: Stream, public readonly dataChannelOptions?: RTCDataChannelInit, ) {} - withTracks(tracks: Track[]) { - const cameraTrack = tracks.find(t => t.type === TrackType.Camera) ?? this.cameraTrack; - const screenShareTrack = tracks.find(t => t.type === TrackType.ScreenShare) ?? this.screenShareTrack; - const microphoneTrack = tracks.find(t => t.type === TrackType.Microphone) ?? this.microphoneTrack; - if (cameraTrack && microphoneTrack && cameraTrack.streamId !== microphoneTrack.streamId) { - throw new Error("The camera and audio track should have the same stream id"); - } - return new LocalMedia(cameraTrack, screenShareTrack, microphoneTrack as AudioTrack, this.dataChannelOptions); + withUserMedia(stream: Stream) { + return new LocalMedia(stream, this.screenShare, this.dataChannelOptions); + } + + withScreenShare(stream: Stream) { + return new LocalMedia(this.userMedia, stream, this.dataChannelOptions); } withDataChannel(options: RTCDataChannelInit): LocalMedia { - return new LocalMedia(this.cameraTrack, this.screenShareTrack, this.microphoneTrack as AudioTrack, options); - } - - get tracks(): Track[] { - const tracks: Track[] = []; - if (this.cameraTrack) { tracks.push(this.cameraTrack); } - if (this.screenShareTrack) { tracks.push(this.screenShareTrack); } - if (this.microphoneTrack) { tracks.push(this.microphoneTrack); } - return tracks; + return new LocalMedia(this.userMedia, this.screenShare, options); } getSDPMetadata(): SDPStreamMetadata { @@ -54,8 +43,8 @@ export class LocalMedia { if (userMediaTrack) { metadata[userMediaTrack.streamId] = { purpose: SDPStreamMetadataPurpose.Usermedia, - audio_muted: this.microphoneTrack?.muted ?? false, - video_muted: this.cameraTrack?.muted ?? false, + audio_muted: this.microphoneTrack?.muted ?? true, + video_muted: this.cameraTrack?.muted ?? true, }; } if (this.screenShareTrack) { @@ -67,13 +56,12 @@ export class LocalMedia { } clone() { - // TODO: implement - return this; + return new LocalMedia(this.userMedia?.clone(), this.screenShare?.clone(), this.dataChannelOptions); } dispose() { - this.cameraTrack?.stop(); - this.microphoneTrack?.stop(); - this.screenShareTrack?.stop(); + this.userMedia?.audioTrack?.stop(); + this.userMedia?.videoTrack?.stop(); + this.screenShare?.videoTrack?.stop(); } } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 8559982c..644ed160 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -701,8 +701,6 @@ export class PeerCall implements IDisposable { private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { this.remoteSDPStreamMetadata = recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); - // will rerequest stream purpose for all tracks and set track.type accordingly - this.peerConnection.notifyStreamPurposeChanged(); for (const track of this.peerConnection.remoteTracks) { const streamMetaData = this.remoteSDPStreamMetadata?.[track.streamId]; if (streamMetaData) { @@ -757,6 +755,8 @@ export class PeerCall implements IDisposable { this.iceDisconnectedTimeout?.abort(); this.iceDisconnectedTimeout = undefined; this.setState(CallState.Connected, log); + const transceivers = this.peerConnection.peerConnection.getTransceivers(); + console.log(transceivers); } else if (state == 'failed') { this.iceDisconnectedTimeout?.abort(); this.iceDisconnectedTimeout = undefined; diff --git a/src/platform/types/MediaDevices.ts b/src/platform/types/MediaDevices.ts index db267871..d0edbbea 100644 --- a/src/platform/types/MediaDevices.ts +++ b/src/platform/types/MediaDevices.ts @@ -18,29 +18,32 @@ export interface MediaDevices { // filter out audiooutput enumerate(): Promise; // to assign to a video element, we downcast to WrappedTrack and use the stream property. - getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise; - getScreenShareTrack(): Promise; + getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise; + getScreenShareTrack(): Promise; } -export enum TrackType { - ScreenShare, - Camera, - Microphone, +export interface Stream { + readonly audioTrack: AudioTrack | undefined; + readonly videoTrack: Track | undefined; + readonly id: string; + clone(): Stream; +} + +export enum TrackKind { + Video = "video", + Audio = "audio" } export interface Track { - get type(): TrackType; - get label(): string; - get id(): string; - get streamId(): string; - get settings(): MediaTrackSettings; - get muted(): boolean; - setMuted(muted: boolean): void; + readonly kind: TrackKind; + readonly label: string; + readonly id: string; + readonly settings: MediaTrackSettings; stop(): void; - clone(): Track; } export interface AudioTrack extends Track { + // TODO: how to emit updates on this? get isSpeaking(): boolean; } diff --git a/src/platform/types/WebRTC.ts b/src/platform/types/WebRTC.ts index a1a7b7cd..a0512163 100644 --- a/src/platform/types/WebRTC.ts +++ b/src/platform/types/WebRTC.ts @@ -14,38 +14,62 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {Track, TrackType} from "./MediaDevices"; +import {Track, Stream} from "./MediaDevices"; import {SDPStreamMetadataPurpose} from "../../matrix/calls/callEventTypes"; export interface WebRTC { createPeerConnection(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize): PeerConnection; } +export interface StreamSender { + get stream(): Stream; + get audioSender(): TrackSender | undefined; + get videoSender(): TrackSender | undefined; +} + +export interface StreamReceiver { + get stream(): Stream; + get audioReceiver(): TrackReceiver | undefined; + get videoReceiver(): TrackReceiver | undefined; +} + +export interface TrackReceiver { + get track(): Track; + get enabled(): boolean; + enable(enabled: boolean); // this modifies the transceiver direction +} + +export interface TrackSender extends TrackReceiver { + /** replaces the track if possible without renegotiation. Can throw. */ + replaceTrack(track: Track): Promise; + /** make any needed adjustments to the sender or transceiver settings + * depending on the purpose, after adding the track to the connection */ + prepareForPurpose(purpose: SDPStreamMetadataPurpose): void; +} + export interface PeerConnectionHandler { onIceConnectionStateChange(state: RTCIceConnectionState); onLocalIceCandidate(candidate: RTCIceCandidate); onIceGatheringStateChange(state: RTCIceGatheringState); - onRemoteTracksChanged(tracks: Track[]); + onRemoteStreamRemoved(stream: Stream); + onRemoteTracksAdded(receiver: TrackReceiver); onRemoteDataChannel(dataChannel: any | undefined); onNegotiationNeeded(); - // request the type of incoming stream - getPurposeForStreamId(streamId: string): SDPStreamMetadataPurpose; } export interface PeerConnection { - notifyStreamPurposeChanged(): void; - get remoteTracks(): Track[]; get iceGatheringState(): RTCIceGatheringState; get signalingState(): RTCSignalingState; get localDescription(): RTCSessionDescription | undefined; + get localStreams(): ReadonlyArray; + get remoteStreams(): ReadonlyArray; createOffer(): Promise; createAnswer(): Promise; setLocalDescription(description?: RTCSessionDescriptionInit): Promise; setRemoteDescription(description: RTCSessionDescriptionInit): Promise; addIceCandidate(candidate: RTCIceCandidate): Promise; - addTrack(track: Track): void; - removeTrack(track: Track): boolean; - replaceTrack(oldTrack: Track, newTrack: Track): Promise; + addTrack(track: Track): TrackSender | undefined; + removeTrack(track: TrackSender): void; createDataChannel(options: RTCDataChannelInit): any; dispose(): void; close(): void; diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index 49fd7fa5..52ca132b 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {MediaDevices as IMediaDevices, TrackType, Track, AudioTrack} from "../../types/MediaDevices"; +import {MediaDevices as IMediaDevices, Stream, Track, TrackKind, AudioTrack} from "../../types/MediaDevices"; const POLLING_INTERVAL = 200; // ms export const SPEAKING_THRESHOLD = -60; // dB @@ -28,22 +28,14 @@ export class MediaDevicesWrapper implements IMediaDevices { return this.mediaDevices.enumerateDevices(); } - async getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise { + async getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise { const stream = await this.mediaDevices.getUserMedia(this.getUserMediaContraints(audio, video)); - const tracks = stream.getTracks().map(t => { - const type = t.kind === "audio" ? TrackType.Microphone : TrackType.Camera; - return wrapTrack(t, stream, type); - }); - return tracks; + return new StreamWrapper(stream); } - async getScreenShareTrack(): Promise { + async getScreenShareTrack(): Promise { const stream = await this.mediaDevices.getDisplayMedia(this.getScreenshareContraints()); - const videoTrack = stream.getTracks().find(t => t.kind === "video"); - if (videoTrack) { - return wrapTrack(videoTrack, stream, TrackType.ScreenShare); - } - return; + return new StreamWrapper(stream); } private getUserMediaContraints(audio: boolean | MediaDeviceInfo, video: boolean | MediaDeviceInfo): MediaStreamConstraints { @@ -78,43 +70,50 @@ export class MediaDevicesWrapper implements IMediaDevices { } } -export function wrapTrack(track: MediaStreamTrack, stream: MediaStream, type: TrackType) { - if (track.kind === "audio") { - return new AudioTrackWrapper(track, stream, type); - } else { - return new TrackWrapper(track, stream, type); +export class StreamWrapper implements Stream { + + public audioTrack: AudioTrackWrapper | undefined; + public videoTrack: TrackWrapper | undefined; + + constructor(public readonly stream: MediaStream) { + for (const track of stream.getTracks()) { + this.update(track); + } + } + + get id(): string { return this.stream.id; } + + clone(): Stream { + return new StreamWrapper(this.stream.clone()); + } + + update(track: MediaStreamTrack): TrackWrapper | undefined { + if (track.kind === "video") { + if (!this.videoTrack || track.id !== this.videoTrack.track.id) { + this.videoTrack = new TrackWrapper(track, this.stream); + return this.videoTrack; + } + } else if (track.kind === "audio") { + if (!this.audioTrack || track.id !== this.audioTrack.track.id) { + this.audioTrack = new AudioTrackWrapper(track, this.stream); + return this.audioTrack; + } + } } } export class TrackWrapper implements Track { constructor( public readonly track: MediaStreamTrack, - public readonly stream: MediaStream, - private _type: TrackType, + public readonly stream: MediaStream ) {} - get type(): TrackType { return this._type; } + get kind(): TrackKind { return this.track.kind as TrackKind; } get label(): string { return this.track.label; } get id(): string { return this.track.id; } - get streamId(): string { return this.stream.id; } - get muted(): boolean { return this.track.muted; } get settings(): MediaTrackSettings { return this.track.getSettings(); } - setMuted(muted: boolean): void { - this.track.enabled = !muted; - } - - setType(type: TrackType): void { - this._type = type; - } - - stop() { - this.track.stop(); - } - - clone() { - return this.track.clone(); - } + stop() { this.track.stop(); } } export class AudioTrackWrapper extends TrackWrapper { @@ -127,8 +126,8 @@ export class AudioTrackWrapper extends TrackWrapper { private volumeLooperTimeout: number; private speakingVolumeSamples: number[]; - constructor(track: MediaStreamTrack, stream: MediaStream, type: TrackType) { - super(track, stream, type); + constructor(track: MediaStreamTrack, stream: MediaStream) { + super(track, stream); this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); this.initVolumeMeasuring(); this.measureVolumeActivity(true); diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 861a91f1..6760182d 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -14,9 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {TrackWrapper, wrapTrack} from "./MediaDevices"; -import {Track, TrackType} from "../../types/MediaDevices"; -import {WebRTC, PeerConnectionHandler, DataChannel, PeerConnection} from "../../types/WebRTC"; +import {StreamWrapper, TrackWrapper, AudioTrackWrapper} from "./MediaDevices"; +import {Stream, Track, AudioTrack, TrackKind} from "../../types/MediaDevices"; +import {WebRTC, PeerConnectionHandler, StreamSender, TrackSender, StreamReceiver, TrackReceiver, PeerConnection} from "../../types/WebRTC"; import {SDPStreamMetadataPurpose} from "../../../matrix/calls/callEventTypes"; const POLLING_INTERVAL = 200; // ms @@ -29,11 +29,171 @@ export class DOMWebRTC implements WebRTC { } } +export class RemoteStreamWrapper extends StreamWrapper { + constructor(stream: MediaStream, private readonly emptyCallback: (stream: RemoteStreamWrapper) => void) { + super(stream); + this.stream.addEventListener("removetrack", this.onTrackRemoved); + } + + onTrackRemoved = (evt: MediaStreamTrackEvent) => { + if (evt.track.id === this.audioTrack?.track.id) { + this.audioTrack = undefined; + } else if (evt.track.id === this.videoTrack?.track.id) { + this.videoTrack = undefined; + } + if (!this.audioTrack && !this.videoTrack) { + this.emptyCallback(this); + } + }; + + dispose() { + this.stream.removeEventListener("removetrack", this.onTrackRemoved); + } +} + +export class DOMStreamSender implements StreamSender { + public audioSender: DOMTrackSender | undefined; + public videoSender: DOMTrackSender | undefined; + + constructor(public readonly stream: StreamWrapper) {} + + update(transceivers: ReadonlyArray, sender: RTCRtpSender): DOMTrackSender | undefined { + const transceiver = transceivers.find(t => t.sender === sender); + if (transceiver && sender.track) { + const trackWrapper = this.stream.update(sender.track); + if (trackWrapper) { + if (trackWrapper.kind === TrackKind.Video) { + this.videoSender = new DOMTrackSender(trackWrapper, transceiver); + return this.videoSender; + } else { + this.audioSender = new DOMTrackSender(trackWrapper, transceiver); + return this.audioSender; + } + } + } + } +} + +export class DOMStreamReceiver implements StreamReceiver { + public audioReceiver: DOMTrackReceiver | undefined; + public videoReceiver: DOMTrackReceiver | undefined; + + constructor(public readonly stream: RemoteStreamWrapper) {} + + update(event: RTCTrackEvent): DOMTrackReceiver | undefined { + const {receiver} = event; + const {track} = receiver; + const trackWrapper = this.stream.update(track); + if (trackWrapper) { + if (trackWrapper.kind === TrackKind.Video) { + this.videoReceiver = new DOMTrackReceiver(trackWrapper, event.transceiver); + return this.videoReceiver; + } else { + this.audioReceiver = new DOMTrackReceiver(trackWrapper, event.transceiver); + return this.audioReceiver; + } + } + } +} + +export class DOMTrackSenderOrReceiver implements TrackReceiver { + constructor( + public readonly track: TrackWrapper, + public readonly transceiver: RTCRtpTransceiver, + private readonly exclusiveValue: RTCRtpTransceiverDirection, + private readonly excludedValue: RTCRtpTransceiverDirection + ) {} + + get enabled(): boolean { + return this.transceiver.currentDirection === "sendrecv" || + this.transceiver.currentDirection === this.exclusiveValue; + } + + enable(enabled: boolean) { + if (enabled !== this.enabled) { + if (enabled) { + if (this.transceiver.currentDirection === "inactive") { + this.transceiver.direction = this.exclusiveValue; + } else { + this.transceiver.direction = "sendrecv"; + } + } else { + if (this.transceiver.currentDirection === "sendrecv") { + this.transceiver.direction = this.excludedValue; + } else { + this.transceiver.direction = "inactive"; + } + } + } + } +} + +export class DOMTrackReceiver extends DOMTrackSenderOrReceiver { + constructor( + track: TrackWrapper, + transceiver: RTCRtpTransceiver, + ) { + super(track, transceiver, "recvonly", "sendonly"); + } +} + +export class DOMTrackSender extends DOMTrackSenderOrReceiver { + constructor( + track: TrackWrapper, + transceiver: RTCRtpTransceiver, + ) { + super(track, transceiver, "sendonly", "recvonly"); + } + /** replaces the track if possible without renegotiation. Can throw. */ + replaceTrack(track: Track): Promise { + return this.transceiver.sender.replaceTrack(track ? (track as TrackWrapper).track : null); + } + + prepareForPurpose(purpose: SDPStreamMetadataPurpose): void { + if (purpose === SDPStreamMetadataPurpose.Screenshare) { + this.getRidOfRTXCodecs(); + } + } + + /** + * This method removes all video/rtx codecs from screensharing video + * transceivers. This is necessary since they can cause problems. Without + * this the following steps should produce an error: + * Chromium calls Firefox + * Firefox answers + * Firefox starts screen-sharing + * Chromium starts screen-sharing + * Call crashes for Chromium with: + * [96685:23:0518/162603.933321:ERROR:webrtc_video_engine.cc(3296)] RTX codec (PT=97) mapped to PT=96 which is not in the codec list. + * [96685:23:0518/162603.933377:ERROR:webrtc_video_engine.cc(1171)] GetChangedRecvParameters called without any video codecs. + * [96685:23:0518/162603.933430:ERROR:sdp_offer_answer.cc(4302)] Failed to set local video description recv parameters for m-section with mid='2'. (INVALID_PARAMETER) + */ + private getRidOfRTXCodecs(): void { + // RTCRtpReceiver.getCapabilities and RTCRtpSender.getCapabilities don't seem to be supported on FF + if (!RTCRtpReceiver.getCapabilities || !RTCRtpSender.getCapabilities) return; + + const recvCodecs = RTCRtpReceiver.getCapabilities("video")?.codecs ?? []; + const sendCodecs = RTCRtpSender.getCapabilities("video")?.codecs ?? []; + const codecs = [...sendCodecs, ...recvCodecs]; + + for (const codec of codecs) { + if (codec.mimeType === "video/rtx") { + const rtxCodecIndex = codecs.indexOf(codec); + codecs.splice(rtxCodecIndex, 1); + } + } + if (this.transceiver.sender.track?.kind === "video" || + this.transceiver.receiver.track?.kind === "video") { + this.transceiver.setCodecPreferences(codecs); + } + } +} + class DOMPeerConnection implements PeerConnection { private readonly peerConnection: RTCPeerConnection; private readonly handler: PeerConnectionHandler; - //private dataChannelWrapper?: DOMDataChannel; - private _remoteTracks: TrackWrapper[] = []; + public readonly localStreams: DOMStreamSender[]; + public readonly remoteStreams: DOMStreamReceiver[]; constructor(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize) { this.handler = handler; @@ -45,7 +205,6 @@ class DOMPeerConnection implements PeerConnection { this.registerHandler(); } - get remoteTracks(): Track[] { return this._remoteTracks; } get iceGatheringState(): RTCIceGatheringState { return this.peerConnection.iceGatheringState; } get localDescription(): RTCSessionDescription | undefined { return this.peerConnection.localDescription ?? undefined; } get signalingState(): RTCSignalingState { return this.peerConnection.signalingState; } @@ -74,48 +233,26 @@ class DOMPeerConnection implements PeerConnection { return this.peerConnection.close(); } - addTrack(track: Track): void { + addTrack(track: Track): DOMTrackSender | undefined { if (!(track instanceof TrackWrapper)) { throw new Error("Not a TrackWrapper"); } - this.peerConnection.addTrack(track.track, track.stream); - if (track.type === TrackType.ScreenShare) { - this.getRidOfRTXCodecs(track); + const sender = this.peerConnection.addTrack(track.track, track.stream); + let streamSender: DOMStreamSender | undefined = this.localStreams.find(s => s.stream.id === track.stream.id); + if (!streamSender) { + streamSender = new DOMStreamSender(new StreamWrapper(track.stream)); + this.localStreams.push(streamSender); } + const trackSender = streamSender.update(this.peerConnection.getTransceivers(), sender); + return trackSender; } - removeTrack(track: Track): boolean { - if (!(track instanceof TrackWrapper)) { - throw new Error("Not a TrackWrapper"); - } - const sender = this.peerConnection.getSenders().find(s => s.track === track.track); - if (sender) { - this.peerConnection.removeTrack(sender); - return true; - } - return false; - } - - async replaceTrack(oldTrack: Track, newTrack: Track): Promise { - if (!(oldTrack instanceof TrackWrapper) || !(newTrack instanceof TrackWrapper)) { - throw new Error("Not a TrackWrapper"); - } - const sender = this.peerConnection.getSenders().find(s => s.track === oldTrack.track); - if (sender) { - await sender.replaceTrack(newTrack.track); - if (newTrack.type === TrackType.ScreenShare) { - this.getRidOfRTXCodecs(newTrack); - } - return true; - } - return false; - } - - notifyStreamPurposeChanged(): void { - for (const track of this.remoteTracks) { - const wrapper = track as TrackWrapper; - wrapper.setType(this.getRemoteTrackType(wrapper.track, wrapper.streamId)); + removeTrack(sender: TrackSender): void { + if (!(sender instanceof DOMTrackSender)) { + throw new Error("Not a DOMTrackSender"); } + this.peerConnection.removeTrack((sender as DOMTrackSender).transceiver.sender); + // TODO: update localStreams } createDataChannel(options: RTCDataChannelInit): any { @@ -170,6 +307,9 @@ class DOMPeerConnection implements PeerConnection { dispose(): void { this.deregisterHandler(); + for (const r of this.remoteStreams) { + r.stream.dispose(); + } } private handleLocalIceCandidate(event: RTCPeerConnectionIceEvent) { @@ -187,67 +327,28 @@ class DOMPeerConnection implements PeerConnection { } } + onRemoteStreamEmpty = (stream: RemoteStreamWrapper): void => { + const idx = this.remoteStreams.findIndex(r => r.stream === stream); + if (idx !== -1) { + this.remoteStreams.splice(idx, 1); + this.handler.onRemoteStreamRemoved(stream); + } + } + private handleRemoteTrack(evt: RTCTrackEvent) { - // TODO: unit test this code somehow - // the tracks on the new stream (with their stream) - const updatedTracks = evt.streams.flatMap(stream => stream.getTracks().map(track => {return {stream, track};})); - // of the tracks we already know about, filter the ones that aren't in the new stream - const withoutRemovedTracks = this._remoteTracks.filter(t => updatedTracks.some(ut => t.track.id === ut.track.id)); - // of the new tracks, filter the ones that we didn't already knew about - const addedTracks = updatedTracks.filter(ut => !this._remoteTracks.some(t => t.track.id === ut.track.id)); - // wrap them - const wrappedAddedTracks = addedTracks.map(t => wrapTrack(t.track, t.stream, this.getRemoteTrackType(t.track, t.stream.id))); - // and concat the tracks for other streams with the added tracks - this._remoteTracks = withoutRemovedTracks.concat(...wrappedAddedTracks); - this.handler.onRemoteTracksChanged(this.remoteTracks); - } - - private getRemoteTrackType(track: MediaStreamTrack, streamId: string): TrackType { - if (track.kind === "video") { - const purpose = this.handler.getPurposeForStreamId(streamId); - return purpose === SDPStreamMetadataPurpose.Usermedia ? TrackType.Camera : TrackType.ScreenShare; - } else { - return TrackType.Microphone; + if (evt.streams.length !== 0) { + throw new Error("track in multiple streams is not supported"); } - } - - /** - * This method removes all video/rtx codecs from screensharing video - * transceivers. This is necessary since they can cause problems. Without - * this the following steps should produce an error: - * Chromium calls Firefox - * Firefox answers - * Firefox starts screen-sharing - * Chromium starts screen-sharing - * Call crashes for Chromium with: - * [96685:23:0518/162603.933321:ERROR:webrtc_video_engine.cc(3296)] RTX codec (PT=97) mapped to PT=96 which is not in the codec list. - * [96685:23:0518/162603.933377:ERROR:webrtc_video_engine.cc(1171)] GetChangedRecvParameters called without any video codecs. - * [96685:23:0518/162603.933430:ERROR:sdp_offer_answer.cc(4302)] Failed to set local video description recv parameters for m-section with mid='2'. (INVALID_PARAMETER) - */ - private getRidOfRTXCodecs(screensharingTrack: TrackWrapper): void { - // RTCRtpReceiver.getCapabilities and RTCRtpSender.getCapabilities don't seem to be supported on FF - if (!RTCRtpReceiver.getCapabilities || !RTCRtpSender.getCapabilities) return; - - const recvCodecs = RTCRtpReceiver.getCapabilities("video")?.codecs ?? []; - const sendCodecs = RTCRtpSender.getCapabilities("video")?.codecs ?? []; - const codecs = [...sendCodecs, ...recvCodecs]; - - for (const codec of codecs) { - if (codec.mimeType === "video/rtx") { - const rtxCodecIndex = codecs.indexOf(codec); - codecs.splice(rtxCodecIndex, 1); - } + const stream = evt.streams[0]; + const transceivers = this.peerConnection.getTransceivers(); + let streamReceiver: DOMStreamReceiver | undefined = this.remoteStreams.find(r => r.stream.id === stream.id); + if (!streamReceiver) { + streamReceiver = new DOMStreamReceiver(new RemoteStreamWrapper(stream, this.onRemoteStreamEmpty)); + this.remoteStreams.push(streamReceiver); } - - for (const trans of this.peerConnection.getTransceivers()) { - if (trans.sender.track === screensharingTrack.track && - ( - trans.sender.track?.kind === "video" || - trans.receiver.track?.kind === "video" - ) - ) { - trans.setCodecPreferences(codecs); - } + const trackReceiver = streamReceiver.update(evt); + if (trackReceiver) { + this.handler.onRemoteTracksAdded(trackReceiver); } } } From bc118b5c0b87c00e0b96b8219c33a86733769353 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 13 Apr 2022 18:34:01 +0200 Subject: [PATCH 067/435] WIP --- src/domain/session/room/CallViewModel.ts | 10 +- src/domain/session/room/RoomViewModel.js | 5 +- .../session/room/timeline/tiles/CallTile.js | 5 +- src/matrix/calls/CallHandler.ts | 2 +- src/matrix/calls/LocalMedia.ts | 18 -- src/matrix/calls/PeerCall.ts | 158 +++++++++++------- src/matrix/calls/group/GroupCall.ts | 2 +- src/matrix/calls/group/Member.ts | 7 +- src/platform/types/WebRTC.ts | 8 +- src/platform/web/dom/MediaDevices.ts | 8 +- src/platform/web/dom/WebRTC.ts | 35 ++-- src/platform/web/ui/session/room/CallView.ts | 16 +- yarn.lock | 8 +- 13 files changed, 154 insertions(+), 128 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 72353e4e..bc7b54ff 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -18,7 +18,7 @@ import {ViewModel, Options as BaseOptions} from "../../ViewModel"; import type {GroupCall} from "../../../matrix/calls/group/GroupCall"; import type {Member} from "../../../matrix/calls/group/Member"; import type {BaseObservableList} from "../../../observable/list/BaseObservableList"; -import type {Track} from "../../../platform/types/MediaDevices"; +import type {Stream} from "../../../platform/types/MediaDevices"; type Options = BaseOptions & {call: GroupCall}; @@ -46,8 +46,8 @@ export class CallViewModel extends ViewModel { return this.call.id; } - get localTracks(): Track[] { - return this.call.localMedia?.tracks ?? []; + get localStream(): Stream | undefined { + return this.call.localMedia?.userMedia; } leave() { @@ -60,8 +60,8 @@ export class CallViewModel extends ViewModel { type MemberOptions = BaseOptions & {member: Member}; export class CallMemberViewModel extends ViewModel { - get tracks(): Track[] { - return this.member.remoteTracks; + get stream(): Stream | undefined { + return this.member.remoteMedia?.userMedia; } private get member(): Member { diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index e7d7dc9a..868ca189 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -365,8 +365,9 @@ export class RoomViewModel extends ViewModel { async startCall() { try { const session = this.getOption("session"); - const mediaTracks = await this.platform.mediaDevices.getMediaTracks(false, true); - const localMedia = new LocalMedia().withTracks(mediaTracks); + const stream = await this.platform.mediaDevices.getMediaTracks(false, true); + const localMedia = new LocalMedia().withUserMedia(stream); + await this._call.join(localMedia); // this will set the callViewModel above as a call will be added to callHandler.calls const call = await session.callHandler.createCall(this._room.id, localMedia, "A call " + Math.round(this.platform.random() * 100)); await call.join(localMedia); diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index 2d2d9dab..0bc12698 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -74,9 +74,8 @@ export class CallTile extends SimpleTile { async join() { if (this.canJoin) { - const mediaTracks = await this.platform.mediaDevices.getMediaTracks(false, true); - // const screenShareTrack = await this.platform.mediaDevices.getScreenShareTrack(); - const localMedia = new LocalMedia().withTracks(mediaTracks); + const stream = await this.platform.mediaDevices.getMediaTracks(false, true); + const localMedia = new LocalMedia().withUserMedia(stream); await this._call.join(localMedia); } } diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 31829396..5208b72e 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -16,7 +16,7 @@ limitations under the License. import {ObservableMap} from "../../observable/map/ObservableMap"; import {WebRTC, PeerConnection, PeerConnectionHandler} from "../../platform/types/WebRTC"; -import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; +import {MediaDevices, Track, AudioTrack} from "../../platform/types/MediaDevices"; import {handlesEventType} from "./PeerCall"; import {EventType, CallIntent} from "./callEventTypes"; import {GroupCall} from "./group/GroupCall"; diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 6eb7a225..6622f641 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -37,24 +37,6 @@ export class LocalMedia { return new LocalMedia(this.userMedia, this.screenShare, options); } - getSDPMetadata(): SDPStreamMetadata { - const metadata = {}; - const userMediaTrack = this.microphoneTrack ?? this.cameraTrack; - if (userMediaTrack) { - metadata[userMediaTrack.streamId] = { - purpose: SDPStreamMetadataPurpose.Usermedia, - audio_muted: this.microphoneTrack?.muted ?? true, - video_muted: this.cameraTrack?.muted ?? true, - }; - } - if (this.screenShareTrack) { - metadata[this.screenShareTrack.streamId] = { - purpose: SDPStreamMetadataPurpose.Screenshare - }; - } - return metadata; - } - clone() { return new LocalMedia(this.userMedia?.clone(), this.screenShare?.clone(), this.dataChannelOptions); } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 644ed160..78ffc818 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -23,8 +23,8 @@ import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; import type {TimeoutCreator, Timeout} from "../../platform/types/types"; -import {WebRTC, PeerConnection, PeerConnectionHandler} from "../../platform/types/WebRTC"; -import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; +import {WebRTC, PeerConnection, PeerConnectionHandler, TrackSender, TrackReceiver} from "../../platform/types/WebRTC"; +import {MediaDevices, Track, AudioTrack, Stream} from "../../platform/types/MediaDevices"; import type {LocalMedia} from "./LocalMedia"; import { @@ -52,6 +52,10 @@ export type Options = { sendSignallingMessage: (message: SignallingMessage, log: ILogItem) => Promise; }; +export class RemoteMedia { + constructor(public userMedia?: Stream | undefined, public screenShare?: Stream | undefined) {} +} + // when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already // do for sharing keys will be best as that already deals with room tracking. /** @@ -89,6 +93,7 @@ export class PeerCall implements IDisposable { private _dataChannel?: any; private _hangupReason?: CallErrorCode; + private _remoteMedia: RemoteMedia; constructor( private callId: string, @@ -96,6 +101,7 @@ export class PeerCall implements IDisposable { private readonly logItem: ILogItem, ) { const outer = this; + this._remoteMedia = new RemoteMedia(); this.peerConnection = options.webRTC.createPeerConnection({ onIceConnectionStateChange(state: RTCIceConnectionState) { outer.logItem.wrap({l: "onIceConnectionStateChange", status: state}, log => { @@ -112,9 +118,14 @@ export class PeerCall implements IDisposable { outer.handleIceGatheringState(state, log); }); }, - onRemoteTracksChanged(tracks: Track[]) { - outer.logItem.wrap({l: "onRemoteTracksChanged", length: tracks.length}, log => { - outer.options.emitUpdate(outer, undefined); + onRemoteStreamRemoved(stream: Stream) { + outer.logItem.wrap("onRemoteStreamRemoved", log => { + outer.updateRemoteMedia(log); + }); + }, + onRemoteTracksAdded(trackReceiver: TrackReceiver) { + outer.logItem.wrap("onRemoteTracksAdded", log => { + outer.updateRemoteMedia(log); }); }, onRemoteDataChannel(dataChannel: any | undefined) { @@ -130,9 +141,6 @@ export class PeerCall implements IDisposable { }); }; outer.responsePromiseChain = outer.responsePromiseChain?.then(promiseCreator) ?? promiseCreator(); - }, - getPurposeForStreamId(streamId: string): SDPStreamMetadataPurpose { - return outer.remoteSDPStreamMetadata?.[streamId]?.purpose ?? SDPStreamMetadataPurpose.Usermedia; } }, this.options.forceTURN, this.options.turnServers, 0); } @@ -143,8 +151,9 @@ export class PeerCall implements IDisposable { get hangupReason(): CallErrorCode | undefined { return this._hangupReason; } - get remoteTracks(): Track[] { - return this.peerConnection.remoteTracks; + // we should keep an object with streams by purpose ... e.g. RemoteMedia? + get remoteMedia(): Readonly { + return this._remoteMedia; } call(localMedia: LocalMedia): Promise { @@ -152,13 +161,10 @@ export class PeerCall implements IDisposable { if (this._state !== CallState.Fledgling) { return; } - this.localMedia = localMedia; this.direction = CallDirection.Outbound; this.setState(CallState.CreateOffer, log); - for (const t of this.localMedia.tracks) { - this.peerConnection.addTrack(t); - } - if (this.localMedia.dataChannelOptions) { + this.setMedia(localMedia); + if (this.localMedia?.dataChannelOptions) { this._dataChannel = this.peerConnection.createDataChannel(this.localMedia.dataChannelOptions); } // after adding the local tracks, and wait for handleNegotiation to be called, @@ -172,11 +178,8 @@ export class PeerCall implements IDisposable { if (this._state !== CallState.Ringing) { return; } - this.localMedia = localMedia; this.setState(CallState.CreateAnswer, log); - for (const t of this.localMedia.tracks) { - this.peerConnection.addTrack(t); - } + this.setMedia(localMedia, log); let myAnswer: RTCSessionDescriptionInit; try { myAnswer = await this.peerConnection.createAnswer(); @@ -205,27 +208,40 @@ export class PeerCall implements IDisposable { }); } - setMedia(localMediaPromise: Promise): Promise { - return this.logItem.wrap("setMedia", async log => { + setMedia(localMedia: LocalMedia, logItem: ILogItem = this.logItem): Promise { + return logItem.wrap("setMedia", async log => { const oldMedia = this.localMedia; - this.localMedia = await localMediaPromise; + this.localMedia = localMedia; + const applyStream = (oldStream: Stream | undefined, stream: Stream | undefined, logLabel: string) => { + const streamSender = oldMedia ? this.peerConnection.localStreams.get(oldStream!.id) : undefined; - const applyTrack = (selectTrack: (media: LocalMedia | undefined) => Track | undefined) => { - const oldTrack = selectTrack(oldMedia); - const newTrack = selectTrack(this.localMedia); - if (oldTrack && newTrack) { - this.peerConnection.replaceTrack(oldTrack, newTrack); - } else if (oldTrack) { - this.peerConnection.removeTrack(oldTrack); - } else if (newTrack) { - this.peerConnection.addTrack(newTrack); + const applyTrack = (oldTrack: Track | undefined, sender: TrackSender | undefined, track: Track | undefined) => { + if (track) { + if (oldTrack && sender) { + log.wrap(`replacing ${logLabel} ${track.kind} track`, log => { + sender.replaceTrack(track); + }); + } else { + log.wrap(`adding ${logLabel} ${track.kind} track`, log => { + this.peerConnection.addTrack(track); + }); + } + } else { + if (sender) { + log.wrap(`replacing ${logLabel} ${sender.track.kind} track`, log => { + this.peerConnection.removeTrack(sender); + }); + } + } } - }; - // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called - applyTrack(m => m?.microphoneTrack); - applyTrack(m => m?.cameraTrack); - applyTrack(m => m?.screenShareTrack); + applyTrack(oldStream?.audioTrack, streamSender?.audioSender, stream?.audioTrack); + applyTrack(oldStream?.videoTrack, streamSender?.videoSender, stream?.videoTrack); + } + + applyStream(oldMedia?.userMedia, localMedia?.userMedia, "userMedia"); + applyStream(oldMedia?.screenShare, localMedia?.screenShare, "screenShare"); + // TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method }); } @@ -321,7 +337,7 @@ export class PeerCall implements IDisposable { const content = { call_id: this.callId, offer, - [SDPStreamMetadataKey]: this.localMedia!.getSDPMetadata(), + [SDPStreamMetadataKey]: this.getSDPMetadata(), version: 1, seq: this.seq++, lifetime: CALL_TIMEOUT_MS @@ -408,7 +424,7 @@ export class PeerCall implements IDisposable { const sdpStreamMetadata = content[SDPStreamMetadataKey]; if (sdpStreamMetadata) { - this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata, log); } else { log.log(`Call did not get any SDPStreamMetadata! Can not send/receive multiple streams`); } @@ -470,7 +486,7 @@ export class PeerCall implements IDisposable { const sdpStreamMetadata = content[SDPStreamMetadataKey]; if (sdpStreamMetadata) { - this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata, log); } else { log.log(`Did not get any SDPStreamMetadata! Can not send/receive multiple streams`); } @@ -596,7 +612,7 @@ export class PeerCall implements IDisposable { // type: EventType.CallNegotiate, // content: { // description: this.peerConnection.localDescription!, - // [SDPStreamMetadataKey]: this.localMedia.getSDPMetadata(), + // [SDPStreamMetadataKey]: this.getSDPMetadata(), // } // }); // } @@ -615,7 +631,7 @@ export class PeerCall implements IDisposable { sdp: localDescription.sdp, type: localDescription.type, }, - [SDPStreamMetadataKey]: this.localMedia!.getSDPMetadata(), + [SDPStreamMetadataKey]: this.getSDPMetadata(), }; // We have just taken the local description from the peerConn which will @@ -699,18 +715,11 @@ export class PeerCall implements IDisposable { }); } - private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { + private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata, log: ILogItem): void { + // this will accumulate all updates into one object, so we still have the old stream info when we change stream id this.remoteSDPStreamMetadata = recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); - for (const track of this.peerConnection.remoteTracks) { - const streamMetaData = this.remoteSDPStreamMetadata?.[track.streamId]; - if (streamMetaData) { - if (track.type === TrackType.Microphone) { - track.setMuted(streamMetaData.audio_muted); - } else { // Camera or ScreenShare - track.setMuted(streamMetaData.video_muted); - } - } - } + this.updateRemoteMedia(log); + // TODO: apply muting } private async addBufferedIceCandidates(log: ILogItem): Promise { @@ -755,8 +764,6 @@ export class PeerCall implements IDisposable { this.iceDisconnectedTimeout?.abort(); this.iceDisconnectedTimeout = undefined; this.setState(CallState.Connected, log); - const transceivers = this.peerConnection.peerConnection.getTransceivers(); - console.log(transceivers); } else if (state == 'failed') { this.iceDisconnectedTimeout?.abort(); this.iceDisconnectedTimeout = undefined; @@ -807,14 +814,53 @@ export class PeerCall implements IDisposable { this.hangupParty = hangupParty; this._hangupReason = hangupReason; this.setState(CallState.Ended, log); - //this.localMedia?.dispose(); - //this.localMedia = undefined; + this.localMedia?.dispose(); + this.localMedia = undefined; if (this.peerConnection && this.peerConnection.signalingState !== 'closed') { this.peerConnection.close(); } } + private getSDPMetadata(): SDPStreamMetadata { + const metadata = {}; + if (this.localMedia?.userMedia) { + const streamId = this.localMedia.userMedia.id; + const streamSender = this.peerConnection.localStreams.get(streamId); + metadata[streamId] = { + purpose: SDPStreamMetadataPurpose.Usermedia, + audio_muted: !(streamSender?.audioSender?.enabled), + video_muted: !(streamSender?.videoSender?.enabled), + }; + console.log("video_muted", streamSender?.videoSender?.enabled, streamSender?.videoSender?.transceiver?.direction, streamSender?.videoSender?.transceiver?.currentDirection, JSON.stringify(metadata)); + } + if (this.localMedia?.screenShare) { + const streamId = this.localMedia.screenShare.id; + metadata[streamId] = { + purpose: SDPStreamMetadataPurpose.Screenshare + }; + } + return metadata; + } + + private updateRemoteMedia(log: ILogItem) { + this._remoteMedia.userMedia = undefined; + this._remoteMedia.screenShare = undefined; + if (this.remoteSDPStreamMetadata) { + for (const [streamId, streamReceiver] of this.peerConnection.remoteStreams.entries()) { + const metaData = this.remoteSDPStreamMetadata[streamId]; + if (metaData) { + if (metaData.purpose === SDPStreamMetadataPurpose.Usermedia) { + this._remoteMedia.userMedia = streamReceiver.stream; + } else if (metaData.purpose === SDPStreamMetadataPurpose.Screenshare) { + this._remoteMedia.screenShare = streamReceiver.stream; + } + } + } + } + this.options.emitUpdate(this, undefined); + } + private async delay(timeoutMs: number): Promise { // Allow a short time for initial candidates to be gathered const timeout = this.disposables.track(this.options.createTimeout(timeoutMs)); diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 886cb53a..47d87e5e 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -165,7 +165,7 @@ export class GroupCall extends EventEmitter<{change: never}> { this._state = GroupCallState.Creating; this.emitChange(); this.callContent = Object.assign({ - "m.type": localMedia.cameraTrack ? "m.video" : "m.voice", + "m.type": localMedia.userMedia?.videoTrack ? "m.video" : "m.voice", }, this.callContent); const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCall, this.id, this.callContent!, {log}); await request.response(); diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 0ac7df99..110aba6d 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -19,10 +19,9 @@ import {makeTxnId, makeId} from "../../common"; import {EventType, CallErrorCode} from "../callEventTypes"; import {formatToDeviceMessagesPayload} from "../../common"; -import type {Options as PeerCallOptions} from "../PeerCall"; +import type {Options as PeerCallOptions, RemoteMedia} from "../PeerCall"; import type {LocalMedia} from "../LocalMedia"; import type {HomeServerApi} from "../../net/HomeServerApi"; -import type {Track} from "../../../platform/types/MediaDevices"; import type {MCallBase, MGroupCallBase, SignallingMessage, CallDeviceMembership} from "../callEventTypes"; import type {GroupCall} from "./GroupCall"; import type {RoomMember} from "../../room/members/RoomMember"; @@ -60,8 +59,8 @@ export class Member { private readonly logItem: ILogItem, ) {} - get remoteTracks(): Track[] { - return this.peerCall?.remoteTracks ?? []; + get remoteMedia(): RemoteMedia | undefined { + return this.peerCall?.remoteMedia; } get isConnected(): boolean { diff --git a/src/platform/types/WebRTC.ts b/src/platform/types/WebRTC.ts index a0512163..edb26c0a 100644 --- a/src/platform/types/WebRTC.ts +++ b/src/platform/types/WebRTC.ts @@ -18,7 +18,7 @@ import {Track, Stream} from "./MediaDevices"; import {SDPStreamMetadataPurpose} from "../../matrix/calls/callEventTypes"; export interface WebRTC { - createPeerConnection(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize): PeerConnection; + createPeerConnection(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize: number): PeerConnection; } export interface StreamSender { @@ -41,7 +41,7 @@ export interface TrackReceiver { export interface TrackSender extends TrackReceiver { /** replaces the track if possible without renegotiation. Can throw. */ - replaceTrack(track: Track): Promise; + replaceTrack(track: Track | undefined): Promise; /** make any needed adjustments to the sender or transceiver settings * depending on the purpose, after adding the track to the connection */ prepareForPurpose(purpose: SDPStreamMetadataPurpose): void; @@ -61,8 +61,8 @@ export interface PeerConnection { get iceGatheringState(): RTCIceGatheringState; get signalingState(): RTCSignalingState; get localDescription(): RTCSessionDescription | undefined; - get localStreams(): ReadonlyArray; - get remoteStreams(): ReadonlyArray; + get localStreams(): ReadonlyMap; + get remoteStreams(): ReadonlyMap; createOffer(): Promise; createAnswer(): Promise; setLocalDescription(description?: RTCSessionDescriptionInit): Promise; diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index 52ca132b..3723162a 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -72,8 +72,8 @@ export class MediaDevicesWrapper implements IMediaDevices { export class StreamWrapper implements Stream { - public audioTrack: AudioTrackWrapper | undefined; - public videoTrack: TrackWrapper | undefined; + public audioTrack: AudioTrackWrapper | undefined = undefined; + public videoTrack: TrackWrapper | undefined = undefined; constructor(public readonly stream: MediaStream) { for (const track of stream.getTracks()) { @@ -91,13 +91,13 @@ export class StreamWrapper implements Stream { if (track.kind === "video") { if (!this.videoTrack || track.id !== this.videoTrack.track.id) { this.videoTrack = new TrackWrapper(track, this.stream); - return this.videoTrack; } + return this.videoTrack; } else if (track.kind === "audio") { if (!this.audioTrack || track.id !== this.audioTrack.track.id) { this.audioTrack = new AudioTrackWrapper(track, this.stream); - return this.audioTrack; } + return this.audioTrack; } } } diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 6760182d..672c14d4 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -62,10 +62,10 @@ export class DOMStreamSender implements StreamSender { if (transceiver && sender.track) { const trackWrapper = this.stream.update(sender.track); if (trackWrapper) { - if (trackWrapper.kind === TrackKind.Video) { + if (trackWrapper.kind === TrackKind.Video && (!this.videoSender || this.videoSender.track.id !== trackWrapper.id)) { this.videoSender = new DOMTrackSender(trackWrapper, transceiver); return this.videoSender; - } else { + } else if (trackWrapper.kind === TrackKind.Audio && (!this.audioSender || this.audioSender.track.id !== trackWrapper.id)) { this.audioSender = new DOMTrackSender(trackWrapper, transceiver); return this.audioSender; } @@ -105,20 +105,20 @@ export class DOMTrackSenderOrReceiver implements TrackReceiver { ) {} get enabled(): boolean { - return this.transceiver.currentDirection === "sendrecv" || - this.transceiver.currentDirection === this.exclusiveValue; + return this.transceiver.direction === "sendrecv" || + this.transceiver.direction === this.exclusiveValue; } enable(enabled: boolean) { if (enabled !== this.enabled) { if (enabled) { - if (this.transceiver.currentDirection === "inactive") { + if (this.transceiver.direction === "inactive") { this.transceiver.direction = this.exclusiveValue; } else { this.transceiver.direction = "sendrecv"; } } else { - if (this.transceiver.currentDirection === "sendrecv") { + if (this.transceiver.direction === "sendrecv") { this.transceiver.direction = this.excludedValue; } else { this.transceiver.direction = "inactive"; @@ -145,7 +145,7 @@ export class DOMTrackSender extends DOMTrackSenderOrReceiver { super(track, transceiver, "sendonly", "recvonly"); } /** replaces the track if possible without renegotiation. Can throw. */ - replaceTrack(track: Track): Promise { + replaceTrack(track: Track | undefined): Promise { return this.transceiver.sender.replaceTrack(track ? (track as TrackWrapper).track : null); } @@ -192,8 +192,8 @@ export class DOMTrackSender extends DOMTrackSenderOrReceiver { class DOMPeerConnection implements PeerConnection { private readonly peerConnection: RTCPeerConnection; private readonly handler: PeerConnectionHandler; - public readonly localStreams: DOMStreamSender[]; - public readonly remoteStreams: DOMStreamReceiver[]; + public readonly localStreams: Map = new Map(); + public readonly remoteStreams: Map = new Map(); constructor(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize) { this.handler = handler; @@ -238,10 +238,11 @@ class DOMPeerConnection implements PeerConnection { throw new Error("Not a TrackWrapper"); } const sender = this.peerConnection.addTrack(track.track, track.stream); - let streamSender: DOMStreamSender | undefined = this.localStreams.find(s => s.stream.id === track.stream.id); + let streamSender = this.localStreams.get(track.stream.id); if (!streamSender) { + // TODO: reuse existing stream wrapper here? streamSender = new DOMStreamSender(new StreamWrapper(track.stream)); - this.localStreams.push(streamSender); + this.localStreams.set(track.stream.id, streamSender); } const trackSender = streamSender.update(this.peerConnection.getTransceivers(), sender); return trackSender; @@ -307,7 +308,7 @@ class DOMPeerConnection implements PeerConnection { dispose(): void { this.deregisterHandler(); - for (const r of this.remoteStreams) { + for (const r of this.remoteStreams.values()) { r.stream.dispose(); } } @@ -328,23 +329,21 @@ class DOMPeerConnection implements PeerConnection { } onRemoteStreamEmpty = (stream: RemoteStreamWrapper): void => { - const idx = this.remoteStreams.findIndex(r => r.stream === stream); - if (idx !== -1) { - this.remoteStreams.splice(idx, 1); + if (this.remoteStreams.delete(stream.id)) { this.handler.onRemoteStreamRemoved(stream); } } private handleRemoteTrack(evt: RTCTrackEvent) { - if (evt.streams.length !== 0) { + if (evt.streams.length !== 1) { throw new Error("track in multiple streams is not supported"); } const stream = evt.streams[0]; const transceivers = this.peerConnection.getTransceivers(); - let streamReceiver: DOMStreamReceiver | undefined = this.remoteStreams.find(r => r.stream.id === stream.id); + let streamReceiver: DOMStreamReceiver | undefined = this.remoteStreams.get(stream.id); if (!streamReceiver) { streamReceiver = new DOMStreamReceiver(new RemoteStreamWrapper(stream, this.onRemoteStreamEmpty)); - this.remoteStreams.push(streamReceiver); + this.remoteStreams.set(stream.id, streamReceiver); } const trackReceiver = streamReceiver.update(evt); if (trackReceiver) { diff --git a/src/platform/web/ui/session/room/CallView.ts b/src/platform/web/ui/session/room/CallView.ts index f8386116..115e6035 100644 --- a/src/platform/web/ui/session/room/CallView.ts +++ b/src/platform/web/ui/session/room/CallView.ts @@ -16,14 +16,14 @@ limitations under the License. import {TemplateView, TemplateBuilder} from "../../general/TemplateView"; import {ListView} from "../../general/ListView"; -import {Track, TrackType} from "../../../../types/MediaDevices"; -import type {TrackWrapper} from "../../../dom/MediaDevices"; +import {Stream} from "../../../../types/MediaDevices"; +import type {StreamWrapper} from "../../../dom/MediaDevices"; import type {CallViewModel, CallMemberViewModel} from "../../../../../domain/session/room/CallViewModel"; -function bindVideoTracks(t: TemplateBuilder, video: HTMLVideoElement, propSelector: (vm: T) => Track[]) { - t.mapSideEffect(propSelector, tracks => { - if (tracks.length) { - video.srcObject = (tracks[0] as TrackWrapper).stream; +function bindVideoTracks(t: TemplateBuilder, video: HTMLVideoElement, propSelector: (vm: T) => Stream | undefined) { + t.mapSideEffect(propSelector, stream => { + if (stream) { + video.srcObject = (stream as StreamWrapper).stream; } }); return video; @@ -33,7 +33,7 @@ export class CallView extends TemplateView { render(t: TemplateBuilder, vm: CallViewModel): HTMLElement { return t.div({class: "CallView"}, [ t.p(vm => `Call ${vm.name} (${vm.id})`), - t.div({class: "CallView_me"}, bindVideoTracks(t, t.video({autoplay: true, width: 240}), vm => vm.localTracks)), + t.div({class: "CallView_me"}, bindVideoTracks(t, t.video({autoplay: true, width: 240}), vm => vm.localStream)), t.view(new ListView({list: vm.memberViewModels}, vm => new MemberView(vm))), t.div({class: "buttons"}, [ t.button({onClick: () => vm.leave()}, "Leave") @@ -44,6 +44,6 @@ export class CallView extends TemplateView { class MemberView extends TemplateView { render(t: TemplateBuilder, vm: CallMemberViewModel) { - return bindVideoTracks(t, t.video({autoplay: true, width: 360}), vm => vm.tracks); + return bindVideoTracks(t, t.video({autoplay: true, width: 360}), vm => vm.stream); } } diff --git a/yarn.lock b/yarn.lock index 0eb74b87..6405fe4d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1485,10 +1485,10 @@ type-fest@^0.20.2: resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.20.2.tgz#1bf207f4b28f91583666cb5fbd327887301cd5f4" integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ== -typescript@^4.3.5: - version "4.6.2" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.6.2.tgz#fe12d2727b708f4eef40f51598b3398baa9611d4" - integrity sha512-HM/hFigTBHZhLXshn9sN37H085+hQGeJHJ/X7LpBWLID/fbc2acUMfU+lGD98X81sKP+pFa9f0DZmCwB9GnbAg== +typescript@^4.4: + version "4.6.3" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.6.3.tgz#eefeafa6afdd31d725584c67a0eaba80f6fc6c6c" + integrity sha512-yNIatDa5iaofVozS/uQJEl3JRWLKKGJKh6Yaiv0GLGSuhpFJe7P3SbHZ8/yjAHRQwKRoA6YZqlfjXWmVzoVSMw== typeson-registry@^1.0.0-alpha.20: version "1.0.0-alpha.39" From 2d00d10161b52e83e56b9a331c0e31adede3d1ec Mon Sep 17 00:00:00 2001 From: Robert Long Date: Wed, 13 Apr 2022 13:08:33 -0700 Subject: [PATCH 068/435] Export LocalMedia --- src/lib.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib.ts b/src/lib.ts index e2da5e16..a49eacbc 100644 --- a/src/lib.ts +++ b/src/lib.ts @@ -71,6 +71,7 @@ export {AvatarView} from "./platform/web/ui/AvatarView.js"; export {RoomType} from "./matrix/room/common"; export {EventEmitter} from "./utils/EventEmitter"; export {Disposables} from "./utils/Disposables"; +export {LocalMedia} from "./matrix/calls/LocalMedia"; // these should eventually be moved to another library export { ObservableArray, From 55097e41542ad9833ff6058b0aa73b5eb106a2ed Mon Sep 17 00:00:00 2001 From: Robert Long Date: Wed, 13 Apr 2022 13:08:47 -0700 Subject: [PATCH 069/435] Add intent to CallHandler --- src/matrix/calls/CallHandler.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 31829396..77492e62 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -107,11 +107,11 @@ export class CallHandler { }); } - async createCall(roomId: string, localMedia: LocalMedia, name: string): Promise { + async createCall(roomId: string, localMedia: LocalMedia, name: string, intent: CallIntent = CallIntent.Ring): Promise { const logItem = this.options.logger.child({l: "call", incoming: false}); const call = new GroupCall(makeId("conf-"), true, { "m.name": name, - "m.intent": CallIntent.Ring + "m.intent": intent }, roomId, this.groupCallOptions, logItem); this._calls.set(call.id, call); From ff856d843ce7dedd2677e4def1c8347ac1c36568 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 14 Apr 2022 13:44:11 +0200 Subject: [PATCH 070/435] ensure all member streams are cloned so we can stop them without affecting the main one also, only stop them when disconnecting from the member, rather then when the peer call ends, as we might want to retry connecting to the peer with the same stream. --- src/matrix/calls/LocalMedia.ts | 4 ++-- src/matrix/calls/PeerCall.ts | 1 - src/matrix/calls/group/GroupCall.ts | 7 ++++--- src/matrix/calls/group/Member.ts | 1 + 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 6622f641..6564adac 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -37,10 +37,10 @@ export class LocalMedia { return new LocalMedia(this.userMedia, this.screenShare, options); } - clone() { + clone(): LocalMedia { return new LocalMedia(this.userMedia?.clone(), this.screenShare?.clone(), this.dataChannelOptions); } - + dispose() { this.userMedia?.audioTrack?.stop(); this.userMedia?.videoTrack?.stop(); diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 78ffc818..5b6e2ca8 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -814,7 +814,6 @@ export class PeerCall implements IDisposable { this.hangupParty = hangupParty; this._hangupReason = hangupReason; this.setState(CallState.Ended, log); - this.localMedia?.dispose(); this.localMedia = undefined; if (this.peerConnection && this.peerConnection.signalingState !== 'closed') { diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 47d87e5e..d077f27c 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -118,7 +118,7 @@ export class GroupCall extends EventEmitter<{change: never}> { this.emitChange(); // send invite to all members that are < my userId for (const [,member] of this._members) { - member.connect(this._localMedia); + member.connect(this._localMedia!.clone()); } }); } @@ -218,6 +218,7 @@ export class GroupCall extends EventEmitter<{change: never}> { ); this._members.add(memberKey, member); if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { + // Safari can't send a MediaStream to multiple sources, so clone it member.connect(this._localMedia!.clone()); } } @@ -267,11 +268,11 @@ export class GroupCall extends EventEmitter<{change: never}> { private removeOwnDevice(log: ILogItem) { if (this._state === GroupCallState.Joined) { log.set("leave_own", true); - this._localMedia?.dispose(); - this._localMedia = undefined; for (const [,member] of this._members) { member.disconnect(); } + this._localMedia?.dispose(); + this._localMedia = undefined; this._state = GroupCallState.Created; this.emitChange(); } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 110aba6d..1305fcf1 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -104,6 +104,7 @@ export class Member { this.peerCall?.close(undefined, log); this.peerCall?.dispose(); this.peerCall = undefined; + this.localMedia?.dispose(); this.localMedia = undefined; }); } From 021b8cdcdc8a8f41edd0bdce008c6bdc859e28d6 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 14 Apr 2022 13:45:21 +0200 Subject: [PATCH 071/435] send hangup when leaving the call but not when somebody else leaves the call through a member event --- src/matrix/calls/group/GroupCall.ts | 4 ++-- src/matrix/calls/group/Member.ts | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index d077f27c..892ed67c 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -269,7 +269,7 @@ export class GroupCall extends EventEmitter<{change: never}> { if (this._state === GroupCallState.Joined) { log.set("leave_own", true); for (const [,member] of this._members) { - member.disconnect(); + member.disconnect(true); } this._localMedia?.dispose(); this._localMedia = undefined; @@ -286,7 +286,7 @@ export class GroupCall extends EventEmitter<{change: never}> { if (member) { log.set("leave", true); this._members.remove(memberKey); - member.disconnect(); + member.disconnect(false); } this.emitChange(); }); diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 1305fcf1..06693030 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -99,9 +99,13 @@ export class Member { } /** @internal */ - disconnect() { + disconnect(hangup: boolean) { this.logItem.wrap("disconnect", log => { - this.peerCall?.close(undefined, log); + if (hangup) { + this.peerCall?.hangup(CallErrorCode.UserHangup); + } else { + this.peerCall?.close(undefined, log); + } this.peerCall?.dispose(); this.peerCall = undefined; this.localMedia?.dispose(); From 382fba88bdf757887ac978712fc8004788006bfb Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 14 Apr 2022 23:19:44 +0200 Subject: [PATCH 072/435] WIP for muting --- src/domain/session/room/CallViewModel.ts | 6 + src/matrix/calls/LocalMedia.ts | 14 +- src/matrix/calls/PeerCall.ts | 256 +++++++++---------- src/matrix/calls/TODO.md | 2 +- src/matrix/calls/group/GroupCall.ts | 11 + src/matrix/calls/group/Member.ts | 8 + src/platform/types/MediaDevices.ts | 3 + src/platform/web/dom/MediaDevices.ts | 34 ++- src/platform/web/dom/WebRTC.ts | 8 + src/platform/web/index.html | 2 +- src/platform/web/ui/general/TemplateView.ts | 6 +- src/platform/web/ui/session/room/CallView.ts | 23 +- 12 files changed, 210 insertions(+), 163 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index bc7b54ff..05245cd3 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -55,6 +55,12 @@ export class CallViewModel extends ViewModel { this.call.leave(); } } + + async toggleVideo() { + const localMedia = this.call.localMedia!; + const toggledMedia = localMedia.withMuted(localMedia.microphoneMuted, !localMedia.cameraMuted); + await this.call.setMedia(toggledMedia); + } } type MemberOptions = BaseOptions & {member: Member}; diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 6564adac..7dbc9e17 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -21,24 +21,30 @@ import {SDPStreamMetadata} from "./callEventTypes"; export class LocalMedia { constructor( public readonly userMedia?: Stream, + public readonly microphoneMuted: boolean = false, + public readonly cameraMuted: boolean = false, public readonly screenShare?: Stream, public readonly dataChannelOptions?: RTCDataChannelInit, ) {} + withMuted(microphone: boolean, camera: boolean) { + return new LocalMedia(this.userMedia, microphone, camera, this.screenShare, this.dataChannelOptions); + } + withUserMedia(stream: Stream) { - return new LocalMedia(stream, this.screenShare, this.dataChannelOptions); + return new LocalMedia(stream, this.microphoneMuted, this.cameraMuted, this.screenShare, this.dataChannelOptions); } withScreenShare(stream: Stream) { - return new LocalMedia(this.userMedia, stream, this.dataChannelOptions); + return new LocalMedia(this.userMedia, this.microphoneMuted, this.cameraMuted, stream, this.dataChannelOptions); } withDataChannel(options: RTCDataChannelInit): LocalMedia { - return new LocalMedia(this.userMedia, this.screenShare, options); + return new LocalMedia(this.userMedia, this.microphoneMuted, this.cameraMuted, this.screenShare, options); } clone(): LocalMedia { - return new LocalMedia(this.userMedia?.clone(), this.screenShare?.clone(), this.dataChannelOptions); + return new LocalMedia(this.userMedia?.clone(), this.microphoneMuted, this.cameraMuted, this.screenShare?.clone(), this.dataChannelOptions); } dispose() { diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 5b6e2ca8..b450ac3f 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -31,6 +31,7 @@ import { SDPStreamMetadataKey, SDPStreamMetadataPurpose, EventType, + CallErrorCode, } from "./callEventTypes"; import type { MCallBase, @@ -66,6 +67,7 @@ export class PeerCall implements IDisposable { private readonly peerConnection: PeerConnection; private _state = CallState.Fledgling; private direction: CallDirection; + // we don't own localMedia and should hence not call dispose on it from here private localMedia?: LocalMedia; private seq: number = 0; // A queue for candidates waiting to go out. @@ -151,7 +153,6 @@ export class PeerCall implements IDisposable { get hangupReason(): CallErrorCode | undefined { return this._hangupReason; } - // we should keep an object with streams by purpose ... e.g. RemoteMedia? get remoteMedia(): Readonly { return this._remoteMedia; } @@ -163,7 +164,7 @@ export class PeerCall implements IDisposable { } this.direction = CallDirection.Outbound; this.setState(CallState.CreateOffer, log); - this.setMedia(localMedia); + this.updateLocalMedia(localMedia, log); if (this.localMedia?.dataChannelOptions) { this._dataChannel = this.peerConnection.createDataChannel(this.localMedia.dataChannelOptions); } @@ -179,7 +180,7 @@ export class PeerCall implements IDisposable { return; } this.setState(CallState.CreateAnswer, log); - this.setMedia(localMedia, log); + this.updateLocalMedia(localMedia, log); let myAnswer: RTCSessionDescriptionInit; try { myAnswer = await this.peerConnection.createAnswer(); @@ -208,48 +209,32 @@ export class PeerCall implements IDisposable { }); } - setMedia(localMedia: LocalMedia, logItem: ILogItem = this.logItem): Promise { - return logItem.wrap("setMedia", async log => { - const oldMedia = this.localMedia; - this.localMedia = localMedia; - const applyStream = (oldStream: Stream | undefined, stream: Stream | undefined, logLabel: string) => { - const streamSender = oldMedia ? this.peerConnection.localStreams.get(oldStream!.id) : undefined; + setMedia(localMedia: LocalMedia): Promise { + return this.logItem.wrap("setMedia", async log => { + log.set("userMedia_audio", !!localMedia.userMedia?.audioTrack); + log.set("userMedia_audio_muted", localMedia.microphoneMuted); + log.set("userMedia_video", !!localMedia.userMedia?.videoTrack); + log.set("userMedia_video_muted", localMedia.cameraMuted); + log.set("screenShare_video", !!localMedia.screenShare?.videoTrack); + log.set("datachannel", !!localMedia.dataChannelOptions); - const applyTrack = (oldTrack: Track | undefined, sender: TrackSender | undefined, track: Track | undefined) => { - if (track) { - if (oldTrack && sender) { - log.wrap(`replacing ${logLabel} ${track.kind} track`, log => { - sender.replaceTrack(track); - }); - } else { - log.wrap(`adding ${logLabel} ${track.kind} track`, log => { - this.peerConnection.addTrack(track); - }); - } - } else { - if (sender) { - log.wrap(`replacing ${logLabel} ${sender.track.kind} track`, log => { - this.peerConnection.removeTrack(sender); - }); - } - } + const oldMetaData = this.getSDPMetadata(); + const willRenegotiate = await this.updateLocalMedia(localMedia, log); + if (!willRenegotiate) { + const newMetaData = this.getSDPMetadata(); + if (JSON.stringify(oldMetaData) !== JSON.stringify(newMetaData)) { + const content: MCallSDPStreamMetadataChanged = { + call_id: this.callId, + version: 1, + seq: this.seq++, + [SDPStreamMetadataKey]: newMetaData + }; + await this.sendSignallingMessage({type: EventType.SDPStreamMetadataChanged, content}, log); } - - applyTrack(oldStream?.audioTrack, streamSender?.audioSender, stream?.audioTrack); - applyTrack(oldStream?.videoTrack, streamSender?.videoSender, stream?.videoTrack); } - - applyStream(oldMedia?.userMedia, localMedia?.userMedia, "userMedia"); - applyStream(oldMedia?.screenShare, localMedia?.screenShare, "screenShare"); - // TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method }); } - /** group calls would handle reject at the group call level, not at the peer call level */ - async reject() { - - } - hangup(errorCode: CallErrorCode): Promise { return this.logItem.wrap("hangup", log => { return this._hangup(errorCode, log); @@ -280,7 +265,14 @@ export class PeerCall implements IDisposable { case EventType.Candidates: await this.handleRemoteIceCandidates(message.content, partyId, log); break; + case EventType.SDPStreamMetadataChanged: + case EventType.SDPStreamMetadataChangedPrefix: + this.updateRemoteSDPStreamMetadata(message.content[SDPStreamMetadataKey], log); + break; case EventType.Hangup: + // TODO: this is a bit hacky, double check its what we need + this.terminate(CallParty.Remote, message.content.reason ?? CallErrorCode.UserHangup, log); + break; default: log.log(`Unknown event type for call: ${message.type}`); break; @@ -444,12 +436,12 @@ export class PeerCall implements IDisposable { // According to previous comments in this file, firefox at some point did not // add streams until media started arriving on them. Testing latest firefox // (81 at time of writing), this is no longer a problem, so let's do it the correct way. - // if (this.peerConnection.remoteTracks.length === 0) { - // await log.wrap(`Call no remote stream or no tracks after setting remote description!`, async log => { - // return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); - // }); - // return; - // } + if (this.peerConnection.remoteStreams.size === 0) { + await log.wrap(`Call no remote stream or no tracks after setting remote description!`, async log => { + return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); + }); + return; + } this.setState(CallState.Ringing, log); @@ -719,7 +711,7 @@ export class PeerCall implements IDisposable { // this will accumulate all updates into one object, so we still have the old stream info when we change stream id this.remoteSDPStreamMetadata = recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); this.updateRemoteMedia(log); - // TODO: apply muting + } private async addBufferedIceCandidates(log: ILogItem): Promise { @@ -828,10 +820,9 @@ export class PeerCall implements IDisposable { const streamSender = this.peerConnection.localStreams.get(streamId); metadata[streamId] = { purpose: SDPStreamMetadataPurpose.Usermedia, - audio_muted: !(streamSender?.audioSender?.enabled), - video_muted: !(streamSender?.videoSender?.enabled), + audio_muted: !(streamSender?.audioSender?.enabled && streamSender?.audioSender?.track?.enabled), + video_muted: !(streamSender?.videoSender?.enabled && streamSender?.videoSender?.track?.enabled), }; - console.log("video_muted", streamSender?.videoSender?.enabled, streamSender?.videoSender?.transceiver?.direction, streamSender?.videoSender?.transceiver?.currentDirection, JSON.stringify(metadata)); } if (this.localMedia?.screenShare) { const streamId = this.localMedia.screenShare.id; @@ -851,6 +842,8 @@ export class PeerCall implements IDisposable { if (metaData) { if (metaData.purpose === SDPStreamMetadataPurpose.Usermedia) { this._remoteMedia.userMedia = streamReceiver.stream; + streamReceiver.audioReceiver?.enable(!metaData.audio_muted); + streamReceiver.videoReceiver?.enable(!metaData.video_muted); } else if (metaData.purpose === SDPStreamMetadataPurpose.Screenshare) { this._remoteMedia.screenShare = streamReceiver.stream; } @@ -860,6 +853,77 @@ export class PeerCall implements IDisposable { this.options.emitUpdate(this, undefined); } + private updateLocalMedia(localMedia: LocalMedia, logItem: ILogItem): Promise { + return logItem.wrap("updateLocalMedia", async log => { + let willRenegotiate = false; + const oldMedia = this.localMedia; + this.localMedia = localMedia; + const applyStream = async (oldStream: Stream | undefined, stream: Stream | undefined, oldMuteSettings: LocalMedia | undefined, mutedSettings: LocalMedia | undefined, logLabel: string) => { + const streamSender = oldStream ? this.peerConnection.localStreams.get(oldStream.id) : undefined; + + const applyTrack = async (oldTrack: Track | undefined, sender: TrackSender | undefined, track: Track | undefined, wasMuted: boolean | undefined, muted: boolean | undefined) => { + const changed = (!track && oldTrack) || + (track && !oldTrack) || + (track && oldTrack && !track.equals(oldTrack)); + if (changed) { + if (track) { + if (oldTrack && sender && !track.equals(oldTrack)) { + try { + await log.wrap(`replacing ${logLabel} ${track.kind} track`, log => { + return sender.replaceTrack(track); + }); + } catch (err) { + // can't replace the track without renegotiating + log.wrap(`adding and removing ${logLabel} ${track.kind} track`, log => { + this.peerConnection.removeTrack(sender); + this.peerConnection.addTrack(track); + willRenegotiate = true; + }); + } + } else { + log.wrap(`adding ${logLabel} ${track.kind} track`, log => { + this.peerConnection.addTrack(track); + willRenegotiate = true; + }); + } + } else { + if (sender) { + // this will be used for muting, do we really want to trigger renegotiation here? + // we want to disable the sender, but also remove the track as we don't want to keep + // using the webcam if we don't need to + log.wrap(`removing ${logLabel} ${sender.track.kind} track`, log => { + sender.track.enabled = false; + this.peerConnection.removeTrack(sender); + willRenegotiate = true; + }); + } + } + } else if (track) { + console.log({muted, wasMuted, wasCameraMuted: oldMedia?.cameraMuted}); + if (sender && muted !== wasMuted) { + // TODO: why does unmuting not work? wasMuted is false + log.wrap(`${logLabel} ${track.kind} ${muted ? "muting" : "unmuting"}`, log => { + sender.track.enabled = !muted; + sender.enable(!muted); + willRenegotiate = true; + }); + } else { + log.log(`${logLabel} ${track.kind} track hasn't changed`); + } + } + } + + await applyTrack(oldStream?.audioTrack, streamSender?.audioSender, stream?.audioTrack, oldMuteSettings?.microphoneMuted, mutedSettings?.microphoneMuted); + await applyTrack(oldStream?.videoTrack, streamSender?.videoSender, stream?.videoTrack, oldMuteSettings?.cameraMuted, mutedSettings?.cameraMuted); + } + + await applyStream(oldMedia?.userMedia, localMedia?.userMedia, oldMedia, localMedia, "userMedia"); + applyStream(oldMedia?.screenShare, localMedia?.screenShare, undefined, undefined, "screenShare"); + return willRenegotiate; + // TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method + }); + } + private async delay(timeoutMs: number): Promise { // Allow a short time for initial candidates to be gathered const timeout = this.disposables.track(this.options.createTimeout(timeoutMs)); @@ -915,100 +979,11 @@ export enum CallDirection { Outbound = 'outbound', } -export enum CallErrorCode { - /** The user chose to end the call */ - UserHangup = 'user_hangup', - - /** An error code when the local client failed to create an offer. */ - LocalOfferFailed = 'local_offer_failed', - /** - * An error code when there is no local mic/camera to use. This may be because - * the hardware isn't plugged in, or the user has explicitly denied access. - */ - NoUserMedia = 'no_user_media', - - /** - * Error code used when a call event failed to send - * because unknown devices were present in the room - */ - UnknownDevices = 'unknown_devices', - - /** - * Error code used when we fail to send the invite - * for some reason other than there being unknown devices - */ - SendInvite = 'send_invite', - - /** - * An answer could not be created - */ - CreateAnswer = 'create_answer', - - /** - * Error code used when we fail to send the answer - * for some reason other than there being unknown devices - */ - SendAnswer = 'send_answer', - - /** - * The session description from the other side could not be set - */ - SetRemoteDescription = 'set_remote_description', - - /** - * The session description from this side could not be set - */ - SetLocalDescription = 'set_local_description', - - /** - * A different device answered the call - */ - AnsweredElsewhere = 'answered_elsewhere', - - /** - * No media connection could be established to the other party - */ - IceFailed = 'ice_failed', - - /** - * The invite timed out whilst waiting for an answer - */ - InviteTimeout = 'invite_timeout', - - /** - * The call was replaced by another call - */ - Replaced = 'replaced', - - /** - * Signalling for the call could not be sent (other than the initial invite) - */ - SignallingFailed = 'signalling_timeout', - - /** - * The remote party is busy - */ - UserBusy = 'user_busy', - - /** - * We transferred the call off to somewhere else - */ - Transfered = 'transferred', - - /** - * A call from the same user was found with a new session id - */ - NewSession = 'new_session', -} - /** * The version field that we set in m.call.* events */ const VOIP_PROTO_VERSION = 1; -/** The fallback ICE server to use for STUN or TURN protocols. */ -const FALLBACK_ICE_SERVER = 'stun:turn.matrix.org'; - /** The length of time a call can be ringing for. */ const CALL_TIMEOUT_MS = 60000; @@ -1029,7 +1004,10 @@ export function handlesEventType(eventType: string): boolean { return eventType === EventType.Invite || eventType === EventType.Candidates || eventType === EventType.Answer || - eventType === EventType.Hangup; + eventType === EventType.Hangup || + eventType === EventType.SDPStreamMetadataChanged || + eventType === EventType.SDPStreamMetadataChangedPrefix || + eventType === EventType.Negotiate; } export function tests() { diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index 8c041fcc..a0a64752 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -15,7 +15,7 @@ - call peers are essentially identified by (userid, deviceid, sessionid). If see a new session id, we first disconnect from the current member so we're ready to connect with a clean slate again (in a member event, also in to_device? no harm I suppose, given olm encryption ensures you can't spoof the deviceid). - implement to_device messages arriving before m.call(.member) state event - implement muting tracks with m.call.sdp_stream_metadata_changed - - implement cloning the localMedia so it works in safari? + - DONE: implement cloning the localMedia so it works in safari? - DONE: implement 3 retries per peer - reeable crypto & implement fetching olm keys before sending encrypted signalling message - local echo for join/leave buttons? diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 892ed67c..9fc8d1ee 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -123,6 +123,17 @@ export class GroupCall extends EventEmitter<{change: never}> { }); } + async setMedia(localMedia: LocalMedia): Promise { + if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { + const oldMedia = this._localMedia; + this._localMedia = localMedia; + await Promise.all(Array.from(this._members.values()).map(m => { + return m.setMedia(localMedia!.clone()); + })); + oldMedia?.dispose(); + } + } + get hasJoined() { return this._state === GroupCallState.Joining || this._state === GroupCallState.Joined; } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 06693030..e821e3da 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -182,6 +182,14 @@ export class Member { } } + /** @internal */ + async setMedia(localMedia: LocalMedia): Promise { + const oldMedia = this.localMedia; + this.localMedia = localMedia; + await this.peerCall?.setMedia(localMedia); + oldMedia?.dispose(); + } + private _createPeerCall(callId: string): PeerCall { return new PeerCall(callId, Object.assign({}, this.options, { emitUpdate: this.emitUpdate, diff --git a/src/platform/types/MediaDevices.ts b/src/platform/types/MediaDevices.ts index d0edbbea..85f64f95 100644 --- a/src/platform/types/MediaDevices.ts +++ b/src/platform/types/MediaDevices.ts @@ -39,6 +39,9 @@ export interface Track { readonly label: string; readonly id: string; readonly settings: MediaTrackSettings; + get enabled(): boolean; + set enabled(value: boolean); + equals(track: Track): boolean; stop(): void; } diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index 3723162a..a04fca91 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -75,27 +75,37 @@ export class StreamWrapper implements Stream { public audioTrack: AudioTrackWrapper | undefined = undefined; public videoTrack: TrackWrapper | undefined = undefined; - constructor(public readonly stream: MediaStream) { - for (const track of stream.getTracks()) { - this.update(track); + constructor(public readonly stream: MediaStream, clonedTracks?: {audioTrack?: AudioTrackWrapper, videoTrack?: TrackWrapper}) { + if (clonedTracks) { + this.audioTrack = clonedTracks.audioTrack; + this.videoTrack = clonedTracks.videoTrack; + } else { + for (const track of stream.getTracks()) { + this.update(track); + } } } get id(): string { return this.stream.id; } clone(): Stream { - return new StreamWrapper(this.stream.clone()); + const clonedStream = this.stream.clone(); + const clonedTracks = { + audioTrack: this.audioTrack ? new AudioTrackWrapper(clonedStream.getAudioTracks()[0], clonedStream, this.audioTrack.id): undefined, + videoTrack: this.videoTrack ? new TrackWrapper(clonedStream.getVideoTracks()[0], clonedStream, this.videoTrack.id): undefined, + }; + return new StreamWrapper(clonedStream, clonedTracks); } update(track: MediaStreamTrack): TrackWrapper | undefined { if (track.kind === "video") { if (!this.videoTrack || track.id !== this.videoTrack.track.id) { - this.videoTrack = new TrackWrapper(track, this.stream); + this.videoTrack = new TrackWrapper(track, this.stream, track.id); } return this.videoTrack; } else if (track.kind === "audio") { if (!this.audioTrack || track.id !== this.audioTrack.track.id) { - this.audioTrack = new AudioTrackWrapper(track, this.stream); + this.audioTrack = new AudioTrackWrapper(track, this.stream, track.id); } return this.audioTrack; } @@ -105,14 +115,18 @@ export class StreamWrapper implements Stream { export class TrackWrapper implements Track { constructor( public readonly track: MediaStreamTrack, - public readonly stream: MediaStream + public readonly stream: MediaStream, + public readonly originalId: string, ) {} get kind(): TrackKind { return this.track.kind as TrackKind; } get label(): string { return this.track.label; } get id(): string { return this.track.id; } get settings(): MediaTrackSettings { return this.track.getSettings(); } - + get enabled(): boolean { return this.track.enabled; } + set enabled(enabled: boolean) { this.track.enabled = enabled; } + // test equality across clones + equals(track: Track): boolean { return (track as TrackWrapper).originalId === this.originalId; } stop() { this.track.stop(); } } @@ -126,8 +140,8 @@ export class AudioTrackWrapper extends TrackWrapper { private volumeLooperTimeout: number; private speakingVolumeSamples: number[]; - constructor(track: MediaStreamTrack, stream: MediaStream) { - super(track, stream); + constructor(track: MediaStreamTrack, stream: MediaStream, originalId: string) { + super(track, stream, originalId); this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); this.initVolumeMeasuring(); this.measureVolumeActivity(true); diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 672c14d4..22096699 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -109,8 +109,16 @@ export class DOMTrackSenderOrReceiver implements TrackReceiver { this.transceiver.direction === this.exclusiveValue; } + enableWithoutRenegotiation(enabled: boolean) { + this.track.track.enabled = enabled; + } + enable(enabled: boolean) { if (enabled !== this.enabled) { + // do this first, so we stop sending track data immediately. + // this will still consume bandwidth though, so also disable the transceiver, + // which will trigger a renegotiation though. + this.enableWithoutRenegotiation(enabled); if (enabled) { if (this.transceiver.direction === "inactive") { this.transceiver.direction = this.exclusiveValue; diff --git a/src/platform/web/index.html b/src/platform/web/index.html index 5950d89f..064c61a1 100644 --- a/src/platform/web/index.html +++ b/src/platform/web/index.html @@ -11,7 +11,7 @@ - + + + diff --git a/scripts/logviewer/main.js b/scripts/logviewer/main.js index 3ae860b2..e552a094 100644 --- a/scripts/logviewer/main.js +++ b/scripts/logviewer/main.js @@ -164,7 +164,11 @@ function getRootItemHeader(prevItem, item) { async function loadFile() { const file = await openFile(); document.getElementById("filename").innerText = file.name; - const json = await readFileAsText(file); + await loadBlob(file); +} + +export async function loadBlob(blob) { + const json = await readFileAsText(blob); const logs = JSON.parse(json); logs.items.sort((a, b) => itemStart(a) - itemStart(b)); rootItem = {c: logs.items}; @@ -181,6 +185,7 @@ async function loadFile() { return fragment; }, document.createDocumentFragment()); main.replaceChildren(fragment); + main.scrollTop = main.scrollHeight; } // TODO: make this use processRecursively From d85f93fb16aa505499e4d9b0bef0936edb4b2970 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 10 May 2022 11:02:39 +0200 Subject: [PATCH 123/435] allow opening the logs straight in the log viewer from settings --- .../session/settings/SettingsViewModel.js | 7 ++++- .../web/ui/session/settings/SettingsView.js | 29 ++++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/domain/session/settings/SettingsViewModel.js b/src/domain/session/settings/SettingsViewModel.js index e7990844..d0f2e91d 100644 --- a/src/domain/session/settings/SettingsViewModel.js +++ b/src/domain/session/settings/SettingsViewModel.js @@ -136,9 +136,14 @@ export class SettingsViewModel extends ViewModel { } async exportLogs() { + const logs = await this.exportLogsBlob(); + this.platform.saveFileAs(logs, `hydrogen-logs-${this.platform.clock.now()}.json`); + } + + async exportLogsBlob() { const persister = this.logger.reporters.find(r => typeof r.export === "function"); const logExport = await persister.export(); - this.platform.saveFileAs(logExport.asBlob(), `hydrogen-logs-${this.platform.clock.now()}.json`); + return logExport.asBlob(); } async togglePushNotifications() { diff --git a/src/platform/web/ui/session/settings/SettingsView.js b/src/platform/web/ui/session/settings/SettingsView.js index 93e44307..58992d9f 100644 --- a/src/platform/web/ui/session/settings/SettingsView.js +++ b/src/platform/web/ui/session/settings/SettingsView.js @@ -98,13 +98,18 @@ export class SettingsView extends TemplateView { t.h3("Preferences"), row(t, vm.i18n`Scale down images when sending`, this._imageCompressionRange(t, vm)), ); + const logButtons = [t.button({onClick: () => vm.exportLogs()}, "Export")]; + if (import.meta.env.DEV) { + logButtons.push(t.button({onClick: () => openLogs(vm)}, "Open logs")); + } settingNodes.push( t.h3("Application"), row(t, vm.i18n`Version`, version), row(t, vm.i18n`Storage usage`, vm => `${vm.storageUsage} / ${vm.storageQuota}`), - row(t, vm.i18n`Debug logs`, t.button({onClick: () => vm.exportLogs()}, "Export")), + row(t, vm.i18n`Debug logs`, logButtons), t.p(["Debug logs contain application usage data including your username, the IDs or aliases of the rooms or groups you have visited, the usernames of other users and the names of files you send. They do not contain messages. For more information, review our ", t.a({href: "https://element.io/privacy", target: "_blank", rel: "noopener"}, "privacy policy"), "."]), + t.p([]) ); return t.main({className: "Settings middle"}, [ @@ -136,3 +141,25 @@ export class SettingsView extends TemplateView { })]; } } + +async function openLogs(vm) { + const logviewerUrl = (await import("../../../../../../scripts/logviewer/index.html?url")).default; + const win = window.open(logviewerUrl); + await new Promise(async r => { + let receivedPong = false; + const waitForPong = event => { + if (event.data.type === "pong") { + window.removeEventListener("message", waitForPong); + receivedPong = true; + r(); + } + }; + window.addEventListener("message", waitForPong); + while (!receivedPong) { + win.postMessage({type: "ping"}); + await new Promise(rr => setTimeout(rr), 100); + } + }); + const logs = await vm.exportLogsBlob(); + win.postMessage({type: "open", logs: logs.nativeBlob}); +} From c823bb125fe21d90b42bbd3e8dd6f20608590d1f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 10 May 2022 11:20:25 +0200 Subject: [PATCH 124/435] fix lint error --- .../web/ui/session/settings/SettingsView.js | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/platform/web/ui/session/settings/SettingsView.js b/src/platform/web/ui/session/settings/SettingsView.js index 58992d9f..e07aaff5 100644 --- a/src/platform/web/ui/session/settings/SettingsView.js +++ b/src/platform/web/ui/session/settings/SettingsView.js @@ -145,20 +145,23 @@ export class SettingsView extends TemplateView { async function openLogs(vm) { const logviewerUrl = (await import("../../../../../../scripts/logviewer/index.html?url")).default; const win = window.open(logviewerUrl); - await new Promise(async r => { + await new Promise((resolve, reject) => { let receivedPong = false; const waitForPong = event => { if (event.data.type === "pong") { window.removeEventListener("message", waitForPong); receivedPong = true; - r(); + resolve(); + } + }; + const sendPings = async () => { + while (!receivedPong) { + win.postMessage({type: "ping"}); + await new Promise(rr => setTimeout(rr), 100); } }; window.addEventListener("message", waitForPong); - while (!receivedPong) { - win.postMessage({type: "ping"}); - await new Promise(rr => setTimeout(rr), 100); - } + sendPings().catch(reject); }); const logs = await vm.exportLogsBlob(); win.postMessage({type: "open", logs: logs.nativeBlob}); From 1d900b518453ebb838fdba7aca2ca51ed4216856 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 10 May 2022 12:14:09 +0200 Subject: [PATCH 125/435] finish open window and poll code for logviewer --- .../web/ui/session/settings/SettingsView.js | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/platform/web/ui/session/settings/SettingsView.js b/src/platform/web/ui/session/settings/SettingsView.js index e07aaff5..09d3cb5a 100644 --- a/src/platform/web/ui/session/settings/SettingsView.js +++ b/src/platform/web/ui/session/settings/SettingsView.js @@ -146,18 +146,24 @@ async function openLogs(vm) { const logviewerUrl = (await import("../../../../../../scripts/logviewer/index.html?url")).default; const win = window.open(logviewerUrl); await new Promise((resolve, reject) => { - let receivedPong = false; + let shouldSendPings = true; + const cleanup = () => { + shouldSendPings = false; + window.removeEventListener("message", waitForPong); + }; const waitForPong = event => { if (event.data.type === "pong") { - window.removeEventListener("message", waitForPong); - receivedPong = true; + cleanup(); resolve(); } }; const sendPings = async () => { - while (!receivedPong) { + while (shouldSendPings) { win.postMessage({type: "ping"}); - await new Promise(rr => setTimeout(rr), 100); + await new Promise(rr => setTimeout(rr, 50)); + if (win.closed) { + cleanup(); + } } }; window.addEventListener("message", waitForPong); From f6ea7803f2bc567039fac8c4b9bd6db8d801b318 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 10 May 2022 18:03:15 +0200 Subject: [PATCH 126/435] move logviewer to own package --- package.json | 1 + scripts/logviewer/file.js | 51 --- scripts/logviewer/html.js | 110 ----- scripts/logviewer/index.html | 237 ---------- scripts/logviewer/main.js | 430 ------------------ .../web/ui/session/settings/SettingsView.js | 2 +- yarn.lock | 5 + 7 files changed, 7 insertions(+), 829 deletions(-) delete mode 100644 scripts/logviewer/file.js delete mode 100644 scripts/logviewer/html.js delete mode 100644 scripts/logviewer/index.html delete mode 100644 scripts/logviewer/main.js diff --git a/package.json b/package.json index b1704e43..28b726e2 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ }, "homepage": "https://github.com/vector-im/hydrogen-web/#readme", "devDependencies": { + "@matrixdotorg/structured-logviewer": "^0.0.1", "@typescript-eslint/eslint-plugin": "^4.29.2", "@typescript-eslint/parser": "^4.29.2", "acorn": "^8.6.0", diff --git a/scripts/logviewer/file.js b/scripts/logviewer/file.js deleted file mode 100644 index 64a8422b..00000000 --- a/scripts/logviewer/file.js +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright 2020 The Matrix.org Foundation C.I.C. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -export function openFile(mimeType = null) { - const input = document.createElement("input"); - input.setAttribute("type", "file"); - input.className = "hidden"; - if (mimeType) { - input.setAttribute("accept", mimeType); - } - const promise = new Promise((resolve, reject) => { - const checkFile = () => { - input.removeEventListener("change", checkFile, true); - const file = input.files[0]; - document.body.removeChild(input); - if (file) { - resolve(file); - } else { - reject(new Error("no file picked")); - } - } - input.addEventListener("change", checkFile, true); - }); - // IE11 needs the input to be attached to the document - document.body.appendChild(input); - input.click(); - return promise; -} - -export function readFileAsText(file) { - const reader = new FileReader(); - const promise = new Promise((resolve, reject) => { - reader.addEventListener("load", evt => resolve(evt.target.result)); - reader.addEventListener("error", evt => reject(evt.target.error)); - }); - reader.readAsText(file); - return promise; -} diff --git a/scripts/logviewer/html.js b/scripts/logviewer/html.js deleted file mode 100644 index a965a6ee..00000000 --- a/scripts/logviewer/html.js +++ /dev/null @@ -1,110 +0,0 @@ -/* -Copyright 2020 Bruno Windels - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// DOM helper functions - -export function isChildren(children) { - // children should be an not-object (that's the attributes), or a domnode, or an array - return typeof children !== "object" || !!children.nodeType || Array.isArray(children); -} - -export function classNames(obj, value) { - return Object.entries(obj).reduce((cn, [name, enabled]) => { - if (typeof enabled === "function") { - enabled = enabled(value); - } - if (enabled) { - return cn + (cn.length ? " " : "") + name; - } else { - return cn; - } - }, ""); -} - -export function setAttribute(el, name, value) { - if (name === "className") { - name = "class"; - } - if (value === false) { - el.removeAttribute(name); - } else { - if (value === true) { - value = name; - } - el.setAttribute(name, value); - } -} - -export function el(elementName, attributes, children) { - return elNS(HTML_NS, elementName, attributes, children); -} - -export function elNS(ns, elementName, attributes, children) { - if (attributes && isChildren(attributes)) { - children = attributes; - attributes = null; - } - - const e = document.createElementNS(ns, elementName); - - if (attributes) { - for (let [name, value] of Object.entries(attributes)) { - if (name === "className" && typeof value === "object" && value !== null) { - value = classNames(value); - } - setAttribute(e, name, value); - } - } - - if (children) { - if (!Array.isArray(children)) { - children = [children]; - } - for (let c of children) { - if (!c.nodeType) { - c = text(c); - } - e.appendChild(c); - } - } - return e; -} - -export function text(str) { - return document.createTextNode(str); -} - -export const HTML_NS = "http://www.w3.org/1999/xhtml"; -export const SVG_NS = "http://www.w3.org/2000/svg"; - -export const TAG_NAMES = { - [HTML_NS]: [ - "br", "a", "ol", "ul", "li", "div", "h1", "h2", "h3", "h4", "h5", "h6", - "p", "strong", "em", "span", "img", "section", "main", "article", "aside", - "pre", "button", "time", "input", "textarea", "label", "form", "progress", "output"], - [SVG_NS]: ["svg", "circle"] -}; - -export const tag = {}; - - -for (const [ns, tags] of Object.entries(TAG_NAMES)) { - for (const tagName of tags) { - tag[tagName] = function(attributes, children) { - return elNS(ns, tagName, attributes, children); - } - } -} diff --git a/scripts/logviewer/index.html b/scripts/logviewer/index.html deleted file mode 100644 index 109cf8d1..00000000 --- a/scripts/logviewer/index.html +++ /dev/null @@ -1,237 +0,0 @@ - - - - - - - -

-
- - - - - diff --git a/scripts/logviewer/main.js b/scripts/logviewer/main.js deleted file mode 100644 index e552a094..00000000 --- a/scripts/logviewer/main.js +++ /dev/null @@ -1,430 +0,0 @@ -/* -Copyright 2020 The Matrix.org Foundation C.I.C. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import {tag as t} from "./html.js"; -import {openFile, readFileAsText} from "./file.js"; - -const main = document.querySelector("main"); - -let selectedItemNode; -let rootItem; -let itemByRef; -let itemsRefFrom; - -const logLevels = [undefined, "All", "Debug", "Detail", "Info", "Warn", "Error", "Fatal", "Off"]; - -main.addEventListener("click", event => { - if (event.target.classList.contains("toggleExpanded")) { - const li = event.target.parentElement.parentElement; - li.classList.toggle("expanded"); - } else { - // allow clicking any links other than .item in the timeline, like refs - if (event.target.tagName === "A" && !event.target.classList.contains("item")) { - return; - } - const itemNode = event.target.closest(".item"); - if (itemNode) { - // we don't want scroll to jump when clicking - // so prevent default behaviour, and select and push to history manually - event.preventDefault(); - selectNode(itemNode); - history.pushState(null, null, `#${itemNode.id}`); - } - } -}); - -window.addEventListener("hashchange", () => { - const id = window.location.hash.substr(1); - const itemNode = document.getElementById(id); - if (itemNode && itemNode.closest("main")) { - ensureParentsExpanded(itemNode); - selectNode(itemNode); - itemNode.scrollIntoView({behavior: "smooth", block: "nearest"}); - } -}); - -function selectNode(itemNode) { - if (selectedItemNode) { - selectedItemNode.classList.remove("selected"); - } - selectedItemNode = itemNode; - selectedItemNode.classList.add("selected"); - let item = rootItem; - let parent; - const indices = selectedItemNode.id.split("/").map(i => parseInt(i, 10)); - for(const i of indices) { - parent = item; - item = itemChildren(item)[i]; - } - showItemDetails(item, parent, selectedItemNode); -} - -function ensureParentsExpanded(itemNode) { - let li = itemNode.parentElement.parentElement; - while (li.tagName === "LI") { - li.classList.add("expanded"); - li = li.parentElement.parentElement; - } -} - -function stringifyItemValue(value) { - if (typeof value === "object" && value !== null) { - return JSON.stringify(value, undefined, 2); - } else { - return value + ""; - } -} - -function showItemDetails(item, parent, itemNode) { - const parentOffset = itemStart(parent) ? `${itemStart(item) - itemStart(parent)}ms` : "none"; - const expandButton = t.button("Expand recursively"); - expandButton.addEventListener("click", () => expandResursively(itemNode.parentElement.parentElement)); - const start = itemStart(item); - const aside = t.aside([ - t.h3(itemCaption(item)), - t.p([t.strong("Log level: "), logLevels[itemLevel(item)]]), - t.p([t.strong("Error: "), itemError(item) ? `${itemError(item).name} ${itemError(item).stack}` : "none"]), - t.p([t.strong("Parent offset: "), parentOffset]), - t.p([t.strong("Start: "), new Date(start).toString(), ` (${start})`]), - t.p([t.strong("Duration: "), `${itemDuration(item)}ms`]), - t.p([t.strong("Child count: "), itemChildren(item) ? `${itemChildren(item).length}` : "none"]), - t.p([t.strong("Forced finish: "), (itemForcedFinish(item) || false) + ""]), - t.p(t.strong("Values:")), - t.ul({class: "values"}, Object.entries(itemValues(item)).map(([key, value]) => { - let valueNode; - if (key === "ref") { - const refItem = itemByRef.get(value); - if (refItem) { - valueNode = t.a({href: `#${refItem.id}`}, itemCaption(refItem)); - } else { - valueNode = `unknown ref ${value}`; - } - } else if (key === "refId") { - const refSources = itemsRefFrom.get(value) ?? []; - valueNode = t.div([t.p([`${value}`, t.br(),`Found these references:`]),t.ul(refSources.map(item => { - return t.li(t.a({href: `#${item.id}`}, itemCaption(item))); - }))]); - } else { - valueNode = stringifyItemValue(value); - } - return t.li([ - t.span({className: "key"}, normalizeValueKey(key)), - t.span({className: "value"}, valueNode) - ]); - })), - t.p(expandButton) - ]); - document.querySelector("aside").replaceWith(aside); -} - -function expandResursively(li) { - li.classList.add("expanded"); - const ol = li.querySelector("ol"); - if (ol) { - const len = ol.children.length; - for (let i = 0; i < len; i += 1) { - expandResursively(ol.children[i]); - } - } -} - -document.getElementById("openFile").addEventListener("click", loadFile); - -function getRootItemHeader(prevItem, item) { - if (prevItem) { - const diff = itemStart(item) - itemEnd(prevItem); - if (diff >= 0) { - return `+ ${formatTime(diff)}`; - } else { - const overlap = -diff; - if (overlap >= itemDuration(item)) { - return `ran entirely in parallel with`; - } else { - return `ran ${formatTime(-diff)} in parallel with`; - } - } - } else { - return new Date(itemStart(item)).toString(); - } -} - -async function loadFile() { - const file = await openFile(); - document.getElementById("filename").innerText = file.name; - await loadBlob(file); -} - -export async function loadBlob(blob) { - const json = await readFileAsText(blob); - const logs = JSON.parse(json); - logs.items.sort((a, b) => itemStart(a) - itemStart(b)); - rootItem = {c: logs.items}; - itemByRef = new Map(); - itemsRefFrom = new Map(); - preprocessRecursively(rootItem, null, itemByRef, itemsRefFrom, []); - - const fragment = logs.items.reduce((fragment, item, i, items) => { - const prevItem = i === 0 ? null : items[i - 1]; - fragment.appendChild(t.section([ - t.h2(getRootItemHeader(prevItem, item)), - t.div({className: "timeline"}, t.ol(itemToNode(item, [i]))) - ])); - return fragment; - }, document.createDocumentFragment()); - main.replaceChildren(fragment); - main.scrollTop = main.scrollHeight; -} - -// TODO: make this use processRecursively -function preprocessRecursively(item, parentElement, refsMap, refsFromMap, path) { - item.s = (parentElement?.s || 0) + item.s; - if (itemRefSource(item)) { - refsMap.set(itemRefSource(item), item); - } - if (itemRef(item)) { - let refs = refsFromMap.get(itemRef(item)); - if (!refs) { - refs = []; - refsFromMap.set(itemRef(item), refs); - } - refs.push(item); - } - if (itemChildren(item)) { - for (let i = 0; i < itemChildren(item).length; i += 1) { - // do it in advance for a child as we don't want to do it for the rootItem - const child = itemChildren(item)[i]; - const childPath = path.concat(i); - child.id = childPath.join("/"); - preprocessRecursively(child, item, refsMap, refsFromMap, childPath); - } - } -} - -const MS_IN_SEC = 1000; -const MS_IN_MIN = MS_IN_SEC * 60; -const MS_IN_HOUR = MS_IN_MIN * 60; -const MS_IN_DAY = MS_IN_HOUR * 24; -function formatTime(ms) { - let str = ""; - if (ms > MS_IN_DAY) { - const days = Math.floor(ms / MS_IN_DAY); - ms -= days * MS_IN_DAY; - str += `${days}d`; - } - if (ms > MS_IN_HOUR) { - const hours = Math.floor(ms / MS_IN_HOUR); - ms -= hours * MS_IN_HOUR; - str += `${hours}h`; - } - if (ms > MS_IN_MIN) { - const mins = Math.floor(ms / MS_IN_MIN); - ms -= mins * MS_IN_MIN; - str += `${mins}m`; - } - if (ms > MS_IN_SEC) { - const secs = ms / MS_IN_SEC; - str += `${secs.toFixed(2)}s`; - } else if (ms > 0 || !str.length) { - str += `${ms}ms`; - } - return str; -} - -function itemChildren(item) { return item.c; } -function itemStart(item) { return item.s; } -function itemEnd(item) { return item.s + item.d; } -function itemDuration(item) { return item.d; } -function itemValues(item) { return item.v; } -function itemLevel(item) { return item.l; } -function itemLabel(item) { return item.v?.l; } -function itemType(item) { return item.v?.t; } -function itemError(item) { return item.e; } -function itemForcedFinish(item) { return item.f; } -function itemRef(item) { return item.v?.ref; } -function itemRefSource(item) { return item.v?.refId; } -function itemShortErrorMessage(item) { - if (itemError(item)) { - const e = itemError(item); - return e.name || e.stack.substr(0, e.stack.indexOf("\n")); - } -} - -function itemCaption(item) { - if (itemLabel(item) && itemError(item)) { - return `${itemLabel(item)} (${itemShortErrorMessage(item)})`; - } if (itemType(item) === "network") { - return `${itemValues(item)?.method} ${itemValues(item)?.url}`; - } else if (itemLabel(item) && itemValues(item)?.id) { - return `${itemLabel(item)} ${itemValues(item).id}`; - } else if (itemLabel(item) && itemValues(item)?.status) { - return `${itemLabel(item)} (${itemValues(item).status})`; - } else if (itemLabel(item) && itemValues(item)?.type) { - return `${itemLabel(item)} (${itemValues(item)?.type})`; - } else if (itemRef(item)) { - const refItem = itemByRef.get(itemRef(item)); - if (refItem) { - return `ref "${itemCaption(refItem)}"` - } else { - return `unknown ref ${itemRef(item)}` - } - } else { - return itemLabel(item) || itemType(item); - } -} -function normalizeValueKey(key) { - switch (key) { - case "t": return "type"; - case "l": return "label"; - default: return key; - } -} - -// returns the node and the total range (recursively) occupied by the node -function itemToNode(item) { - const hasChildren = !!itemChildren(item)?.length; - const className = { - item: true, - "has-children": hasChildren, - error: itemError(item), - [`type-${itemType(item)}`]: !!itemType(item), - [`level-${itemLevel(item)}`]: true, - }; - - const id = item.id; - let captionNode; - if (itemRef(item)) { - const refItem = itemByRef.get(itemRef(item)); - if (refItem) { - captionNode = ["ref ", t.a({href: `#${refItem.id}`}, itemCaption(refItem))]; - } - } - if (!captionNode) { - captionNode = itemCaption(item); - } - const li = t.li([ - t.div([ - hasChildren ? t.button({className: "toggleExpanded"}) : "", - t.a({className, id, href: `#${id}`}, [ - t.span({class: "caption"}, captionNode), - t.span({class: "duration"}, `(${formatTime(itemDuration(item))})`), - ]) - ]) - ]); - if (itemChildren(item) && itemChildren(item).length) { - li.appendChild(t.ol(itemChildren(item).map(item => { - return itemToNode(item); - }))); - } - return li; -} - -const highlightForm = document.getElementById("highlightForm"); - -highlightForm.addEventListener("submit", evt => { - evt.preventDefault(); - const matchesOutput = document.getElementById("highlightMatches"); - const query = document.getElementById("highlight").value; - if (query) { - matchesOutput.innerText = "Searching…"; - let matches = 0; - processRecursively(rootItem, item => { - let domNode = document.getElementById(item.id); - if (itemMatchesFilter(item, query)) { - matches += 1; - domNode.classList.add("highlighted"); - domNode = domNode.parentElement; - while (domNode.nodeName !== "SECTION") { - if (domNode.nodeName === "LI") { - domNode.classList.add("expanded"); - } - domNode = domNode.parentElement; - } - } else { - domNode.classList.remove("highlighted"); - } - }); - matchesOutput.innerText = `${matches} matches`; - } else { - for (const node of document.querySelectorAll(".highlighted")) { - node.classList.remove("highlighted"); - } - matchesOutput.innerText = ""; - } -}); - -function itemMatchesFilter(item, query) { - if (itemError(item)) { - if (valueMatchesQuery(itemError(item), query)) { - return true; - } - } - return valueMatchesQuery(itemValues(item), query); -} - -function valueMatchesQuery(value, query) { - if (typeof value === "string") { - return value.includes(query); - } else if (typeof value === "object" && value !== null) { - for (const key in value) { - if (value.hasOwnProperty(key) && valueMatchesQuery(value[key], query)) { - return true; - } - } - } else if (typeof value === "number") { - return value.toString().includes(query); - } - return false; -} - -function processRecursively(item, callback, parentItem) { - if (item.id) { - callback(item, parentItem); - } - if (itemChildren(item)) { - for (let i = 0; i < itemChildren(item).length; i += 1) { - // do it in advance for a child as we don't want to do it for the rootItem - const child = itemChildren(item)[i]; - processRecursively(child, callback, item); - } - } -} - -document.getElementById("collapseAll").addEventListener("click", () => { - for (const node of document.querySelectorAll(".expanded")) { - node.classList.remove("expanded"); - } -}); -document.getElementById("hideCollapsed").addEventListener("click", () => { - for (const node of document.querySelectorAll("section > div.timeline > ol > li:not(.expanded)")) { - node.closest("section").classList.add("hidden"); - } -}); -document.getElementById("hideHighlightedSiblings").addEventListener("click", () => { - for (const node of document.querySelectorAll(".highlighted")) { - const list = node.closest("ol"); - const siblings = Array.from(list.querySelectorAll("li > div > a:not(.highlighted)")).map(n => n.closest("li")); - for (const sibling of siblings) { - if (!sibling.classList.contains("expanded")) { - sibling.classList.add("hidden"); - } - } - } -}); -document.getElementById("showAll").addEventListener("click", () => { - for (const node of document.querySelectorAll(".hidden")) { - node.classList.remove("hidden"); - } -}); diff --git a/src/platform/web/ui/session/settings/SettingsView.js b/src/platform/web/ui/session/settings/SettingsView.js index 09d3cb5a..ffc20f25 100644 --- a/src/platform/web/ui/session/settings/SettingsView.js +++ b/src/platform/web/ui/session/settings/SettingsView.js @@ -143,7 +143,7 @@ export class SettingsView extends TemplateView { } async function openLogs(vm) { - const logviewerUrl = (await import("../../../../../../scripts/logviewer/index.html?url")).default; + const logviewerUrl = (await import("@matrixdotorg/structured-logviewer/index.html?url")).default; const win = window.open(logviewerUrl); await new Promise((resolve, reject) => { let shouldSendPings = true; diff --git a/yarn.lock b/yarn.lock index 6405fe4d..cbce8b06 100644 --- a/yarn.lock +++ b/yarn.lock @@ -56,6 +56,11 @@ version "3.2.3" resolved "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.3.tgz#cc332fdd25c08ef0e40f4d33fc3f822a0f98b6f4" +"@matrixdotorg/structured-logviewer@^0.0.1": + version "0.0.1" + resolved "https://registry.yarnpkg.com/@matrixdotorg/structured-logviewer/-/structured-logviewer-0.0.1.tgz#9c29470b552f874afbb1df16c6e8e9e0c55cbf59" + integrity sha512-IdPYxAFDEoEs2G1ImKCkCxFI3xF1DDctP3N9JOtHRvIPbPPdTT9DyNqKTewCb5zwjNB1mGBrnWyURnHDiOOL3w== + "@nodelib/fs.scandir@2.1.5": version "2.1.5" resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5" From e2621015e179c2d00ef7129a6c2384be3902b189 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 10 May 2022 20:08:58 +0200 Subject: [PATCH 127/435] don't include log viewer in production build --- src/platform/web/ui/session/settings/SettingsView.js | 7 +++++-- vite.config.js | 4 +++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/platform/web/ui/session/settings/SettingsView.js b/src/platform/web/ui/session/settings/SettingsView.js index ffc20f25..f375a4be 100644 --- a/src/platform/web/ui/session/settings/SettingsView.js +++ b/src/platform/web/ui/session/settings/SettingsView.js @@ -142,9 +142,12 @@ export class SettingsView extends TemplateView { } } + async function openLogs(vm) { - const logviewerUrl = (await import("@matrixdotorg/structured-logviewer/index.html?url")).default; - const win = window.open(logviewerUrl); + // Use vite-specific url so this asset doesn't get picked up by vite and included in the production build, + // as opening the logs is only available during dev time, and @matrixdotorg/structured-logviewer is a dev dependency + // This url is what import "@matrixdotorg/structured-logviewer/index.html?url" resolves to with vite. + const win = window.open(`/@fs/${DEFINE_PROJECT_DIR}/node_modules/@matrixdotorg/structured-logviewer/index.html`); await new Promise((resolve, reject) => { let shouldSendPings = true; const cleanup = () => { diff --git a/vite.config.js b/vite.config.js index 87e3d063..97fb8885 100644 --- a/vite.config.js +++ b/vite.config.js @@ -37,6 +37,8 @@ export default defineConfig(({mode}) => { "sw": definePlaceholders }), ], - define: definePlaceholders, + define: Object.assign({ + DEFINE_PROJECT_DIR: JSON.stringify(__dirname) + }, definePlaceholders), }); }); From 21065791a86d53fda3c478bf9853646670338dad Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 10 May 2022 16:58:03 -0700 Subject: [PATCH 128/435] Fix removing members in handleCallMemberEvent --- src/matrix/calls/CallHandler.ts | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 7fbe6103..92f7314c 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -40,11 +40,15 @@ export type Options = Omit & { clock: Clock }; +function getRoomMemberKey(roomId: string, userId: string) { + return JSON.stringify(roomId)+`,`+JSON.stringify(userId); +} + export class CallHandler { // group calls by call id private readonly _calls: ObservableMap = new ObservableMap(); - // map of userId to set of conf_id's they are in - private memberToCallIds: Map> = new Map(); + // map of `"roomId","userId"` to set of conf_id's they are in + private roomMemberToCallIds: Map> = new Map(); private groupCallOptions: GroupCallOptions; private sessionId = makeId("s"); @@ -98,7 +102,7 @@ export class CallHandler { // } const callsMemberEvents = await txn.roomState.getAllForType(roomId, EventType.GroupCallMember); for (const entry of callsMemberEvents) { - this.handleCallMemberEvent(entry.event, log); + this.handleCallMemberEvent(entry.event, roomId, log); } // TODO: we should be loading the other members as well at some point })); @@ -149,7 +153,7 @@ export class CallHandler { // then update members for (const event of events) { if (event.type === EventType.GroupCallMember) { - this.handleCallMemberEvent(event, log); + this.handleCallMemberEvent(event, room.id, log); } } } @@ -194,8 +198,9 @@ export class CallHandler { } } - private handleCallMemberEvent(event: StateEvent, log: ILogItem) { + private handleCallMemberEvent(event: StateEvent, roomId: string, log: ILogItem) { const userId = event.state_key; + const roomMemberKey = getRoomMemberKey(roomId, userId) const calls = event.content["m.calls"] ?? []; for (const call of calls) { const callId = call["m.call_id"]; @@ -204,7 +209,8 @@ export class CallHandler { groupCall?.updateMembership(userId, call, log); }; const newCallIdsMemberOf = new Set(calls.map(call => call["m.call_id"])); - let previousCallIdsMemberOf = this.memberToCallIds.get(userId); + let previousCallIdsMemberOf = this.roomMemberToCallIds.get(roomMemberKey); + // remove user as member of any calls not present anymore if (previousCallIdsMemberOf) { for (const previousCallId of previousCallIdsMemberOf) { @@ -215,9 +221,9 @@ export class CallHandler { } } if (newCallIdsMemberOf.size === 0) { - this.memberToCallIds.delete(userId); + this.roomMemberToCallIds.delete(roomMemberKey); } else { - this.memberToCallIds.set(userId, newCallIdsMemberOf); + this.roomMemberToCallIds.set(roomMemberKey, newCallIdsMemberOf); } } } From 5ee4e39bc78eb2ab28b7c7ac26379fd219ec8195 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 11 May 2022 10:06:05 +0200 Subject: [PATCH 129/435] add return type --- src/matrix/calls/CallHandler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 92f7314c..07ed8492 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -40,7 +40,7 @@ export type Options = Omit & { clock: Clock }; -function getRoomMemberKey(roomId: string, userId: string) { +function getRoomMemberKey(roomId: string, userId: string): string { return JSON.stringify(roomId)+`,`+JSON.stringify(userId); } From a923e7e5e1ef22aabdb615cd52d1c9acfbe7b5da Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 11 May 2022 13:14:23 +0200 Subject: [PATCH 130/435] don't pass errors as log levels --- src/matrix/calls/PeerCall.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 963d6c30..e6ce3571 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -438,7 +438,7 @@ export class PeerCall implements IDisposable { if (this.callId! > newCallId) { log.log( "Glare detected: answering incoming call " + newCallId + - " and canceling outgoing call ", + " and canceling outgoing call " ); // How do we interrupt `call()`? well, perhaps we need to not just await InviteSent but also CreateAnswer? if (this._state === CallState.Fledgling || this._state === CallState.CreateOffer) { @@ -452,7 +452,7 @@ export class PeerCall implements IDisposable { } else { log.log( "Glare detected: rejecting incoming call " + newCallId + - " and keeping outgoing call ", + " and keeping outgoing call " ); await this.sendHangupWithCallId(newCallId, CallErrorCode.Replaced, log); } @@ -625,7 +625,7 @@ export class PeerCall implements IDisposable { if (this.opponentPartyId !== partyId) { log.log( `Ignoring candidates from party ID ${partyId}: ` + - `we have chosen party ID ${this.opponentPartyId}`, + `we have chosen party ID ${this.opponentPartyId}` ); return; @@ -680,7 +680,7 @@ export class PeerCall implements IDisposable { await this.sendSignallingMessage({type: EventType.Negotiate, content}, log); } } catch (err) { - log.log(`Failed to complete negotiation`, err); + log.log(`Failed to complete negotiation`).catch(err); } } @@ -801,12 +801,12 @@ export class PeerCall implements IDisposable { log.log(`Ignoring remote ICE candidate with no sdpMid or sdpMLineIndex`); continue; } - log.log(`Got remote ICE ${candidate.sdpMid} candidate: ${candidate.candidate}`); + const logItem = log.log(`Adding remote ICE ${candidate.sdpMid} candidate: ${candidate.candidate}`); try { await this.peerConnection.addIceCandidate(candidate); } catch (err) { if (!this.ignoreOffer) { - log.log(`Failed to add remote ICE candidate`, err); + logItem.catch(err); } } } From ec1568cf1c1a2545d32b3e3bf3342760fdfb75b7 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 12 May 2022 11:53:29 +0200 Subject: [PATCH 131/435] fix lint error --- .eslintrc.js | 1 + 1 file changed, 1 insertion(+) diff --git a/.eslintrc.js b/.eslintrc.js index cb28f4c8..eb23d387 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -17,6 +17,7 @@ module.exports = { "globals": { "DEFINE_VERSION": "readonly", "DEFINE_GLOBAL_HASH": "readonly", + "DEFINE_PROJECT_DIR": "readonly", // only available in sw.js "DEFINE_UNHASHED_PRECACHED_ASSETS": "readonly", "DEFINE_HASHED_PRECACHED_ASSETS": "readonly", From d727dfd843a5167197c783c5d1ba07748bde17a3 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 12 May 2022 11:58:28 +0200 Subject: [PATCH 132/435] add session.observeRoomState to observe state changes in all rooms and use it for calls this won't be called for state already received and stored in storage, that you need to still do yourself --- src/lib.ts | 8 +++++++ src/matrix/RoomStateHandlerSet.ts | 37 +++++++++++++++++++++++++++++++ src/matrix/Session.js | 9 +++++++- src/matrix/calls/CallHandler.ts | 19 ++++++---------- src/matrix/room/Room.js | 30 ++++++++++++------------- src/matrix/room/common.ts | 11 +++++++++ 6 files changed, 86 insertions(+), 28 deletions(-) create mode 100644 src/matrix/RoomStateHandlerSet.ts diff --git a/src/lib.ts b/src/lib.ts index 0fc6f539..8c5c4e8e 100644 --- a/src/lib.ts +++ b/src/lib.ts @@ -15,11 +15,19 @@ limitations under the License. */ export {Logger} from "./logging/Logger"; +export type {ILogItem} from "./logging/types"; export {IDBLogPersister} from "./logging/IDBLogPersister"; export {ConsoleReporter} from "./logging/ConsoleReporter"; export {Platform} from "./platform/web/Platform.js"; export {Client, LoadStatus} from "./matrix/Client.js"; export {RoomStatus} from "./matrix/room/common"; +// export everything needed to observe state events on all rooms using session.observeRoomState +export type {RoomStateHandler} from "./matrix/room/common"; +export type {MemberChange} from "./matrix/room/members/RoomMember"; +export type {Transaction} from "./matrix/storage/idb/Transaction"; +export type {Room} from "./matrix/room/Room"; +export type {StateEvent} from "./matrix/storage/types"; + // export main view & view models export {createNavigation, createRouter} from "./domain/navigation/index.js"; export {RootViewModel} from "./domain/RootViewModel.js"; diff --git a/src/matrix/RoomStateHandlerSet.ts b/src/matrix/RoomStateHandlerSet.ts new file mode 100644 index 00000000..cf202097 --- /dev/null +++ b/src/matrix/RoomStateHandlerSet.ts @@ -0,0 +1,37 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import type {ILogItem} from "../logging/types"; +import type {StateEvent} from "./storage/types"; +import type {Transaction} from "./storage/idb/Transaction"; +import type {Room} from "./room/Room"; +import type {MemberChange} from "./room/members/RoomMember"; +import type {RoomStateHandler} from "./room/common"; +import {BaseObservable} from "../observable/BaseObservable"; + +/** keeps track of all handlers registered with Session.observeRoomState */ +export class RoomStateHandlerSet extends BaseObservable implements RoomStateHandler { + handleRoomState(room: Room, stateEvent: StateEvent, txn: Transaction, log: ILogItem) { + for(let h of this._handlers) { + h.handleRoomState(room, stateEvent, txn, log); + } + } + updateRoomMembers(room: Room, memberChanges: Map) { + for(let h of this._handlers) { + h.updateRoomMembers(room, memberChanges); + } + } +} diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 9d63d335..6211c456 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -48,6 +48,7 @@ import {SecretStorage} from "./ssss/SecretStorage"; import {ObservableValue} from "../observable/value/ObservableValue"; import {RetainedObservableValue} from "../observable/value/RetainedObservableValue"; import {CallHandler} from "./calls/CallHandler"; +import {RoomStateHandlerSet} from "./RoomStateHandlerSet"; const PICKLE_KEY = "DEFAULT_KEY"; const PUSHER_KEY = "pusher"; @@ -101,6 +102,8 @@ export class Session { }], forceTURN: false, }); + this._roomStateHandler = new RoomStateHandlerSet(); + this.observeRoomState(this._callHandler); this._deviceMessageHandler = new DeviceMessageHandler({storage, callHandler: this._callHandler}); this._olm = olm; this._olmUtil = null; @@ -595,7 +598,7 @@ export class Session { user: this._user, createRoomEncryption: this._createRoomEncryption, platform: this._platform, - callHandler: this._callHandler + roomStateHandler: this._roomStateHandler }); } @@ -937,6 +940,10 @@ export class Session { return observable; } + observeRoomState(roomStateHandler) { + return this._roomStateHandler.subscribe(roomStateHandler); + } + /** Creates an empty (summary isn't loaded) the archived room if it isn't loaded already, assuming sync will either remove it (when rejoining) or diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 07ed8492..2386076b 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -35,6 +35,7 @@ import type {Options as GroupCallOptions} from "./group/GroupCall"; import type {Transaction} from "../storage/idb/Transaction"; import type {CallEntry} from "../storage/idb/stores/CallStore"; import type {Clock} from "../../platform/web/dom/Clock"; +import type {RoomStateHandler} from "../room/common"; export type Options = Omit & { clock: Clock @@ -44,7 +45,7 @@ function getRoomMemberKey(roomId: string, userId: string): string { return JSON.stringify(roomId)+`,`+JSON.stringify(userId); } -export class CallHandler { +export class CallHandler implements RoomStateHandler { // group calls by call id private readonly _calls: ObservableMap = new ObservableMap(); // map of `"roomId","userId"` to set of conf_id's they are in @@ -143,18 +144,12 @@ export class CallHandler { // TODO: check and poll turn server credentials here /** @internal */ - handleRoomState(room: Room, events: StateEvent[], txn: Transaction, log: ILogItem) { - // first update call events - for (const event of events) { - if (event.type === EventType.GroupCall) { - this.handleCallEvent(event, room.id, txn, log); - } + handleRoomState(room: Room, event: StateEvent, txn: Transaction, log: ILogItem) { + if (event.type === EventType.GroupCall) { + this.handleCallEvent(event, room.id, txn, log); } - // then update members - for (const event of events) { - if (event.type === EventType.GroupCallMember) { - this.handleCallMemberEvent(event, room.id, log); - } + if (event.type === EventType.GroupCallMember) { + this.handleCallMemberEvent(event, room.id, log); } } diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js index 34f35af8..556cef7a 100644 --- a/src/matrix/room/Room.js +++ b/src/matrix/room/Room.js @@ -30,7 +30,7 @@ const EVENT_ENCRYPTED_TYPE = "m.room.encrypted"; export class Room extends BaseRoom { constructor(options) { super(options); - this._callHandler = options.callHandler; + this._roomStateHandler = options.roomStateHandler; // TODO: pass pendingEvents to start like pendingOperations? const {pendingEvents} = options; const relationWriter = new RelationWriter({ @@ -179,7 +179,7 @@ export class Room extends BaseRoom { removedPendingEvents = await this._sendQueue.removeRemoteEchos(roomResponse.timeline.events, txn, log); } const powerLevelsEvent = this._getPowerLevelsEvent(roomResponse); - this._updateCallHandler(roomResponse, txn, log); + this._updateRoomStateHandler(roomResponse, txn, log); return { summaryChanges, roomEncryption, @@ -217,9 +217,7 @@ export class Room extends BaseRoom { if (this._memberList) { this._memberList.afterSync(memberChanges); } - if (this._callHandler) { - this._callHandler.updateRoomMembers(this, memberChanges); - } + this._roomStateHandler.updateRoomMembers(this, memberChanges); if (this._observedMembers) { this._updateObservedMembers(memberChanges); } @@ -447,17 +445,19 @@ export class Room extends BaseRoom { return this._sendQueue.pendingEvents; } - _updateCallHandler(roomResponse, txn, log) { - if (this._callHandler) { - const stateEvents = roomResponse.state?.events; - if (stateEvents?.length) { - this._callHandler.handleRoomState(this, stateEvents, txn, log); + _updateRoomStateHandler(roomResponse, txn, log) { + const stateEvents = roomResponse.state?.events; + if (stateEvents) { + for (let i = 0; i < stateEvents.length; i++) { + this._roomStateHandler.handleRoomState(this, stateEvents[i], txn, log); } - let timelineEvents = roomResponse.timeline?.events; - if (timelineEvents) { - const timelineStateEvents = timelineEvents.filter(e => typeof e.state_key === "string"); - if (timelineEvents.length !== 0) { - this._callHandler.handleRoomState(this, timelineStateEvents, txn, log); + } + let timelineEvents = roomResponse.timeline?.events; + if (timelineEvents) { + for (let i = 0; i < timelineEvents.length; i++) { + const event = timelineEvents[i]; + if (typeof event.state_key === "string") { + this._roomStateHandler.handleRoomState(this, event, txn, log); } } } diff --git a/src/matrix/room/common.ts b/src/matrix/room/common.ts index 57ab7023..2fdc2483 100644 --- a/src/matrix/room/common.ts +++ b/src/matrix/room/common.ts @@ -14,6 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. */ +import type {Room} from "./Room"; +import type {StateEvent} from "../storage/types"; +import type {Transaction} from "../storage/idb/Transaction"; +import type {ILogItem} from "../../logging/types"; +import type {MemberChange} from "./members/RoomMember"; + export function getPrevContentFromStateEvent(event) { // where to look for prev_content is a bit of a mess, // see https://matrix.to/#/!NasysSDfxKxZBzJJoE:matrix.org/$DvrAbZJiILkOmOIuRsNoHmh2v7UO5CWp_rYhlGk34fQ?via=matrix.org&via=pixie.town&via=amorgan.xyz @@ -40,3 +46,8 @@ export enum RoomType { Private, Public } + +export interface RoomStateHandler { + handleRoomState(room: Room, stateEvent: StateEvent, txn: Transaction, log: ILogItem); + updateRoomMembers(room: Room, memberChanges: Map); +} From db053385962e0272c86f0ca9b998633f8325a3a1 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 12 May 2022 17:26:29 +0200 Subject: [PATCH 133/435] extract function to iterate over room response state events --- src/matrix/room/ArchivedRoom.js | 8 ++-- src/matrix/room/Room.js | 32 +++++++-------- src/matrix/room/RoomSummary.js | 25 ++---------- src/matrix/room/common.ts | 69 ++++++++++++++++++++++++++++++++- 4 files changed, 91 insertions(+), 43 deletions(-) diff --git a/src/matrix/room/ArchivedRoom.js b/src/matrix/room/ArchivedRoom.js index 1a23d25b..86595163 100644 --- a/src/matrix/room/ArchivedRoom.js +++ b/src/matrix/room/ArchivedRoom.js @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {reduceStateEvents} from "./RoomSummary.js"; +import {iterateResponseStateEvents} from "./common"; import {BaseRoom} from "./BaseRoom.js"; import {RoomMember, EVENT_TYPE as MEMBER_EVENT_TYPE} from "./members/RoomMember.js"; @@ -173,15 +173,15 @@ export class ArchivedRoom extends BaseRoom { } function findKickDetails(roomResponse, ownUserId) { - const kickEvent = reduceStateEvents(roomResponse, (kickEvent, event) => { + let kickEvent; + iterateResponseStateEvents(roomResponse, event => { if (event.type === MEMBER_EVENT_TYPE) { // did we get kicked? if (event.state_key === ownUserId && event.sender !== event.state_key) { kickEvent = event; } } - return kickEvent; - }, null); + }); if (kickEvent) { return { // this is different from the room membership in the sync section, which can only be leave diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js index 556cef7a..a2eadfeb 100644 --- a/src/matrix/room/Room.js +++ b/src/matrix/room/Room.js @@ -23,6 +23,7 @@ import {WrappedError} from "../error.js" import {Heroes} from "./members/Heroes.js"; import {AttachmentUpload} from "./AttachmentUpload.js"; import {DecryptionSource} from "../e2ee/common.js"; +import {iterateResponseStateEvents} from "./common.js"; import {PowerLevels, EVENT_TYPE as POWERLEVELS_EVENT_TYPE } from "./PowerLevels.js"; const EVENT_ENCRYPTED_TYPE = "m.room.encrypted"; @@ -179,7 +180,7 @@ export class Room extends BaseRoom { removedPendingEvents = await this._sendQueue.removeRemoteEchos(roomResponse.timeline.events, txn, log); } const powerLevelsEvent = this._getPowerLevelsEvent(roomResponse); - this._updateRoomStateHandler(roomResponse, txn, log); + this._runRoomStateHandlers(roomResponse, txn, log); return { summaryChanges, roomEncryption, @@ -275,8 +276,13 @@ export class Room extends BaseRoom { } _getPowerLevelsEvent(roomResponse) { - const isPowerlevelEvent = event => event.state_key === "" && event.type === POWERLEVELS_EVENT_TYPE; - const powerLevelEvent = roomResponse.timeline?.events.find(isPowerlevelEvent) ?? roomResponse.state?.events.find(isPowerlevelEvent); + let powerLevelEvent; + iterateResponseStateEvents(roomResponse, event => { + if(event.state_key === "" && event.type === POWERLEVELS_EVENT_TYPE) { + powerLevelEvent = event; + } + + }); return powerLevelEvent; } @@ -445,20 +451,12 @@ export class Room extends BaseRoom { return this._sendQueue.pendingEvents; } - _updateRoomStateHandler(roomResponse, txn, log) { - const stateEvents = roomResponse.state?.events; - if (stateEvents) { - for (let i = 0; i < stateEvents.length; i++) { - this._roomStateHandler.handleRoomState(this, stateEvents[i], txn, log); - } - } - let timelineEvents = roomResponse.timeline?.events; - if (timelineEvents) { - for (let i = 0; i < timelineEvents.length; i++) { - const event = timelineEvents[i]; - if (typeof event.state_key === "string") { - this._roomStateHandler.handleRoomState(this, event, txn, log); - } + /** global room state handlers, run during write sync step */ + _runRoomStateHandlers(roomResponse, txn, log) { + iterateResponseStateEvents(roomResponse, event => { + this._roomStateHandler.handleRoomState(this, event, txn, log); + }); + } } } } diff --git a/src/matrix/room/RoomSummary.js b/src/matrix/room/RoomSummary.js index a3dec467..62608683 100644 --- a/src/matrix/room/RoomSummary.js +++ b/src/matrix/room/RoomSummary.js @@ -15,7 +15,7 @@ limitations under the License. */ import {MEGOLM_ALGORITHM} from "../e2ee/common.js"; - +import {iterateResponseStateEvents} from "./common"; function applyTimelineEntries(data, timelineEntries, isInitialSync, canMarkUnread, ownUserId) { if (timelineEntries.length) { @@ -27,25 +27,6 @@ function applyTimelineEntries(data, timelineEntries, isInitialSync, canMarkUnrea return data; } -export function reduceStateEvents(roomResponse, callback, value) { - const stateEvents = roomResponse?.state?.events; - // state comes before timeline - if (Array.isArray(stateEvents)) { - value = stateEvents.reduce(callback, value); - } - const timelineEvents = roomResponse?.timeline?.events; - // and after that state events in the timeline - if (Array.isArray(timelineEvents)) { - value = timelineEvents.reduce((data, event) => { - if (typeof event.state_key === "string") { - value = callback(value, event); - } - return value; - }, value); - } - return value; -} - function applySyncResponse(data, roomResponse, membership, ownUserId) { if (roomResponse.summary) { data = updateSummary(data, roomResponse.summary); @@ -60,7 +41,9 @@ function applySyncResponse(data, roomResponse, membership, ownUserId) { // process state events in state and in timeline. // non-state events are handled by applyTimelineEntries // so decryption is handled properly - data = reduceStateEvents(roomResponse, (data, event) => processStateEvent(data, event, ownUserId), data); + iterateResponseStateEvents(roomResponse, event => { + data = processStateEvent(data, event, ownUserId); + }); const unreadNotifications = roomResponse.unread_notifications; if (unreadNotifications) { data = processNotificationCounts(data, unreadNotifications); diff --git a/src/matrix/room/common.ts b/src/matrix/room/common.ts index 2fdc2483..38070925 100644 --- a/src/matrix/room/common.ts +++ b/src/matrix/room/common.ts @@ -15,7 +15,7 @@ limitations under the License. */ import type {Room} from "./Room"; -import type {StateEvent} from "../storage/types"; +import type {StateEvent, TimelineEvent} from "../storage/types"; import type {Transaction} from "../storage/idb/Transaction"; import type {ILogItem} from "../../logging/types"; import type {MemberChange} from "./members/RoomMember"; @@ -50,4 +50,71 @@ export enum RoomType { export interface RoomStateHandler { handleRoomState(room: Room, stateEvent: StateEvent, txn: Transaction, log: ILogItem); updateRoomMembers(room: Room, memberChanges: Map); +type RoomResponse = { + state?: { + events?: Array + }, + timeline?: { + events?: Array + } +} + +/** iterates over any state events in a sync room response, in the order that they should be applied (from older to younger events) */ +export function iterateResponseStateEvents(roomResponse: RoomResponse, callback: (StateEvent) => void) { + // first iterate over state events, they precede the timeline + const stateEvents = roomResponse.state?.events; + if (stateEvents) { + for (let i = 0; i < stateEvents.length; i++) { + callback(stateEvents[i]); + } + } + // now see if there are any state events within the timeline + let timelineEvents = roomResponse.timeline?.events; + if (timelineEvents) { + for (let i = 0; i < timelineEvents.length; i++) { + const event = timelineEvents[i]; + if (typeof event.state_key === "string") { + callback(event); + } + } + } +} + +export function tests() { + return { + "test iterateResponseStateEvents with both state and timeline sections": assert => { + const roomResponse = { + state: { + events: [ + {type: "m.room.member", state_key: "1"}, + {type: "m.room.member", state_key: "2", content: {a: 1}}, + ] + }, + timeline: { + events: [ + {type: "m.room.message"}, + {type: "m.room.member", state_key: "3"}, + {type: "m.room.message"}, + {type: "m.room.member", state_key: "2", content: {a: 2}}, + ] + } + } as unknown as RoomResponse; + const expectedStateKeys = ["1", "2", "3", "2"]; + const expectedAForMember2 = [1, 2]; + iterateResponseStateEvents(roomResponse, event => { + assert.strictEqual(event.type, "m.room.member"); + assert.strictEqual(expectedStateKeys.shift(), event.state_key); + if (event.state_key === "2") { + assert.strictEqual(expectedAForMember2.shift(), event.content.a); + } + }); + assert.strictEqual(expectedStateKeys.length, 0); + assert.strictEqual(expectedAForMember2.length, 0); + }, + "test iterateResponseStateEvents with empty response": assert => { + iterateResponseStateEvents({}, () => { + assert.fail("no events expected"); + }); + } + } } From a50ea7e77b84dbe955bf4dc062d0a098924e81d4 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 12 May 2022 17:27:03 +0200 Subject: [PATCH 134/435] add support for observing room state for single room + initial state --- src/lib.ts | 2 +- src/matrix/Session.js | 2 +- src/matrix/calls/CallHandler.ts | 2 +- src/matrix/room/BaseRoom.js | 26 +++++++++ src/matrix/room/Room.js | 12 +++- src/matrix/room/common.ts | 3 - .../room/state/ObservedStateKeyValue.ts | 55 +++++++++++++++++++ src/matrix/room/state/ObservedStateTypeMap.ts | 53 ++++++++++++++++++ .../{ => room/state}/RoomStateHandlerSet.ts | 14 ++--- src/matrix/room/state/types.ts | 38 +++++++++++++ 10 files changed, 192 insertions(+), 15 deletions(-) create mode 100644 src/matrix/room/state/ObservedStateKeyValue.ts create mode 100644 src/matrix/room/state/ObservedStateTypeMap.ts rename src/matrix/{ => room/state}/RoomStateHandlerSet.ts (75%) create mode 100644 src/matrix/room/state/types.ts diff --git a/src/lib.ts b/src/lib.ts index 8c5c4e8e..839fbb15 100644 --- a/src/lib.ts +++ b/src/lib.ts @@ -22,7 +22,7 @@ export {Platform} from "./platform/web/Platform.js"; export {Client, LoadStatus} from "./matrix/Client.js"; export {RoomStatus} from "./matrix/room/common"; // export everything needed to observe state events on all rooms using session.observeRoomState -export type {RoomStateHandler} from "./matrix/room/common"; +export type {RoomStateHandler} from "./matrix/room/state/types"; export type {MemberChange} from "./matrix/room/members/RoomMember"; export type {Transaction} from "./matrix/storage/idb/Transaction"; export type {Room} from "./matrix/room/Room"; diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 6211c456..cd676fc5 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -48,7 +48,7 @@ import {SecretStorage} from "./ssss/SecretStorage"; import {ObservableValue} from "../observable/value/ObservableValue"; import {RetainedObservableValue} from "../observable/value/RetainedObservableValue"; import {CallHandler} from "./calls/CallHandler"; -import {RoomStateHandlerSet} from "./RoomStateHandlerSet"; +import {RoomStateHandlerSet} from "./room/state/RoomStateHandlerSet"; const PICKLE_KEY = "DEFAULT_KEY"; const PUSHER_KEY = "pusher"; diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 2386076b..e585bb40 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -35,7 +35,7 @@ import type {Options as GroupCallOptions} from "./group/GroupCall"; import type {Transaction} from "../storage/idb/Transaction"; import type {CallEntry} from "../storage/idb/stores/CallStore"; import type {Clock} from "../../platform/web/dom/Clock"; -import type {RoomStateHandler} from "../room/common"; +import type {RoomStateHandler} from "../room/state/types"; export type Options = Omit & { clock: Clock diff --git a/src/matrix/room/BaseRoom.js b/src/matrix/room/BaseRoom.js index cc65a320..b8f172d0 100644 --- a/src/matrix/room/BaseRoom.js +++ b/src/matrix/room/BaseRoom.js @@ -31,6 +31,8 @@ import {ensureLogItem} from "../../logging/utils"; import {PowerLevels} from "./PowerLevels.js"; import {RetainedObservableValue} from "../../observable/value/RetainedObservableValue"; import {TimelineReader} from "./timeline/persistence/TimelineReader"; +import {ObservedStateTypeMap} from "./state/ObservedStateTypeMap"; +import {ObservedStateKeyValue} from "./state/ObservedStateKeyValue"; const EVENT_ENCRYPTED_TYPE = "m.room.encrypted"; @@ -53,11 +55,35 @@ export class BaseRoom extends EventEmitter { this._getSyncToken = getSyncToken; this._platform = platform; this._observedEvents = null; + this._roomStateObservers = new Set(); this._powerLevels = null; this._powerLevelLoading = null; this._observedMembers = null; } + async observeStateType(type, txn = undefined) { + const map = new ObservedStateTypeMap(type); + await this._addStateObserver(map, txn); + return map; + } + + async observeStateTypeAndKey(type, stateKey, txn = undefined) { + const value = new ObservedStateKeyValue(type, stateKey); + await this._addStateObserver(value, txn); + return value; + } + + async _addStateObserver(stateObserver, txn) { + if (!txn) { + txn = await this._storage.readTxn([this._storage.storeNames.roomState]); + } + await stateObserver.load(this.id, txn); + this._roomStateObservers.add(stateObserver); + stateObserver.setRemoveCallback(() => { + this._roomStateObservers.delete(stateObserver); + }); + } + async _eventIdsToEntries(eventIds, txn) { const retryEntries = []; await Promise.all(eventIds.map(async eventId => { diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js index a2eadfeb..796474d3 100644 --- a/src/matrix/room/Room.js +++ b/src/matrix/room/Room.js @@ -182,6 +182,7 @@ export class Room extends BaseRoom { const powerLevelsEvent = this._getPowerLevelsEvent(roomResponse); this._runRoomStateHandlers(roomResponse, txn, log); return { + roomResponse, summaryChanges, roomEncryption, newEntries, @@ -204,7 +205,7 @@ export class Room extends BaseRoom { const { summaryChanges, newEntries, updatedEntries, newLiveKey, removedPendingEvents, memberChanges, powerLevelsEvent, - heroChanges, roomEncryption + heroChanges, roomEncryption, roomResponse } = changes; log.set("id", this.id); this._syncWriter.afterSync(newLiveKey); @@ -264,6 +265,7 @@ export class Room extends BaseRoom { if (removedPendingEvents) { this._sendQueue.emitRemovals(removedPendingEvents); } + this._emitSyncRoomState(roomResponse); } _updateObservedMembers(memberChanges) { @@ -457,8 +459,14 @@ export class Room extends BaseRoom { this._roomStateHandler.handleRoomState(this, event, txn, log); }); } + + /** local room state observers, run during after sync step */ + _emitSyncRoomState(roomResponse) { + iterateResponseStateEvents(roomResponse, event => { + for (const handler of this._roomStateObservers) { + handler.handleStateEvent(event); } - } + }); } /** @package */ diff --git a/src/matrix/room/common.ts b/src/matrix/room/common.ts index 38070925..7556cfb0 100644 --- a/src/matrix/room/common.ts +++ b/src/matrix/room/common.ts @@ -47,9 +47,6 @@ export enum RoomType { Public } -export interface RoomStateHandler { - handleRoomState(room: Room, stateEvent: StateEvent, txn: Transaction, log: ILogItem); - updateRoomMembers(room: Room, memberChanges: Map); type RoomResponse = { state?: { events?: Array diff --git a/src/matrix/room/state/ObservedStateKeyValue.ts b/src/matrix/room/state/ObservedStateKeyValue.ts new file mode 100644 index 00000000..41cc3c7b --- /dev/null +++ b/src/matrix/room/state/ObservedStateKeyValue.ts @@ -0,0 +1,55 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import type {StateObserver} from "./types"; +import type {StateEvent} from "../../storage/types"; +import type {Transaction} from "../../storage/idb/Transaction"; +import {BaseObservableValue} from "../../../observable/value/BaseObservableValue"; + +/** + * Observable value for a state event with a given type and state key. + * Unsubscribes when last subscription is removed */ +export class ObservedStateKeyValue extends BaseObservableValue implements StateObserver { + private event?: StateEvent; + private removeCallback?: () => void; + + constructor(private readonly type: string, private readonly stateKey: string) { + super(); + } + /** @internal */ + async load(roomId: string, txn: Transaction): Promise { + this.event = (await txn.roomState.get(roomId, this.type, this.stateKey))?.event; + } + /** @internal */ + handleStateEvent(event: StateEvent) { + if (event.type === this.type && event.state_key === this.stateKey) { + this.event = event; + this.emit(this.get()); + } + } + + get(): StateEvent | undefined { + return this.event; + } + + setRemoveCallback(callback: () => void) { + this.removeCallback = callback; + } + + onUnsubscribeLast() { + this.removeCallback?.(); + } +} diff --git a/src/matrix/room/state/ObservedStateTypeMap.ts b/src/matrix/room/state/ObservedStateTypeMap.ts new file mode 100644 index 00000000..e8fa6f7b --- /dev/null +++ b/src/matrix/room/state/ObservedStateTypeMap.ts @@ -0,0 +1,53 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import type {StateObserver} from "./types"; +import type {StateEvent} from "../../storage/types"; +import type {Transaction} from "../../storage/idb/Transaction"; +import {ObservableMap} from "../../../observable/map/ObservableMap"; + +/** + * Observable map for a given type with state keys as map keys. + * Unsubscribes when last subscription is removed */ +export class ObservedStateTypeMap extends ObservableMap implements StateObserver { + private removeCallback?: () => void; + + constructor(private readonly type: string) { + super(); + } + /** @internal */ + async load(roomId: string, txn: Transaction): Promise { + const events = await txn.roomState.getAllForType(roomId, this.type); + for (let i = 0; i < events.length; ++i) { + const {event} = events[i]; + this.add(event.state_key, event); + } + } + /** @internal */ + handleStateEvent(event: StateEvent) { + if (event.type === this.type) { + this.set(event.state_key, event); + } + } + + setRemoveCallback(callback: () => void) { + this.removeCallback = callback; + } + + onUnsubscribeLast() { + this.removeCallback?.(); + } +} diff --git a/src/matrix/RoomStateHandlerSet.ts b/src/matrix/room/state/RoomStateHandlerSet.ts similarity index 75% rename from src/matrix/RoomStateHandlerSet.ts rename to src/matrix/room/state/RoomStateHandlerSet.ts index cf202097..986cb0f9 100644 --- a/src/matrix/RoomStateHandlerSet.ts +++ b/src/matrix/room/state/RoomStateHandlerSet.ts @@ -14,13 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -import type {ILogItem} from "../logging/types"; -import type {StateEvent} from "./storage/types"; -import type {Transaction} from "./storage/idb/Transaction"; -import type {Room} from "./room/Room"; -import type {MemberChange} from "./room/members/RoomMember"; -import type {RoomStateHandler} from "./room/common"; -import {BaseObservable} from "../observable/BaseObservable"; +import type {ILogItem} from "../../../logging/types"; +import type {StateEvent} from "../../storage/types"; +import type {Transaction} from "../../storage/idb/Transaction"; +import type {Room} from "../Room"; +import type {MemberChange} from "../members/RoomMember"; +import type {RoomStateHandler} from "./types"; +import {BaseObservable} from "../../../observable/BaseObservable"; /** keeps track of all handlers registered with Session.observeRoomState */ export class RoomStateHandlerSet extends BaseObservable implements RoomStateHandler { diff --git a/src/matrix/room/state/types.ts b/src/matrix/room/state/types.ts new file mode 100644 index 00000000..ef99c727 --- /dev/null +++ b/src/matrix/room/state/types.ts @@ -0,0 +1,38 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import type {Room} from "../Room"; +import type {StateEvent} from "../../storage/types"; +import type {Transaction} from "../../storage/idb/Transaction"; +import type {ILogItem} from "../../../logging/types"; +import type {MemberChange} from "../members/RoomMember"; + +/** used for Session.observeRoomState, which observes in all room, but without loading from storage + * It receives the sync write transaction, so other stores can be updated as part of the same transaction. */ +export interface RoomStateHandler { + handleRoomState(room: Room, stateEvent: StateEvent, syncWriteTxn: Transaction, log: ILogItem); + updateRoomMembers(room: Room, memberChanges: Map); +} + +/** + * used for Room.observeStateType and Room.observeStateTypeAndKey + * @internal + * */ +export interface StateObserver { + handleStateEvent(event: StateEvent); + load(roomId: string, txn: Transaction): Promise; + setRemoveCallback(callback: () => void); +} From 6225574df61650e3d792c9640b77e18e0af6ddac Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 12 May 2022 17:52:17 +0200 Subject: [PATCH 135/435] write test for ObservedStateKeyValue --- .../room/state/ObservedStateKeyValue.ts | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/src/matrix/room/state/ObservedStateKeyValue.ts b/src/matrix/room/state/ObservedStateKeyValue.ts index 41cc3c7b..ce380458 100644 --- a/src/matrix/room/state/ObservedStateKeyValue.ts +++ b/src/matrix/room/state/ObservedStateKeyValue.ts @@ -53,3 +53,52 @@ export class ObservedStateKeyValue extends BaseObservableValue { + const storage = await createMockStorage(); + const writeTxn = await storage.readWriteTxn([storage.storeNames.roomState]); + writeTxn.roomState.set("!abc", { + event_id: "$abc", + type: "m.room.member", + state_key: "@alice", + sender: "@alice", + origin_server_ts: 5, + content: {} + }); + await writeTxn.complete(); + const txn = await storage.readTxn([storage.storeNames.roomState]); + const value = new ObservedStateKeyValue("m.room.member", "@alice"); + await value.load("!abc", txn); + const updates: Array = []; + assert.strictEqual(value.get()?.origin_server_ts, 5); + const unsubscribe = value.subscribe(value => updates.push(value)); + value.handleStateEvent({ + event_id: "$abc", + type: "m.room.member", + state_key: "@bob", + sender: "@alice", + origin_server_ts: 10, + content: {} + }); + assert.strictEqual(updates.length, 0); + value.handleStateEvent({ + event_id: "$abc", + type: "m.room.member", + state_key: "@alice", + sender: "@alice", + origin_server_ts: 10, + content: {} + }); + assert.strictEqual(updates.length, 1); + assert.strictEqual(updates[0]?.origin_server_ts, 10); + let removed = false; + value.setRemoveCallback(() => removed = true); + unsubscribe(); + assert(removed); + } + } +} From b2d787b96c4c5fcf41d912b94389faeb6ae0f250 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 17 May 2022 15:55:15 +0200 Subject: [PATCH 136/435] fix wrong extension in import --- src/matrix/room/Room.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js index 796474d3..38982786 100644 --- a/src/matrix/room/Room.js +++ b/src/matrix/room/Room.js @@ -23,7 +23,7 @@ import {WrappedError} from "../error.js" import {Heroes} from "./members/Heroes.js"; import {AttachmentUpload} from "./AttachmentUpload.js"; import {DecryptionSource} from "../e2ee/common.js"; -import {iterateResponseStateEvents} from "./common.js"; +import {iterateResponseStateEvents} from "./common"; import {PowerLevels, EVENT_TYPE as POWERLEVELS_EVENT_TYPE } from "./PowerLevels.js"; const EVENT_ENCRYPTED_TYPE = "m.room.encrypted"; From 6f0ebeacb7eb3def10caa4c83fce8088a18c884e Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 1 Jun 2022 15:27:00 +0200 Subject: [PATCH 137/435] fetch single device key in DeviceTracker --- src/matrix/e2ee/DeviceTracker.js | 52 ++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/matrix/e2ee/DeviceTracker.js b/src/matrix/e2ee/DeviceTracker.js index f8c3bca8..49d9ffab 100644 --- a/src/matrix/e2ee/DeviceTracker.js +++ b/src/matrix/e2ee/DeviceTracker.js @@ -309,6 +309,7 @@ export class DeviceTracker { return await this._devicesForUserIds(roomId, userIds, txn, hsApi, log); } + /** gets devices for the given user ids that are in the given room */ async devicesForRoomMembers(roomId, userIds, hsApi, log) { const txn = await this._storage.readTxn([ this._storage.storeNames.userIdentities, @@ -316,6 +317,57 @@ export class DeviceTracker { return await this._devicesForUserIds(roomId, userIds, txn, hsApi, log); } + /** gets a single device */ + async deviceForId(userId, deviceId, hsApi, log) { + const txn = await this._storage.readTxn([ + this._storage.storeNames.deviceIdentities, + ]); + let device = await txn.deviceIdentities.get(userId, deviceId); + if (device) { + log.set("existingDevice", true); + } else { + //// BEGIN EXTRACT (deviceKeysMap) + const deviceKeyResponse = await hsApi.queryKeys({ + "timeout": 10000, + "device_keys": { + [userId]: [deviceId] + }, + "token": this._getSyncToken() + }, {log}).response(); + // verify signature + const verifiedKeysPerUser = log.wrap("verify", log => this._filterVerifiedDeviceKeys(deviceKeyResponse["device_keys"], log)); + //// END EXTRACT + + // there should only be one device in here, but still check the HS sends us the right one + const verifiedKeys = verifiedKeysPerUser + .find(vkpu => vkpu.userId === userId).verifiedKeys + .find(vk => vk["device_id"] === deviceId); + device = deviceKeysAsDeviceIdentity(verifiedKeys); + const txn = await this._storage.readWriteTxn([ + this._storage.storeNames.deviceIdentities, + ]); + // check again we don't have the device already. + // when updating all keys for a user we allow updating the + // device when the key hasn't changed so the device display name + // can be updated, but here we don't. + const existingDevice = await txn.deviceIdentities.get(userId, deviceId); + if (existingDevice) { + device = existingDevice; + log.set("existingDeviceAfterFetch", true); + } else { + try { + txn.deviceIdentities.set(device); + log.set("newDevice", true); + } catch (err) { + txn.abort(); + throw err; + } + await txn.complete(); + } + } + return device; + } + /** * @param {string} roomId [description] * @param {Array} userIds a set of user ids to try and find the identity for. Will be check to belong to roomId. From 50ae51e8937892ca838f3f2160c6114d14a8462b Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 1 Jun 2022 15:28:49 +0200 Subject: [PATCH 138/435] encrypt call signalling message only for given device --- src/matrix/Session.js | 16 ++++++++-------- src/matrix/calls/group/GroupCall.ts | 6 +++--- src/matrix/calls/group/Member.ts | 24 ++++++++++++------------ 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index cd676fc5..26ab1702 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -79,18 +79,18 @@ export class Session { this._callHandler = new CallHandler({ clock: this._platform.clock, hsApi: this._hsApi, - encryptDeviceMessage: async (roomId, userId, message, log) => { + encryptDeviceMessage: async (roomId, userId, deviceId, message, log) => { if (!this._deviceTracker || !this._olmEncryption) { throw new Error("encryption is not enabled"); } - // TODO: just get the devices we're sending the message to, not all the room devices - // although we probably already fetched all devices to send messages in the likely e2ee room - const devices = await log.wrap("get device keys", async log => { - await this._deviceTracker.trackRoom(this.rooms.get(roomId), log); - return this._deviceTracker.devicesForRoomMembers(roomId, [userId], this._hsApi, log); + const device = await log.wrap("get device key", async log => { + return this._deviceTracker.deviceForId(userId, deviceId, this._hsApi, log); }); - const encryptedMessage = await this._olmEncryption.encrypt(message.type, message.content, devices, this._hsApi, log); - return encryptedMessage; + if (!device) { + throw new Error(`Could not find device key ${deviceId} for ${userId} in ${roomId}`); + } + const encryptedMessages = await this._olmEncryption.encrypt(message.type, message.content, [device], this._hsApi, log); + return encryptedMessages; }, storage: this._storage, webRTC: this._platform.webRTC, diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index af3966a0..b2b52e9b 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -55,7 +55,7 @@ function getDeviceFromMemberKey(key: string): string { export type Options = Omit & { emitUpdate: (call: GroupCall, params?: any) => void; - encryptDeviceMessage: (roomId: string, userId: string, message: SignallingMessage, log: ILogItem) => Promise, + encryptDeviceMessage: (roomId: string, userId: string, deviceId: string, message: SignallingMessage, log: ILogItem) => Promise, storage: Storage, logger: ILogger, }; @@ -93,8 +93,8 @@ export class GroupCall extends EventEmitter<{change: never}> { this._memberOptions = Object.assign({}, options, { confId: this.id, emitUpdate: member => this._members.update(getMemberKey(member.userId, member.deviceId), member), - encryptDeviceMessage: (userId: string, message: SignallingMessage, log) => { - return this.options.encryptDeviceMessage(this.roomId, userId, message, log); + encryptDeviceMessage: (userId: string, deviceId: string, message: SignallingMessage, log) => { + return this.options.encryptDeviceMessage(this.roomId, userId, deviceId, message, log); } }); } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 69e1eeea..e3abc49e 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -36,7 +36,7 @@ export type Options = Omit, log: ILogItem) => Promise, + encryptDeviceMessage: (userId: string, deviceId: string, message: SignallingMessage, log: ILogItem) => Promise, emitUpdate: (participant: Member, params?: any) => void, } @@ -217,20 +217,20 @@ export class Member { groupMessage.content.party_id = this.options.ownDeviceId; groupMessage.content.sender_session_id = this.options.sessionId; groupMessage.content.dest_session_id = this.sessionId; - // const encryptedMessages = await this.options.encryptDeviceMessage(this.member.userId, groupMessage, log); - // const payload = formatToDeviceMessagesPayload(encryptedMessages); - const payload = { - messages: { - [this.member.userId]: { - [this.deviceId]: groupMessage.content - } - } - }; + let payload; + let type: string = message.type; + const encryptedMessages = await this.options.encryptDeviceMessage(this.member.userId, this.deviceId, groupMessage, log); + if (encryptedMessages) { + payload = formatToDeviceMessagesPayload(encryptedMessages); + type = "m.room.encrypted"; + } else { + // device needs deviceId and userId + payload = formatToDeviceMessagesPayload([{content: groupMessage.content, device: this}]); + } // TODO: remove this for release log.set("payload", groupMessage.content); const request = this.options.hsApi.sendToDevice( - message.type, - //"m.room.encrypted", + type, payload, makeTxnId(), {log} From 9efe294a79bdf50e6fd7ba1ffa0a32c23fb58d3b Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 1 Jun 2022 15:29:24 +0200 Subject: [PATCH 139/435] fetch and verify keys on olm call signalling message --- src/matrix/DeviceMessageHandler.js | 62 ++++++++++++++---------------- src/matrix/Session.js | 11 +++++- src/matrix/Sync.js | 2 +- src/matrix/e2ee/README.md | 3 ++ 4 files changed, 41 insertions(+), 37 deletions(-) diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index c33236aa..507e07ef 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -38,7 +38,6 @@ export class DeviceMessageHandler { async prepareSync(toDeviceEvents, lock, txn, log) { log.set("messageTypes", countBy(toDeviceEvents, e => e.type)); - this._handleUnencryptedCallEvents(toDeviceEvents, log); const encryptedEvents = toDeviceEvents.filter(e => e.type === "m.room.encrypted"); if (!this._olmDecryption) { log.log("can't decrypt, encryption not enabled", log.level.Warn); @@ -54,20 +53,6 @@ export class DeviceMessageHandler { } const newRoomKeys = this._megolmDecryption.roomKeysFromDeviceMessages(olmDecryptChanges.results, log); - // const callMessages = olmDecryptChanges.results.filter(dr => this._callHandler.handlesDeviceMessageEventType(dr.event?.type)); - // // load devices by sender key - // await Promise.all(callMessages.map(async dr => { - // dr.setDevice(await this._getDevice(dr.senderCurve25519Key, txn)); - // })); - // // TODO: pass this in the prep and run it in afterSync or afterSyncComplete (as callHandler can send events as well)? - // for (const dr of callMessages) { - // if (dr.device) { - // this._callHandler.handleDeviceMessage(dr.event, dr.device.userId, dr.device.deviceId, log); - // } else { - // console.error("could not deliver message because don't have device for sender key", dr.event); - // } - // } - // TODO: somehow include rooms that received a call to_device message in the sync state? // or have updates flow through event emitter? // well, we don't really need to update the room other then when a call starts or stops @@ -76,33 +61,42 @@ export class DeviceMessageHandler { } } - _handleUnencryptedCallEvents(toDeviceEvents, log) { - const callMessages = toDeviceEvents.filter(e => this._callHandler.handlesDeviceMessageEventType(e.type)); - for (const event of callMessages) { - const userId = event.sender; - const deviceId = event.content.device_id; - this._callHandler.handleDeviceMessage(event, userId, deviceId, log); - } - } - /** check that prep is not undefined before calling this */ async writeSync(prep, txn) { // write olm changes prep.olmDecryptChanges.write(txn); const didWriteValues = await Promise.all(prep.newRoomKeys.map(key => this._megolmDecryption.writeRoomKey(key, txn))); - return didWriteValues.some(didWrite => !!didWrite); + const hasNewRoomKeys = didWriteValues.some(didWrite => !!didWrite); + return { + hasNewRoomKeys, + decryptionResults: prep.olmDecryptChanges.results + }; } - - async _getDevice(senderKey, txn) { - let device = this._senderDeviceCache.get(senderKey); - if (!device) { - device = await txn.deviceIdentities.getByCurve25519Key(senderKey); - if (device) { - this._senderDeviceCache.set(device); - } + async afterSyncCompleted(decryptionResults, deviceTracker, hsApi, log) { + // if we don't have a device, we need to fetch the device keys the message claims + // and check the keys, and we should only do network requests during + // sync processing in the afterSyncCompleted step. + const callMessages = decryptionResults.filter(dr => this._callHandler.handlesDeviceMessageEventType(dr.event?.type)); + if (callMessages.length) { + await log.wrap("process call signalling messages", async log => { + for (const dr of callMessages) { + // serialize device loading, so subsequent messages for the same device take advantage of the cache + const device = await deviceTracker.deviceForId(dr.event.sender, dr.event.content.device_id, hsApi, log); + dr.setDevice(device); + if (dr.isVerified) { + this._callHandler.handleDeviceMessage(dr.event, dr.userId, dr.deviceId, log); + } else { + log.log({ + l: "could not verify olm fingerprint key matches, ignoring", + ed25519Key: dr.device.ed25519Key, + claimedEd25519Key: dr.claimedEd25519Key, + deviceId, userId, + }); + } + } + }); } - return device; } } diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 26ab1702..c80cf527 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -685,7 +685,9 @@ export class Session { async writeSync(syncResponse, syncFilterId, preparation, txn, log) { const changes = { syncInfo: null, - e2eeAccountChanges: null + e2eeAccountChanges: null, + hasNewRoomKeys: false, + deviceMessageDecryptionResults: null, }; const syncToken = syncResponse.next_batch; if (syncToken !== this.syncToken) { @@ -706,7 +708,9 @@ export class Session { } if (preparation) { - changes.hasNewRoomKeys = await log.wrap("deviceMsgs", log => this._deviceMessageHandler.writeSync(preparation, txn, log)); + const {hasNewRoomKeys, decryptionResults} = await log.wrap("deviceMsgs", log => this._deviceMessageHandler.writeSync(preparation, txn, log)); + changes.hasNewRoomKeys = hasNewRoomKeys; + changes.deviceMessageDecryptionResults = decryptionResults; } // store account data @@ -747,6 +751,9 @@ export class Session { if (changes.hasNewRoomKeys) { this._keyBackup.get()?.flush(log); } + if (changes.deviceMessageDecryptionResults) { + await this._deviceMessageHandler.afterSyncCompleted(changes.deviceMessageDecryptionResults, this._deviceTracker, this._hsApi, log); + } } _tryReplaceRoomBeingCreated(roomId, log) { diff --git a/src/matrix/Sync.js b/src/matrix/Sync.js index 4f907563..f77fa79a 100644 --- a/src/matrix/Sync.js +++ b/src/matrix/Sync.js @@ -160,7 +160,7 @@ export class Sync { const isCatchupSync = this._status.get() === SyncStatus.CatchupSync; const sessionPromise = (async () => { try { - await log.wrap("session", log => this._session.afterSyncCompleted(sessionChanges, isCatchupSync, log), log.level.Detail); + await log.wrap("session", log => this._session.afterSyncCompleted(sessionChanges, isCatchupSync, log)); } catch (err) {} // error is logged, but don't fail sessionPromise })(); diff --git a/src/matrix/e2ee/README.md b/src/matrix/e2ee/README.md index fab53880..fdb4866c 100644 --- a/src/matrix/e2ee/README.md +++ b/src/matrix/e2ee/README.md @@ -41,5 +41,8 @@ Runs before any room.prepareSync, so the new room keys can be passed to each roo - e2ee account - generate more otks if needed - upload new otks if needed or device keys if not uploaded before + - device message handler: + - fetch keys we don't know about yet for (call) to_device messages identity + - pass signalling messages to call handler - rooms - share new room keys if needed From 3edfbd2cf6f468b90e198079f2aefa4670c731f4 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 1 Jun 2022 15:30:25 +0200 Subject: [PATCH 140/435] await hangup here, so log doesn't terminate early --- src/matrix/calls/group/Member.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index e3abc49e..6a8158b6 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -157,7 +157,7 @@ export class Member { connection.logItem.wrap("disconnect", async log => { disconnectLogItem = log; if (hangup) { - connection.peerCall?.hangup(CallErrorCode.UserHangup, log); + await connection.peerCall?.hangup(CallErrorCode.UserHangup, log); } else { await connection.peerCall?.close(undefined, log); } From 83eef2be9d76417c47e54cc2ac61768b7c7624ba Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 1 Jun 2022 15:30:41 +0200 Subject: [PATCH 141/435] log lack of persisted storage in ... persisted logs! --- src/matrix/storage/idb/StorageFactory.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/storage/idb/StorageFactory.ts b/src/matrix/storage/idb/StorageFactory.ts index 5cb1b6e5..264d8670 100644 --- a/src/matrix/storage/idb/StorageFactory.ts +++ b/src/matrix/storage/idb/StorageFactory.ts @@ -67,7 +67,7 @@ export class StorageFactory { requestPersistedStorage().then(persisted => { // Firefox lies here though, and returns true even if the user denied the request if (!persisted) { - console.warn("no persisted storage, database can be evicted by browser"); + log.log("no persisted storage, database can be evicted by browser", log.level.Warn); } }); From a014740e72d3343c616a39e2a8d03537fa518b07 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 1 Jun 2022 15:55:43 +0200 Subject: [PATCH 142/435] don't throw when we can't encrypt, just fall back to sending unencrypted --- src/matrix/Session.js | 16 ++++++++++------ src/matrix/calls/group/GroupCall.ts | 2 +- src/matrix/calls/group/Member.ts | 2 +- src/matrix/e2ee/DeviceTracker.js | 5 ++++- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index c80cf527..f82ad555 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -81,16 +81,20 @@ export class Session { hsApi: this._hsApi, encryptDeviceMessage: async (roomId, userId, deviceId, message, log) => { if (!this._deviceTracker || !this._olmEncryption) { - throw new Error("encryption is not enabled"); + log.set("encryption_disabled", true); + return; } const device = await log.wrap("get device key", async log => { - return this._deviceTracker.deviceForId(userId, deviceId, this._hsApi, log); + const device = this._deviceTracker.deviceForId(userId, deviceId, this._hsApi, log); + if (!device) { + log.set("not_found", true); + } + return device; }); - if (!device) { - throw new Error(`Could not find device key ${deviceId} for ${userId} in ${roomId}`); + if (device) { + const encryptedMessages = await this._olmEncryption.encrypt(message.type, message.content, [device], this._hsApi, log); + return encryptedMessages; } - const encryptedMessages = await this._olmEncryption.encrypt(message.type, message.content, [device], this._hsApi, log); - return encryptedMessages; }, storage: this._storage, webRTC: this._platform.webRTC, diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index b2b52e9b..ddc0b517 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -55,7 +55,7 @@ function getDeviceFromMemberKey(key: string): string { export type Options = Omit & { emitUpdate: (call: GroupCall, params?: any) => void; - encryptDeviceMessage: (roomId: string, userId: string, deviceId: string, message: SignallingMessage, log: ILogItem) => Promise, + encryptDeviceMessage: (roomId: string, userId: string, deviceId: string, message: SignallingMessage, log: ILogItem) => Promise, storage: Storage, logger: ILogger, }; diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 6a8158b6..4c30a6cf 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -36,7 +36,7 @@ export type Options = Omit, log: ILogItem) => Promise, + encryptDeviceMessage: (userId: string, deviceId: string, message: SignallingMessage, log: ILogItem) => Promise, emitUpdate: (participant: Member, params?: any) => void, } diff --git a/src/matrix/e2ee/DeviceTracker.js b/src/matrix/e2ee/DeviceTracker.js index 49d9ffab..0d0b0e8b 100644 --- a/src/matrix/e2ee/DeviceTracker.js +++ b/src/matrix/e2ee/DeviceTracker.js @@ -338,10 +338,13 @@ export class DeviceTracker { const verifiedKeysPerUser = log.wrap("verify", log => this._filterVerifiedDeviceKeys(deviceKeyResponse["device_keys"], log)); //// END EXTRACT - // there should only be one device in here, but still check the HS sends us the right one const verifiedKeys = verifiedKeysPerUser .find(vkpu => vkpu.userId === userId).verifiedKeys .find(vk => vk["device_id"] === deviceId); + // user hasn't uploaded keys for device? + if (!verifiedKeys) { + return undefined; + } device = deviceKeysAsDeviceIdentity(verifiedKeys); const txn = await this._storage.readWriteTxn([ this._storage.storeNames.deviceIdentities, From a139571e201271b19944369eafec8c41d25e6751 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Jun 2022 10:58:04 +0200 Subject: [PATCH 143/435] move setting seq on outbound messages to member, is specific to_device --- src/matrix/calls/PeerCall.ts | 9 --------- src/matrix/calls/callEventTypes.ts | 2 +- src/matrix/calls/group/Member.ts | 2 ++ 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index e6ce3571..ea7e9d06 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -71,7 +71,6 @@ export class PeerCall implements IDisposable { private localMedia?: LocalMedia; private localMuteSettings?: MuteSettings; // TODO: this should go in member - private seq: number = 0; // A queue for candidates waiting to go out. // We try to amalgamate candidates into a single candidate message where // possible @@ -238,7 +237,6 @@ export class PeerCall implements IDisposable { const content: MCallSDPStreamMetadataChanged = { call_id: this.callId, version: 1, - seq: this.seq++, [SDPStreamMetadataKey]: this.getSDPMetadata() }; await this.sendSignallingMessage({type: EventType.SDPStreamMetadataChangedPrefix, content}, log); @@ -263,7 +261,6 @@ export class PeerCall implements IDisposable { const content: MCallSDPStreamMetadataChanged = { call_id: this.callId, version: 1, - seq: this.seq++, [SDPStreamMetadataKey]: this.getSDPMetadata() }; await this.sendSignallingMessage({type: EventType.SDPStreamMetadataChangedPrefix, content}, log); @@ -350,7 +347,6 @@ export class PeerCall implements IDisposable { const content = { call_id: callId, version: 1, - seq: this.seq++, }; // TODO: Don't send UserHangup reason to older clients if (reason) { @@ -398,7 +394,6 @@ export class PeerCall implements IDisposable { offer, [SDPStreamMetadataKey]: this.getSDPMetadata(), version: 1, - seq: this.seq++, lifetime: CALL_TIMEOUT_MS }; await this.sendSignallingMessage({type: EventType.Invite, content}, log); @@ -409,7 +404,6 @@ export class PeerCall implements IDisposable { description: offer, [SDPStreamMetadataKey]: this.getSDPMetadata(), version: 1, - seq: this.seq++, lifetime: CALL_TIMEOUT_MS }; await this.sendSignallingMessage({type: EventType.Negotiate, content}, log); @@ -674,7 +668,6 @@ export class PeerCall implements IDisposable { description: this.peerConnection.localDescription!, [SDPStreamMetadataKey]: this.getSDPMetadata(), version: 1, - seq: this.seq++, lifetime: CALL_TIMEOUT_MS }; await this.sendSignallingMessage({type: EventType.Negotiate, content}, log); @@ -689,7 +682,6 @@ export class PeerCall implements IDisposable { const answerContent: MCallAnswer = { call_id: this.callId, version: 1, - seq: this.seq++, answer: { sdp: localDescription.sdp, type: localDescription.type, @@ -755,7 +747,6 @@ export class PeerCall implements IDisposable { content: { call_id: this.callId, version: 1, - seq: this.seq++, candidates }, }, log); diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index 0490b44f..09376c85 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -65,7 +65,6 @@ export interface CallReplacesTarget { export type MCallBase = { call_id: string; version: string | number; - seq: number; } export type MGroupCallBase = MCallBase & { @@ -74,6 +73,7 @@ export type MGroupCallBase = MCallBase & { sender_session_id: string; dest_session_id: string; party_id: string; // Should not need this? + seq: number; } export type MCallAnswer = Base & { diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 4c30a6cf..90765569 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -53,6 +53,7 @@ const errorCodesWithoutRetry = [ class MemberConnection { public retryCount: number = 0; public peerCall?: PeerCall; + public outboundSeqCounter: number = 0; constructor( public localMedia: LocalMedia, @@ -212,6 +213,7 @@ export class Member { /** @internal */ sendSignallingMessage = async (message: SignallingMessage, log: ILogItem): Promise => { const groupMessage = message as SignallingMessage; + groupMessage.content.seq = ++this.connection!.outboundSeqCounter; groupMessage.content.conf_id = this.options.confId; groupMessage.content.device_id = this.options.ownDeviceId; groupMessage.content.party_id = this.options.ownDeviceId; From 513c05945988aabf5835fa0bdde2da9b96caad09 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Jun 2022 10:59:14 +0200 Subject: [PATCH 144/435] buffer messages as long as seq numbers in between haven't been received --- src/matrix/calls/group/Member.ts | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 90765569..a40100d2 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -18,7 +18,7 @@ import {PeerCall, CallState} from "../PeerCall"; import {makeTxnId, makeId} from "../../common"; import {EventType, CallErrorCode} from "../callEventTypes"; import {formatToDeviceMessagesPayload} from "../../common"; - +import {sortedIndex} from "../../../utils/sortedIndex"; import type {MuteSettings} from "../common"; import type {Options as PeerCallOptions, RemoteMedia} from "../PeerCall"; import type {LocalMedia} from "../LocalMedia"; @@ -53,6 +53,8 @@ const errorCodesWithoutRetry = [ class MemberConnection { public retryCount: number = 0; public peerCall?: PeerCall; + public lastProcessedSeqNr: number | undefined; + public queuedSignallingMessages: SignallingMessage[] = []; public outboundSeqCounter: number = 0; constructor( @@ -253,11 +255,19 @@ export class Member { if (message.type === EventType.Invite && !connection.peerCall) { connection.peerCall = this._createPeerCall(message.content.call_id); } + const idx = sortedIndex(connection.queuedSignallingMessages, message, (a, b) => a.content.seq - b.content.seq); + connection.queuedSignallingMessages.splice(idx, 0, message); if (connection.peerCall) { - const item = connection.peerCall.handleIncomingSignallingMessage(message, this.deviceId, connection.logItem); - syncLog.refDetached(item); - } else { - // TODO: need to buffer events until invite comes? + while ( + connection.queuedSignallingMessages.length && ( + connection.lastProcessedSeqNr === undefined || + connection.queuedSignallingMessages[0].content.seq === connection.lastProcessedSeqNr + 1 + )) { + const dequeuedMessage = connection.queuedSignallingMessages.shift()!; + const item = connection.peerCall!.handleIncomingSignallingMessage(dequeuedMessage, this.deviceId, connection.logItem); + connection.lastProcessedSeqNr = dequeuedMessage.content.seq; + syncLog.refDetached(item); + } } } else { syncLog.log({l: "member not connected", userId: this.userId, deviceId: this.deviceId}); From a530944f7d9023c16f40883be88ad6e68489ff75 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Jun 2022 11:11:32 +0200 Subject: [PATCH 145/435] add logging to seq queueing --- src/matrix/calls/group/Member.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index a40100d2..541cdf13 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -257,18 +257,26 @@ export class Member { } const idx = sortedIndex(connection.queuedSignallingMessages, message, (a, b) => a.content.seq - b.content.seq); connection.queuedSignallingMessages.splice(idx, 0, message); + let hasBeenDequeued = false; if (connection.peerCall) { while ( connection.queuedSignallingMessages.length && ( - connection.lastProcessedSeqNr === undefined || - connection.queuedSignallingMessages[0].content.seq === connection.lastProcessedSeqNr + 1 - )) { + connection.lastProcessedSeqNr === undefined || + connection.queuedSignallingMessages[0].content.seq === connection.lastProcessedSeqNr + 1 + ) + ) { const dequeuedMessage = connection.queuedSignallingMessages.shift()!; + if (dequeuedMessage === message) { + hasBeenDequeued = true; + } const item = connection.peerCall!.handleIncomingSignallingMessage(dequeuedMessage, this.deviceId, connection.logItem); - connection.lastProcessedSeqNr = dequeuedMessage.content.seq; syncLog.refDetached(item); + connection.lastProcessedSeqNr = dequeuedMessage.content.seq; } } + if (!hasBeenDequeued) { + syncLog.refDetached(connection.logItem.log({l: "queued signalling message", type: message.type})); + } } else { syncLog.log({l: "member not connected", userId: this.userId, deviceId: this.deviceId}); } From a52740ed1b4fd7fcacc8bd2f9a9c78ceb17b3d9f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Jun 2022 15:55:08 +0200 Subject: [PATCH 146/435] give room state handler access to member sync to get sender profile info --- src/matrix/room/Room.js | 14 ++++++++------ src/matrix/room/state/RoomStateHandlerSet.ts | 7 +++++-- src/matrix/room/state/types.ts | 5 +++-- .../room/timeline/persistence/MemberWriter.js | 6 +++++- src/matrix/room/timeline/persistence/SyncWriter.js | 2 +- 5 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js index 38982786..425401d8 100644 --- a/src/matrix/room/Room.js +++ b/src/matrix/room/Room.js @@ -123,7 +123,7 @@ export class Room extends BaseRoom { txn.roomState.removeAllForRoom(this.id); txn.roomMembers.removeAllForRoom(this.id); } - const {entries: newEntries, updatedEntries, newLiveKey, memberChanges} = + const {entries: newEntries, updatedEntries, newLiveKey, memberChanges, memberSync} = await log.wrap("syncWriter", log => this._syncWriter.writeSync( roomResponse, isRejoin, summaryChanges.hasFetchedMembers, txn, log), log.level.Detail); if (decryptChanges) { @@ -180,7 +180,7 @@ export class Room extends BaseRoom { removedPendingEvents = await this._sendQueue.removeRemoteEchos(roomResponse.timeline.events, txn, log); } const powerLevelsEvent = this._getPowerLevelsEvent(roomResponse); - this._runRoomStateHandlers(roomResponse, txn, log); + await this._runRoomStateHandlers(roomResponse, memberSync, txn, log); return { roomResponse, summaryChanges, @@ -453,14 +453,16 @@ export class Room extends BaseRoom { return this._sendQueue.pendingEvents; } - /** global room state handlers, run during write sync step */ - _runRoomStateHandlers(roomResponse, txn, log) { + /** global room state handlers, run during writeSync step */ + _runRoomStateHandlers(roomResponse, memberSync, txn, log) { + const promises = []; iterateResponseStateEvents(roomResponse, event => { - this._roomStateHandler.handleRoomState(this, event, txn, log); + promises.push(this._roomStateHandler.handleRoomState(this, event, memberSync, txn, log)); }); + return Promise.all(promises); } - /** local room state observers, run during after sync step */ + /** local room state observers, run during afterSync step */ _emitSyncRoomState(roomResponse) { iterateResponseStateEvents(roomResponse, event => { for (const handler of this._roomStateObservers) { diff --git a/src/matrix/room/state/RoomStateHandlerSet.ts b/src/matrix/room/state/RoomStateHandlerSet.ts index 986cb0f9..491b02ce 100644 --- a/src/matrix/room/state/RoomStateHandlerSet.ts +++ b/src/matrix/room/state/RoomStateHandlerSet.ts @@ -20,14 +20,17 @@ import type {Transaction} from "../../storage/idb/Transaction"; import type {Room} from "../Room"; import type {MemberChange} from "../members/RoomMember"; import type {RoomStateHandler} from "./types"; +import type {MemberSync} from "../timeline/persistence/MemberWriter.js"; import {BaseObservable} from "../../../observable/BaseObservable"; /** keeps track of all handlers registered with Session.observeRoomState */ export class RoomStateHandlerSet extends BaseObservable implements RoomStateHandler { - handleRoomState(room: Room, stateEvent: StateEvent, txn: Transaction, log: ILogItem) { + async handleRoomState(room: Room, stateEvent: StateEvent, memberSync: MemberSync, txn: Transaction, log: ILogItem): Promise { + const promises: Promise[] = []; for(let h of this._handlers) { - h.handleRoomState(room, stateEvent, txn, log); + promises.push(h.handleRoomState(room, stateEvent, memberSync, txn, log)); } + await Promise.all(promises); } updateRoomMembers(room: Room, memberChanges: Map) { for(let h of this._handlers) { diff --git a/src/matrix/room/state/types.ts b/src/matrix/room/state/types.ts index ef99c727..2e7167d2 100644 --- a/src/matrix/room/state/types.ts +++ b/src/matrix/room/state/types.ts @@ -19,12 +19,13 @@ import type {StateEvent} from "../../storage/types"; import type {Transaction} from "../../storage/idb/Transaction"; import type {ILogItem} from "../../../logging/types"; import type {MemberChange} from "../members/RoomMember"; +import type {MemberSync} from "../timeline/persistence/MemberWriter"; /** used for Session.observeRoomState, which observes in all room, but without loading from storage * It receives the sync write transaction, so other stores can be updated as part of the same transaction. */ export interface RoomStateHandler { - handleRoomState(room: Room, stateEvent: StateEvent, syncWriteTxn: Transaction, log: ILogItem); - updateRoomMembers(room: Room, memberChanges: Map); + handleRoomState(room: Room, stateEvent: StateEvent, memberSync: MemberSync, syncWriteTxn: Transaction, log: ILogItem): Promise; + updateRoomMembers(room: Room, memberChanges: Map): void; } /** diff --git a/src/matrix/room/timeline/persistence/MemberWriter.js b/src/matrix/room/timeline/persistence/MemberWriter.js index 1cdcb7d5..9345324e 100644 --- a/src/matrix/room/timeline/persistence/MemberWriter.js +++ b/src/matrix/room/timeline/persistence/MemberWriter.js @@ -56,7 +56,11 @@ export class MemberWriter { } } -class MemberSync { +/** Represents the member changes in a given sync. + * Used to write the changes to storage and historical member + * information for events in the same sync. + **/ +export class MemberSync { constructor(memberWriter, stateEvents, timelineEvents, hasFetchedMembers) { this._memberWriter = memberWriter; this._timelineEvents = timelineEvents; diff --git a/src/matrix/room/timeline/persistence/SyncWriter.js b/src/matrix/room/timeline/persistence/SyncWriter.js index dc5ae3a8..76c7bec7 100644 --- a/src/matrix/room/timeline/persistence/SyncWriter.js +++ b/src/matrix/room/timeline/persistence/SyncWriter.js @@ -244,7 +244,7 @@ export class SyncWriter { const {currentKey, entries, updatedEntries} = await this._writeTimeline(timelineEvents, timeline, memberSync, this._lastLiveKey, txn, log); const memberChanges = await memberSync.write(txn); - return {entries, updatedEntries, newLiveKey: currentKey, memberChanges}; + return {entries, updatedEntries, newLiveKey: currentKey, memberChanges, memberSync}; } afterSync(newLiveKey) { From 90b6a5ccb625066325b4a45e4ae3fc1c1fb3b026 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Jun 2022 15:56:23 +0200 Subject: [PATCH 147/435] update call member info with room member info --- src/domain/session/room/CallViewModel.ts | 5 +++- src/matrix/calls/CallHandler.ts | 38 ++++++++++++++++-------- src/matrix/calls/group/GroupCall.ts | 19 ++++++++++-- src/matrix/calls/group/Member.ts | 9 +++++- 4 files changed, 53 insertions(+), 18 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 08bc3691..a770ddbe 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -117,7 +117,10 @@ class OwnMemberViewModel extends ViewModel implements IStreamV } } -type MemberOptions = BaseOptions & {member: Member, mediaRepository: MediaRepository}; +type MemberOptions = BaseOptions & { + member: Member, + mediaRepository: MediaRepository +}; export class CallMemberViewModel extends ViewModel implements IStreamViewModel { get stream(): Stream | undefined { diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index e585bb40..7524200c 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -22,6 +22,7 @@ import {EventType, CallIntent} from "./callEventTypes"; import {GroupCall} from "./group/GroupCall"; import {makeId} from "../common"; import {CALL_LOG_TYPE} from "./common"; +import {EVENT_TYPE as MEMBER_EVENT_TYPE, RoomMember} from "../room/members/RoomMember"; import type {LocalMedia} from "./LocalMedia"; import type {Room} from "../room/Room"; @@ -36,6 +37,7 @@ import type {Transaction} from "../storage/idb/Transaction"; import type {CallEntry} from "../storage/idb/stores/CallStore"; import type {Clock} from "../../platform/web/dom/Clock"; import type {RoomStateHandler} from "../room/state/types"; +import type {MemberSync} from "../room/timeline/persistence/MemberWriter"; export type Options = Omit & { clock: Clock @@ -77,7 +79,7 @@ export class CallHandler implements RoomStateHandler { const names = this.options.storage.storeNames; const txn = await this.options.storage.readTxn([ names.calls, - names.roomState + names.roomState, ]); return txn; } @@ -97,15 +99,17 @@ export class CallHandler implements RoomStateHandler { })); const roomIds = Array.from(new Set(callEntries.map(e => e.roomId))); await Promise.all(roomIds.map(async roomId => { - // const ownCallsMemberEvent = await txn.roomState.get(roomId, EventType.GroupCallMember, this.options.ownUserId); - // if (ownCallsMemberEvent) { - // this.handleCallMemberEvent(ownCallsMemberEvent.event, log); - // } + // TODO: don't load all members until we need them const callsMemberEvents = await txn.roomState.getAllForType(roomId, EventType.GroupCallMember); - for (const entry of callsMemberEvents) { - this.handleCallMemberEvent(entry.event, roomId, log); - } - // TODO: we should be loading the other members as well at some point + await Promise.all(callsMemberEvents.map(async entry => { + const roomMemberState = await txn.roomState.get(roomId, MEMBER_EVENT_TYPE, entry.event.sender); + if (roomMemberState) { + const roomMember = RoomMember.fromMemberEvent(roomMemberState.event); + if (roomMember) { + this.handleCallMemberEvent(entry.event, roomMember, roomId, log); + } + } + })); })); log.set("newSize", this._calls.size); }); @@ -144,12 +148,15 @@ export class CallHandler implements RoomStateHandler { // TODO: check and poll turn server credentials here /** @internal */ - handleRoomState(room: Room, event: StateEvent, txn: Transaction, log: ILogItem) { + async handleRoomState(room: Room, event: StateEvent, memberSync: MemberSync, txn: Transaction, log: ILogItem) { if (event.type === EventType.GroupCall) { this.handleCallEvent(event, room.id, txn, log); } if (event.type === EventType.GroupCallMember) { - this.handleCallMemberEvent(event, room.id, log); + const member: RoomMember | undefined = await memberSync.lookupMemberAtEvent(event.sender, event, txn); + if (member) { // should always have a member? + this.handleCallMemberEvent(event, member, room.id, log); + } } } @@ -157,6 +164,11 @@ export class CallHandler implements RoomStateHandler { updateRoomMembers(room: Room, memberChanges: Map) { // TODO: also have map for roomId to calls, so we can easily update members // we will also need this to get the call for a room + for (const call of this._calls.values()) { + if (call.roomId === room.id) { + call.updateRoomMembers(memberChanges); + } + } } /** @internal */ @@ -193,7 +205,7 @@ export class CallHandler implements RoomStateHandler { } } - private handleCallMemberEvent(event: StateEvent, roomId: string, log: ILogItem) { + private handleCallMemberEvent(event: StateEvent, member: RoomMember, roomId: string, log: ILogItem) { const userId = event.state_key; const roomMemberKey = getRoomMemberKey(roomId, userId) const calls = event.content["m.calls"] ?? []; @@ -201,7 +213,7 @@ export class CallHandler implements RoomStateHandler { const callId = call["m.call_id"]; const groupCall = this._calls.get(callId); // TODO: also check the member when receiving the m.call event - groupCall?.updateMembership(userId, call, log); + groupCall?.updateMembership(userId, member, call, log); }; const newCallIdsMemberOf = new Set(calls.map(call => call["m.call_id"])); let previousCallIdsMemberOf = this.roomMemberToCallIds.get(roomMemberKey); diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index ddc0b517..ebacf844 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -18,7 +18,7 @@ import {ObservableMap} from "../../../observable/map/ObservableMap"; import {Member} from "./Member"; import {LocalMedia} from "../LocalMedia"; import {MuteSettings, CALL_LOG_TYPE} from "../common"; -import {RoomMember} from "../../room/members/RoomMember"; +import {MemberChange, RoomMember} from "../../room/members/RoomMember"; import {EventEmitter} from "../../../utils/EventEmitter"; import {EventType, CallIntent} from "../callEventTypes"; @@ -258,7 +258,20 @@ export class GroupCall extends EventEmitter<{change: never}> { } /** @internal */ - updateMembership(userId: string, callMembership: CallMembership, syncLog: ILogItem) { + updateRoomMembers(memberChanges: Map) { + for (const change of memberChanges.values()) { + const {member} = change; + for (const callMember of this._members.values()) { + // find all call members for a room member (can be multiple, for every device) + if (callMember.userId === member.userId) { + callMember.updateRoomMember(member); + } + } + } + } + + /** @internal */ + updateMembership(userId: string, roomMember: RoomMember, callMembership: CallMembership, syncLog: ILogItem) { syncLog.wrap({l: "update call membership", t: CALL_LOG_TYPE, id: this.id, userId}, log => { const devices = callMembership["m.devices"]; const previousDeviceIds = this.getDeviceIdsForUserId(userId); @@ -290,7 +303,7 @@ export class GroupCall extends EventEmitter<{change: never}> { } log.set("add", true); member = new Member( - RoomMember.fromUserId(this.roomId, userId, "join"), + roomMember, device, this._memberOptions, ); this._members.add(memberKey, member); diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 541cdf13..2a4eefb7 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -68,7 +68,7 @@ export class Member { private connection?: MemberConnection; constructor( - public readonly member: RoomMember, + public member: RoomMember, private callDeviceMembership: CallDeviceMembership, private readonly options: Options, ) {} @@ -179,6 +179,13 @@ export class Member { this.connection.logItem.refDetached(causeItem); } } + + /** @internal */ + updateRoomMember(roomMember: RoomMember) { + this.member = roomMember; + // TODO: this emits an update during the writeSync phase, which we usually try to avoid + this.options.emitUpdate(this); + } /** @internal */ emitUpdateFromPeerCall = (peerCall: PeerCall, params: any, log: ILogItem): void => { From 0c20beb1c0a43dff5275d8711c565b718cf7c36d Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Jun 2022 15:58:03 +0200 Subject: [PATCH 148/435] always pass mediaRepo to call vm --- src/domain/session/room/RoomViewModel.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index c60ce649..f8b70508 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -67,8 +67,9 @@ export class RoomViewModel extends ViewModel { this.emitChange("callViewModel"); })); const call = this._callObservable.get(); + // TODO: cleanup this duplication to create CallViewModel if (call) { - this._callViewModel = new CallViewModel(this.childOptions({call})); + this._callViewModel = this.track(new CallViewModel(this.childOptions({call, mediaRepository: this._room.mediaRepository}))); } } From d66d810fe2a4daa3dcbed4d2f97fbce5a4a393e1 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Jun 2022 15:58:26 +0200 Subject: [PATCH 149/435] pass updates to avatar view --- src/platform/web/ui/session/room/CallView.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/platform/web/ui/session/room/CallView.ts b/src/platform/web/ui/session/room/CallView.ts index 92dfe6b9..ac31973c 100644 --- a/src/platform/web/ui/session/room/CallView.ts +++ b/src/platform/web/ui/session/room/CallView.ts @@ -49,7 +49,7 @@ class StreamView extends TemplateView { t.div({className: { StreamView_avatar: true, hidden: vm => !vm.isCameraMuted - }}, t.view(new AvatarView(vm, 64), {parentProvidesUpdates: true})), + }}, t.view(new AvatarView(vm, 96), {parentProvidesUpdates: true})), t.div({ className: { StreamView_muteStatus: true, @@ -60,4 +60,10 @@ class StreamView extends TemplateView { }) ]); } + + update(value, props) { + super.update(value); + // update the AvatarView as we told it to not subscribe itself with parentProvidesUpdates + this.updateSubViews(value, props); + } } From f452c3ff4c56f1ae4fb42a46107f10ff02b33668 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Jun 2022 15:58:38 +0200 Subject: [PATCH 150/435] enable 96px avatars --- src/platform/web/ui/css/avatar.css | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/platform/web/ui/css/avatar.css b/src/platform/web/ui/css/avatar.css index 2ee9ca0c..a8f8080d 100644 --- a/src/platform/web/ui/css/avatar.css +++ b/src/platform/web/ui/css/avatar.css @@ -46,6 +46,14 @@ limitations under the License. font-size: calc(var(--avatar-size) * 0.6); } +.hydrogen .avatar.size-96 { + --avatar-size: 96px; + width: var(--avatar-size); + height: var(--avatar-size); + line-height: var(--avatar-size); + font-size: calc(var(--avatar-size) * 0.6); +} + .hydrogen .avatar.size-64 { --avatar-size: 64px; width: var(--avatar-size); From 8ba1d085f6a1ed011f464d9e38ea3d414fe991d7 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Jun 2022 15:58:50 +0200 Subject: [PATCH 151/435] fix refactor mistake in logging --- src/matrix/DeviceMessageHandler.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index 507e07ef..05276549 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -91,7 +91,8 @@ export class DeviceMessageHandler { l: "could not verify olm fingerprint key matches, ignoring", ed25519Key: dr.device.ed25519Key, claimedEd25519Key: dr.claimedEd25519Key, - deviceId, userId, + deviceId: device.deviceId, + userId: device.userId, }); } } From c8b5c6dd41e89a20a3a3913a4efbed9046c8f7b8 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Jun 2022 17:30:17 +0200 Subject: [PATCH 152/435] expose own user on BaseRoom so we don't have to pass session around everywhere we need this --- src/matrix/room/BaseRoom.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/matrix/room/BaseRoom.js b/src/matrix/room/BaseRoom.js index b8f172d0..9ca75a24 100644 --- a/src/matrix/room/BaseRoom.js +++ b/src/matrix/room/BaseRoom.js @@ -446,6 +446,10 @@ export class BaseRoom extends EventEmitter { return this._summary.data.membership; } + get user() { + return this._user; + } + isDirectMessageForUserId(userId) { if (this._summary.data.dmUserId === userId) { return true; From 5280467e6612de359b332ad28133cede54f569a3 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Jun 2022 17:30:43 +0200 Subject: [PATCH 153/435] return type is actual subclass options, not the options of ViewModel --- src/domain/ViewModel.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/ViewModel.ts b/src/domain/ViewModel.ts index 0bc52f6e..debf8e34 100644 --- a/src/domain/ViewModel.ts +++ b/src/domain/ViewModel.ts @@ -47,7 +47,7 @@ export class ViewModel extends EventEmitter<{change this._options = options; } - childOptions(explicitOptions: T): T & Options { + childOptions(explicitOptions: T): T & O { return Object.assign({}, this._options, explicitOptions); } From f8b01ac3ccd09fe3527aa7704360d76565f10cd2 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Jun 2022 17:31:17 +0200 Subject: [PATCH 154/435] show profile info for own call member by observing member on room --- src/domain/session/room/CallViewModel.ts | 55 ++++++++++++++++++------ src/domain/session/room/RoomViewModel.js | 4 +- 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index a770ddbe..265648bf 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -20,15 +20,18 @@ import {getStreamVideoTrack, getStreamAudioTrack} from "../../../matrix/calls/co import {avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../avatar"; import {EventObservableValue} from "../../../observable/value/EventObservableValue"; import {ObservableValueMap} from "../../../observable/map/ObservableValueMap"; +import type {Room} from "../../../matrix/room/Room"; import type {GroupCall} from "../../../matrix/calls/group/GroupCall"; import type {Member} from "../../../matrix/calls/group/Member"; +import type {RoomMember} from "../../../matrix/room/members/RoomMember"; import type {BaseObservableList} from "../../../observable/list/BaseObservableList"; +import type {BaseObservableValue} from "../../../observable/value/BaseObservableValue"; import type {Stream} from "../../../platform/types/MediaDevices"; import type {MediaRepository} from "../../../matrix/net/MediaRepository"; type Options = BaseOptions & { call: GroupCall, - mediaRepository: MediaRepository + room: Room, }; export class CallViewModel extends ViewModel { @@ -37,10 +40,10 @@ export class CallViewModel extends ViewModel { constructor(options: Options) { super(options); const ownMemberViewModelMap = new ObservableValueMap("self", new EventObservableValue(this.call, "change")) - .mapValues(call => new OwnMemberViewModel(this.childOptions({call: this.call, mediaRepository: this.getOption("mediaRepository")})), () => {}); + .mapValues((call, emitChange) => new OwnMemberViewModel(this.childOptions({call, emitChange})), () => {}); this.memberViewModels = this.call.members .filterValues(member => member.isConnected) - .mapValues(member => new CallMemberViewModel(this.childOptions({member, mediaRepository: this.getOption("mediaRepository")}))) + .mapValues(member => new CallMemberViewModel(this.childOptions({member, mediaRepository: this.getOption("room").mediaRepository}))) .join(ownMemberViewModelMap) .sortValues((a, b) => a.compare(b)); } @@ -74,12 +77,22 @@ export class CallViewModel extends ViewModel { } } -type OwnMemberOptions = BaseOptions & { - call: GroupCall, - mediaRepository: MediaRepository -} +class OwnMemberViewModel extends ViewModel implements IStreamViewModel { + private memberObservable: undefined | BaseObservableValue; + + constructor(options: Options) { + super(options); + this.init(); + } + + async init() { + const room = this.getOption("room"); + this.memberObservable = await room.observeMember(room.user.id); + this.track(this.memberObservable!.subscribe(() => { + this.emitChange(undefined); + })); + } -class OwnMemberViewModel extends ViewModel implements IStreamViewModel { get stream(): Stream | undefined { return this.call.localMedia?.userMedia; } @@ -97,19 +110,37 @@ class OwnMemberViewModel extends ViewModel implements IStreamV } get avatarLetter(): string { - return "I"; + const member = this.memberObservable?.get(); + if (member) { + return avatarInitials(member.name); + } else { + return ""; + } } get avatarColorNumber(): number { - return 3; + const member = this.memberObservable?.get(); + if (member) { + return getIdentifierColorNumber(member.userId); + } else { + return 0; + } } avatarUrl(size: number): string | undefined { - return undefined; + const member = this.memberObservable?.get(); + if (member) { + return getAvatarHttpUrl(member.avatarUrl, size, this.platform, this.getOption("room").mediaRepository); + } } get avatarTitle(): string { - return "Me"; + const member = this.memberObservable?.get(); + if (member) { + return member.name; + } else { + return ""; + } } compare(other: OwnMemberViewModel | CallMemberViewModel): number { diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index f8b70508..077a996e 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -62,14 +62,14 @@ export class RoomViewModel extends ViewModel { } this._callViewModel = this.disposeTracked(this._callViewModel); if (call) { - this._callViewModel = this.track(new CallViewModel(this.childOptions({call, mediaRepository: this._room.mediaRepository}))); + this._callViewModel = this.track(new CallViewModel(this.childOptions({call, room: this._room}))); } this.emitChange("callViewModel"); })); const call = this._callObservable.get(); // TODO: cleanup this duplication to create CallViewModel if (call) { - this._callViewModel = this.track(new CallViewModel(this.childOptions({call, mediaRepository: this._room.mediaRepository}))); + this._callViewModel = this.track(new CallViewModel(this.childOptions({call, room: this._room}))); } } From ed5fdb8154513b67d07d8fc3f7494a63e9f8c3e7 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 3 Jun 2022 12:43:51 +0200 Subject: [PATCH 155/435] don't withhold member event for call just because we don't have profile --- src/matrix/calls/CallHandler.ts | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 7524200c..ca58c4a8 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -102,13 +102,18 @@ export class CallHandler implements RoomStateHandler { // TODO: don't load all members until we need them const callsMemberEvents = await txn.roomState.getAllForType(roomId, EventType.GroupCallMember); await Promise.all(callsMemberEvents.map(async entry => { - const roomMemberState = await txn.roomState.get(roomId, MEMBER_EVENT_TYPE, entry.event.sender); + const userId = entry.event.sender; + const roomMemberState = await txn.roomState.get(roomId, MEMBER_EVENT_TYPE, userId); + let roomMember; if (roomMemberState) { - const roomMember = RoomMember.fromMemberEvent(roomMemberState.event); - if (roomMember) { - this.handleCallMemberEvent(entry.event, roomMember, roomId, log); - } + roomMember = RoomMember.fromMemberEvent(roomMemberState.event); } + if (!roomMember) { + // we'll be missing the member here if we received a call and it's members + // as pre-gap state and the members weren't active in the timeline we got. + roomMember = RoomMember.fromUserId(roomId, userId, "join"); + } + this.handleCallMemberEvent(entry.event, roomMember, roomId, log); })); })); log.set("newSize", this._calls.size); @@ -153,10 +158,13 @@ export class CallHandler implements RoomStateHandler { this.handleCallEvent(event, room.id, txn, log); } if (event.type === EventType.GroupCallMember) { - const member: RoomMember | undefined = await memberSync.lookupMemberAtEvent(event.sender, event, txn); - if (member) { // should always have a member? - this.handleCallMemberEvent(event, member, room.id, log); + let member = await memberSync.lookupMemberAtEvent(event.sender, event, txn); + if (!member) { + // we'll be missing the member here if we received a call and it's members + // as pre-gap state and the members weren't active in the timeline we got. + member = RoomMember.fromUserId(room.id, event.sender, "join"); } + this.handleCallMemberEvent(event, member, room.id, log); } } From 1fab314dd51c4b7356d047a07bdd04bd3d35ad44 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 3 Jun 2022 15:49:45 +0200 Subject: [PATCH 156/435] return user id for own avatar in call if member hasn't been found --- src/domain/session/room/CallViewModel.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 265648bf..cd974605 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -114,17 +114,12 @@ class OwnMemberViewModel extends ViewModel implements IStreamViewModel if (member) { return avatarInitials(member.name); } else { - return ""; + return this.getOption("room").user.id; } } get avatarColorNumber(): number { - const member = this.memberObservable?.get(); - if (member) { - return getIdentifierColorNumber(member.userId); - } else { - return 0; - } + return getIdentifierColorNumber(this.getOption("room").user.id); } avatarUrl(size: number): string | undefined { @@ -139,7 +134,7 @@ class OwnMemberViewModel extends ViewModel implements IStreamViewModel if (member) { return member.name; } else { - return ""; + return this.getOption("room").user.id; } } From bfdea03bbdff2c44fda85bc922146786164739eb Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 3 Jun 2022 15:50:02 +0200 Subject: [PATCH 157/435] start with seq 1, like Element Call does --- src/matrix/calls/group/Member.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 2a4eefb7..cf6bb6f0 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -222,7 +222,7 @@ export class Member { /** @internal */ sendSignallingMessage = async (message: SignallingMessage, log: ILogItem): Promise => { const groupMessage = message as SignallingMessage; - groupMessage.content.seq = ++this.connection!.outboundSeqCounter; + groupMessage.content.seq = this.connection!.outboundSeqCounter++; groupMessage.content.conf_id = this.options.confId; groupMessage.content.device_id = this.options.ownDeviceId; groupMessage.content.party_id = this.options.ownDeviceId; From 10caba68727707bbfd11d607aedd3d42cadfd210 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 9 Jun 2022 15:33:59 +0200 Subject: [PATCH 158/435] improve calls view --- src/domain/ViewModel.ts | 2 +- src/domain/session/room/CallViewModel.ts | 52 ++++- .../web/ui/css/themes/element/call.css | 206 ++++++++++++++++++ .../ui/css/themes/element/icons/cam-muted.svg | 1 + .../css/themes/element/icons/cam-unmuted.svg | 1 + .../ui/css/themes/element/icons/hangup.svg | 3 + .../ui/css/themes/element/icons/mic-muted.svg | 19 ++ .../css/themes/element/icons/mic-unmuted.svg | 16 ++ .../css/themes/element/icons/video-call.svg | 4 + .../css/themes/element/icons/voice-call.svg | 3 + .../web/ui/css/themes/element/theme.css | 54 +---- src/platform/web/ui/session/room/CallView.ts | 65 +++++- 12 files changed, 359 insertions(+), 67 deletions(-) create mode 100644 src/platform/web/ui/css/themes/element/call.css create mode 100644 src/platform/web/ui/css/themes/element/icons/cam-muted.svg create mode 100644 src/platform/web/ui/css/themes/element/icons/cam-unmuted.svg create mode 100644 src/platform/web/ui/css/themes/element/icons/hangup.svg create mode 100644 src/platform/web/ui/css/themes/element/icons/mic-muted.svg create mode 100644 src/platform/web/ui/css/themes/element/icons/mic-unmuted.svg create mode 100644 src/platform/web/ui/css/themes/element/icons/video-call.svg create mode 100644 src/platform/web/ui/css/themes/element/icons/voice-call.svg diff --git a/src/domain/ViewModel.ts b/src/domain/ViewModel.ts index debf8e34..257624ea 100644 --- a/src/domain/ViewModel.ts +++ b/src/domain/ViewModel.ts @@ -115,7 +115,7 @@ export class ViewModel extends EventEmitter<{change return result; } - emitChange(changedProps: any): void { + emitChange(changedProps?: any): void { if (this._options.emitChange) { this._options.emitChange(changedProps); } else { diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index cd974605..3e7fd951 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -46,12 +46,32 @@ export class CallViewModel extends ViewModel { .mapValues(member => new CallMemberViewModel(this.childOptions({member, mediaRepository: this.getOption("room").mediaRepository}))) .join(ownMemberViewModelMap) .sortValues((a, b) => a.compare(b)); + this.track(this.memberViewModels.subscribe({ + onRemove: () => { + this.emitChange(); // update memberCount + }, + onAdd: () => { + this.emitChange(); // update memberCount + }, + onUpdate: () => {}, + onReset: () => {}, + onMove: () => {} + })) } - private get call(): GroupCall { - return this.getOption("call"); + get isCameraMuted(): boolean { + return isLocalCameraMuted(this.call); } + get isMicrophoneMuted(): boolean { + return isLocalMicrophoneMuted(this.call); + } + + get memberCount(): number { + return this.memberViewModels.length; + } + + get name(): string { return this.call.name; } @@ -60,19 +80,27 @@ export class CallViewModel extends ViewModel { return this.call.id; } - get stream(): Stream | undefined { - return this.call.localMedia?.userMedia; + private get call(): GroupCall { + return this.getOption("call"); } - leave() { + hangup() { if (this.call.hasJoined) { this.call.leave(); } } - async toggleVideo() { + async toggleCamera() { if (this.call.muteSettings) { this.call.setMuted(this.call.muteSettings.toggleCamera()); + this.emitChange(); + } + } + + async toggleMicrophone() { + if (this.call.muteSettings) { + this.call.setMuted(this.call.muteSettings.toggleMicrophone()); + this.emitChange(); } } } @@ -102,11 +130,11 @@ class OwnMemberViewModel extends ViewModel implements IStreamViewModel } get isCameraMuted(): boolean { - return isMuted(this.call.muteSettings?.camera, !!getStreamVideoTrack(this.stream)); + return isLocalCameraMuted(this.call); } get isMicrophoneMuted(): boolean { - return isMuted(this.call.muteSettings?.microphone, !!getStreamAudioTrack(this.stream)); + return isLocalMicrophoneMuted(this.call); } get avatarLetter(): string { @@ -209,3 +237,11 @@ function isMuted(muted: boolean | undefined, hasTrack: boolean) { return !hasTrack; } } + +function isLocalCameraMuted(call: GroupCall): boolean { + return isMuted(call.muteSettings?.camera, !!getStreamVideoTrack(call.localMedia?.userMedia)); +} + +function isLocalMicrophoneMuted(call: GroupCall): boolean { + return isMuted(call.muteSettings?.microphone, !!getStreamAudioTrack(call.localMedia?.userMedia)); +} diff --git a/src/platform/web/ui/css/themes/element/call.css b/src/platform/web/ui/css/themes/element/call.css new file mode 100644 index 00000000..4398f9c6 --- /dev/null +++ b/src/platform/web/ui/css/themes/element/call.css @@ -0,0 +1,206 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.CallView { + height: 40vh; + display: grid; +} + +.CallView > * { + grid-column: 1; + grid-row: 1; +} + +.CallView_members { + display: grid; + gap: 12px; + background: var(--background-color-secondary--darker-60); + padding: 12px; + margin: 0; + min-height: 0; + list-style: none; + align-self: stretch; +} + +.StreamView { + display: grid; + border-radius: 8px; + overflow: hidden; + background-color: black; +} + +.StreamView > * { + grid-column: 1; + grid-row: 1; +} + +.StreamView video { + width: 100%; + height: 100%; + object-fit: contain; +} + +.StreamView_avatar { + align-self: center; + justify-self: center; +} + +.StreamView_muteStatus { + align-self: start; + justify-self: end; + width: 24px; + height: 24px; + background-position: center; + background-repeat: no-repeat; + background-size: 14px; + display: block; + background-color: var(--text-color); + border-radius: 4px; + margin: 4px; +} + +.StreamView_muteStatus.microphoneMuted { + background-image: url("./icons/mic-muted.svg?primary=text-color--lighter-80"); +} + +.StreamView_muteStatus.cameraMuted { + background-image: url("./icons/cam-muted.svg?primary=text-color--lighter-80"); +} + +.CallView_buttons { + align-self: end; + justify-self: center; + display: flex; + gap: 12px; + margin-bottom: 16px; +} + +.CallView_buttons button { + border-radius: 100%; + width: 48px; + height: 48px; + border: none; + background-color: var(--accent-color); + background-position: center; + background-repeat: no-repeat; +} + +.CallView_buttons .CallView_hangup { + background-color: var(--error-color); + background-image: url("./icons/hangup.svg?primary=background-color-primary"); +} + +.CallView_buttons .CallView_mutedMicrophone { + background-color: var(--background-color-primary); + background-image: url("./icons/mic-muted.svg?primary=text-color"); +} + +.CallView_buttons .CallView_unmutedMicrophone { + background-image: url("./icons/mic-unmuted.svg?primary=background-color-primary"); +} + +.CallView_buttons .CallView_mutedCamera { + background-color: var(--background-color-primary); + background-image: url("./icons/cam-muted.svg?primary=text-color"); +} + +.CallView_buttons .CallView_unmutedCamera { + background-image: url("./icons/cam-unmuted.svg?primary=background-color-primary"); +} + +.CallView_members.size1 { + grid-template-columns: 1fr; + grid-template-rows: 1fr; +} + +.CallView_members.size2 { + grid-template-columns: 1fr; + grid-template-rows: repeat(2, 1fr); +} + +/* square */ +.CallView_members.square.size3, +.CallView_members.square.size4 { + grid-template-columns: repeat(2, 1fr); + grid-template-rows: repeat(2, 1fr); +} +.CallView_members.square.size5, +.CallView_members.square.size6 { + grid-template-columns: repeat(3, 1fr); + grid-template-rows: repeat(2, 1fr); +} +.CallView_members.square.size7, +.CallView_members.square.size8, +.CallView_members.square.size9 { + grid-template-columns: repeat(3, 1fr); + grid-template-rows: repeat(3, 1fr); +} +.CallView_members.square.size10 { + grid-template-columns: repeat(3, 1fr); + grid-template-rows: repeat(4, 1fr); +} +/** tall */ +.CallView_members.tall.size3 { + grid-template-columns: 1fr; + grid-template-rows: repeat(3, 1fr); +} +.CallView_members.tall.size4 { + grid-template-columns: 1fr; + grid-template-rows: repeat(4, 1fr); +} +.CallView_members.tall.size5, +.CallView_members.tall.size6 { + grid-template-rows: repeat(3, 1fr); + grid-template-columns: repeat(2, 1fr); +} +.CallView_members.tall.size7, +.CallView_members.tall.size8 { + grid-template-rows: repeat(4, 1fr); + grid-template-columns: repeat(2, 1fr); +} +.CallView_members.tall.size9, +.CallView_members.tall.size10 { + grid-template-rows: repeat(5, 1fr); + grid-template-columns: repeat(2, 1fr); +} +/** wide */ +.CallView_members.wide.size2 { + grid-template-columns: repeat(2, 1fr); + grid-template-rows: 1fr; +} +.CallView_members.wide.size3 { + grid-template-rows: 1fr; + grid-template-columns: repeat(3, 1fr); +} +.CallView_members.wide.size4 { + grid-template-rows: 1fr; + grid-template-columns: repeat(4, 1fr); +} +.CallView_members.wide.size5, +.CallView_members.wide.size6 { + grid-template-rows: repeat(2, 1fr); + grid-template-columns: repeat(3, 1fr); +} +.CallView_members.wide.size7, +.CallView_members.wide.size8 { + grid-template-rows: repeat(2, 1fr); + grid-template-columns: repeat(4, 1fr); +} +.CallView_members.wide.size9, +.CallView_members.wide.size10 { + grid-template-rows: repeat(3, 1fr); + grid-template-columns: repeat(4, 1fr); +} diff --git a/src/platform/web/ui/css/themes/element/icons/cam-muted.svg b/src/platform/web/ui/css/themes/element/icons/cam-muted.svg new file mode 100644 index 00000000..6a739ae2 --- /dev/null +++ b/src/platform/web/ui/css/themes/element/icons/cam-muted.svg @@ -0,0 +1 @@ + diff --git a/src/platform/web/ui/css/themes/element/icons/cam-unmuted.svg b/src/platform/web/ui/css/themes/element/icons/cam-unmuted.svg new file mode 100644 index 00000000..9497e075 --- /dev/null +++ b/src/platform/web/ui/css/themes/element/icons/cam-unmuted.svg @@ -0,0 +1 @@ + diff --git a/src/platform/web/ui/css/themes/element/icons/hangup.svg b/src/platform/web/ui/css/themes/element/icons/hangup.svg new file mode 100644 index 00000000..c56fe7a4 --- /dev/null +++ b/src/platform/web/ui/css/themes/element/icons/hangup.svg @@ -0,0 +1,3 @@ + + + diff --git a/src/platform/web/ui/css/themes/element/icons/mic-muted.svg b/src/platform/web/ui/css/themes/element/icons/mic-muted.svg new file mode 100644 index 00000000..35669ee0 --- /dev/null +++ b/src/platform/web/ui/css/themes/element/icons/mic-muted.svg @@ -0,0 +1,19 @@ + + + + + + diff --git a/src/platform/web/ui/css/themes/element/icons/mic-unmuted.svg b/src/platform/web/ui/css/themes/element/icons/mic-unmuted.svg new file mode 100644 index 00000000..94b81510 --- /dev/null +++ b/src/platform/web/ui/css/themes/element/icons/mic-unmuted.svg @@ -0,0 +1,16 @@ + + + + + diff --git a/src/platform/web/ui/css/themes/element/icons/video-call.svg b/src/platform/web/ui/css/themes/element/icons/video-call.svg new file mode 100644 index 00000000..bc3688b5 --- /dev/null +++ b/src/platform/web/ui/css/themes/element/icons/video-call.svg @@ -0,0 +1,4 @@ + + + + diff --git a/src/platform/web/ui/css/themes/element/icons/voice-call.svg b/src/platform/web/ui/css/themes/element/icons/voice-call.svg new file mode 100644 index 00000000..02a79969 --- /dev/null +++ b/src/platform/web/ui/css/themes/element/icons/voice-call.svg @@ -0,0 +1,3 @@ + + + diff --git a/src/platform/web/ui/css/themes/element/theme.css b/src/platform/web/ui/css/themes/element/theme.css index db0ad66b..6403bb60 100644 --- a/src/platform/web/ui/css/themes/element/theme.css +++ b/src/platform/web/ui/css/themes/element/theme.css @@ -18,6 +18,7 @@ limitations under the License. @import url('../../main.css'); @import url('inter.css'); @import url('timeline.css'); +@import url('call.css'); :root { font-size: 10px; @@ -1155,56 +1156,3 @@ button.RoomDetailsView_row::after { background-position: center; background-size: 36px; } - -.CallView { - max-height: 50vh; - overflow-y: auto; -} - -.CallView ul { - display: flex; - margin: 0; - gap: 12px; - padding: 0; - flex-wrap: wrap; - justify-content: center; -} - -.StreamView { - width: 360px; - min-height: 200px; - border: 2px var(--accent-color) solid; - display: grid; - border-radius: 8px; - overflow: hidden; - background-color: black; -} - -.StreamView > * { - grid-column: 1; - grid-row: 1; -} - -.StreamView video { - width: 100%; -} - -.StreamView_avatar { - align-self: center; - justify-self: center; -} - -.StreamView_muteStatus { - align-self: end; - justify-self: start; - color: var(--text-color--lighter-80); -} - -.StreamView_muteStatus.microphoneMuted::before { - content: "mic muted"; -} - -.StreamView_muteStatus.cameraMuted::before { - content: "cam muted"; -} - diff --git a/src/platform/web/ui/session/room/CallView.ts b/src/platform/web/ui/session/room/CallView.ts index ac31973c..5ccdaa84 100644 --- a/src/platform/web/ui/session/room/CallView.ts +++ b/src/platform/web/ui/session/room/CallView.ts @@ -17,26 +17,81 @@ limitations under the License. import {TemplateView, Builder} from "../../general/TemplateView"; import {AvatarView} from "../../AvatarView"; import {ListView} from "../../general/ListView"; +import {classNames} from "../../general/html"; import {Stream} from "../../../../types/MediaDevices"; import type {CallViewModel, CallMemberViewModel, IStreamViewModel} from "../../../../../domain/session/room/CallViewModel"; export class CallView extends TemplateView { + private resizeObserver?: ResizeObserver; + render(t: Builder, vm: CallViewModel): Element { + const members = t.view(new ListView({ + className: "CallView_members", + list: vm.memberViewModels + }, vm => new StreamView(vm))) as HTMLElement; + this.bindMembersCssClasses(t, members); return t.div({class: "CallView"}, [ - t.p(vm => `Call ${vm.name} (${vm.id})`), - t.view(new ListView({list: vm.memberViewModels}, vm => new StreamView(vm))), - t.div({class: "buttons"}, [ - t.button({onClick: () => vm.leave()}, "Leave"), - t.button({onClick: () => vm.toggleVideo()}, "Toggle video"), + members, + //t.p(vm => `Call ${vm.name}`), + t.div({class: "CallView_buttons"}, [ + t.button({className: { + "CallView_mutedMicrophone": vm => vm.isMicrophoneMuted, + "CallView_unmutedMicrophone": vm => !vm.isMicrophoneMuted, + }, onClick: () => vm.toggleMicrophone()}), + t.button({className: { + "CallView_mutedCamera": vm => vm.isCameraMuted, + "CallView_unmutedCamera": vm => !vm.isCameraMuted, + }, onClick: () => vm.toggleCamera()}), + t.button({className: "CallView_hangup", onClick: () => vm.hangup()}), ]) ]); } + + private bindMembersCssClasses(t, members) { + t.mapSideEffect(vm => vm.memberCount, count => { + members.classList.forEach((c, _, list) => { + if (c.startsWith("size")) { + list.remove(c); + } + }); + members.classList.add(`size${count}`); + }); + // update classes describing aspect ratio categories + if (typeof ResizeObserver === "function") { + const set = (c, flag) => { + if (flag) { + members.classList.add(c); + } else { + members.classList.remove(c); + } + }; + this.resizeObserver = new ResizeObserver(() => { + const ar = members.clientWidth / members.clientHeight; + const isTall = ar < 0.5; + const isSquare = !isTall && ar < 1.8 + const isWide = !isTall && !isSquare; + set("tall", isTall); + set("square", isSquare); + set("wide", isWide); + }); + this.resizeObserver!.observe(members); + } + } + + public unmount() { + if (this.resizeObserver) { + this.resizeObserver.unobserve((this.root()! as Element).querySelector(".CallView_members")); + this.resizeObserver = undefined; + } + super.unmount(); + } } class StreamView extends TemplateView { render(t: Builder, vm: IStreamViewModel): Element { const video = t.video({ autoplay: true, + disablePictureInPicture: true, className: { hidden: vm => vm.isCameraMuted } From 41288683fc9547ef1cf80cfe56ac472bf6afd712 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 10 Jun 2022 17:10:23 +0200 Subject: [PATCH 159/435] allow unmuting when we don't yet have a mic/cam track --- scripts/postcss/css-url-to-variables.js | 8 ++- src/domain/session/room/CallViewModel.ts | 55 +++++++++---------- src/matrix/calls/PeerCall.ts | 12 ++-- src/matrix/calls/common.ts | 28 +++++++++- src/matrix/calls/group/GroupCall.ts | 25 +++++++-- .../web/ui/css/themes/element/call.css | 9 +++ src/platform/web/ui/session/room/CallView.ts | 14 ++++- 7 files changed, 105 insertions(+), 46 deletions(-) diff --git a/scripts/postcss/css-url-to-variables.js b/scripts/postcss/css-url-to-variables.js index 1d4666f4..9e2e0077 100644 --- a/scripts/postcss/css-url-to-variables.js +++ b/scripts/postcss/css-url-to-variables.js @@ -26,7 +26,13 @@ const idToPrepend = "icon-url"; function findAndReplaceUrl(decl) { const value = decl.value; - const parsed = valueParser(value); + let parsed; + try { + parsed = valueParser(value); + } catch (err) { + console.log(`Error trying to parse ${decl}`); + throw err; + } parsed.walk(node => { if (node.type !== "function" || node.value !== "url") { return; diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 3e7fd951..5b6f05b4 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -60,18 +60,17 @@ export class CallViewModel extends ViewModel { } get isCameraMuted(): boolean { - return isLocalCameraMuted(this.call); + return this.call.muteSettings?.camera ?? true; } get isMicrophoneMuted(): boolean { - return isLocalMicrophoneMuted(this.call); + return this.call.muteSettings?.microphone ?? true; } get memberCount(): number { return this.memberViewModels.length; } - get name(): string { return this.call.name; } @@ -84,22 +83,36 @@ export class CallViewModel extends ViewModel { return this.getOption("call"); } - hangup() { + async hangup() { if (this.call.hasJoined) { - this.call.leave(); + await this.call.leave(); } } async toggleCamera() { - if (this.call.muteSettings) { - this.call.setMuted(this.call.muteSettings.toggleCamera()); + const {localMedia, muteSettings} = this.call; + if (muteSettings && localMedia) { + // unmute but no track? + if (muteSettings.camera && !getStreamVideoTrack(localMedia.userMedia)) { + const stream = await this.platform.mediaDevices.getMediaTracks(!muteSettings.microphone, true); + await this.call.setMedia(localMedia.withUserMedia(stream)); + } else { + await this.call.setMuted(muteSettings.toggleCamera()); + } this.emitChange(); } } async toggleMicrophone() { - if (this.call.muteSettings) { - this.call.setMuted(this.call.muteSettings.toggleMicrophone()); + const {localMedia, muteSettings} = this.call; + if (muteSettings && localMedia) { + // unmute but no track? + if (muteSettings.microphone && !getStreamAudioTrack(localMedia.userMedia)) { + const stream = await this.platform.mediaDevices.getMediaTracks(true, !muteSettings.camera); + await this.call.setMedia(localMedia.withUserMedia(stream)); + } else { + await this.call.setMuted(muteSettings.toggleMicrophone()); + } this.emitChange(); } } @@ -130,11 +143,11 @@ class OwnMemberViewModel extends ViewModel implements IStreamViewModel } get isCameraMuted(): boolean { - return isLocalCameraMuted(this.call); + return this.call.muteSettings?.camera ?? true; } get isMicrophoneMuted(): boolean { - return isLocalMicrophoneMuted(this.call); + return this.call.muteSettings?.microphone ?? true; } get avatarLetter(): string { @@ -186,11 +199,11 @@ export class CallMemberViewModel extends ViewModel implements ISt } get isCameraMuted(): boolean { - return isMuted(this.member.remoteMuteSettings?.camera, !!getStreamVideoTrack(this.stream)); + return this.member.remoteMuteSettings?.camera ?? true; } get isMicrophoneMuted(): boolean { - return isMuted(this.member.remoteMuteSettings?.microphone, !!getStreamAudioTrack(this.stream)); + return this.member.remoteMuteSettings?.microphone ?? true; } get avatarLetter(): string { @@ -229,19 +242,3 @@ export interface IStreamViewModel extends AvatarSource, ViewModel { get isCameraMuted(): boolean; get isMicrophoneMuted(): boolean; } - -function isMuted(muted: boolean | undefined, hasTrack: boolean) { - if (muted) { - return true; - } else { - return !hasTrack; - } -} - -function isLocalCameraMuted(call: GroupCall): boolean { - return isMuted(call.muteSettings?.camera, !!getStreamVideoTrack(call.localMedia?.userMedia)); -} - -function isLocalMicrophoneMuted(call: GroupCall): boolean { - return isMuted(call.muteSettings?.microphone, !!getStreamAudioTrack(call.localMedia?.userMedia)); -} diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index ea7e9d06..809decbf 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -887,8 +887,8 @@ export class PeerCall implements IDisposable { const streamId = this.localMedia.userMedia.id; metadata[streamId] = { purpose: SDPStreamMetadataPurpose.Usermedia, - audio_muted: this.localMuteSettings?.microphone || !getStreamAudioTrack(this.localMedia.userMedia), - video_muted: this.localMuteSettings?.camera || !getStreamVideoTrack(this.localMedia.userMedia), + audio_muted: this.localMuteSettings?.microphone ?? false, + video_muted: this.localMuteSettings?.camera ?? false, }; } if (this.localMedia?.screenShare) { @@ -936,7 +936,7 @@ export class PeerCall implements IDisposable { this.updateRemoteMedia(log); } } - }) + }); }; stream.addEventListener("removetrack", listener); const disposeListener = () => { @@ -971,8 +971,10 @@ export class PeerCall implements IDisposable { videoReceiver.track.enabled = !metaData.video_muted; } this._remoteMuteSettings = new MuteSettings( - metaData.audio_muted || !audioReceiver?.track, - metaData.video_muted || !videoReceiver?.track + metaData.audio_muted ?? false, + metaData.video_muted ?? false, + !!audioReceiver?.track ?? false, + !!videoReceiver?.track ?? false ); log.log({ l: "setting userMedia", diff --git a/src/matrix/calls/common.ts b/src/matrix/calls/common.ts index db49a168..66db6edc 100644 --- a/src/matrix/calls/common.ts +++ b/src/matrix/calls/common.ts @@ -25,14 +25,36 @@ export function getStreamVideoTrack(stream: Stream | undefined): Track | undefin } export class MuteSettings { - constructor (public readonly microphone: boolean = false, public readonly camera: boolean = false) {} + constructor ( + private readonly isMicrophoneMuted: boolean = false, + private readonly isCameraMuted: boolean = false, + private hasMicrophoneTrack: boolean = false, + private hasCameraTrack: boolean = false, + ) {} + + updateTrackInfo(userMedia: Stream | undefined) { + this.hasMicrophoneTrack = !!getStreamAudioTrack(userMedia); + this.hasCameraTrack = !!getStreamVideoTrack(userMedia); + } + + get microphone(): boolean { + return !this.hasMicrophoneTrack || this.isMicrophoneMuted; + } + + get camera(): boolean { + return !this.hasCameraTrack || this.isCameraMuted; + } toggleCamera(): MuteSettings { - return new MuteSettings(this.microphone, !this.camera); + return new MuteSettings(this.microphone, !this.camera, this.hasMicrophoneTrack, this.hasCameraTrack); } toggleMicrophone(): MuteSettings { - return new MuteSettings(!this.microphone, this.camera); + return new MuteSettings(!this.microphone, this.camera, this.hasMicrophoneTrack, this.hasCameraTrack); + } + + equals(other: MuteSettings) { + return this.microphone === other.microphone && this.camera === other.camera; } } diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index ebacf844..19e40dbf 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -137,11 +137,13 @@ export class GroupCall extends EventEmitter<{change: never}> { ownSessionId: this.options.sessionId }); const membersLogItem = logItem.child("member connections"); + const localMuteSettings = new MuteSettings(); + localMuteSettings.updateTrackInfo(localMedia.userMedia); const joinedData = new JoinedData( logItem, membersLogItem, localMedia, - new MuteSettings() + localMuteSettings ); this.joinedData = joinedData; await joinedData.logItem.wrap("join", async log => { @@ -163,9 +165,14 @@ export class GroupCall extends EventEmitter<{change: never}> { if ((this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) && this.joinedData) { const oldMedia = this.joinedData.localMedia; this.joinedData.localMedia = localMedia; + // reflect the fact we gained or lost local tracks in the local mute settings + // and update the track info so PeerCall can use it to send up to date metadata, + this.joinedData.localMuteSettings.updateTrackInfo(localMedia.userMedia); + this.emitChange(); //allow listeners to see new media/mute settings await Promise.all(Array.from(this._members.values()).map(m => { return m.setMedia(localMedia, oldMedia); })); + oldMedia?.stopExcept(localMedia); } } @@ -175,11 +182,19 @@ export class GroupCall extends EventEmitter<{change: never}> { if (!joinedData) { return; } + const prevMuteSettings = joinedData.localMuteSettings; + // we still update the mute settings if nothing changed because + // you might be muted because you don't have a track or because + // you actively chosen to mute + // (which we want to respect in the future when you add a track) joinedData.localMuteSettings = muteSettings; - await Promise.all(Array.from(this._members.values()).map(m => { - return m.setMuted(joinedData.localMuteSettings); - })); - this.emitChange(); + joinedData.localMuteSettings.updateTrackInfo(joinedData.localMedia.userMedia); + if (!prevMuteSettings.equals(muteSettings)) { + await Promise.all(Array.from(this._members.values()).map(m => { + return m.setMuted(joinedData.localMuteSettings); + })); + this.emitChange(); + } } get muteSettings(): MuteSettings | undefined { diff --git a/src/platform/web/ui/css/themes/element/call.css b/src/platform/web/ui/css/themes/element/call.css index 4398f9c6..9bdc4abb 100644 --- a/src/platform/web/ui/css/themes/element/call.css +++ b/src/platform/web/ui/css/themes/element/call.css @@ -40,6 +40,7 @@ limitations under the License. border-radius: 8px; overflow: hidden; background-color: black; + display: block; } .StreamView > * { @@ -98,11 +99,19 @@ limitations under the License. background-repeat: no-repeat; } +.CallView_buttons button:disabled { + background-color: var(--accent-color--lighter-10); +} + .CallView_buttons .CallView_hangup { background-color: var(--error-color); background-image: url("./icons/hangup.svg?primary=background-color-primary"); } +.CallView_buttons .CallView_hangup:disabled { + background-color: var(--error-color--lighter-10); +} + .CallView_buttons .CallView_mutedMicrophone { background-color: var(--background-color-primary); background-image: url("./icons/mic-muted.svg?primary=text-color"); diff --git a/src/platform/web/ui/session/room/CallView.ts b/src/platform/web/ui/session/room/CallView.ts index 5ccdaa84..6961dc53 100644 --- a/src/platform/web/ui/session/room/CallView.ts +++ b/src/platform/web/ui/session/room/CallView.ts @@ -37,12 +37,12 @@ export class CallView extends TemplateView { t.button({className: { "CallView_mutedMicrophone": vm => vm.isMicrophoneMuted, "CallView_unmutedMicrophone": vm => !vm.isMicrophoneMuted, - }, onClick: () => vm.toggleMicrophone()}), + }, onClick: disableTargetCallback(() => vm.toggleMicrophone())}), t.button({className: { "CallView_mutedCamera": vm => vm.isCameraMuted, "CallView_unmutedCamera": vm => !vm.isCameraMuted, - }, onClick: () => vm.toggleCamera()}), - t.button({className: "CallView_hangup", onClick: () => vm.hangup()}), + }, onClick: disableTargetCallback(() => vm.toggleCamera())}), + t.button({className: "CallView_hangup", onClick: disableTargetCallback(() => vm.hangup())}), ]) ]); } @@ -122,3 +122,11 @@ class StreamView extends TemplateView { this.updateSubViews(value, props); } } + +function disableTargetCallback(callback: (evt: Event) => Promise): (evt: Event) => Promise { + return async (evt: Event) => { + (evt.target as HTMLElement)?.setAttribute("disabled", "disabled"); + await callback(evt); + (evt.target as HTMLElement)?.removeAttribute("disabled"); + } +} From 595a15c533d3a22bd79487a8ac42fd932f9d7c38 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 14 Jun 2022 12:12:10 +0200 Subject: [PATCH 160/435] make overlay buttons on call view clickable on chrome --- src/platform/web/ui/css/themes/element/call.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/platform/web/ui/css/themes/element/call.css b/src/platform/web/ui/css/themes/element/call.css index 9bdc4abb..59e4046b 100644 --- a/src/platform/web/ui/css/themes/element/call.css +++ b/src/platform/web/ui/css/themes/element/call.css @@ -87,6 +87,10 @@ limitations under the License. display: flex; gap: 12px; margin-bottom: 16px; + /** Chrome (v100) requires this to make the buttons clickable + * where they overlap with the video element, even though + * the buttons come later in the DOM. */ + z-index: 1; } .CallView_buttons button { From d4ee19c4e4d915714b52da05276e6010e51029d2 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 14 Jun 2022 12:24:38 +0200 Subject: [PATCH 161/435] fix video elements not respecting parent height in callview grid --- src/platform/web/ui/css/themes/element/call.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/platform/web/ui/css/themes/element/call.css b/src/platform/web/ui/css/themes/element/call.css index 59e4046b..10388fb4 100644 --- a/src/platform/web/ui/css/themes/element/call.css +++ b/src/platform/web/ui/css/themes/element/call.css @@ -40,7 +40,6 @@ limitations under the License. border-radius: 8px; overflow: hidden; background-color: black; - display: block; } .StreamView > * { @@ -51,6 +50,7 @@ limitations under the License. .StreamView video { width: 100%; height: 100%; + min-height: 0; object-fit: contain; } From 8a90c48d1e0c844827e0e08306c3f915e0acb3b2 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Sat, 25 Jun 2022 05:56:43 +0200 Subject: [PATCH 162/435] debugging unmuting not working --- src/domain/session/room/CallViewModel.ts | 1 + src/matrix/calls/LocalMedia.ts | 12 +++++++++--- src/matrix/calls/PeerCall.ts | 15 +++++++++------ src/matrix/calls/group/GroupCall.ts | 13 ++++++++----- src/matrix/calls/group/Member.ts | 2 +- src/platform/web/dom/MediaDevices.ts | 3 +++ 6 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 5b6f05b4..020c9f17 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -109,6 +109,7 @@ export class CallViewModel extends ViewModel { // unmute but no track? if (muteSettings.microphone && !getStreamAudioTrack(localMedia.userMedia)) { const stream = await this.platform.mediaDevices.getMediaTracks(true, !muteSettings.camera); + console.log("got tracks", Array.from(stream.getTracks()).map((t: MediaStreamTrack) => { return {kind: t.kind, id: t.id};})) await this.call.setMedia(localMedia.withUserMedia(stream)); } else { await this.call.setMuted(muteSettings.toggleMicrophone()); diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index b79fe098..922e6861 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -45,13 +45,19 @@ export class LocalMedia { const cloneOrAdoptStream = (oldOriginalStream: Stream | undefined, oldCloneStream: Stream | undefined, newStream: Stream | undefined): Stream | undefined => { let stream; if (oldOriginalStream?.id === newStream?.id) { - stream = oldCloneStream; + return oldCloneStream; } else { - stream = newStream?.clone(); + const clonedStream = newStream?.clone(); + // clonedStream.addEventListener("removetrack", evt => { + // console.log(`removing track ${evt.track.id} (${evt.track.kind}) from clonedStream ${clonedStream.id}`); + // }); + // console.log("got cloned tracks", Array.from(clonedStream.getTracks()).map((t: MediaStreamTrack) => { return {kind: t.kind, id: t.id};}), "from new stream", Array.from(newStream.getTracks()).map((t: MediaStreamTrack) => { return {kind: t.kind, id: t.id};})); + // console.log("stopping old audio stream", getStreamAudioTrack(oldCloneStream)?.id); + // console.log("stopping old video stream", getStreamVideoTrack(oldCloneStream)?.id); getStreamAudioTrack(oldCloneStream)?.stop(); getStreamVideoTrack(oldCloneStream)?.stop(); + return clonedStream; } - return stream; } return new LocalMedia( cloneOrAdoptStream(oldOriginal?.userMedia, oldClone?.userMedia, this.userMedia), diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 809decbf..1e22a2ff 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -383,8 +383,7 @@ export class PeerCall implements IDisposable { const offer = this.peerConnection.localDescription!; // Get rid of any candidates waiting to be sent: they'll be included in the local // description we just got and will send in the offer. - log.log(`Discarding ${ - this.candidateSendQueue.length} candidates that will be sent in offer`); + log.set("includedCandidates", this.candidateSendQueue.length); this.candidateSendQueue = []; // need to queue this @@ -414,16 +413,20 @@ export class PeerCall implements IDisposable { this.sendCandidateQueue(log); - await log.wrap("invite timeout", async log => { - if (this._state === CallState.InviteSent) { + if (this._state === CallState.InviteSent) { + const timeoutLog = this.logItem.child("invite timeout"); + log.refDetached(timeoutLog); + // don't await this, as it would block other negotationneeded events from being processed + // as they are processed serially + timeoutLog.run(async log => { try { await this.delay(CALL_TIMEOUT_MS); } catch (err) { return; } // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between if (this._state === CallState.InviteSent) { this._hangup(CallErrorCode.InviteTimeout, log); } - } - }); + }).catch(err => {}); // prevent error from being unhandled, it will be logged already by run above + } }; private async handleInviteGlare(content: MCallInvite, partyId: PartyId, log: ILogItem): Promise { diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 19e40dbf..087dc1ff 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -149,11 +149,14 @@ export class GroupCall extends EventEmitter<{change: never}> { await joinedData.logItem.wrap("join", async log => { this._state = GroupCallState.Joining; this.emitChange(); - const memberContent = await this._createJoinPayload(); - // send m.call.member state event - const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCallMember, this.options.ownUserId, memberContent, {log}); - await request.response(); - this.emitChange(); + await log.wrap("update member state", async log => { + const memberContent = await this._createJoinPayload(); + log.set("payload", memberContent); + // send m.call.member state event + const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCallMember, this.options.ownUserId, memberContent, {log}); + await request.response(); + this.emitChange(); + }); // send invite to all members that are < my userId for (const [,member] of this._members) { this.connectToMember(member, joinedData, log); diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index cf6bb6f0..3003c3df 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -293,7 +293,7 @@ export class Member { async setMedia(localMedia: LocalMedia, previousMedia: LocalMedia): Promise { const {connection} = this; if (connection) { - connection.localMedia = connection.localMedia.replaceClone(connection.localMedia, previousMedia); + connection.localMedia = localMedia.replaceClone(connection.localMedia, previousMedia); await connection.peerCall?.setMedia(connection.localMedia, connection.logItem); } } diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index c34ab85b..d6439faa 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -30,6 +30,9 @@ export class MediaDevicesWrapper implements IMediaDevices { async getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise { const stream = await this.mediaDevices.getUserMedia(this.getUserMediaContraints(audio, video)); + stream.addEventListener("removetrack", evt => { + console.log(`removing track ${evt.track.id} (${evt.track.kind}) from stream ${stream.id}`); + }); return stream as Stream; } From 5527e2b22cd1adf1fb91e42c10c9385e0572eb99 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 5 Jul 2022 11:02:36 +0200 Subject: [PATCH 163/435] also remove deferred log items from open list when finishing them otherwise they end up in the logs twice when exporting --- src/logging/Logger.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/logging/Logger.ts b/src/logging/Logger.ts index 395181ef..37a3326c 100644 --- a/src/logging/Logger.ts +++ b/src/logging/Logger.ts @@ -159,6 +159,11 @@ export class Logger implements ILogger { this._openItems.clear(); } + /** @internal */ + _removeItemFromOpenList(item: LogItem): void { + this._openItems.delete(item); + } + /** @internal */ _persistItem(item: LogItem, filter?: LogFilter, forced?: boolean): void { for (var i = 0; i < this.reporters.length; i += 1) { @@ -186,6 +191,7 @@ class DeferredPersistRootLogItem extends LogItem { finish() { super.finish(); (this._logger as Logger)._persistItem(this, undefined, false); + (this._logger as Logger)._removeItemFromOpenList(this); } forceFinish() { From 206ac6e2dd2e85dbee2c908cad3a3c89fd41f6ed Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 5 Jul 2022 18:22:36 +0200 Subject: [PATCH 164/435] WIP: prevent stream id from changing when upgrading call --- src/matrix/calls/PeerCall.ts | 84 ++++++++++++++++++------------ src/matrix/calls/group/Member.ts | 1 + src/platform/types/MediaDevices.ts | 2 + 3 files changed, 53 insertions(+), 34 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 1e22a2ff..29eccf2c 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -999,49 +999,55 @@ export class PeerCall implements IDisposable { private updateLocalMedia(localMedia: LocalMedia, logItem: ILogItem): Promise { return logItem.wrap("updateLocalMedia", async log => { - const oldMedia = this.localMedia; - this.localMedia = localMedia; + const senders = this.peerConnection.getSenders(); const applyStream = async (oldStream: Stream | undefined, stream: Stream | undefined, streamPurpose: SDPStreamMetadataPurpose) => { const applyTrack = async (oldTrack: Track | undefined, newTrack: Track | undefined) => { - if (!oldTrack && newTrack) { - log.wrap(`adding ${streamPurpose} ${newTrack.kind} track`, log => { - const sender = this.peerConnection.addTrack(newTrack, stream!); - this.options.webRTC.prepareSenderForPurpose(this.peerConnection, sender, streamPurpose); - }); - } else if (oldTrack) { - const sender = this.peerConnection.getSenders().find(s => s.track && s.track.id === oldTrack.id); - if (sender) { - if (newTrack && oldTrack.id !== newTrack.id) { - try { - await log.wrap(`replacing ${streamPurpose} ${newTrack.kind} track`, log => { - return sender.replaceTrack(newTrack); - }); - } catch (err) { - // can't replace the track without renegotiating{ - log.wrap(`adding and removing ${streamPurpose} ${newTrack.kind} track`, log => { - this.peerConnection.removeTrack(sender); - const newSender = this.peerConnection.addTrack(newTrack); - this.options.webRTC.prepareSenderForPurpose(this.peerConnection, newSender, streamPurpose); - }); - } - } else if (!newTrack) { - log.wrap(`removing ${streamPurpose} ${sender.track!.kind} track`, log => { - this.peerConnection.removeTrack(sender); - }); - } else { - log.log(`${streamPurpose} ${oldTrack.kind} track hasn't changed`); - } + const oldSender = senders.find(s => s.track === oldTrack); + const streamToKeep = (oldStream ?? stream)!; + if (streamToKeep !== stream) { + if (oldTrack) { + streamToKeep.removeTrack(oldTrack); } - // TODO: should we do something if we didn't find the sender? e.g. some other code already removed the sender but didn't update localMedia + if (newTrack) { + streamToKeep.addTrack(newTrack); + } + } + if (newTrack && oldSender) { + try { + await log.wrap(`attempting to replace ${streamPurpose} ${newTrack.kind} track`, log => { + return oldSender.replaceTrack(newTrack); + }); + // replaceTrack succeeded, nothing left to do + return; + } catch (err) {} + } + if(oldSender) { + log.wrap(`removing ${streamPurpose} ${oldSender.track!.kind} track`, log => { + this.peerConnection.removeTrack(oldSender); + }); + } + if (newTrack) { + log.wrap(`adding ${streamPurpose} ${newTrack.kind} track`, log => { + const newSender = this.peerConnection.addTrack(newTrack, streamToKeep); + this.options.webRTC.prepareSenderForPurpose(this.peerConnection, newSender, streamPurpose); + }); } } - + if (!oldStream && !stream) { + return; + } await applyTrack(getStreamAudioTrack(oldStream), getStreamAudioTrack(stream)); await applyTrack(getStreamVideoTrack(oldStream), getStreamVideoTrack(stream)); }; - await applyStream(oldMedia?.userMedia, localMedia?.userMedia, SDPStreamMetadataPurpose.Usermedia); - await applyStream(oldMedia?.screenShare, localMedia?.screenShare, SDPStreamMetadataPurpose.Screenshare); + await applyStream(this.localMedia?.userMedia, localMedia?.userMedia, SDPStreamMetadataPurpose.Usermedia); + await applyStream(this.localMedia?.screenShare, localMedia?.screenShare, SDPStreamMetadataPurpose.Screenshare); + // we explicitly don't replace this.localMedia if already set + // as we need to keep the old stream so the stream id doesn't change + // instead we add and remove tracks in the stream in applyTrack + if (!this.localMedia) { + this.localMedia = localMedia; + } // TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method }); } @@ -1158,3 +1164,13 @@ function enableTransceiver(transceiver: Transceiver, enabled: boolean, exclusive } } } + +/** + * tests to write: + * + * upgradeCall: adding a track with setMedia calls the correct methods on the peerConnection + * upgradeCall: removing a track with setMedia calls the correct methods on the peerConnection + * upgradeCall: replacing compatible track with setMedia calls the correct methods on the peerConnection + * upgradeCall: replacing incompatible track (sender.replaceTrack throws) with setMedia calls the correct methods on the peerConnection + * + * */ diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 3003c3df..b6461e6a 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -293,6 +293,7 @@ export class Member { async setMedia(localMedia: LocalMedia, previousMedia: LocalMedia): Promise { const {connection} = this; if (connection) { + // TODO: see if we can simplify this connection.localMedia = localMedia.replaceClone(connection.localMedia, previousMedia); await connection.peerCall?.setMedia(connection.localMedia, connection.logItem); } diff --git a/src/platform/types/MediaDevices.ts b/src/platform/types/MediaDevices.ts index 1b5f7afd..c5439ac4 100644 --- a/src/platform/types/MediaDevices.ts +++ b/src/platform/types/MediaDevices.ts @@ -56,6 +56,8 @@ export interface Stream { clone(): Stream; addEventListener(type: K, listener: (this: Stream, ev: StreamEventMap[K]) => any, options?: boolean | AddEventListenerOptions): void; removeEventListener(type: K, listener: (this: Stream, ev: StreamEventMap[K]) => any, options?: boolean | EventListenerOptions): void; + addTrack(track: Track); + removeTrack(track: Track); } export enum TrackKind { From f187a51c97ea09c8c8af90136eb8fe64b6f14337 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 7 Jul 2022 15:36:30 +0200 Subject: [PATCH 165/435] stop replaced track in PeerCall --- src/matrix/calls/LocalMedia.ts | 11 +---------- src/matrix/calls/PeerCall.ts | 1 + 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 922e6861..90a257fb 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -47,16 +47,7 @@ export class LocalMedia { if (oldOriginalStream?.id === newStream?.id) { return oldCloneStream; } else { - const clonedStream = newStream?.clone(); - // clonedStream.addEventListener("removetrack", evt => { - // console.log(`removing track ${evt.track.id} (${evt.track.kind}) from clonedStream ${clonedStream.id}`); - // }); - // console.log("got cloned tracks", Array.from(clonedStream.getTracks()).map((t: MediaStreamTrack) => { return {kind: t.kind, id: t.id};}), "from new stream", Array.from(newStream.getTracks()).map((t: MediaStreamTrack) => { return {kind: t.kind, id: t.id};})); - // console.log("stopping old audio stream", getStreamAudioTrack(oldCloneStream)?.id); - // console.log("stopping old video stream", getStreamVideoTrack(oldCloneStream)?.id); - getStreamAudioTrack(oldCloneStream)?.stop(); - getStreamVideoTrack(oldCloneStream)?.stop(); - return clonedStream; + return newStream?.clone(); } } return new LocalMedia( diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 29eccf2c..118e5de4 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -1007,6 +1007,7 @@ export class PeerCall implements IDisposable { if (streamToKeep !== stream) { if (oldTrack) { streamToKeep.removeTrack(oldTrack); + oldTrack.stop(); } if (newTrack) { streamToKeep.addTrack(newTrack); From 2f08cd89843062a22c545d2e45d97bd9843d7ae6 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 7 Jul 2022 15:36:49 +0200 Subject: [PATCH 166/435] clone localMedia in Member when connection, like we do for setMedia --- src/matrix/calls/group/GroupCall.ts | 3 +-- src/matrix/calls/group/Member.ts | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 087dc1ff..32052cc3 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -510,8 +510,7 @@ export class GroupCall extends EventEmitter<{change: never}> { const logItem = joinedData.membersLogItem.child({l: "member", id: memberKey}); logItem.set("sessionId", member.sessionId); log.wrap({l: "connect", id: memberKey}, log => { - // Safari can't send a MediaStream to multiple sources, so clone it - const connectItem = member.connect(joinedData.localMedia.clone(), joinedData.localMuteSettings, logItem); + const connectItem = member.connect(joinedData.localMedia, joinedData.localMuteSettings, logItem); if (connectItem) { log.refDetached(connectItem); } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index b6461e6a..9f370d6f 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -116,7 +116,8 @@ export class Member { if (this.connection) { return; } - const connection = new MemberConnection(localMedia, localMuteSettings, memberLogItem); + // Safari can't send a MediaStream to multiple sources, so clone it + const connection = new MemberConnection(localMedia.clone(), localMuteSettings, memberLogItem); this.connection = connection; let connectLogItem; connection.logItem.wrap("connect", async log => { @@ -293,7 +294,6 @@ export class Member { async setMedia(localMedia: LocalMedia, previousMedia: LocalMedia): Promise { const {connection} = this; if (connection) { - // TODO: see if we can simplify this connection.localMedia = localMedia.replaceClone(connection.localMedia, previousMedia); await connection.peerCall?.setMedia(connection.localMedia, connection.logItem); } From e9649ec7c2dbc0f69ef3719f0b77ede611feb31b Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 7 Jul 2022 15:47:09 +0200 Subject: [PATCH 167/435] different streams never have the same id, even for same devices --- src/matrix/calls/LocalMedia.ts | 16 +++------------- src/matrix/calls/group/GroupCall.ts | 3 +-- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 90a257fb..f02cd11b 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -40,8 +40,6 @@ export class LocalMedia { /** @internal */ replaceClone(oldClone: LocalMedia | undefined, oldOriginal: LocalMedia | undefined): LocalMedia { - let userMedia; - let screenShare; const cloneOrAdoptStream = (oldOriginalStream: Stream | undefined, oldCloneStream: Stream | undefined, newStream: Stream | undefined): Stream | undefined => { let stream; if (oldOriginalStream?.id === newStream?.id) { @@ -63,16 +61,8 @@ export class LocalMedia { } dispose() { - this.stopExcept(undefined); - } - - stopExcept(newMedia: LocalMedia | undefined) { - if(newMedia?.userMedia?.id !== this.userMedia?.id) { - getStreamAudioTrack(this.userMedia)?.stop(); - getStreamVideoTrack(this.userMedia)?.stop(); - } - if(newMedia?.screenShare?.id !== this.screenShare?.id) { - getStreamVideoTrack(this.screenShare)?.stop(); - } + getStreamAudioTrack(this.userMedia)?.stop(); + getStreamVideoTrack(this.userMedia)?.stop(); + getStreamVideoTrack(this.screenShare)?.stop(); } } diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 32052cc3..07aa2a08 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -175,8 +175,7 @@ export class GroupCall extends EventEmitter<{change: never}> { await Promise.all(Array.from(this._members.values()).map(m => { return m.setMedia(localMedia, oldMedia); })); - - oldMedia?.stopExcept(localMedia); + oldMedia?.dispose(); } } From 3346f68d25e4ba6b6d4d09a8b961b67627cd354c Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 12 Jul 2022 11:59:52 +0200 Subject: [PATCH 168/435] WIP --- src/matrix/calls/PeerCall.ts | 62 +++++++++++++++++--------------- src/matrix/calls/group/Member.ts | 2 +- src/platform/web/dom/WebRTC.ts | 15 +++++++- 3 files changed, 49 insertions(+), 30 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 118e5de4..fc5b48a6 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -177,6 +177,7 @@ export class PeerCall implements IDisposable { if (this._state !== CallState.Fledgling) { return; } + log.set("signalingState", this.peerConnection.signalingState); this.direction = CallDirection.Outbound; this.setState(CallState.CreateOffer, log); this.localMuteSettings = localMuteSettings; @@ -309,35 +310,39 @@ export class PeerCall implements IDisposable { }, async log => { logItem = log; - switch (message.type) { - case EventType.Invite: - if (this.callId !== message.content.call_id) { - await this.handleInviteGlare(message.content, partyId, log); - } else { + const callIdMatches = this.callId === message.content.call_id; + + if (message.type === EventType.Invite && !callIdMatches) { + await this.handleInviteGlare(message.content, partyId, log); + } else if (callIdMatches) { + switch (message.type) { + case EventType.Invite: await this.handleFirstInvite(message.content, partyId, log); - } - break; - case EventType.Answer: - await this.handleAnswer(message.content, partyId, log); - break; - case EventType.Negotiate: - await this.onNegotiateReceived(message.content, log); - break; - case EventType.Candidates: - await this.handleRemoteIceCandidates(message.content, partyId, log); - break; - case EventType.SDPStreamMetadataChanged: - case EventType.SDPStreamMetadataChangedPrefix: - this.updateRemoteSDPStreamMetadata(message.content[SDPStreamMetadataKey], log); - break; - case EventType.Hangup: - // TODO: this is a bit hacky, double check its what we need - log.set("reason", message.content.reason); - this.terminate(CallParty.Remote, message.content.reason ?? CallErrorCode.UserHangup, log); - break; - default: - log.log(`Unknown event type for call: ${message.type}`); - break; + break; + case EventType.Answer: + await this.handleAnswer(message.content, partyId, log); + break; + case EventType.Negotiate: + await this.onNegotiateReceived(message.content, log); + break; + case EventType.Candidates: + await this.handleRemoteIceCandidates(message.content, partyId, log); + break; + case EventType.SDPStreamMetadataChanged: + case EventType.SDPStreamMetadataChangedPrefix: + this.updateRemoteSDPStreamMetadata(message.content[SDPStreamMetadataKey], log); + break; + case EventType.Hangup: + // TODO: this is a bit hacky, double check its what we need + log.set("reason", message.content.reason); + this.terminate(CallParty.Remote, message.content.reason ?? CallErrorCode.UserHangup, log); + break; + default: + log.log(`Unknown event type for call: ${message.type}`); + break; + } + } else if (!callIdMatches) { + log.set("wrongCallId", true); } }); return logItem; @@ -879,6 +884,7 @@ export class PeerCall implements IDisposable { this.setState(CallState.Ended, log); this.localMedia = undefined; + // TODO: change signalingState to connectionState? if (this.peerConnection && this.peerConnection.signalingState !== 'closed') { this.peerConnection.close(); } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 9f370d6f..9e87fa22 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -283,7 +283,7 @@ export class Member { } } if (!hasBeenDequeued) { - syncLog.refDetached(connection.logItem.log({l: "queued signalling message", type: message.type})); + syncLog.refDetached(connection.logItem.log({l: "queued signalling message", type: message.type, seq: message.content.seq})); } } else { syncLog.log({l: "member not connected", userId: this.userId, deviceId: this.deviceId}); diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 61d481d4..05e032ca 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -24,11 +24,24 @@ const SPEAKING_SAMPLE_COUNT = 8; // samples export class DOMWebRTC implements WebRTC { createPeerConnection(forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize): PeerConnection { - return new RTCPeerConnection({ + const peerConn = new RTCPeerConnection({ iceTransportPolicy: forceTURN ? 'relay' : undefined, iceServers: turnServers, iceCandidatePoolSize: iceCandidatePoolSize, }) as PeerConnection; + return new Proxy(peerConn, { + get(target, prop, receiver) { + if (prop === "close") { + console.trace("calling peerConnection.close"); + } + const value = target[prop]; + if (typeof value === "function") { + return value.bind(target); + } else { + return value; + } + } + }); } prepareSenderForPurpose(peerConnection: PeerConnection, sender: Sender, purpose: SDPStreamMetadataPurpose): void { From bb2e63b05ba568918f6e7caf56dc33e7f0b60f6f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 20 Sep 2022 17:27:39 +0200 Subject: [PATCH 169/435] don't queue messages for different callIds so last seq doesn't corrupt this includes handling invite glares differently --- src/matrix/calls/PeerCall.ts | 137 ++++++++++++++++++------------- src/matrix/calls/group/Member.ts | 71 +++++++++++----- 2 files changed, 131 insertions(+), 77 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index fc5b48a6..d004dedc 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -53,6 +53,12 @@ export type Options = { sendSignallingMessage: (message: SignallingMessage, log: ILogItem) => Promise; }; +export enum IncomingMessageAction { + InviteGlare, + Handle, + Ignore +}; + export class RemoteMedia { constructor(public userMedia?: Stream | undefined, public screenShare?: Stream | undefined) {} } @@ -299,6 +305,17 @@ export class PeerCall implements IDisposable { await this.sendHangupWithCallId(this.callId, errorCode, log); } + getMessageAction(message: SignallingMessage): IncomingMessageAction { + const callIdMatches = this.callId === message.content.call_id; + if (message.type === EventType.Invite && !callIdMatches) { + return IncomingMessageAction.InviteGlare; + } if (callIdMatches) { + return IncomingMessageAction.Handle; + } else { + return IncomingMessageAction.Ignore; + } + } + handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId, log: ILogItem): ILogItem { // return logItem item immediately so it can be references in sync manner let logItem; @@ -309,40 +326,35 @@ export class PeerCall implements IDisposable { payload: message.content }, async log => { logItem = log; - - const callIdMatches = this.callId === message.content.call_id; - - if (message.type === EventType.Invite && !callIdMatches) { - await this.handleInviteGlare(message.content, partyId, log); - } else if (callIdMatches) { - switch (message.type) { - case EventType.Invite: - await this.handleFirstInvite(message.content, partyId, log); - break; - case EventType.Answer: - await this.handleAnswer(message.content, partyId, log); - break; - case EventType.Negotiate: - await this.onNegotiateReceived(message.content, log); - break; - case EventType.Candidates: - await this.handleRemoteIceCandidates(message.content, partyId, log); - break; - case EventType.SDPStreamMetadataChanged: - case EventType.SDPStreamMetadataChangedPrefix: - this.updateRemoteSDPStreamMetadata(message.content[SDPStreamMetadataKey], log); - break; - case EventType.Hangup: - // TODO: this is a bit hacky, double check its what we need - log.set("reason", message.content.reason); - this.terminate(CallParty.Remote, message.content.reason ?? CallErrorCode.UserHangup, log); - break; - default: - log.log(`Unknown event type for call: ${message.type}`); - break; - } - } else if (!callIdMatches) { + if (this.getMessageAction(message) !== IncomingMessageAction.Handle) { log.set("wrongCallId", true); + return; + } + switch (message.type) { + case EventType.Invite: + await this.handleFirstInvite(message.content, partyId, log); + break; + case EventType.Answer: + await this.handleAnswer(message.content, partyId, log); + break; + case EventType.Negotiate: + await this.onNegotiateReceived(message.content, log); + break; + case EventType.Candidates: + await this.handleRemoteIceCandidates(message.content, partyId, log); + break; + case EventType.SDPStreamMetadataChanged: + case EventType.SDPStreamMetadataChangedPrefix: + this.updateRemoteSDPStreamMetadata(message.content[SDPStreamMetadataKey], log); + break; + case EventType.Hangup: + // TODO: this is a bit hacky, double check its what we need + log.set("reason", message.content.reason); + this.terminate(CallParty.Remote, message.content.reason ?? CallErrorCode.UserHangup, log); + break; + default: + log.log(`Unknown event type for call: ${message.type}`); + break; } }); return logItem; @@ -434,30 +446,43 @@ export class PeerCall implements IDisposable { } }; - private async handleInviteGlare(content: MCallInvite, partyId: PartyId, log: ILogItem): Promise { - // this is only called when the ids are different - const newCallId = content.call_id; - if (this.callId! > newCallId) { - log.log( - "Glare detected: answering incoming call " + newCallId + - " and canceling outgoing call " - ); - // How do we interrupt `call()`? well, perhaps we need to not just await InviteSent but also CreateAnswer? - if (this._state === CallState.Fledgling || this._state === CallState.CreateOffer) { - // TODO: don't send invite! - } else { - await this.sendHangupWithCallId(this.callId, CallErrorCode.Replaced, log); - } - await this.handleInvite(content, partyId, log); - // TODO: need to skip state check - await this.answer(this.localMedia!, this.localMuteSettings!, log); - } else { - log.log( - "Glare detected: rejecting incoming call " + newCallId + - " and keeping outgoing call " - ); - await this.sendHangupWithCallId(newCallId, CallErrorCode.Replaced, log); + /** + * @returns {boolean} whether or not this call should be replaced + * */ + handleInviteGlare(message: SignallingMessage, partyId: PartyId, log: ILogItem): {shouldReplace: boolean, log?: ILogItem} { + if (message.type !== EventType.Invite) { + return {shouldReplace: false}; } + + const {content} = message; + const newCallId = content.call_id; + const shouldReplace = this.callId! > newCallId; + + let logItem; + log.wrap("handling call glare", async log => { + logItem = log; + if (shouldReplace) { + log.log( + "Glare detected: answering incoming call " + newCallId + + " and canceling outgoing call " + ); + // TODO: How do we interrupt `call()`? well, perhaps we need to not just await InviteSent but also CreateAnswer? + if (this._state !== CallState.Fledgling && this._state !== CallState.CreateOffer) { + await this.sendHangupWithCallId(this.callId, CallErrorCode.Replaced, log); + } + // since this method isn't awaited, we dispose ourselves once we hung up + this.close(CallErrorCode.Replaced, log); + this.dispose(); + } else { + log.log( + "Glare detected: rejecting incoming call " + newCallId + + " and keeping outgoing call " + ); + await this.sendHangupWithCallId(newCallId, CallErrorCode.Replaced, log); + } + }); + + return {shouldReplace, log: logItem}; } private handleHangupReceived(content: MCallHangupReject, log: ILogItem) { diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 9e87fa22..98e2aa4d 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {PeerCall, CallState} from "../PeerCall"; +import {PeerCall, CallState, IncomingMessageAction} from "../PeerCall"; import {makeTxnId, makeId} from "../../common"; import {EventType, CallErrorCode} from "../callEventTypes"; import {formatToDeviceMessagesPayload} from "../../common"; @@ -251,7 +251,7 @@ export class Member { } /** @internal */ - handleDeviceMessage(message: SignallingMessage, syncLog: ILogItem): void{ + handleDeviceMessage(message: SignallingMessage, syncLog: ILogItem): void { const {connection} = this; if (connection) { const destSessionId = message.content.dest_session_id; @@ -260,36 +260,65 @@ export class Member { syncLog.refDetached(logItem); return; } + // if there is no peerCall, we either create it with an invite and Handle is implied or we'll ignore it + let action = IncomingMessageAction.Handle; + if (connection.peerCall) { + action = connection.peerCall.getMessageAction(message); + // deal with glare and replacing the call before creating new calls + if (action === IncomingMessageAction.InviteGlare) { + const {shouldReplace, log} = connection.peerCall.handleInviteGlare(message, this.deviceId, connection.logItem); + if (log) { + syncLog.refDetached(log); + } + if (shouldReplace) { + connection.peerCall = undefined; + } + } + } if (message.type === EventType.Invite && !connection.peerCall) { connection.peerCall = this._createPeerCall(message.content.call_id); } - const idx = sortedIndex(connection.queuedSignallingMessages, message, (a, b) => a.content.seq - b.content.seq); - connection.queuedSignallingMessages.splice(idx, 0, message); - let hasBeenDequeued = false; - if (connection.peerCall) { - while ( - connection.queuedSignallingMessages.length && ( - connection.lastProcessedSeqNr === undefined || - connection.queuedSignallingMessages[0].content.seq === connection.lastProcessedSeqNr + 1 - ) - ) { - const dequeuedMessage = connection.queuedSignallingMessages.shift()!; - if (dequeuedMessage === message) { - hasBeenDequeued = true; + if (action === IncomingMessageAction.Handle) { + const idx = sortedIndex(connection.queuedSignallingMessages, message, (a, b) => a.content.seq - b.content.seq); + connection.queuedSignallingMessages.splice(idx, 0, message); + if (connection.peerCall) { + const hasNewMessageBeenDequeued = this.dequeueSignallingMessages(connection, connection.peerCall, message, syncLog); + if (!hasNewMessageBeenDequeued) { + syncLog.refDetached(connection.logItem.log({l: "queued signalling message", type: message.type, seq: message.content.seq})); } - const item = connection.peerCall!.handleIncomingSignallingMessage(dequeuedMessage, this.deviceId, connection.logItem); - syncLog.refDetached(item); - connection.lastProcessedSeqNr = dequeuedMessage.content.seq; } - } - if (!hasBeenDequeued) { - syncLog.refDetached(connection.logItem.log({l: "queued signalling message", type: message.type, seq: message.content.seq})); + } else if (action === IncomingMessageAction.Ignore && connection.peerCall) { + const logItem = connection.logItem.log({l: "ignoring to_device event with wrong call_id", callId: message.content.call_id, type: message.type}); + syncLog.refDetached(logItem); } } else { syncLog.log({l: "member not connected", userId: this.userId, deviceId: this.deviceId}); } } + private dequeueSignallingMessages(connection: MemberConnection, peerCall: PeerCall, newMessage: SignallingMessage, syncLog: ILogItem): boolean { + let hasNewMessageBeenDequeued = false; + while ( + connection.queuedSignallingMessages.length && ( + connection.lastProcessedSeqNr === undefined || + connection.queuedSignallingMessages[0].content.seq === connection.lastProcessedSeqNr + 1 + ) + ) { + const message = connection.queuedSignallingMessages.shift()!; + if (message === newMessage) { + hasNewMessageBeenDequeued = true; + } + // ignore items in the queue that should not be handled and prevent + // the lastProcessedSeqNr being corrupted with the `seq` for other call ids + if (peerCall.getMessageAction(message) === IncomingMessageAction.Handle) { + const item = peerCall.handleIncomingSignallingMessage(message, this.deviceId, connection.logItem); + syncLog.refDetached(item); + connection.lastProcessedSeqNr = message.content.seq; + } + } + return hasNewMessageBeenDequeued; + } + /** @internal */ async setMedia(localMedia: LocalMedia, previousMedia: LocalMedia): Promise { const {connection} = this; From 7ce5cdfc4ae4840028af2a3ea96ea431920c71bd Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 22 Sep 2022 13:19:56 +0200 Subject: [PATCH 170/435] prevent concurrent calls of OlmEncryption.encrypt to OOM olm wasm heap This is being triggered by connecting to many call members at once, while encrypting the signaling messages. This keeps many olm.Session objects into memory at the same time, which makes olm run out of wasm heap memory. --- src/matrix/e2ee/olm/Encryption.ts | 38 ++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/src/matrix/e2ee/olm/Encryption.ts b/src/matrix/e2ee/olm/Encryption.ts index dcc9f0b1..5fd1f25b 100644 --- a/src/matrix/e2ee/olm/Encryption.ts +++ b/src/matrix/e2ee/olm/Encryption.ts @@ -21,6 +21,7 @@ import {createSessionEntry} from "./Session"; import type {OlmMessage, OlmPayload, OlmEncryptedMessageContent} from "./types"; import type {Account} from "../Account"; import type {LockMap} from "../../../utils/LockMap"; +import {Lock, MultiLock, ILock} from "../../../utils/Lock"; import type {Storage} from "../../storage/idb/Storage"; import type {Transaction} from "../../storage/idb/Transaction"; import type {DeviceIdentity} from "../../storage/idb/stores/DeviceIdentityStore"; @@ -62,6 +63,9 @@ const OTK_ALGORITHM = "signed_curve25519"; const MAX_BATCH_SIZE = 20; export class Encryption { + + private _batchLocks: Array; + constructor( private readonly account: Account, private readonly pickleKey: string, @@ -71,14 +75,42 @@ export class Encryption { private readonly ownUserId: string, private readonly olmUtil: Olm.Utility, private readonly senderKeyLock: LockMap - ) {} + ) { + this._batchLocks = new Array(MAX_BATCH_SIZE); + for (let i = 0; i < MAX_BATCH_SIZE; i += 1) { + this._batchLocks[i] = new Lock(); + } + } + + /** A hack to prevent olm OOMing when `encrypt` is called several times concurrently, + * which is the case when encrypting voip signalling message to send over to_device. + * A better fix will be to extract the common bits from megolm/KeyLoader in a super class + * and have some sort of olm/SessionLoader that is shared between encryption and decryption + * and only keeps the olm session in wasm memory for a brief moment, like we already do for RoomKeys, + * and get the benefit of an optimal cache at the same time. + * */ + private async _takeBatchLock(amount: number): Promise { + const locks = this._batchLocks.filter(l => !l.isTaken).slice(0, amount); + if (locks.length < amount) { + const takenLocks = this._batchLocks.filter(l => l.isTaken).slice(0, amount - locks.length); + locks.push(...takenLocks); + } + await Promise.all(locks.map(l => l.take())); + return new MultiLock(locks); + } async encrypt(type: string, content: Record, devices: DeviceIdentity[], hsApi: HomeServerApi, log: ILogItem): Promise { let messages: EncryptedMessage[] = []; for (let i = 0; i < devices.length ; i += MAX_BATCH_SIZE) { const batchDevices = devices.slice(i, i + MAX_BATCH_SIZE); - const batchMessages = await this._encryptForMaxDevices(type, content, batchDevices, hsApi, log); - messages = messages.concat(batchMessages); + const batchLock = await this._takeBatchLock(batchDevices.length); + try { + const batchMessages = await this._encryptForMaxDevices(type, content, batchDevices, hsApi, log); + messages = messages.concat(batchMessages); + } + finally { + batchLock.release(); + } } return messages; } From 8aa62b257326cdf94ac0c168d83174cfa169b06c Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 25 Sep 2022 21:39:38 +0100 Subject: [PATCH 171/435] don't ignore end-of-ICE candidates --- src/matrix/calls/PeerCall.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index d004dedc..9a67fae1 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -818,14 +818,16 @@ export class PeerCall implements IDisposable { private async addIceCandidates(candidates: RTCIceCandidate[], log: ILogItem): Promise { for (const candidate of candidates) { + let logItem; if ( (candidate.sdpMid === null || candidate.sdpMid === undefined) && (candidate.sdpMLineIndex === null || candidate.sdpMLineIndex === undefined) ) { - log.log(`Ignoring remote ICE candidate with no sdpMid or sdpMLineIndex`); - continue; + logItem = log.log(`Got remote ICE end-of-ICE candidates`); + } + else { + logItem = log.log(`Adding remote ICE ${candidate.sdpMid} candidate: ${candidate.candidate}`); } - const logItem = log.log(`Adding remote ICE ${candidate.sdpMid} candidate: ${candidate.candidate}`); try { await this.peerConnection.addIceCandidate(candidate); } catch (err) { From 31e67142bdc1ffa2d503c7492371a703a26e07b2 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 25 Sep 2022 22:11:35 +0100 Subject: [PATCH 172/435] typo --- src/matrix/calls/PeerCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 9a67fae1..1ee945c5 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -823,7 +823,7 @@ export class PeerCall implements IDisposable { (candidate.sdpMid === null || candidate.sdpMid === undefined) && (candidate.sdpMLineIndex === null || candidate.sdpMLineIndex === undefined) ) { - logItem = log.log(`Got remote ICE end-of-ICE candidates`); + logItem = log.log(`Got remote end-of-ICE candidates`); } else { logItem = log.log(`Adding remote ICE ${candidate.sdpMid} candidate: ${candidate.candidate}`); From af2098327bc023062da0b6ac6607f1051eea9549 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 26 Sep 2022 00:47:12 +0100 Subject: [PATCH 173/435] first cut at implementing TURN --- src/matrix/Session.js | 13 +++++++++++++ src/matrix/calls/CallHandler.ts | 6 ++++++ src/matrix/net/HomeServerApi.ts | 6 +++++- 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index f82ad555..de05f040 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -513,6 +513,7 @@ export class Session { // TODO: what can we do if this throws? await txn.complete(); } + await this._updateTurnServers(); // enable session backup, this requests the latest backup version if (!this._keyBackup.get()) { if (dehydratedDevice) { @@ -559,6 +560,18 @@ export class Session { } } + async _updateTurnServers() { + const turnServersData = await this._hsApi.getTurnServers().response(); + this._callHandler.setTurnServers({ + urls: turnServerData.uris, + username: turnServerData.username, + credential: turnServerData.password, + }); + if (turnServersData.ttl > 0) { + setTimeout(this._updateTurnServers, turnServersData.ttl * 1000); + } + } + async _getPendingEventsByRoom(txn) { const pendingEvents = await txn.pendingEvents.getAll(); return pendingEvents.reduce((groups, pe) => { diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index ca58c4a8..93491e07 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -75,6 +75,12 @@ export class CallHandler implements RoomStateHandler { this._loadCallEntries(callEntries, txn); } + async setTurnServers(turnServers: RTCIceServer) { + this.options.turnServers = turnServers; + this.groupCallOptions.turnServers = turnServers; + // TODO: we should update any ongoing peerconnections if the TURN server details have changed + } + private async _getLoadTxn(): Promise { const names = this.options.storage.storeNames; const txn = await this.options.storage.readTxn([ diff --git a/src/matrix/net/HomeServerApi.ts b/src/matrix/net/HomeServerApi.ts index 30406c34..828a1c82 100644 --- a/src/matrix/net/HomeServerApi.ts +++ b/src/matrix/net/HomeServerApi.ts @@ -305,10 +305,14 @@ export class HomeServerApi { createRoom(payload: Record, options?: BaseRequestOptions): IHomeServerRequest { return this._post(`/createRoom`, {}, payload, options); } - + setAccountData(ownUserId: string, type: string, content: Record, options?: BaseRequestOptions): IHomeServerRequest { return this._put(`/user/${encodeURIComponent(ownUserId)}/account_data/${encodeURIComponent(type)}`, {}, content, options); } + + getTurnServer(options?: BaseRequestOptions): IHomeServerRequest { + return this._get(`/voip/turnServer`, undefined, undefined, options); + } } import {Request as MockRequest} from "../../mocks/Request.js"; From eab87e5157c795d255ae3760515b29ddd477aec8 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 26 Sep 2022 00:52:41 +0100 Subject: [PATCH 174/435] fix plurals --- src/matrix/Session.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index de05f040..527c0e00 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -561,14 +561,14 @@ export class Session { } async _updateTurnServers() { - const turnServersData = await this._hsApi.getTurnServers().response(); + const turnServerData = await this._hsApi.getTurnServer().response(); this._callHandler.setTurnServers({ urls: turnServerData.uris, username: turnServerData.username, credential: turnServerData.password, }); - if (turnServersData.ttl > 0) { - setTimeout(this._updateTurnServers, turnServersData.ttl * 1000); + if (turnServerData.ttl > 0) { + setTimeout(this._updateTurnServers, turnServerData.ttl * 1000); } } From 3d7852a469bd4584798dad39e834cbe38451f763 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 26 Sep 2022 01:07:24 +0100 Subject: [PATCH 175/435] set array of turnservers --- src/matrix/Session.js | 4 ++-- src/matrix/calls/CallHandler.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 527c0e00..3c49182d 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -562,11 +562,11 @@ export class Session { async _updateTurnServers() { const turnServerData = await this._hsApi.getTurnServer().response(); - this._callHandler.setTurnServers({ + this._callHandler.setTurnServers([{ urls: turnServerData.uris, username: turnServerData.username, credential: turnServerData.password, - }); + }]); if (turnServerData.ttl > 0) { setTimeout(this._updateTurnServers, turnServerData.ttl * 1000); } diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 93491e07..183ec5a1 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -75,7 +75,7 @@ export class CallHandler implements RoomStateHandler { this._loadCallEntries(callEntries, txn); } - async setTurnServers(turnServers: RTCIceServer) { + async setTurnServers(turnServers: RTCIceServer[]) { this.options.turnServers = turnServers; this.groupCallOptions.turnServers = turnServers; // TODO: we should update any ongoing peerconnections if the TURN server details have changed From 6570ec46f4a9ba500215b7b15f65e858f74cb0a2 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 26 Sep 2022 09:02:22 +0100 Subject: [PATCH 176/435] setTurnServers doesn't need to be async --- src/matrix/calls/CallHandler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 183ec5a1..17aa0e55 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -75,7 +75,7 @@ export class CallHandler implements RoomStateHandler { this._loadCallEntries(callEntries, txn); } - async setTurnServers(turnServers: RTCIceServer[]) { + setTurnServers(turnServers: RTCIceServer[]) { this.options.turnServers = turnServers; this.groupCallOptions.turnServers = turnServers; // TODO: we should update any ongoing peerconnections if the TURN server details have changed From f4e633beb4faebe1eaced7ea64f3903ecc57f08b Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:24:14 +0200 Subject: [PATCH 177/435] support onfirstsubscribe callback in ROV will use it to start polling for turnServer updates --- src/observable/value/RetainedObservableValue.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/observable/value/RetainedObservableValue.ts b/src/observable/value/RetainedObservableValue.ts index edfb6c15..16058f8e 100644 --- a/src/observable/value/RetainedObservableValue.ts +++ b/src/observable/value/RetainedObservableValue.ts @@ -17,15 +17,17 @@ limitations under the License. import {ObservableValue} from "./ObservableValue"; export class RetainedObservableValue extends ObservableValue { - private _freeCallback: () => void; - constructor(initialValue: T, freeCallback: () => void) { + constructor(initialValue: T, private freeCallback: () => void, private startCallback: () => void = () => {}) { super(initialValue); - this._freeCallback = freeCallback; + } + + onSubscribeFirst() { + this.startCallback(); } onUnsubscribeLast() { super.onUnsubscribeLast(); - this._freeCallback(); + this.freeCallback(); } } From 035ead0d5b47c7ebb742f74aae0e4949f556fcb9 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:24:41 +0200 Subject: [PATCH 178/435] implement polling of voip turnServer settings from HS in separate class --- src/matrix/calls/TurnServerSource.ts | 222 +++++++++++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100644 src/matrix/calls/TurnServerSource.ts diff --git a/src/matrix/calls/TurnServerSource.ts b/src/matrix/calls/TurnServerSource.ts new file mode 100644 index 00000000..cc6923af --- /dev/null +++ b/src/matrix/calls/TurnServerSource.ts @@ -0,0 +1,222 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {RetainedObservableValue} from "../../observable/value/RetainedObservableValue"; + +import type {HomeServerApi} from "../net/HomeServerApi"; +import type {IHomeServerRequest} from "../net/HomeServerRequest"; +import type {BaseObservableValue} from "../../observable/value/BaseObservableValue"; +import type {ObservableValue} from "../../observable/value/ObservableValue"; +import type {Clock, Timeout} from "../../platform/web/dom/Clock"; +import type {ILogItem} from "../../logging/types"; + +type TurnServerSettings = { + urls: string[], + username: string, + password: string, + ttl: number +}; + +const DEFAULT_TTL = 5 * 60; // 5min +const DEFAULT_SETTINGS: RTCIceServer = { + urls: ["stun:turn.matrix.org"], + username: "", + credential: "", +}; + +export class TurnServerSource { + private currentObservable?: ObservableValue; + private pollTimeout?: Timeout; + private pollRequest?: IHomeServerRequest; + private isPolling = false; + + constructor( + private hsApi: HomeServerApi, + private clock: Clock, + private defaultSettings: RTCIceServer = DEFAULT_SETTINGS + ) {} + + getSettings(log: ILogItem): Promise> { + return log.wrap("get turn server", async log => { + if (!this.isPolling) { + const settings = await this.doRequest(log); + const iceServer = settings ? toIceServer(settings) : this.defaultSettings; + if (this.currentObservable) { + this.currentObservable.set(iceServer); + } else { + this.currentObservable = new RetainedObservableValue(iceServer, + () => { + this.stopPollLoop(); + }, + () => { + // start loop on first subscribe + this.runLoop(this.currentObservable!, settings?.ttl ?? DEFAULT_TTL); + }); + } + } + return this.currentObservable!; + }); + } + + private async runLoop(observable: ObservableValue, initialTtl: number): Promise { + let ttl = initialTtl; + this.isPolling = true; + while(this.isPolling) { + try { + this.pollTimeout = this.clock.createTimeout(ttl * 1000); + await this.pollTimeout.elapsed(); + this.pollTimeout = undefined; + const settings = await this.doRequest(undefined); + if (settings) { + const iceServer = toIceServer(settings); + if (shouldUpdate(observable, iceServer)) { + observable.set(iceServer); + } + if (settings.ttl > 0) { + ttl = settings.ttl; + } else { + // stop polling is settings are good indefinitely + this.stopPollLoop(); + } + } else { + ttl = DEFAULT_TTL; + } + } catch (err) { + if (err.name === "AbortError") { + /* ignore, the loop will exit because isPolling is false */ + } else { + // TODO: log error + } + } + } + } + + private async doRequest(log: ILogItem | undefined): Promise { + try { + this.pollRequest = this.hsApi.getTurnServer({log}); + const settings = await this.pollRequest.response(); + return settings; + } catch (err) { + if (err.name === "HomeServerError") { + return undefined; + } + throw err; + } finally { + this.pollRequest = undefined; + } + } + + stopPollLoop() { + this.isPolling = false; + this.currentObservable = undefined; + this.pollTimeout?.dispose(); + this.pollTimeout = undefined; + this.pollRequest?.abort(); + this.pollRequest = undefined; + } + + dispose() { + this.stopPollLoop(); + } +} + +function shouldUpdate(observable: BaseObservableValue, settings: RTCIceServer): boolean { + const currentSettings = observable.get(); + if (!currentSettings) { + return true; + } + // same length and new settings doesn't contain any uri the old settings don't contain + const currentUrls = Array.isArray(currentSettings.urls) ? currentSettings.urls : [currentSettings.urls]; + const newUrls = Array.isArray(settings.urls) ? settings.urls : [settings.urls]; + const arraysEqual = currentUrls.length === newUrls.length && + !newUrls.some(uri => !currentUrls.includes(uri)); + return !arraysEqual || settings.username !== currentSettings.username || + settings.credential !== currentSettings.credential; +} + +function toIceServer(settings: TurnServerSettings): RTCIceServer { + return { + urls: settings.urls, + username: settings.username, + credential: settings.password, + credentialType: "password" + } +} + +export function tests() { + return { + "shouldUpdate returns false for same object": assert => { + const observable = {get() { + return { + urls: ["a", "b"], + username: "alice", + credential: "f00", + }; + }}; + const same = { + urls: ["a", "b"], + username: "alice", + credential: "f00", + }; + assert.equal(false, shouldUpdate(observable as any as BaseObservableValue, same)); + }, + "shouldUpdate returns true for 1 different uri": assert => { + const observable = {get() { + return { + urls: ["a", "c"], + username: "alice", + credential: "f00", + }; + }}; + const same = { + urls: ["a", "b"], + username: "alice", + credential: "f00", + }; + assert.equal(true, shouldUpdate(observable as any as BaseObservableValue, same)); + }, + "shouldUpdate returns true for different user": assert => { + const observable = {get() { + return { + urls: ["a", "b"], + username: "alice", + credential: "f00", + }; + }}; + const same = { + urls: ["a", "b"], + username: "bob", + credential: "f00", + }; + assert.equal(true, shouldUpdate(observable as any as BaseObservableValue, same)); + }, + "shouldUpdate returns true for different password": assert => { + const observable = {get() { + return { + urls: ["a", "b"], + username: "alice", + credential: "f00", + }; + }}; + const same = { + urls: ["a", "b"], + username: "alice", + credential: "b4r", + }; + assert.equal(true, shouldUpdate(observable as any as BaseObservableValue, same)); + } + } +} From 3a4c38086c066c31534263d98c19e875b3627a0f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:25:24 +0200 Subject: [PATCH 179/435] fetch turn server settings when joining a call, and pass down --- src/matrix/calls/CallHandler.ts | 8 ++------ src/matrix/calls/PeerCall.ts | 3 ++- src/matrix/calls/group/GroupCall.ts | 19 +++++++++++++++---- src/matrix/calls/group/Member.ts | 22 ++++++++++++++++------ 4 files changed, 35 insertions(+), 17 deletions(-) diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 17aa0e55..354c9e6a 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -23,6 +23,7 @@ import {GroupCall} from "./group/GroupCall"; import {makeId} from "../common"; import {CALL_LOG_TYPE} from "./common"; import {EVENT_TYPE as MEMBER_EVENT_TYPE, RoomMember} from "../room/members/RoomMember"; +import {TurnServerSource} from "./TurnServerSource"; import type {LocalMedia} from "./LocalMedia"; import type {Room} from "../room/Room"; @@ -57,6 +58,7 @@ export class CallHandler implements RoomStateHandler { constructor(private readonly options: Options) { this.groupCallOptions = Object.assign({}, this.options, { + turnServerSource: new TurnServerSource(this.options.hsApi, this.options.clock), emitUpdate: (groupCall, params) => this._calls.update(groupCall.id, params), createTimeout: this.options.clock.createTimeout, sessionId: this.sessionId @@ -75,12 +77,6 @@ export class CallHandler implements RoomStateHandler { this._loadCallEntries(callEntries, txn); } - setTurnServers(turnServers: RTCIceServer[]) { - this.options.turnServers = turnServers; - this.groupCallOptions.turnServers = turnServers; - // TODO: we should update any ongoing peerconnections if the TURN server details have changed - } - private async _getLoadTxn(): Promise { const names = this.options.storage.storeNames; const txn = await this.options.storage.readTxn([ diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index d004dedc..ef906d97 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -15,6 +15,7 @@ limitations under the License. */ import {ObservableMap} from "../../observable/map/ObservableMap"; +import {BaseObservableValue} from "../../observable/value/BaseObservableValue"; import {recursivelyAssign} from "../../utils/recursivelyAssign"; import {Disposables, Disposable, IDisposable} from "../../utils/Disposables"; import {WebRTC, PeerConnection, Transceiver, TransceiverDirection, Sender, Receiver, PeerConnectionEventMap} from "../../platform/types/WebRTC"; @@ -47,7 +48,7 @@ import type { export type Options = { webRTC: WebRTC, forceTURN: boolean, - turnServers: RTCIceServer[], + turnServer: BaseObservableValue, createTimeout: TimeoutCreator, emitUpdate: (peerCall: PeerCall, params: any, log: ILogItem) => void; sendSignallingMessage: (message: SignallingMessage, log: ILogItem) => Promise; diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 07aa2a08..4c038be9 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -23,6 +23,7 @@ import {EventEmitter} from "../../../utils/EventEmitter"; import {EventType, CallIntent} from "../callEventTypes"; import type {Options as MemberOptions} from "./Member"; +import type {TurnServerSource} from "../TurnServerSource"; import type {BaseObservableMap} from "../../../observable/map/BaseObservableMap"; import type {Track} from "../../../platform/types/MediaDevices"; import type {SignallingMessage, MGroupCallBase, CallMembership} from "../callEventTypes"; @@ -32,6 +33,7 @@ import type {Platform} from "../../../platform/web/Platform"; import type {EncryptedMessage} from "../../e2ee/olm/Encryption"; import type {ILogItem, ILogger} from "../../../logging/types"; import type {Storage} from "../../storage/idb/Storage"; +import type {BaseObservableValue} from "../../../observable/value/BaseObservableValue"; export enum GroupCallState { Fledgling = "fledgling", @@ -53,11 +55,12 @@ function getDeviceFromMemberKey(key: string): string { return JSON.parse(`[${key}]`)[1]; } -export type Options = Omit & { +export type Options = Omit & { emitUpdate: (call: GroupCall, params?: any) => void; encryptDeviceMessage: (roomId: string, userId: string, deviceId: string, message: SignallingMessage, log: ILogItem) => Promise, storage: Storage, logger: ILogger, + turnServerSource: TurnServerSource }; class JoinedData { @@ -65,7 +68,8 @@ class JoinedData { public readonly logItem: ILogItem, public readonly membersLogItem: ILogItem, public localMedia: LocalMedia, - public localMuteSettings: MuteSettings + public localMuteSettings: MuteSettings, + public turnServer: BaseObservableValue ) {} dispose() { @@ -136,6 +140,7 @@ export class GroupCall extends EventEmitter<{change: never}> { id: this.id, ownSessionId: this.options.sessionId }); + const turnServer = await this.options.turnServerSource.getSettings(logItem); const membersLogItem = logItem.child("member connections"); const localMuteSettings = new MuteSettings(); localMuteSettings.updateTrackInfo(localMedia.userMedia); @@ -143,7 +148,8 @@ export class GroupCall extends EventEmitter<{change: never}> { logItem, membersLogItem, localMedia, - localMuteSettings + localMuteSettings, + turnServer ); this.joinedData = joinedData; await joinedData.logItem.wrap("join", async log => { @@ -509,7 +515,12 @@ export class GroupCall extends EventEmitter<{change: never}> { const logItem = joinedData.membersLogItem.child({l: "member", id: memberKey}); logItem.set("sessionId", member.sessionId); log.wrap({l: "connect", id: memberKey}, log => { - const connectItem = member.connect(joinedData.localMedia, joinedData.localMuteSettings, logItem); + const connectItem = member.connect( + joinedData.localMedia, + joinedData.localMuteSettings, + joinedData.turnServer, + logItem + ); if (connectItem) { log.refDetached(connectItem); } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 98e2aa4d..7cd900f2 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -19,6 +19,7 @@ import {makeTxnId, makeId} from "../../common"; import {EventType, CallErrorCode} from "../callEventTypes"; import {formatToDeviceMessagesPayload} from "../../common"; import {sortedIndex} from "../../../utils/sortedIndex"; + import type {MuteSettings} from "../common"; import type {Options as PeerCallOptions, RemoteMedia} from "../PeerCall"; import type {LocalMedia} from "../LocalMedia"; @@ -28,8 +29,9 @@ import type {GroupCall} from "./GroupCall"; import type {RoomMember} from "../../room/members/RoomMember"; import type {EncryptedMessage} from "../../e2ee/olm/Encryption"; import type {ILogItem} from "../../../logging/types"; +import type {BaseObservableValue} from "../../../observable/value/BaseObservableValue"; -export type Options = Omit & { +export type Options = Omit & { confId: string, ownUserId: string, ownDeviceId: string, @@ -60,6 +62,7 @@ class MemberConnection { constructor( public localMedia: LocalMedia, public localMuteSettings: MuteSettings, + public turnServer: BaseObservableValue, public readonly logItem: ILogItem ) {} } @@ -112,12 +115,17 @@ export class Member { } /** @internal */ - connect(localMedia: LocalMedia, localMuteSettings: MuteSettings, memberLogItem: ILogItem): ILogItem | undefined { + connect(localMedia: LocalMedia, localMuteSettings: MuteSettings, turnServer: BaseObservableValue, memberLogItem: ILogItem): ILogItem | undefined { if (this.connection) { return; } // Safari can't send a MediaStream to multiple sources, so clone it - const connection = new MemberConnection(localMedia.clone(), localMuteSettings, memberLogItem); + const connection = new MemberConnection( + localMedia.clone(), + localMuteSettings, + turnServer, + memberLogItem + ); this.connection = connection; let connectLogItem; connection.logItem.wrap("connect", async log => { @@ -204,7 +212,7 @@ export class Member { if (hangupReason && !errorCodesWithoutRetry.includes(hangupReason)) { connection.retryCount += 1; const {retryCount} = connection; - connection.logItem.wrap({l: "retry connection", retryCount}, async retryLog => { + connection.logItem.wrap({l: "retry connection", retryCount}, async retryLog => { log.refDetached(retryLog); if (retryCount <= 3) { await this.callIfNeeded(retryLog); @@ -337,9 +345,11 @@ export class Member { } private _createPeerCall(callId: string): PeerCall { + const connection = this.connection!; return new PeerCall(callId, Object.assign({}, this.options, { emitUpdate: this.emitUpdateFromPeerCall, - sendSignallingMessage: this.sendSignallingMessage - }), this.connection!.logItem); + sendSignallingMessage: this.sendSignallingMessage, + turnServer: connection.turnServer + }), connection.logItem); } } From 917ad52302a78d7367c19a56287bc4284e411f5a Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:25:53 +0200 Subject: [PATCH 180/435] use observable for turnserver in peercall and subscribe if changing config is supported --- src/matrix/calls/PeerCall.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index ef906d97..ccff5868 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -115,8 +115,18 @@ export class PeerCall implements IDisposable { ) { logItem.log({l: "create PeerCall", id: callId}); this._remoteMedia = new RemoteMedia(); - this.peerConnection = options.webRTC.createPeerConnection(this.options.forceTURN, this.options.turnServers, 0); - + this.peerConnection = options.webRTC.createPeerConnection( + this.options.forceTURN, + [this.options.turnServer.get()], + 0 + ); + // update turn servers when they change (see TurnServerSource) if possible + if (typeof this.peerConnection["setConfiguration"] === "function") { + this.disposables.track(this.options.turnServer.subscribe(turnServer => { + this.logItem.log({l: "updating turn server", turnServer}) + this.peerConnection["setConfiguration"]({iceServers: [turnServer]}); + })); + } const listen = (type: K, listener: (this: PeerConnection, ev: PeerConnectionEventMap[K]) => any, options?: boolean | EventListenerOptions): void => { this.peerConnection.addEventListener(type, listener); const dispose = () => { From f74c4e642590395e18f135d6266578123c7a7350 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:26:16 +0200 Subject: [PATCH 181/435] remove previous approach --- src/matrix/Session.js | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 3c49182d..f82ad555 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -513,7 +513,6 @@ export class Session { // TODO: what can we do if this throws? await txn.complete(); } - await this._updateTurnServers(); // enable session backup, this requests the latest backup version if (!this._keyBackup.get()) { if (dehydratedDevice) { @@ -560,18 +559,6 @@ export class Session { } } - async _updateTurnServers() { - const turnServerData = await this._hsApi.getTurnServer().response(); - this._callHandler.setTurnServers([{ - urls: turnServerData.uris, - username: turnServerData.username, - credential: turnServerData.password, - }]); - if (turnServerData.ttl > 0) { - setTimeout(this._updateTurnServers, turnServerData.ttl * 1000); - } - } - async _getPendingEventsByRoom(txn) { const pendingEvents = await txn.pendingEvents.getAll(); return pendingEvents.reduce((groups, pe) => { From e6bf49a6ccbee379613848d6f25fd7fa9981f817 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:27:41 +0200 Subject: [PATCH 182/435] remove previous hardcoded turnServers setting --- src/matrix/Session.js | 3 --- src/matrix/calls/CallHandler.ts | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index f82ad555..7d187b30 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -101,9 +101,6 @@ export class Session { ownDeviceId: sessionInfo.deviceId, ownUserId: sessionInfo.userId, logger: this._platform.logger, - turnServers: [{ - urls: ["stun:turn.matrix.org"], - }], forceTURN: false, }); this._roomStateHandler = new RoomStateHandlerSet(); diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 354c9e6a..7b8265c5 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -40,7 +40,7 @@ import type {Clock} from "../../platform/web/dom/Clock"; import type {RoomStateHandler} from "../room/state/types"; import type {MemberSync} from "../room/timeline/persistence/MemberWriter"; -export type Options = Omit & { +export type Options = Omit & { clock: Clock }; From 05c2da95c4ffa99b2581e7b1a3077a5fcb41901d Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:38:04 +0200 Subject: [PATCH 183/435] fix typo --- src/matrix/calls/TurnServerSource.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/matrix/calls/TurnServerSource.ts b/src/matrix/calls/TurnServerSource.ts index cc6923af..cac08bdf 100644 --- a/src/matrix/calls/TurnServerSource.ts +++ b/src/matrix/calls/TurnServerSource.ts @@ -24,7 +24,7 @@ import type {Clock, Timeout} from "../../platform/web/dom/Clock"; import type {ILogItem} from "../../logging/types"; type TurnServerSettings = { - urls: string[], + uris: string[], username: string, password: string, ttl: number @@ -54,6 +54,7 @@ export class TurnServerSource { if (!this.isPolling) { const settings = await this.doRequest(log); const iceServer = settings ? toIceServer(settings) : this.defaultSettings; + log.set("iceServer", iceServer); if (this.currentObservable) { this.currentObservable.set(iceServer); } else { @@ -149,7 +150,7 @@ function shouldUpdate(observable: BaseObservableValue, function toIceServer(settings: TurnServerSettings): RTCIceServer { return { - urls: settings.urls, + urls: settings.uris, username: settings.username, credential: settings.password, credentialType: "password" From d36b9be24f1d041c2c668f213041165a23921fd0 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:41:30 +0200 Subject: [PATCH 184/435] fix screwing up whitespace --- src/matrix/calls/group/Member.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 7cd900f2..b1c7b430 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -212,7 +212,7 @@ export class Member { if (hangupReason && !errorCodesWithoutRetry.includes(hangupReason)) { connection.retryCount += 1; const {retryCount} = connection; - connection.logItem.wrap({l: "retry connection", retryCount}, async retryLog => { + connection.logItem.wrap({l: "retry connection", retryCount}, async retryLog => { log.refDetached(retryLog); if (retryCount <= 3) { await this.callIfNeeded(retryLog); From ac319bdafd747b1dbf3575dc2aeba59145db50cb Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:45:41 +0200 Subject: [PATCH 185/435] we can assume setConfiguration is available --- src/matrix/calls/PeerCall.ts | 12 +++++------- src/platform/types/WebRTC.ts | 1 + 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index ccff5868..a7ae7b04 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -120,13 +120,11 @@ export class PeerCall implements IDisposable { [this.options.turnServer.get()], 0 ); - // update turn servers when they change (see TurnServerSource) if possible - if (typeof this.peerConnection["setConfiguration"] === "function") { - this.disposables.track(this.options.turnServer.subscribe(turnServer => { - this.logItem.log({l: "updating turn server", turnServer}) - this.peerConnection["setConfiguration"]({iceServers: [turnServer]}); - })); - } + // update turn servers when they change (see TurnServerSource) + this.disposables.track(this.options.turnServer.subscribe(turnServer => { + this.logItem.log({l: "updating turn server", turnServer}) + this.peerConnection.setConfiguration({iceServers: [turnServer]}); + })); const listen = (type: K, listener: (this: PeerConnection, ev: PeerConnectionEventMap[K]) => any, options?: boolean | EventListenerOptions): void => { this.peerConnection.addEventListener(type, listener); const dispose = () => { diff --git a/src/platform/types/WebRTC.ts b/src/platform/types/WebRTC.ts index 39ad49c5..236e8354 100644 --- a/src/platform/types/WebRTC.ts +++ b/src/platform/types/WebRTC.ts @@ -148,6 +148,7 @@ export interface PeerConnection { addEventListener(type: K, listener: (this: PeerConnection, ev: PeerConnectionEventMap[K]) => any, options?: boolean | AddEventListenerOptions): void; removeEventListener(type: K, listener: (this: PeerConnection, ev: PeerConnectionEventMap[K]) => any, options?: boolean | EventListenerOptions): void; getStats(selector?: Track | null): Promise; + setConfiguration(configuration?: RTCConfiguration): void; } From 24ebf6c559154aa6dcbc4685613b1b57e561f7de Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:47:48 +0200 Subject: [PATCH 186/435] cleanup --- src/matrix/calls/TurnServerSource.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/matrix/calls/TurnServerSource.ts b/src/matrix/calls/TurnServerSource.ts index cac08bdf..ca0f6848 100644 --- a/src/matrix/calls/TurnServerSource.ts +++ b/src/matrix/calls/TurnServerSource.ts @@ -64,7 +64,7 @@ export class TurnServerSource { }, () => { // start loop on first subscribe - this.runLoop(this.currentObservable!, settings?.ttl ?? DEFAULT_TTL); + this.runLoop(settings?.ttl ?? DEFAULT_TTL); }); } } @@ -72,7 +72,7 @@ export class TurnServerSource { }); } - private async runLoop(observable: ObservableValue, initialTtl: number): Promise { + private async runLoop(initialTtl: number): Promise { let ttl = initialTtl; this.isPolling = true; while(this.isPolling) { @@ -83,8 +83,8 @@ export class TurnServerSource { const settings = await this.doRequest(undefined); if (settings) { const iceServer = toIceServer(settings); - if (shouldUpdate(observable, iceServer)) { - observable.set(iceServer); + if (shouldUpdate(this.currentObservable!, iceServer)) { + this.currentObservable!.set(iceServer); } if (settings.ttl > 0) { ttl = settings.ttl; From eccbab1491d6245ae95084ff120907be66cfd7f2 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:57:20 +0200 Subject: [PATCH 187/435] cleanup polling and also hangup on all ongoing calls on dispose session --- src/matrix/Session.js | 2 ++ src/matrix/calls/CallHandler.ts | 10 +++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 7d187b30..31b57b57 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -488,6 +488,8 @@ export class Session { this._megolmDecryption = undefined; this._e2eeAccount?.dispose(); this._e2eeAccount = undefined; + this._callHandler?.dispose(); + this._callHandler = undefined; for (const room of this._rooms.values()) { room.dispose(); } diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 7b8265c5..67cf3dc7 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -55,10 +55,12 @@ export class CallHandler implements RoomStateHandler { private roomMemberToCallIds: Map> = new Map(); private groupCallOptions: GroupCallOptions; private sessionId = makeId("s"); + private turnServerSource: TurnServerSource; constructor(private readonly options: Options) { + this.turnServerSource = new TurnServerSource(this.options.hsApi, this.options.clock); this.groupCallOptions = Object.assign({}, this.options, { - turnServerSource: new TurnServerSource(this.options.hsApi, this.options.clock), + turnServerSource: this.turnServerSource, emitUpdate: (groupCall, params) => this._calls.update(groupCall.id, params), createTimeout: this.options.clock.createTimeout, sessionId: this.sessionId @@ -243,5 +245,11 @@ export class CallHandler implements RoomStateHandler { this.roomMemberToCallIds.set(roomMemberKey, newCallIdsMemberOf); } } + + dispose() { + this.turnServerSource.dispose(); + const joinedCalls = Array.from(this._calls.values()).filter(c => c.hasJoined); + Promise.all(joinedCalls.map(c => c.leave())).then(() => {}, () => {}); + } } From c660d82d6a47620a78ac18b9eed443b103cec3c6 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 26 Sep 2022 16:00:25 +0200 Subject: [PATCH 188/435] make internal method private --- src/matrix/calls/TurnServerSource.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/TurnServerSource.ts b/src/matrix/calls/TurnServerSource.ts index ca0f6848..1066f7a6 100644 --- a/src/matrix/calls/TurnServerSource.ts +++ b/src/matrix/calls/TurnServerSource.ts @@ -120,7 +120,7 @@ export class TurnServerSource { } } - stopPollLoop() { + private stopPollLoop() { this.isPolling = false; this.currentObservable = undefined; this.pollTimeout?.dispose(); From 6f0e781b4970274a7d3452b2821faa745f488f56 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 26 Sep 2022 19:15:13 +0200 Subject: [PATCH 189/435] WIP --- src/matrix/Session.js | 1 + src/matrix/calls/CallHandler.ts | 11 +- src/matrix/calls/PeerCall.ts | 8 +- src/matrix/calls/callEventTypes.ts | 4 +- src/matrix/calls/common.ts | 2 + src/matrix/calls/group/GroupCall.ts | 149 ++++++++++++++++++++-------- src/matrix/calls/group/Member.ts | 48 ++++++++- src/matrix/e2ee/DeviceTracker.js | 2 +- 8 files changed, 175 insertions(+), 50 deletions(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 31b57b57..291002c5 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -78,6 +78,7 @@ export class Session { this._user = new User(sessionInfo.userId); this._callHandler = new CallHandler({ clock: this._platform.clock, + random: this._platform.random, hsApi: this._hsApi, encryptDeviceMessage: async (roomId, userId, deviceId, message, log) => { if (!this._deviceTracker || !this._olmEncryption) { diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 67cf3dc7..2b914a2f 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -55,12 +55,10 @@ export class CallHandler implements RoomStateHandler { private roomMemberToCallIds: Map> = new Map(); private groupCallOptions: GroupCallOptions; private sessionId = makeId("s"); - private turnServerSource: TurnServerSource; constructor(private readonly options: Options) { - this.turnServerSource = new TurnServerSource(this.options.hsApi, this.options.clock); this.groupCallOptions = Object.assign({}, this.options, { - turnServerSource: this.turnServerSource, + turnServerSource: new TurnServerSource(this.options.hsApi, this.options.clock), emitUpdate: (groupCall, params) => this._calls.update(groupCall.id, params), createTimeout: this.options.clock.createTimeout, sessionId: this.sessionId @@ -247,9 +245,10 @@ export class CallHandler implements RoomStateHandler { } dispose() { - this.turnServerSource.dispose(); - const joinedCalls = Array.from(this._calls.values()).filter(c => c.hasJoined); - Promise.all(joinedCalls.map(c => c.leave())).then(() => {}, () => {}); + this.groupCallOptions.turnServerSource.dispose(); + for(const call of this._calls.values()) { + call.dispose(); + } } } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 0f16b207..d7d815fa 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -1098,7 +1098,13 @@ export class PeerCall implements IDisposable { private async delay(timeoutMs: number): Promise { // Allow a short time for initial candidates to be gathered const timeout = this.disposables.track(this.options.createTimeout(timeoutMs)); - await timeout.elapsed(); + try { + await timeout.elapsed(); + } catch (err) { + if (err.name !== "AbortError") { + throw err; + } + } this.disposables.untrack(timeout); } diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index 09376c85..ca5c870c 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -24,7 +24,9 @@ export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata"; export interface CallDeviceMembership { device_id: string, - session_id: string + session_id: string, + ["m.expires_ts"]?: number, + feeds?: Array<{purpose: string}> } export interface CallMembership { diff --git a/src/matrix/calls/common.ts b/src/matrix/calls/common.ts index 66db6edc..dde55ddc 100644 --- a/src/matrix/calls/common.ts +++ b/src/matrix/calls/common.ts @@ -59,3 +59,5 @@ export class MuteSettings { } export const CALL_LOG_TYPE = "call"; +//export const CALL_MEMBER_VALIDITY_PERIOD_MS = 3600 * 1000; // 1h +export const CALL_MEMBER_VALIDITY_PERIOD_MS = 60 * 1000; diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 4c038be9..cb027be8 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -15,9 +15,9 @@ limitations under the License. */ import {ObservableMap} from "../../../observable/map/ObservableMap"; -import {Member} from "./Member"; +import {Member, isMemberExpired, memberExpiresAt} from "./Member"; import {LocalMedia} from "../LocalMedia"; -import {MuteSettings, CALL_LOG_TYPE} from "../common"; +import {MuteSettings, CALL_LOG_TYPE, CALL_MEMBER_VALIDITY_PERIOD_MS} from "../common"; import {MemberChange, RoomMember} from "../../room/members/RoomMember"; import {EventEmitter} from "../../../utils/EventEmitter"; import {EventType, CallIntent} from "../callEventTypes"; @@ -26,7 +26,7 @@ import type {Options as MemberOptions} from "./Member"; import type {TurnServerSource} from "../TurnServerSource"; import type {BaseObservableMap} from "../../../observable/map/BaseObservableMap"; import type {Track} from "../../../platform/types/MediaDevices"; -import type {SignallingMessage, MGroupCallBase, CallMembership} from "../callEventTypes"; +import type {SignallingMessage, MGroupCallBase, CallMembership, CallMemberContent, CallDeviceMembership} from "../callEventTypes"; import type {Room} from "../../room/Room"; import type {StateEvent} from "../../storage/types"; import type {Platform} from "../../../platform/web/Platform"; @@ -34,6 +34,7 @@ import type {EncryptedMessage} from "../../e2ee/olm/Encryption"; import type {ILogItem, ILogger} from "../../../logging/types"; import type {Storage} from "../../storage/idb/Storage"; import type {BaseObservableValue} from "../../../observable/value/BaseObservableValue"; +import type {Clock, Timeout} from "../../../platform/web/dom/Clock"; export enum GroupCallState { Fledgling = "fledgling", @@ -59,11 +60,14 @@ export type Options = Omit void; encryptDeviceMessage: (roomId: string, userId: string, deviceId: string, message: SignallingMessage, log: ILogItem) => Promise, storage: Storage, + random: () => number, logger: ILogger, turnServerSource: TurnServerSource }; class JoinedData { + public renewMembershipTimeout?: Timeout; + constructor( public readonly logItem: ILogItem, public readonly membersLogItem: ILogItem, @@ -75,6 +79,7 @@ class JoinedData { dispose() { this.localMedia.dispose(); this.logItem.finish(); + this.renewMembershipTimeout?.dispose(); } } @@ -96,7 +101,17 @@ export class GroupCall extends EventEmitter<{change: never}> { this._state = newCall ? GroupCallState.Fledgling : GroupCallState.Created; this._memberOptions = Object.assign({}, options, { confId: this.id, - emitUpdate: member => this._members.update(getMemberKey(member.userId, member.deviceId), member), + emitUpdate: member => { + if (!member.isExpired) { + this._members.update(getMemberKey(member.userId, member.deviceId), member); + } else if (!member.isConnected) { + // don't just kick out connected members, even if their timestamp expired + this._members.remove(getMemberKey(member.userId, member.deviceId)); + } + if (member.isExpired && member.isConnected) { + console.trace("was about to kick a connected but expired member"); + } + }, encryptDeviceMessage: (userId: string, deviceId: string, message: SignallingMessage, log) => { return this.options.encryptDeviceMessage(this.roomId, userId, deviceId, message, log); } @@ -156,7 +171,7 @@ export class GroupCall extends EventEmitter<{change: never}> { this._state = GroupCallState.Joining; this.emitChange(); await log.wrap("update member state", async log => { - const memberContent = await this._createJoinPayload(); + const memberContent = await this._createMemberPayload(true); log.set("payload", memberContent); // send m.call.member state event const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCallMember, this.options.ownUserId, memberContent, {log}); @@ -220,7 +235,9 @@ export class GroupCall extends EventEmitter<{change: never}> { } await joinedData.logItem.wrap("leave", async log => { try { - const memberContent = await this._leaveCallMemberContent(); + joinedData.renewMembershipTimeout?.dispose(); + joinedData.renewMembershipTimeout = undefined; + const memberContent = await this._createMemberPayload(false); // send m.call.member state event if (memberContent) { const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCallMember, this.options.ownUserId, memberContent, {log}); @@ -296,19 +313,32 @@ export class GroupCall extends EventEmitter<{change: never}> { /** @internal */ updateMembership(userId: string, roomMember: RoomMember, callMembership: CallMembership, syncLog: ILogItem) { syncLog.wrap({l: "update call membership", t: CALL_LOG_TYPE, id: this.id, userId}, log => { + const now = this.options.clock.now(); const devices = callMembership["m.devices"]; const previousDeviceIds = this.getDeviceIdsForUserId(userId); for (const device of devices) { const deviceId = device.device_id; const memberKey = getMemberKey(userId, deviceId); - log.wrap({l: "update device membership", id: memberKey, sessionId: device.session_id}, log => { - if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) { + if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) { + log.wrap("update own membership", log => { + // TODO: should we check if new device is expired? + if (this.hasJoined) { + this.joinedData!.logItem.refDetached(log); + this._setupRenewMembershipTimeout(device, log); + } if (this._state === GroupCallState.Joining) { - log.set("update_own", true); + log.set("joined", true); this._state = GroupCallState.Joined; this.emitChange(); } - } else { + }); + } else { + log.wrap({l: "update device membership", id: memberKey, sessionId: device.session_id}, log => { + if (isMemberExpired(device, now)) { + log.set("expired", true); + this._members.remove(memberKey); + return; + } let member = this._members.get(memberKey); const sessionIdChanged = member && member.sessionId !== device.session_id; if (member && !sessionIdChanged) { @@ -328,6 +358,7 @@ export class GroupCall extends EventEmitter<{change: never}> { member = new Member( roomMember, device, this._memberOptions, + log ); this._members.add(memberKey, member); if (this.joinedData) { @@ -337,8 +368,8 @@ export class GroupCall extends EventEmitter<{change: never}> { // flush pending messages, either after having created the member, // or updated the session id with updateCallInfo this.flushPendingIncomingDeviceMessages(member, log); - } - }); + }); + } } const newDeviceIds = new Set(devices.map(call => call.device_id)); @@ -466,14 +497,14 @@ export class GroupCall extends EventEmitter<{change: never}> { } } - private async _createJoinPayload() { + private async _createMemberPayload(includeOwn: boolean): Promise { const {storage} = this.options; const txn = await storage.readTxn([storage.storeNames.roomState]); const stateEvent = await txn.roomState.get(this.roomId, EventType.GroupCallMember, this.options.ownUserId); - const stateContent = stateEvent?.event?.content ?? { + const stateContent: CallMemberContent = stateEvent?.event?.content as CallMemberContent ?? { ["m.calls"]: [] }; - const callsInfo = stateContent["m.calls"]; + let callsInfo = stateContent["m.calls"]; let callInfo = callsInfo.find(c => c["m.call_id"] === this.id); if (!callInfo) { callInfo = { @@ -482,32 +513,29 @@ export class GroupCall extends EventEmitter<{change: never}> { }; callsInfo.push(callInfo); } - callInfo["m.devices"] = callInfo["m.devices"].filter(d => d["device_id"] !== this.options.ownDeviceId); - callInfo["m.devices"].push({ - ["device_id"]: this.options.ownDeviceId, - ["session_id"]: this.options.sessionId, - feeds: [{purpose: "m.usermedia"}] - }); - return stateContent; - } - - private async _leaveCallMemberContent(): Promise | undefined> { - const {storage} = this.options; - const txn = await storage.readTxn([storage.storeNames.roomState]); - const stateEvent = await txn.roomState.get(this.roomId, EventType.GroupCallMember, this.options.ownUserId); - if (stateEvent) { - const content = stateEvent.event.content; - const callInfo = content["m.calls"]?.find(c => c["m.call_id"] === this.id); - if (callInfo) { - const devicesInfo = callInfo["m.devices"]; - const deviceIndex = devicesInfo.findIndex(d => d["device_id"] === this.options.ownDeviceId); - if (deviceIndex !== -1) { - devicesInfo.splice(deviceIndex, 1); - return content; - } + const now = this.options.clock.now(); + callInfo["m.devices"] = callInfo["m.devices"].filter(d => { + // remove our own device (to add it again below) + if (d["device_id"] === this.options.ownDeviceId) { + return false; } - + // also remove any expired devices (+ the validity period added again) + if (memberExpiresAt(d) === undefined || isMemberExpired(d, now, CALL_MEMBER_VALIDITY_PERIOD_MS)) { + return false; + } + return true; + }); + if (includeOwn) { + callInfo["m.devices"].push({ + ["device_id"]: this.options.ownDeviceId, + ["session_id"]: this.options.sessionId, + ["m.expires_ts"]: now + CALL_MEMBER_VALIDITY_PERIOD_MS, + feeds: [{purpose: "m.usermedia"}] + }); } + // filter out empty call membership + stateContent["m.calls"] = callsInfo.filter(c => c["m.devices"].length !== 0); + return stateContent; } private connectToMember(member: Member, joinedData: JoinedData, log: ILogItem) { @@ -524,11 +552,52 @@ export class GroupCall extends EventEmitter<{change: never}> { if (connectItem) { log.refDetached(connectItem); } - }) + }); } protected emitChange() { this.emit("change"); this.options.emitUpdate(this); } + + private _setupRenewMembershipTimeout(callDeviceMembership: CallDeviceMembership, log: ILogItem) { + const {joinedData} = this; + if (!joinedData) { + return; + } + joinedData.renewMembershipTimeout?.dispose(); + joinedData.renewMembershipTimeout = undefined; + const expiresAt = memberExpiresAt(callDeviceMembership); + if (typeof expiresAt !== "number") { + return; + } + const expiresFromNow = expiresAt - this.options.clock.now(); + // renew 1 to 5 minutes (8.3% of 1h) before expiring + // do it a bit beforehand and somewhat random to not collide with + // other clients trying to renew as well + const timeToRenewBeforeExpiration = Math.max(4000, Math.ceil((0.2 +(this.options.random() * 0.8)) * (0.08333 * CALL_MEMBER_VALIDITY_PERIOD_MS))); + const renewFromNow = Math.max(0, expiresFromNow - timeToRenewBeforeExpiration); + log.set("expiresIn", expiresFromNow); + log.set("renewIn", renewFromNow); + joinedData.renewMembershipTimeout = this.options.clock.createTimeout(renewFromNow); + joinedData.renewMembershipTimeout.elapsed().then( + () => { + joinedData.logItem.wrap("renew membership", async log => { + const memberContent = await this._createMemberPayload(true); + log.set("payload", memberContent); + // send m.call.member state event + const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCallMember, this.options.ownUserId, memberContent, {log}); + await request.response(); + }); + }, + () => { /* assume we're swallowing AbortError from dispose above */ } + ); + } + + dispose() { + this.joinedData?.dispose(); + for (const member of this._members.values()) { + member.dispose(); + } + } } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index b1c7b430..8e266fac 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -30,6 +30,7 @@ import type {RoomMember} from "../../room/members/RoomMember"; import type {EncryptedMessage} from "../../e2ee/olm/Encryption"; import type {ILogItem} from "../../../logging/types"; import type {BaseObservableValue} from "../../../observable/value/BaseObservableValue"; +import type {Clock, Timeout} from "../../../platform/web/dom/Clock"; export type Options = Omit & { confId: string, @@ -40,6 +41,7 @@ export type Options = Omit, log: ILogItem) => Promise, emitUpdate: (participant: Member, params?: any) => void, + clock: Clock } const errorCodesWithoutRetry = [ @@ -69,12 +71,33 @@ class MemberConnection { export class Member { private connection?: MemberConnection; + private expireTimeout?: Timeout; constructor( public member: RoomMember, private callDeviceMembership: CallDeviceMembership, private readonly options: Options, - ) {} + updateMemberLog: ILogItem + ) { + this._renewExpireTimeout(updateMemberLog); + } + + private _renewExpireTimeout(log: ILogItem) { + this.expireTimeout?.dispose(); + this.expireTimeout = undefined; + const expiresAt = memberExpiresAt(this.callDeviceMembership); + if (typeof expiresAt !== "number") { + return; + } + const expiresFromNow = Math.max(0, expiresAt - this.options.clock.now()); + log?.set("expiresIn", expiresFromNow); + // add 10ms to make sure isExpired returns true + this.expireTimeout = this.options.clock.createTimeout(expiresFromNow + 10); + this.expireTimeout.elapsed().then( + () => { this.options.emitUpdate(this, "isExpired"); }, + (err) => { /* ignore abort error */ }, + ); + } /** * Gives access the log item for this item once joined to the group call. @@ -89,6 +112,11 @@ export class Member { return this.connection?.peerCall?.remoteMedia; } + get isExpired(): boolean { + // never consider a peer we're connected to, to be expired + return !this.isConnected && isMemberExpired(this.callDeviceMembership, this.options.clock.now()); + } + get remoteMuteSettings(): MuteSettings | undefined { return this.connection?.peerCall?.remoteMuteSettings; } @@ -184,6 +212,7 @@ export class Member { /** @internal */ updateCallInfo(callDeviceMembership: CallDeviceMembership, causeItem: ILogItem) { this.callDeviceMembership = callDeviceMembership; + this._renewExpireTimeout(causeItem); if (this.connection) { this.connection.logItem.refDetached(causeItem); } @@ -352,4 +381,21 @@ export class Member { turnServer: connection.turnServer }), connection.logItem); } + + dispose() { + this.expireTimeout?.dispose(); + this.connection?.peerCall?.dispose(); + } +} + +export function memberExpiresAt(callDeviceMembership: CallDeviceMembership): number | undefined { + const expiresAt = callDeviceMembership["m.expires_ts"]; + if (Number.isSafeInteger(expiresAt)) { + return expiresAt; + } +} + +export function isMemberExpired(callDeviceMembership: CallDeviceMembership, now: number, margin: number = 0) { + const expiresAt = memberExpiresAt(callDeviceMembership); + return typeof expiresAt === "number" && ((expiresAt + margin) <= now); } diff --git a/src/matrix/e2ee/DeviceTracker.js b/src/matrix/e2ee/DeviceTracker.js index 0d0b0e8b..e0eea4fc 100644 --- a/src/matrix/e2ee/DeviceTracker.js +++ b/src/matrix/e2ee/DeviceTracker.js @@ -337,7 +337,7 @@ export class DeviceTracker { // verify signature const verifiedKeysPerUser = log.wrap("verify", log => this._filterVerifiedDeviceKeys(deviceKeyResponse["device_keys"], log)); //// END EXTRACT - + // TODO: what if verifiedKeysPerUser is empty or does not contain userId? const verifiedKeys = verifiedKeysPerUser .find(vkpu => vkpu.userId === userId).verifiedKeys .find(vk => vk["device_id"] === deviceId); From 05bb360c004f89ae48908e9f8497903df7c45cb7 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 30 Sep 2022 11:12:04 +0200 Subject: [PATCH 190/435] allow to dequeue signalling messages with repeated seq (from other call) they will just be ignored by peerCall.getMessageAction() but we don't want to block dequeuing on these --- src/matrix/calls/group/Member.ts | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 8e266fac..a21de30f 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -67,6 +67,22 @@ class MemberConnection { public turnServer: BaseObservableValue, public readonly logItem: ILogItem ) {} + + get canDequeueNextSignallingMessage() { + if (this.queuedSignallingMessages.length === 0) { + return false; + } + if (this.lastProcessedSeqNr === undefined) { + return true; + } + const first = this.queuedSignallingMessages[0]; + // allow messages with both a seq we've just seen and + // the next one to be dequeued as it can happen + // that messages for other callIds (which could repeat seq) + // are present in the queue + return first.content.seq === this.lastProcessedSeqNr || + first.content.seq === this.lastProcessedSeqNr + 1; + } } export class Member { @@ -335,12 +351,7 @@ export class Member { private dequeueSignallingMessages(connection: MemberConnection, peerCall: PeerCall, newMessage: SignallingMessage, syncLog: ILogItem): boolean { let hasNewMessageBeenDequeued = false; - while ( - connection.queuedSignallingMessages.length && ( - connection.lastProcessedSeqNr === undefined || - connection.queuedSignallingMessages[0].content.seq === connection.lastProcessedSeqNr + 1 - ) - ) { + while (connection.canDequeueNextSignallingMessage) { const message = connection.queuedSignallingMessages.shift()!; if (message === newMessage) { hasNewMessageBeenDequeued = true; From 8f8a964b3bc0159f98991f45e6972f90414b45fb Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 30 Sep 2022 11:15:48 +0200 Subject: [PATCH 191/435] allow to actually process the invite in the new peer call --- src/matrix/calls/group/Member.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index a21de30f..d4cc93f4 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -325,6 +325,7 @@ export class Member { } if (shouldReplace) { connection.peerCall = undefined; + action = IncomingMessageAction.Handle; } } } From 167a19a85babc34845700eb19caf687ae19666d3 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 30 Sep 2022 11:16:09 +0200 Subject: [PATCH 192/435] untracking from a disposed Disposables is actually not alarming it happens under normal conditions when an abortable action is aborted and you untrack afterwards. --- src/utils/Disposables.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/Disposables.ts b/src/utils/Disposables.ts index f7c7eb53..f27846be 100644 --- a/src/utils/Disposables.ts +++ b/src/utils/Disposables.ts @@ -50,8 +50,8 @@ export class Disposables { } untrack(disposable: Disposable): undefined { - if (this.isDisposed) { - console.warn("Disposables already disposed, cannot untrack"); + // already disposed + if (!this._disposables) { return undefined; } const idx = this._disposables!.indexOf(disposable); From 282cba0ff16016fe75a10c257e1a5310460dc8ec Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 30 Sep 2022 11:17:07 +0200 Subject: [PATCH 193/435] call points for PeerCall.delay rely on AbortError actually being thrown + fix this in one call point where it wasn't the case --- src/matrix/calls/PeerCall.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index d7d815fa..a64a1524 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -771,7 +771,8 @@ export class PeerCall implements IDisposable { const {flushCandidatesLog} = this; // MSC2746 recommends these values (can be quite long when calling because the // callee will need a while to answer the call) - await this.delay(this.direction === CallDirection.Inbound ? 500 : 2000); + try { await this.delay(this.direction === CallDirection.Inbound ? 500 : 2000); } + catch (err) { return; } this.sendCandidateQueue(flushCandidatesLog); this.flushCandidatesLog = undefined; } @@ -1100,12 +1101,9 @@ export class PeerCall implements IDisposable { const timeout = this.disposables.track(this.options.createTimeout(timeoutMs)); try { await timeout.elapsed(); - } catch (err) { - if (err.name !== "AbortError") { - throw err; - } + } finally { + this.disposables.untrack(timeout); } - this.disposables.untrack(timeout); } private sendSignallingMessage(message: SignallingMessage, log: ILogItem) { From b5f686b10e052586bbeef935e10d9280301bdeb1 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 30 Sep 2022 11:17:54 +0200 Subject: [PATCH 194/435] also allow clean up while still in Joining state, otherwise we get stuck in joined state but without joinedData --- src/matrix/calls/group/GroupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index cb027be8..0c9d3e90 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -437,7 +437,7 @@ export class GroupCall extends EventEmitter<{change: never}> { /** @internal */ disconnect(log: ILogItem) { - if (this._state === GroupCallState.Joined) { + if (this.hasJoined) { for (const [,member] of this._members) { const disconnectLogItem = member.disconnect(true); if (disconnectLogItem) { From f2564ed5cc4ad9965d6c4670c56def57ba13a91d Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 30 Sep 2022 11:20:00 +0200 Subject: [PATCH 195/435] also emit updates for expired, connected members that we didn't kick --- src/matrix/calls/group/GroupCall.ts | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 0c9d3e90..db908004 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -102,14 +102,15 @@ export class GroupCall extends EventEmitter<{change: never}> { this._memberOptions = Object.assign({}, options, { confId: this.id, emitUpdate: member => { - if (!member.isExpired) { - this._members.update(getMemberKey(member.userId, member.deviceId), member); - } else if (!member.isConnected) { - // don't just kick out connected members, even if their timestamp expired - this._members.remove(getMemberKey(member.userId, member.deviceId)); - } - if (member.isExpired && member.isConnected) { - console.trace("was about to kick a connected but expired member"); + const memberKey = getMemberKey(member.userId, member.deviceId); + // only remove expired members from the call if we don't have a peer conn with them + if (member.isExpired && !member.isConnected) { + this._members.remove(memberKey); + } else { + if (member.isExpired && member.isConnected) { + console.trace("was about to kick a connected but expired member"); + } + this._members.update(memberKey, member); } }, encryptDeviceMessage: (userId: string, deviceId: string, message: SignallingMessage, log) => { From b694d13348e4f23141c49fa3a18b470a4893f834 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 30 Sep 2022 11:20:33 +0200 Subject: [PATCH 196/435] can be readonly --- src/matrix/calls/group/GroupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index db908004..729c152e 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -73,7 +73,7 @@ class JoinedData { public readonly membersLogItem: ILogItem, public localMedia: LocalMedia, public localMuteSettings: MuteSettings, - public turnServer: BaseObservableValue + public readonly turnServer: BaseObservableValue ) {} dispose() { From 1dddabc038b8a39e7d2c0a9f18928c1e67ae8884 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 30 Sep 2022 11:21:26 +0200 Subject: [PATCH 197/435] make removing own membership a bit clearer in the logs --- src/matrix/calls/group/GroupCall.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 729c152e..60099c03 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -432,8 +432,9 @@ export class GroupCall extends EventEmitter<{change: never}> { } private removeOwnDevice(log: ILogItem) { - log.set("leave_own", true); - this.disconnect(log); + log.wrap("remove own membership", log => { + this.disconnect(log); + }); } /** @internal */ From 56ecd39f267de4fb770e9ecae2ca3f307ef0ee95 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 30 Sep 2022 17:46:57 +0200 Subject: [PATCH 198/435] don't assume joinedData is set here although not entirely sure why it wouldn't be --- src/matrix/calls/group/GroupCall.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 60099c03..ac41a772 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -324,7 +324,9 @@ export class GroupCall extends EventEmitter<{change: never}> { log.wrap("update own membership", log => { // TODO: should we check if new device is expired? if (this.hasJoined) { - this.joinedData!.logItem.refDetached(log); + if (this.joinedData) { + this.joinedData.logItem.refDetached(log); + } this._setupRenewMembershipTimeout(device, log); } if (this._state === GroupCallState.Joining) { From 7eb8015ace99a3ce02455ade28af2eab91703544 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 6 Oct 2022 15:20:05 +0200 Subject: [PATCH 199/435] ensure member.dispose is called when removing member so expiration timer is always stopped --- src/matrix/calls/group/GroupCall.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index ac41a772..c787b00a 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -105,6 +105,7 @@ export class GroupCall extends EventEmitter<{change: never}> { const memberKey = getMemberKey(member.userId, member.deviceId); // only remove expired members from the call if we don't have a peer conn with them if (member.isExpired && !member.isConnected) { + member.dispose(); this._members.remove(memberKey); } else { if (member.isExpired && member.isConnected) { @@ -339,6 +340,7 @@ export class GroupCall extends EventEmitter<{change: never}> { log.wrap({l: "update device membership", id: memberKey, sessionId: device.session_id}, log => { if (isMemberExpired(device, now)) { log.set("expired", true); + this._members.get(memberKey)?.dispose(); this._members.remove(memberKey); return; } @@ -354,6 +356,7 @@ export class GroupCall extends EventEmitter<{change: never}> { if (disconnectLogItem) { log.refDetached(disconnectLogItem); } + member.dispose(); this._members.remove(memberKey); member = undefined; } @@ -462,11 +465,12 @@ export class GroupCall extends EventEmitter<{change: never}> { const member = this._members.get(memberKey); if (member) { log.set("leave", true); - this._members.remove(memberKey); const disconnectLogItem = member.disconnect(false); if (disconnectLogItem) { log.refDetached(disconnectLogItem); } + member.dispose(); + this._members.remove(memberKey); } this.emitChange(); }); From 2ecfb8f13911cd1ec35fcb92ba1227ecf5162d40 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 6 Oct 2022 15:22:15 +0200 Subject: [PATCH 200/435] unify dispose logic in Member --- src/matrix/calls/group/Member.ts | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index d4cc93f4..5f6325b5 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -83,6 +83,12 @@ class MemberConnection { return first.content.seq === this.lastProcessedSeqNr || first.content.seq === this.lastProcessedSeqNr + 1; } + + dispose() { + this.peerCall?.dispose(); + this.localMedia.dispose(); + this.logItem.finish(); + } } export class Member { @@ -210,18 +216,15 @@ export class Member { return; } let disconnectLogItem; + // if if not sending the hangup, still log disconnect connection.logItem.wrap("disconnect", async log => { disconnectLogItem = log; - if (hangup) { - await connection.peerCall?.hangup(CallErrorCode.UserHangup, log); - } else { - await connection.peerCall?.close(undefined, log); + if (hangup && connection.peerCall) { + await connection.peerCall.hangup(CallErrorCode.UserHangup, log); } - connection.peerCall?.dispose(); - connection.localMedia?.dispose(); - this.connection = undefined; }); - connection.logItem.finish(); + connection.dispose(); + this.connection = undefined; return disconnectLogItem; } @@ -395,8 +398,10 @@ export class Member { } dispose() { + this.connection?.dispose(); + this.connection = undefined; this.expireTimeout?.dispose(); - this.connection?.peerCall?.dispose(); + this.expireTimeout = undefined; } } From c87fd4dfed64346449876faacf48f8fecaf25109 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 6 Oct 2022 15:59:45 +0200 Subject: [PATCH 201/435] cleanup --- src/matrix/calls/group/GroupCall.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index c787b00a..64249176 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -548,8 +548,11 @@ export class GroupCall extends EventEmitter<{change: never}> { private connectToMember(member: Member, joinedData: JoinedData, log: ILogItem) { const memberKey = getMemberKey(member.userId, member.deviceId); - const logItem = joinedData.membersLogItem.child({l: "member", id: memberKey}); - logItem.set("sessionId", member.sessionId); + const logItem = joinedData.membersLogItem.child({ + l: "member", + id: memberKey, + sessionId: member.sessionId + }); log.wrap({l: "connect", id: memberKey}, log => { const connectItem = member.connect( joinedData.localMedia, From 7ae9c3af0d7cec33cb82b8a9a87ad82d8220a985 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 6 Oct 2022 16:00:38 +0200 Subject: [PATCH 202/435] renew at least 10s before own membership expires --- src/matrix/calls/group/GroupCall.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 64249176..bdb3844b 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -583,10 +583,10 @@ export class GroupCall extends EventEmitter<{change: never}> { return; } const expiresFromNow = expiresAt - this.options.clock.now(); - // renew 1 to 5 minutes (8.3% of 1h) before expiring + // renew 1 to 5 minutes (8.3% of 1h, but min 10s) before expiring // do it a bit beforehand and somewhat random to not collide with // other clients trying to renew as well - const timeToRenewBeforeExpiration = Math.max(4000, Math.ceil((0.2 +(this.options.random() * 0.8)) * (0.08333 * CALL_MEMBER_VALIDITY_PERIOD_MS))); + const timeToRenewBeforeExpiration = Math.max(10000, Math.ceil((0.2 +(this.options.random() * 0.8)) * (0.08333 * CALL_MEMBER_VALIDITY_PERIOD_MS))); const renewFromNow = Math.max(0, expiresFromNow - timeToRenewBeforeExpiration); log.set("expiresIn", expiresFromNow); log.set("renewIn", renewFromNow); From a8ac504efdc0966badb670fd4186f601d8f7d957 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 6 Oct 2022 16:10:36 +0200 Subject: [PATCH 203/435] explain when joinedData is set --- src/matrix/calls/group/GroupCall.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index bdb3844b..c90c7d2c 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -88,6 +88,7 @@ export class GroupCall extends EventEmitter<{change: never}> { private _memberOptions: MemberOptions; private _state: GroupCallState; private bufferedDeviceMessages = new Map>>(); + /** Set between calling join and leave. */ private joinedData?: JoinedData; constructor( From a07be730f94bc290df5b65af345e33f73ee05e2f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 6 Oct 2022 16:10:50 +0200 Subject: [PATCH 204/435] bring member validity period back to 1h --- src/matrix/calls/common.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/matrix/calls/common.ts b/src/matrix/calls/common.ts index dde55ddc..015e0df8 100644 --- a/src/matrix/calls/common.ts +++ b/src/matrix/calls/common.ts @@ -59,5 +59,4 @@ export class MuteSettings { } export const CALL_LOG_TYPE = "call"; -//export const CALL_MEMBER_VALIDITY_PERIOD_MS = 3600 * 1000; // 1h -export const CALL_MEMBER_VALIDITY_PERIOD_MS = 60 * 1000; +export const CALL_MEMBER_VALIDITY_PERIOD_MS = 3600 * 1000; // 1h From 4a36fd96e464727c06a1c45ed64a52cf1d5f610c Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 6 Oct 2022 16:11:16 +0200 Subject: [PATCH 205/435] update bundled logviewer --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index e3f82fa1..f470bd9f 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ }, "homepage": "https://github.com/vector-im/hydrogen-web/#readme", "devDependencies": { - "@matrixdotorg/structured-logviewer": "^0.0.1", + "@matrixdotorg/structured-logviewer": "^0.0.3", "@typescript-eslint/eslint-plugin": "^4.29.2", "@typescript-eslint/parser": "^4.29.2", "acorn": "^8.6.0", diff --git a/yarn.lock b/yarn.lock index 0408a6e0..9b9e5453 100644 --- a/yarn.lock +++ b/yarn.lock @@ -56,10 +56,10 @@ version "3.2.8" resolved "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.8.tgz#8d53636d045e1776e2a2ec6613e57330dd9ce856" -"@matrixdotorg/structured-logviewer@^0.0.1": - version "0.0.1" - resolved "https://registry.yarnpkg.com/@matrixdotorg/structured-logviewer/-/structured-logviewer-0.0.1.tgz#9c29470b552f874afbb1df16c6e8e9e0c55cbf59" - integrity sha512-IdPYxAFDEoEs2G1ImKCkCxFI3xF1DDctP3N9JOtHRvIPbPPdTT9DyNqKTewCb5zwjNB1mGBrnWyURnHDiOOL3w== +"@matrixdotorg/structured-logviewer@^0.0.3": + version "0.0.3" + resolved "https://registry.yarnpkg.com/@matrixdotorg/structured-logviewer/-/structured-logviewer-0.0.3.tgz#1555111159d83cde0cfd5ba1a571e1faa1a90871" + integrity sha512-QqFglx0M8ix0IoRsJXDg1If26ltbYfuLjJ0MQrJYze3yz4ayEESRpQEA0YxJRVVtbco5M94tmrDpikokTFnn3A== "@nodelib/fs.scandir@2.1.5": version "2.1.5" From 400df6a4fff84ac8fa0f7460541abcd1659e62ce Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 7 Oct 2022 10:07:10 +0200 Subject: [PATCH 206/435] ensure removing expired members from call is always logged also return the log item from logger.log so we can ref it --- src/logging/Logger.ts | 3 ++- src/logging/types.ts | 2 +- src/matrix/calls/group/GroupCall.ts | 11 +++++++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/logging/Logger.ts b/src/logging/Logger.ts index 37a3326c..d3aa1d6d 100644 --- a/src/logging/Logger.ts +++ b/src/logging/Logger.ts @@ -30,10 +30,11 @@ export class Logger implements ILogger { this._platform = platform; } - log(labelOrValues: LabelOrValues, logLevel: LogLevel = LogLevel.Info): void { + log(labelOrValues: LabelOrValues, logLevel: LogLevel = LogLevel.Info): ILogItem { const item = new LogItem(labelOrValues, logLevel, this); item.end = item.start; this._persistItem(item, undefined, false); + return item; } /** Prefer `run()` or `log()` above this method; only use it if you have a long-running operation diff --git a/src/logging/types.ts b/src/logging/types.ts index 0ed3b0dc..0f4f6150 100644 --- a/src/logging/types.ts +++ b/src/logging/types.ts @@ -68,7 +68,7 @@ export interface ILogItemCreator { */ export interface ILogger { - log(labelOrValues: LabelOrValues, logLevel?: LogLevel): void; + log(labelOrValues: LabelOrValues, logLevel?: LogLevel): ILogItem; child(labelOrValues: LabelOrValues, logLevel?: LogLevel, filterCreator?: FilterCreator): ILogItem; wrapOrRun(item: ILogItem | undefined, labelOrValues: LabelOrValues, callback: LogCallback, logLevel?: LogLevel, filterCreator?: FilterCreator): T; runDetached(labelOrValues: LabelOrValues, callback: LogCallback, logLevel?: LogLevel, filterCreator?: FilterCreator): ILogItem; diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index c90c7d2c..a8b6605f 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -104,14 +104,17 @@ export class GroupCall extends EventEmitter<{change: never}> { confId: this.id, emitUpdate: member => { const memberKey = getMemberKey(member.userId, member.deviceId); - // only remove expired members from the call if we don't have a peer conn with them + // only remove expired members to whom we're not already connected if (member.isExpired && !member.isConnected) { + const logItem = this.options.logger.log({ + l: "removing expired member from call", + memberKey, + callId: this.id + }) + member.logItem?.refDetached(logItem); member.dispose(); this._members.remove(memberKey); } else { - if (member.isExpired && member.isConnected) { - console.trace("was about to kick a connected but expired member"); - } this._members.update(memberKey, member); } }, From 793686393414006c98f6a7dd73e5c092f2200362 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 7 Oct 2022 10:07:46 +0200 Subject: [PATCH 207/435] remove options on dispose in Member to ensure callback can't be called anymore, as we don't check that the member argument is the one currently in GroupCall._members. --- src/matrix/calls/group/Member.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 5f6325b5..8763bf5e 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -98,7 +98,7 @@ export class Member { constructor( public member: RoomMember, private callDeviceMembership: CallDeviceMembership, - private readonly options: Options, + private options: Options, updateMemberLog: ILogItem ) { this._renewExpireTimeout(updateMemberLog); @@ -402,6 +402,8 @@ export class Member { this.connection = undefined; this.expireTimeout?.dispose(); this.expireTimeout = undefined; + // ensure the emitUpdate callback can't be called anymore + this.options = undefined as any as Options; } } From 4350537004c3e9af6d2a6c633a1bbd440d321fd7 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 7 Oct 2022 10:58:12 +0200 Subject: [PATCH 208/435] cleanup --- src/matrix/calls/group/GroupCall.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index a8b6605f..84fb9be9 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -344,8 +344,12 @@ export class GroupCall extends EventEmitter<{change: never}> { log.wrap({l: "update device membership", id: memberKey, sessionId: device.session_id}, log => { if (isMemberExpired(device, now)) { log.set("expired", true); - this._members.get(memberKey)?.dispose(); - this._members.remove(memberKey); + const member = this._members.get(memberKey); + if (member) { + member.dispose(); + this._members.remove(memberKey); + log.set("removed", true); + } return; } let member = this._members.get(memberKey); From 7d10bec4acb65c8fd2c696a0994a0f98c6467982 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 7 Oct 2022 10:59:13 +0200 Subject: [PATCH 209/435] remove comment not relevant anymore --- src/matrix/calls/group/GroupCall.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 84fb9be9..f2561f32 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -327,7 +327,6 @@ export class GroupCall extends EventEmitter<{change: never}> { const memberKey = getMemberKey(userId, deviceId); if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) { log.wrap("update own membership", log => { - // TODO: should we check if new device is expired? if (this.hasJoined) { if (this.joinedData) { this.joinedData.logItem.refDetached(log); From e9b8cfbd9e328437f2484d0a384a4a76b6e2493f Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 18 Oct 2022 12:56:58 -0700 Subject: [PATCH 210/435] Member should be expired if there's no expires at key --- src/matrix/calls/group/Member.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 8763bf5e..0aba3b54 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -416,5 +416,5 @@ export function memberExpiresAt(callDeviceMembership: CallDeviceMembership): num export function isMemberExpired(callDeviceMembership: CallDeviceMembership, now: number, margin: number = 0) { const expiresAt = memberExpiresAt(callDeviceMembership); - return typeof expiresAt === "number" && ((expiresAt + margin) <= now); + return typeof expiresAt === "number" ? ((expiresAt + margin) <= now) : true; } From c0306b45a606023793aaa869a175e7661bad2824 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 18 Oct 2022 13:43:19 -0700 Subject: [PATCH 211/435] Fix typechecking --- src/logging/NullLogger.ts | 4 +++- src/platform/web/ui/session/room/CallView.ts | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/logging/NullLogger.ts b/src/logging/NullLogger.ts index f6f877b3..bf7b3f40 100644 --- a/src/logging/NullLogger.ts +++ b/src/logging/NullLogger.ts @@ -21,7 +21,9 @@ function noop (): void {} export class NullLogger implements ILogger { public readonly item: ILogItem = new NullLogItem(this); - log(): void {} + log(): ILogItem { + return this.item; + } addReporter() {} diff --git a/src/platform/web/ui/session/room/CallView.ts b/src/platform/web/ui/session/room/CallView.ts index 6961dc53..619afc2e 100644 --- a/src/platform/web/ui/session/room/CallView.ts +++ b/src/platform/web/ui/session/room/CallView.ts @@ -80,7 +80,7 @@ export class CallView extends TemplateView { public unmount() { if (this.resizeObserver) { - this.resizeObserver.unobserve((this.root()! as Element).querySelector(".CallView_members")); + this.resizeObserver.unobserve((this.root()! as Element).querySelector(".CallView_members")!); this.resizeObserver = undefined; } super.unmount(); From 32835e26b9b2435dc7db32d76a82cf698ee11216 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 10 Nov 2022 14:08:34 +0100 Subject: [PATCH 212/435] adjust query for finding all state events of a type we were looking for state events with state key between \0 and \u10FFFF but an empty state key is also valid and would come before \0, so allow empty state keys at the beginning of the range and include them in the result by opening the lower bound. --- src/matrix/storage/idb/stores/RoomStateStore.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/matrix/storage/idb/stores/RoomStateStore.ts b/src/matrix/storage/idb/stores/RoomStateStore.ts index 99315e9e..e7661378 100644 --- a/src/matrix/storage/idb/stores/RoomStateStore.ts +++ b/src/matrix/storage/idb/stores/RoomStateStore.ts @@ -43,9 +43,9 @@ export class RoomStateStore { getAllForType(roomId: string, type: string): Promise { const range = this._roomStateStore.IDBKeyRange.bound( - encodeKey(roomId, type, MIN_UNICODE), + encodeKey(roomId, type, ""), encodeKey(roomId, type, MAX_UNICODE), - true, + false, true ); return this._roomStateStore.selectAll(range); From 4125165760b866e7aa4e7f18c36e4fe05c025eb8 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 10 Nov 2022 20:59:27 -0600 Subject: [PATCH 213/435] Add full MXID tooltip to message sender display name Follow-up to https://github.com/vector-im/hydrogen-web/pull/917 --- .../web/ui/session/room/timeline/BaseMessageView.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/platform/web/ui/session/room/timeline/BaseMessageView.js b/src/platform/web/ui/session/room/timeline/BaseMessageView.js index ee0a37db..d998e826 100644 --- a/src/platform/web/ui/session/room/timeline/BaseMessageView.js +++ b/src/platform/web/ui/session/room/timeline/BaseMessageView.js @@ -63,7 +63,13 @@ export class BaseMessageView extends TemplateView { li.removeChild(li.querySelector(".Timeline_messageSender")); } else if (!isContinuation && !this._isReplyPreview) { const avatar = tag.a({href: vm.memberPanelLink, className: "Timeline_messageAvatar"}, [renderStaticAvatar(vm, 30)]); - const sender = tag.div({className: `Timeline_messageSender usercolor${vm.avatarColorNumber}`}, vm.displayName); + const sender = tag.div( + { + className: `Timeline_messageSender usercolor${vm.avatarColorNumber}`, + title: vm.sender, + }, + vm.displayName, + ); li.insertBefore(avatar, li.firstChild); li.insertBefore(sender, li.firstChild); } From 1aba18ca1db687d78cc86d2062cf3154ead443da Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 10 Nov 2022 21:14:57 -0600 Subject: [PATCH 214/435] Add ,
, and HTML elements to templating --- src/platform/web/ui/general/html.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/platform/web/ui/general/html.ts b/src/platform/web/ui/general/html.ts index 4e1fe78d..93512897 100644 --- a/src/platform/web/ui/general/html.ts +++ b/src/platform/web/ui/general/html.ts @@ -102,10 +102,10 @@ export const SVG_NS: string = "http://www.w3.org/2000/svg"; export const TAG_NAMES = { [HTML_NS]: [ "br", "a", "ol", "ul", "li", "div", "h1", "h2", "h3", "h4", "h5", "h6", - "p", "strong", "em", "span", "img", "section", "header", "main", "footer", - "article", "aside", "del", "blockquote", + "p", "strong", "em", "span", "img", "section", "header", "main", "footer", "dialog", + "article", "aside", "del", "blockquote", "details", "summary", "table", "thead", "tbody", "tr", "th", "td", "hr", - "pre", "code", "button", "time", "input", "textarea", "select", "option", "label", "form", + "pre", "code", "button", "time", "input", "textarea", "select", "option", "optgroup", "label", "form", "progress", "output", "video", "style"], [SVG_NS]: ["svg", "g", "path", "circle", "ellipse", "rect", "use"] } as const; From 40f16a40fc04471bc1183fb96d7abcebe6802d7a Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 15 Nov 2022 09:52:06 +0000 Subject: [PATCH 215/435] Update README.md Fix #909 --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 6c447024..336b5b53 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,8 @@ You can run Hydrogen locally by the following commands in the terminal: Now point your browser to `http://localhost:3000`. If you prefer, you can also [use docker](doc/docker.md). +PS: You need nodejs, running yarn on top of any other js platform is not support. + # FAQ Some frequently asked questions are answered [here](doc/FAQ.md). From 2b218abda38cded949fa8c6d8701b0c1296227bc Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 15 Nov 2022 09:52:34 +0000 Subject: [PATCH 216/435] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 336b5b53..a4c1529f 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ You can run Hydrogen locally by the following commands in the terminal: Now point your browser to `http://localhost:3000`. If you prefer, you can also [use docker](doc/docker.md). -PS: You need nodejs, running yarn on top of any other js platform is not support. +PS: You need nodejs, running yarn on top of any other js platform is not supported. # FAQ From 12e378eb62cb1cef52639c15ff2eab7cb22f760f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 18 Nov 2022 09:52:57 +0100 Subject: [PATCH 217/435] some thoughts on how different room types could be implemented --- doc/impl-thoughts/room-types.ts | 55 +++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 doc/impl-thoughts/room-types.ts diff --git a/doc/impl-thoughts/room-types.ts b/doc/impl-thoughts/room-types.ts new file mode 100644 index 00000000..f683e927 --- /dev/null +++ b/doc/impl-thoughts/room-types.ts @@ -0,0 +1,55 @@ +/* +different room types create different kind of "sync listeners", who implement the sync lifecycle handlers + +they would each have a factory, +*/ + +interface IRoomSyncHandler { + prepareSync() + afterPrepareSync() + writeSync() + afterSync() + afterSyncCompleted() +} + +interface IRoom extends IRoomSyncHandler { + start(): void; + load(): void; + get id(): string; +} + +interface IRoomFactory { + createRoom(type, roomId, syncResponse): T + createSchema(db, txn, oldVersion, version, log) + get storesForSync(): string[]; + get rooms(): ObservableMap +} + +class InstantMessageRoom implements IRoom { +} + +class InstantMessageRoomFactory implements IRoomFactory{ + loadLastMessages(): Promise + /* + get all room ids and sort them according to idb sorting order + open cursor 'f' on `timelineFragments` + open a cursor 'e' on `timelineEvents` + for each room: + with cursor 'f', go to last fragment id and go up from there to find live fragment + with cursor 'e', go to last event index for fragment id and room id and go up until we have acceptable event type + for encrypted rooms: + decrypt message if needed (m.room.encrypted is likely something we want to display) + */ +} + +class SpaceRoom implements IRoom {} + +class SpaceRoomFactory implements IRoomFactory { + createRoom(type, roomId, syncResponse): IRoomSyncHandler +} + +class Session { + constructor(roomFactoriesByType: Map) { + + } +} From 3f7c1577e0fdd9d22682b9bd7e4578c0c0174db1 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 18 Nov 2022 23:26:59 +0100 Subject: [PATCH 218/435] WIP --- .../session/room/timeline/TilesCollection.js | 76 ++++++++-- .../room/timeline/tiles/BaseMessageTile.js | 3 +- .../session/room/timeline/tiles/DateTile.ts | 141 ++++++++++++++++++ .../session/room/timeline/tiles/GapTile.js | 5 + .../session/room/timeline/tiles/ITile.ts | 44 ++++++ .../session/room/timeline/tiles/SimpleTile.js | 32 +++- .../session/room/timeline/tiles/index.ts | 4 +- src/platform/web/ui/session/room/common.ts | 27 ++-- .../session/room/timeline/DateHeaderView.ts | 33 ++++ 9 files changed, 332 insertions(+), 33 deletions(-) create mode 100644 src/domain/session/room/timeline/tiles/DateTile.ts create mode 100644 src/domain/session/room/timeline/tiles/ITile.ts create mode 100644 src/platform/web/ui/session/room/timeline/DateHeaderView.ts diff --git a/src/domain/session/room/timeline/TilesCollection.js b/src/domain/session/room/timeline/TilesCollection.js index 173b0cf6..8dfbf2a1 100644 --- a/src/domain/session/room/timeline/TilesCollection.js +++ b/src/domain/session/room/timeline/TilesCollection.js @@ -16,6 +16,7 @@ limitations under the License. import {BaseObservableList} from "../../../../observable/list/BaseObservableList"; import {sortedIndex} from "../../../../utils/sortedIndex"; +import {TileShape} from "./tiles/ITile"; // maps 1..n entries to 0..1 tile. Entries are what is stored in the timeline, either an event or fragmentboundary // for now, tileClassForEntry should be stable in whether it returns a tile or not. @@ -51,12 +52,14 @@ export class TilesCollection extends BaseObservableList { } _populateTiles() { + this._silent = true; this._tiles = []; let currentTile = null; for (let entry of this._entries) { if (!currentTile || !currentTile.tryIncludeEntry(entry)) { currentTile = this._createTile(entry); if (currentTile) { + console.log("adding initial tile", currentTile.shape, currentTile.eventId, "at", this._tiles.length); this._tiles.push(currentTile); } } @@ -72,11 +75,20 @@ export class TilesCollection extends BaseObservableList { if (prevTile) { prevTile.updateNextSibling(null); } + // add date headers here + for (let idx = 0; idx < this._tiles.length; idx += 1) { + const tile = this._tiles[idx]; + if (tile.needsDateSeparator) { + this._addTileAt(idx, tile.createDateSeparator(), true); + idx += 1; // tile's index moved one up, don't process it again + } + } // now everything is wired up, // allow tiles to emit updates for (const tile of this._tiles) { tile.setUpdateEmit(this._emitSpontanousUpdate); } + this._silent = false; } _findTileIdx(entry) { @@ -130,25 +142,61 @@ export class TilesCollection extends BaseObservableList { const newTile = this._createTile(entry); if (newTile) { - if (prevTile) { - prevTile.updateNextSibling(newTile); - // this emits an update while the add hasn't been emitted yet - newTile.updatePreviousSibling(prevTile); - } - if (nextTile) { - newTile.updateNextSibling(nextTile); - nextTile.updatePreviousSibling(newTile); - } - this._tiles.splice(tileIdx, 0, newTile); - this.emitAdd(tileIdx, newTile); - // add event is emitted, now the tile - // can emit updates - newTile.setUpdateEmit(this._emitSpontanousUpdate); + console.log("adding tile", newTile.shape, newTile.eventId, "at", tileIdx); + this._addTileAt(tileIdx, newTile); + this._evaluateDateHeaderAtIdx(tileIdx); } // find position by sort key // ask siblings to be included? both? yes, twice: a (insert c here) b, ask a(c), if yes ask b(a), else ask b(c)? if yes then b(a)? } + _evaluateDateHeaderAtIdx(tileIdx) { + //console.log("_evaluateDateHeaderAtIdx", tileIdx); + // consider the two adjacent tiles where the previous sibling changed: + // the new tile and the next tile + for (let i = 0; i < 5; i += 1) { + const idx = Math.max(tileIdx + i - 2, 0); + const tile = this._tiles[idx]; + const prevTile = idx > 0 ? this._tiles[idx - 1] : undefined; + const hasDateSeparator = prevTile?.shape === TileShape.DateHeader; + if (tile.needsDateSeparator) { + if (hasDateSeparator) { + // TODO: replace this by return UpdateAction from updateNextSibling + // and do this in onAdd + //console.log(" update", idx - 1, prevTile?.shape, prevTile?.eventId); + this.emitUpdate(idx - 1, prevTile, "date"); + } else { + //console.log(" add", idx, tile.shape, tile.eventId); + tileIdx += 1; + this._addTileAt(idx, tile.createDateSeparator()); + } + // TODO must be looking at the wrong index to find the old date separator?? + } else if (hasDateSeparator) { + // this is never triggered because needsDateSeparator is not cleared + // when loading more items because we don't do anything once the + // direct sibling is a DateTile + //console.log(" remove", idx -1, prevTile?.shape, prevTile?.eventId); + this._removeTile(idx - 1, prevTile); + } + } + } + + _addTileAt(idx, newTile, silent = false) { + const prevTile = idx > 0 ? this._tiles[idx - 1] : undefined; + const nextTile = this._tiles[idx]; + prevTile?.updateNextSibling(newTile); + newTile.updatePreviousSibling(prevTile); + newTile.updateNextSibling(nextTile); + nextTile?.updatePreviousSibling(newTile); + this._tiles.splice(idx, 0, newTile); + if (!silent) { + this.emitAdd(idx, newTile); + } + // add event is emitted, now the tile + // can emit updates + newTile.setUpdateEmit(this._emitSpontanousUpdate); + } + onUpdate(index, entry, params) { // if an update is emitted while calling source.subscribe() from onSubscribeFirst, ignore it if (!this._tiles) { diff --git a/src/domain/session/room/timeline/tiles/BaseMessageTile.js b/src/domain/session/room/timeline/tiles/BaseMessageTile.js index cfa27a94..1ad1ba44 100644 --- a/src/domain/session/room/timeline/tiles/BaseMessageTile.js +++ b/src/domain/session/room/timeline/tiles/BaseMessageTile.js @@ -15,13 +15,13 @@ limitations under the License. */ import {SimpleTile} from "./SimpleTile.js"; +import {TileShape} from "./ITile"; import {ReactionsViewModel} from "../ReactionsViewModel.js"; import {getIdentifierColorNumber, avatarInitials, getAvatarHttpUrl} from "../../../../avatar"; export class BaseMessageTile extends SimpleTile { constructor(entry, options) { super(entry, options); - this._date = this._entry.timestamp ? new Date(this._entry.timestamp) : null; this._isContinuation = false; this._reactions = null; this._replyTile = null; @@ -78,6 +78,7 @@ export class BaseMessageTile extends SimpleTile { return this.sender; } + // TODO: remove? get date() { return this._date && this._date.toLocaleDateString({}, {month: "numeric", day: "numeric"}); } diff --git a/src/domain/session/room/timeline/tiles/DateTile.ts b/src/domain/session/room/timeline/tiles/DateTile.ts new file mode 100644 index 00000000..413ec2b2 --- /dev/null +++ b/src/domain/session/room/timeline/tiles/DateTile.ts @@ -0,0 +1,141 @@ +import {ITile, TileShape, EmitUpdateFn} from "./ITile"; +import {UpdateAction} from "../UpdateAction"; +import {BaseEntry} from "../../../../../matrix/room/timeline/entries/BaseEntry"; +import {BaseEventEntry} from "../../../../../matrix/room/timeline/entries/BaseEventEntry"; +import {ViewModel} from "../../../../ViewModel"; +import type {Options} from "../../../../ViewModel"; + +/** + * edge cases: + * - be able to remove the tile in response to the sibling changing, + * probably by letting updateNextSibling/updatePreviousSibling + * return an UpdateAction and change TilesCollection accordingly. + * this is relevant when next becomes undefined there when + * a pending event is removed on remote echo. + * */ + +export class DateTile extends ViewModel implements ITile { + private _emitUpdate?: EmitUpdateFn; + private _dateString?: string; + + constructor(private _firstTileInDay: ITile, options: Options) { + super(options); + } + + setUpdateEmit(emitUpdate: EmitUpdateFn): void { + this._emitUpdate = emitUpdate; + } + + get upperEntry(): BaseEventEntry { + return this.refEntry; + } + + get lowerEntry(): BaseEventEntry { + return this.refEntry; + } + + /** the entry reference by this datetile, e.g. the entry of the first tile for this day */ + private get refEntry(): BaseEventEntry { + // lowerEntry is the first entry... i think? + // so given the date header always comes before, + // this is our closest entry. + return this._firstTileInDay.lowerEntry; + } + + compare(tile: ITile): number { + return this.compareEntry(tile.upperEntry); + } + + get date(): string { + if (!this._dateString) { + const date = new Date(this.refEntry.timestamp); + this._dateString = date.toLocaleDateString({}, { + weekday: 'long', + year: 'numeric', + month: 'long', + day: 'numeric' + }); + } + return this._dateString; + } + + get shape(): TileShape { + return TileShape.DateHeader; + } + + get needsDateSeparator(): boolean { + return false; + } + + createDateSeparator(): undefined { + return undefined; + } + +/** + * _findTileIdx in TilesCollection should never return + * the index of a DateTile as that is mainly used + * for mapping incoming event indices coming from the Timeline + * to the tile index to propage the event. + * This is not a path that is relevant to date headers as they + * are added as a side-effect of adding other tiles and are generally + * not updated (only removed in some cases). _findTileIdx is also + * used for emitting spontanous updates, but that should also not be + * needed for a DateTile. + * The problem is basically that _findTileIdx maps an entry to + * a tile, and DateTile adopts the entry of it's sibling tile (_firstTileInDay) + * so now we have the entry pointing to two tiles. So we should avoid + * returning the DateTile itself from the compare method. + * We will always return -1 or 1 from here to signal an entry comes before or after us, + * never 0 + * */ + compareEntry(entry: BaseEntry): number { + const result = this.refEntry.compare(entry); + if (result === 0) { + // if it's a match for the reference entry (e.g. _firstTileInDay), + // say it comes after us as the date tile always comes at the top + // of the day. + return -1; + } + // otherwise, assume the given entry is never for ourselves + // as we don't have our own entry, we only borrow one from _firstTileInDay + return result; + } + + // update received for already included (falls within sort keys) entry + updateEntry(entry, param): UpdateAction { + return UpdateAction.Nothing(); + } + + // return whether the tile should be removed + // as SimpleTile only has one entry, the tile should be removed + removeEntry(entry: BaseEntry): boolean { + return false; + } + + // SimpleTile can only contain 1 entry + tryIncludeEntry(): boolean { + return false; + } + + // let item know it has a new sibling + updatePreviousSibling(prev: ITile | undefined): void { + this._firstTileInDay.updatePreviousSibling(prev); + } + + // let item know it has a new sibling + updateNextSibling(next: ITile | undefined): UpdateAction { + // TODO: next can be undefined when a pending event is removed + // TODO: we need a way to remove this date header + this._firstTileInDay = next!; + this._dateString = undefined; + // TODO: do we need to reevaluate our date here and emit an update? + } + + notifyVisible(): void { + // trigger sticky logic here? + } + + dispose(): void { + + } +} diff --git a/src/domain/session/room/timeline/tiles/GapTile.js b/src/domain/session/room/timeline/tiles/GapTile.js index bb7d8086..1e6bdd08 100644 --- a/src/domain/session/room/timeline/tiles/GapTile.js +++ b/src/domain/session/room/timeline/tiles/GapTile.js @@ -19,6 +19,7 @@ import {UpdateAction} from "../UpdateAction.js"; import {ConnectionError} from "../../../../../matrix/error.js"; import {ConnectionStatus} from "../../../../../matrix/net/Reconnector"; +// TODO: should this become an ITile and SimpleTile become EventTile? export class GapTile extends SimpleTile { constructor(entry, options) { super(entry, options); @@ -29,6 +30,10 @@ export class GapTile extends SimpleTile { this._showSpinner = false; } + get needsDateSeparator() { + return false; + } + async fill() { if (!this._loading && !this._entry.edgeReached) { this._loading = true; diff --git a/src/domain/session/room/timeline/tiles/ITile.ts b/src/domain/session/room/timeline/tiles/ITile.ts new file mode 100644 index 00000000..dd6c0f81 --- /dev/null +++ b/src/domain/session/room/timeline/tiles/ITile.ts @@ -0,0 +1,44 @@ +import {UpdateAction} from "../UpdateAction.js"; +import {BaseEntry} from "../../../../../matrix/room/timeline/entries/BaseEntry"; +import {BaseEventEntry} from "../../../../../matrix/room/timeline/entries/BaseEventEntry"; +import {IDisposable} from "../../../../../utils/Disposables"; + +export type EmitUpdateFn = (tile: ITile, props: any) => void + +export enum TileShape { + Message = "message", + MessageStatus = "message-status", + Announcement = "announcement", + File = "file", + Gap = "gap", + Image = "image", + Location = "location", + MissingAttachment = "missing-attachment", + Redacted = "redacted", + Video = "video", + DateHeader = "date-header" +} + +// TODO: should we imply inheriting from view model here? +export interface ITile extends IDisposable { + setUpdateEmit(emitUpdate: EmitUpdateFn): void; + get upperEntry(): E; + get lowerEntry(): E; + compare(tile: ITile): number; + compareEntry(entry: BaseEntry): number; + // update received for already included (falls within sort keys) entry + updateEntry(entry: BaseEntry, param: any): UpdateAction; + // return whether the tile should be removed + // as SimpleTile only has one entry, the tile should be removed + removeEntry(entry: BaseEntry): boolean + // SimpleTile can only contain 1 entry + tryIncludeEntry(): boolean; + // let item know it has a new sibling + updatePreviousSibling(prev: ITile | undefined): void; + // let item know it has a new sibling + updateNextSibling(next: ITile | undefined): void; + notifyVisible(): void; + get needsDateSeparator(): boolean; + createDateSeparator(): ITile | undefined; + get shape(): TileShape; +} diff --git a/src/domain/session/room/timeline/tiles/SimpleTile.js b/src/domain/session/room/timeline/tiles/SimpleTile.js index 04141576..7e394d25 100644 --- a/src/domain/session/room/timeline/tiles/SimpleTile.js +++ b/src/domain/session/room/timeline/tiles/SimpleTile.js @@ -15,13 +15,17 @@ limitations under the License. */ import {UpdateAction} from "../UpdateAction.js"; +import {TileShape} from "./ITile"; import {ViewModel} from "../../../../ViewModel"; import {SendStatus} from "../../../../../matrix/room/sending/PendingEvent.js"; +import {DateTile} from "./DateTile"; export class SimpleTile extends ViewModel { constructor(entry, options) { super(options); this._entry = entry; + this._date = this._entry.timestamp ? new Date(this._entry.timestamp) : undefined; + this._needsDateSeparator = false; this._emitUpdate = undefined; } // view model props for all subclasses @@ -37,8 +41,26 @@ export class SimpleTile extends ViewModel { return false; } - get hasDateSeparator() { - return false; + get needsDateSeparator() { + return this._needsDateSeparator; + } + + createDateSeparator() { + return new DateTile(this, this.childOptions({})); + } + + _updateDateSeparator(prev) { + if (prev && prev._date && this._date) { + const neededDateSeparator = this._needsDateSeparator; + this._needsDateSeparator = prev._date.getFullYear() !== this._date.getFullYear() || + prev._date.getMonth() !== this._date.getMonth() || + prev._date.getDate() !== this._date.getDate(); + if (neededDateSeparator && !this._needsDateSeparator) { + console.log("clearing needsDateSeparator", {this: this._entry.content, prev: prev.content}); + } + } else { + this._needsDateSeparator = !!this._date; + } } get id() { @@ -123,8 +145,10 @@ export class SimpleTile extends ViewModel { return false; } // let item know it has a new sibling - updatePreviousSibling(/*prev*/) { - + updatePreviousSibling(prev) { + if (prev?.shape !== TileShape.DateHeader) { + this._updateDateSeparator(prev); + } } // let item know it has a new sibling diff --git a/src/domain/session/room/timeline/tiles/index.ts b/src/domain/session/room/timeline/tiles/index.ts index 242bea2f..6822a91d 100644 --- a/src/domain/session/room/timeline/tiles/index.ts +++ b/src/domain/session/room/timeline/tiles/index.ts @@ -27,7 +27,7 @@ import {EncryptedEventTile} from "./EncryptedEventTile.js"; import {EncryptionEnabledTile} from "./EncryptionEnabledTile.js"; import {MissingAttachmentTile} from "./MissingAttachmentTile.js"; -import type {SimpleTile} from "./SimpleTile.js"; +import type {ITile, TileShape} from "./ITile"; import type {Room} from "../../../../../matrix/room/Room"; import type {Timeline} from "../../../../../matrix/room/timeline/Timeline"; import type {FragmentBoundaryEntry} from "../../../../../matrix/room/timeline/entries/FragmentBoundaryEntry"; @@ -42,7 +42,7 @@ export type Options = ViewModelOptions & { timeline: Timeline tileClassForEntry: TileClassForEntryFn; }; -export type TileConstructor = new (entry: TimelineEntry, options: Options) => SimpleTile; +export type TileConstructor = new (entry: TimelineEntry, options: Options) => ITile; export function tileClassForEntry(entry: TimelineEntry): TileConstructor | undefined { if (entry.isGap) { diff --git a/src/platform/web/ui/session/room/common.ts b/src/platform/web/ui/session/room/common.ts index 7b62630f..6a47ff90 100644 --- a/src/platform/web/ui/session/room/common.ts +++ b/src/platform/web/ui/session/room/common.ts @@ -22,31 +22,34 @@ import {LocationView} from "./timeline/LocationView.js"; import {MissingAttachmentView} from "./timeline/MissingAttachmentView.js"; import {AnnouncementView} from "./timeline/AnnouncementView.js"; import {RedactedView} from "./timeline/RedactedView.js"; -import {SimpleTile} from "../../../../../domain/session/room/timeline/tiles/SimpleTile.js"; +import {ITile, TileShape} from "../../../../../domain/session/room/timeline/tiles/ITile.js"; import {GapView} from "./timeline/GapView.js"; +import {DateHeaderView} from "./timeline/DateHeaderView"; import type {TileViewConstructor, ViewClassForEntryFn} from "./TimelineView"; -export function viewClassForTile(vm: SimpleTile): TileViewConstructor { +export function viewClassForTile(vm: ITile): TileViewConstructor { switch (vm.shape) { - case "gap": + case TileShape.Gap: return GapView; - case "announcement": + case TileShape.Announcement: return AnnouncementView; - case "message": - case "message-status": + case TileShape.Message: + case TileShape.MessageStatus: return TextMessageView; - case "image": + case TileShape.Image: return ImageView; - case "video": + case TileShape.Video: return VideoView; - case "file": + case TileShape.File: return FileView; - case "location": + case TileShape.Location: return LocationView; - case "missing-attachment": + case TileShape.MissingAttachment: return MissingAttachmentView; - case "redacted": + case TileShape.Redacted: return RedactedView; + case TileShape.DateHeader: + return DateHeaderView; default: throw new Error(`Tiles of shape "${vm.shape}" are not supported, check the tileClassForEntry function in the view model`); } diff --git a/src/platform/web/ui/session/room/timeline/DateHeaderView.ts b/src/platform/web/ui/session/room/timeline/DateHeaderView.ts new file mode 100644 index 00000000..63003e91 --- /dev/null +++ b/src/platform/web/ui/session/room/timeline/DateHeaderView.ts @@ -0,0 +1,33 @@ +/* +Copyright 2020 Bruno Windels + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {TemplateView} from "../../../general/TemplateView"; +import {spinner} from "../../../common.js"; +import type {DateTile} from "../../../../../../domain/session/room/timeline/tiles/DateTile"; + +export class DateHeaderView extends TemplateView { + // ignore other argument + constructor(vm) { + super(vm); + } + + render(t, vm) { + return t.div({className: "DateHeader"}, t.div(vm.date)); + } + + /* This is called by the parent ListView, which just has 1 listener for the whole list */ + onClick() {} +} From e9053372d64fd380970f7282610878ad6af0df03 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Fri, 25 Nov 2022 11:13:47 +0530 Subject: [PATCH 219/435] Encode ids as URI component --- src/domain/navigation/URLRouter.ts | 2 +- src/domain/navigation/index.ts | 30 ++++++++++++++----- .../session/rightpanel/MemberTileViewModel.js | 2 +- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/domain/navigation/URLRouter.ts b/src/domain/navigation/URLRouter.ts index bf1c218d..23503530 100644 --- a/src/domain/navigation/URLRouter.ts +++ b/src/domain/navigation/URLRouter.ts @@ -144,7 +144,7 @@ export class URLRouter implements IURLRou openRoomActionUrl(roomId: string): string { // not a segment to navigation knowns about, so append it manually - const urlPath = `${this._stringifyPath(this._navigation.path.until("session"))}/open-room/${roomId}`; + const urlPath = `${this._stringifyPath(this._navigation.path.until("session"))}/open-room/${encodeURIComponent(roomId)}`; return this._history.pathAsUrl(urlPath); } diff --git a/src/domain/navigation/index.ts b/src/domain/navigation/index.ts index af3c35bd..f580206e 100644 --- a/src/domain/navigation/index.ts +++ b/src/domain/navigation/index.ts @@ -147,7 +147,7 @@ export function parseUrlPath(urlPath: string, currentNavPath: Path, segments.push(new Segment("empty-grid-tile", selectedIndex)); } } else if (type === "open-room") { - const roomId = iterator.next().value; + const roomId = decodeURIComponent(iterator.next().value); if (!roomId) { break; } const rooms = currentNavPath.get("rooms"); if (rooms) { @@ -176,7 +176,7 @@ export function parseUrlPath(urlPath: string, currentNavPath: Path, } else if (type === "details" || type === "members") { pushRightPanelSegment(segments, type); } else if (type === "member") { - const userId = iterator.next().value; + const userId = decodeURIComponent(iterator.next().value); if (!userId) { break; } pushRightPanelSegment(segments, type, userId); } else if (type.includes("loginToken")) { @@ -185,7 +185,7 @@ export function parseUrlPath(urlPath: string, currentNavPath: Path, segments.push(new Segment("sso", loginToken)); } else { // might be undefined, which will be turned into true by Segment - const value = iterator.next().value; + const value = decodeURIComponent(iterator.next().value); segments.push(new Segment(type, value)); } } @@ -196,19 +196,20 @@ export function stringifyPath(path: Path): string { let urlPath = ""; let prevSegment: Segment | undefined; for (const segment of path.segments) { + const encodedSegmentValue = encodeSegmentValue(segment.value); switch (segment.type) { case "rooms": - urlPath += `/rooms/${segment.value.join(",")}`; + urlPath += `/rooms/${(encodedSegmentValue as string[]).join(",")}`; break; case "empty-grid-tile": - urlPath += `/${segment.value}`; + urlPath += `/${encodedSegmentValue}`; break; case "room": if (prevSegment?.type === "rooms") { const index = prevSegment.value.indexOf(segment.value); urlPath += `/${index}`; } else { - urlPath += `/${segment.type}/${segment.value}`; + urlPath += `/${segment.type}/${encodedSegmentValue}`; } break; case "right-panel": @@ -217,8 +218,8 @@ export function stringifyPath(path: Path): string { continue; default: urlPath += `/${segment.type}`; - if (segment.value && segment.value !== true) { - urlPath += `/${segment.value}`; + if (encodedSegmentValue && encodedSegmentValue !== true) { + urlPath += `/${encodedSegmentValue}`; } } prevSegment = segment; @@ -226,6 +227,19 @@ export function stringifyPath(path: Path): string { return urlPath; } +function encodeSegmentValue(value: SegmentType[keyof SegmentType]) { + if (typeof value === "boolean") { + // Nothing to encode for boolean + return value; + } + else if (Array.isArray(value)) { + return value.map(v => encodeURIComponent(v)); + } + else { + return encodeURIComponent(value); + } +} + export function tests() { function createEmptyPath() { const nav: Navigation = new Navigation(allowsChild); diff --git a/src/domain/session/rightpanel/MemberTileViewModel.js b/src/domain/session/rightpanel/MemberTileViewModel.js index c8dcf63a..9f9a5483 100644 --- a/src/domain/session/rightpanel/MemberTileViewModel.js +++ b/src/domain/session/rightpanel/MemberTileViewModel.js @@ -48,7 +48,7 @@ export class MemberTileViewModel extends ViewModel { get detailsUrl() { const roomId = this.navigation.path.get("room").value; - return `${this.urlRouter.openRoomActionUrl(roomId)}/member/${this._member.userId}`; + return `${this.urlRouter.openRoomActionUrl(roomId)}/member/${encodeURIComponent(this._member.userId)}`; } _updatePreviousName(newName) { From 80080074fa668c4e52c23517e8807a2e2b9880d7 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Fri, 25 Nov 2022 11:20:47 +0530 Subject: [PATCH 220/435] Don't encode unknown segment values --- src/domain/navigation/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/navigation/index.ts b/src/domain/navigation/index.ts index f580206e..60da2435 100644 --- a/src/domain/navigation/index.ts +++ b/src/domain/navigation/index.ts @@ -185,7 +185,7 @@ export function parseUrlPath(urlPath: string, currentNavPath: Path, segments.push(new Segment("sso", loginToken)); } else { // might be undefined, which will be turned into true by Segment - const value = decodeURIComponent(iterator.next().value); + const value = iterator.next().value; segments.push(new Segment(type, value)); } } From 7bfadf37b42e953b3e12051c970fcc7894d89233 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Fri, 25 Nov 2022 11:33:02 +0530 Subject: [PATCH 221/435] Decode all segment values that aren't undefined --- src/domain/navigation/index.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/domain/navigation/index.ts b/src/domain/navigation/index.ts index 60da2435..935f7138 100644 --- a/src/domain/navigation/index.ts +++ b/src/domain/navigation/index.ts @@ -185,7 +185,11 @@ export function parseUrlPath(urlPath: string, currentNavPath: Path, segments.push(new Segment("sso", loginToken)); } else { // might be undefined, which will be turned into true by Segment - const value = iterator.next().value; + let value = iterator.next().value; + if (value) { + // decode only if value isn't undefined! + value = decodeURIComponent(value) + } segments.push(new Segment(type, value)); } } From d889c7deeb1d2a87b618a2823b0e3fd71d373007 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 25 Nov 2022 10:49:26 +0100 Subject: [PATCH 222/435] bound checks in date header loop and cleanup Co-authored-by: R Midhun Suresh --- src/domain/session/room/timeline/TilesCollection.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/domain/session/room/timeline/TilesCollection.js b/src/domain/session/room/timeline/TilesCollection.js index 8dfbf2a1..29e0bdfc 100644 --- a/src/domain/session/room/timeline/TilesCollection.js +++ b/src/domain/session/room/timeline/TilesCollection.js @@ -154,8 +154,14 @@ export class TilesCollection extends BaseObservableList { //console.log("_evaluateDateHeaderAtIdx", tileIdx); // consider the two adjacent tiles where the previous sibling changed: // the new tile and the next tile - for (let i = 0; i < 5; i += 1) { - const idx = Math.max(tileIdx + i - 2, 0); + for (let i = -2; i < 3; i += 1) { + const idx = tileIdx + i; + if (idx < 0) { + continue; + } + if (idx >= this._tiles.length) { + break; + } const tile = this._tiles[idx]; const prevTile = idx > 0 ? this._tiles[idx - 1] : undefined; const hasDateSeparator = prevTile?.shape === TileShape.DateHeader; From 4abf18a5f11f4e70b8596fb81a0ed9c56b84472a Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 25 Nov 2022 11:26:53 +0100 Subject: [PATCH 223/435] don't look at tiles before the inserted tile, there is no need also clarify with comments how the current algorithm works --- src/domain/session/room/timeline/TilesCollection.js | 12 ++++++++---- src/domain/session/room/timeline/tiles/DateTile.ts | 3 +++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/domain/session/room/timeline/TilesCollection.js b/src/domain/session/room/timeline/TilesCollection.js index 29e0bdfc..e4ed9d24 100644 --- a/src/domain/session/room/timeline/TilesCollection.js +++ b/src/domain/session/room/timeline/TilesCollection.js @@ -152,9 +152,11 @@ export class TilesCollection extends BaseObservableList { _evaluateDateHeaderAtIdx(tileIdx) { //console.log("_evaluateDateHeaderAtIdx", tileIdx); - // consider the two adjacent tiles where the previous sibling changed: - // the new tile and the next tile - for (let i = -2; i < 3; i += 1) { + // consider two tiles after the inserted tile, because + // the first of the two tiles may be a DateTile in which case, + // we remove it after looking at the needsDateSeparator prop of the + // next next tile + for (let i = 0; i < 2; i += 1) { const idx = tileIdx + i; if (idx < 0) { continue; @@ -173,10 +175,12 @@ export class TilesCollection extends BaseObservableList { this.emitUpdate(idx - 1, prevTile, "date"); } else { //console.log(" add", idx, tile.shape, tile.eventId); + // adding a tile shift all the indices we need to consider + // especially given we consider removals for the tile that + // comes after a datetile tileIdx += 1; this._addTileAt(idx, tile.createDateSeparator()); } - // TODO must be looking at the wrong index to find the old date separator?? } else if (hasDateSeparator) { // this is never triggered because needsDateSeparator is not cleared // when loading more items because we don't do anything once the diff --git a/src/domain/session/room/timeline/tiles/DateTile.ts b/src/domain/session/room/timeline/tiles/DateTile.ts index 413ec2b2..92bd42e5 100644 --- a/src/domain/session/room/timeline/tiles/DateTile.ts +++ b/src/domain/session/room/timeline/tiles/DateTile.ts @@ -119,6 +119,9 @@ export class DateTile extends ViewModel implements ITile { // let item know it has a new sibling updatePreviousSibling(prev: ITile | undefined): void { + // forward the sibling update to our next tile, so it is informed + // about it's previous sibling beyond the date header (which is it's direct previous sibling) + // so it can recalculate whether it still needs a date header this._firstTileInDay.updatePreviousSibling(prev); } From 31f53d27c164347a48836f068625294e02a97ed9 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 25 Nov 2022 11:51:22 +0100 Subject: [PATCH 224/435] emit update from datetile when date might have changed --- .../session/room/timeline/TilesCollection.js | 14 ++------------ src/domain/session/room/timeline/tiles/DateTile.ts | 5 ++++- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/domain/session/room/timeline/TilesCollection.js b/src/domain/session/room/timeline/TilesCollection.js index e4ed9d24..6652e89c 100644 --- a/src/domain/session/room/timeline/TilesCollection.js +++ b/src/domain/session/room/timeline/TilesCollection.js @@ -151,7 +151,6 @@ export class TilesCollection extends BaseObservableList { } _evaluateDateHeaderAtIdx(tileIdx) { - //console.log("_evaluateDateHeaderAtIdx", tileIdx); // consider two tiles after the inserted tile, because // the first of the two tiles may be a DateTile in which case, // we remove it after looking at the needsDateSeparator prop of the @@ -167,25 +166,16 @@ export class TilesCollection extends BaseObservableList { const tile = this._tiles[idx]; const prevTile = idx > 0 ? this._tiles[idx - 1] : undefined; const hasDateSeparator = prevTile?.shape === TileShape.DateHeader; - if (tile.needsDateSeparator) { - if (hasDateSeparator) { - // TODO: replace this by return UpdateAction from updateNextSibling - // and do this in onAdd - //console.log(" update", idx - 1, prevTile?.shape, prevTile?.eventId); - this.emitUpdate(idx - 1, prevTile, "date"); - } else { - //console.log(" add", idx, tile.shape, tile.eventId); + if (tile.needsDateSeparator && !hasDateSeparator) { // adding a tile shift all the indices we need to consider // especially given we consider removals for the tile that // comes after a datetile tileIdx += 1; this._addTileAt(idx, tile.createDateSeparator()); - } - } else if (hasDateSeparator) { + } else if (!tile.needsDateSeparator && hasDateSeparator) { // this is never triggered because needsDateSeparator is not cleared // when loading more items because we don't do anything once the // direct sibling is a DateTile - //console.log(" remove", idx -1, prevTile?.shape, prevTile?.eventId); this._removeTile(idx - 1, prevTile); } } diff --git a/src/domain/session/room/timeline/tiles/DateTile.ts b/src/domain/session/room/timeline/tiles/DateTile.ts index 92bd42e5..4e7cbda2 100644 --- a/src/domain/session/room/timeline/tiles/DateTile.ts +++ b/src/domain/session/room/timeline/tiles/DateTile.ts @@ -130,8 +130,11 @@ export class DateTile extends ViewModel implements ITile { // TODO: next can be undefined when a pending event is removed // TODO: we need a way to remove this date header this._firstTileInDay = next!; + const prevDateString = this._dateString; this._dateString = undefined; - // TODO: do we need to reevaluate our date here and emit an update? + if (prevDateString && prevDateString !== this.date) { + this._emitUpdate?.(this, "date"); + } } notifyVisible(): void { From cb0ab589de1d411f24006cce65fd0e3ecdada1c2 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 25 Nov 2022 11:51:38 +0100 Subject: [PATCH 225/435] remove lower bound check as we don't go negative anymore --- src/domain/session/room/timeline/TilesCollection.js | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/domain/session/room/timeline/TilesCollection.js b/src/domain/session/room/timeline/TilesCollection.js index 6652e89c..c9a9d6df 100644 --- a/src/domain/session/room/timeline/TilesCollection.js +++ b/src/domain/session/room/timeline/TilesCollection.js @@ -157,9 +157,6 @@ export class TilesCollection extends BaseObservableList { // next next tile for (let i = 0; i < 2; i += 1) { const idx = tileIdx + i; - if (idx < 0) { - continue; - } if (idx >= this._tiles.length) { break; } @@ -167,11 +164,11 @@ export class TilesCollection extends BaseObservableList { const prevTile = idx > 0 ? this._tiles[idx - 1] : undefined; const hasDateSeparator = prevTile?.shape === TileShape.DateHeader; if (tile.needsDateSeparator && !hasDateSeparator) { - // adding a tile shift all the indices we need to consider - // especially given we consider removals for the tile that - // comes after a datetile - tileIdx += 1; - this._addTileAt(idx, tile.createDateSeparator()); + // adding a tile shift all the indices we need to consider + // especially given we consider removals for the tile that + // comes after a datetile + tileIdx += 1; + this._addTileAt(idx, tile.createDateSeparator()); } else if (!tile.needsDateSeparator && hasDateSeparator) { // this is never triggered because needsDateSeparator is not cleared // when loading more items because we don't do anything once the From 7c6d651b32a79159dae0c313d18606329f4f2118 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 25 Nov 2022 12:09:28 +0100 Subject: [PATCH 226/435] remove date headers when removing pending tiles --- src/domain/session/room/timeline/TilesCollection.js | 4 ++++ src/domain/session/room/timeline/tiles/DateTile.ts | 11 ++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/domain/session/room/timeline/TilesCollection.js b/src/domain/session/room/timeline/TilesCollection.js index c9a9d6df..95eac7ac 100644 --- a/src/domain/session/room/timeline/TilesCollection.js +++ b/src/domain/session/room/timeline/TilesCollection.js @@ -255,6 +255,10 @@ export class TilesCollection extends BaseObservableList { this.emitRemove(tileIdx, tile); prevTile?.updateNextSibling(nextTile); nextTile?.updatePreviousSibling(prevTile); + + if (prevTile && prevTile.shape === TileShape.DateHeader && (!nextTile || !nextTile.needsDateSeparator)) { + this._removeTile(tileIdx - 1, prevTile); + } } // would also be called when unloading a part of the timeline diff --git a/src/domain/session/room/timeline/tiles/DateTile.ts b/src/domain/session/room/timeline/tiles/DateTile.ts index 4e7cbda2..7e860511 100644 --- a/src/domain/session/room/timeline/tiles/DateTile.ts +++ b/src/domain/session/room/timeline/tiles/DateTile.ts @@ -127,9 +127,14 @@ export class DateTile extends ViewModel implements ITile { // let item know it has a new sibling updateNextSibling(next: ITile | undefined): UpdateAction { - // TODO: next can be undefined when a pending event is removed - // TODO: we need a way to remove this date header - this._firstTileInDay = next!; + if(!next) { + // If we are the DateTile for the last tile in the timeline, + // and that tile gets removed, next would be undefined + // and this DateTile would be removed as well, + // so do nothing + return; + } + this._firstTileInDay = next; const prevDateString = this._dateString; this._dateString = undefined; if (prevDateString && prevDateString !== this.date) { From b5d5adaa3638367e0dfb506752f5e9324d778fa6 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 25 Nov 2022 14:42:24 +0100 Subject: [PATCH 227/435] add tests Co-authored-by: R Midhun Suresh --- .../session/room/timeline/TilesCollection.js | 99 ++++++++++++++++++- .../session/room/timeline/tiles/DateTile.ts | 26 +++++ .../session/room/timeline/tiles/SimpleTile.js | 62 ++++++++++++ 3 files changed, 185 insertions(+), 2 deletions(-) diff --git a/src/domain/session/room/timeline/TilesCollection.js b/src/domain/session/room/timeline/TilesCollection.js index 95eac7ac..4dc6e4aa 100644 --- a/src/domain/session/room/timeline/TilesCollection.js +++ b/src/domain/session/room/timeline/TilesCollection.js @@ -59,7 +59,6 @@ export class TilesCollection extends BaseObservableList { if (!currentTile || !currentTile.tryIncludeEntry(entry)) { currentTile = this._createTile(entry); if (currentTile) { - console.log("adding initial tile", currentTile.shape, currentTile.eventId, "at", this._tiles.length); this._tiles.push(currentTile); } } @@ -142,7 +141,6 @@ export class TilesCollection extends BaseObservableList { const newTile = this._createTile(entry); if (newTile) { - console.log("adding tile", newTile.shape, newTile.eventId, "at", tileIdx); this._addTileAt(tileIdx, newTile); this._evaluateDateHeaderAtIdx(tileIdx); } @@ -264,6 +262,7 @@ export class TilesCollection extends BaseObservableList { // would also be called when unloading a part of the timeline onRemove(index, entry) { const tileIdx = this._findTileIdx(entry); + const tile = this._findTileAtIdx(entry, tileIdx); if (tile) { const removeTile = tile.removeEntry(entry); @@ -317,6 +316,7 @@ export function tests() { constructor(entry) { this.entry = entry; this.update = null; + this.needsDateSeparator = false; } setUpdateEmit(update) { this.update = update; @@ -346,6 +346,34 @@ export function tests() { dispose() {} } + class DateHeaderTile extends TestTile { + get shape() { return TileShape.DateHeader; } + updateNextSibling(next) { + this.next = next; + } + updatePreviousSibling(prev) { + this.next?.updatePreviousSibling(prev); + } + compareEntry(b) { + // important that date tiles as sorted before their next item, but after their previous sibling + return this.next.compareEntry(b) - 0.5; + } + } + + class MessageNeedingDateHeaderTile extends TestTile { + get shape() { return TileShape.Message; } + + createDateSeparator() { + return new DateHeaderTile(this.entry); + } + updatePreviousSibling(prev) { + if (prev?.shape !== TileShape.DateHeader) { + // 1 day is 10 + this.needsDateSeparator = !prev || Math.floor(prev.entry.n / 10) !== Math.floor(this.entry.n / 10); + } + } + } + return { "don't emit update before add": assert => { class UpdateOnSiblingTile extends TestTile { @@ -404,6 +432,73 @@ export function tests() { }); entries.remove(1); assert.deepEqual(events, ["remove", "update"]); + }, + "date tile is added when needed when populating": assert => { + const entries = new ObservableArray([{n: 15}]); + const tileOptions = { + tileClassForEntry: () => MessageNeedingDateHeaderTile, + }; + const tiles = new TilesCollection(entries, tileOptions); + tiles.subscribe({}); + const tilesArray = Array.from(tiles); + assert.equal(tilesArray.length, 2); + assert.equal(tilesArray[0].shape, TileShape.DateHeader); + assert.equal(tilesArray[1].shape, TileShape.Message); + }, + "date header is added when receiving addition": assert => { + const entries = new ObservableArray([{n: 15}]); + const tileOptions = { + tileClassForEntry: () => MessageNeedingDateHeaderTile, + }; + const tiles = new TilesCollection(entries, tileOptions); + tiles.subscribe({ + onAdd() {}, + onRemove() {} + }); + entries.insert(0, {n: 5}); + const tilesArray = Array.from(tiles); + assert.equal(tilesArray[0].shape, TileShape.DateHeader); + assert.equal(tilesArray[1].shape, TileShape.Message); + assert.equal(tilesArray[2].shape, TileShape.DateHeader); + assert.equal(tilesArray[3].shape, TileShape.Message); + assert.equal(tilesArray.length, 4); + }, + "date header is removed and added when loading more messages for the same day": assert => { + const entries = new ObservableArray([{n: 15}]); + const tileOptions = { + tileClassForEntry: () => MessageNeedingDateHeaderTile, + }; + const tiles = new TilesCollection(entries, tileOptions); + tiles.subscribe({ + onAdd() {}, + onRemove() {} + }); + entries.insert(0, {n: 12}); + const tilesArray = Array.from(tiles); + assert.equal(tilesArray[0].shape, TileShape.DateHeader); + assert.equal(tilesArray[1].shape, TileShape.Message); + assert.equal(tilesArray[2].shape, TileShape.Message); + assert.equal(tilesArray.length, 3); + }, + "date header is removed at the end of the timeline": assert => { + const entries = new ObservableArray([{n: 5}, {n: 15}]); + const tileOptions = { + tileClassForEntry: () => MessageNeedingDateHeaderTile, + }; + const tiles = new TilesCollection(entries, tileOptions); + let removals = 0; + tiles.subscribe({ + onAdd() {}, + onRemove() { + removals += 1; + } + }); + entries.remove(1); + const tilesArray = Array.from(tiles); + assert.equal(tilesArray[0].shape, TileShape.DateHeader); + assert.equal(tilesArray[1].shape, TileShape.Message); + assert.equal(tilesArray.length, 2); + assert.equal(removals, 2); } } } diff --git a/src/domain/session/room/timeline/tiles/DateTile.ts b/src/domain/session/room/timeline/tiles/DateTile.ts index 7e860511..36150a68 100644 --- a/src/domain/session/room/timeline/tiles/DateTile.ts +++ b/src/domain/session/room/timeline/tiles/DateTile.ts @@ -150,3 +150,29 @@ export class DateTile extends ViewModel implements ITile { } } + +import { EventEntry } from "../../../../../matrix/room/timeline/entries/EventEntry.js"; +import { SimpleTile } from "./SimpleTile"; + +export function tests() { + return { + "date tile sorts before reference tile": assert => { + const a = new SimpleTile(new EventEntry({ + event: {}, + eventIndex: 2, + fragmentId: 1 + }, undefined), {}); + const b = new SimpleTile(new EventEntry({ + event: {}, + eventIndex: 3, + fragmentId: 1 + }, undefined), {}); + const d = new DateTile(b, {} as any); + const tiles = [d, b, a]; + tiles.sort((a, b) => a.compare(b)); + assert.equal(tiles[0], a); + assert.equal(tiles[1], d); + assert.equal(tiles[2], b); + } + } +} \ No newline at end of file diff --git a/src/domain/session/room/timeline/tiles/SimpleTile.js b/src/domain/session/room/timeline/tiles/SimpleTile.js index 7e394d25..f466e4d4 100644 --- a/src/domain/session/room/timeline/tiles/SimpleTile.js +++ b/src/domain/session/room/timeline/tiles/SimpleTile.js @@ -184,3 +184,65 @@ export class SimpleTile extends ViewModel { return this._options.timeline.me; } } + +import { EventEntry } from "../../../../../matrix/room/timeline/entries/EventEntry.js"; + +export function tests() { + return { + "needsDateSeparator is false when previous sibling is for same date": assert => { + const fridayEntry = new EventEntry({ + event: { + origin_server_ts: 1669376446222, + type: "m.room.message", + content: {} + } + }, undefined); + const thursdayEntry = new EventEntry({ + event: { + origin_server_ts: fridayEntry.timestamp - (60 * 60 * 8 * 1000), + type: "m.room.message", + content: {} + } + }, undefined); + const fridayTile = new SimpleTile(fridayEntry, {}); + const thursdayTile = new SimpleTile(thursdayEntry, {}); + assert.equal(fridayTile.needsDateSeparator, false); + fridayTile.updatePreviousSibling(thursdayTile); + assert.equal(fridayTile.needsDateSeparator, false); + }, + "needsDateSeparator is true when previous sibling is for different date": assert => { + const fridayEntry = new EventEntry({ + event: { + origin_server_ts: 1669376446222, + type: "m.room.message", + content: {} + } + }, undefined); + const thursdayEntry = new EventEntry({ + event: { + origin_server_ts: fridayEntry.timestamp - (60 * 60 * 24 * 1000), + type: "m.room.message", + content: {} + } + }, undefined); + const fridayTile = new SimpleTile(fridayEntry, {}); + const thursdayTile = new SimpleTile(thursdayEntry, {}); + assert.equal(fridayTile.needsDateSeparator, false); + fridayTile.updatePreviousSibling(thursdayTile); + assert.equal(fridayTile.needsDateSeparator, true); + }, + "needsDateSeparator is true when previous sibling is undefined": assert => { + const fridayEntry = new EventEntry({ + event: { + origin_server_ts: 1669376446222, + type: "m.room.message", + content: {} + } + }, undefined); + const fridayTile = new SimpleTile(fridayEntry, {}); + assert.equal(fridayTile.needsDateSeparator, false); + fridayTile.updatePreviousSibling(undefined); + assert.equal(fridayTile.needsDateSeparator, true); + }, + } +} \ No newline at end of file From 9b235f7c80b105ab84eb8e1321dd0ae4725c55a6 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 25 Nov 2022 14:42:54 +0100 Subject: [PATCH 228/435] also test next next tile, change this back after testing before --- src/domain/session/room/timeline/TilesCollection.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/session/room/timeline/TilesCollection.js b/src/domain/session/room/timeline/TilesCollection.js index 4dc6e4aa..458697ca 100644 --- a/src/domain/session/room/timeline/TilesCollection.js +++ b/src/domain/session/room/timeline/TilesCollection.js @@ -153,7 +153,7 @@ export class TilesCollection extends BaseObservableList { // the first of the two tiles may be a DateTile in which case, // we remove it after looking at the needsDateSeparator prop of the // next next tile - for (let i = 0; i < 2; i += 1) { + for (let i = 0; i < 3; i += 1) { const idx = tileIdx + i; if (idx >= this._tiles.length) { break; From 9c2c5af291b14c755e216ad978bcbe424a8cc1ec Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 25 Nov 2022 16:12:35 +0100 Subject: [PATCH 229/435] small UI refinements for date header --- src/platform/web/ui/css/themes/element/timeline.css | 7 +++++++ .../web/ui/session/room/timeline/DateHeaderView.ts | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/platform/web/ui/css/themes/element/timeline.css b/src/platform/web/ui/css/themes/element/timeline.css index 43c57d19..87b3e8c7 100644 --- a/src/platform/web/ui/css/themes/element/timeline.css +++ b/src/platform/web/ui/css/themes/element/timeline.css @@ -422,3 +422,10 @@ only loads when the top comes into view*/ .GapView.isAtTop { padding: 52px 20px 12px 20px; } + +.DateHeader { + color: var(--light-text-color); + font-weight: bold; + padding: 12px 4px; + text-align: center; +} \ No newline at end of file diff --git a/src/platform/web/ui/session/room/timeline/DateHeaderView.ts b/src/platform/web/ui/session/room/timeline/DateHeaderView.ts index 63003e91..f9159c5e 100644 --- a/src/platform/web/ui/session/room/timeline/DateHeaderView.ts +++ b/src/platform/web/ui/session/room/timeline/DateHeaderView.ts @@ -25,7 +25,7 @@ export class DateHeaderView extends TemplateView { } render(t, vm) { - return t.div({className: "DateHeader"}, t.div(vm.date)); + return t.div({className: "DateHeader"}, t.time(vm.date)); } /* This is called by the parent ListView, which just has 1 listener for the whole list */ From c4e239a4011805a68043194989195d0e358e7402 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 25 Nov 2022 16:13:22 +0100 Subject: [PATCH 230/435] add timeformatter, shared between all view models --- src/domain/ViewModel.ts | 5 ++ .../session/room/timeline/tiles/DateTile.ts | 8 +- src/platform/types/types.ts | 6 +- src/platform/web/Platform.js | 2 + src/platform/web/dom/TimeFormatter.ts | 73 +++++++++++++++++++ 5 files changed, 86 insertions(+), 8 deletions(-) create mode 100644 src/platform/web/dom/TimeFormatter.ts diff --git a/src/domain/ViewModel.ts b/src/domain/ViewModel.ts index 8dbc37ea..878b43ba 100644 --- a/src/domain/ViewModel.ts +++ b/src/domain/ViewModel.ts @@ -29,6 +29,7 @@ import type {ILogger} from "../logging/types"; import type {Navigation} from "./navigation/Navigation"; import type {SegmentType} from "./navigation/index"; import type {IURLRouter} from "./navigation/URLRouter"; +import type { ITimeFormatter } from "../platform/types/types"; export type Options = { platform: Platform; @@ -145,4 +146,8 @@ export class ViewModel = Op // typescript needs a little help here return this._options.navigation as unknown as Navigation; } + + get timeFormatter(): ITimeFormatter { + return this._options.platform.timeFormatter; + } } diff --git a/src/domain/session/room/timeline/tiles/DateTile.ts b/src/domain/session/room/timeline/tiles/DateTile.ts index 36150a68..d8dac808 100644 --- a/src/domain/session/room/timeline/tiles/DateTile.ts +++ b/src/domain/session/room/timeline/tiles/DateTile.ts @@ -48,13 +48,7 @@ export class DateTile extends ViewModel implements ITile { get date(): string { if (!this._dateString) { - const date = new Date(this.refEntry.timestamp); - this._dateString = date.toLocaleDateString({}, { - weekday: 'long', - year: 'numeric', - month: 'long', - day: 'numeric' - }); + this._dateString = this.timeFormatter.formatRelativeDate(new Date(this.refEntry.timestamp)); } return this._dateString; } diff --git a/src/platform/types/types.ts b/src/platform/types/types.ts index 1d359a09..5e982545 100644 --- a/src/platform/types/types.ts +++ b/src/platform/types/types.ts @@ -16,7 +16,7 @@ limitations under the License. import type {RequestResult} from "../web/dom/request/fetch.js"; import type {RequestBody} from "../../matrix/net/common"; -import type {ILogItem} from "../../logging/types"; +import type { BaseObservableValue } from "../../observable/ObservableValue"; export interface IRequestOptions { uploadProgress?: (loadedBytes: number) => void; @@ -43,3 +43,7 @@ export type File = { readonly name: string; readonly blob: IBlobHandle; } + +export interface ITimeFormatter { + formatRelativeDate(date: Date): string; +} \ No newline at end of file diff --git a/src/platform/web/Platform.js b/src/platform/web/Platform.js index 29a83e1f..0d95e585 100644 --- a/src/platform/web/Platform.js +++ b/src/platform/web/Platform.js @@ -39,6 +39,7 @@ import {Disposables} from "../../utils/Disposables"; import {parseHTML} from "./parsehtml.js"; import {handleAvatarError} from "./ui/avatar"; import {ThemeLoader} from "./theming/ThemeLoader"; +import {TimeFormatter} from "./dom/TimeFormatter"; function addScript(src) { return new Promise(function (resolve, reject) { @@ -139,6 +140,7 @@ export class Platform { this._createLogger(options?.development); this.history = new History(); this.onlineStatus = new OnlineStatus(); + this.timeFormatter = new TimeFormatter(); this._serviceWorkerHandler = null; if (assetPaths.serviceWorker && "serviceWorker" in navigator) { this._serviceWorkerHandler = new ServiceWorkerHandler(); diff --git a/src/platform/web/dom/TimeFormatter.ts b/src/platform/web/dom/TimeFormatter.ts new file mode 100644 index 00000000..ab9d454e --- /dev/null +++ b/src/platform/web/dom/TimeFormatter.ts @@ -0,0 +1,73 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import type { ITimeFormatter } from "../../types/types"; +import {Clock} from "./Clock"; + +enum TimeScope { + Minute = 60 * 1000, + Day = 24 * 60 * 60 * 1000, +} + +export class TimeFormatter implements ITimeFormatter { + + private todayMidnight: Date; + private relativeDayFormatter: Intl.RelativeTimeFormat; + private weekdayFormatter: Intl.DateTimeFormat; + private currentYearFormatter: Intl.DateTimeFormat; + private otherYearFormatter: Intl.DateTimeFormat; + + constructor(private clock: Clock) { + // don't use the clock time here as the DOM relative formatters don't support setting the reference date anyway + this.todayMidnight = new Date(); + this.todayMidnight.setHours(0, 0, 0, 0); + this.relativeDayFormatter = new Intl.RelativeTimeFormat(undefined, {numeric: "auto"}); + this.weekdayFormatter = new Intl.DateTimeFormat(undefined, {weekday: 'long'}); + this.currentYearFormatter = new Intl.DateTimeFormat(undefined, { + weekday: 'long', + month: 'long', + day: 'numeric' + }); + this.otherYearFormatter = new Intl.DateTimeFormat(undefined, { + weekday: 'long', + year: 'numeric', + month: 'long', + day: 'numeric' + }); + } + + formatRelativeDate(date: Date): string { + let daysDiff = Math.floor((date.getTime() - this.todayMidnight.getTime()) / TimeScope.Day); + console.log("formatRelativeDate daysDiff", daysDiff, date); + if (daysDiff >= -1 && daysDiff <= 1) { + // Tomorrow, Today, Yesterday + return capitalizeFirstLetter(this.relativeDayFormatter.format(daysDiff, "day")); + } else if (daysDiff > -7 && daysDiff < 0) { + // Wednesday + return this.weekdayFormatter.format(date); + } else if (this.todayMidnight.getFullYear() === date.getFullYear()) { + // Friday, November 6 + return this.currentYearFormatter.format(date); + } else { + // Friday, November 5, 2021 + return this.otherYearFormatter.format(date); + } + } +} + +function capitalizeFirstLetter(str: string) { + return str.slice(0, 1).toLocaleUpperCase() + str.slice(1); +} \ No newline at end of file From 2136b051a0ee8bf984df3eab38f4703253e384b8 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 25 Nov 2022 16:13:31 +0100 Subject: [PATCH 231/435] cleanup --- src/domain/session/room/timeline/tiles/SimpleTile.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/domain/session/room/timeline/tiles/SimpleTile.js b/src/domain/session/room/timeline/tiles/SimpleTile.js index f466e4d4..91e023cb 100644 --- a/src/domain/session/room/timeline/tiles/SimpleTile.js +++ b/src/domain/session/room/timeline/tiles/SimpleTile.js @@ -55,9 +55,6 @@ export class SimpleTile extends ViewModel { this._needsDateSeparator = prev._date.getFullYear() !== this._date.getFullYear() || prev._date.getMonth() !== this._date.getMonth() || prev._date.getDate() !== this._date.getDate(); - if (neededDateSeparator && !this._needsDateSeparator) { - console.log("clearing needsDateSeparator", {this: this._entry.content, prev: prev.content}); - } } else { this._needsDateSeparator = !!this._date; } From c538f5dbb14e3a647be8da93eda28e6a2af054e3 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 25 Nov 2022 16:31:44 +0100 Subject: [PATCH 232/435] make date header a bit more accessible --- .../session/room/timeline/tiles/DateTile.ts | 15 ++++++++++++--- src/platform/types/types.ts | 1 + src/platform/web/dom/TimeFormatter.ts | 3 +++ .../web/ui/css/themes/element/timeline.css | 1 + .../ui/session/room/timeline/DateHeaderView.ts | 2 +- 5 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/domain/session/room/timeline/tiles/DateTile.ts b/src/domain/session/room/timeline/tiles/DateTile.ts index d8dac808..590333b8 100644 --- a/src/domain/session/room/timeline/tiles/DateTile.ts +++ b/src/domain/session/room/timeline/tiles/DateTile.ts @@ -17,6 +17,7 @@ import type {Options} from "../../../../ViewModel"; export class DateTile extends ViewModel implements ITile { private _emitUpdate?: EmitUpdateFn; private _dateString?: string; + private _machineReadableString?: string; constructor(private _firstTileInDay: ITile, options: Options) { super(options); @@ -46,13 +47,20 @@ export class DateTile extends ViewModel implements ITile { return this.compareEntry(tile.upperEntry); } - get date(): string { + get relativeDate(): string { if (!this._dateString) { this._dateString = this.timeFormatter.formatRelativeDate(new Date(this.refEntry.timestamp)); } return this._dateString; } + get machineReadableDate(): string { + if (!this._machineReadableString) { + this._machineReadableString = this.timeFormatter.formatMachineReadableDate(new Date(this.refEntry.timestamp)); + } + return this._machineReadableString; + } + get shape(): TileShape { return TileShape.DateHeader; } @@ -131,8 +139,9 @@ export class DateTile extends ViewModel implements ITile { this._firstTileInDay = next; const prevDateString = this._dateString; this._dateString = undefined; - if (prevDateString && prevDateString !== this.date) { - this._emitUpdate?.(this, "date"); + this._machineReadableString = undefined; + if (prevDateString && prevDateString !== this.relativeDate) { + this._emitUpdate?.(this, "relativeDate"); } } diff --git a/src/platform/types/types.ts b/src/platform/types/types.ts index 5e982545..9147f6b8 100644 --- a/src/platform/types/types.ts +++ b/src/platform/types/types.ts @@ -46,4 +46,5 @@ export type File = { export interface ITimeFormatter { formatRelativeDate(date: Date): string; + formatMachineReadableDate(date: Date): string; } \ No newline at end of file diff --git a/src/platform/web/dom/TimeFormatter.ts b/src/platform/web/dom/TimeFormatter.ts index ab9d454e..3f354713 100644 --- a/src/platform/web/dom/TimeFormatter.ts +++ b/src/platform/web/dom/TimeFormatter.ts @@ -48,6 +48,9 @@ export class TimeFormatter implements ITimeFormatter { day: 'numeric' }); } + formatMachineReadableDate(date: Date): string { + return `${date.getFullYear()}-${date.getMonth() + 1}-${date.getDate()}`; + } formatRelativeDate(date: Date): string { let daysDiff = Math.floor((date.getTime() - this.todayMidnight.getTime()) / TimeScope.Day); diff --git a/src/platform/web/ui/css/themes/element/timeline.css b/src/platform/web/ui/css/themes/element/timeline.css index 87b3e8c7..16597a7b 100644 --- a/src/platform/web/ui/css/themes/element/timeline.css +++ b/src/platform/web/ui/css/themes/element/timeline.css @@ -428,4 +428,5 @@ only loads when the top comes into view*/ font-weight: bold; padding: 12px 4px; text-align: center; + font-size: 1.5rem; } \ No newline at end of file diff --git a/src/platform/web/ui/session/room/timeline/DateHeaderView.ts b/src/platform/web/ui/session/room/timeline/DateHeaderView.ts index f9159c5e..3d640568 100644 --- a/src/platform/web/ui/session/room/timeline/DateHeaderView.ts +++ b/src/platform/web/ui/session/room/timeline/DateHeaderView.ts @@ -25,7 +25,7 @@ export class DateHeaderView extends TemplateView { } render(t, vm) { - return t.div({className: "DateHeader"}, t.time(vm.date)); + return t.h2({className: "DateHeader"}, t.time({dateTime: vm.machineReadableDate}, vm.relativeDate)); } /* This is called by the parent ListView, which just has 1 listener for the whole list */ From 85a06876cd8476ff70be8e42db4014da8885a512 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 25 Nov 2022 16:47:00 +0100 Subject: [PATCH 233/435] make date header sticky with css this has the limitation that it needs to have a fixed with and an opaque background, but that's better than not making it sticky for now. --- .../web/ui/css/themes/element/timeline.css | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/platform/web/ui/css/themes/element/timeline.css b/src/platform/web/ui/css/themes/element/timeline.css index 16597a7b..8645bc3f 100644 --- a/src/platform/web/ui/css/themes/element/timeline.css +++ b/src/platform/web/ui/css/themes/element/timeline.css @@ -424,9 +424,20 @@ only loads when the top comes into view*/ } .DateHeader { - color: var(--light-text-color); font-weight: bold; - padding: 12px 4px; - text-align: center; font-size: 1.5rem; -} \ No newline at end of file + position: sticky; + top: 5px; +} + +.DateHeader time { + margin: 0 auto; + padding: 12px 4px; + width: 250px; + padding: 12px; + display: block; + color: var(--light-text-color); + background-color: var(--background-color-primary); + border-radius: 8px; + text-align: center; + } \ No newline at end of file From 77fd7e7aca680282a1d7ec64da189f0074f9431a Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 25 Nov 2022 16:47:54 +0100 Subject: [PATCH 234/435] format message time in timeFormatter as well --- src/domain/session/room/timeline/tiles/BaseMessageTile.js | 2 +- src/platform/types/types.ts | 1 + src/platform/web/dom/TimeFormatter.ts | 7 +++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/domain/session/room/timeline/tiles/BaseMessageTile.js b/src/domain/session/room/timeline/tiles/BaseMessageTile.js index 1ad1ba44..05ff1867 100644 --- a/src/domain/session/room/timeline/tiles/BaseMessageTile.js +++ b/src/domain/session/room/timeline/tiles/BaseMessageTile.js @@ -84,7 +84,7 @@ export class BaseMessageTile extends SimpleTile { } get time() { - return this._date && this._date.toLocaleTimeString({}, {hour: "numeric", minute: "2-digit"}); + return this._date && this.timeFormatter.formatTime(this._date); } get isOwn() { diff --git a/src/platform/types/types.ts b/src/platform/types/types.ts index 9147f6b8..df7ce6ac 100644 --- a/src/platform/types/types.ts +++ b/src/platform/types/types.ts @@ -45,6 +45,7 @@ export type File = { } export interface ITimeFormatter { + formatTime(date: Date): string; formatRelativeDate(date: Date): string; formatMachineReadableDate(date: Date): string; } \ No newline at end of file diff --git a/src/platform/web/dom/TimeFormatter.ts b/src/platform/web/dom/TimeFormatter.ts index 3f354713..7db879ea 100644 --- a/src/platform/web/dom/TimeFormatter.ts +++ b/src/platform/web/dom/TimeFormatter.ts @@ -29,6 +29,7 @@ export class TimeFormatter implements ITimeFormatter { private weekdayFormatter: Intl.DateTimeFormat; private currentYearFormatter: Intl.DateTimeFormat; private otherYearFormatter: Intl.DateTimeFormat; + private timeFormatter: Intl.DateTimeFormat; constructor(private clock: Clock) { // don't use the clock time here as the DOM relative formatters don't support setting the reference date anyway @@ -47,7 +48,13 @@ export class TimeFormatter implements ITimeFormatter { month: 'long', day: 'numeric' }); + this.timeFormatter = new Intl.DateTimeFormat(undefined, {hour: "numeric", minute: "2-digit"}); } + + formatTime(date: Date): string { + return this.timeFormatter.format(date); + } + formatMachineReadableDate(date: Date): string { return `${date.getFullYear()}-${date.getMonth() + 1}-${date.getDate()}`; } From efa45cade68a6f48edda48fef4cc1a7fc08d9924 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 25 Nov 2022 16:48:46 +0100 Subject: [PATCH 235/435] remove date from message tiles --- src/domain/session/room/timeline/tiles/BaseMessageTile.js | 5 ----- src/platform/web/ui/session/room/timeline/BaseMediaView.js | 2 +- src/platform/web/ui/session/room/timeline/FileView.js | 2 +- src/platform/web/ui/session/room/timeline/LocationView.js | 2 +- src/platform/web/ui/session/room/timeline/TextMessageView.js | 2 +- 5 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/domain/session/room/timeline/tiles/BaseMessageTile.js b/src/domain/session/room/timeline/tiles/BaseMessageTile.js index 05ff1867..45c2a05d 100644 --- a/src/domain/session/room/timeline/tiles/BaseMessageTile.js +++ b/src/domain/session/room/timeline/tiles/BaseMessageTile.js @@ -78,11 +78,6 @@ export class BaseMessageTile extends SimpleTile { return this.sender; } - // TODO: remove? - get date() { - return this._date && this._date.toLocaleDateString({}, {month: "numeric", day: "numeric"}); - } - get time() { return this._date && this.timeFormatter.formatTime(this._date); } diff --git a/src/platform/web/ui/session/room/timeline/BaseMediaView.js b/src/platform/web/ui/session/room/timeline/BaseMediaView.js index 9d534fd1..bc49b3f6 100644 --- a/src/platform/web/ui/session/room/timeline/BaseMediaView.js +++ b/src/platform/web/ui/session/room/timeline/BaseMediaView.js @@ -34,7 +34,7 @@ export class BaseMediaView extends BaseMessageView { const children = [ t.div({className: "spacer", style: spacerStyle}), this.renderMedia(t, vm), - t.time(vm.date + " " + vm.time), + t.time(vm.time), ]; const status = t.div({ className: { diff --git a/src/platform/web/ui/session/room/timeline/FileView.js b/src/platform/web/ui/session/room/timeline/FileView.js index 6a2d418e..ca0eb10e 100644 --- a/src/platform/web/ui/session/room/timeline/FileView.js +++ b/src/platform/web/ui/session/room/timeline/FileView.js @@ -24,7 +24,7 @@ export class FileView extends BaseMessageView { } else { children.push( t.button({className: "link", onClick: () => vm.download()}, vm => vm.label), - t.time(vm.date + " " + vm.time) + t.time(vm.time) ); } return t.p({className: "Timeline_messageBody statusMessage"}, children); diff --git a/src/platform/web/ui/session/room/timeline/LocationView.js b/src/platform/web/ui/session/room/timeline/LocationView.js index de605c6a..e0d2656c 100644 --- a/src/platform/web/ui/session/room/timeline/LocationView.js +++ b/src/platform/web/ui/session/room/timeline/LocationView.js @@ -21,7 +21,7 @@ export class LocationView extends BaseMessageView { return t.p({className: "Timeline_messageBody statusMessage"}, [ t.span(vm.label), t.a({className: "Timeline_locationLink", href: vm.mapsLink, target: "_blank", rel: "noopener"}, vm.i18n`Open in maps`), - t.time(vm.date + " " + vm.time) + t.time(vm.time) ]); } } diff --git a/src/platform/web/ui/session/room/timeline/TextMessageView.js b/src/platform/web/ui/session/room/timeline/TextMessageView.js index 8d6cb4dc..a6741de7 100644 --- a/src/platform/web/ui/session/room/timeline/TextMessageView.js +++ b/src/platform/web/ui/session/room/timeline/TextMessageView.js @@ -20,7 +20,7 @@ import {ReplyPreviewError, ReplyPreviewView} from "./ReplyPreviewView.js"; export class TextMessageView extends BaseMessageView { renderMessageBody(t, vm) { - const time = t.time({className: {hidden: !vm.date}}, vm.date + " " + vm.time); + const time = t.time({className: {hidden: !vm.time}}, vm.time); const container = t.div({ className: { "Timeline_messageBody": true, From 195142c2492a5fae91b2d5c89cd11fad3d9b6730 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 25 Nov 2022 16:56:01 +0100 Subject: [PATCH 236/435] fix lint warnings --- src/domain/session/room/timeline/tiles/BaseMessageTile.js | 1 - src/domain/session/room/timeline/tiles/DateTile.ts | 2 +- src/domain/session/room/timeline/tiles/SimpleTile.js | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/domain/session/room/timeline/tiles/BaseMessageTile.js b/src/domain/session/room/timeline/tiles/BaseMessageTile.js index 45c2a05d..a7dc82cd 100644 --- a/src/domain/session/room/timeline/tiles/BaseMessageTile.js +++ b/src/domain/session/room/timeline/tiles/BaseMessageTile.js @@ -15,7 +15,6 @@ limitations under the License. */ import {SimpleTile} from "./SimpleTile.js"; -import {TileShape} from "./ITile"; import {ReactionsViewModel} from "../ReactionsViewModel.js"; import {getIdentifierColorNumber, avatarInitials, getAvatarHttpUrl} from "../../../../avatar"; diff --git a/src/domain/session/room/timeline/tiles/DateTile.ts b/src/domain/session/room/timeline/tiles/DateTile.ts index 590333b8..ac4e6329 100644 --- a/src/domain/session/room/timeline/tiles/DateTile.ts +++ b/src/domain/session/room/timeline/tiles/DateTile.ts @@ -56,7 +56,7 @@ export class DateTile extends ViewModel implements ITile { get machineReadableDate(): string { if (!this._machineReadableString) { - this._machineReadableString = this.timeFormatter.formatMachineReadableDate(new Date(this.refEntry.timestamp)); + this._machineReadableString = this.timeFormatter.formatMachineReadableDate(new Date(this.refEntry.timestamp)); } return this._machineReadableString; } diff --git a/src/domain/session/room/timeline/tiles/SimpleTile.js b/src/domain/session/room/timeline/tiles/SimpleTile.js index 91e023cb..7cb9617d 100644 --- a/src/domain/session/room/timeline/tiles/SimpleTile.js +++ b/src/domain/session/room/timeline/tiles/SimpleTile.js @@ -51,7 +51,6 @@ export class SimpleTile extends ViewModel { _updateDateSeparator(prev) { if (prev && prev._date && this._date) { - const neededDateSeparator = this._needsDateSeparator; this._needsDateSeparator = prev._date.getFullYear() !== this._date.getFullYear() || prev._date.getMonth() !== this._date.getMonth() || prev._date.getDate() !== this._date.getDate(); From 7bdd23e7679ea4faf09d1bae3ac3dddd1bea6bab Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 25 Nov 2022 17:21:58 +0100 Subject: [PATCH 237/435] restore active reaction bg color to be transparent version of accent color --- doc/THEMING.md | 1 + src/platform/web/theming/shared/color.mjs | 3 +++ src/platform/web/ui/css/themes/element/timeline.css | 4 ++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/THEMING.md b/doc/THEMING.md index 599434bd..c00ab155 100644 --- a/doc/THEMING.md +++ b/doc/THEMING.md @@ -80,6 +80,7 @@ Currently supported operations are: | -------- | -------- | -------- | | darker | percentage | color | | lighter | percentage | color | +| alpha | alpha percentage | color | ## Aliases It is possible give aliases to variables in the `theme.css` file: diff --git a/src/platform/web/theming/shared/color.mjs b/src/platform/web/theming/shared/color.mjs index 8af76b6b..31d40185 100644 --- a/src/platform/web/theming/shared/color.mjs +++ b/src/platform/web/theming/shared/color.mjs @@ -36,5 +36,8 @@ export function derive(value, operation, argument, isDark) { const newColorString = offColor(value).lighten(argumentAsNumber / 100).hex(); return newColorString; } + case "alpha": { + return offColor(value).rgba(argumentAsNumber / 100); + } } } diff --git a/src/platform/web/ui/css/themes/element/timeline.css b/src/platform/web/ui/css/themes/element/timeline.css index 8645bc3f..7ff35eb1 100644 --- a/src/platform/web/ui/css/themes/element/timeline.css +++ b/src/platform/web/ui/css/themes/element/timeline.css @@ -364,7 +364,7 @@ only loads when the top comes into view*/ } .Timeline_messageReactions button.active { - background-color: var(--background-color-secondary); + background-color: var(--accent-color--alpha-11); border-color: var(--accent-color); } @@ -440,4 +440,4 @@ only loads when the top comes into view*/ background-color: var(--background-color-primary); border-radius: 8px; text-align: center; - } \ No newline at end of file + } From 2d0122dda7781995d143d59812ed6d9667198bdf Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 25 Nov 2022 17:38:05 +0100 Subject: [PATCH 238/435] apply PR suggestions --- src/domain/navigation/index.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/domain/navigation/index.ts b/src/domain/navigation/index.ts index 935f7138..5a4b9abc 100644 --- a/src/domain/navigation/index.ts +++ b/src/domain/navigation/index.ts @@ -147,8 +147,9 @@ export function parseUrlPath(urlPath: string, currentNavPath: Path, segments.push(new Segment("empty-grid-tile", selectedIndex)); } } else if (type === "open-room") { - const roomId = decodeURIComponent(iterator.next().value); + let roomId = iterator.next().value; if (!roomId) { break; } + roomId = decodeURIComponent(roomId); const rooms = currentNavPath.get("rooms"); if (rooms) { segments.push(roomsSegmentWithRoom(rooms, roomId, currentNavPath)); @@ -176,8 +177,9 @@ export function parseUrlPath(urlPath: string, currentNavPath: Path, } else if (type === "details" || type === "members") { pushRightPanelSegment(segments, type); } else if (type === "member") { - const userId = decodeURIComponent(iterator.next().value); + let userId = iterator.next().value; if (!userId) { break; } + userId = decodeURIComponent(userId); pushRightPanelSegment(segments, type, userId); } else if (type.includes("loginToken")) { // Special case for SSO-login with query parameter loginToken= From b8444a32c0f74cadabcc4db654d7378fe8b98fff Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 25 Nov 2022 17:38:13 +0100 Subject: [PATCH 239/435] add missing decode --- src/domain/navigation/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/navigation/index.ts b/src/domain/navigation/index.ts index 5a4b9abc..a75260d0 100644 --- a/src/domain/navigation/index.ts +++ b/src/domain/navigation/index.ts @@ -137,7 +137,7 @@ export function parseUrlPath(urlPath: string, currentNavPath: Path, if (type === "rooms") { const roomsValue = iterator.next().value; if (roomsValue === undefined) { break; } - const roomIds = roomsValue.split(","); + const roomIds = decodeURIComponent(roomsValue).split(","); segments.push(new Segment(type, roomIds)); const selectedIndex = parseInt(iterator.next().value || "0", 10); const roomId = roomIds[selectedIndex]; From 467a76c2231b9555a6d7deb4f19b4a5eddd6b3e3 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 25 Nov 2022 17:45:06 +0100 Subject: [PATCH 240/435] decode ids separately, as we encode them separately as well --- src/domain/navigation/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/navigation/index.ts b/src/domain/navigation/index.ts index a75260d0..3bbe4d3a 100644 --- a/src/domain/navigation/index.ts +++ b/src/domain/navigation/index.ts @@ -137,7 +137,7 @@ export function parseUrlPath(urlPath: string, currentNavPath: Path, if (type === "rooms") { const roomsValue = iterator.next().value; if (roomsValue === undefined) { break; } - const roomIds = decodeURIComponent(roomsValue).split(","); + const roomIds = roomsValue.split(",").map(id => decodeURIComponent(id)); segments.push(new Segment(type, roomIds)); const selectedIndex = parseInt(iterator.next().value || "0", 10); const roomId = roomIds[selectedIndex]; From f7132a48d9a30fcb10ac58981cb5ee8dd5c2629f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 25 Nov 2022 17:45:31 +0100 Subject: [PATCH 241/435] implement PR suggestion to do all encoding in fn, so return type is str --- src/domain/navigation/index.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/domain/navigation/index.ts b/src/domain/navigation/index.ts index 3bbe4d3a..a2705944 100644 --- a/src/domain/navigation/index.ts +++ b/src/domain/navigation/index.ts @@ -205,7 +205,7 @@ export function stringifyPath(path: Path): string { const encodedSegmentValue = encodeSegmentValue(segment.value); switch (segment.type) { case "rooms": - urlPath += `/rooms/${(encodedSegmentValue as string[]).join(",")}`; + urlPath += `/rooms/${encodedSegmentValue}`; break; case "empty-grid-tile": urlPath += `/${encodedSegmentValue}`; @@ -224,7 +224,7 @@ export function stringifyPath(path: Path): string { continue; default: urlPath += `/${segment.type}`; - if (encodedSegmentValue && encodedSegmentValue !== true) { + if (encodedSegmentValue) { urlPath += `/${encodedSegmentValue}`; } } @@ -233,13 +233,13 @@ export function stringifyPath(path: Path): string { return urlPath; } -function encodeSegmentValue(value: SegmentType[keyof SegmentType]) { - if (typeof value === "boolean") { +function encodeSegmentValue(value: SegmentType[keyof SegmentType]): string { + if (value === true) { // Nothing to encode for boolean - return value; + return ""; } else if (Array.isArray(value)) { - return value.map(v => encodeURIComponent(v)); + return value.map(v => encodeURIComponent(v)).join(","); } else { return encodeURIComponent(value); From 308bbee5ede6f4d21cd69e2f9239aec4e667b379 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 25 Nov 2022 17:47:54 +0100 Subject: [PATCH 242/435] release v0.3.5 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5429e0cf..1a21d58c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hydrogen-web", - "version": "0.3.4", + "version": "0.3.5", "description": "A javascript matrix client prototype, trying to minize RAM usage by offloading as much as possible to IndexedDB", "directories": { "doc": "doc" From d1649ea4991fbe407277cd40ea6619b5a1a785f6 Mon Sep 17 00:00:00 2001 From: Paulo Pinto Date: Thu, 15 Dec 2022 15:22:55 +0000 Subject: [PATCH 243/435] Make sure fonts are cached by service worker --- .../web/ui/css/themes/element/inter.css | 72 +++++++++---------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/src/platform/web/ui/css/themes/element/inter.css b/src/platform/web/ui/css/themes/element/inter.css index 5c69d7e6..d4cd1e8a 100644 --- a/src/platform/web/ui/css/themes/element/inter.css +++ b/src/platform/web/ui/css/themes/element/inter.css @@ -3,16 +3,16 @@ font-style: normal; font-weight: 100; font-display: swap; - src: url("inter/Inter-Thin.woff2?v=3.13") format("woff2"), - url("inter/Inter-Thin.woff?v=3.13") format("woff"); + src: url("inter/Inter-Thin.woff2") format("woff2"), + url("inter/Inter-Thin.woff") format("woff"); } @font-face { font-family: 'Inter'; font-style: italic; font-weight: 100; font-display: swap; - src: url("inter/Inter-ThinItalic.woff2?v=3.13") format("woff2"), - url("inter/Inter-ThinItalic.woff?v=3.13") format("woff"); + src: url("inter/Inter-ThinItalic.woff2") format("woff2"), + url("inter/Inter-ThinItalic.woff") format("woff"); } @font-face { @@ -20,16 +20,16 @@ font-style: normal; font-weight: 200; font-display: swap; - src: url("inter/Inter-ExtraLight.woff2?v=3.13") format("woff2"), - url("inter/Inter-ExtraLight.woff?v=3.13") format("woff"); + src: url("inter/Inter-ExtraLight.woff2") format("woff2"), + url("inter/Inter-ExtraLight.woff") format("woff"); } @font-face { font-family: 'Inter'; font-style: italic; font-weight: 200; font-display: swap; - src: url("inter/Inter-ExtraLightItalic.woff2?v=3.13") format("woff2"), - url("inter/Inter-ExtraLightItalic.woff?v=3.13") format("woff"); + src: url("inter/Inter-ExtraLightItalic.woff2") format("woff2"), + url("inter/Inter-ExtraLightItalic.woff") format("woff"); } @font-face { @@ -37,16 +37,16 @@ font-style: normal; font-weight: 300; font-display: swap; - src: url("inter/Inter-Light.woff2?v=3.13") format("woff2"), - url("inter/Inter-Light.woff?v=3.13") format("woff"); + src: url("inter/Inter-Light.woff2") format("woff2"), + url("inter/Inter-Light.woff") format("woff"); } @font-face { font-family: 'Inter'; font-style: italic; font-weight: 300; font-display: swap; - src: url("inter/Inter-LightItalic.woff2?v=3.13") format("woff2"), - url("inter/Inter-LightItalic.woff?v=3.13") format("woff"); + src: url("inter/Inter-LightItalic.woff2") format("woff2"), + url("inter/Inter-LightItalic.woff") format("woff"); } @font-face { @@ -54,16 +54,16 @@ font-style: normal; font-weight: 400; font-display: swap; - src: url("inter/Inter-Regular.woff2?v=3.13") format("woff2"), - url("inter/Inter-Regular.woff?v=3.13") format("woff"); + src: url("inter/Inter-Regular.woff2") format("woff2"), + url("inter/Inter-Regular.woff") format("woff"); } @font-face { font-family: 'Inter'; font-style: italic; font-weight: 400; font-display: swap; - src: url("inter/Inter-Italic.woff2?v=3.13") format("woff2"), - url("inter/Inter-Italic.woff?v=3.13") format("woff"); + src: url("inter/Inter-Italic.woff2") format("woff2"), + url("inter/Inter-Italic.woff") format("woff"); } @font-face { @@ -71,16 +71,16 @@ font-style: normal; font-weight: 500; font-display: swap; - src: url("inter/Inter-Medium.woff2?v=3.13") format("woff2"), - url("inter/Inter-Medium.woff?v=3.13") format("woff"); + src: url("inter/Inter-Medium.woff2") format("woff2"), + url("inter/Inter-Medium.woff") format("woff"); } @font-face { font-family: 'Inter'; font-style: italic; font-weight: 500; font-display: swap; - src: url("inter/Inter-MediumItalic.woff2?v=3.13") format("woff2"), - url("inter/Inter-MediumItalic.woff?v=3.13") format("woff"); + src: url("inter/Inter-MediumItalic.woff2") format("woff2"), + url("inter/Inter-MediumItalic.woff") format("woff"); } @font-face { @@ -88,16 +88,16 @@ font-style: normal; font-weight: 600; font-display: swap; - src: url("inter/Inter-SemiBold.woff2?v=3.13") format("woff2"), - url("inter/Inter-SemiBold.woff?v=3.13") format("woff"); + src: url("inter/Inter-SemiBold.woff2") format("woff2"), + url("inter/Inter-SemiBold.woff") format("woff"); } @font-face { font-family: 'Inter'; font-style: italic; font-weight: 600; font-display: swap; - src: url("inter/Inter-SemiBoldItalic.woff2?v=3.13") format("woff2"), - url("inter/Inter-SemiBoldItalic.woff?v=3.13") format("woff"); + src: url("inter/Inter-SemiBoldItalic.woff2") format("woff2"), + url("inter/Inter-SemiBoldItalic.woff") format("woff"); } @font-face { @@ -105,16 +105,16 @@ font-style: normal; font-weight: 700; font-display: swap; - src: url("inter/Inter-Bold.woff2?v=3.13") format("woff2"), - url("inter/Inter-Bold.woff?v=3.13") format("woff"); + src: url("inter/Inter-Bold.woff2") format("woff2"), + url("inter/Inter-Bold.woff") format("woff"); } @font-face { font-family: 'Inter'; font-style: italic; font-weight: 700; font-display: swap; - src: url("inter/Inter-BoldItalic.woff2?v=3.13") format("woff2"), - url("inter/Inter-BoldItalic.woff?v=3.13") format("woff"); + src: url("inter/Inter-BoldItalic.woff2") format("woff2"), + url("inter/Inter-BoldItalic.woff") format("woff"); } @font-face { @@ -122,16 +122,16 @@ font-style: normal; font-weight: 800; font-display: swap; - src: url("inter/Inter-ExtraBold.woff2?v=3.13") format("woff2"), - url("inter/Inter-ExtraBold.woff?v=3.13") format("woff"); + src: url("inter/Inter-ExtraBold.woff2") format("woff2"), + url("inter/Inter-ExtraBold.woff") format("woff"); } @font-face { font-family: 'Inter'; font-style: italic; font-weight: 800; font-display: swap; - src: url("inter/Inter-ExtraBoldItalic.woff2?v=3.13") format("woff2"), - url("inter/Inter-ExtraBoldItalic.woff?v=3.13") format("woff"); + src: url("inter/Inter-ExtraBoldItalic.woff2") format("woff2"), + url("inter/Inter-ExtraBoldItalic.woff") format("woff"); } @font-face { @@ -139,14 +139,14 @@ font-style: normal; font-weight: 900; font-display: swap; - src: url("inter/Inter-Black.woff2?v=3.13") format("woff2"), - url("inter/Inter-Black.woff?v=3.13") format("woff"); + src: url("inter/Inter-Black.woff2") format("woff2"), + url("inter/Inter-Black.woff") format("woff"); } @font-face { font-family: 'Inter'; font-style: italic; font-weight: 900; font-display: swap; - src: url("inter/Inter-BlackItalic.woff2?v=3.13") format("woff2"), - url("inter/Inter-BlackItalic.woff?v=3.13") format("woff"); + src: url("inter/Inter-BlackItalic.woff2") format("woff2"), + url("inter/Inter-BlackItalic.woff") format("woff"); } From dbbbb1c29ababbcbe3f933b2a9424e3c0fc00ec5 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 16 Dec 2022 09:07:37 +0000 Subject: [PATCH 244/435] clarify ublock origin breaks the service worker --- doc/FAQ.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/FAQ.md b/doc/FAQ.md index ce372dc3..d445500e 100644 --- a/doc/FAQ.md +++ b/doc/FAQ.md @@ -10,6 +10,9 @@ TorBrowser ships a crippled IndexedDB implementation and will not work. At some It used work in pre-webkit Edge, to have it work on Windows Phone, but that support has probably bit-rotted as it isn't tested anymore. +The following browser extensions are known to break Hydrogen + - uBlock Origin (seems to block the service worker script) + ## Is there a way to run the app as a desktop app? You can install Hydrogen as a PWA using Chrome/Chromium on any platform or Edge on Windows. Gnome Web/Ephiphany also allows to "Install site as web application". There is no Electron build of Hydrogen, and there will likely be none in the near future, as Electron complicates the release process considerably. Once Hydrogen is more mature and feature complete, we might reconsider and use [Tauri](https://tauri.studio) if there are compelling use cases not possible with PWAs. For now though, we want to keep development and releasing fast and nimble ;) From 1473e5647d6b778bcb5979ce8273d7fe447df4d8 Mon Sep 17 00:00:00 2001 From: ElonSatoshi Date: Fri, 16 Dec 2022 03:21:22 -0600 Subject: [PATCH 245/435] Update FAQ.md Added some extra notes about uBlock Origin compatibility --- doc/FAQ.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/FAQ.md b/doc/FAQ.md index d445500e..3f195a0e 100644 --- a/doc/FAQ.md +++ b/doc/FAQ.md @@ -11,7 +11,8 @@ TorBrowser ships a crippled IndexedDB implementation and will not work. At some It used work in pre-webkit Edge, to have it work on Windows Phone, but that support has probably bit-rotted as it isn't tested anymore. The following browser extensions are known to break Hydrogen - - uBlock Origin (seems to block the service worker script) + - uBlock Origin (Some custom filters seem to block the service worker script) + - Try locating the filter that is blocking the service worker script in the uBlock Origin logger, and disabling that filter. Otherwise, the easiest solution is to disable uBlock Origin for the Hydrogen site. It is possible to re-enable it after logging in, but it may possibly break again when there is an update. ## Is there a way to run the app as a desktop app? From 03b0cfb47bda50ccf470b02954ef699fc1266ee7 Mon Sep 17 00:00:00 2001 From: ElonSatoshi Date: Fri, 16 Dec 2022 03:26:40 -0600 Subject: [PATCH 246/435] How to disable uBlock Origin Added a little note on how to disable uBlock Origin for a specific site --- doc/FAQ.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/FAQ.md b/doc/FAQ.md index 3f195a0e..0823ee08 100644 --- a/doc/FAQ.md +++ b/doc/FAQ.md @@ -12,7 +12,7 @@ It used work in pre-webkit Edge, to have it work on Windows Phone, but that supp The following browser extensions are known to break Hydrogen - uBlock Origin (Some custom filters seem to block the service worker script) - - Try locating the filter that is blocking the service worker script in the uBlock Origin logger, and disabling that filter. Otherwise, the easiest solution is to disable uBlock Origin for the Hydrogen site. It is possible to re-enable it after logging in, but it may possibly break again when there is an update. + - Try locating the filter that is blocking the service worker script in the uBlock Origin logger, and disabling that filter. Otherwise, the easiest solution is to disable uBlock Origin for the Hydrogen site (by opening the uBlock Origin popup and clicking the large power button symbol). It is possible to re-enable it after logging in, but it may possibly break again when there is an update. ## Is there a way to run the app as a desktop app? From 6ecff485ecf74c0001b8b86ac55b8980c254330e Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 20 Dec 2022 18:52:38 +0100 Subject: [PATCH 247/435] make tile comparison commutative again, allow DateTile to do comparison --- src/domain/session/room/timeline/tiles/DateTile.ts | 10 +++++++++- src/domain/session/room/timeline/tiles/ITile.ts | 13 +++++++++++++ .../session/room/timeline/tiles/SimpleTile.js | 12 ++++++++++-- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/src/domain/session/room/timeline/tiles/DateTile.ts b/src/domain/session/room/timeline/tiles/DateTile.ts index ac4e6329..2174b9c5 100644 --- a/src/domain/session/room/timeline/tiles/DateTile.ts +++ b/src/domain/session/room/timeline/tiles/DateTile.ts @@ -119,6 +119,14 @@ export class DateTile extends ViewModel implements ITile { return false; } + /** + * This tile needs to do the comparison between tiles, as it uses the entry + * from another tile to determine its sorting order. + * */ + get comparisonIsNotCommutative(): boolean { + return true; + } + // let item know it has a new sibling updatePreviousSibling(prev: ITile | undefined): void { // forward the sibling update to our next tile, so it is informed @@ -178,4 +186,4 @@ export function tests() { assert.equal(tiles[2], b); } } -} \ No newline at end of file +} diff --git a/src/domain/session/room/timeline/tiles/ITile.ts b/src/domain/session/room/timeline/tiles/ITile.ts index dd6c0f81..f36c1c51 100644 --- a/src/domain/session/room/timeline/tiles/ITile.ts +++ b/src/domain/session/room/timeline/tiles/ITile.ts @@ -24,7 +24,20 @@ export interface ITile extends IDisposable { setUpdateEmit(emitUpdate: EmitUpdateFn): void; get upperEntry(): E; get lowerEntry(): E; + /** compare two tiles, returning: + * - 0 if both tiles are considered equal + * - a negative value if this tiles is sorted before the given tile + * - a positive value if this tiles is sorted after the given tile + **/ compare(tile: ITile): number; + /** Some tiles might need comparison mechanisms that are not commutative, + * (e.g. `tileA.compare(tileB)` not being the same as `tileB.compare(tileA)`), + * a property needed for reliably sorting the tiles in TilesCollection. + * To counteract this, tiles can indicate this is not the case for them and + * when any other tile is being compared to a tile where this flag is true, + * it should delegate the comparison to the given tile. + * E.g. one example where this flag is used is DateTile. */ + get comparisonIsNotCommutative(): boolean; compareEntry(entry: BaseEntry): number; // update received for already included (falls within sort keys) entry updateEntry(entry: BaseEntry, param: any): UpdateAction; diff --git a/src/domain/session/room/timeline/tiles/SimpleTile.js b/src/domain/session/room/timeline/tiles/SimpleTile.js index 7cb9617d..93f89d66 100644 --- a/src/domain/session/room/timeline/tiles/SimpleTile.js +++ b/src/domain/session/room/timeline/tiles/SimpleTile.js @@ -110,8 +110,16 @@ export class SimpleTile extends ViewModel { return this._entry; } + get comparisonIsNotCommutative() { + return false; + } + compare(tile) { - return this.upperEntry.compare(tile.upperEntry); + if (tile.comparisonIsNotCommutative) { + return -tile.compare(this); + } else { + return this.upperEntry.compare(tile.upperEntry); + } } compareEntry(entry) { @@ -241,4 +249,4 @@ export function tests() { assert.equal(fridayTile.needsDateSeparator, true); }, } -} \ No newline at end of file +} From c38ee5239f68d0e945b70b6e54d1032dbc2ef5ac Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 20 Dec 2022 18:53:13 +0100 Subject: [PATCH 248/435] remove debug logging --- src/platform/web/dom/TimeFormatter.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/platform/web/dom/TimeFormatter.ts b/src/platform/web/dom/TimeFormatter.ts index 7db879ea..2a98a716 100644 --- a/src/platform/web/dom/TimeFormatter.ts +++ b/src/platform/web/dom/TimeFormatter.ts @@ -61,7 +61,6 @@ export class TimeFormatter implements ITimeFormatter { formatRelativeDate(date: Date): string { let daysDiff = Math.floor((date.getTime() - this.todayMidnight.getTime()) / TimeScope.Day); - console.log("formatRelativeDate daysDiff", daysDiff, date); if (daysDiff >= -1 && daysDiff <= 1) { // Tomorrow, Today, Yesterday return capitalizeFirstLetter(this.relativeDayFormatter.format(daysDiff, "day")); @@ -80,4 +79,4 @@ export class TimeFormatter implements ITimeFormatter { function capitalizeFirstLetter(str: string) { return str.slice(0, 1).toLocaleUpperCase() + str.slice(1); -} \ No newline at end of file +} From 3fecedfeb4cb23aea75c457e89c10eef9ccec2fc Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 20 Dec 2022 18:57:09 +0100 Subject: [PATCH 249/435] release v0.3.6 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1a21d58c..9bf21a7c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hydrogen-web", - "version": "0.3.5", + "version": "0.3.6", "description": "A javascript matrix client prototype, trying to minize RAM usage by offloading as much as possible to IndexedDB", "directories": { "doc": "doc" From 3735e4420addf904dfe2623818a8c9162753e2f1 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 5 Jan 2023 11:34:58 +0100 Subject: [PATCH 250/435] fix TS definition --- src/domain/session/room/timeline/tiles/ITile.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/session/room/timeline/tiles/ITile.ts b/src/domain/session/room/timeline/tiles/ITile.ts index f36c1c51..24425a07 100644 --- a/src/domain/session/room/timeline/tiles/ITile.ts +++ b/src/domain/session/room/timeline/tiles/ITile.ts @@ -45,7 +45,7 @@ export interface ITile extends IDisposable { // as SimpleTile only has one entry, the tile should be removed removeEntry(entry: BaseEntry): boolean // SimpleTile can only contain 1 entry - tryIncludeEntry(): boolean; + tryIncludeEntry(entry: BaseEntry): boolean; // let item know it has a new sibling updatePreviousSibling(prev: ITile | undefined): void; // let item know it has a new sibling From 508d88edb54d308e27ecc7c0a3d0fd794ac24ba7 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 5 Jan 2023 11:35:17 +0100 Subject: [PATCH 251/435] add some ideas to CSS doc how to document css classes --- doc/CSS.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/CSS.md b/doc/CSS.md index 7365ec5b..36e02a3f 100644 --- a/doc/CSS.md +++ b/doc/CSS.md @@ -6,6 +6,10 @@ We could do top to bottom gradients in default avatars to make them look a bit c Can take ideas/adopt from OOCSS and SMACSS. +## Documentation + +Whether we use OOCSS, SMACSS or BEM, we should write a tool that uses a JS parser (acorn?) to find all css classes used in the view code by looking for a `{className: "..."}` pattern. E.g. if using BEM, use all the found classes to construct a doc with a section for every block, with therein all elements and modifiers. + ### Root - maybe we should not assume `body` is the root, but rather a `.brawl` class. The root is where we'd set root level css variables, fonts?, etc. Should we scope all css to this root class? That could get painful with just vanilla css. We could use something like https://github.com/domwashburn/postcss-parent-selector to only do this at build time. Other useful plugin for postcss: https://github.com/postcss/postcss-selector-parser From e26eb30b82fb04e36d2cf8e510ee2ae02dda1efb Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 5 Jan 2023 15:47:01 +0100 Subject: [PATCH 252/435] rename m.expires_ts to expires_ts to be compatible with MSC/Element --- src/matrix/calls/callEventTypes.ts | 2 +- src/matrix/calls/group/GroupCall.ts | 2 +- src/matrix/calls/group/Member.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index ca5c870c..8fd7b23d 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -25,7 +25,7 @@ export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata"; export interface CallDeviceMembership { device_id: string, session_id: string, - ["m.expires_ts"]?: number, + ["expires_ts"]?: number, feeds?: Array<{purpose: string}> } diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index f2561f32..0931d4ea 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -544,7 +544,7 @@ export class GroupCall extends EventEmitter<{change: never}> { callInfo["m.devices"].push({ ["device_id"]: this.options.ownDeviceId, ["session_id"]: this.options.sessionId, - ["m.expires_ts"]: now + CALL_MEMBER_VALIDITY_PERIOD_MS, + ["expires_ts"]: now + CALL_MEMBER_VALIDITY_PERIOD_MS, feeds: [{purpose: "m.usermedia"}] }); } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 0aba3b54..9debef22 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -408,7 +408,7 @@ export class Member { } export function memberExpiresAt(callDeviceMembership: CallDeviceMembership): number | undefined { - const expiresAt = callDeviceMembership["m.expires_ts"]; + const expiresAt = callDeviceMembership["expires_ts"]; if (Number.isSafeInteger(expiresAt)) { return expiresAt; } From fcb4f2a62dd1569eb1bf8c4c3e44358cbe08f8a3 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 6 Jan 2023 12:05:11 +0100 Subject: [PATCH 253/435] fix microphone audio being played back through local video preview --- src/domain/session/room/CallViewModel.ts | 2 +- src/matrix/calls/LocalMedia.ts | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 020c9f17..9c0eae9a 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -136,7 +136,7 @@ class OwnMemberViewModel extends ViewModel implements IStreamViewModel } get stream(): Stream | undefined { - return this.call.localMedia?.userMedia; + return this.call.localMedia?.userMediaPreview; } private get call(): GroupCall { diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index f02cd11b..25d6862f 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -20,11 +20,24 @@ import {SDPStreamMetadata} from "./callEventTypes"; import {getStreamVideoTrack, getStreamAudioTrack} from "./common"; export class LocalMedia { + // the userMedia stream without audio, to play in the UI + // without our own audio being played back to us + public readonly userMediaPreview?: Stream; + constructor( public readonly userMedia?: Stream, public readonly screenShare?: Stream, public readonly dataChannelOptions?: RTCDataChannelInit, - ) {} + ) { + if (userMedia && userMedia.getVideoTracks().length > 0) { + this.userMediaPreview = userMedia.clone(); + const audioTrack = getStreamAudioTrack(this.userMediaPreview); + if (audioTrack) { + audioTrack.stop(); + this.userMediaPreview.removeTrack(audioTrack); + } + } + } withUserMedia(stream: Stream) { return new LocalMedia(stream, this.screenShare, this.dataChannelOptions); @@ -63,6 +76,7 @@ export class LocalMedia { dispose() { getStreamAudioTrack(this.userMedia)?.stop(); getStreamVideoTrack(this.userMedia)?.stop(); + getStreamVideoTrack(this.userMediaPreview)?.stop(); getStreamVideoTrack(this.screenShare)?.stop(); } } From cb0f803276106d608238aebeedc62256644d83fd Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 6 Jan 2023 12:05:34 +0100 Subject: [PATCH 254/435] make this code a bit clearer --- src/matrix/calls/group/GroupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 0931d4ea..d65ea838 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -216,8 +216,8 @@ export class GroupCall extends EventEmitter<{change: never}> { // you might be muted because you don't have a track or because // you actively chosen to mute // (which we want to respect in the future when you add a track) + muteSettings.updateTrackInfo(joinedData.localMedia.userMedia); joinedData.localMuteSettings = muteSettings; - joinedData.localMuteSettings.updateTrackInfo(joinedData.localMedia.userMedia); if (!prevMuteSettings.equals(muteSettings)) { await Promise.all(Array.from(this._members.values()).map(m => { return m.setMuted(joinedData.localMuteSettings); From 014ae4185f4226b2ebc43f367cec2a0498fdc6cf Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 6 Jan 2023 17:01:57 +0100 Subject: [PATCH 255/435] WIP --- src/utils/ErrorBoundary.ts | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 src/utils/ErrorBoundary.ts diff --git a/src/utils/ErrorBoundary.ts b/src/utils/ErrorBoundary.ts new file mode 100644 index 00000000..e9f297c5 --- /dev/null +++ b/src/utils/ErrorBoundary.ts @@ -0,0 +1,38 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {ObservableValue, BaseObservableValue} from "../observable/ObservableValue"; + +export class ErrorBoundary { + constructor(private readonly errorCallback: (Error) => void) {} + + try(callback: () => T): T | undefined; + try(callback: () => Promise): Promise | undefined { + try { + let result: T | Promise = callback(); + if (result instanceof Promise) { + result = result.catch(err => { + this.errorCallback(err); + return undefined; + }); + } + return result; + } catch (err) { + this.errorCallback(err); + return undefined; + } + } +} From a9a72f88e5f5b44e148b104f630f4bc7f74445ad Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 9 Jan 2023 10:36:28 +0100 Subject: [PATCH 256/435] finish ErrorBoundary --- src/utils/ErrorBoundary.ts | 56 ++++++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 6 deletions(-) diff --git a/src/utils/ErrorBoundary.ts b/src/utils/ErrorBoundary.ts index e9f297c5..520d2a35 100644 --- a/src/utils/ErrorBoundary.ts +++ b/src/utils/ErrorBoundary.ts @@ -14,25 +14,69 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {ObservableValue, BaseObservableValue} from "../observable/ObservableValue"; +export const ErrorValue = Symbol("ErrorBoundary:Error"); export class ErrorBoundary { + private _error?: Error; + constructor(private readonly errorCallback: (Error) => void) {} - try(callback: () => T): T | undefined; - try(callback: () => Promise): Promise | undefined { + /** + * Executes callback() and then runs errorCallback() on error. + * This will never throw but instead return `errorValue` if an error occured. + */ + try(callback: () => T): T | typeof ErrorValue; + try(callback: () => Promise): Promise | typeof ErrorValue { try { - let result: T | Promise = callback(); + let result: T | Promise = callback(); if (result instanceof Promise) { result = result.catch(err => { + this._error = err; this.errorCallback(err); - return undefined; + return ErrorValue; }); } return result; } catch (err) { + this._error = err; this.errorCallback(err); - return undefined; + return ErrorValue; } } + + get error(): Error | undefined { + return this._error; + } } + +export function tests() { + return { + "catches sync error": assert => { + let emitted = false; + const boundary = new ErrorBoundary(() => emitted = true); + const result = boundary.try(() => { + throw new Error("fail!"); + }); + assert(emitted); + assert.strictEqual(result, ErrorValue); + }, + "return value of callback is forwarded": assert => { + let emitted = false; + const boundary = new ErrorBoundary(() => emitted = true); + const result = boundary.try(() => { + return "hello"; + }); + assert(!emitted); + assert.strictEqual(result, "hello"); + }, + "catches async error": async assert => { + let emitted = false; + const boundary = new ErrorBoundary(() => emitted = true); + const result = await boundary.try(async () => { + throw new Error("fail!"); + }); + assert(emitted); + assert.strictEqual(result, ErrorValue); + } + } +} \ No newline at end of file From 47a8b4d72805574f8d12c3de80b2ec298e56d71d Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 6 Jan 2023 17:01:57 +0100 Subject: [PATCH 257/435] WIP --- src/utils/ErrorBoundary.ts | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 src/utils/ErrorBoundary.ts diff --git a/src/utils/ErrorBoundary.ts b/src/utils/ErrorBoundary.ts new file mode 100644 index 00000000..e9f297c5 --- /dev/null +++ b/src/utils/ErrorBoundary.ts @@ -0,0 +1,38 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {ObservableValue, BaseObservableValue} from "../observable/ObservableValue"; + +export class ErrorBoundary { + constructor(private readonly errorCallback: (Error) => void) {} + + try(callback: () => T): T | undefined; + try(callback: () => Promise): Promise | undefined { + try { + let result: T | Promise = callback(); + if (result instanceof Promise) { + result = result.catch(err => { + this.errorCallback(err); + return undefined; + }); + } + return result; + } catch (err) { + this.errorCallback(err); + return undefined; + } + } +} From 50f46a21bd6c0583b70a08a8e5d95b40289ad71a Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 9 Jan 2023 10:36:28 +0100 Subject: [PATCH 258/435] finish ErrorBoundary --- src/utils/ErrorBoundary.ts | 56 ++++++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 6 deletions(-) diff --git a/src/utils/ErrorBoundary.ts b/src/utils/ErrorBoundary.ts index e9f297c5..520d2a35 100644 --- a/src/utils/ErrorBoundary.ts +++ b/src/utils/ErrorBoundary.ts @@ -14,25 +14,69 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {ObservableValue, BaseObservableValue} from "../observable/ObservableValue"; +export const ErrorValue = Symbol("ErrorBoundary:Error"); export class ErrorBoundary { + private _error?: Error; + constructor(private readonly errorCallback: (Error) => void) {} - try(callback: () => T): T | undefined; - try(callback: () => Promise): Promise | undefined { + /** + * Executes callback() and then runs errorCallback() on error. + * This will never throw but instead return `errorValue` if an error occured. + */ + try(callback: () => T): T | typeof ErrorValue; + try(callback: () => Promise): Promise | typeof ErrorValue { try { - let result: T | Promise = callback(); + let result: T | Promise = callback(); if (result instanceof Promise) { result = result.catch(err => { + this._error = err; this.errorCallback(err); - return undefined; + return ErrorValue; }); } return result; } catch (err) { + this._error = err; this.errorCallback(err); - return undefined; + return ErrorValue; } } + + get error(): Error | undefined { + return this._error; + } } + +export function tests() { + return { + "catches sync error": assert => { + let emitted = false; + const boundary = new ErrorBoundary(() => emitted = true); + const result = boundary.try(() => { + throw new Error("fail!"); + }); + assert(emitted); + assert.strictEqual(result, ErrorValue); + }, + "return value of callback is forwarded": assert => { + let emitted = false; + const boundary = new ErrorBoundary(() => emitted = true); + const result = boundary.try(() => { + return "hello"; + }); + assert(!emitted); + assert.strictEqual(result, "hello"); + }, + "catches async error": async assert => { + let emitted = false; + const boundary = new ErrorBoundary(() => emitted = true); + const result = await boundary.try(async () => { + throw new Error("fail!"); + }); + assert(emitted); + assert.strictEqual(result, ErrorValue); + } + } +} \ No newline at end of file From 3706ff4f35cdc1c03dca826276be083b9e4a9d7b Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Mon, 9 Jan 2023 18:49:03 +0000 Subject: [PATCH 259/435] Docker builds are currently hosted on GHCR not GitLab --- doc/docker.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/docker.md b/doc/docker.md index 910938f0..6b138dab 100644 --- a/doc/docker.md +++ b/doc/docker.md @@ -42,8 +42,8 @@ docker build -t hydrogen . Or, pull the docker image from GitLab: ``` -docker pull registry.gitlab.com/jcgruenhage/hydrogen-web -docker tag registry.gitlab.com/jcgruenhage/hydrogen-web hydrogen +docker pull ghcr.io/vector-im/hydrogen-web +docker tag ghcr.io/vector-im/hydrogen-web hydrogen ``` ### Start container image From 21d0c4d9b0e0baf0b78c0547cc99e58469851bf1 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Mon, 9 Jan 2023 18:56:18 +0000 Subject: [PATCH 260/435] Make consistent --- doc/docker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/docker.md b/doc/docker.md index 6b138dab..f1eeb596 100644 --- a/doc/docker.md +++ b/doc/docker.md @@ -39,7 +39,7 @@ In this repository, create a Docker image: docker build -t hydrogen . ``` -Or, pull the docker image from GitLab: +Or, pull the docker image from GitHub Container Registry: ``` docker pull ghcr.io/vector-im/hydrogen-web From df6474b63754ca6d579d41cb00d743481a87cdcc Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Wed, 11 Jan 2023 18:06:25 +0000 Subject: [PATCH 261/435] Allow config override in docker image --- Dockerfile | 5 +++++ doc/docker.md | 24 ++++++++++++++++++++++++ docker/dynamic-config.sh | 8 ++++++++ 3 files changed, 37 insertions(+) create mode 100755 docker/dynamic-config.sh diff --git a/Dockerfile b/Dockerfile index f9e32313..07153148 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,4 +6,9 @@ RUN yarn install \ && yarn build FROM docker.io/nginx:alpine + +# Copy the dynamic config script +COPY ./docker/dynamic-config.sh /docker-entrypoint.d/99-dynamic-config.sh + +# Copy the built app from the first build stage COPY --from=builder /app/target /usr/share/nginx/html diff --git a/doc/docker.md b/doc/docker.md index 910938f0..fc934999 100644 --- a/doc/docker.md +++ b/doc/docker.md @@ -56,3 +56,27 @@ docker run \ --publish 80:80 \ hydrogen ``` + +You can override the default `config.json` using the `CONFIG_OVERRIDE` environment variable. For example to specify a different Homeserver and : + +``` +docker run \ + --name hydrogen \ + --publish 80:80 \ + --env CONFIG_OVERRIDE='{ + "push": { + "appId": "io.element.hydrogen.web", + "gatewayUrl": "https://matrix.org", + "applicationServerKey": "BC-gpSdVHEXhvHSHS0AzzWrQoukv2BE7KzpoPO_FfPacqOo3l1pdqz7rSgmB04pZCWaHPz7XRe6fjLaC-WPDopM" + }, + "defaultHomeServer": "https://fosdem.org", + "themeManifests": [ + "assets/theme-element.json" + ], + "defaultTheme": { + "light": "element-light", + "dark": "element-dark" + } +}' \ + hydrogen +``` diff --git a/docker/dynamic-config.sh b/docker/dynamic-config.sh new file mode 100755 index 00000000..952cb969 --- /dev/null +++ b/docker/dynamic-config.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +set -eux + +# Use config override environment variable if set +if [ -n "${CONFIG_OVERRIDE:-}" ]; then + echo "$CONFIG_OVERRIDE" > /usr/share/nginx/html/config.json +fi From 6fa73ec21496d6edee1189366f2365cc4cd1e1ec Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Wed, 11 Jan 2023 18:55:28 +0000 Subject: [PATCH 262/435] Install npm deps in separate docker layer --- Dockerfile | 11 ++++++++--- Dockerfile-dev | 7 ++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index f9e32313..b0ec5128 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,14 @@ FROM docker.io/node:alpine as builder RUN apk add --no-cache git python3 build-base -COPY . /app + WORKDIR /app -RUN yarn install \ - && yarn build + +# Copy package.json and yarn.lock and install dependencies first to speed up subsequent builds +COPY package.json yarn.lock /app/ +RUN yarn install + +COPY . /app +RUN yarn build FROM docker.io/nginx:alpine COPY --from=builder /app/target /usr/share/nginx/html diff --git a/Dockerfile-dev b/Dockerfile-dev index 08dd9abd..7212a4ae 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -1,7 +1,12 @@ FROM docker.io/node:alpine RUN apk add --no-cache git python3 build-base -COPY . /code + WORKDIR /code + +# Copy package.json and yarn.lock and install dependencies first to speed up subsequent builds +COPY package.json yarn.lock /code/ RUN yarn install + +COPY . /code EXPOSE 3000 ENTRYPOINT ["yarn", "start"] From f6c94ecc5aa514d5d1d42557a36e97579daebb2a Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Mon, 16 Jan 2023 13:41:47 +0530 Subject: [PATCH 263/435] WIP --- src/domain/session/room/CallViewModel.ts | 2 +- src/matrix/calls/LocalMedia.ts | 31 +++++++++++++----------- src/matrix/calls/PeerCall.ts | 27 ++------------------- src/matrix/calls/common.ts | 25 +++++++++++++++++++ src/matrix/calls/group/GroupCall.ts | 15 +++++++++++- 5 files changed, 59 insertions(+), 41 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 9c0eae9a..37f30840 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -136,7 +136,7 @@ class OwnMemberViewModel extends ViewModel implements IStreamViewModel } get stream(): Stream | undefined { - return this.call.localMedia?.userMediaPreview; + return this.call.localPreviewMedia?.userMedia; } private get call(): GroupCall { diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 25d6862f..7a673ba9 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -20,24 +20,12 @@ import {SDPStreamMetadata} from "./callEventTypes"; import {getStreamVideoTrack, getStreamAudioTrack} from "./common"; export class LocalMedia { - // the userMedia stream without audio, to play in the UI - // without our own audio being played back to us - public readonly userMediaPreview?: Stream; constructor( public readonly userMedia?: Stream, public readonly screenShare?: Stream, public readonly dataChannelOptions?: RTCDataChannelInit, - ) { - if (userMedia && userMedia.getVideoTracks().length > 0) { - this.userMediaPreview = userMedia.clone(); - const audioTrack = getStreamAudioTrack(this.userMediaPreview); - if (audioTrack) { - audioTrack.stop(); - this.userMediaPreview.removeTrack(audioTrack); - } - } - } + ) {} withUserMedia(stream: Stream) { return new LocalMedia(stream, this.screenShare, this.dataChannelOptions); @@ -51,6 +39,22 @@ export class LocalMedia { return new LocalMedia(this.userMedia, this.screenShare, options); } + /** + * Create an instance of LocalMedia without audio track (for user preview) + */ + asPreview(): LocalMedia { + const media = new LocalMedia(this.userMedia, this.screenShare, this.dataChannelOptions); + const userMedia = media.userMedia; + if (userMedia && userMedia.getVideoTracks().length > 0) { + const audioTrack = getStreamAudioTrack(userMedia); + if (audioTrack) { + audioTrack.stop(); + userMedia.removeTrack(audioTrack); + } + } + return media; + } + /** @internal */ replaceClone(oldClone: LocalMedia | undefined, oldOriginal: LocalMedia | undefined): LocalMedia { const cloneOrAdoptStream = (oldOriginalStream: Stream | undefined, oldCloneStream: Stream | undefined, newStream: Stream | undefined): Stream | undefined => { @@ -76,7 +80,6 @@ export class LocalMedia { dispose() { getStreamAudioTrack(this.userMedia)?.stop(); getStreamVideoTrack(this.userMedia)?.stop(); - getStreamVideoTrack(this.userMediaPreview)?.stop(); getStreamVideoTrack(this.screenShare)?.stop(); } } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index a64a1524..ae200ea2 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -20,7 +20,7 @@ import {recursivelyAssign} from "../../utils/recursivelyAssign"; import {Disposables, Disposable, IDisposable} from "../../utils/Disposables"; import {WebRTC, PeerConnection, Transceiver, TransceiverDirection, Sender, Receiver, PeerConnectionEventMap} from "../../platform/types/WebRTC"; import {MediaDevices, Track, TrackKind, Stream, StreamTrackEvent} from "../../platform/types/MediaDevices"; -import {getStreamVideoTrack, getStreamAudioTrack, MuteSettings} from "./common"; +import {getStreamVideoTrack, getStreamAudioTrack, MuteSettings, mute} from "./common"; import { SDPStreamMetadataKey, SDPStreamMetadataPurpose, @@ -266,14 +266,7 @@ export class PeerCall implements IDisposable { log.set("microphoneMuted", localMuteSettings.microphone); if (this.localMedia) { - const userMediaAudio = getStreamAudioTrack(this.localMedia.userMedia); - if (userMediaAudio) { - this.muteTrack(userMediaAudio, this.localMuteSettings.microphone, log); - } - const userMediaVideo = getStreamVideoTrack(this.localMedia.userMedia); - if (userMediaVideo) { - this.muteTrack(userMediaVideo, this.localMuteSettings.camera, log); - } + mute(this.localMedia, localMuteSettings, log); const content: MCallSDPStreamMetadataChanged = { call_id: this.callId, version: 1, @@ -290,22 +283,6 @@ export class PeerCall implements IDisposable { }); } - private muteTrack(track: Track, muted: boolean, log: ILogItem): void { - log.wrap({l: "track", kind: track.kind, id: track.id}, log => { - const enabled = !muted; - log.set("enabled", enabled); - const transceiver = this.findTransceiverForTrack(track); - if (transceiver) { - if (transceiver.sender.track) { - transceiver.sender.track.enabled = enabled; - } - log.set("fromDirection", transceiver.direction); - // enableSenderOnTransceiver(transceiver, enabled); - log.set("toDirection", transceiver.direction); - } - }); - } - private async _hangup(errorCode: CallErrorCode, log: ILogItem): Promise { if (this._state === CallState.Ended) { return; diff --git a/src/matrix/calls/common.ts b/src/matrix/calls/common.ts index 015e0df8..b99ed303 100644 --- a/src/matrix/calls/common.ts +++ b/src/matrix/calls/common.ts @@ -14,7 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */ +import {ILogItem} from "../../logging/types"; import type {Track, Stream} from "../../platform/types/MediaDevices"; +import {LocalMedia} from "./LocalMedia"; export function getStreamAudioTrack(stream: Stream | undefined): Track | undefined { return stream?.getAudioTracks()[0]; @@ -24,6 +26,29 @@ export function getStreamVideoTrack(stream: Stream | undefined): Track | undefin return stream?.getVideoTracks()[0]; } +export function mute(localMedia: LocalMedia, localMuteSettings: MuteSettings, log: ILogItem) { + return log.wrap("setMuted", log => { + log.set("cameraMuted", localMuteSettings.camera); + log.set("microphoneMuted", localMuteSettings.microphone); + + // Mute audio + const userMediaAudio = getStreamAudioTrack(localMedia.userMedia); + if (userMediaAudio) { + const enabled = !localMuteSettings.microphone; + log.set("microphone enabled", enabled); + userMediaAudio.enabled = enabled; + } + + // Mute video + const userMediaVideo = getStreamVideoTrack(localMedia.userMedia); + if (userMediaVideo) { + const enabled = !localMuteSettings.camera; + log.set("camera enabled", enabled); + userMediaVideo.enabled = enabled; + } + }); +} + export class MuteSettings { constructor ( private readonly isMicrophoneMuted: boolean = false, diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index d65ea838..79f0e66c 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -17,7 +17,7 @@ limitations under the License. import {ObservableMap} from "../../../observable/map/ObservableMap"; import {Member, isMemberExpired, memberExpiresAt} from "./Member"; import {LocalMedia} from "../LocalMedia"; -import {MuteSettings, CALL_LOG_TYPE, CALL_MEMBER_VALIDITY_PERIOD_MS} from "../common"; +import {MuteSettings, CALL_LOG_TYPE, CALL_MEMBER_VALIDITY_PERIOD_MS, mute} from "../common"; import {MemberChange, RoomMember} from "../../room/members/RoomMember"; import {EventEmitter} from "../../../utils/EventEmitter"; import {EventType, CallIntent} from "../callEventTypes"; @@ -72,12 +72,14 @@ class JoinedData { public readonly logItem: ILogItem, public readonly membersLogItem: ILogItem, public localMedia: LocalMedia, + public localPreviewMedia: LocalMedia, public localMuteSettings: MuteSettings, public readonly turnServer: BaseObservableValue ) {} dispose() { this.localMedia.dispose(); + this.localPreviewMedia.dispose(); this.logItem.finish(); this.renewMembershipTimeout?.dispose(); } @@ -125,6 +127,7 @@ export class GroupCall extends EventEmitter<{change: never}> { } get localMedia(): LocalMedia | undefined { return this.joinedData?.localMedia; } + get localPreviewMedia(): LocalMedia | undefined { return this.joinedData?.localPreviewMedia; } get members(): BaseObservableMap { return this._members; } get isTerminated(): boolean { @@ -165,10 +168,12 @@ export class GroupCall extends EventEmitter<{change: never}> { const membersLogItem = logItem.child("member connections"); const localMuteSettings = new MuteSettings(); localMuteSettings.updateTrackInfo(localMedia.userMedia); + const localPreviewMedia = localMedia.asPreview(); const joinedData = new JoinedData( logItem, membersLogItem, localMedia, + localPreviewMedia, localMuteSettings, turnServer ); @@ -219,6 +224,14 @@ export class GroupCall extends EventEmitter<{change: never}> { muteSettings.updateTrackInfo(joinedData.localMedia.userMedia); joinedData.localMuteSettings = muteSettings; if (!prevMuteSettings.equals(muteSettings)) { + // Mute our copies of LocalMedias; + // otherwise the camera lights will still be on. + if (this.localPreviewMedia) { + mute(this.localPreviewMedia, muteSettings, this.joinedData!.logItem); + } + if (this.localMedia) { + mute(this.localMedia, muteSettings, this.joinedData!.logItem); + } await Promise.all(Array.from(this._members.values()).map(m => { return m.setMuted(joinedData.localMuteSettings); })); From c064336e359a2d271a30a6af3aa73e8eeb83c035 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Mon, 16 Jan 2023 15:28:50 +0530 Subject: [PATCH 264/435] Create localPreviewMedia in GroupCall.setMedia --- src/matrix/calls/group/GroupCall.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 79f0e66c..f3f37a2b 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -200,6 +200,8 @@ export class GroupCall extends EventEmitter<{change: never}> { if ((this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) && this.joinedData) { const oldMedia = this.joinedData.localMedia; this.joinedData.localMedia = localMedia; + this.localPreviewMedia?.dispose(); + this.joinedData.localPreviewMedia = localMedia.asPreview(); // reflect the fact we gained or lost local tracks in the local mute settings // and update the track info so PeerCall can use it to send up to date metadata, this.joinedData.localMuteSettings.updateTrackInfo(localMedia.userMedia); From ce5c5779bdf59f4ab13182c7c7b7b81b4d6af556 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 17 Jan 2023 14:24:20 +0530 Subject: [PATCH 265/435] Set mute -> mute --- src/matrix/calls/common.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/common.ts b/src/matrix/calls/common.ts index b99ed303..5036eae4 100644 --- a/src/matrix/calls/common.ts +++ b/src/matrix/calls/common.ts @@ -27,7 +27,7 @@ export function getStreamVideoTrack(stream: Stream | undefined): Track | undefin } export function mute(localMedia: LocalMedia, localMuteSettings: MuteSettings, log: ILogItem) { - return log.wrap("setMuted", log => { + return log.wrap("mute", log => { log.set("cameraMuted", localMuteSettings.camera); log.set("microphoneMuted", localMuteSettings.microphone); From 1cd6540476dfa80b9b6e53d7ed83213d4079fe64 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 17 Jan 2023 14:32:02 +0530 Subject: [PATCH 266/435] Don't use getter here --- src/matrix/calls/group/GroupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index f3f37a2b..cda5a40f 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -200,7 +200,7 @@ export class GroupCall extends EventEmitter<{change: never}> { if ((this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) && this.joinedData) { const oldMedia = this.joinedData.localMedia; this.joinedData.localMedia = localMedia; - this.localPreviewMedia?.dispose(); + this.joinedData.localPreviewMedia?.dispose(); this.joinedData.localPreviewMedia = localMedia.asPreview(); // reflect the fact we gained or lost local tracks in the local mute settings // and update the track info so PeerCall can use it to send up to date metadata, From 7f004193d30de325e01d26047b91b7f6254e4199 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 18 Jan 2023 08:33:02 +0100 Subject: [PATCH 267/435] add user agent in log export --- src/logging/IDBLogger.ts | 1 + src/platform/web/Platform.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/logging/IDBLogger.ts b/src/logging/IDBLogger.ts index ab9474b0..863ff5a3 100644 --- a/src/logging/IDBLogger.ts +++ b/src/logging/IDBLogger.ts @@ -201,6 +201,7 @@ class IDBLogExport implements ILogExport { const log = { formatVersion: 1, appVersion: this._platform.updateService?.version, + platform: this._platform.description, items: this._items.map(i => JSON.parse(i.json)) }; const json = JSON.stringify(log); diff --git a/src/platform/web/Platform.js b/src/platform/web/Platform.js index 0d95e585..a138e70c 100644 --- a/src/platform/web/Platform.js +++ b/src/platform/web/Platform.js @@ -366,7 +366,7 @@ export class Platform { } get description() { - return navigator.userAgent ?? ""; + return "web-" + (navigator.userAgent ?? ""); } dispose() { From 5a124809afec177b6828469f5b77c9fd9c254de3 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 17 Jan 2023 17:19:12 +0100 Subject: [PATCH 268/435] write docs how updates work --- doc/updates.md | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 doc/updates.md diff --git a/doc/updates.md b/doc/updates.md new file mode 100644 index 00000000..6522fd10 --- /dev/null +++ b/doc/updates.md @@ -0,0 +1,58 @@ +# Updates + +How updates flow from the model to the view model to the UI. + +## EventEmitter, single values + +When interested in updates from a single object, chances are it inherits from `EventEmitter` and it supports a `change` event. + +`ViewModel` by default follows this pattern, but it can be overwritten, see Collections below. + +### Parameters + +Often a `parameters` or `params` argument is passed with the name of the field who's value has now changed. This parameter is currently only sometimes used, e.g. when it is too complicated or costly to check every possible field. An example of this is `TilesListView.onUpdate` to see if the `shape` property of a tile changed and hence the view needs to be recreated. Other than that, bindings in the web UI just reevaluate all bindings when receiving an update. This is a soft convention that could probably be more standardized, and it's not always clear what to pass (e.g. when multiple fields are being updated). + +Another reason to keep this convention around is that if one day we decide to add support for a different platform with a different UI, it may not be feasible to reevaluate all data-bindings in the UI for a given view model when receiving an update. + +## Collections + +As an optimization, Hydrogen uses a pattern to let updates flow over an observable collection where this makes sense. There is an `update` event for this in both `ObservableMap` and `ObservableList`. This prevents having to listen for updates on each individual item in large collections. The `update` event uses the same `params` argument as explained above. + +Some values like `BaseRoom` emit both with a `change` event on the event emitter and also over the collection. This way consumers can use what fits best for their case: the left panel can listen for updates on the room over the collection to power the room list, and the room view model can listen to the event emitter to get updates from the current room only. + +### MappedMap and mapping models to `ViewModel`s + +This can get a little complicated when using `MappedMap`, e.g. when mapping a model from `matrix/` +to a view model in `domain/`. Often, view models will want to emit updates _spontanously_, +e.g. without a prior update being sent from the lower-lying model. An example would be to change the value of a field after the view has called a method on the view model. +To support this pattern while having updates still flow over the collection requires some extra work; +`ViewModel` has a `emitChange` option which you can pass in to override +what `ViewModel.emitChange` does (by default it emits the `change` event on the view model). +`MappedMap` passes a callback to emit an update over the collection to the mapper function. +You can pass this callback as the `emitChange` option and updates will now flow over the collection. + +`MappedMap` also accepts an updater function, which you can use to make the view model respond to updates +from the lower-lying model. + +Here is an example: + +```ts +const viewModels = someCollection.mapValues( + (model, emitChange) => new SomeViewModel(this.childOptions({ + model, + // will make ViewModel.emitChange go over + // the collection rather than emit a "change" event + emitChange, + })), + // an update came in from the model, let the vm know + (vm: SomeViewModel) => vm.onUpdate(), + ); +``` + +### `ListView` & the `parentProvidesUpdates` flag. + +`ObservableList` is always rendered in the UI using `ListView`. When receiving an update over the collection, it will find the child view for the given index and call `update(params)` on it. Views will typically need to be told whether they should listen to the `change` event in their view model or rather wait for their `update()` method to be called by their parent view, `ListView`. That's why the `mount(args)` method on a view supports a `parentProvidesUpdates` flag. If `true`, the view should not subscribe to its view model, but rather updates the DOM when its `update()` method is called. Also see `BaseUpdateView` and `TemplateView` for how this is implemented in the child view. + +## `ObservableValue` + +When some method wants to return an object that can be updated, often an `ObservableValue` is used rather than an `EventEmitter`. It's not 100% clear cut when to use the former or the latter, but `ObservableValue` is often used when the returned value in it's entirety will change rather than just a property on it. `ObservableValue` also has some nice facilities like lazy evaluation when subscribed to and the `waitFor` method to work with promises. \ No newline at end of file From 8b0b81368008e9a64c43d760f61bd4589df0e617 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 17 Jan 2023 17:28:29 +0100 Subject: [PATCH 269/435] organize documention and remove obsolete documents --- doc/FAQ.md => FAQ.md | 0 doc/GOAL.md | 8 -- doc/TODO.md | 77 --------------- doc/api.md | 90 ------------------ doc/{ => architecture}/SKINNING.md | 0 doc/{ => architecture}/THEMING.md | 0 doc/{ => architecture}/UI/index.md | 0 .../UI/render-dom-elements.md | 0 doc/{ => architecture}/UI/ui.md | 0 doc/{ => architecture}/architecture.md | 0 .../images/coloring-process.png | Bin .../images/svg-icon-example.png | Bin .../images/theming-architecture.png | Bin .../persisted-network-calls.md | 0 doc/{ => architecture}/sync-updates.md | 0 doc/{ => architecture}/updates.md | 0 .../CATCHUP-BACKFILL.md | 0 doc/{ => implementation planning}/CSS.md | 0 .../DESIGN.md | 0 .../E2EE.md | 0 .../FRAGMENTS.md | 0 .../LOCAL-ECHO-STATE.md | 0 .../LOGIN.md | 0 .../MEMBERS.md | 0 .../PENDING_REPLIES.md | 0 .../PUSH.md | 0 .../QUESTIONS.md | 0 .../READ-RECEIPTS.md | 0 .../RECONNECTING.md | 0 .../RELATIONS.md | 0 doc/{ => implementation planning}/RELEASE.md | 0 .../REPLIES.md | 0 .../ROOM-VERSIONS.md | 0 .../SDK.md | 0 doc/{ => implementation planning}/SENDING.md | 0 .../SSO.md | 0 .../VIEW-UPDATES.md | 0 .../background-tasks.md | 0 .../html-messages.md | 0 doc/{ => implementation planning}/invites.md | 0 .../room-types.ts | 0 .../session-container.md | 0 .../timeline-member.md | 0 doc/{ => problem solving}/IMPORT-ISSUES.md | 0 doc/{ => problem solving}/INDEXEDDB.md | 0 .../domexception_mapping.md | 0 doc/viewhierarchy.md | 21 ---- 47 files changed, 196 deletions(-) rename doc/FAQ.md => FAQ.md (100%) delete mode 100644 doc/GOAL.md delete mode 100644 doc/TODO.md delete mode 100644 doc/api.md rename doc/{ => architecture}/SKINNING.md (100%) rename doc/{ => architecture}/THEMING.md (100%) rename doc/{ => architecture}/UI/index.md (100%) rename doc/{ => architecture}/UI/render-dom-elements.md (100%) rename doc/{ => architecture}/UI/ui.md (100%) rename doc/{ => architecture}/architecture.md (100%) rename doc/{ => architecture}/images/coloring-process.png (100%) rename doc/{ => architecture}/images/svg-icon-example.png (100%) rename doc/{ => architecture}/images/theming-architecture.png (100%) rename doc/{ => architecture}/persisted-network-calls.md (100%) rename doc/{ => architecture}/sync-updates.md (100%) rename doc/{ => architecture}/updates.md (100%) rename doc/{impl-thoughts => implementation planning}/CATCHUP-BACKFILL.md (100%) rename doc/{ => implementation planning}/CSS.md (100%) rename doc/{impl-thoughts => implementation planning}/DESIGN.md (100%) rename doc/{impl-thoughts => implementation planning}/E2EE.md (100%) rename doc/{impl-thoughts => implementation planning}/FRAGMENTS.md (100%) rename doc/{impl-thoughts => implementation planning}/LOCAL-ECHO-STATE.md (100%) rename doc/{impl-thoughts => implementation planning}/LOGIN.md (100%) rename doc/{impl-thoughts => implementation planning}/MEMBERS.md (100%) rename doc/{impl-thoughts => implementation planning}/PENDING_REPLIES.md (100%) rename doc/{impl-thoughts => implementation planning}/PUSH.md (100%) rename doc/{ => implementation planning}/QUESTIONS.md (100%) rename doc/{impl-thoughts => implementation planning}/READ-RECEIPTS.md (100%) rename doc/{impl-thoughts => implementation planning}/RECONNECTING.md (100%) rename doc/{impl-thoughts => implementation planning}/RELATIONS.md (100%) rename doc/{ => implementation planning}/RELEASE.md (100%) rename doc/{impl-thoughts => implementation planning}/REPLIES.md (100%) rename doc/{impl-thoughts => implementation planning}/ROOM-VERSIONS.md (100%) rename doc/{impl-thoughts => implementation planning}/SDK.md (100%) rename doc/{ => implementation planning}/SENDING.md (100%) rename doc/{impl-thoughts => implementation planning}/SSO.md (100%) rename doc/{impl-thoughts => implementation planning}/VIEW-UPDATES.md (100%) rename doc/{impl-thoughts => implementation planning}/background-tasks.md (100%) rename doc/{impl-thoughts => implementation planning}/html-messages.md (100%) rename doc/{ => implementation planning}/invites.md (100%) rename doc/{impl-thoughts => implementation planning}/room-types.ts (100%) rename doc/{impl-thoughts => implementation planning}/session-container.md (100%) rename doc/{impl-thoughts => implementation planning}/timeline-member.md (100%) rename doc/{ => problem solving}/IMPORT-ISSUES.md (100%) rename doc/{ => problem solving}/INDEXEDDB.md (100%) rename doc/{ => problem solving}/domexception_mapping.md (100%) delete mode 100644 doc/viewhierarchy.md diff --git a/doc/FAQ.md b/FAQ.md similarity index 100% rename from doc/FAQ.md rename to FAQ.md diff --git a/doc/GOAL.md b/doc/GOAL.md deleted file mode 100644 index 3883cf27..00000000 --- a/doc/GOAL.md +++ /dev/null @@ -1,8 +0,0 @@ -goal: - -write client that works on lumia 950 phone, so I can use matrix on my phone. - -try approach offline to indexeddb. go low-memory, and test the performance of storing every event individually in indexeddb. - -try to use little bandwidth, mainly by being an offline application and storing all requested data in indexeddb. -be as functional as possible while offline diff --git a/doc/TODO.md b/doc/TODO.md deleted file mode 100644 index 7d16400d..00000000 --- a/doc/TODO.md +++ /dev/null @@ -1,77 +0,0 @@ -# Minimal thing to get working - - - DONE: finish summary store - - DONE: move "sdk" bits over to "matrix" directory - - DONE: add eventemitter - - DONE: make sync work - - DONE: store summaries - - DONE: setup editorconfig - - DONE: setup linting (also in editor) - - DONE: store timeline - - DONE: store state - - DONE: make summary work better (name and joined/inviteCount doesn't seem to work well) - - DONE: timeline doesn't seem to recover it's key well upon loading, the query in load seems to never yield an event in the persister - - DONE: map DOMException to something better - - it's pretty opaque now when something idb related fails. DOMException has these fields: - code: 0 - message: "Key already exists in the object store." - name: "ConstraintError" - - DONE: emit events so we can start showing something on the screen maybe? - - DONE: move session._rooms over to Map, so we can iterate over it, ... - - DONE: build a very basic interface with - - DONE: a start/stop sync button - - DONE: a room list sorted alphabetically - - DONE: do some preprocessing on sync response which can then be used by persister, summary, timeline - - DONE: support timeline - - DONE: clicking on a room list, you see messages (userId -> body) - - DONE: style minimal UI - - DONE: implement gap filling and fragments (see FRAGMENTS.md) - - DONE: allow collection items (especially tiles) to self-update - - improve fragmentidcomparer::add - - DONE: better UI - - fix MappedMap update mechanism - - see if in BaseObservableMap we need to change ...params - - DONE: put sync button and status label inside SessionView - - fix some errors: - - find out if `(this._emitCollectionUpdate)(this)` is different than `this._emitCollectionUpdate(this)` - - got "database tried to mutate when not allowed" or something error as well - - find out why when RoomPersister.(\_createGapEntry/\_createEventEntry) we remove .buffer the transaction fails (good), but upon fixing and refreshing is missing a message! syncToken should not be saved, so why isn't this again in the sync response and now the txn does succeed? - - DONE: take access token out of IDB? this way it can be stored in a more secure thing for non-web clients, together wit encryption key for olm sessions ... ? like macos keychain, gnome keyring, ... maybe using https://www.npmjs.com/package/keytar - - DONE: experiment with using just a normal array with 2 numbers for sortkeys, to work in Edge as well. - - DONE: send messages - - DONE: fill gaps with call to /messages - - - DONE: build script - - DONE: take dev index.html, run some dom modifications to change script tag with `parse5`. - - DONE: create js bundle, rollup - - DONE: create css bundle, postcss, probably just need postcss-import for now, but good to have more options - - DONE: put all in /target - - have option to run it locally to test - - - deploy script - - upload /target to github pages - - - DONE: offline available - - both offline mechanisms have (filelist, version) as input for their template: - - create appcache manifest with (index.html, brawl.js, brawl.css) and print version number in it - - create service worker wit file list to cache (at top const files = "%%FILES_ARRAY%%", version = "%%VERSION%%") - - write web manifest - - DONE: delete and clear sessions from picker - - option to close current session and go back to picker - - - accept invite - - member list - - e2e encryption - - sync retry strategy - - instead of stopping sync on fetch error, show spinner and status and have auto retry strategy - - - create room - - join room - - leave room - - unread rooms, badge count, sort rooms by activity - - - DONE: create sync filter - - DONE: lazy loading members - - decide denormalized data in summary vs reading from multiple stores PER room on load - - allow Room/Summary class to be subclassed and store additional data? - - store account data, support read markers diff --git a/doc/api.md b/doc/api.md deleted file mode 100644 index 89e03639..00000000 --- a/doc/api.md +++ /dev/null @@ -1,90 +0,0 @@ -Session - properties: - rooms -> Rooms - -# storage -Storage - key...() -> KeyRange - start...Txn() -> Transaction -Transaction - store(name) -> ObjectStore - finish() - rollback() -ObjectStore : QueryTarget - index(name) -Index : QueryTarget - - -Rooms: EventEmitter, Iterator - get(id) -> RoomSummary ? -InternalRoom: EventEmitter - applySync(roomResponse, membership, txn) - - this method updates the room summary - - persists the room summary - - persists room state & timeline with RoomPersister - - updates the OpenRoom if present - - - applyAndPersistSync(roomResponse, membership, txn) { - this._summary.applySync(roomResponse, membership); - this._summary.persist(txn); - this._roomPersister.persist(roomResponse, membership, txn); - if (this._openRoom) { - this._openRoom.applySync(roomResponse); - } - } - -RoomPersister - RoomPersister (persists timeline and room state) - RoomSummary (persists room summary) -RoomSummary : EventEmitter - methods: - async open() - id - name - lastMessage - unreadCount - mentionCount - isEncrypted - isDirectMessage - membership - - should this have a custom reducer for custom fields? - - events - propChange(fieldName) - -OpenRoom : EventEmitter - properties: - timeline - events: - - -RoomState: EventEmitter - [room_id, event_type, state_key] -> [sort_key, event] -Timeline: EventEmitter - // should have a cache of recently lookup sender members? - // can we disambiguate members like this? - methods: - lastEvents(amount) - firstEvents(amount) - eventsAfter(sortKey, amount) - eventsBefore(sortKey, amount) - events: - eventsApppended - -RoomMembers : EventEmitter, Iterator - // no order, but need to be able to get all members somehow, needs to map to a ReactiveMap or something - events: - added(ids, values) - removed(ids, values) - changed(id, fieldName) -RoomMember: EventEmitter - properties: - id - name - powerLevel - membership - avatar - events: - propChange(fieldName) \ No newline at end of file diff --git a/doc/SKINNING.md b/doc/architecture/SKINNING.md similarity index 100% rename from doc/SKINNING.md rename to doc/architecture/SKINNING.md diff --git a/doc/THEMING.md b/doc/architecture/THEMING.md similarity index 100% rename from doc/THEMING.md rename to doc/architecture/THEMING.md diff --git a/doc/UI/index.md b/doc/architecture/UI/index.md similarity index 100% rename from doc/UI/index.md rename to doc/architecture/UI/index.md diff --git a/doc/UI/render-dom-elements.md b/doc/architecture/UI/render-dom-elements.md similarity index 100% rename from doc/UI/render-dom-elements.md rename to doc/architecture/UI/render-dom-elements.md diff --git a/doc/UI/ui.md b/doc/architecture/UI/ui.md similarity index 100% rename from doc/UI/ui.md rename to doc/architecture/UI/ui.md diff --git a/doc/architecture.md b/doc/architecture/architecture.md similarity index 100% rename from doc/architecture.md rename to doc/architecture/architecture.md diff --git a/doc/images/coloring-process.png b/doc/architecture/images/coloring-process.png similarity index 100% rename from doc/images/coloring-process.png rename to doc/architecture/images/coloring-process.png diff --git a/doc/images/svg-icon-example.png b/doc/architecture/images/svg-icon-example.png similarity index 100% rename from doc/images/svg-icon-example.png rename to doc/architecture/images/svg-icon-example.png diff --git a/doc/images/theming-architecture.png b/doc/architecture/images/theming-architecture.png similarity index 100% rename from doc/images/theming-architecture.png rename to doc/architecture/images/theming-architecture.png diff --git a/doc/persisted-network-calls.md b/doc/architecture/persisted-network-calls.md similarity index 100% rename from doc/persisted-network-calls.md rename to doc/architecture/persisted-network-calls.md diff --git a/doc/sync-updates.md b/doc/architecture/sync-updates.md similarity index 100% rename from doc/sync-updates.md rename to doc/architecture/sync-updates.md diff --git a/doc/updates.md b/doc/architecture/updates.md similarity index 100% rename from doc/updates.md rename to doc/architecture/updates.md diff --git a/doc/impl-thoughts/CATCHUP-BACKFILL.md b/doc/implementation planning/CATCHUP-BACKFILL.md similarity index 100% rename from doc/impl-thoughts/CATCHUP-BACKFILL.md rename to doc/implementation planning/CATCHUP-BACKFILL.md diff --git a/doc/CSS.md b/doc/implementation planning/CSS.md similarity index 100% rename from doc/CSS.md rename to doc/implementation planning/CSS.md diff --git a/doc/impl-thoughts/DESIGN.md b/doc/implementation planning/DESIGN.md similarity index 100% rename from doc/impl-thoughts/DESIGN.md rename to doc/implementation planning/DESIGN.md diff --git a/doc/impl-thoughts/E2EE.md b/doc/implementation planning/E2EE.md similarity index 100% rename from doc/impl-thoughts/E2EE.md rename to doc/implementation planning/E2EE.md diff --git a/doc/impl-thoughts/FRAGMENTS.md b/doc/implementation planning/FRAGMENTS.md similarity index 100% rename from doc/impl-thoughts/FRAGMENTS.md rename to doc/implementation planning/FRAGMENTS.md diff --git a/doc/impl-thoughts/LOCAL-ECHO-STATE.md b/doc/implementation planning/LOCAL-ECHO-STATE.md similarity index 100% rename from doc/impl-thoughts/LOCAL-ECHO-STATE.md rename to doc/implementation planning/LOCAL-ECHO-STATE.md diff --git a/doc/impl-thoughts/LOGIN.md b/doc/implementation planning/LOGIN.md similarity index 100% rename from doc/impl-thoughts/LOGIN.md rename to doc/implementation planning/LOGIN.md diff --git a/doc/impl-thoughts/MEMBERS.md b/doc/implementation planning/MEMBERS.md similarity index 100% rename from doc/impl-thoughts/MEMBERS.md rename to doc/implementation planning/MEMBERS.md diff --git a/doc/impl-thoughts/PENDING_REPLIES.md b/doc/implementation planning/PENDING_REPLIES.md similarity index 100% rename from doc/impl-thoughts/PENDING_REPLIES.md rename to doc/implementation planning/PENDING_REPLIES.md diff --git a/doc/impl-thoughts/PUSH.md b/doc/implementation planning/PUSH.md similarity index 100% rename from doc/impl-thoughts/PUSH.md rename to doc/implementation planning/PUSH.md diff --git a/doc/QUESTIONS.md b/doc/implementation planning/QUESTIONS.md similarity index 100% rename from doc/QUESTIONS.md rename to doc/implementation planning/QUESTIONS.md diff --git a/doc/impl-thoughts/READ-RECEIPTS.md b/doc/implementation planning/READ-RECEIPTS.md similarity index 100% rename from doc/impl-thoughts/READ-RECEIPTS.md rename to doc/implementation planning/READ-RECEIPTS.md diff --git a/doc/impl-thoughts/RECONNECTING.md b/doc/implementation planning/RECONNECTING.md similarity index 100% rename from doc/impl-thoughts/RECONNECTING.md rename to doc/implementation planning/RECONNECTING.md diff --git a/doc/impl-thoughts/RELATIONS.md b/doc/implementation planning/RELATIONS.md similarity index 100% rename from doc/impl-thoughts/RELATIONS.md rename to doc/implementation planning/RELATIONS.md diff --git a/doc/RELEASE.md b/doc/implementation planning/RELEASE.md similarity index 100% rename from doc/RELEASE.md rename to doc/implementation planning/RELEASE.md diff --git a/doc/impl-thoughts/REPLIES.md b/doc/implementation planning/REPLIES.md similarity index 100% rename from doc/impl-thoughts/REPLIES.md rename to doc/implementation planning/REPLIES.md diff --git a/doc/impl-thoughts/ROOM-VERSIONS.md b/doc/implementation planning/ROOM-VERSIONS.md similarity index 100% rename from doc/impl-thoughts/ROOM-VERSIONS.md rename to doc/implementation planning/ROOM-VERSIONS.md diff --git a/doc/impl-thoughts/SDK.md b/doc/implementation planning/SDK.md similarity index 100% rename from doc/impl-thoughts/SDK.md rename to doc/implementation planning/SDK.md diff --git a/doc/SENDING.md b/doc/implementation planning/SENDING.md similarity index 100% rename from doc/SENDING.md rename to doc/implementation planning/SENDING.md diff --git a/doc/impl-thoughts/SSO.md b/doc/implementation planning/SSO.md similarity index 100% rename from doc/impl-thoughts/SSO.md rename to doc/implementation planning/SSO.md diff --git a/doc/impl-thoughts/VIEW-UPDATES.md b/doc/implementation planning/VIEW-UPDATES.md similarity index 100% rename from doc/impl-thoughts/VIEW-UPDATES.md rename to doc/implementation planning/VIEW-UPDATES.md diff --git a/doc/impl-thoughts/background-tasks.md b/doc/implementation planning/background-tasks.md similarity index 100% rename from doc/impl-thoughts/background-tasks.md rename to doc/implementation planning/background-tasks.md diff --git a/doc/impl-thoughts/html-messages.md b/doc/implementation planning/html-messages.md similarity index 100% rename from doc/impl-thoughts/html-messages.md rename to doc/implementation planning/html-messages.md diff --git a/doc/invites.md b/doc/implementation planning/invites.md similarity index 100% rename from doc/invites.md rename to doc/implementation planning/invites.md diff --git a/doc/impl-thoughts/room-types.ts b/doc/implementation planning/room-types.ts similarity index 100% rename from doc/impl-thoughts/room-types.ts rename to doc/implementation planning/room-types.ts diff --git a/doc/impl-thoughts/session-container.md b/doc/implementation planning/session-container.md similarity index 100% rename from doc/impl-thoughts/session-container.md rename to doc/implementation planning/session-container.md diff --git a/doc/impl-thoughts/timeline-member.md b/doc/implementation planning/timeline-member.md similarity index 100% rename from doc/impl-thoughts/timeline-member.md rename to doc/implementation planning/timeline-member.md diff --git a/doc/IMPORT-ISSUES.md b/doc/problem solving/IMPORT-ISSUES.md similarity index 100% rename from doc/IMPORT-ISSUES.md rename to doc/problem solving/IMPORT-ISSUES.md diff --git a/doc/INDEXEDDB.md b/doc/problem solving/INDEXEDDB.md similarity index 100% rename from doc/INDEXEDDB.md rename to doc/problem solving/INDEXEDDB.md diff --git a/doc/domexception_mapping.md b/doc/problem solving/domexception_mapping.md similarity index 100% rename from doc/domexception_mapping.md rename to doc/problem solving/domexception_mapping.md diff --git a/doc/viewhierarchy.md b/doc/viewhierarchy.md deleted file mode 100644 index c4e6355a..00000000 --- a/doc/viewhierarchy.md +++ /dev/null @@ -1,21 +0,0 @@ -view hierarchy: -``` - BrawlView - SwitchView - SessionView - SyncStatusBar - ListView(left-panel) - RoomTile - SwitchView - RoomPlaceholderView - RoomView - MiddlePanel - ListView(timeline) - event tiles (see ui/session/room/timeline/) - ComposerView - RightPanel - SessionPickView - ListView - SessionPickerItemView - LoginView -``` From 13aea539fa65811952ca9085c538ae58f099e901 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 17 Jan 2023 17:30:48 +0100 Subject: [PATCH 270/435] move ts style guide to own dir --- doc/{TS-MIGRATION.md => style guide/typescript.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename doc/{TS-MIGRATION.md => style guide/typescript.md} (100%) diff --git a/doc/TS-MIGRATION.md b/doc/style guide/typescript.md similarity index 100% rename from doc/TS-MIGRATION.md rename to doc/style guide/typescript.md From 887dea528d8721e0ac8b287c7662f245ecd551ca Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 17 Jan 2023 17:31:48 +0100 Subject: [PATCH 271/435] update dirs in docs --- FAQ.md | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/FAQ.md b/FAQ.md index 0823ee08..43053c2f 100644 --- a/FAQ.md +++ b/FAQ.md @@ -36,4 +36,4 @@ Published builds can be found at https://github.com/vector-im/hydrogen-web/relea ## I want to embed Hydrogen in my website, how should I do that? -Hydrogen aims to be usable as an SDK, and while it is still early days, you can find some documentation how to do that in [SDK.md](SDK.md). +Hydrogen aims to be usable as an SDK, and while it is still early days, you can find some documentation how to do that in [SDK.md](doc/SDK.md). diff --git a/README.md b/README.md index a4c1529f..7ad012a5 100644 --- a/README.md +++ b/README.md @@ -43,4 +43,4 @@ PS: You need nodejs, running yarn on top of any other js platform is not support # FAQ -Some frequently asked questions are answered [here](doc/FAQ.md). +Some frequently asked questions are answered [here](FAQ.md). From bd648c1de33082830c9f9a5382cadc4ecb5ab341 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 19 Jan 2023 09:25:11 +0100 Subject: [PATCH 272/435] skinning support is broken, remove doc --- doc/architecture/SKINNING.md | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 doc/architecture/SKINNING.md diff --git a/doc/architecture/SKINNING.md b/doc/architecture/SKINNING.md deleted file mode 100644 index 5f1c735d..00000000 --- a/doc/architecture/SKINNING.md +++ /dev/null @@ -1,22 +0,0 @@ -# Replacing javascript files - -Any source file can be replaced at build time by mapping the path in a JSON file passed in to the build command, e.g. `yarn build --override-imports customizations.json`. The file should be written like so: - -```json -{ - "src/platform/web/ui/session/room/timeline/TextMessageView.js": "src/platform/web/ui/session/room/timeline/MyTextMessageView.js" -} -``` -The paths are relative to the location of the mapping file, but the mapping file should be in a parent directory of the files you want to replace. - -You should see a "replacing x with y" line (twice actually, for the normal and legacy build). - -# Injecting CSS - -You can override the location of the main css file with the `--override-css ` option to the build script. The default is `src/platform/web/ui/css/main.css`, which you probably want to import from your custom css file like so: - -```css -@import url('src/platform/web/ui/css/main.css'); - -/* additions */ -``` From f0d2c191849ed0154c3ac57418a624fb997d4d71 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 9 Jan 2023 13:50:03 +0100 Subject: [PATCH 273/435] allow an explicit error value again in ErrorBoundary --- src/utils/ErrorBoundary.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/utils/ErrorBoundary.ts b/src/utils/ErrorBoundary.ts index 520d2a35..d13065f9 100644 --- a/src/utils/ErrorBoundary.ts +++ b/src/utils/ErrorBoundary.ts @@ -25,22 +25,22 @@ export class ErrorBoundary { * Executes callback() and then runs errorCallback() on error. * This will never throw but instead return `errorValue` if an error occured. */ - try(callback: () => T): T | typeof ErrorValue; - try(callback: () => Promise): Promise | typeof ErrorValue { + try(callback: () => T, errorValue?: E): T | typeof errorValue; + try(callback: () => Promise, errorValue?: E): Promise | typeof errorValue { try { - let result: T | Promise = callback(); + let result: T | Promise = callback(); if (result instanceof Promise) { result = result.catch(err => { this._error = err; this.errorCallback(err); - return ErrorValue; + return errorValue; }); } return result; } catch (err) { this._error = err; this.errorCallback(err); - return ErrorValue; + return errorValue; } } @@ -56,9 +56,9 @@ export function tests() { const boundary = new ErrorBoundary(() => emitted = true); const result = boundary.try(() => { throw new Error("fail!"); - }); + }, 0); assert(emitted); - assert.strictEqual(result, ErrorValue); + assert.strictEqual(result, 0); }, "return value of callback is forwarded": assert => { let emitted = false; @@ -74,9 +74,9 @@ export function tests() { const boundary = new ErrorBoundary(() => emitted = true); const result = await boundary.try(async () => { throw new Error("fail!"); - }); + }, 0); assert(emitted); - assert.strictEqual(result, ErrorValue); + assert.strictEqual(result, 0); } } } \ No newline at end of file From b1687d71155ffaf50a4b1ed541c2bd57d7a16602 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 9 Jan 2023 13:50:27 +0100 Subject: [PATCH 274/435] introduce error boundary in call member --- src/matrix/calls/group/Member.ts | 185 +++++++++++++++++-------------- 1 file changed, 103 insertions(+), 82 deletions(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 9debef22..4ceaf46f 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -19,6 +19,7 @@ import {makeTxnId, makeId} from "../../common"; import {EventType, CallErrorCode} from "../callEventTypes"; import {formatToDeviceMessagesPayload} from "../../common"; import {sortedIndex} from "../../../utils/sortedIndex"; +import { ErrorBoundary } from "../../../utils/ErrorBoundary"; import type {MuteSettings} from "../common"; import type {Options as PeerCallOptions, RemoteMedia} from "../PeerCall"; @@ -94,6 +95,9 @@ class MemberConnection { export class Member { private connection?: MemberConnection; private expireTimeout?: Timeout; + private errorBoundary = new ErrorBoundary(err => { + this.options.emitUpdate(this, "error"); + }); constructor( public member: RoomMember, @@ -104,6 +108,10 @@ export class Member { this._renewExpireTimeout(updateMemberLog); } + get error(): Error | undefined { + return this.errorBoundary.error; + } + private _renewExpireTimeout(log: ILogItem) { this.expireTimeout?.dispose(); this.expireTimeout = undefined; @@ -166,23 +174,26 @@ export class Member { /** @internal */ connect(localMedia: LocalMedia, localMuteSettings: MuteSettings, turnServer: BaseObservableValue, memberLogItem: ILogItem): ILogItem | undefined { - if (this.connection) { - return; - } - // Safari can't send a MediaStream to multiple sources, so clone it - const connection = new MemberConnection( - localMedia.clone(), - localMuteSettings, - turnServer, - memberLogItem - ); - this.connection = connection; - let connectLogItem; - connection.logItem.wrap("connect", async log => { - connectLogItem = log; - await this.callIfNeeded(log); + return this.errorBoundary.try(() => { + if (this.connection) { + return; + } + // Safari can't send a MediaStream to multiple sources, so clone it + const connection = new MemberConnection( + localMedia.clone(), + localMuteSettings, + turnServer, + memberLogItem + ); + this.connection = connection; + let connectLogItem: ILogItem | undefined; + connection.logItem.wrap("connect", async log => { + connectLogItem = log; + await this.callIfNeeded(log); + }); + throw new Error("connect failed!"); + return connectLogItem; }); - return connectLogItem; } private callIfNeeded(log: ILogItem): Promise { @@ -211,30 +222,34 @@ export class Member { /** @internal */ disconnect(hangup: boolean): ILogItem | undefined { - const {connection} = this; - if (!connection) { - return; - } - let disconnectLogItem; - // if if not sending the hangup, still log disconnect - connection.logItem.wrap("disconnect", async log => { - disconnectLogItem = log; - if (hangup && connection.peerCall) { - await connection.peerCall.hangup(CallErrorCode.UserHangup, log); + return this.errorBoundary.try(() => { + const {connection} = this; + if (!connection) { + return; } + let disconnectLogItem; + // if if not sending the hangup, still log disconnect + connection.logItem.wrap("disconnect", async log => { + disconnectLogItem = log; + if (hangup && connection.peerCall) { + await connection.peerCall.hangup(CallErrorCode.UserHangup, log); + } + }); + connection.dispose(); + this.connection = undefined; + return disconnectLogItem; }); - connection.dispose(); - this.connection = undefined; - return disconnectLogItem; } /** @internal */ updateCallInfo(callDeviceMembership: CallDeviceMembership, causeItem: ILogItem) { - this.callDeviceMembership = callDeviceMembership; - this._renewExpireTimeout(causeItem); - if (this.connection) { - this.connection.logItem.refDetached(causeItem); - } + this.errorBoundary.try(() => { + this.callDeviceMembership = callDeviceMembership; + this._renewExpireTimeout(causeItem); + if (this.connection) { + this.connection.logItem.refDetached(causeItem); + } + }); } /** @internal */ @@ -308,49 +323,51 @@ export class Member { /** @internal */ handleDeviceMessage(message: SignallingMessage, syncLog: ILogItem): void { - const {connection} = this; - if (connection) { - const destSessionId = message.content.dest_session_id; - if (destSessionId !== this.options.sessionId) { - const logItem = connection.logItem.log({l: "ignoring to_device event with wrong session_id", destSessionId, type: message.type}); - syncLog.refDetached(logItem); - return; - } - // if there is no peerCall, we either create it with an invite and Handle is implied or we'll ignore it - let action = IncomingMessageAction.Handle; - if (connection.peerCall) { - action = connection.peerCall.getMessageAction(message); - // deal with glare and replacing the call before creating new calls - if (action === IncomingMessageAction.InviteGlare) { - const {shouldReplace, log} = connection.peerCall.handleInviteGlare(message, this.deviceId, connection.logItem); - if (log) { - syncLog.refDetached(log); - } - if (shouldReplace) { - connection.peerCall = undefined; - action = IncomingMessageAction.Handle; - } + this.errorBoundary.try(() => { + const {connection} = this; + if (connection) { + const destSessionId = message.content.dest_session_id; + if (destSessionId !== this.options.sessionId) { + const logItem = connection.logItem.log({l: "ignoring to_device event with wrong session_id", destSessionId, type: message.type}); + syncLog.refDetached(logItem); + return; } - } - if (message.type === EventType.Invite && !connection.peerCall) { - connection.peerCall = this._createPeerCall(message.content.call_id); - } - if (action === IncomingMessageAction.Handle) { - const idx = sortedIndex(connection.queuedSignallingMessages, message, (a, b) => a.content.seq - b.content.seq); - connection.queuedSignallingMessages.splice(idx, 0, message); + // if there is no peerCall, we either create it with an invite and Handle is implied or we'll ignore it + let action = IncomingMessageAction.Handle; if (connection.peerCall) { - const hasNewMessageBeenDequeued = this.dequeueSignallingMessages(connection, connection.peerCall, message, syncLog); - if (!hasNewMessageBeenDequeued) { - syncLog.refDetached(connection.logItem.log({l: "queued signalling message", type: message.type, seq: message.content.seq})); + action = connection.peerCall.getMessageAction(message); + // deal with glare and replacing the call before creating new calls + if (action === IncomingMessageAction.InviteGlare) { + const {shouldReplace, log} = connection.peerCall.handleInviteGlare(message, this.deviceId, connection.logItem); + if (log) { + syncLog.refDetached(log); + } + if (shouldReplace) { + connection.peerCall = undefined; + action = IncomingMessageAction.Handle; + } } } - } else if (action === IncomingMessageAction.Ignore && connection.peerCall) { - const logItem = connection.logItem.log({l: "ignoring to_device event with wrong call_id", callId: message.content.call_id, type: message.type}); - syncLog.refDetached(logItem); + if (message.type === EventType.Invite && !connection.peerCall) { + connection.peerCall = this._createPeerCall(message.content.call_id); + } + if (action === IncomingMessageAction.Handle) { + const idx = sortedIndex(connection.queuedSignallingMessages, message, (a, b) => a.content.seq - b.content.seq); + connection.queuedSignallingMessages.splice(idx, 0, message); + if (connection.peerCall) { + const hasNewMessageBeenDequeued = this.dequeueSignallingMessages(connection, connection.peerCall, message, syncLog); + if (!hasNewMessageBeenDequeued) { + syncLog.refDetached(connection.logItem.log({l: "queued signalling message", type: message.type, seq: message.content.seq})); + } + } + } else if (action === IncomingMessageAction.Ignore && connection.peerCall) { + const logItem = connection.logItem.log({l: "ignoring to_device event with wrong call_id", callId: message.content.call_id, type: message.type}); + syncLog.refDetached(logItem); + } + } else { + syncLog.log({l: "member not connected", userId: this.userId, deviceId: this.deviceId}); } - } else { - syncLog.log({l: "member not connected", userId: this.userId, deviceId: this.deviceId}); - } + }); } private dequeueSignallingMessages(connection: MemberConnection, peerCall: PeerCall, newMessage: SignallingMessage, syncLog: ILogItem): boolean { @@ -373,19 +390,23 @@ export class Member { /** @internal */ async setMedia(localMedia: LocalMedia, previousMedia: LocalMedia): Promise { - const {connection} = this; - if (connection) { - connection.localMedia = localMedia.replaceClone(connection.localMedia, previousMedia); - await connection.peerCall?.setMedia(connection.localMedia, connection.logItem); - } + return this.errorBoundary.try(async () => { + const {connection} = this; + if (connection) { + connection.localMedia = localMedia.replaceClone(connection.localMedia, previousMedia); + await connection.peerCall?.setMedia(connection.localMedia, connection.logItem); + } + }); } async setMuted(muteSettings: MuteSettings): Promise { - const {connection} = this; - if (connection) { - connection.localMuteSettings = muteSettings; - await connection.peerCall?.setMuted(muteSettings, connection.logItem); - } + return this.errorBoundary.try(async () => { + const {connection} = this; + if (connection) { + connection.localMuteSettings = muteSettings; + await connection.peerCall?.setMuted(muteSettings, connection.logItem); + } + }); } private _createPeerCall(callId: string): PeerCall { From 7f9edbb742bb78b6a782bce5f0670a46e4bcc3f5 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 9 Jan 2023 14:03:14 +0100 Subject: [PATCH 275/435] this can be private --- src/matrix/calls/group/GroupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index cda5a40f..01ff4d31 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -276,7 +276,7 @@ export class GroupCall extends EventEmitter<{change: never}> { }); } - terminate(log?: ILogItem): Promise { + private terminate(log?: ILogItem): Promise { return this.options.logger.wrapOrRun(log, {l: "terminate call", t: CALL_LOG_TYPE}, async log => { if (this._state === GroupCallState.Fledgling) { return; From 1e4180a71ff4f4ef5218eae861035a4bc5ca6387 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 9 Jan 2023 14:03:52 +0100 Subject: [PATCH 276/435] add error boundary to GroupCall --- src/matrix/calls/group/GroupCall.ts | 321 +++++++++++++++------------- 1 file changed, 177 insertions(+), 144 deletions(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 01ff4d31..53a87d5f 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -21,6 +21,7 @@ import {MuteSettings, CALL_LOG_TYPE, CALL_MEMBER_VALIDITY_PERIOD_MS, mute} from import {MemberChange, RoomMember} from "../../room/members/RoomMember"; import {EventEmitter} from "../../../utils/EventEmitter"; import {EventType, CallIntent} from "../callEventTypes"; +import { ErrorBoundary } from "../../../utils/ErrorBoundary"; import type {Options as MemberOptions} from "./Member"; import type {TurnServerSource} from "../TurnServerSource"; @@ -92,6 +93,9 @@ export class GroupCall extends EventEmitter<{change: never}> { private bufferedDeviceMessages = new Map>>(); /** Set between calling join and leave. */ private joinedData?: JoinedData; + private errorBoundary = new ErrorBoundary(err => { + this.emitChange(); + }); constructor( public readonly id: string, @@ -154,6 +158,10 @@ export class GroupCall extends EventEmitter<{change: never}> { return this.joinedData?.logItem; } + get error(): Error | undefined { + return this.errorBoundary.error; + } + async join(localMedia: LocalMedia): Promise { if (this._state !== GroupCallState.Created || this.joinedData) { return; @@ -206,6 +214,9 @@ export class GroupCall extends EventEmitter<{change: never}> { // and update the track info so PeerCall can use it to send up to date metadata, this.joinedData.localMuteSettings.updateTrackInfo(localMedia.userMedia); this.emitChange(); //allow listeners to see new media/mute settings + // TODO: if setMedia fails on one of the members, we should revert to the old media + // on the members processed so far, and show an error that we could not set the new media + // for this, we will need to remove the usage of the errorBoundary in member.setMedia. await Promise.all(Array.from(this._members.values()).map(m => { return m.setMedia(localMedia, oldMedia); })); @@ -234,6 +245,9 @@ export class GroupCall extends EventEmitter<{change: never}> { if (this.localMedia) { mute(this.localMedia, muteSettings, this.joinedData!.logItem); } + // TODO: if setMuted fails on one of the members, we should revert to the old media + // on the members processed so far, and show an error that we could not set the new media + // for this, we will need to remove the usage of the errorBoundary in member.setMuted. await Promise.all(Array.from(this._members.values()).map(m => { return m.setMuted(joinedData.localMuteSettings); })); @@ -271,7 +285,14 @@ export class GroupCall extends EventEmitter<{change: never}> { log.set("already_left", true); } } finally { - this.disconnect(log); + // disconnect is called both from the sync loop and from methods like this one that + // are called from the view model. We want errors during the sync loop being caught + // by the errorboundary, but since leave is called from the view model, we want + // the error to be thrown. So here we check if disconnect succeeded, and if not + // we rethrow the error put into the errorBoundary. + if(!this.disconnect(log)) { + throw this.errorBoundary.error; + } } }); } @@ -308,126 +329,134 @@ export class GroupCall extends EventEmitter<{change: never}> { /** @internal */ updateCallEvent(callContent: Record, syncLog: ILogItem) { - syncLog.wrap({l: "update call", t: CALL_LOG_TYPE, id: this.id}, log => { - this.callContent = callContent; - if (this._state === GroupCallState.Creating) { - this._state = GroupCallState.Created; - } - log.set("status", this._state); - this.emitChange(); + this.errorBoundary.try(() => { + syncLog.wrap({l: "update call", t: CALL_LOG_TYPE, id: this.id}, log => { + this.callContent = callContent; + if (this._state === GroupCallState.Creating) { + this._state = GroupCallState.Created; + } + log.set("status", this._state); + this.emitChange(); + }); }); } /** @internal */ updateRoomMembers(memberChanges: Map) { - for (const change of memberChanges.values()) { - const {member} = change; - for (const callMember of this._members.values()) { - // find all call members for a room member (can be multiple, for every device) - if (callMember.userId === member.userId) { - callMember.updateRoomMember(member); + this.errorBoundary.try(() => { + for (const change of memberChanges.values()) { + const {member} = change; + for (const callMember of this._members.values()) { + // find all call members for a room member (can be multiple, for every device) + if (callMember.userId === member.userId) { + callMember.updateRoomMember(member); + } } } - } - } - - /** @internal */ - updateMembership(userId: string, roomMember: RoomMember, callMembership: CallMembership, syncLog: ILogItem) { - syncLog.wrap({l: "update call membership", t: CALL_LOG_TYPE, id: this.id, userId}, log => { - const now = this.options.clock.now(); - const devices = callMembership["m.devices"]; - const previousDeviceIds = this.getDeviceIdsForUserId(userId); - for (const device of devices) { - const deviceId = device.device_id; - const memberKey = getMemberKey(userId, deviceId); - if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) { - log.wrap("update own membership", log => { - if (this.hasJoined) { - if (this.joinedData) { - this.joinedData.logItem.refDetached(log); - } - this._setupRenewMembershipTimeout(device, log); - } - if (this._state === GroupCallState.Joining) { - log.set("joined", true); - this._state = GroupCallState.Joined; - this.emitChange(); - } - }); - } else { - log.wrap({l: "update device membership", id: memberKey, sessionId: device.session_id}, log => { - if (isMemberExpired(device, now)) { - log.set("expired", true); - const member = this._members.get(memberKey); - if (member) { - member.dispose(); - this._members.remove(memberKey); - log.set("removed", true); - } - return; - } - let member = this._members.get(memberKey); - const sessionIdChanged = member && member.sessionId !== device.session_id; - if (member && !sessionIdChanged) { - log.set("update", true); - member.updateCallInfo(device, log); - } else { - if (member && sessionIdChanged) { - log.set("removedSessionId", member.sessionId); - const disconnectLogItem = member.disconnect(false); - if (disconnectLogItem) { - log.refDetached(disconnectLogItem); - } - member.dispose(); - this._members.remove(memberKey); - member = undefined; - } - log.set("add", true); - member = new Member( - roomMember, - device, this._memberOptions, - log - ); - this._members.add(memberKey, member); - if (this.joinedData) { - this.connectToMember(member, this.joinedData, log); - } - } - // flush pending messages, either after having created the member, - // or updated the session id with updateCallInfo - this.flushPendingIncomingDeviceMessages(member, log); - }); - } - } - - const newDeviceIds = new Set(devices.map(call => call.device_id)); - // remove user as member of any calls not present anymore - for (const previousDeviceId of previousDeviceIds) { - if (!newDeviceIds.has(previousDeviceId)) { - this.removeMemberDevice(userId, previousDeviceId, log); - } - } - if (userId === this.options.ownUserId && !newDeviceIds.has(this.options.ownDeviceId)) { - this.removeOwnDevice(log); - } + }); + } + + /** @internal */ + updateMembership(userId: string, roomMember: RoomMember, callMembership: CallMembership, syncLog: ILogItem) { + this.errorBoundary.try(() => { + syncLog.wrap({l: "update call membership", t: CALL_LOG_TYPE, id: this.id, userId}, log => { + const now = this.options.clock.now(); + const devices = callMembership["m.devices"]; + const previousDeviceIds = this.getDeviceIdsForUserId(userId); + for (const device of devices) { + const deviceId = device.device_id; + const memberKey = getMemberKey(userId, deviceId); + if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) { + log.wrap("update own membership", log => { + if (this.hasJoined) { + if (this.joinedData) { + this.joinedData.logItem.refDetached(log); + } + this._setupRenewMembershipTimeout(device, log); + } + if (this._state === GroupCallState.Joining) { + log.set("joined", true); + this._state = GroupCallState.Joined; + this.emitChange(); + } + }); + } else { + log.wrap({l: "update device membership", id: memberKey, sessionId: device.session_id}, log => { + if (isMemberExpired(device, now)) { + log.set("expired", true); + const member = this._members.get(memberKey); + if (member) { + member.dispose(); + this._members.remove(memberKey); + log.set("removed", true); + } + return; + } + let member = this._members.get(memberKey); + const sessionIdChanged = member && member.sessionId !== device.session_id; + if (member && !sessionIdChanged) { + log.set("update", true); + member.updateCallInfo(device, log); + } else { + if (member && sessionIdChanged) { + log.set("removedSessionId", member.sessionId); + const disconnectLogItem = member.disconnect(false); + if (disconnectLogItem) { + log.refDetached(disconnectLogItem); + } + member.dispose(); + this._members.remove(memberKey); + member = undefined; + } + log.set("add", true); + member = new Member( + roomMember, + device, this._memberOptions, + log + ); + this._members.add(memberKey, member); + if (this.joinedData) { + this.connectToMember(member, this.joinedData, log); + } + } + // flush pending messages, either after having created the member, + // or updated the session id with updateCallInfo + this.flushPendingIncomingDeviceMessages(member, log); + }); + } + } + + const newDeviceIds = new Set(devices.map(call => call.device_id)); + // remove user as member of any calls not present anymore + for (const previousDeviceId of previousDeviceIds) { + if (!newDeviceIds.has(previousDeviceId)) { + this.removeMemberDevice(userId, previousDeviceId, log); + } + } + if (userId === this.options.ownUserId && !newDeviceIds.has(this.options.ownDeviceId)) { + this.removeOwnDevice(log); + } + }); }); } /** @internal */ removeMembership(userId: string, syncLog: ILogItem) { - const deviceIds = this.getDeviceIdsForUserId(userId); - syncLog.wrap({ - l: "remove call member", - t: CALL_LOG_TYPE, - id: this.id, - userId - }, log => { - for (const deviceId of deviceIds) { - this.removeMemberDevice(userId, deviceId, log); - } - if (userId === this.options.ownUserId) { - this.removeOwnDevice(log); - } + this.errorBoundary.try(() => { + const deviceIds = this.getDeviceIdsForUserId(userId); + syncLog.wrap({ + l: "remove call member", + t: CALL_LOG_TYPE, + id: this.id, + userId + }, log => { + for (const deviceId of deviceIds) { + this.removeMemberDevice(userId, deviceId, log); + } + if (userId === this.options.ownUserId) { + this.removeOwnDevice(log); + } + }); }); } @@ -465,19 +494,21 @@ export class GroupCall extends EventEmitter<{change: never}> { } /** @internal */ - disconnect(log: ILogItem) { - if (this.hasJoined) { - for (const [,member] of this._members) { - const disconnectLogItem = member.disconnect(true); - if (disconnectLogItem) { - log.refDetached(disconnectLogItem); + disconnect(log: ILogItem): boolean { + return this.errorBoundary.try(() => { + if (this.hasJoined) { + for (const [,member] of this._members) { + const disconnectLogItem = member.disconnect(true); + if (disconnectLogItem) { + log.refDetached(disconnectLogItem); + } } + this._state = GroupCallState.Created; } - this._state = GroupCallState.Created; - } - this.joinedData?.dispose(); - this.joinedData = undefined; - this.emitChange(); + this.joinedData?.dispose(); + this.joinedData = undefined; + this.emitChange(); + }, false) || true; } /** @internal */ @@ -500,31 +531,33 @@ export class GroupCall extends EventEmitter<{change: never}> { /** @internal */ handleDeviceMessage(message: SignallingMessage, userId: string, deviceId: string, syncLog: ILogItem) { - // TODO: return if we are not membering to the call - const key = getMemberKey(userId, deviceId); - let member = this._members.get(key); - if (member && message.content.sender_session_id === member.sessionId) { - member.handleDeviceMessage(message, syncLog); - } else { - const item = syncLog.log({ - l: "call: buffering to_device message, member not found", - t: CALL_LOG_TYPE, - id: this.id, - userId, - deviceId, - sessionId: message.content.sender_session_id, - type: message.type - }); - syncLog.refDetached(item); - // we haven't received the m.call.member yet for this caller (or with this session id). - // buffer the device messages or create the member/call as it should arrive in a moment - let messages = this.bufferedDeviceMessages.get(key); - if (!messages) { - messages = new Set(); - this.bufferedDeviceMessages.set(key, messages); + this.errorBoundary.try(() => { + // TODO: return if we are not membering to the call + const key = getMemberKey(userId, deviceId); + let member = this._members.get(key); + if (member && message.content.sender_session_id === member.sessionId) { + member.handleDeviceMessage(message, syncLog); + } else { + const item = syncLog.log({ + l: "call: buffering to_device message, member not found", + t: CALL_LOG_TYPE, + id: this.id, + userId, + deviceId, + sessionId: message.content.sender_session_id, + type: message.type + }); + syncLog.refDetached(item); + // we haven't received the m.call.member yet for this caller (or with this session id). + // buffer the device messages or create the member/call as it should arrive in a moment + let messages = this.bufferedDeviceMessages.get(key); + if (!messages) { + messages = new Set(); + this.bufferedDeviceMessages.set(key, messages); + } + messages.add(message); } - messages.add(message); - } + }); } private async _createMemberPayload(includeOwn: boolean): Promise { From fef7af3b31fa4fb48f2d72a038493beac5f915cc Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 9 Jan 2023 14:04:13 +0100 Subject: [PATCH 277/435] report errors from ErrorBoundary on GroupCall and Member in UI UI is still very crude fwiw --- src/domain/session/room/CallViewModel.ts | 19 ++++++++++++++++++- .../session/room/timeline/tiles/CallTile.js | 11 ++++++++--- .../web/ui/css/themes/element/call.css | 16 ++++++++++++++++ src/platform/web/ui/session/room/CallView.ts | 8 +++++++- 4 files changed, 49 insertions(+), 5 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 37f30840..358b26f0 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -39,7 +39,11 @@ export class CallViewModel extends ViewModel { constructor(options: Options) { super(options); - const ownMemberViewModelMap = new ObservableValueMap("self", new EventObservableValue(this.call, "change")) + const callObservable = new EventObservableValue(this.call, "change"); + this.track(callObservable.subscribe(() => { + this.emitChange(); + })); + const ownMemberViewModelMap = new ObservableValueMap("self", callObservable) .mapValues((call, emitChange) => new OwnMemberViewModel(this.childOptions({call, emitChange})), () => {}); this.memberViewModels = this.call.members .filterValues(member => member.isConnected) @@ -79,6 +83,10 @@ export class CallViewModel extends ViewModel { return this.call.id; } + get error(): string | undefined { + return this.call.error?.message; + } + private get call(): GroupCall { return this.getOption("call"); } @@ -135,6 +143,10 @@ class OwnMemberViewModel extends ViewModel implements IStreamViewModel })); } + get error(): string | undefined { + return undefined; + } + get stream(): Stream | undefined { return this.call.localPreviewMedia?.userMedia; } @@ -195,6 +207,10 @@ export class CallMemberViewModel extends ViewModel implements ISt return this.member.remoteMedia?.userMedia; } + get error(): string | undefined { + return this.member.error?.message; + } + private get member(): Member { return this.getOption("member"); } @@ -242,4 +258,5 @@ export interface IStreamViewModel extends AvatarSource, ViewModel { get stream(): Stream | undefined; get isCameraMuted(): boolean; get isMicrophoneMuted(): boolean; + get error(): string | undefined; } diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index 0bc12698..a54af5d8 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -74,9 +74,14 @@ export class CallTile extends SimpleTile { async join() { if (this.canJoin) { - const stream = await this.platform.mediaDevices.getMediaTracks(false, true); - const localMedia = new LocalMedia().withUserMedia(stream); - await this._call.join(localMedia); + try { + const stream = await this.platform.mediaDevices.getMediaTracks(false, true); + const localMedia = new LocalMedia().withUserMedia(stream); + await this._call.join(localMedia); + } catch (err) { + this._error = err; + this.emitChange("error"); + } } } diff --git a/src/platform/web/ui/css/themes/element/call.css b/src/platform/web/ui/css/themes/element/call.css index 10388fb4..f4d674b4 100644 --- a/src/platform/web/ui/css/themes/element/call.css +++ b/src/platform/web/ui/css/themes/element/call.css @@ -24,6 +24,14 @@ limitations under the License. grid-row: 1; } +.CallView_error { + color: red; + font-weight: bold; + align-self: start; + justify-self: center; + margin: 16px; +} + .CallView_members { display: grid; gap: 12px; @@ -59,6 +67,14 @@ limitations under the License. justify-self: center; } +.StreamView_error { + color: red; + font-weight: bold; + align-self: start; + justify-self: center; + margin: 16px; +} + .StreamView_muteStatus { align-self: start; justify-self: end; diff --git a/src/platform/web/ui/session/room/CallView.ts b/src/platform/web/ui/session/room/CallView.ts index 619afc2e..87cac99f 100644 --- a/src/platform/web/ui/session/room/CallView.ts +++ b/src/platform/web/ui/session/room/CallView.ts @@ -43,7 +43,10 @@ export class CallView extends TemplateView { "CallView_unmutedCamera": vm => !vm.isCameraMuted, }, onClick: disableTargetCallback(() => vm.toggleCamera())}), t.button({className: "CallView_hangup", onClick: disableTargetCallback(() => vm.hangup())}), - ]) + ]), + t.if(vm => !!vm.error, t => { + return t.div({className: "CallView_error"}, vm => vm.error); + }) ]); } @@ -112,6 +115,9 @@ class StreamView extends TemplateView { microphoneMuted: vm => vm.isMicrophoneMuted && !vm.isCameraMuted, cameraMuted: vm => vm.isCameraMuted, } + }), + t.if(vm => !!vm.error, t => { + return t.div({className: "StreamView_error"}, vm => vm.error); }) ]); } From bd3499056ae358d1de1f172dc33e2d3f939a827d Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 10 Jan 2023 12:05:30 +0100 Subject: [PATCH 278/435] provider higher-level rageshake fn for opened session Co-authored-by: R Midhun Suresh --- src/domain/rageshake.ts | 33 +++++++++++++++++-- .../session/settings/SettingsViewModel.js | 30 ++++------------- 2 files changed, 38 insertions(+), 25 deletions(-) diff --git a/src/domain/rageshake.ts b/src/domain/rageshake.ts index cb06e638..08e82d8c 100644 --- a/src/domain/rageshake.ts +++ b/src/domain/rageshake.ts @@ -16,11 +16,15 @@ limitations under the License. import type {BlobHandle} from "../platform/web/dom/BlobHandle"; import type {RequestFunction} from "../platform/types/types"; +import type {Platform} from "../platform/web/Platform"; +import type {ILogger} from "../logging/types"; +import type { IDBLogPersister } from "../logging/IDBLogPersister"; +import type { Session } from "../matrix/Session"; // see https://github.com/matrix-org/rageshake#readme type RageshakeData = { // A textual description of the problem. Included in the details.log.gz file. - text: string | undefined; + text?: string; // Application user-agent. Included in the details.log.gz file. userAgent: string; // Identifier for the application (eg 'riot-web'). Should correspond to a mapping configured in the configuration file for github issue reporting to work. @@ -28,7 +32,7 @@ type RageshakeData = { // Application version. Included in the details.log.gz file. version: string; // Label to attach to the github issue, and include in the details file. - label: string | undefined; + label?: string; }; export async function submitLogsToRageshakeServer(data: RageshakeData, logsBlob: BlobHandle, submitUrl: string, request: RequestFunction): Promise { @@ -63,3 +67,28 @@ export async function submitLogsToRageshakeServer(data: RageshakeData, logsBlob: // we don't bother with reading report_url from the body as the rageshake server doesn't always return it // and would have to have CORS setup properly for us to be able to read it. } + +/** @throws {Error} */ +export async function submitLogsFromSessionToDefaultServer(session: Session, platform: Platform): Promise { + const {bugReportEndpointUrl} = platform.config; + if (!bugReportEndpointUrl) { + throw new Error("no server configured to submit logs"); + } + const logReporters = (platform.logger as ILogger).reporters; + const exportReporter = logReporters.find(r => !!r["export"]) as IDBLogPersister | undefined; + if (!exportReporter) { + throw new Error("No logger that can export configured"); + } + const logExport = await exportReporter.export(); + await submitLogsToRageshakeServer( + { + app: "hydrogen", + userAgent: platform.description, + version: platform.version, + text: `Submit logs from settings for user ${session.userId} on device ${session.deviceId}`, + }, + logExport.asBlob(), + bugReportEndpointUrl, + platform.request + ); +} \ No newline at end of file diff --git a/src/domain/session/settings/SettingsViewModel.js b/src/domain/session/settings/SettingsViewModel.js index 0d61bcac..d60a6327 100644 --- a/src/domain/session/settings/SettingsViewModel.js +++ b/src/domain/session/settings/SettingsViewModel.js @@ -16,7 +16,7 @@ limitations under the License. import {ViewModel} from "../../ViewModel"; import {KeyBackupViewModel} from "./KeyBackupViewModel.js"; -import {submitLogsToRageshakeServer} from "../../../domain/rageshake"; +import {submitLogsFromSessionToDefaultServer} from "../../../domain/rageshake"; class PushNotificationStatus { constructor() { @@ -175,29 +175,13 @@ export class SettingsViewModel extends ViewModel { } async sendLogsToServer() { - const {bugReportEndpointUrl} = this.platform.config; - if (bugReportEndpointUrl) { - this._logsFeedbackMessage = this.i18n`Sending logs…`; + this._logsFeedbackMessage = this.i18n`Sending logs…`; + try { + await submitLogsFromSessionToDefaultServer(this._session, this.platform); + this._logsFeedbackMessage = this.i18n`Logs sent succesfully!`; + } catch (err) { + this._logsFeedbackMessage = err.message; this.emitChange(); - try { - const logExport = await this.logger.export(); - await submitLogsToRageshakeServer( - { - app: "hydrogen", - userAgent: this.platform.description, - version: DEFINE_VERSION, - text: `Submit logs from settings for user ${this._session.userId} on device ${this._session.deviceId}`, - }, - logExport.asBlob(), - bugReportEndpointUrl, - this.platform.request - ); - this._logsFeedbackMessage = this.i18n`Logs sent succesfully!`; - this.emitChange(); - } catch (err) { - this._logsFeedbackMessage = err.message; - this.emitChange(); - } } } From b8bc6edbc0240731b883437a8a062f1f0d36c645 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 10 Jan 2023 12:06:49 +0100 Subject: [PATCH 279/435] add ErrorView(Model) to easily report errors and submit logs from UI --- src/domain/ErrorViewModel.ts | 44 +++++++++++++++ .../web/ui/css/themes/element/error.css | 50 +++++++++++++++++ .../web/ui/css/themes/element/theme.css | 1 + src/platform/web/ui/general/ErrorView.ts | 56 +++++++++++++++++++ 4 files changed, 151 insertions(+) create mode 100644 src/domain/ErrorViewModel.ts create mode 100644 src/platform/web/ui/css/themes/element/error.css create mode 100644 src/platform/web/ui/general/ErrorView.ts diff --git a/src/domain/ErrorViewModel.ts b/src/domain/ErrorViewModel.ts new file mode 100644 index 00000000..9045af80 --- /dev/null +++ b/src/domain/ErrorViewModel.ts @@ -0,0 +1,44 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { ViewModel, Options as BaseOptions } from "./ViewModel"; +import {submitLogsFromSessionToDefaultServer} from "./rageshake"; +import type { Session } from "../matrix/Session"; + +type Options = { + error: Error + session: Session, + onClose: () => void +} & BaseOptions; + +export class ErrorViewModel extends ViewModel { + get message(): string { + return this.getOption("error")?.message; + } + + close() { + this.getOption("onClose")(); + } + + async submitLogs(): Promise { + try { + await submitLogsFromSessionToDefaultServer(this.getOption("session"), this.platform); + return true; + } catch (err) { + return false; + } + } +} \ No newline at end of file diff --git a/src/platform/web/ui/css/themes/element/error.css b/src/platform/web/ui/css/themes/element/error.css new file mode 100644 index 00000000..f4e4fb6f --- /dev/null +++ b/src/platform/web/ui/css/themes/element/error.css @@ -0,0 +1,50 @@ +.ErrorView_block { + background: var(--error-color); + color: var(--fixed-white); + margin: 16px; +} + +.ErrorView_inline { + color: var(--error-color); + margin: 4px; +} + +.ErrorView { + font-weight: bold; + margin: 16px; + border-radius: 8px; + padding: 12px; + display: flex; + gap: 4px; +} + +.ErrorView_message { + flex-basis: 0; + flex-grow: 1; + margin: 0px; + word-break: break-all; + word-break: break-word; +} + +.ErrorView_submit { + align-self: end; +} + +.ErrorView_close { + align-self: start; + width: 16px; + height: 16px; + border: none; + background: none; + background-repeat: no-repeat; + background-size: contain; + cursor: pointer; +} + +.ErrorView_block .ErrorView_close { + background-image: url('icons/clear.svg?primary=fixed-white'); +} + +.ErrorView_inline .ErrorView_close { + background-image: url('icons/clear.svg?primary=text-color'); +} \ No newline at end of file diff --git a/src/platform/web/ui/css/themes/element/theme.css b/src/platform/web/ui/css/themes/element/theme.css index 6403bb60..84a27f13 100644 --- a/src/platform/web/ui/css/themes/element/theme.css +++ b/src/platform/web/ui/css/themes/element/theme.css @@ -19,6 +19,7 @@ limitations under the License. @import url('inter.css'); @import url('timeline.css'); @import url('call.css'); +@import url('error.css'); :root { font-size: 10px; diff --git a/src/platform/web/ui/general/ErrorView.ts b/src/platform/web/ui/general/ErrorView.ts new file mode 100644 index 00000000..f44361c3 --- /dev/null +++ b/src/platform/web/ui/general/ErrorView.ts @@ -0,0 +1,56 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {TemplateView, Builder} from "./TemplateView"; +import { disableTargetCallback } from "./utils"; + +import type { ViewNode } from "./types"; +import type {ErrorViewModel} from "../../../../domain/ErrorViewModel"; + + +export class ErrorView extends TemplateView { + constructor(vm: ErrorViewModel, private readonly options: {inline: boolean} = {inline: false}) { + super(vm); + } + override render(t: Builder, vm: ErrorViewModel): ViewNode { + const submitLogsButton = t.button({ + className: "ErrorView_submit", + onClick: disableTargetCallback(async () => { + if (await vm.submitLogs()) { + alert("Logs submitted!"); + } else { + alert("Could not submit logs"); + } + }) + }, "Submit logs"); + const closeButton = t.button({ + className: "ErrorView_close", + onClick: () => vm.close(), + title: "Dismiss error" + }); + return t.div({ + className: { + "ErrorView": true, + "ErrorView_inline": this.options.inline, + "ErrorView_block": !this.options.inline + }}, [ + t.p({className: "ErrorView_message"}, vm.message), + submitLogsButton, + closeButton + ]); + } +} + From f15e849f5496bca0b571c9f47d5f069882e38e82 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 10 Jan 2023 12:08:10 +0100 Subject: [PATCH 280/435] user error view model in room, also when starting call --- src/domain/session/room/RoomViewModel.js | 68 +++++++++++--------- src/platform/web/ui/session/room/RoomView.js | 5 +- 2 files changed, 40 insertions(+), 33 deletions(-) diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 077a996e..d744e5d5 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -23,6 +23,7 @@ import {avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../ import {ViewModel} from "../../ViewModel"; import {imageToInfo} from "../common.js"; import {LocalMedia} from "../../../matrix/calls/LocalMedia"; +import { ErrorViewModel } from "../../ErrorViewModel"; // TODO: remove fallback so default isn't included in bundle for SDK users that have their custom tileClassForEntry // this is a breaking SDK change though to make this option mandatory import {tileClassForEntry as defaultTileClassForEntry} from "./timeline/tiles/index"; @@ -36,8 +37,7 @@ export class RoomViewModel extends ViewModel { this._tileClassForEntry = tileClassForEntry ?? defaultTileClassForEntry; this._tileOptions = undefined; this._onRoomChange = this._onRoomChange.bind(this); - this._timelineError = null; - this._sendError = null; + this._errorViewModel = null; this._composerVM = null; if (room.isArchived) { this._composerVM = new ArchivedViewModel(this.childOptions({archivedRoom: room})); @@ -73,6 +73,14 @@ export class RoomViewModel extends ViewModel { } } + _reportError(error) { + this._errorViewModel = new ErrorViewModel(this.childOptions({error, onClose: () => { + this._errorViewModel = null; + this.emitChange("errorViewModel"); + }})); + this.emitChange("errorViewModel"); + } + async load() { this._room.on("change", this._onRoomChange); try { @@ -88,10 +96,8 @@ export class RoomViewModel extends ViewModel { timeline, }))); this.emitChange("timelineViewModel"); - } catch (err) { - console.error(`room.openTimeline(): ${err.message}:\n${err.stack}`); - this._timelineError = err; - this.emitChange("error"); + } catch (error) { + this._reportError(error); } this._clearUnreadAfterDelay(); } @@ -143,14 +149,8 @@ export class RoomViewModel extends ViewModel { get timelineViewModel() { return this._timelineVM; } get isEncrypted() { return this._room.isEncrypted; } - get error() { - if (this._timelineError) { - return `Something went wrong loading the timeline: ${this._timelineError.message}`; - } - if (this._sendError) { - return `Something went wrong sending your message: ${this._sendError.message}`; - } - return ""; + get errorViewModel() { + return this._errorViewModel; } get avatarLetter() { @@ -215,11 +215,8 @@ export class RoomViewModel extends ViewModel { } else { await this._room.sendEvent("m.room.message", {msgtype, body: message}); } - } catch (err) { - console.error(`room.sendMessage(): ${err.message}:\n${err.stack}`); - this._sendError = err; - this._timelineError = null; - this.emitChange("error"); + } catch (error) { + this._reportError(error); return false; } return true; @@ -289,10 +286,8 @@ export class RoomViewModel extends ViewModel { attachments["info.thumbnail_url"] = this._room.createAttachment(thumbnail.blob, file.name); await this._room.sendEvent("m.room.message", content, attachments); - } catch (err) { - this._sendError = err; - this.emitChange("error"); - console.error(err.stack); + } catch (error) { + this._reportError(error); } } @@ -331,10 +326,8 @@ export class RoomViewModel extends ViewModel { this._room.createAttachment(thumbnail.blob, file.name); } await this._room.sendEvent("m.room.message", content, attachments); - } catch (err) { - this._sendError = err; - this.emitChange("error"); - console.error(err.stack); + } catch (error) { + this._reportError(error); } } @@ -364,15 +357,28 @@ export class RoomViewModel extends ViewModel { } async startCall() { + let localMedia; try { - const session = this.getOption("session"); const stream = await this.platform.mediaDevices.getMediaTracks(false, true); - const localMedia = new LocalMedia().withUserMedia(stream); + localMedia = new LocalMedia().withUserMedia(stream); + } catch (err) { + this._reportError(new Error(`Could not get local audio and/or video stream: ${err.message}`)); + return; + } + const session = this.getOption("session"); + let call; + try { // this will set the callViewModel above as a call will be added to callHandler.calls - const call = await session.callHandler.createCall(this._room.id, "m.video", "A call " + Math.round(this.platform.random() * 100)); + call = await session.callHandler.createCall(this._room.id, "m.video", "A call " + Math.round(this.platform.random() * 100)); + } catch (err) { + this._reportError(new Error(`Could not create call: ${err.message}`)); + return; + } + try { await call.join(localMedia); } catch (err) { - console.error(err.stack); + this._reportError(new Error(`Could not join call: ${err.message}`)); + return; } } } diff --git a/src/platform/web/ui/session/room/RoomView.js b/src/platform/web/ui/session/room/RoomView.js index 4f9e40d9..9478f5e6 100644 --- a/src/platform/web/ui/session/room/RoomView.js +++ b/src/platform/web/ui/session/room/RoomView.js @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {TemplateView} from "../../general/TemplateView"; +import {TemplateView} from "../../general/TemplateView"; import {Popup} from "../../general/Popup.js"; import {Menu} from "../../general/Menu.js"; import {TimelineView} from "./TimelineView"; @@ -24,6 +24,7 @@ import {MessageComposer} from "./MessageComposer.js"; import {RoomArchivedView} from "./RoomArchivedView.js"; import {AvatarView} from "../../AvatarView.js"; import {CallView} from "./CallView"; +import { ErrorView } from "../../general/ErrorView"; export class RoomView extends TemplateView { constructor(vm, viewClassForTile) { @@ -53,7 +54,7 @@ export class RoomView extends TemplateView { }) ]), t.div({className: "RoomView_body"}, [ - t.div({className: "RoomView_error"}, vm => vm.error), + t.if(vm => vm.errorViewModel, t => t.div({className: "RoomView_error"}, t.view(new ErrorView(vm.errorViewModel)))), t.mapView(vm => vm.callViewModel, callViewModel => callViewModel ? new CallView(callViewModel) : null), t.mapView(vm => vm.timelineViewModel, timelineViewModel => { return timelineViewModel ? From 42ee2d294b1f9f4f3afd3d6ccb737d70ccf95bb6 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 10 Jan 2023 12:09:10 +0100 Subject: [PATCH 281/435] use error view model from call tile --- .../session/room/timeline/tiles/CallTile.js | 26 +++++++++++++++---- .../ui/session/room/timeline/CallTileView.ts | 21 ++++++++++----- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index a54af5d8..a3008cb9 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -16,7 +16,7 @@ limitations under the License. import {SimpleTile} from "./SimpleTile.js"; import {LocalMedia} from "../../../../../matrix/calls/LocalMedia"; - +import {ErrorViewModel} from "../../../../ErrorViewModel" // TODO: timeline entries for state events with the same state key and type // should also update previous entries in the timeline, so we can update the name of the call, whether it is terminated, etc ... @@ -28,6 +28,7 @@ export class CallTile extends SimpleTile { const calls = this.getOption("session").callHandler.calls; this._call = calls.get(this._entry.stateKey); this._callSubscription = undefined; + this._errorViewModel = undefined; if (this._call) { this._callSubscription = this._call.disposableOn("change", () => { // unsubscribe when terminated @@ -60,6 +61,10 @@ export class CallTile extends SimpleTile { return this._call && this._call.hasJoined; } + get errorViewModel() { + return this._errorViewModel; + } + get label() { if (this._call) { if (this._call.hasJoined) { @@ -78,16 +83,19 @@ export class CallTile extends SimpleTile { const stream = await this.platform.mediaDevices.getMediaTracks(false, true); const localMedia = new LocalMedia().withUserMedia(stream); await this._call.join(localMedia); - } catch (err) { - this._error = err; - this.emitChange("error"); + } catch (error) { + this._reportError(error); } } } async leave() { if (this.canLeave) { - this._call.leave(); + try { + this._call.leave(); + } catch (err) { + this._reportError(err); + } } } @@ -96,4 +104,12 @@ export class CallTile extends SimpleTile { this._callSubscription = this._callSubscription(); } } + + _reportError(error) { + this._errorViewModel = new ErrorViewModel(this.childOptions({error, onClose: () => { + this._errorViewModel = undefined; + this.emitChange("errorViewModel"); + }})); + this.emitChange("errorViewModel"); + } } diff --git a/src/platform/web/ui/session/room/timeline/CallTileView.ts b/src/platform/web/ui/session/room/timeline/CallTileView.ts index e0ca00bd..5bede510 100644 --- a/src/platform/web/ui/session/room/timeline/CallTileView.ts +++ b/src/platform/web/ui/session/room/timeline/CallTileView.ts @@ -14,17 +14,24 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {TemplateView} from "../../../general/TemplateView"; +import {Builder, TemplateView} from "../../../general/TemplateView"; import type {CallTile} from "../../../../../../domain/session/room/timeline/tiles/CallTile"; +import {ErrorView} from "../../../general/ErrorView"; export class CallTileView extends TemplateView { - render(t, vm) { + render(t: Builder, vm: CallTile) { return t.li( - {className: "AnnouncementView"}, - t.div([ - vm => vm.label, - t.button({className: "CallTileView_join", hidden: vm => !vm.canJoin}, "Join"), - t.button({className: "CallTileView_leave", hidden: vm => !vm.canLeave}, "Leave") + {className: "CallTileView AnnouncementView"}, + t.div( + [ + t.if(vm => vm.errorViewModel, t => { + return t.div({className: "CallTileView_error"}, t.view(new ErrorView(vm.errorViewModel, {inline: true}))); + }), + t.div([ + vm => vm.label, + t.button({className: "CallTileView_join", hidden: vm => !vm.canJoin}, "Join"), + t.button({className: "CallTileView_leave", hidden: vm => !vm.canLeave}, "Leave") + ]) ]) ); } From 64d6db556a0e3a6a0c5fed3dc8a27345962e928f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 12 Jan 2023 14:36:39 +0100 Subject: [PATCH 282/435] fix updates from call and member classes in VM this fixes this.emitChange sending the update over the collection in the call member VM, which is how updates are subscribed to by the UI. It also adds a callback to the VM for when the member sends an update, so we can check later on if the error on the member has been set. --- src/domain/session/room/CallViewModel.ts | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 358b26f0..1080015f 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -40,14 +40,19 @@ export class CallViewModel extends ViewModel { constructor(options: Options) { super(options); const callObservable = new EventObservableValue(this.call, "change"); - this.track(callObservable.subscribe(() => { - this.emitChange(); - })); + this.track(callObservable.subscribe(() => this.onUpdate())); const ownMemberViewModelMap = new ObservableValueMap("self", callObservable) .mapValues((call, emitChange) => new OwnMemberViewModel(this.childOptions({call, emitChange})), () => {}); this.memberViewModels = this.call.members .filterValues(member => member.isConnected) - .mapValues(member => new CallMemberViewModel(this.childOptions({member, mediaRepository: this.getOption("room").mediaRepository}))) + .mapValues( + (member, emitChange) => new CallMemberViewModel(this.childOptions({ + member, + emitChange, + mediaRepository: this.getOption("room").mediaRepository + })), + (vm: CallMemberViewModel) => vm.onUpdate(), + ) .join(ownMemberViewModelMap) .sortValues((a, b) => a.compare(b)); this.track(this.memberViewModels.subscribe({ @@ -91,6 +96,9 @@ export class CallViewModel extends ViewModel { return this.getOption("call"); } + private onUpdate() { + } + async hangup() { if (this.call.hasJoined) { await this.call.leave(); @@ -241,6 +249,8 @@ export class CallMemberViewModel extends ViewModel implements ISt return this.member.member.name; } + onUpdate() { + } compare(other: OwnMemberViewModel | CallMemberViewModel): number { if (other instanceof OwnMemberViewModel) { return -other.compare(this); From 4070d422cdd0883af3c4709cab3afc781c13c14b Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 12 Jan 2023 14:37:28 +0100 Subject: [PATCH 283/435] use error view (model) in call view (model) --- src/domain/ErrorViewModel.ts | 4 ++ src/domain/session/room/CallViewModel.ts | 68 ++++++++++++++++---- src/platform/web/ui/session/room/CallView.ts | 9 +-- 3 files changed, 65 insertions(+), 16 deletions(-) diff --git a/src/domain/ErrorViewModel.ts b/src/domain/ErrorViewModel.ts index 9045af80..51a34565 100644 --- a/src/domain/ErrorViewModel.ts +++ b/src/domain/ErrorViewModel.ts @@ -29,6 +29,10 @@ export class ErrorViewModel extends ViewModel { return this.getOption("error")?.message; } + get error(): Error { + return this.getOption("error"); + } + close() { this.getOption("onClose")(); } diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 1080015f..e87af12a 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -20,6 +20,7 @@ import {getStreamVideoTrack, getStreamAudioTrack} from "../../../matrix/calls/co import {avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../avatar"; import {EventObservableValue} from "../../../observable/value/EventObservableValue"; import {ObservableValueMap} from "../../../observable/map/ObservableValueMap"; +import { ErrorViewModel } from "../../ErrorViewModel"; import type {Room} from "../../../matrix/room/Room"; import type {GroupCall} from "../../../matrix/calls/group/GroupCall"; import type {Member} from "../../../matrix/calls/group/Member"; @@ -28,14 +29,17 @@ import type {BaseObservableList} from "../../../observable/list/BaseObservableLi import type {BaseObservableValue} from "../../../observable/value/BaseObservableValue"; import type {Stream} from "../../../platform/types/MediaDevices"; import type {MediaRepository} from "../../../matrix/net/MediaRepository"; +import type { Session } from "../../../matrix/Session"; type Options = BaseOptions & { call: GroupCall, room: Room, + session: Session }; export class CallViewModel extends ViewModel { public readonly memberViewModels: BaseObservableList; + private _errorViewModel?: ErrorViewModel; constructor(options: Options) { super(options); @@ -88,8 +92,8 @@ export class CallViewModel extends ViewModel { return this.call.id; } - get error(): string | undefined { - return this.call.error?.message; + get errorViewModel(): ErrorViewModel | undefined { + return this._errorViewModel; } private get call(): GroupCall { @@ -97,11 +101,18 @@ export class CallViewModel extends ViewModel { } private onUpdate() { + if (this.call.error) { + this._reportError(this.call.error); + } } async hangup() { - if (this.call.hasJoined) { - await this.call.leave(); + try { + if (this.call.hasJoined) { + await this.call.leave(); + } + } catch (err) { + this._reportError(err); } } @@ -125,7 +136,6 @@ export class CallViewModel extends ViewModel { // unmute but no track? if (muteSettings.microphone && !getStreamAudioTrack(localMedia.userMedia)) { const stream = await this.platform.mediaDevices.getMediaTracks(true, !muteSettings.camera); - console.log("got tracks", Array.from(stream.getTracks()).map((t: MediaStreamTrack) => { return {kind: t.kind, id: t.id};})) await this.call.setMedia(localMedia.withUserMedia(stream)); } else { await this.call.setMuted(muteSettings.toggleMicrophone()); @@ -133,6 +143,21 @@ export class CallViewModel extends ViewModel { this.emitChange(); } } + + private _reportError(error: Error) { + if (this._errorViewModel?.error === error) { + return; + } + this.disposeTracked(this._errorViewModel); + this._errorViewModel = new ErrorViewModel(this.childOptions({ + error, + onClose: () => { + this._errorViewModel = this.disposeTracked(this._errorViewModel); + this.emitChange("errorViewModel"); + } + })); + this.emitChange("errorViewModel"); + } } class OwnMemberViewModel extends ViewModel implements IStreamViewModel { @@ -151,7 +176,7 @@ class OwnMemberViewModel extends ViewModel implements IStreamViewModel })); } - get error(): string | undefined { + get errorViewModel(): ErrorViewModel | undefined { return undefined; } @@ -207,22 +232,25 @@ class OwnMemberViewModel extends ViewModel implements IStreamViewModel type MemberOptions = BaseOptions & { member: Member, - mediaRepository: MediaRepository + mediaRepository: MediaRepository, + session: Session }; export class CallMemberViewModel extends ViewModel implements IStreamViewModel { + private _errorViewModel?: ErrorViewModel; + get stream(): Stream | undefined { return this.member.remoteMedia?.userMedia; } - get error(): string | undefined { - return this.member.error?.message; - } - private get member(): Member { return this.getOption("member"); } + get errorViewModel(): ErrorViewModel | undefined { + return this._errorViewModel; + } + get isCameraMuted(): boolean { return this.member.remoteMuteSettings?.camera ?? true; } @@ -250,7 +278,23 @@ export class CallMemberViewModel extends ViewModel implements ISt } onUpdate() { + this.mapMemberSyncErrorIfNeeded(); } + + private mapMemberSyncErrorIfNeeded() { + if (this.member.error && (!this._errorViewModel || this._errorViewModel.error !== this.member.error)) { + this.disposeTracked(this._errorViewModel); + this._errorViewModel = this.track(new ErrorViewModel(this.childOptions({ + error: this.member.error, + onClose: () => { + this._errorViewModel = this.disposeTracked(this._errorViewModel); + this.emitChange("errorViewModel"); + }, + }))); + this.emitChange("errorViewModel"); + } + } + compare(other: OwnMemberViewModel | CallMemberViewModel): number { if (other instanceof OwnMemberViewModel) { return -other.compare(this); @@ -268,5 +312,5 @@ export interface IStreamViewModel extends AvatarSource, ViewModel { get stream(): Stream | undefined; get isCameraMuted(): boolean; get isMicrophoneMuted(): boolean; - get error(): string | undefined; + get errorViewModel(): ErrorViewModel | undefined; } diff --git a/src/platform/web/ui/session/room/CallView.ts b/src/platform/web/ui/session/room/CallView.ts index 87cac99f..eacb3144 100644 --- a/src/platform/web/ui/session/room/CallView.ts +++ b/src/platform/web/ui/session/room/CallView.ts @@ -20,6 +20,7 @@ import {ListView} from "../../general/ListView"; import {classNames} from "../../general/html"; import {Stream} from "../../../../types/MediaDevices"; import type {CallViewModel, CallMemberViewModel, IStreamViewModel} from "../../../../../domain/session/room/CallViewModel"; +import { ErrorView } from "../../general/ErrorView"; export class CallView extends TemplateView { private resizeObserver?: ResizeObserver; @@ -44,8 +45,8 @@ export class CallView extends TemplateView { }, onClick: disableTargetCallback(() => vm.toggleCamera())}), t.button({className: "CallView_hangup", onClick: disableTargetCallback(() => vm.hangup())}), ]), - t.if(vm => !!vm.error, t => { - return t.div({className: "CallView_error"}, vm => vm.error); + t.if(vm => !!vm.errorViewModel, t => { + return t.div({className: "CallView_error"}, t.view(new ErrorView(vm.errorViewModel!))); }) ]); } @@ -116,8 +117,8 @@ class StreamView extends TemplateView { cameraMuted: vm => vm.isCameraMuted, } }), - t.if(vm => !!vm.error, t => { - return t.div({className: "StreamView_error"}, vm => vm.error); + t.if(vm => !!vm.errorViewModel, t => { + return t.div({className: "StreamView_error"}, t.view(new ErrorView(vm.errorViewModel!))); }) ]); } From 7b32a2729e09c31778e9fb9f3c82f09586ddd499 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 13 Jan 2023 10:39:44 +0100 Subject: [PATCH 284/435] don't allow other click handlers to run in parent elements --- src/platform/web/ui/general/ErrorView.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/platform/web/ui/general/ErrorView.ts b/src/platform/web/ui/general/ErrorView.ts index f44361c3..eb1e7838 100644 --- a/src/platform/web/ui/general/ErrorView.ts +++ b/src/platform/web/ui/general/ErrorView.ts @@ -28,7 +28,8 @@ export class ErrorView extends TemplateView { override render(t: Builder, vm: ErrorViewModel): ViewNode { const submitLogsButton = t.button({ className: "ErrorView_submit", - onClick: disableTargetCallback(async () => { + onClick: disableTargetCallback(async evt => { + evt.stopPropagation(); if (await vm.submitLogs()) { alert("Logs submitted!"); } else { @@ -38,7 +39,10 @@ export class ErrorView extends TemplateView { }, "Submit logs"); const closeButton = t.button({ className: "ErrorView_close", - onClick: () => vm.close(), + onClick: evt => { + evt.stopPropagation(); + vm.close(); + }, title: "Dismiss error" }); return t.div({ From 75839007ea41b7a398a2adb95651629d5b8648ff Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 13 Jan 2023 10:40:04 +0100 Subject: [PATCH 285/435] make buttons clickable in the first place --- src/platform/web/ui/css/themes/element/call.css | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/platform/web/ui/css/themes/element/call.css b/src/platform/web/ui/css/themes/element/call.css index f4d674b4..c0f1550e 100644 --- a/src/platform/web/ui/css/themes/element/call.css +++ b/src/platform/web/ui/css/themes/element/call.css @@ -68,11 +68,12 @@ limitations under the License. } .StreamView_error { - color: red; - font-weight: bold; align-self: start; justify-self: center; - margin: 16px; + /** Chrome (v100) requires this to make the buttons clickable + * where they overlap with the video element, even though + * the buttons come later in the DOM. */ + z-index: 1; } .StreamView_muteStatus { From 2bba9f8675bfefc0122db7f0b1e8d7dc41f7c0ec Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 13 Jan 2023 10:40:15 +0100 Subject: [PATCH 286/435] some layout improvements --- src/platform/web/ui/css/themes/element/error.css | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/platform/web/ui/css/themes/element/error.css b/src/platform/web/ui/css/themes/element/error.css index f4e4fb6f..ce252056 100644 --- a/src/platform/web/ui/css/themes/element/error.css +++ b/src/platform/web/ui/css/themes/element/error.css @@ -15,7 +15,7 @@ border-radius: 8px; padding: 12px; display: flex; - gap: 4px; + gap: 8px; } .ErrorView_message { @@ -24,6 +24,7 @@ margin: 0px; word-break: break-all; word-break: break-word; + align-self: center; } .ErrorView_submit { From d3b5a7066383a8d8d0e3addc6ac1e6f232bdd8ab Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 13 Jan 2023 12:42:52 +0100 Subject: [PATCH 287/435] join errors thrown from matrix layer with sync errors caught by error boundary. this adds a new base view model that facilitates reporting errors with the ErrorViewModel --- src/domain/ErrorReportViewModel.ts | 62 +++++++++++ src/domain/session/room/CallViewModel.ts | 101 ++++++------------ src/domain/session/room/RoomViewModel.js | 27 ++--- .../session/room/timeline/tiles/CallTile.js | 31 ++---- .../session/room/timeline/tiles/SimpleTile.js | 4 +- 5 files changed, 114 insertions(+), 111 deletions(-) create mode 100644 src/domain/ErrorReportViewModel.ts diff --git a/src/domain/ErrorReportViewModel.ts b/src/domain/ErrorReportViewModel.ts new file mode 100644 index 00000000..eb6ed0aa --- /dev/null +++ b/src/domain/ErrorReportViewModel.ts @@ -0,0 +1,62 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { ViewModel } from "./ViewModel"; +import type { Options as BaseOptions } from "./ViewModel"; +import type { Session } from "../matrix/Session"; +import { ErrorViewModel } from "./ErrorViewModel"; +import type { LogCallback, LabelOrValues } from "../logging/types"; + +export type Options = BaseOptions & { + session: Session +}; + +/** Base class for view models that need to report errors to the UI. */ +export class ErrorReportViewModel extends ViewModel { + private _errorViewModel?: ErrorViewModel; + + get errorViewModel(): ErrorViewModel | undefined { + return this._errorViewModel; + } + + protected reportError(error: Error) { + if (this._errorViewModel?.error === error) { + return; + } + this.disposeTracked(this._errorViewModel); + this._errorViewModel = this.track(new ErrorViewModel(this.childOptions({ + error, + onClose: () => { + this._errorViewModel = this.disposeTracked(this._errorViewModel); + this.emitChange("errorViewModel"); + } + }))); + this.emitChange("errorViewModel"); + } + + protected logAndCatch(labelOrValues: LabelOrValues, callback: LogCallback, errorValue: T = undefined as unknown as T): T { + try { + const result = this.logger.run(labelOrValues, callback); + if (result instanceof Promise) { + result.catch(err => this.reportError(err)); + } + return result; + } catch (err) { + this.reportError(err); + return errorValue; + } + } +} diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index e87af12a..ee0ad502 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -15,7 +15,8 @@ limitations under the License. */ import {AvatarSource} from "../../AvatarSource"; -import {ViewModel, Options as BaseOptions} from "../../ViewModel"; +import type {ViewModel} from "../../ViewModel"; +import {ErrorReportViewModel, Options as BaseOptions} from "../../ErrorReportViewModel"; import {getStreamVideoTrack, getStreamAudioTrack} from "../../../matrix/calls/common"; import {avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../avatar"; import {EventObservableValue} from "../../../observable/value/EventObservableValue"; @@ -34,12 +35,10 @@ import type { Session } from "../../../matrix/Session"; type Options = BaseOptions & { call: GroupCall, room: Room, - session: Session }; -export class CallViewModel extends ViewModel { +export class CallViewModel extends ErrorReportViewModel { public readonly memberViewModels: BaseObservableList; - private _errorViewModel?: ErrorViewModel; constructor(options: Options) { super(options); @@ -92,75 +91,58 @@ export class CallViewModel extends ViewModel { return this.call.id; } - get errorViewModel(): ErrorViewModel | undefined { - return this._errorViewModel; - } - private get call(): GroupCall { return this.getOption("call"); } private onUpdate() { if (this.call.error) { - this._reportError(this.call.error); + this.reportError(this.call.error); } } async hangup() { - try { + this.logAndCatch("Call.hangup", async log => { if (this.call.hasJoined) { await this.call.leave(); } - } catch (err) { - this._reportError(err); - } + }); } async toggleCamera() { - const {localMedia, muteSettings} = this.call; - if (muteSettings && localMedia) { - // unmute but no track? - if (muteSettings.camera && !getStreamVideoTrack(localMedia.userMedia)) { - const stream = await this.platform.mediaDevices.getMediaTracks(!muteSettings.microphone, true); - await this.call.setMedia(localMedia.withUserMedia(stream)); - } else { - await this.call.setMuted(muteSettings.toggleCamera()); + this.logAndCatch("Call.toggleCamera", async log => { + const {localMedia, muteSettings} = this.call; + if (muteSettings && localMedia) { + // unmute but no track? + if (muteSettings.camera && !getStreamVideoTrack(localMedia.userMedia)) { + const stream = await this.platform.mediaDevices.getMediaTracks(!muteSettings.microphone, true); + await this.call.setMedia(localMedia.withUserMedia(stream)); + } else { + await this.call.setMuted(muteSettings.toggleCamera()); + } + this.emitChange(); } - this.emitChange(); - } + }); } async toggleMicrophone() { - const {localMedia, muteSettings} = this.call; - if (muteSettings && localMedia) { - // unmute but no track? - if (muteSettings.microphone && !getStreamAudioTrack(localMedia.userMedia)) { - const stream = await this.platform.mediaDevices.getMediaTracks(true, !muteSettings.camera); - await this.call.setMedia(localMedia.withUserMedia(stream)); - } else { - await this.call.setMuted(muteSettings.toggleMicrophone()); + this.logAndCatch("Call.toggleMicrophone", async log => { + const {localMedia, muteSettings} = this.call; + if (muteSettings && localMedia) { + // unmute but no track? + if (muteSettings.microphone && !getStreamAudioTrack(localMedia.userMedia)) { + const stream = await this.platform.mediaDevices.getMediaTracks(true, !muteSettings.camera); + await this.call.setMedia(localMedia.withUserMedia(stream)); + } else { + await this.call.setMuted(muteSettings.toggleMicrophone()); + } + this.emitChange(); } - this.emitChange(); - } - } - - private _reportError(error: Error) { - if (this._errorViewModel?.error === error) { - return; - } - this.disposeTracked(this._errorViewModel); - this._errorViewModel = new ErrorViewModel(this.childOptions({ - error, - onClose: () => { - this._errorViewModel = this.disposeTracked(this._errorViewModel); - this.emitChange("errorViewModel"); - } - })); - this.emitChange("errorViewModel"); + }); } } -class OwnMemberViewModel extends ViewModel implements IStreamViewModel { +class OwnMemberViewModel extends ErrorReportViewModel implements IStreamViewModel { private memberObservable: undefined | BaseObservableValue; constructor(options: Options) { @@ -233,12 +215,9 @@ class OwnMemberViewModel extends ViewModel implements IStreamViewModel type MemberOptions = BaseOptions & { member: Member, mediaRepository: MediaRepository, - session: Session }; -export class CallMemberViewModel extends ViewModel implements IStreamViewModel { - private _errorViewModel?: ErrorViewModel; - +export class CallMemberViewModel extends ErrorReportViewModel implements IStreamViewModel { get stream(): Stream | undefined { return this.member.remoteMedia?.userMedia; } @@ -247,10 +226,6 @@ export class CallMemberViewModel extends ViewModel implements ISt return this.getOption("member"); } - get errorViewModel(): ErrorViewModel | undefined { - return this._errorViewModel; - } - get isCameraMuted(): boolean { return this.member.remoteMuteSettings?.camera ?? true; } @@ -282,16 +257,8 @@ export class CallMemberViewModel extends ViewModel implements ISt } private mapMemberSyncErrorIfNeeded() { - if (this.member.error && (!this._errorViewModel || this._errorViewModel.error !== this.member.error)) { - this.disposeTracked(this._errorViewModel); - this._errorViewModel = this.track(new ErrorViewModel(this.childOptions({ - error: this.member.error, - onClose: () => { - this._errorViewModel = this.disposeTracked(this._errorViewModel); - this.emitChange("errorViewModel"); - }, - }))); - this.emitChange("errorViewModel"); + if (this.member.error) { + this.reportError(this.member.error); } } diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index d744e5d5..a5dfdc8f 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -20,7 +20,7 @@ import {ComposerViewModel} from "./ComposerViewModel.js" import {CallViewModel} from "./CallViewModel" import {PickMapObservableValue} from "../../../observable/value/PickMapObservableValue"; import {avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../avatar"; -import {ViewModel} from "../../ViewModel"; +import {ErrorReportViewModel} from "../../ErrorReportViewModel"; import {imageToInfo} from "../common.js"; import {LocalMedia} from "../../../matrix/calls/LocalMedia"; import { ErrorViewModel } from "../../ErrorViewModel"; @@ -28,7 +28,7 @@ import { ErrorViewModel } from "../../ErrorViewModel"; // this is a breaking SDK change though to make this option mandatory import {tileClassForEntry as defaultTileClassForEntry} from "./timeline/tiles/index"; -export class RoomViewModel extends ViewModel { +export class RoomViewModel extends ErrorReportViewModel { constructor(options) { super(options); const {room, tileClassForEntry} = options; @@ -37,7 +37,6 @@ export class RoomViewModel extends ViewModel { this._tileClassForEntry = tileClassForEntry ?? defaultTileClassForEntry; this._tileOptions = undefined; this._onRoomChange = this._onRoomChange.bind(this); - this._errorViewModel = null; this._composerVM = null; if (room.isArchived) { this._composerVM = new ArchivedViewModel(this.childOptions({archivedRoom: room})); @@ -73,14 +72,6 @@ export class RoomViewModel extends ViewModel { } } - _reportError(error) { - this._errorViewModel = new ErrorViewModel(this.childOptions({error, onClose: () => { - this._errorViewModel = null; - this.emitChange("errorViewModel"); - }})); - this.emitChange("errorViewModel"); - } - async load() { this._room.on("change", this._onRoomChange); try { @@ -97,7 +88,7 @@ export class RoomViewModel extends ViewModel { }))); this.emitChange("timelineViewModel"); } catch (error) { - this._reportError(error); + this.reportError(error); } this._clearUnreadAfterDelay(); } @@ -216,7 +207,7 @@ export class RoomViewModel extends ViewModel { await this._room.sendEvent("m.room.message", {msgtype, body: message}); } } catch (error) { - this._reportError(error); + this.reportError(error); return false; } return true; @@ -287,7 +278,7 @@ export class RoomViewModel extends ViewModel { this._room.createAttachment(thumbnail.blob, file.name); await this._room.sendEvent("m.room.message", content, attachments); } catch (error) { - this._reportError(error); + this.reportError(error); } } @@ -327,7 +318,7 @@ export class RoomViewModel extends ViewModel { } await this._room.sendEvent("m.room.message", content, attachments); } catch (error) { - this._reportError(error); + this.reportError(error); } } @@ -362,7 +353,7 @@ export class RoomViewModel extends ViewModel { const stream = await this.platform.mediaDevices.getMediaTracks(false, true); localMedia = new LocalMedia().withUserMedia(stream); } catch (err) { - this._reportError(new Error(`Could not get local audio and/or video stream: ${err.message}`)); + this.reportError(new Error(`Could not get local audio and/or video stream: ${err.message}`)); return; } const session = this.getOption("session"); @@ -371,13 +362,13 @@ export class RoomViewModel extends ViewModel { // this will set the callViewModel above as a call will be added to callHandler.calls call = await session.callHandler.createCall(this._room.id, "m.video", "A call " + Math.round(this.platform.random() * 100)); } catch (err) { - this._reportError(new Error(`Could not create call: ${err.message}`)); + this.reportError(new Error(`Could not create call: ${err.message}`)); return; } try { await call.join(localMedia); } catch (err) { - this._reportError(new Error(`Could not join call: ${err.message}`)); + this.reportError(new Error(`Could not join call: ${err.message}`)); return; } } diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index a3008cb9..44c39eef 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -28,7 +28,6 @@ export class CallTile extends SimpleTile { const calls = this.getOption("session").callHandler.calls; this._call = calls.get(this._entry.stateKey); this._callSubscription = undefined; - this._errorViewModel = undefined; if (this._call) { this._callSubscription = this._call.disposableOn("change", () => { // unsubscribe when terminated @@ -61,10 +60,6 @@ export class CallTile extends SimpleTile { return this._call && this._call.hasJoined; } - get errorViewModel() { - return this._errorViewModel; - } - get label() { if (this._call) { if (this._call.hasJoined) { @@ -78,25 +73,21 @@ export class CallTile extends SimpleTile { } async join() { - if (this.canJoin) { - try { + await this.logAndCatch("Call.join", async log => { + if (this.canJoin) { const stream = await this.platform.mediaDevices.getMediaTracks(false, true); const localMedia = new LocalMedia().withUserMedia(stream); await this._call.join(localMedia); - } catch (error) { - this._reportError(error); } - } + }); } async leave() { - if (this.canLeave) { - try { - this._call.leave(); - } catch (err) { - this._reportError(err); + await this.logAndCatch("Call.leave", async log => { + if (this.canLeave) { + await this._call.leave(); } - } + }); } dispose() { @@ -104,12 +95,4 @@ export class CallTile extends SimpleTile { this._callSubscription = this._callSubscription(); } } - - _reportError(error) { - this._errorViewModel = new ErrorViewModel(this.childOptions({error, onClose: () => { - this._errorViewModel = undefined; - this.emitChange("errorViewModel"); - }})); - this.emitChange("errorViewModel"); - } } diff --git a/src/domain/session/room/timeline/tiles/SimpleTile.js b/src/domain/session/room/timeline/tiles/SimpleTile.js index 18e6ba17..aa26da81 100644 --- a/src/domain/session/room/timeline/tiles/SimpleTile.js +++ b/src/domain/session/room/timeline/tiles/SimpleTile.js @@ -15,10 +15,10 @@ limitations under the License. */ import {UpdateAction} from "../UpdateAction.js"; -import {ViewModel} from "../../../../ViewModel"; +import {ErrorReportViewModel} from "../../../../ErrorReportViewModel"; import {SendStatus} from "../../../../../matrix/room/sending/PendingEvent.js"; -export class SimpleTile extends ViewModel { +export class SimpleTile extends ErrorReportViewModel { constructor(entry, options) { super(options); this._entry = entry; From 80be2b74570df5e4d78cdb160ae5271ab07a5eb0 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 13 Jan 2023 14:20:29 +0100 Subject: [PATCH 288/435] fix missing import --- src/domain/session/room/RoomViewModel.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index a5dfdc8f..c864222e 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -21,6 +21,7 @@ import {CallViewModel} from "./CallViewModel" import {PickMapObservableValue} from "../../../observable/value/PickMapObservableValue"; import {avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../avatar"; import {ErrorReportViewModel} from "../../ErrorReportViewModel"; +import {ViewModel} from "../../ViewModel"; import {imageToInfo} from "../common.js"; import {LocalMedia} from "../../../matrix/calls/LocalMedia"; import { ErrorViewModel } from "../../ErrorViewModel"; From 29a7b0451e9b6f4167ae36298841677dd87a6a5f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 13 Jan 2023 14:20:33 +0100 Subject: [PATCH 289/435] prevent errors in promises from being uncaught by returning a promise that has the error swallowed --- src/domain/ErrorReportViewModel.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/domain/ErrorReportViewModel.ts b/src/domain/ErrorReportViewModel.ts index eb6ed0aa..68fe580c 100644 --- a/src/domain/ErrorReportViewModel.ts +++ b/src/domain/ErrorReportViewModel.ts @@ -49,9 +49,12 @@ export class ErrorReportViewModel extends ViewModel protected logAndCatch(labelOrValues: LabelOrValues, callback: LogCallback, errorValue: T = undefined as unknown as T): T { try { - const result = this.logger.run(labelOrValues, callback); + let result = this.logger.run(labelOrValues, callback); if (result instanceof Promise) { - result.catch(err => this.reportError(err)); + result = result.catch(err => { + this.reportError(err); + return errorValue; + }) as unknown as T; } return result; } catch (err) { From cec9f6b691b7473c98d6cbaf8cb2b86cf8494e70 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 13 Jan 2023 21:26:01 +0100 Subject: [PATCH 290/435] WIP for docs on error handling --- doc/error-handling.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/error-handling.md diff --git a/doc/error-handling.md b/doc/error-handling.md new file mode 100644 index 00000000..5e136735 --- /dev/null +++ b/doc/error-handling.md @@ -0,0 +1,13 @@ +# Error handling + +Ideally, every error that is unexpected and can't be automatically recovered from without degrading the experience is shown in the UI. This is the task of the view model, and you can use `ErrorReportViewModel`, a dedicated base view model class for this purpose. It exposes a child view model, `ErrorViewModel`, when `reportError` is called which can be paired with `ErrorView` in the view to present an error message from which debug logs can also be sent. + +Methods on classes from the `matrix` layer can often throw errors and those errors should be caught in the view model and reported with `reportError`. As a convenience method, there is also `logAndCatch` which calls a callback within a log item and also a try catch that reports the error. + +## Sync errors + +There are some errors that are throw during background processes though, most notably the sync loop. These processes are not triggered by the view model directly, and hence there is not always a method call they can wrap in a try/catch. For this, there is the `ErrorBoundary` utility class. Since almost all aspects of the client can be updated through the sync loop, it is not too helpful if there is only one try/catch around the whole sync and we stop sync if something goes wrong. + +Instead, it's more helpful to split up the error handling into different scopes, where errors are stored and not rethrown when leaving the scope. One example is to have a scope per room. In this way, we can isolate an error occuring during sync to a specific room, and report it in the UI of that room. + +There is an extra complication though. The `writeSync` sync lifecycle step should not swallow any errors, or data loss can occur. This is because the whole `writeSync` lifecycle step is writes all changes (for all rooms, the session, ...) for a sync response in one transaction. This includes the sync token. So if there is an error in `writeSync` of a given room preventing storing all changes the sync response would cause, \ No newline at end of file From f853871722254a460855369ed2cc19a8ee868cb0 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 16 Jan 2023 10:11:04 +0100 Subject: [PATCH 291/435] some rewording on error handling doc --- doc/error-handling.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/error-handling.md b/doc/error-handling.md index 5e136735..21c67ee5 100644 --- a/doc/error-handling.md +++ b/doc/error-handling.md @@ -1,13 +1,13 @@ # Error handling -Ideally, every error that is unexpected and can't be automatically recovered from without degrading the experience is shown in the UI. This is the task of the view model, and you can use `ErrorReportViewModel`, a dedicated base view model class for this purpose. It exposes a child view model, `ErrorViewModel`, when `reportError` is called which can be paired with `ErrorView` in the view to present an error message from which debug logs can also be sent. +Ideally, every error that is unexpected and can't be automatically recovered from without degrading the experience is shown in the UI. This is the task of the view model, and you can use `ErrorReportViewModel` for this purpose, a dedicated base view model class. It exposes a child view model, `ErrorViewModel`, when `reportError` is called which can be paired with `ErrorView` in the view to present an error message from which debug logs can also be sent. -Methods on classes from the `matrix` layer can often throw errors and those errors should be caught in the view model and reported with `reportError`. As a convenience method, there is also `logAndCatch` which calls a callback within a log item and also a try catch that reports the error. +Methods on classes from the `matrix` layer can often throw errors and those errors should be caught in the view model and reported with `reportError`. As a convenience method, there is also `logAndCatch` when inheriting from `ErrorReportViewModel` which combines logging and error reporting; it calls a callback within a log item and also a try catch that reports the error. -## Sync errors +## Sync errors & ErrorBoundary -There are some errors that are throw during background processes though, most notably the sync loop. These processes are not triggered by the view model directly, and hence there is not always a method call they can wrap in a try/catch. For this, there is the `ErrorBoundary` utility class. Since almost all aspects of the client can be updated through the sync loop, it is not too helpful if there is only one try/catch around the whole sync and we stop sync if something goes wrong. +There are some errors that are thrown during background processes though, most notably the sync loop. These processes are not triggered by the view model directly, and hence there is not always a method call they can wrap in a try/catch. For this, there is the `ErrorBoundary` utility class. Since almost all aspects of the client can be updated through the sync loop, it is also not too helpful if there is only one try/catch around the whole sync and we stop sync if something goes wrong. Instead, it's more helpful to split up the error handling into different scopes, where errors are stored and not rethrown when leaving the scope. One example is to have a scope per room. In this way, we can isolate an error occuring during sync to a specific room, and report it in the UI of that room. -There is an extra complication though. The `writeSync` sync lifecycle step should not swallow any errors, or data loss can occur. This is because the whole `writeSync` lifecycle step is writes all changes (for all rooms, the session, ...) for a sync response in one transaction. This includes the sync token. So if there is an error in `writeSync` of a given room preventing storing all changes the sync response would cause, \ No newline at end of file +There is an extra complication though. The `writeSync` sync lifecycle step should not swallow any errors, or data loss can occur. This is because the whole `writeSync` lifecycle step writes all changes (for all rooms, the session, ...) for a sync response in one transaction (including the sync token), and aborts the transaction and stops sync if there is an error thrown during this step. So if there is an error in `writeSync` of a given room, it's fair to assume not all changes it was planning to write were passed to the transaction, as it got interrupted by the exception. Therefore, if we would swallow the error, data loss can occur as we'd not get another chance to write these changes to disk as we would have advanced the sync token. Therefore, code in the `writeSync` lifecycle step should be written defensively but always throw. From 1de92af2eb4622ae14a7b51816e1db99add2f3c5 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 17 Jan 2023 08:25:08 +0000 Subject: [PATCH 292/435] Update src/domain/session/room/CallViewModel.ts Co-authored-by: R Midhun Suresh --- src/domain/session/room/CallViewModel.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index ee0ad502..98473888 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -21,7 +21,7 @@ import {getStreamVideoTrack, getStreamAudioTrack} from "../../../matrix/calls/co import {avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../avatar"; import {EventObservableValue} from "../../../observable/value/EventObservableValue"; import {ObservableValueMap} from "../../../observable/map/ObservableValueMap"; -import { ErrorViewModel } from "../../ErrorViewModel"; +import {ErrorViewModel} from "../../ErrorViewModel"; import type {Room} from "../../../matrix/room/Room"; import type {GroupCall} from "../../../matrix/calls/group/GroupCall"; import type {Member} from "../../../matrix/calls/group/Member"; From 7d80fbda4cadc14b96d4a46f7958f747d79fb2e7 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 17 Jan 2023 08:25:16 +0000 Subject: [PATCH 293/435] Update src/domain/session/room/CallViewModel.ts Co-authored-by: R Midhun Suresh --- src/domain/session/room/CallViewModel.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 98473888..df2da3c6 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -30,7 +30,7 @@ import type {BaseObservableList} from "../../../observable/list/BaseObservableLi import type {BaseObservableValue} from "../../../observable/value/BaseObservableValue"; import type {Stream} from "../../../platform/types/MediaDevices"; import type {MediaRepository} from "../../../matrix/net/MediaRepository"; -import type { Session } from "../../../matrix/Session"; +import type {Session} from "../../../matrix/Session"; type Options = BaseOptions & { call: GroupCall, From f421cdd4f2e2c400c243b683ce2a623b5219010b Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 17 Jan 2023 08:25:33 +0000 Subject: [PATCH 294/435] Update src/domain/session/room/RoomViewModel.js Co-authored-by: R Midhun Suresh --- src/domain/session/room/RoomViewModel.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index c864222e..421041e9 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -24,7 +24,7 @@ import {ErrorReportViewModel} from "../../ErrorReportViewModel"; import {ViewModel} from "../../ViewModel"; import {imageToInfo} from "../common.js"; import {LocalMedia} from "../../../matrix/calls/LocalMedia"; -import { ErrorViewModel } from "../../ErrorViewModel"; +import {ErrorViewModel} from "../../ErrorViewModel"; // TODO: remove fallback so default isn't included in bundle for SDK users that have their custom tileClassForEntry // this is a breaking SDK change though to make this option mandatory import {tileClassForEntry as defaultTileClassForEntry} from "./timeline/tiles/index"; From cc653884a515e04d38d37f814b9447f38659fb06 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 17 Jan 2023 09:28:23 +0100 Subject: [PATCH 295/435] remove getter that is now in parent class --- src/domain/session/room/RoomViewModel.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 421041e9..0c815949 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -141,10 +141,6 @@ export class RoomViewModel extends ErrorReportViewModel { get timelineViewModel() { return this._timelineVM; } get isEncrypted() { return this._room.isEncrypted; } - get errorViewModel() { - return this._errorViewModel; - } - get avatarLetter() { return avatarInitials(this.name); } From a2c44484b21a96dfcc47121fd2af936a08b29cb2 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 17 Jan 2023 09:29:03 +0100 Subject: [PATCH 296/435] newline --- src/domain/rageshake.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/rageshake.ts b/src/domain/rageshake.ts index 08e82d8c..cc8aec07 100644 --- a/src/domain/rageshake.ts +++ b/src/domain/rageshake.ts @@ -91,4 +91,4 @@ export async function submitLogsFromSessionToDefaultServer(session: Session, pla bugReportEndpointUrl, platform.request ); -} \ No newline at end of file +} From 0dbb7d4e50d7f947cd1d2b4e978c91b988a16c9d Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 17 Jan 2023 11:37:49 +0100 Subject: [PATCH 297/435] use logAndCatch in RoomViewModel, everything reporting errors also logs --- src/domain/session/room/RoomViewModel.js | 127 +++++++++++------------ 1 file changed, 61 insertions(+), 66 deletions(-) diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 0c815949..a3542ebb 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -74,9 +74,9 @@ export class RoomViewModel extends ErrorReportViewModel { } async load() { - this._room.on("change", this._onRoomChange); - try { - const timeline = await this._room.openTimeline(); + this.logAndCatch("Room.load", async log => { + this._room.on("change", this._onRoomChange); + const timeline = await this._room.openTimeline(log); this._tileOptions = this.childOptions({ session: this.getOption("session"), roomVM: this, @@ -88,13 +88,11 @@ export class RoomViewModel extends ErrorReportViewModel { timeline, }))); this.emitChange("timelineViewModel"); - } catch (error) { - this.reportError(error); - } - this._clearUnreadAfterDelay(); + await this._clearUnreadAfterDelay(log); + }); } - async _clearUnreadAfterDelay() { + async _clearUnreadAfterDelay(log) { if (this._room.isArchived || this._clearUnreadTimout) { return; } @@ -104,7 +102,9 @@ export class RoomViewModel extends ErrorReportViewModel { await this._room.clearUnread(); this._clearUnreadTimout = null; } catch (err) { - if (err.name !== "AbortError") { + if (err.name === "AbortError") { + log.set("clearUnreadCancelled", true); + } else { throw err; } } @@ -190,62 +190,61 @@ export class RoomViewModel extends ErrorReportViewModel { } } - async _sendMessage(message, replyingTo) { - if (!this._room.isArchived && message) { - try { + _sendMessage(message, replyingTo) { + return this.logAndCatch("sendMessage", async log => { + let success = false; + if (!this._room.isArchived && message) { let msgtype = "m.text"; if (message.startsWith("/me ")) { message = message.substr(4).trim(); msgtype = "m.emote"; } if (replyingTo) { + log.set("replyingTo", replyingTo.eventId); + // TODO: reply should not send? it should just return the content for the reply and we send it ourselves await replyingTo.reply(msgtype, message); } else { - await this._room.sendEvent("m.room.message", {msgtype, body: message}); + await this._room.sendEvent("m.room.message", {msgtype, body: message}, undefined, log); } - } catch (error) { - this.reportError(error); - return false; + success = true; } - return true; - } - return false; + log.set("success", success); + return success; + }, false); } - async _pickAndSendFile() { - try { + _pickAndSendFile() { + return this.logAndCatch("sendFile", async log => { const file = await this.platform.openFile(); if (!file) { + log.set("cancelled", true); return; } - return this._sendFile(file); - } catch (err) { - console.error(err); - } + return this._sendFile(file, log); + }); } - async _sendFile(file) { + async _sendFile(file, log) { const content = { body: file.name, msgtype: "m.file" }; await this._room.sendEvent("m.room.message", content, { "url": this._room.createAttachment(file.blob, file.name) - }); + }, log); } - async _pickAndSendVideo() { - try { + _pickAndSendVideo() { + return this.logAndCatch("sendVideo", async log => { if (!this.platform.hasReadPixelPermission()) { - alert("Please allow canvas image data access, so we can scale your images down."); - return; + throw new Error("Please allow canvas image data access, so we can scale your images down."); } const file = await this.platform.openFile("video/*"); if (!file) { return; } if (!file.blob.mimeType.startsWith("video/")) { - return this._sendFile(file); + return this._sendFile(file, log); } let video; try { @@ -273,24 +272,23 @@ export class RoomViewModel extends ErrorReportViewModel { content.info.thumbnail_info = imageToInfo(thumbnail); attachments["info.thumbnail_url"] = this._room.createAttachment(thumbnail.blob, file.name); - await this._room.sendEvent("m.room.message", content, attachments); - } catch (error) { - this.reportError(error); - } + await this._room.sendEvent("m.room.message", content, attachments, log); + }); } async _pickAndSendPicture() { - try { + this.logAndCatch("sendPicture", async log => { if (!this.platform.hasReadPixelPermission()) { alert("Please allow canvas image data access, so we can scale your images down."); return; } const file = await this.platform.openFile("image/*"); if (!file) { + log.set("cancelled", true); return; } if (!file.blob.mimeType.startsWith("image/")) { - return this._sendFile(file); + return this._sendFile(file, log); } let image = await this.platform.loadImage(file.blob); const limit = await this.platform.settingsStorage.getInt("sentImageSizeLimit"); @@ -313,10 +311,8 @@ export class RoomViewModel extends ErrorReportViewModel { attachments["info.thumbnail_url"] = this._room.createAttachment(thumbnail.blob, file.name); } - await this._room.sendEvent("m.room.message", content, attachments); - } catch (error) { - this.reportError(error); - } + await this._room.sendEvent("m.room.message", content, attachments, log); + }); } get room() { @@ -344,30 +340,29 @@ export class RoomViewModel extends ErrorReportViewModel { } } - async startCall() { - let localMedia; - try { - const stream = await this.platform.mediaDevices.getMediaTracks(false, true); - localMedia = new LocalMedia().withUserMedia(stream); - } catch (err) { - this.reportError(new Error(`Could not get local audio and/or video stream: ${err.message}`)); - return; - } - const session = this.getOption("session"); - let call; - try { - // this will set the callViewModel above as a call will be added to callHandler.calls - call = await session.callHandler.createCall(this._room.id, "m.video", "A call " + Math.round(this.platform.random() * 100)); - } catch (err) { - this.reportError(new Error(`Could not create call: ${err.message}`)); - return; - } - try { - await call.join(localMedia); - } catch (err) { - this.reportError(new Error(`Could not join call: ${err.message}`)); - return; - } + startCall() { + return this.logAndCatch("startCall", async log => { + let localMedia; + try { + const stream = await this.platform.mediaDevices.getMediaTracks(false, true); + localMedia = new LocalMedia().withUserMedia(stream); + } catch (err) { + throw new Error(`Could not get local audio and/or video stream: ${err.message}`); + } + const session = this.getOption("session"); + let call; + try { + // this will set the callViewModel above as a call will be added to callHandler.calls + call = await session.callHandler.createCall(this._room.id, "m.video", "A call " + Math.round(this.platform.random() * 100)); + } catch (err) { + throw new Error(`Could not create call: ${err.message}`); + } + try { + await call.join(localMedia); + } catch (err) { + throw new Error(`Could not join call: ${err.message}`); + } + }); } } From e33209b747140c51d83a585e01358251adb70f5a Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 17 Jan 2023 12:23:20 +0100 Subject: [PATCH 298/435] start logging in view model and pass it on to model methods (calls+room) --- src/domain/session/room/CallViewModel.ts | 4 +- src/domain/session/room/RoomViewModel.js | 30 ++-- .../session/room/timeline/tiles/CallTile.js | 9 +- src/matrix/calls/CallHandler.ts | 141 ++++++++++-------- src/matrix/calls/group/GroupCall.ts | 95 ++++++------ 5 files changed, 151 insertions(+), 128 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index df2da3c6..dd942878 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -102,9 +102,9 @@ export class CallViewModel extends ErrorReportViewModel { } async hangup() { - this.logAndCatch("Call.hangup", async log => { + this.logAndCatch("CallViewModel.hangup", async log => { if (this.call.hasJoined) { - await this.call.leave(); + await this.call.leave(log); } }); } diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index a3542ebb..5a29bd85 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -24,7 +24,6 @@ import {ErrorReportViewModel} from "../../ErrorReportViewModel"; import {ViewModel} from "../../ViewModel"; import {imageToInfo} from "../common.js"; import {LocalMedia} from "../../../matrix/calls/LocalMedia"; -import {ErrorViewModel} from "../../ErrorViewModel"; // TODO: remove fallback so default isn't included in bundle for SDK users that have their custom tileClassForEntry // this is a breaking SDK change though to make this option mandatory import {tileClassForEntry as defaultTileClassForEntry} from "./timeline/tiles/index"; @@ -74,7 +73,7 @@ export class RoomViewModel extends ErrorReportViewModel { } async load() { - this.logAndCatch("Room.load", async log => { + this.logAndCatch("RoomViewModel.load", async log => { this._room.on("change", this._onRoomChange); const timeline = await this._room.openTimeline(log); this._tileOptions = this.childOptions({ @@ -99,7 +98,7 @@ export class RoomViewModel extends ErrorReportViewModel { this._clearUnreadTimout = this.clock.createTimeout(2000); try { await this._clearUnreadTimout.elapsed(); - await this._room.clearUnread(); + await this._room.clearUnread(log); this._clearUnreadTimout = null; } catch (err) { if (err.name === "AbortError") { @@ -111,7 +110,9 @@ export class RoomViewModel extends ErrorReportViewModel { } focus() { - this._clearUnreadAfterDelay(); + this.logAndCatch("RoomViewModel.focus", async log => { + this._clearUnreadAfterDelay(log); + }); } dispose() { @@ -191,7 +192,7 @@ export class RoomViewModel extends ErrorReportViewModel { } _sendMessage(message, replyingTo) { - return this.logAndCatch("sendMessage", async log => { + return this.logAndCatch("RoomViewModel.sendMessage", async log => { let success = false; if (!this._room.isArchived && message) { let msgtype = "m.text"; @@ -214,7 +215,7 @@ export class RoomViewModel extends ErrorReportViewModel { } _pickAndSendFile() { - return this.logAndCatch("sendFile", async log => { + return this.logAndCatch("RoomViewModel.sendFile", async log => { const file = await this.platform.openFile(); if (!file) { log.set("cancelled", true); @@ -235,7 +236,7 @@ export class RoomViewModel extends ErrorReportViewModel { } _pickAndSendVideo() { - return this.logAndCatch("sendVideo", async log => { + return this.logAndCatch("RoomViewModel.sendVideo", async log => { if (!this.platform.hasReadPixelPermission()) { throw new Error("Please allow canvas image data access, so we can scale your images down."); } @@ -277,7 +278,7 @@ export class RoomViewModel extends ErrorReportViewModel { } async _pickAndSendPicture() { - this.logAndCatch("sendPicture", async log => { + this.logAndCatch("RoomViewModel.sendPicture", async log => { if (!this.platform.hasReadPixelPermission()) { alert("Please allow canvas image data access, so we can scale your images down."); return; @@ -341,7 +342,8 @@ export class RoomViewModel extends ErrorReportViewModel { } startCall() { - return this.logAndCatch("startCall", async log => { + return this.logAndCatch("RoomViewModel.startCall", async log => { + log.set("roomId", this._room.id); let localMedia; try { const stream = await this.platform.mediaDevices.getMediaTracks(false, true); @@ -353,12 +355,18 @@ export class RoomViewModel extends ErrorReportViewModel { let call; try { // this will set the callViewModel above as a call will be added to callHandler.calls - call = await session.callHandler.createCall(this._room.id, "m.video", "A call " + Math.round(this.platform.random() * 100)); + call = await session.callHandler.createCall( + this._room.id, + "m.video", + "A call " + Math.round(this.platform.random() * 100), + undefined, + log + ); } catch (err) { throw new Error(`Could not create call: ${err.message}`); } try { - await call.join(localMedia); + await call.join(localMedia, log); } catch (err) { throw new Error(`Could not join call: ${err.message}`); } diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index 44c39eef..c792496e 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -16,7 +16,6 @@ limitations under the License. import {SimpleTile} from "./SimpleTile.js"; import {LocalMedia} from "../../../../../matrix/calls/LocalMedia"; -import {ErrorViewModel} from "../../../../ErrorViewModel" // TODO: timeline entries for state events with the same state key and type // should also update previous entries in the timeline, so we can update the name of the call, whether it is terminated, etc ... @@ -73,19 +72,19 @@ export class CallTile extends SimpleTile { } async join() { - await this.logAndCatch("Call.join", async log => { + await this.logAndCatch("CallTile.join", async log => { if (this.canJoin) { const stream = await this.platform.mediaDevices.getMediaTracks(false, true); const localMedia = new LocalMedia().withUserMedia(stream); - await this._call.join(localMedia); + await this._call.join(localMedia, log); } }); } async leave() { - await this.logAndCatch("Call.leave", async log => { + await this.logAndCatch("CallTile.leave", async log => { if (this.canLeave) { - await this._call.leave(); + await this._call.leave(log); } }); } diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 2b914a2f..51afcda9 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -65,16 +65,26 @@ export class CallHandler implements RoomStateHandler { }); } - async loadCalls(intent: CallIntent = CallIntent.Ring) { - const txn = await this._getLoadTxn(); - const callEntries = await txn.calls.getByIntent(intent); - this._loadCallEntries(callEntries, txn); + loadCalls(intent?: CallIntent, log?: ILogItem) { + return this.options.logger.wrapOrRun(log, "CallHandler.loadCalls", async log => { + if (!intent) { + intent = CallIntent.Ring; + } + log.set("intent", intent); + const txn = await this._getLoadTxn(); + const callEntries = await txn.calls.getByIntent(intent); + await this._loadCallEntries(callEntries, txn, log); + }); } - async loadCallsForRoom(intent: CallIntent, roomId: string) { - const txn = await this._getLoadTxn(); - const callEntries = await txn.calls.getByIntentAndRoom(intent, roomId); - this._loadCallEntries(callEntries, txn); + loadCallsForRoom(intent: CallIntent, roomId: string, log?: ILogItem) { + return this.options.logger.wrapOrRun(log, "CallHandler.loadCallsForRoom", async log => { + log.set("intent", intent); + log.set("roomId", roomId); + const txn = await this._getLoadTxn(); + const callEntries = await txn.calls.getByIntentAndRoom(intent, roomId); + await this._loadCallEntries(callEntries, txn, log); + }); } private async _getLoadTxn(): Promise { @@ -86,68 +96,71 @@ export class CallHandler implements RoomStateHandler { return txn; } - private async _loadCallEntries(callEntries: CallEntry[], txn: Transaction): Promise { - return this.options.logger.run({l: "loading calls", t: CALL_LOG_TYPE}, async log => { - log.set("entries", callEntries.length); - await Promise.all(callEntries.map(async callEntry => { - if (this._calls.get(callEntry.callId)) { - return; + private async _loadCallEntries(callEntries: CallEntry[], txn: Transaction, log: ILogItem): Promise { + log.set("entries", callEntries.length); + await Promise.all(callEntries.map(async callEntry => { + if (this._calls.get(callEntry.callId)) { + return; + } + const event = await txn.roomState.get(callEntry.roomId, EventType.GroupCall, callEntry.callId); + if (event) { + const call = new GroupCall(event.event.state_key, false, event.event.content, event.roomId, this.groupCallOptions); + this._calls.set(call.id, call); + } + })); + const roomIds = Array.from(new Set(callEntries.map(e => e.roomId))); + await Promise.all(roomIds.map(async roomId => { + // TODO: don't load all members until we need them + const callsMemberEvents = await txn.roomState.getAllForType(roomId, EventType.GroupCallMember); + await Promise.all(callsMemberEvents.map(async entry => { + const userId = entry.event.sender; + const roomMemberState = await txn.roomState.get(roomId, MEMBER_EVENT_TYPE, userId); + let roomMember; + if (roomMemberState) { + roomMember = RoomMember.fromMemberEvent(roomMemberState.event); } - const event = await txn.roomState.get(callEntry.roomId, EventType.GroupCall, callEntry.callId); - if (event) { - const call = new GroupCall(event.event.state_key, false, event.event.content, event.roomId, this.groupCallOptions); - this._calls.set(call.id, call); + if (!roomMember) { + // we'll be missing the member here if we received a call and it's members + // as pre-gap state and the members weren't active in the timeline we got. + roomMember = RoomMember.fromUserId(roomId, userId, "join"); } + this.handleCallMemberEvent(entry.event, roomMember, roomId, log); })); - const roomIds = Array.from(new Set(callEntries.map(e => e.roomId))); - await Promise.all(roomIds.map(async roomId => { - // TODO: don't load all members until we need them - const callsMemberEvents = await txn.roomState.getAllForType(roomId, EventType.GroupCallMember); - await Promise.all(callsMemberEvents.map(async entry => { - const userId = entry.event.sender; - const roomMemberState = await txn.roomState.get(roomId, MEMBER_EVENT_TYPE, userId); - let roomMember; - if (roomMemberState) { - roomMember = RoomMember.fromMemberEvent(roomMemberState.event); - } - if (!roomMember) { - // we'll be missing the member here if we received a call and it's members - // as pre-gap state and the members weren't active in the timeline we got. - roomMember = RoomMember.fromUserId(roomId, userId, "join"); - } - this.handleCallMemberEvent(entry.event, roomMember, roomId, log); - })); - })); - log.set("newSize", this._calls.size); - }); + })); + log.set("newSize", this._calls.size); } - async createCall(roomId: string, type: "m.video" | "m.voice", name: string, intent: CallIntent = CallIntent.Ring): Promise { - const call = new GroupCall(makeId("conf-"), true, { - "m.name": name, - "m.intent": intent - }, roomId, this.groupCallOptions); - this._calls.set(call.id, call); + createCall(roomId: string, type: "m.video" | "m.voice", name: string, intent?: CallIntent, log?: ILogItem): Promise { + return this.options.logger.wrapOrRun(log, "CallHandler.createCall", async log => { + if (!intent) { + intent = CallIntent.Ring; + } + const call = new GroupCall(makeId("conf-"), true, { + "m.name": name, + "m.intent": intent + }, roomId, this.groupCallOptions); + this._calls.set(call.id, call); - try { - await call.create(type); - // store call info so it will ring again when reopening the app - const txn = await this.options.storage.readWriteTxn([this.options.storage.storeNames.calls]); - txn.calls.add({ - intent: call.intent, - callId: call.id, - timestamp: this.options.clock.now(), - roomId: roomId - }); - await txn.complete(); - } catch (err) { - //if (err.name === "ConnectionError") { - // if we're offline, give up and remove the call again - this._calls.remove(call.id); - //} - throw err; - } - return call; + try { + await call.create(type, log); + // store call info so it will ring again when reopening the app + const txn = await this.options.storage.readWriteTxn([this.options.storage.storeNames.calls]); + txn.calls.add({ + intent: call.intent, + callId: call.id, + timestamp: this.options.clock.now(), + roomId: roomId + }); + await txn.complete(); + } catch (err) { + //if (err.name === "ConnectionError") { + // if we're offline, give up and remove the call again + this._calls.remove(call.id); + //} + throw err; + } + return call; + }); } get calls(): BaseObservableMap { return this._calls; } diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 53a87d5f..d700f2a0 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -162,45 +162,48 @@ export class GroupCall extends EventEmitter<{change: never}> { return this.errorBoundary.error; } - async join(localMedia: LocalMedia): Promise { - if (this._state !== GroupCallState.Created || this.joinedData) { - return; - } - const logItem = this.options.logger.child({ - l: "answer call", - t: CALL_LOG_TYPE, - id: this.id, - ownSessionId: this.options.sessionId - }); - const turnServer = await this.options.turnServerSource.getSettings(logItem); - const membersLogItem = logItem.child("member connections"); - const localMuteSettings = new MuteSettings(); - localMuteSettings.updateTrackInfo(localMedia.userMedia); - const localPreviewMedia = localMedia.asPreview(); - const joinedData = new JoinedData( - logItem, - membersLogItem, - localMedia, - localPreviewMedia, - localMuteSettings, - turnServer - ); - this.joinedData = joinedData; - await joinedData.logItem.wrap("join", async log => { - this._state = GroupCallState.Joining; - this.emitChange(); - await log.wrap("update member state", async log => { - const memberContent = await this._createMemberPayload(true); - log.set("payload", memberContent); - // send m.call.member state event - const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCallMember, this.options.ownUserId, memberContent, {log}); - await request.response(); - this.emitChange(); - }); - // send invite to all members that are < my userId - for (const [,member] of this._members) { - this.connectToMember(member, joinedData, log); + join(localMedia: LocalMedia, log?: ILogItem): Promise { + return this.options.logger.wrapOrRun(log, "Call.join", async joinLog => { + if (this._state !== GroupCallState.Created || this.joinedData) { + return; } + const logItem = this.options.logger.child({ + l: "Call.connection", + t: CALL_LOG_TYPE, + id: this.id, + ownSessionId: this.options.sessionId + }); + const turnServer = await this.options.turnServerSource.getSettings(logItem); + const membersLogItem = logItem.child("member connections"); + const localMuteSettings = new MuteSettings(); + localMuteSettings.updateTrackInfo(localMedia.userMedia); + const localPreviewMedia = localMedia.asPreview(); + const joinedData = new JoinedData( + logItem, + membersLogItem, + localMedia, + localPreviewMedia, + localMuteSettings, + turnServer + ); + this.joinedData = joinedData; + await joinedData.logItem.wrap("join", async log => { + joinLog.refDetached(log); + this._state = GroupCallState.Joining; + this.emitChange(); + await log.wrap("update member state", async log => { + const memberContent = await this._createMemberPayload(true); + log.set("payload", memberContent); + // send m.call.member state event + const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCallMember, this.options.ownUserId, memberContent, {log}); + await request.response(); + this.emitChange(); + }); + // send invite to all members that are < my userId + for (const [,member] of this._members) { + this.connectToMember(member, joinedData, log); + } + }); }); } @@ -263,12 +266,12 @@ export class GroupCall extends EventEmitter<{change: never}> { return this._state === GroupCallState.Joining || this._state === GroupCallState.Joined; } - async leave(): Promise { - const {joinedData} = this; - if (!joinedData) { - return; - } - await joinedData.logItem.wrap("leave", async log => { + async leave(log?: ILogItem): Promise { + await this.options.logger.wrapOrRun(log, "Call.leave", async log => { + const {joinedData} = this; + if (!joinedData) { + return; + } try { joinedData.renewMembershipTimeout?.dispose(); joinedData.renewMembershipTimeout = undefined; @@ -310,8 +313,8 @@ export class GroupCall extends EventEmitter<{change: never}> { } /** @internal */ - create(type: "m.video" | "m.voice", log?: ILogItem): Promise { - return this.options.logger.wrapOrRun(log, {l: "create call", t: CALL_LOG_TYPE}, async log => { + create(type: "m.video" | "m.voice", log: ILogItem): Promise { + return log.wrap({l: "create call", t: CALL_LOG_TYPE}, async log => { if (this._state !== GroupCallState.Fledgling) { return; } From dfaaf6d23470b4d74707fba84ef5939be22213f0 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 17 Jan 2023 12:32:05 +0100 Subject: [PATCH 299/435] cleanup reply code a bit, have only 1 path to send message --- doc/impl-thoughts/RELATIONS.md | 2 +- src/domain/session/room/RoomViewModel.js | 7 ++++--- src/domain/session/room/timeline/tiles/BaseMessageTile.js | 4 ++-- src/matrix/room/timeline/entries/BaseEventEntry.js | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/doc/impl-thoughts/RELATIONS.md b/doc/impl-thoughts/RELATIONS.md index 5d91c28e..ac4e43ce 100644 --- a/doc/impl-thoughts/RELATIONS.md +++ b/doc/impl-thoughts/RELATIONS.md @@ -237,7 +237,7 @@ room.sendEvent(eventEntry.eventType, replacement); ## Replies ```js -const reply = eventEntry.reply({}); +const reply = eventEntry.createReplyContent({}); room.sendEvent("m.room.message", reply); ``` diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 5a29bd85..8c551172 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -200,13 +200,14 @@ export class RoomViewModel extends ErrorReportViewModel { message = message.substr(4).trim(); msgtype = "m.emote"; } + let content; if (replyingTo) { log.set("replyingTo", replyingTo.eventId); - // TODO: reply should not send? it should just return the content for the reply and we send it ourselves - await replyingTo.reply(msgtype, message); + content = await replyingTo.createReplyContent(msgtype, message); } else { - await this._room.sendEvent("m.room.message", {msgtype, body: message}, undefined, log); + content = {msgtype, body: message}; } + await this._room.sendEvent("m.room.message", content, undefined, log); success = true; } log.set("success", success); diff --git a/src/domain/session/room/timeline/tiles/BaseMessageTile.js b/src/domain/session/room/timeline/tiles/BaseMessageTile.js index 5a28e1b3..44e0c5b0 100644 --- a/src/domain/session/room/timeline/tiles/BaseMessageTile.js +++ b/src/domain/session/room/timeline/tiles/BaseMessageTile.js @@ -146,8 +146,8 @@ export class BaseMessageTile extends SimpleTile { this._roomVM.startReply(this._entry); } - reply(msgtype, body, log = null) { - return this._room.sendEvent("m.room.message", this._entry.reply(msgtype, body), null, log); + createReplyContent(msgtype, body) { + return this._entry.createReplyContent(msgtype, body); } redact(reason, log) { diff --git a/src/matrix/room/timeline/entries/BaseEventEntry.js b/src/matrix/room/timeline/entries/BaseEventEntry.js index 44fdcaec..39698557 100644 --- a/src/matrix/room/timeline/entries/BaseEventEntry.js +++ b/src/matrix/room/timeline/entries/BaseEventEntry.js @@ -181,7 +181,7 @@ export class BaseEventEntry extends BaseEntry { return createAnnotation(this.id, key); } - reply(msgtype, body) { + createReplyContent(msgtype, body) { return createReplyContent(this, msgtype, body); } From bf9c868c8bc8f71cb9ed99301ccd0f580bcfb98d Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 17 Jan 2023 17:18:07 +0100 Subject: [PATCH 300/435] make it clearer that logAndCatch is probably what you want --- doc/error-handling.md | 6 ++++-- src/domain/ErrorReportViewModel.ts | 7 +++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/doc/error-handling.md b/doc/error-handling.md index 21c67ee5..ac2c4e59 100644 --- a/doc/error-handling.md +++ b/doc/error-handling.md @@ -2,12 +2,14 @@ Ideally, every error that is unexpected and can't be automatically recovered from without degrading the experience is shown in the UI. This is the task of the view model, and you can use `ErrorReportViewModel` for this purpose, a dedicated base view model class. It exposes a child view model, `ErrorViewModel`, when `reportError` is called which can be paired with `ErrorView` in the view to present an error message from which debug logs can also be sent. -Methods on classes from the `matrix` layer can often throw errors and those errors should be caught in the view model and reported with `reportError`. As a convenience method, there is also `logAndCatch` when inheriting from `ErrorReportViewModel` which combines logging and error reporting; it calls a callback within a log item and also a try catch that reports the error. +Methods on classes from the `matrix` layer can often throw errors and those errors should be caught in the view model and reported to the UI. When inheriting from `ErrorReportViewModel`, there is the low-level `reportError` method, but typically you'd use the convenience method `logAndCatch`. The latter makes it easy to get both error handlikng and logging right. You would typically use `logAndCatch` for every public method in the view model (e.g methods called from the view or from the parent view model). It calls a callback within a log item and also a try catch that reports the error. ## Sync errors & ErrorBoundary There are some errors that are thrown during background processes though, most notably the sync loop. These processes are not triggered by the view model directly, and hence there is not always a method call they can wrap in a try/catch. For this, there is the `ErrorBoundary` utility class. Since almost all aspects of the client can be updated through the sync loop, it is also not too helpful if there is only one try/catch around the whole sync and we stop sync if something goes wrong. -Instead, it's more helpful to split up the error handling into different scopes, where errors are stored and not rethrown when leaving the scope. One example is to have a scope per room. In this way, we can isolate an error occuring during sync to a specific room, and report it in the UI of that room. +Instead, it's more helpful to split up the error handling into different scopes, where errors are stored and not rethrown when leaving the scope. One example is to have a scope per room. In this way, we can isolate an error occuring during sync to a specific room, and report it in the UI of that room. This is typically where you would use `reportError` from `ErrorReportViewModel` rather than `logAndCatch`. You observe changes from your model in the view model (see docs on updates), and if the `error` property is set (by the `ErrorBoundary`), you call reportError with it. You can do this repeatedly without problems, if the same error is already reported, it's a No-Op. + +### `writeSync` and preventing data loss when dealing with errors. There is an extra complication though. The `writeSync` sync lifecycle step should not swallow any errors, or data loss can occur. This is because the whole `writeSync` lifecycle step writes all changes (for all rooms, the session, ...) for a sync response in one transaction (including the sync token), and aborts the transaction and stops sync if there is an error thrown during this step. So if there is an error in `writeSync` of a given room, it's fair to assume not all changes it was planning to write were passed to the transaction, as it got interrupted by the exception. Therefore, if we would swallow the error, data loss can occur as we'd not get another chance to write these changes to disk as we would have advanced the sync token. Therefore, code in the `writeSync` lifecycle step should be written defensively but always throw. diff --git a/src/domain/ErrorReportViewModel.ts b/src/domain/ErrorReportViewModel.ts index 68fe580c..da46b2df 100644 --- a/src/domain/ErrorReportViewModel.ts +++ b/src/domain/ErrorReportViewModel.ts @@ -32,6 +32,10 @@ export class ErrorReportViewModel extends ViewModel return this._errorViewModel; } + /** Typically you'd want to use `logAndCatch` when implementing a view model method. + * Use `reportError` when showing errors on your model that were set by + * background processes using `ErrorBoundary` or you have some other + * special low-level need to write your try/catch yourself. */ protected reportError(error: Error) { if (this._errorViewModel?.error === error) { return; @@ -47,6 +51,9 @@ export class ErrorReportViewModel extends ViewModel this.emitChange("errorViewModel"); } + /** Combines logging and error reporting in one method. + * Wrap the implementation of public view model methods + * with this to ensure errors are logged and reported.*/ protected logAndCatch(labelOrValues: LabelOrValues, callback: LogCallback, errorValue: T = undefined as unknown as T): T { try { let result = this.logger.run(labelOrValues, callback); From 3842f450dda28992767c62f382793fc8dc8f5d06 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 19 Jan 2023 11:30:23 +0100 Subject: [PATCH 301/435] ensure errors caught by boundary are logged in calls code --- src/matrix/calls/group/GroupCall.ts | 5 +++++ src/matrix/calls/group/Member.ts | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index d700f2a0..25e201e5 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -94,6 +94,11 @@ export class GroupCall extends EventEmitter<{change: never}> { /** Set between calling join and leave. */ private joinedData?: JoinedData; private errorBoundary = new ErrorBoundary(err => { + if (this.joinedData) { + // in case the error happens in code that does not log, + // log it here to make sure it isn't swallowed + this.joinedData.logItem.log("error at boundary").catch(err); + } this.emitChange(); }); diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 4ceaf46f..257628f5 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -97,6 +97,11 @@ export class Member { private expireTimeout?: Timeout; private errorBoundary = new ErrorBoundary(err => { this.options.emitUpdate(this, "error"); + if (this.connection) { + // in case the error happens in code that does not log, + // log it here to make sure it isn't swallowed + this.connection.logItem.log("error at boundary").catch(err); + } }); constructor( From e6b17cc74aeaa6a06e3799914fb4a6e7a0e79412 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 19 Jan 2023 11:30:52 +0100 Subject: [PATCH 302/435] fix callback type --- src/utils/ErrorBoundary.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/ErrorBoundary.ts b/src/utils/ErrorBoundary.ts index d13065f9..a4ac459c 100644 --- a/src/utils/ErrorBoundary.ts +++ b/src/utils/ErrorBoundary.ts @@ -19,7 +19,7 @@ export const ErrorValue = Symbol("ErrorBoundary:Error"); export class ErrorBoundary { private _error?: Error; - constructor(private readonly errorCallback: (Error) => void) {} + constructor(private readonly errorCallback: (err: Error) => void) {} /** * Executes callback() and then runs errorCallback() on error. @@ -79,4 +79,4 @@ export function tests() { assert.strictEqual(result, 0); } } -} \ No newline at end of file +} From 24088506785034a3164aac9f85811b561dcd8e19 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 19 Jan 2023 11:33:21 +0100 Subject: [PATCH 303/435] emit change before logging --- src/matrix/calls/group/GroupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 25e201e5..f6436eeb 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -94,12 +94,12 @@ export class GroupCall extends EventEmitter<{change: never}> { /** Set between calling join and leave. */ private joinedData?: JoinedData; private errorBoundary = new ErrorBoundary(err => { + this.emitChange(); if (this.joinedData) { // in case the error happens in code that does not log, // log it here to make sure it isn't swallowed this.joinedData.logItem.log("error at boundary").catch(err); } - this.emitChange(); }); constructor( From daad19c0604305ed00e40cb06be42a987e3d56c8 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 19 Jan 2023 11:37:39 +0100 Subject: [PATCH 304/435] swallow errors in errorCallback in ErrorBoundary nothing should be able to make ErrorBoundary.try throw --- src/utils/ErrorBoundary.ts | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/utils/ErrorBoundary.ts b/src/utils/ErrorBoundary.ts index a4ac459c..a74da479 100644 --- a/src/utils/ErrorBoundary.ts +++ b/src/utils/ErrorBoundary.ts @@ -32,18 +32,26 @@ export class ErrorBoundary { if (result instanceof Promise) { result = result.catch(err => { this._error = err; - this.errorCallback(err); + this.reportError(err); return errorValue; }); } return result; } catch (err) { this._error = err; - this.errorCallback(err); + this.reportError(err); return errorValue; } } + private reportError(err: Error) { + try { + this.errorCallback(err); + } catch (err) { + console.error("error in ErrorBoundary callback", err); + } + } + get error(): Error | undefined { return this._error; } @@ -77,6 +85,15 @@ export function tests() { }, 0); assert(emitted); assert.strictEqual(result, 0); + }, + "exception in error callback is swallowed": async assert => { + let emitted = false; + const boundary = new ErrorBoundary(() => { throw new Error("bug in errorCallback"); }); + assert.doesNotThrow(() => { + boundary.try(() => { + throw new Error("fail!"); + }); + }); } } } From 289414702736ed42c314850f5334ed2304c81125 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Thu, 19 Jan 2023 12:02:57 +0000 Subject: [PATCH 305/435] Set Quentin and Hugh as codeowners for docker related files --- .github/CODEOWNERS | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..3dd1a6ab --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,7 @@ +# Docker related files are not maintained by the core Hydrogen team +/.dockerignore @hughns @sandhose +/Dockerfile @hughns @sandhose +/Dockerfile-dev @hughns @sandhose +/.github/workflows/docker-publish.yml @hughns @sandhose +/docker/ @hughns @sandhose +/doc/docker.md @hughns @sandhose From ba097d23bc526d586b0578ccacc236be837532e4 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Thu, 19 Jan 2023 15:01:51 +0000 Subject: [PATCH 306/435] Upgrade GitHub Actions versions to use Node 16 based runtime Node 14 is deprecated and is EOL on 30 Apr 2023 --- .github/workflows/docker-publish.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 0a322a3d..dad55339 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -21,10 +21,10 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Log into registry ${{ env.REGISTRY }} - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -32,12 +32,12 @@ jobs: - name: Extract Docker metadata id: meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@v4 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - name: Build and push Docker image - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} From b993bc09556935e1c0b9bc685bf9da4b1115a331 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Thu, 19 Jan 2023 17:07:55 +0000 Subject: [PATCH 307/435] Build docker image for multiple architectures including ARM This is based on https://www.docker.com/blog/faster-multi-platform-builds-dockerfile-cross-compilation-guide/ which doesn't use QEMU --- .github/workflows/docker-publish.yml | 4 ++++ Dockerfile | 4 ++-- doc/docker.md | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index dad55339..a70eea6c 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -23,6 +23,9 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Log into registry ${{ env.REGISTRY }} uses: docker/login-action@v2 with: @@ -42,3 +45,4 @@ jobs: push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + platforms: linux/amd64,linux/arm64,linux/arm/v7 diff --git a/Dockerfile b/Dockerfile index f35dc559..4d8be56b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/node:alpine as builder +FROM --platform=${BUILDPLATFORM} docker.io/node:alpine as builder RUN apk add --no-cache git python3 build-base WORKDIR /app @@ -10,7 +10,7 @@ RUN yarn install COPY . /app RUN yarn build -FROM docker.io/nginx:alpine +FROM --platform=${BUILDPLATFORM} docker.io/nginx:alpine # Copy the dynamic config script COPY ./docker/dynamic-config.sh /docker-entrypoint.d/99-dynamic-config.sh diff --git a/doc/docker.md b/doc/docker.md index db779fe1..752c492f 100644 --- a/doc/docker.md +++ b/doc/docker.md @@ -35,7 +35,9 @@ To stop the container, simply hit `ctrl+c`. In this repository, create a Docker image: -``` +```sh +# Enable BuildKit https://docs.docker.com/develop/develop-images/build_enhancements/ +export DOCKER_BUILDKIT=1 docker build -t hydrogen . ``` From d27aa137cfb22f22ddaa87a7cf793143f5c69a91 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Fri, 20 Jan 2023 13:31:14 +0000 Subject: [PATCH 308/435] Build a docker image using unprivileged nginx BREAKING CHANGE the exposed port has changed from 80 to 8080 --- Dockerfile | 9 ++++++++- doc/docker.md | 4 ++-- docker/dynamic-config.sh | 7 +++++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4d8be56b..011c7f89 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,10 +10,17 @@ RUN yarn install COPY . /app RUN yarn build -FROM --platform=${BUILDPLATFORM} docker.io/nginx:alpine +# Because we will be running as an unprivileged user, we need to make sure that the config file is writable +# So, we will copy the default config to the /tmp folder that will be writable at runtime +RUN mv -f target/config.json /config.json.bundled \ + && ln -sf /tmp/config.json target/config.json + +FROM --platform=${BUILDPLATFORM} docker.io/nginxinc/nginx-unprivileged:alpine # Copy the dynamic config script COPY ./docker/dynamic-config.sh /docker-entrypoint.d/99-dynamic-config.sh +# And the bundled config file +COPY --from=builder /config.json.bundled /config.json.bundled # Copy the built app from the first build stage COPY --from=builder /app/target /usr/share/nginx/html diff --git a/doc/docker.md b/doc/docker.md index 752c492f..336d658a 100644 --- a/doc/docker.md +++ b/doc/docker.md @@ -55,7 +55,7 @@ Then, start up a container from that image: ``` docker run \ --name hydrogen \ - --publish 80:80 \ + --publish 8080:8080 \ hydrogen ``` @@ -64,7 +64,7 @@ You can override the default `config.json` using the `CONFIG_OVERRIDE` environme ``` docker run \ --name hydrogen \ - --publish 80:80 \ + --publish 8080:8080 \ --env CONFIG_OVERRIDE='{ "push": { "appId": "io.element.hydrogen.web", diff --git a/docker/dynamic-config.sh b/docker/dynamic-config.sh index 952cb969..99858a27 100755 --- a/docker/dynamic-config.sh +++ b/docker/dynamic-config.sh @@ -2,7 +2,10 @@ set -eux -# Use config override environment variable if set if [ -n "${CONFIG_OVERRIDE:-}" ]; then - echo "$CONFIG_OVERRIDE" > /usr/share/nginx/html/config.json + # Use config override environment variable if set + echo "$CONFIG_OVERRIDE" > /tmp/config.json +else + # Otherwise, use the default config that was bundled in the image + cp /config.json.bundled /tmp/config.json fi From b2feaf2b4e83d72718140da5892ce82e6d645e01 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 20 Jan 2023 16:50:07 +0100 Subject: [PATCH 309/435] fix mixing up params, causing us to call onUpdate on params rather than vm --- src/domain/session/room/CallViewModel.ts | 2 +- src/matrix/calls/group/GroupCall.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 0ba774a2..484543b7 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -55,7 +55,7 @@ export class CallViewModel extends ErrorReportViewModel vm.onUpdate(), + (param, vm) => vm?.onUpdate(), ) as BaseObservableMap; this.memberViewModels = otherMemberViewModels .join(ownMemberViewModelMap) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index efb321b2..0b8c7db5 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -126,7 +126,7 @@ export class GroupCall extends EventEmitter<{change: never}> { member.dispose(); this._members.remove(memberKey); } else { - this._members.update(memberKey, member); + this._members.update(memberKey); } }, encryptDeviceMessage: (userId: string, deviceId: string, message: SignallingMessage, log) => { From 6645f8f43b1ad2ffd532462bdf4584101dae43f1 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 20 Jan 2023 16:50:15 +0100 Subject: [PATCH 310/435] remove debug throw --- src/matrix/calls/group/Member.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 71c8b7f9..e097804c 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -196,7 +196,6 @@ export class Member { connectLogItem = log; await this.callIfNeeded(log); }); - throw new Error("connect failed!"); return connectLogItem; }); } From 590e9500c179b7d5238cdd60abfc7b9e24163716 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 20 Jan 2023 17:33:16 +0100 Subject: [PATCH 311/435] clone localMedia,so we don't remove audio track from stream sent to peer --- src/matrix/calls/LocalMedia.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 7a673ba9..adecfc18 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -43,7 +43,7 @@ export class LocalMedia { * Create an instance of LocalMedia without audio track (for user preview) */ asPreview(): LocalMedia { - const media = new LocalMedia(this.userMedia, this.screenShare, this.dataChannelOptions); + const media = this.clone(); const userMedia = media.userMedia; if (userMedia && userMedia.getVideoTracks().length > 0) { const audioTrack = getStreamAudioTrack(userMedia); From 2bd37970ba8a8267545378a407057c317fa6aa1a Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 20 Jan 2023 17:34:15 +0100 Subject: [PATCH 312/435] each LocalMedia own their streams, so a copy should have their own clone --- src/matrix/calls/LocalMedia.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index adecfc18..b7a14a9a 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -28,15 +28,15 @@ export class LocalMedia { ) {} withUserMedia(stream: Stream) { - return new LocalMedia(stream, this.screenShare, this.dataChannelOptions); + return new LocalMedia(stream, this.screenShare?.clone(), this.dataChannelOptions); } withScreenShare(stream: Stream) { - return new LocalMedia(this.userMedia, stream, this.dataChannelOptions); + return new LocalMedia(this.userMedia?.clone(), stream, this.dataChannelOptions); } withDataChannel(options: RTCDataChannelInit): LocalMedia { - return new LocalMedia(this.userMedia, this.screenShare, options); + return new LocalMedia(this.userMedia?.clone(), this.screenShare?.clone(), options); } /** From a808d9395cb064701b9c1028c8a3659e989df616 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Fri, 20 Jan 2023 17:54:01 +0000 Subject: [PATCH 313/435] Add note about change in port number and writable /tmp --- doc/docker.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/docker.md b/doc/docker.md index 336d658a..41632909 100644 --- a/doc/docker.md +++ b/doc/docker.md @@ -59,6 +59,8 @@ docker run \ hydrogen ``` +n.b. the image is now based on the unprivileged nginx base, so the port is now `8080` instead of `80` and you need a writable `/tmp` volume. + You can override the default `config.json` using the `CONFIG_OVERRIDE` environment variable. For example to specify a different Homeserver and : ``` From d763e697996fa4b4bfc02faa0c2aec00abc85173 Mon Sep 17 00:00:00 2001 From: Matthew Cengia Date: Mon, 23 Jan 2023 08:28:20 +0700 Subject: [PATCH 314/435] Use target platform (rather than build platform) for multi-platform container images Signed-off-by: Matthew Cengia --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 011c7f89..3a430d6c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ RUN yarn build RUN mv -f target/config.json /config.json.bundled \ && ln -sf /tmp/config.json target/config.json -FROM --platform=${BUILDPLATFORM} docker.io/nginxinc/nginx-unprivileged:alpine +FROM --platform=${TARGETPLATFORM} docker.io/nginxinc/nginx-unprivileged:alpine # Copy the dynamic config script COPY ./docker/dynamic-config.sh /docker-entrypoint.d/99-dynamic-config.sh From 3bb889ed9c1efb6c093811adb9bc1e79a885f74d Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 23 Jan 2023 14:53:08 +0100 Subject: [PATCH 315/435] WIP --- .../session/room/timeline/tiles/CallTile.js | 61 ++++++++++++++++--- src/matrix/calls/CallHandler.ts | 4 +- src/matrix/calls/callEventTypes.ts | 5 ++ src/matrix/calls/group/GroupCall.ts | 8 ++- .../web/ui/css/themes/element/timeline.css | 9 ++- .../ui/session/room/timeline/CallTileView.ts | 12 +++- 6 files changed, 83 insertions(+), 16 deletions(-) diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index c792496e..1758e6bb 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -15,7 +15,11 @@ limitations under the License. */ import {SimpleTile} from "./SimpleTile.js"; +import {ViewModel} from "../../../../ViewModel"; import {LocalMedia} from "../../../../../matrix/calls/LocalMedia"; +import {CallType} from "../../../../../matrix/calls/callEventTypes"; +import {avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../../../avatar"; + // TODO: timeline entries for state events with the same state key and type // should also update previous entries in the timeline, so we can update the name of the call, whether it is terminated, etc ... @@ -28,17 +32,28 @@ export class CallTile extends SimpleTile { this._call = calls.get(this._entry.stateKey); this._callSubscription = undefined; if (this._call) { - this._callSubscription = this._call.disposableOn("change", () => { + this._callSubscription = this.track(this._call.disposableOn("change", () => { // unsubscribe when terminated if (this._call.isTerminated) { this._callSubscription = this._callSubscription(); this._call = undefined; } this.emitChange(); - }); + })); + this.memberViewModels = this._setupMembersList(this._call); } } + _setupMembersList(call) { + return call.members.mapValues( + (member, emitChange) => new MemberAvatarViewModel(this.childOptions({ + member, + emitChange, + mediaRepository: this.getOption("room").mediaRepository + })), + ).sortValues((a, b) => a.avatarTitle < b.avatarTitle ? -1 : 1); + } + get confId() { return this._entry.stateKey; } @@ -61,16 +76,24 @@ export class CallTile extends SimpleTile { get label() { if (this._call) { - if (this._call.hasJoined) { - return `Ongoing call (${this.name}, ${this.confId})`; + if (this._type === CallType.Video) { + return `${this.displayName} started a video call`; } else { - return `${this.displayName} started a call (${this.name}, ${this.confId})`; + return `${this.displayName} started a voice call`; } } else { - return `Call finished, started by ${this.displayName} (${this.name}, ${this.confId})`; + if (this._type === CallType.Video) { + return `Video call ended`; + } else { + return `Voice call ended`; + } } } + get _type() { + return this._entry.event.content["m.type"]; + } + async join() { await this.logAndCatch("CallTile.join", async log => { if (this.canJoin) { @@ -88,10 +111,28 @@ export class CallTile extends SimpleTile { } }); } +} - dispose() { - if (this._callSubscription) { - this._callSubscription = this._callSubscription(); - } +class MemberAvatarViewModel extends ViewModel { + get _member() { + return this.getOption("member"); + } + + get avatarLetter() { + return avatarInitials(this._member.member.name); + } + + get avatarColorNumber() { + return getIdentifierColorNumber(this._member.userId); + } + + avatarUrl(size) { + const {avatarUrl} = this._member.member; + const mediaRepository = this.getOption("mediaRepository"); + return getAvatarHttpUrl(avatarUrl, size, this.platform, mediaRepository); + } + + get avatarTitle() { + return this._member.member.name; } } diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 6be6b193..f6533f4f 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -18,7 +18,7 @@ import {ObservableMap} from "../../observable/map"; import {WebRTC, PeerConnection} from "../../platform/types/WebRTC"; import {MediaDevices, Track} from "../../platform/types/MediaDevices"; import {handlesEventType} from "./PeerCall"; -import {EventType, CallIntent} from "./callEventTypes"; +import {EventType, CallIntent, CallType} from "./callEventTypes"; import {GroupCall} from "./group/GroupCall"; import {makeId} from "../common"; import {CALL_LOG_TYPE} from "./common"; @@ -130,7 +130,7 @@ export class CallHandler implements RoomStateHandler { log.set("newSize", this._calls.size); } - createCall(roomId: string, type: "m.video" | "m.voice", name: string, intent?: CallIntent, log?: ILogItem): Promise { + createCall(roomId: string, type: CallType, name: string, intent?: CallIntent, log?: ILogItem): Promise { return this.options.logger.wrapOrRun(log, "CallHandler.createCall", async log => { if (!intent) { intent = CallIntent.Ring; diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index 8fd7b23d..a0eb986c 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -227,3 +227,8 @@ export enum CallIntent { Prompt = "m.prompt", Room = "m.room", }; + +export enum CallType { + Video = "m.video", + Voice = "m.voice", +} diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 0b8c7db5..241e486c 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -20,7 +20,7 @@ import {LocalMedia} from "../LocalMedia"; import {MuteSettings, CALL_LOG_TYPE, CALL_MEMBER_VALIDITY_PERIOD_MS, mute} from "../common"; import {MemberChange, RoomMember} from "../../room/members/RoomMember"; import {EventEmitter} from "../../../utils/EventEmitter"; -import {EventType, CallIntent} from "../callEventTypes"; +import {EventType, CallIntent, CallType} from "../callEventTypes"; import { ErrorBoundary } from "../../../utils/ErrorBoundary"; import type {Options as MemberOptions} from "./Member"; @@ -155,6 +155,10 @@ export class GroupCall extends EventEmitter<{change: never}> { return this.callContent?.["m.intent"]; } + get type(): CallType { + return this.callContent?.["m.type"]; + } + /** * Gives access the log item for this call while joined. * Can be used for call diagnostics while in the call. @@ -318,7 +322,7 @@ export class GroupCall extends EventEmitter<{change: never}> { } /** @internal */ - create(type: "m.video" | "m.voice", log: ILogItem): Promise { + create(type: CallType, log: ILogItem): Promise { return log.wrap({l: "create call", t: CALL_LOG_TYPE}, async log => { if (this._state !== GroupCallState.Fledgling) { return; diff --git a/src/platform/web/ui/css/themes/element/timeline.css b/src/platform/web/ui/css/themes/element/timeline.css index 7ff35eb1..81a81960 100644 --- a/src/platform/web/ui/css/themes/element/timeline.css +++ b/src/platform/web/ui/css/themes/element/timeline.css @@ -440,4 +440,11 @@ only loads when the top comes into view*/ background-color: var(--background-color-primary); border-radius: 8px; text-align: center; - } +} + +.CallTileView_members > * { + margin-left: -16px; +} +.CallTileView_members { + display: flex; +} diff --git a/src/platform/web/ui/session/room/timeline/CallTileView.ts b/src/platform/web/ui/session/room/timeline/CallTileView.ts index 5bede510..c99c2136 100644 --- a/src/platform/web/ui/session/room/timeline/CallTileView.ts +++ b/src/platform/web/ui/session/room/timeline/CallTileView.ts @@ -17,7 +17,16 @@ limitations under the License. import {Builder, TemplateView} from "../../../general/TemplateView"; import type {CallTile} from "../../../../../../domain/session/room/timeline/tiles/CallTile"; import {ErrorView} from "../../../general/ErrorView"; - +import {ListView} from "../../../general/ListView"; +import {AvatarView} from "../../../AvatarView"; +/* +.CallTileView_members > * { + margin-left: -16px; +} +.CallTileView_members { + display: flex; +} +*/ export class CallTileView extends TemplateView { render(t: Builder, vm: CallTile) { return t.li( @@ -29,6 +38,7 @@ export class CallTileView extends TemplateView { }), t.div([ vm => vm.label, + t.view(new ListView({className: "CallTileView_members", list: vm.memberViewModels}, vm => new AvatarView(vm, 24))), t.button({className: "CallTileView_join", hidden: vm => !vm.canJoin}, "Join"), t.button({className: "CallTileView_leave", hidden: vm => !vm.canLeave}, "Leave") ]) From af5cc0f62b709724e661de8220f5c0b89b34886c Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 23 Jan 2023 15:35:28 +0100 Subject: [PATCH 316/435] sort by userId, sorting order needs to be stable --- src/domain/session/room/timeline/tiles/CallTile.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index 1758e6bb..b762cef4 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -51,7 +51,7 @@ export class CallTile extends SimpleTile { emitChange, mediaRepository: this.getOption("room").mediaRepository })), - ).sortValues((a, b) => a.avatarTitle < b.avatarTitle ? -1 : 1); + ).sortValues((a, b) => a.userId.localeCompare(b.userId)); } get confId() { @@ -118,6 +118,10 @@ class MemberAvatarViewModel extends ViewModel { return this.getOption("member"); } + get userId() { + return this._member.userId; + } + get avatarLetter() { return avatarInitials(this._member.member.name); } From 1df8d31ab5f4b812544b6ff18c7a4a389aae9854 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 23 Jan 2023 17:06:33 +0100 Subject: [PATCH 317/435] show call duration in tile --- .../session/room/timeline/tiles/CallTile.js | 30 +++++++++++++++---- src/matrix/calls/CallHandler.ts | 8 ++--- src/matrix/calls/group/GroupCall.ts | 15 ++++++++-- src/platform/types/types.ts | 1 + src/platform/web/dom/TimeFormatter.ts | 24 +++++++++++++++ .../ui/session/room/timeline/CallTileView.ts | 10 ++----- 6 files changed, 68 insertions(+), 20 deletions(-) diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index b762cef4..71297646 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -33,17 +33,27 @@ export class CallTile extends SimpleTile { this._callSubscription = undefined; if (this._call) { this._callSubscription = this.track(this._call.disposableOn("change", () => { - // unsubscribe when terminated - if (this._call.isTerminated) { - this._callSubscription = this._callSubscription(); - this._call = undefined; - } - this.emitChange(); + this._onCallUpdate(); })); + this._onCallUpdate(); this.memberViewModels = this._setupMembersList(this._call); } } + _onCallUpdate() { + // unsubscribe when terminated + if (this._call.isTerminated) { + this._durationInterval = this.disposeTracked(this._durationInterval); + this._callSubscription = this.disposeTracked(this._callSubscription); + this._call = undefined; + } else if (!this._durationInterval) { + this._durationInterval = this.track(this.platform.clock.createInterval(() => { + this.emitChange("duration"); + }, 1000)); + } + this.emitChange(); + } + _setupMembersList(call) { return call.members.mapValues( (member, emitChange) => new MemberAvatarViewModel(this.childOptions({ @@ -57,6 +67,14 @@ export class CallTile extends SimpleTile { get confId() { return this._entry.stateKey; } + + get duration() { + if (this._call && this._call.duration) { + return this.timeFormatter.formatDuration(this._call.duration); + } else { + return ""; + } + } get shape() { return "call"; diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index f6533f4f..c3f5d917 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -104,7 +104,7 @@ export class CallHandler implements RoomStateHandler { } const event = await txn.roomState.get(callEntry.roomId, EventType.GroupCall, callEntry.callId); if (event) { - const call = new GroupCall(event.event.state_key, false, event.event.content, event.roomId, this.groupCallOptions); + const call = new GroupCall(event.event.state_key, false, callEntry.timestamp, event.event.content, event.roomId, this.groupCallOptions); this._calls.set(call.id, call); } })); @@ -135,7 +135,7 @@ export class CallHandler implements RoomStateHandler { if (!intent) { intent = CallIntent.Ring; } - const call = new GroupCall(makeId("conf-"), true, { + const call = new GroupCall(makeId("conf-"), true, undefined, { "m.name": name, "m.intent": intent }, roomId, this.groupCallOptions); @@ -210,14 +210,14 @@ export class CallHandler implements RoomStateHandler { const callId = event.state_key; let call = this._calls.get(callId); if (call) { - call.updateCallEvent(event.content, log); + call.updateCallEvent(event, log); if (call.isTerminated) { call.disconnect(log); this._calls.remove(call.id); txn.calls.remove(call.intent, roomId, call.id); } } else { - call = new GroupCall(event.state_key, false, event.content, roomId, this.groupCallOptions); + call = new GroupCall(event.state_key, false, event.origin_server_ts, event.content, roomId, this.groupCallOptions); this._calls.set(call.id, call); txn.calls.add({ intent: call.intent, diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 241e486c..5748601f 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -105,6 +105,7 @@ export class GroupCall extends EventEmitter<{change: never}> { constructor( public readonly id: string, newCall: boolean, + private startTime: number | undefined, private callContent: Record, public readonly roomId: string, private readonly options: Options, @@ -143,6 +144,12 @@ export class GroupCall extends EventEmitter<{change: never}> { return this.callContent?.["m.terminated"] === true; } + get duration(): number | undefined { + if (typeof this.startTime === "number") { + return (this.options.clock.now() - this.startTime); + } + } + get isRinging(): boolean { return this._state === GroupCallState.Created && this.intent === "m.ring" && !this.isMember(this.options.ownUserId); } @@ -340,10 +347,14 @@ export class GroupCall extends EventEmitter<{change: never}> { } /** @internal */ - updateCallEvent(callContent: Record, syncLog: ILogItem) { + updateCallEvent(event: StateEvent, syncLog: ILogItem) { this.errorBoundary.try(() => { syncLog.wrap({l: "update call", t: CALL_LOG_TYPE, id: this.id}, log => { - this.callContent = callContent; + + if (typeof this.startTime !== "number") { + this.startTime = event.origin_server_ts; + } + this.callContent = event.content; if (this._state === GroupCallState.Creating) { this._state = GroupCallState.Created; } diff --git a/src/platform/types/types.ts b/src/platform/types/types.ts index 77953c4a..0e2f536e 100644 --- a/src/platform/types/types.ts +++ b/src/platform/types/types.ts @@ -55,4 +55,5 @@ export interface ITimeFormatter { formatTime(date: Date): string; formatRelativeDate(date: Date): string; formatMachineReadableDate(date: Date): string; + formatDuration(milliseconds: number): string; } diff --git a/src/platform/web/dom/TimeFormatter.ts b/src/platform/web/dom/TimeFormatter.ts index 2a98a716..cebc6ec4 100644 --- a/src/platform/web/dom/TimeFormatter.ts +++ b/src/platform/web/dom/TimeFormatter.ts @@ -22,6 +22,9 @@ enum TimeScope { Day = 24 * 60 * 60 * 1000, } +const MINUTES_IN_MS = 60 * 1000; +const HOURS_IN_MS = MINUTES_IN_MS * 60; + export class TimeFormatter implements ITimeFormatter { private todayMidnight: Date; @@ -75,6 +78,27 @@ export class TimeFormatter implements ITimeFormatter { return this.otherYearFormatter.format(date); } } + + formatDuration(milliseconds: number): string { + let hours = 0; + let minutes = 0; + if (milliseconds > HOURS_IN_MS) { + hours = Math.floor(milliseconds / HOURS_IN_MS); + milliseconds -= hours * HOURS_IN_MS; + } + if (milliseconds > MINUTES_IN_MS) { + minutes = Math.floor(milliseconds / MINUTES_IN_MS); + milliseconds -= minutes * MINUTES_IN_MS; + } + const seconds = Math.floor(milliseconds / 1000); + if (hours) { + return `${hours}h ${minutes}m ${seconds}s`; + } + if (minutes) { + return `${minutes}m ${seconds}s`; + } + return `${seconds}s`; + } } function capitalizeFirstLetter(str: string) { diff --git a/src/platform/web/ui/session/room/timeline/CallTileView.ts b/src/platform/web/ui/session/room/timeline/CallTileView.ts index c99c2136..a5c6ae09 100644 --- a/src/platform/web/ui/session/room/timeline/CallTileView.ts +++ b/src/platform/web/ui/session/room/timeline/CallTileView.ts @@ -19,14 +19,7 @@ import type {CallTile} from "../../../../../../domain/session/room/timeline/tile import {ErrorView} from "../../../general/ErrorView"; import {ListView} from "../../../general/ListView"; import {AvatarView} from "../../../AvatarView"; -/* -.CallTileView_members > * { - margin-left: -16px; -} -.CallTileView_members { - display: flex; -} -*/ + export class CallTileView extends TemplateView { render(t: Builder, vm: CallTile) { return t.li( @@ -38,6 +31,7 @@ export class CallTileView extends TemplateView { }), t.div([ vm => vm.label, + vm => vm.duration, t.view(new ListView({className: "CallTileView_members", list: vm.memberViewModels}, vm => new AvatarView(vm, 24))), t.button({className: "CallTileView_join", hidden: vm => !vm.canJoin}, "Join"), t.button({className: "CallTileView_leave", hidden: vm => !vm.canLeave}, "Leave") From 5035d2357349032aa28b8a3c866f8a1de861d03f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 23 Jan 2023 17:42:33 +0100 Subject: [PATCH 318/435] add all elements to call tile that need to be shown, style buttons --- .../session/room/timeline/tiles/CallTile.js | 28 +++++++++++++------ .../web/ui/css/themes/element/timeline.css | 14 ++++++++++ .../ui/session/room/timeline/CallTileView.ts | 22 ++++++++++----- 3 files changed, 49 insertions(+), 15 deletions(-) diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index 71297646..8887b834 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -64,6 +64,14 @@ export class CallTile extends SimpleTile { ).sortValues((a, b) => a.userId.localeCompare(b.userId)); } + get memberCount() { + // TODO: emit updates for this property + if (this._call) { + return this._call.members.size; + } + return 0; + } + get confId() { return this._entry.stateKey; } @@ -80,10 +88,6 @@ export class CallTile extends SimpleTile { return "call"; } - get name() { - return this._entry.content["m.name"]; - } - get canJoin() { return this._call && !this._call.hasJoined; } @@ -92,15 +96,15 @@ export class CallTile extends SimpleTile { return this._call && this._call.hasJoined; } - get label() { + get title() { if (this._call) { - if (this._type === CallType.Video) { + if (this.type === CallType.Video) { return `${this.displayName} started a video call`; } else { return `${this.displayName} started a voice call`; } } else { - if (this._type === CallType.Video) { + if (this.type === CallType.Video) { return `Video call ended`; } else { return `Voice call ended`; @@ -108,7 +112,15 @@ export class CallTile extends SimpleTile { } } - get _type() { + get typeLabel() { + if (this.type === CallType.Video) { + return `Video call`; + } else { + return `Voice call`; + } + } + + get type() { return this._entry.event.content["m.type"]; } diff --git a/src/platform/web/ui/css/themes/element/timeline.css b/src/platform/web/ui/css/themes/element/timeline.css index 81a81960..5dc86276 100644 --- a/src/platform/web/ui/css/themes/element/timeline.css +++ b/src/platform/web/ui/css/themes/element/timeline.css @@ -442,9 +442,23 @@ only loads when the top comes into view*/ text-align: center; } +.CallTileView > div > div { + display: flex; + flex-direction: column; + gap: 4px; +} + .CallTileView_members > * { margin-left: -16px; } .CallTileView_members { display: flex; } + +.CallTileView_memberCount::before { + content: url('./icons/room-members.svg?primary=text-color'); + background-repeat: no-repeat; + background-size: 24px; + width: 24px; + height: 24px; +} diff --git a/src/platform/web/ui/session/room/timeline/CallTileView.ts b/src/platform/web/ui/session/room/timeline/CallTileView.ts index a5c6ae09..a9a872b3 100644 --- a/src/platform/web/ui/session/room/timeline/CallTileView.ts +++ b/src/platform/web/ui/session/room/timeline/CallTileView.ts @@ -30,11 +30,19 @@ export class CallTileView extends TemplateView { return t.div({className: "CallTileView_error"}, t.view(new ErrorView(vm.errorViewModel, {inline: true}))); }), t.div([ - vm => vm.label, - vm => vm.duration, - t.view(new ListView({className: "CallTileView_members", list: vm.memberViewModels}, vm => new AvatarView(vm, 24))), - t.button({className: "CallTileView_join", hidden: vm => !vm.canJoin}, "Join"), - t.button({className: "CallTileView_leave", hidden: vm => !vm.canLeave}, "Leave") + t.div({className: "CallTileView_title"}, vm => vm.title), + t.div({className: "CallTileView_subtitle"}, [ + vm.typeLabel, " • ", + t.span({className: "CallTileView_memberCount"}, vm => vm.memberCount) + ]), + t.view(new ListView({className: "CallTileView_members", tagName: "div", list: vm.memberViewModels}, vm => { + return new AvatarView(vm, 24); + })), + t.div(vm => vm.duration), + t.div([ + t.button({className: "CallTileView_join button-action primary", hidden: vm => !vm.canJoin}, "Join"), + t.button({className: "CallTileView_leave button-action primary destructive", hidden: vm => !vm.canLeave}, "Leave") + ]) ]) ]) ); @@ -42,9 +50,9 @@ export class CallTileView extends TemplateView { /* This is called by the parent ListView, which just has 1 listener for the whole list */ onClick(evt) { - if (evt.target.className === "CallTileView_join") { + if (evt.target.classList.contains("CallTileView_join")) { this.value.join(); - } else if (evt.target.className === "CallTileView_leave") { + } else if (evt.target.classList.contains("CallTileView_leave")) { this.value.leave(); } } From fa5cb684b0917949cb40fae9267a6e980cd86393 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 24 Jan 2023 11:53:49 +0530 Subject: [PATCH 319/435] WIP --- src/domain/avatar.ts | 8 ++ src/domain/session/SessionViewModel.js | 9 ++ .../toast/BaseToastNotificationViewModel.ts | 34 ++++++ .../toast/CallToastNotificationViewModel.ts | 97 +++++++++++++++++ .../session/toast/ToastCollectionViewModel.ts | 88 +++++++++++++++ src/matrix/calls/CallHandler.ts | 6 +- src/matrix/calls/group/GroupCall.ts | 1 + .../web/ui/css/themes/element/theme.css | 100 ++++++++++++++++++ src/platform/web/ui/session/SessionView.js | 2 + .../toast/CallToastNotificationView.ts | 51 +++++++++ .../ui/session/toast/ToastCollectionView.ts | 33 ++++++ 11 files changed, 426 insertions(+), 3 deletions(-) create mode 100644 src/domain/session/toast/BaseToastNotificationViewModel.ts create mode 100644 src/domain/session/toast/CallToastNotificationViewModel.ts create mode 100644 src/domain/session/toast/ToastCollectionViewModel.ts create mode 100644 src/platform/web/ui/session/toast/CallToastNotificationView.ts create mode 100644 src/platform/web/ui/session/toast/ToastCollectionView.ts diff --git a/src/domain/avatar.ts b/src/domain/avatar.ts index ecaccebe..aaf700ef 100644 --- a/src/domain/avatar.ts +++ b/src/domain/avatar.ts @@ -58,3 +58,11 @@ export function getAvatarHttpUrl(avatarUrl: string | undefined, cssSize: number, } return undefined; } + +// move to AvatarView.js when converting to typescript +export interface IAvatarContract { + avatarLetter: string; + avatarColorNumber: number; + avatarUrl: (size: number) => string | undefined; + avatarTitle: string; +} diff --git a/src/domain/session/SessionViewModel.js b/src/domain/session/SessionViewModel.js index eb371ca8..1247f5d7 100644 --- a/src/domain/session/SessionViewModel.js +++ b/src/domain/session/SessionViewModel.js @@ -30,6 +30,7 @@ import {ViewModel} from "../ViewModel"; import {RoomViewModelObservable} from "./RoomViewModelObservable.js"; import {RightPanelViewModel} from "./rightpanel/RightPanelViewModel.js"; import {SyncStatus} from "../../matrix/Sync.js"; +import {ToastCollectionViewModel} from "./toast/ToastCollectionViewModel"; export class SessionViewModel extends ViewModel { constructor(options) { @@ -47,6 +48,10 @@ export class SessionViewModel extends ViewModel { this._gridViewModel = null; this._createRoomViewModel = null; this._joinRoomViewModel = null; + this._toastCollectionViewModel = this.track(new ToastCollectionViewModel(this.childOptions({ + callHandler: this._client.session.callHandler, + session: this._client.session, + }))); this._setupNavigation(); this._setupForcedLogoutOnAccessTokenInvalidation(); } @@ -173,6 +178,10 @@ export class SessionViewModel extends ViewModel { return this._joinRoomViewModel; } + get toastCollectionViewModel() { + return this._toastCollectionViewModel; + } + _updateGrid(roomIds) { const changed = !(this._gridViewModel && roomIds); const currentRoomId = this.navigation.path.get("room"); diff --git a/src/domain/session/toast/BaseToastNotificationViewModel.ts b/src/domain/session/toast/BaseToastNotificationViewModel.ts new file mode 100644 index 00000000..70fefea7 --- /dev/null +++ b/src/domain/session/toast/BaseToastNotificationViewModel.ts @@ -0,0 +1,34 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { ErrorReportViewModel } from "../../ErrorReportViewModel"; +import {Options as BaseOptions} from "../../ViewModel"; +import type {Session} from "../../../matrix/Session.js"; + +export type BaseClassOptions = { + dismiss: () => void; + session: Session; +} & BaseOptions; + +export abstract class BaseToastNotificationViewModel extends ErrorReportViewModel { + constructor(options: T) { + super(options); + } + + dismiss() { + this.getOption("dismiss")(); + } +} diff --git a/src/domain/session/toast/CallToastNotificationViewModel.ts b/src/domain/session/toast/CallToastNotificationViewModel.ts new file mode 100644 index 00000000..0e72c3f9 --- /dev/null +++ b/src/domain/session/toast/CallToastNotificationViewModel.ts @@ -0,0 +1,97 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import type {GroupCall} from "../../../matrix/calls/group/GroupCall"; +import type {Room} from "../../../matrix/room/Room.js"; +import {IAvatarContract, avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../avatar"; +import {LocalMedia} from "../../../matrix/calls/LocalMedia"; +import {BaseClassOptions, BaseToastNotificationViewModel} from "./BaseToastNotificationViewModel"; + +type Options = { + call: GroupCall; + room: Room; +} & BaseClassOptions; + + +export class CallToastNotificationViewModel extends BaseToastNotificationViewModel implements IAvatarContract { + constructor(options: Options) { + super(options); + this.track( + this.call.members.subscribe({ + onAdd: (_, __) => { + this.emitChange("memberCount"); + }, + onUpdate: (_, __) => { + this.emitChange("memberCount"); + }, + onRemove: (_, __) => { + this.emitChange("memberCount"); + }, + onReset: () => { /** noop */ }, + }) + ); + // Dismiss the toast if the room is opened manually + this.track( + this.navigation.observe("room").subscribe(roomId => { + if (roomId === this.call.roomId) { + this.dismiss(); + } + })); + } + + async join() { + await this.logAndCatch("CallToastNotificationViewModel.join", async (log) => { + const stream = await this.platform.mediaDevices.getMediaTracks(false, true); + const localMedia = new LocalMedia().withUserMedia(stream); + await this.call.join(localMedia, log); + const url = this.urlCreator.openRoomActionUrl(this.call.roomId); + this.urlCreator.pushUrl(url); + }); + } + + get call(): GroupCall { + return this.getOption("call"); + } + + private get room(): Room { + return this.getOption("room"); + } + + get roomName(): string { + return this.room.name; + } + + get memberCount(): number { + return this.call.members.size; + } + + get avatarLetter() { + return avatarInitials(this.roomName); + } + + get avatarColorNumber() { + return getIdentifierColorNumber(this.room.avatarColorId); + } + + avatarUrl(size: number) { + return getAvatarHttpUrl(this.room.avatarUrl, size, this.platform, this.room.mediaRepository); + } + + get avatarTitle() { + return this.roomName; + } +} + + diff --git a/src/domain/session/toast/ToastCollectionViewModel.ts b/src/domain/session/toast/ToastCollectionViewModel.ts new file mode 100644 index 00000000..f2fe65e3 --- /dev/null +++ b/src/domain/session/toast/ToastCollectionViewModel.ts @@ -0,0 +1,88 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {ObservableArray, ObservableMap} from "../../../observable"; +import {ViewModel, Options as BaseOptions} from "../../ViewModel"; +import type {GroupCall} from "../../../matrix/calls/group/GroupCall"; +import type {Room} from "../../../matrix/room/Room.js"; +import type {CallHandler} from "../../../matrix/calls/CallHandler"; +import type {Session} from "../../../matrix/Session.js"; +import {CallToastNotificationViewModel} from "./CallToastNotificationViewModel"; + +type Options = { + callHandler: CallHandler; + session: Session; +} & BaseOptions; + +export class ToastCollectionViewModel extends ViewModel { + public readonly toastViewModels: ObservableArray = new ObservableArray(); + + constructor(options: Options) { + super(options); + const callsObservableMap = this.getOption("callHandler").calls; + this.track(callsObservableMap.subscribe(this)); + } + + onAdd(_, call) { + if (this.shouldShowNotification(call)) { + const room = this._findRoomForCall(call); + const dismiss = () => { + const idx = this.toastViewModels.array.findIndex(vm => vm.call === call); + if (idx !== -1) { + this.toastViewModels.remove(idx); + } + }; + this.toastViewModels.append( + new CallToastNotificationViewModel(this.childOptions({ call, room, dismiss })) + ); + } + } + + onRemove(_, call) { + const idx = this.toastViewModels.array.findIndex(vm => vm.call === call); + if (idx !== -1) { + this.toastViewModels.remove(idx); + } + } + + onUpdate(_, call) { + const idx = this.toastViewModels.array.findIndex(vm => vm.call === call); + if (idx !== -1) { + // todo: is this correct? + this.toastViewModels.update(idx, this.toastViewModels.at(idx)!); + } + } + + onReset() { + for (let i = 0; i < this.toastViewModels.length; ++i) { + this.toastViewModels.remove(i); + } + } + + private _findRoomForCall(call: GroupCall): Room { + const id = call.roomId; + const rooms = this.getOption("session").rooms; + return rooms.get(id); + } + + private shouldShowNotification(call: GroupCall): boolean { + const currentlyOpenedRoomId = this.navigation.path.get("room")?.value; + if (!call.isLoadedFromStorage && call.roomId !== currentlyOpenedRoomId) { + return true; + } + return false; + } +} diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 6be6b193..e153c761 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -104,7 +104,7 @@ export class CallHandler implements RoomStateHandler { } const event = await txn.roomState.get(callEntry.roomId, EventType.GroupCall, callEntry.callId); if (event) { - const call = new GroupCall(event.event.state_key, false, event.event.content, event.roomId, this.groupCallOptions); + const call = new GroupCall(event.event.state_key, true, false, event.event.content, event.roomId, this.groupCallOptions); this._calls.set(call.id, call); } })); @@ -135,7 +135,7 @@ export class CallHandler implements RoomStateHandler { if (!intent) { intent = CallIntent.Ring; } - const call = new GroupCall(makeId("conf-"), true, { + const call = new GroupCall(makeId("conf-"), false, true, { "m.name": name, "m.intent": intent }, roomId, this.groupCallOptions); @@ -217,7 +217,7 @@ export class CallHandler implements RoomStateHandler { txn.calls.remove(call.intent, roomId, call.id); } } else { - call = new GroupCall(event.state_key, false, event.content, roomId, this.groupCallOptions); + call = new GroupCall(event.state_key, false, false, event.content, roomId, this.groupCallOptions); this._calls.set(call.id, call); txn.calls.add({ intent: call.intent, diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 0b8c7db5..108e1fb9 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -104,6 +104,7 @@ export class GroupCall extends EventEmitter<{change: never}> { constructor( public readonly id: string, + public readonly isLoadedFromStorage: boolean, newCall: boolean, private callContent: Record, public readonly roomId: string, diff --git a/src/platform/web/ui/css/themes/element/theme.css b/src/platform/web/ui/css/themes/element/theme.css index 1c9a233b..3aec2a8a 100644 --- a/src/platform/web/ui/css/themes/element/theme.css +++ b/src/platform/web/ui/css/themes/element/theme.css @@ -1224,3 +1224,103 @@ button.RoomDetailsView_row::after { .JoinRoomView_status .spinner { margin-right: 5px; } + +/* Toast */ +.ToastCollectionView { + display: flex; + position: fixed; + flex-direction: column; + z-index: 1000; + left: 44px; + top: 52px; +} + +.ToastCollectionView ul { + margin: 0; + padding: 0; +} + +.CallToastNotificationView:not(:first-child) { + margin-top: 12px; +} + +.CallToastNotificationView { + display: grid; + grid-template-rows: 40px 1fr 1fr 48px; + row-gap: 4px; + width: 260px; + background-color: var(--background-color-secondary); + border-radius: 8px; + color: var(--text-color); + box-shadow: 2px 2px 10px rgba(0, 0, 0, 0.5); +} + +.CallToastNotificationView__top { + display: grid; + grid-template-columns: auto 176px auto; + align-items: center; + justify-items: center; +} +.CallToastNotificationView__dismiss-btn { + background: center var(--background-color-secondary--darker-5) url("./icons/dismiss.svg?primary=text-color") no-repeat; + border-radius: 100%; + height: 15px; + width: 15px; +} + +.CallToastNotificationView__name { + font-weight: 600; + width: 100%; +} + +.CallToastNotificationView__description { + margin-left: 42px; +} + +.CallToastNotificationView__call-type::before { + content: url("./icons/video-call.svg?primary=light-text-color"); + display: flex; + width: 20px; + height: 20px; + padding-right: 5px; +} + +.CallToastNotificationView__call-type::after { + content: ""; + width: 4px; + height: 4px; + background-color: var(--text-color); + border-radius: 100%; + align-self: center; + margin: 5px; +} + +.CallToastNotificationView__member-count::before { + content: url("./icons/room-members.svg?primary=light-text-color"); + display: flex; + width: 20px; + height: 20px; + padding-right: 5px; +} + +.CallToastNotificationView__member-count, +.CallToastNotificationView__call-type { + display: flex; + align-items: center; +} + +.CallToastNotificationView__info { + display: flex; + margin-left: 42px; +} + +.CallToastNotificationView__action { + display: flex; + justify-content: end; + margin-right: 10px; +} + +.CallToastNotificationView__action .button-action { + width: 100px; + height: 40px; +} diff --git a/src/platform/web/ui/session/SessionView.js b/src/platform/web/ui/session/SessionView.js index 7bcd8c0f..9f84e872 100644 --- a/src/platform/web/ui/session/SessionView.js +++ b/src/platform/web/ui/session/SessionView.js @@ -30,6 +30,7 @@ import {CreateRoomView} from "./CreateRoomView.js"; import {RightPanelView} from "./rightpanel/RightPanelView.js"; import {viewClassForTile} from "./room/common"; import {JoinRoomView} from "./JoinRoomView"; +import {ToastCollectionView} from "./toast/ToastCollectionView"; export class SessionView extends TemplateView { render(t, vm) { @@ -40,6 +41,7 @@ export class SessionView extends TemplateView { "right-shown": vm => !!vm.rightPanelViewModel }, }, [ + t.view(new ToastCollectionView(vm.toastCollectionViewModel)), t.view(new SessionStatusView(vm.sessionStatusViewModel)), t.view(new LeftPanelView(vm.leftPanelViewModel)), t.mapView(vm => vm.activeMiddleViewModel, () => { diff --git a/src/platform/web/ui/session/toast/CallToastNotificationView.ts b/src/platform/web/ui/session/toast/CallToastNotificationView.ts new file mode 100644 index 00000000..50adcc7b --- /dev/null +++ b/src/platform/web/ui/session/toast/CallToastNotificationView.ts @@ -0,0 +1,51 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {AvatarView} from "../../AvatarView.js"; +import {ErrorView} from "../../general/ErrorView"; +import {TemplateView, Builder} from "../../general/TemplateView"; +import type {CallToastNotificationViewModel} from "../../../../../domain/session/toast/CallToastNotificationViewModel"; + +export class CallToastNotificationView extends TemplateView { + render(t: Builder, vm: CallToastNotificationViewModel) { + return t.div({ className: "CallToastNotificationView" }, [ + t.div({ className: "CallToastNotificationView__top" }, [ + t.view(new AvatarView(vm, 24)), + t.span({ className: "CallToastNotificationView__name" }, (vm) => vm.roomName), + t.button({ + className: "button-action CallToastNotificationView__dismiss-btn", + onClick: () => vm.dismiss(), + }), + ]), + t.div({ className: "CallToastNotificationView__description" }, [ + t.span(vm.i18n`Video call started`) + ]), + t.div({ className: "CallToastNotificationView__info" }, [ + t.span({className: "CallToastNotificationView__call-type"}, vm.i18n`Video`), + t.span({className: "CallToastNotificationView__member-count"}, (vm) => vm.memberCount), + ]), + t.div({ className: "CallToastNotificationView__action" }, [ + t.button({ + className: "button-action primary", + onClick: () => vm.join(), + }, vm.i18n`Join`), + ]), + t.if(vm => !!vm.errorViewModel, t => { + return t.div({className: "CallView_error"}, t.view(new ErrorView(vm.errorViewModel!))); + }), + ]); + } +} diff --git a/src/platform/web/ui/session/toast/ToastCollectionView.ts b/src/platform/web/ui/session/toast/ToastCollectionView.ts new file mode 100644 index 00000000..3dc99c77 --- /dev/null +++ b/src/platform/web/ui/session/toast/ToastCollectionView.ts @@ -0,0 +1,33 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {CallToastNotificationView} from "./CallToastNotificationView"; +import {ListView} from "../../general/ListView"; +import {TemplateView, Builder} from "../../general/TemplateView"; +import type {CallToastNotificationViewModel} from "../../../../../domain/session/toast/CallToastNotificationViewModel"; +import type {ToastCollectionViewModel} from "../../../../../domain/session/toast/ToastCollectionViewModel"; + +export class ToastCollectionView extends TemplateView { + render(t: Builder, vm: ToastCollectionViewModel) { + const view = new ListView({ + list: vm.toastViewModels, + parentProvidesUpdates: false, + }, (vm: CallToastNotificationViewModel) => new CallToastNotificationView(vm)); + return t.div({ className: "ToastCollectionView" }, [ + t.view(view), + ]); + } +} From d2b1fc7fef1eace1a14ea43bc4d4b54b804f0d87 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 24 Jan 2023 11:58:49 +0530 Subject: [PATCH 320/435] No need to pass in callHandler separately because session is already passed --- src/domain/session/SessionViewModel.js | 1 - src/domain/session/toast/ToastCollectionViewModel.ts | 7 +++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/domain/session/SessionViewModel.js b/src/domain/session/SessionViewModel.js index 1247f5d7..072e1b74 100644 --- a/src/domain/session/SessionViewModel.js +++ b/src/domain/session/SessionViewModel.js @@ -49,7 +49,6 @@ export class SessionViewModel extends ViewModel { this._createRoomViewModel = null; this._joinRoomViewModel = null; this._toastCollectionViewModel = this.track(new ToastCollectionViewModel(this.childOptions({ - callHandler: this._client.session.callHandler, session: this._client.session, }))); this._setupNavigation(); diff --git a/src/domain/session/toast/ToastCollectionViewModel.ts b/src/domain/session/toast/ToastCollectionViewModel.ts index f2fe65e3..3860830d 100644 --- a/src/domain/session/toast/ToastCollectionViewModel.ts +++ b/src/domain/session/toast/ToastCollectionViewModel.ts @@ -14,16 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {ObservableArray, ObservableMap} from "../../../observable"; +import {ObservableArray} from "../../../observable"; import {ViewModel, Options as BaseOptions} from "../../ViewModel"; import type {GroupCall} from "../../../matrix/calls/group/GroupCall"; import type {Room} from "../../../matrix/room/Room.js"; -import type {CallHandler} from "../../../matrix/calls/CallHandler"; import type {Session} from "../../../matrix/Session.js"; import {CallToastNotificationViewModel} from "./CallToastNotificationViewModel"; type Options = { - callHandler: CallHandler; session: Session; } & BaseOptions; @@ -32,7 +30,8 @@ export class ToastCollectionViewModel extends ViewModel { constructor(options: Options) { super(options); - const callsObservableMap = this.getOption("callHandler").calls; + const session = this.getOption("session"); + const callsObservableMap = session.callHandler.calls; this.track(callsObservableMap.subscribe(this)); } From e3a8c184f6088d57f49bd9584e06df8a06d26eb9 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 24 Jan 2023 12:03:04 +0530 Subject: [PATCH 321/435] Fix formatting --- src/domain/session/toast/BaseToastNotificationViewModel.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/session/toast/BaseToastNotificationViewModel.ts b/src/domain/session/toast/BaseToastNotificationViewModel.ts index 70fefea7..0ace7fdf 100644 --- a/src/domain/session/toast/BaseToastNotificationViewModel.ts +++ b/src/domain/session/toast/BaseToastNotificationViewModel.ts @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { ErrorReportViewModel } from "../../ErrorReportViewModel"; +import {ErrorReportViewModel} from "../../ErrorReportViewModel"; import {Options as BaseOptions} from "../../ViewModel"; import type {Session} from "../../../matrix/Session.js"; From afee565eb7cd116015a96289035780f7693ffbbd Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 24 Jan 2023 12:05:17 +0530 Subject: [PATCH 322/435] Also emit on reset event --- src/domain/session/toast/CallToastNotificationViewModel.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/domain/session/toast/CallToastNotificationViewModel.ts b/src/domain/session/toast/CallToastNotificationViewModel.ts index 0e72c3f9..ebc66167 100644 --- a/src/domain/session/toast/CallToastNotificationViewModel.ts +++ b/src/domain/session/toast/CallToastNotificationViewModel.ts @@ -39,7 +39,9 @@ export class CallToastNotificationViewModel extends BaseToastNotificationViewMod onRemove: (_, __) => { this.emitChange("memberCount"); }, - onReset: () => { /** noop */ }, + onReset: () => { + this.emitChange("memberCount"); + }, }) ); // Dismiss the toast if the room is opened manually From b86fdd476f51ed4fb0372ddd84297a9926949011 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 24 Jan 2023 12:12:35 +0530 Subject: [PATCH 323/435] Add return types --- .../session/toast/BaseToastNotificationViewModel.ts | 2 +- .../session/toast/CallToastNotificationViewModel.ts | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/domain/session/toast/BaseToastNotificationViewModel.ts b/src/domain/session/toast/BaseToastNotificationViewModel.ts index 0ace7fdf..01ec688b 100644 --- a/src/domain/session/toast/BaseToastNotificationViewModel.ts +++ b/src/domain/session/toast/BaseToastNotificationViewModel.ts @@ -28,7 +28,7 @@ export abstract class BaseToastNotificationViewModel { await this.logAndCatch("CallToastNotificationViewModel.join", async (log) => { const stream = await this.platform.mediaDevices.getMediaTracks(false, true); const localMedia = new LocalMedia().withUserMedia(stream); @@ -79,19 +79,19 @@ export class CallToastNotificationViewModel extends BaseToastNotificationViewMod return this.call.members.size; } - get avatarLetter() { + get avatarLetter(): string { return avatarInitials(this.roomName); } - get avatarColorNumber() { + get avatarColorNumber(): number { return getIdentifierColorNumber(this.room.avatarColorId); } - avatarUrl(size: number) { + avatarUrl(size: number): string | undefined { return getAvatarHttpUrl(this.room.avatarUrl, size, this.platform, this.room.mediaRepository); } - get avatarTitle() { + get avatarTitle(): string { return this.roomName; } } From eb7fcc6da2eaabcf216b96d1c4fd59e15ea7f012 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 24 Jan 2023 13:24:15 +0530 Subject: [PATCH 324/435] Add return types --- .../session/toast/ToastCollectionViewModel.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/domain/session/toast/ToastCollectionViewModel.ts b/src/domain/session/toast/ToastCollectionViewModel.ts index 3860830d..30e424ae 100644 --- a/src/domain/session/toast/ToastCollectionViewModel.ts +++ b/src/domain/session/toast/ToastCollectionViewModel.ts @@ -14,12 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. */ +import {CallToastNotificationViewModel} from "./CallToastNotificationViewModel"; import {ObservableArray} from "../../../observable"; import {ViewModel, Options as BaseOptions} from "../../ViewModel"; import type {GroupCall} from "../../../matrix/calls/group/GroupCall"; import type {Room} from "../../../matrix/room/Room.js"; import type {Session} from "../../../matrix/Session.js"; -import {CallToastNotificationViewModel} from "./CallToastNotificationViewModel"; type Options = { session: Session; @@ -35,8 +35,8 @@ export class ToastCollectionViewModel extends ViewModel { this.track(callsObservableMap.subscribe(this)); } - onAdd(_, call) { - if (this.shouldShowNotification(call)) { + onAdd(_, call: GroupCall) { + if (this._shouldShowNotification(call)) { const room = this._findRoomForCall(call); const dismiss = () => { const idx = this.toastViewModels.array.findIndex(vm => vm.call === call); @@ -50,17 +50,16 @@ export class ToastCollectionViewModel extends ViewModel { } } - onRemove(_, call) { + onRemove(_, call: GroupCall) { const idx = this.toastViewModels.array.findIndex(vm => vm.call === call); if (idx !== -1) { this.toastViewModels.remove(idx); } } - onUpdate(_, call) { + onUpdate(_, call: GroupCall) { const idx = this.toastViewModels.array.findIndex(vm => vm.call === call); if (idx !== -1) { - // todo: is this correct? this.toastViewModels.update(idx, this.toastViewModels.at(idx)!); } } @@ -77,7 +76,7 @@ export class ToastCollectionViewModel extends ViewModel { return rooms.get(id); } - private shouldShowNotification(call: GroupCall): boolean { + private _shouldShowNotification(call: GroupCall): boolean { const currentlyOpenedRoomId = this.navigation.path.get("room")?.value; if (!call.isLoadedFromStorage && call.roomId !== currentlyOpenedRoomId) { return true; From 26476324dce53b908d051003a20917dc8f8d5e37 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 24 Jan 2023 18:07:48 +0530 Subject: [PATCH 325/435] lockfile --- yarn.lock | 908 +----------------------------------------------------- 1 file changed, 8 insertions(+), 900 deletions(-) diff --git a/yarn.lock b/yarn.lock index 4b7e83f3..876917a8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -23,43 +23,6 @@ chalk "^2.0.0" js-tokens "^4.0.0" -"@colors/colors@1.5.0": - version "1.5.0" - resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.5.0.tgz#bb504579c1cae923e6576a4f5da43d25f97bdbd9" - integrity sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ== - -"@cypress/request@^2.88.10": - version "2.88.10" - resolved "https://registry.yarnpkg.com/@cypress/request/-/request-2.88.10.tgz#b66d76b07f860d3a4b8d7a0604d020c662752cce" - integrity sha512-Zp7F+R93N0yZyG34GutyTNr+okam7s/Fzc1+i3kcqOP8vk6OuajuE9qZJ6Rs+10/1JFtXFYMdyarnU1rZuJesg== - dependencies: - aws-sign2 "~0.7.0" - aws4 "^1.8.0" - caseless "~0.12.0" - combined-stream "~1.0.6" - extend "~3.0.2" - forever-agent "~0.6.1" - form-data "~2.3.2" - http-signature "~1.3.6" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.19" - performance-now "^2.1.0" - qs "~6.5.2" - safe-buffer "^5.1.2" - tough-cookie "~2.5.0" - tunnel-agent "^0.6.0" - uuid "^8.3.2" - -"@cypress/xvfb@^1.2.4": - version "1.2.4" - resolved "https://registry.yarnpkg.com/@cypress/xvfb/-/xvfb-1.2.4.tgz#2daf42e8275b39f4aa53c14214e557bd14e7748a" - integrity sha512-skbBzPggOVYCbnGgV+0dmBdW/s77ZkAOXIC1knS8NagwDjBrNC1LuXtQJeiN6l+m7lzmHtaoUw/ctJKdqkG57Q== - dependencies: - debug "^3.1.0" - lodash.once "^4.1.1" - "@eslint/eslintrc@^0.4.3": version "0.4.3" resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-0.4.3.tgz#9e42981ef035beb3dd49add17acb96e8ff6f394c" @@ -142,28 +105,6 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-18.7.13.tgz#23e6c5168333480d454243378b69e861ab5c011a" integrity sha512-46yIhxSe5xEaJZXWdIBP7GU4HDTG8/eo0qd9atdiL+lFpA03y8KS+lkTN834TWJj5767GbWv4n/P6efyTFt1Dw== -"@types/node@^14.14.31": - version "14.18.26" - resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.26.tgz#239e19f8b4ea1a9eb710528061c1d733dc561996" - integrity sha512-0b+utRBSYj8L7XAp0d+DX7lI4cSmowNaaTkk6/1SKzbKkG+doLuPusB9EOvzLJ8ahJSk03bTLIL6cWaEd4dBKA== - -"@types/sinonjs__fake-timers@8.1.1": - version "8.1.1" - resolved "https://registry.yarnpkg.com/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-8.1.1.tgz#b49c2c70150141a15e0fa7e79cf1f92a72934ce3" - integrity sha512-0kSuKjAS0TrGLJ0M/+8MaFkGsQhZpB6pxOmvS3K8FYI72K//YmdfoW9X2qPsAKh1mkwxGD5zib9s1FIFed6E8g== - -"@types/sizzle@^2.3.2": - version "2.3.3" - resolved "https://registry.yarnpkg.com/@types/sizzle/-/sizzle-2.3.3.tgz#ff5e2f1902969d305225a047c8a0fd5c915cebef" - integrity sha512-JYM8x9EGF163bEyhdJBpR2QX1R5naCJHC8ucJylJ3w9/CVBaskdQ8WqBf8MmQrd1kRvp/a4TS8HJ+bxzR7ZJYQ== - -"@types/yauzl@^2.9.1": - version "2.10.0" - resolved "https://registry.yarnpkg.com/@types/yauzl/-/yauzl-2.10.0.tgz#b3248295276cf8c6f153ebe6a9aba0c988cb2599" - integrity sha512-Cn6WYCm0tXv8p6k+A8PvbDG763EDpBoTzHdA+Q/MF6H3sapGjCm9NzoaJncJS9tUKSuCoDs9XHxYYsQDgxR6kw== - dependencies: - "@types/node" "*" - "@typescript-eslint/eslint-plugin@^4.29.2": version "4.29.2" resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-4.29.2.tgz#f54dc0a32b8f61c6024ab8755da05363b733838d" @@ -258,14 +199,6 @@ aes-js@^3.1.2: resolved "https://registry.yarnpkg.com/aes-js/-/aes-js-3.1.2.tgz#db9aabde85d5caabbfc0d4f2a4446960f627146a" integrity sha512-e5pEa2kBnBOgR4Y/p20pskXI74UEz7de8ZGVo58asOtvSVG5YAbJeELPZxOmt+Bnz3rX753YKhfIn4X4l1PPRQ== -aggregate-error@^3.0.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/aggregate-error/-/aggregate-error-3.1.0.tgz#92670ff50f5359bdb7a3e0d40d0ec30c5737687a" - integrity sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA== - dependencies: - clean-stack "^2.0.0" - indent-string "^4.0.0" - ajv@^6.10.0, ajv@^6.12.4: version "6.12.6" resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" @@ -296,23 +229,11 @@ ansi-colors@^4.1.1: resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-4.1.1.tgz#cbb9ae256bf750af1eab344f229aa27fe94ba348" integrity sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA== -ansi-escapes@^4.3.0: - version "4.3.2" - resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-4.3.2.tgz#6b2291d1db7d98b6521d5f1efa42d0f3a9feb65e" - integrity sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ== - dependencies: - type-fest "^0.21.3" - ansi-regex@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75" integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg== -ansi-regex@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" - integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== - ansi-styles@^3.2.1: version "3.2.1" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d" @@ -327,11 +248,6 @@ ansi-styles@^4.0.0, ansi-styles@^4.1.0: dependencies: color-convert "^2.0.1" -arch@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/arch/-/arch-2.2.0.tgz#1bc47818f305764f23ab3306b0bfc086c5a29d11" - integrity sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ== - argparse@^1.0.7: version "1.0.10" resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" @@ -344,48 +260,11 @@ array-union@^2.1.0: resolved "https://registry.yarnpkg.com/array-union/-/array-union-2.1.0.tgz#b798420adbeb1de828d84acd8a2e23d3efe85e8d" integrity sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw== -asn1@~0.2.3: - version "0.2.6" - resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.6.tgz#0d3a7bb6e64e02a90c0303b31f292868ea09a08d" - integrity sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ== - dependencies: - safer-buffer "~2.1.0" - -assert-plus@1.0.0, assert-plus@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525" - integrity sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw== - astral-regex@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/astral-regex/-/astral-regex-2.0.0.tgz#483143c567aeed4785759c0865786dc77d7d2e31" integrity sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ== -async@^3.2.0: - version "3.2.4" - resolved "https://registry.yarnpkg.com/async/-/async-3.2.4.tgz#2d22e00f8cddeb5fde5dd33522b56d1cf569a81c" - integrity sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ== - -asynckit@^0.4.0: - version "0.4.0" - resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" - integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q== - -at-least-node@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/at-least-node/-/at-least-node-1.0.0.tgz#602cd4b46e844ad4effc92a8011a3c46e0238dc2" - integrity sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg== - -aws-sign2@~0.7.0: - version "0.7.0" - resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" - integrity sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA== - -aws4@^1.8.0: - version "1.11.0" - resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59" - integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA== - balanced-match@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" @@ -408,28 +287,6 @@ base64-arraybuffer@^0.2.0: resolved "https://registry.yarnpkg.com/base64-arraybuffer/-/base64-arraybuffer-0.2.0.tgz#4b944fac0191aa5907afe2d8c999ccc57ce80f45" integrity sha512-7emyCsu1/xiBXgQZrscw/8KPRT44I4Yq9Pe6EGs3aPRTsWuggML1/1DTuZUuIaJPIm1FTDUVXl4x/yW8s0kQDQ== -base64-js@^1.3.1: - version "1.5.1" - resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" - integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== - -bcrypt-pbkdf@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e" - integrity sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w== - dependencies: - tweetnacl "^0.14.3" - -blob-util@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/blob-util/-/blob-util-2.0.2.tgz#3b4e3c281111bb7f11128518006cdc60b403a1eb" - integrity sha512-T7JQa+zsXXEa6/8ZhHcQEW1UFfVM49Ts65uBkFL6fz2QmrElqmbajIDJvuA0tEhRe5eIjpV9ZF+0RfZR9voJFQ== - -bluebird@^3.7.2: - version "3.7.2" - resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" - integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== - boolbase@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e" @@ -457,34 +314,11 @@ bs58@^4.0.1: dependencies: base-x "^3.0.2" -buffer-crc32@~0.2.3: - version "0.2.13" - resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242" - integrity sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ== - -buffer@^5.6.0: - version "5.7.1" - resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0" - integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ== - dependencies: - base64-js "^1.3.1" - ieee754 "^1.1.13" - -cachedir@^2.3.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/cachedir/-/cachedir-2.3.0.tgz#0c75892a052198f0b21c7c1804d8331edfcae0e8" - integrity sha512-A+Fezp4zxnit6FanDmv9EqXNAi3vt9DWp51/71UEhXukb7QUuvtv9344h91dyAxuTLoSYJFU299qzR3tzwPAhw== - callsites@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73" integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ== -caseless@~0.12.0: - version "0.12.0" - resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" - integrity sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw== - chalk@^2.0.0: version "2.4.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" @@ -502,53 +336,6 @@ chalk@^4.0.0: ansi-styles "^4.1.0" supports-color "^7.1.0" -chalk@^4.1.0: - version "4.1.2" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" - integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== - dependencies: - ansi-styles "^4.1.0" - supports-color "^7.1.0" - -check-more-types@^2.24.0: - version "2.24.0" - resolved "https://registry.yarnpkg.com/check-more-types/-/check-more-types-2.24.0.tgz#1420ffb10fd444dcfc79b43891bbfffd32a84600" - integrity sha512-Pj779qHxV2tuapviy1bSZNEL1maXr13bPYpsvSDB68HlYcYuhlDrmGd63i0JHMCLKzc7rUSNIrpdJlhVlNwrxA== - -ci-info@^3.2.0: - version "3.3.2" - resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-3.3.2.tgz#6d2967ffa407466481c6c90b6e16b3098f080128" - integrity sha512-xmDt/QIAdeZ9+nfdPsaBCpMvHNLFiLdjj59qjqn+6iPe6YmHGQ35sBnQ8uslRBXFmXkiZQOJRjvQeoGppoTjjg== - -clean-stack@^2.0.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-2.2.0.tgz#ee8472dbb129e727b31e8a10a427dee9dfe4008b" - integrity sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A== - -cli-cursor@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-3.1.0.tgz#264305a7ae490d1d03bf0c9ba7c925d1753af307" - integrity sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw== - dependencies: - restore-cursor "^3.1.0" - -cli-table3@~0.6.1: - version "0.6.2" - resolved "https://registry.yarnpkg.com/cli-table3/-/cli-table3-0.6.2.tgz#aaf5df9d8b5bf12634dc8b3040806a0c07120d2a" - integrity sha512-QyavHCaIC80cMivimWu4aWHilIpiDpfm3hGmqAmXVL1UsnbLuBSMd21hTX6VY4ZSDSM73ESLeF8TOYId3rBTbw== - dependencies: - string-width "^4.2.0" - optionalDependencies: - "@colors/colors" "1.5.0" - -cli-truncate@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/cli-truncate/-/cli-truncate-2.1.0.tgz#c39e28bf05edcde5be3b98992a22deed5a2b93c7" - integrity sha512-n8fOixwDD6b/ObinzTrp1ZKFzbgvKZvuz/TvejnLn1aQfC6r52XEx85FmuC+3HI+JM7coBRXUvNqEU2PHVrHpg== - dependencies: - slice-ansi "^3.0.0" - string-width "^4.2.0" - color-convert@^1.9.0: version "1.9.3" resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" @@ -573,28 +360,11 @@ color-name@~1.1.4: resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== -colorette@^2.0.16: - version "2.0.19" - resolved "https://registry.yarnpkg.com/colorette/-/colorette-2.0.19.tgz#cdf044f47ad41a0f4b56b3a0d5b4e6e1a2d5a798" - integrity sha512-3tlv/dIP7FWvj3BsbHrGLJ6l/oKh1O3TcgBqMn+yyCagOxc23fyzDS6HypQbgxWbkpDnf52p1LuR4eWDQ/K9WQ== - colors@^1.3.3: version "1.4.0" resolved "https://registry.yarnpkg.com/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78" integrity sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA== -combined-stream@^1.0.6, combined-stream@~1.0.6: - version "1.0.8" - resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" - integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== - dependencies: - delayed-stream "~1.0.0" - -commander@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/commander/-/commander-5.1.0.tgz#46abbd1652f8e059bddaef99bbdcb2ad9cf179ae" - integrity sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg== - commander@^6.1.0: version "6.2.1" resolved "https://registry.yarnpkg.com/commander/-/commander-6.2.1.tgz#0792eb682dfbc325999bb2b84fddddba110ac73c" @@ -605,11 +375,6 @@ commander@^7.2.0: resolved "https://registry.yarnpkg.com/commander/-/commander-7.2.0.tgz#a36cb57d0b501ce108e4d20559a150a391d97ab7" integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw== -common-tags@^1.8.0: - version "1.8.2" - resolved "https://registry.yarnpkg.com/common-tags/-/common-tags-1.8.2.tgz#94ebb3c076d26032745fd54face7f688ef5ac9c6" - integrity sha512-gk/Z852D2Wtb//0I+kRFNKKE9dIIVirjoqPoA1wJU+XePVXZfGeBpk45+A1rKO4Q43prqWBNY/MiIeRLbPWUaA== - concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" @@ -625,12 +390,7 @@ core-js@^3.6.5: resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.19.3.tgz#6df8142a996337503019ff3235a7022d7cdf4559" integrity sha512-LeLBMgEGSsG7giquSzvgBrTS7V5UL6ks3eQlUSbN8dJStlLFiRzUm5iqsRyzUB8carhfKjkJ2vzKqE6z1Vga9g== -core-util-is@1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" - integrity sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ== - -cross-spawn@^7.0.0, cross-spawn@^7.0.2: +cross-spawn@^7.0.2: version "7.0.3" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== @@ -675,73 +435,6 @@ cuint@^0.2.2: resolved "https://registry.yarnpkg.com/cuint/-/cuint-0.2.2.tgz#408086d409550c2631155619e9fa7bcadc3b991b" integrity sha1-QICG1AlVDCYxFVYZ6fp7ytw7mRs= -cypress@^10.6.0: - version "10.6.0" - resolved "https://registry.yarnpkg.com/cypress/-/cypress-10.6.0.tgz#13f46867febf2c3715874ed5dce9c2e946b175fe" - integrity sha512-6sOpHjostp8gcLO34p6r/Ci342lBs8S5z9/eb3ZCQ22w2cIhMWGUoGKkosabPBfKcvRS9BE4UxybBtlIs8gTQA== - dependencies: - "@cypress/request" "^2.88.10" - "@cypress/xvfb" "^1.2.4" - "@types/node" "^14.14.31" - "@types/sinonjs__fake-timers" "8.1.1" - "@types/sizzle" "^2.3.2" - arch "^2.2.0" - blob-util "^2.0.2" - bluebird "^3.7.2" - buffer "^5.6.0" - cachedir "^2.3.0" - chalk "^4.1.0" - check-more-types "^2.24.0" - cli-cursor "^3.1.0" - cli-table3 "~0.6.1" - commander "^5.1.0" - common-tags "^1.8.0" - dayjs "^1.10.4" - debug "^4.3.2" - enquirer "^2.3.6" - eventemitter2 "^6.4.3" - execa "4.1.0" - executable "^4.1.1" - extract-zip "2.0.1" - figures "^3.2.0" - fs-extra "^9.1.0" - getos "^3.2.1" - is-ci "^3.0.0" - is-installed-globally "~0.4.0" - lazy-ass "^1.6.0" - listr2 "^3.8.3" - lodash "^4.17.21" - log-symbols "^4.0.0" - minimist "^1.2.6" - ospath "^1.2.2" - pretty-bytes "^5.6.0" - proxy-from-env "1.0.0" - request-progress "^3.0.0" - semver "^7.3.2" - supports-color "^8.1.1" - tmp "~0.2.1" - untildify "^4.0.0" - yauzl "^2.10.0" - -dashdash@^1.12.0: - version "1.14.1" - resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" - integrity sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g== - dependencies: - assert-plus "^1.0.0" - -dayjs@^1.10.4: - version "1.11.5" - resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.5.tgz#00e8cc627f231f9499c19b38af49f56dc0ac5e93" - integrity sha512-CAdX5Q3YW3Gclyo5Vpqkgpj8fSdLQcRuzfX6mC6Phy0nfJ0eGYOeS7m4mt2plDWLAtA4TqTakvbboHvUxfe4iA== - -debug@^3.1.0: - version "3.2.7" - resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a" - integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== - dependencies: - ms "^2.1.1" - debug@^4.0.1, debug@^4.1.1: version "4.3.1" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee" @@ -756,13 +449,6 @@ debug@^4.3.1: dependencies: ms "2.1.2" -debug@^4.3.2: - version "4.3.4" - resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" - integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== - dependencies: - ms "2.1.2" - deep-is@^0.1.3: version "0.1.3" resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34" @@ -773,11 +459,6 @@ deep-is@~0.1.3: resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.4.tgz#a6f2dce612fadd2ef1f519b73551f17e85199831" integrity sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ== -delayed-stream@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" - integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== - dir-glob@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f" @@ -839,27 +520,12 @@ domutils@^2.6.0: domelementtype "^2.2.0" domhandler "^4.2.0" -ecc-jsbn@~0.1.1: - version "0.1.2" - resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" - integrity sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw== - dependencies: - jsbn "~0.1.0" - safer-buffer "^2.1.0" - emoji-regex@^8.0.0: version "8.0.0" resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== -end-of-stream@^1.1.0: - version "1.4.4" - resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" - integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q== - dependencies: - once "^1.4.0" - -enquirer@^2.3.5, enquirer@^2.3.6: +enquirer@^2.3.5: version "2.3.6" resolved "https://registry.yarnpkg.com/enquirer/-/enquirer-2.3.6.tgz#2a7fe5dd634a1e4125a975ec994ff5456dc3734d" integrity sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg== @@ -1259,59 +925,11 @@ esutils@^2.0.2: resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64" integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g== -eventemitter2@^6.4.3: - version "6.4.7" - resolved "https://registry.yarnpkg.com/eventemitter2/-/eventemitter2-6.4.7.tgz#a7f6c4d7abf28a14c1ef3442f21cb306a054271d" - integrity sha512-tYUSVOGeQPKt/eC1ABfhHy5Xd96N3oIijJvN3O9+TsC28T5V9yX9oEfEK5faP0EFSNVOG97qtAS68GBrQB2hDg== - -execa@4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/execa/-/execa-4.1.0.tgz#4e5491ad1572f2f17a77d388c6c857135b22847a" - integrity sha512-j5W0//W7f8UxAn8hXVnwG8tLwdiUy4FJLcSupCg6maBYZDpyBvTApK7KyuI4bKj8KOh1r2YH+6ucuYtJv1bTZA== - dependencies: - cross-spawn "^7.0.0" - get-stream "^5.0.0" - human-signals "^1.1.1" - is-stream "^2.0.0" - merge-stream "^2.0.0" - npm-run-path "^4.0.0" - onetime "^5.1.0" - signal-exit "^3.0.2" - strip-final-newline "^2.0.0" - -executable@^4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/executable/-/executable-4.1.1.tgz#41532bff361d3e57af4d763b70582db18f5d133c" - integrity sha512-8iA79xD3uAch729dUG8xaaBBFGaEa0wdD2VkYLFHwlqosEj/jT66AzcreRDSgV7ehnNLBW2WR5jIXwGKjVdTLg== - dependencies: - pify "^2.2.0" - -extend@^3.0.1, extend@~3.0.2: +extend@^3.0.1: version "3.0.2" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== -extract-zip@2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-2.0.1.tgz#663dca56fe46df890d5f131ef4a06d22bb8ba13a" - integrity sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg== - dependencies: - debug "^4.1.1" - get-stream "^5.1.0" - yauzl "^2.10.0" - optionalDependencies: - "@types/yauzl" "^2.9.1" - -extsprintf@1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" - integrity sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g== - -extsprintf@^1.2.0: - version "1.4.1" - resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.1.tgz#8d172c064867f235c0c84a596806d279bf4bcc07" - integrity sha512-Wrk35e8ydCKDj/ArClo1VrPVmN8zph5V4AtHwIuHhvMXsKf73UT3BOD+azBIW+3wOJ4FhEH7zyaJCFvChjYvMA== - fake-indexeddb@^3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/fake-indexeddb/-/fake-indexeddb-3.1.2.tgz#8073a12ed3b254f7afc064f3cc2629f0110a5303" @@ -1353,20 +971,6 @@ fastq@^1.6.0: dependencies: reusify "^1.0.4" -fd-slicer@~1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/fd-slicer/-/fd-slicer-1.1.0.tgz#25c7c89cb1f9077f8891bbe61d8f390eae256f1e" - integrity sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g== - dependencies: - pend "~1.2.0" - -figures@^3.2.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/figures/-/figures-3.2.0.tgz#625c18bd293c604dc4a8ddb2febf0c88341746af" - integrity sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg== - dependencies: - escape-string-regexp "^1.0.5" - file-entry-cache@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-6.0.1.tgz#211b2dd9659cb0394b073e7323ac3c933d522027" @@ -1394,30 +998,6 @@ flatted@^3.1.0: resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.1.1.tgz#c4b489e80096d9df1dfc97c79871aea7c617c469" integrity sha512-zAoAQiudy+r5SvnSw3KJy5os/oRJYHzrzja/tBDqrZtNhUw8bt6y8OBzMWcjWr+8liV8Eb6yOhw8WZ7VFZ5ZzA== -forever-agent@~0.6.1: - version "0.6.1" - resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" - integrity sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw== - -form-data@~2.3.2: - version "2.3.3" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6" - integrity sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ== - dependencies: - asynckit "^0.4.0" - combined-stream "^1.0.6" - mime-types "^2.1.12" - -fs-extra@^9.1.0: - version "9.1.0" - resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-9.1.0.tgz#5954460c764a8da2094ba3554bf839e6b9a7c86d" - integrity sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ== - dependencies: - at-least-node "^1.0.0" - graceful-fs "^4.2.0" - jsonfile "^6.0.1" - universalify "^2.0.0" - fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" @@ -1438,27 +1018,6 @@ functional-red-black-tree@^1.0.1: resolved "https://registry.yarnpkg.com/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz#1b0ab3bd553b2a0d6399d29c0e3ea0b252078327" integrity sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc= -get-stream@^5.0.0, get-stream@^5.1.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-5.2.0.tgz#4966a1795ee5ace65e706c4b7beb71257d6e22d3" - integrity sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA== - dependencies: - pump "^3.0.0" - -getos@^3.2.1: - version "3.2.1" - resolved "https://registry.yarnpkg.com/getos/-/getos-3.2.1.tgz#0134d1f4e00eb46144c5a9c0ac4dc087cbb27dc5" - integrity sha512-U56CfOK17OKgTVqozZjUKNdkfEv6jk5WISBJ8SHoagjE6L69zOwl3Z+O8myjY9MEW3i2HPWQBt/LTbCgcC973Q== - dependencies: - async "^3.2.0" - -getpass@^0.1.1: - version "0.1.7" - resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" - integrity sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng== - dependencies: - assert-plus "^1.0.0" - glob-parent@^5.1.2: version "5.1.2" resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4" @@ -1478,13 +1037,6 @@ glob@^7.1.3: once "^1.3.0" path-is-absolute "^1.0.0" -global-dirs@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/global-dirs/-/global-dirs-3.0.0.tgz#70a76fe84ea315ab37b1f5576cbde7d48ef72686" - integrity sha512-v8ho2DS5RiCjftj1nD9NmnfaOzTdud7RRnVd9kFNOjqZbISlx5DQ+OrTkywgd0dIt7oFCvKetZSHoHcP3sDdiA== - dependencies: - ini "2.0.0" - globals@^13.6.0: version "13.8.0" resolved "https://registry.yarnpkg.com/globals/-/globals-13.8.0.tgz#3e20f504810ce87a8d72e55aecf8435b50f4c1b3" @@ -1511,11 +1063,6 @@ globby@^11.0.3: merge2 "^1.3.0" slash "^3.0.0" -graceful-fs@^4.1.6, graceful-fs@^4.2.0: - version "4.2.10" - resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.10.tgz#147d3a006da4ca3ce14728c7aefc287c367d7a6c" - integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA== - has-flag@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" @@ -1538,25 +1085,6 @@ he@1.2.0: resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f" integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw== -http-signature@~1.3.6: - version "1.3.6" - resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.3.6.tgz#cb6fbfdf86d1c974f343be94e87f7fc128662cf9" - integrity sha512-3adrsD6zqo4GsTqtO7FyrejHNv+NgiIfAfv68+jVlFmSr9OGy7zrxONceFRLKvnnZA5jbxQBX1u9PpB6Wi32Gw== - dependencies: - assert-plus "^1.0.0" - jsprim "^2.0.2" - sshpk "^1.14.1" - -human-signals@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-1.1.1.tgz#c5b1cd14f50aeae09ab6c59fe63ba3395fe4dfa3" - integrity sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw== - -ieee754@^1.1.13: - version "1.2.1" - resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" - integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== - ignore@^4.0.6: version "4.0.6" resolved "https://registry.yarnpkg.com/ignore/-/ignore-4.0.6.tgz#750e3db5862087b4737ebac8207ffd1ef27b25fc" @@ -1589,11 +1117,6 @@ imurmurhash@^0.1.4: resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" integrity sha1-khi5srkoojixPcT7a21XbyMUU+o= -indent-string@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-4.0.0.tgz#624f8f4497d619b2d9768531d58f4122854d7251" - integrity sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg== - inflight@^1.0.4: version "1.0.6" resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" @@ -1607,18 +1130,6 @@ inherits@2: resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== -ini@2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/ini/-/ini-2.0.0.tgz#e5fd556ecdd5726be978fa1001862eacb0a94bc5" - integrity sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA== - -is-ci@^3.0.0: - version "3.0.1" - resolved "https://registry.yarnpkg.com/is-ci/-/is-ci-3.0.1.tgz#db6ecbed1bd659c43dac0f45661e7674103d1867" - integrity sha512-ZYvCgrefwqoQ6yTyYUbQu64HsITZ3NfKX1lzaEYdkTDcfKzzCI/wthRRYKkdjHKFVgNiXKAKm65Zo1pk2as/QQ== - dependencies: - ci-info "^3.2.0" - is-core-module@^2.8.1: version "2.8.1" resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.8.1.tgz#f59fdfca701d5879d0a6b100a40aa1560ce27211" @@ -1643,54 +1154,21 @@ is-glob@^4.0.0, is-glob@^4.0.1: dependencies: is-extglob "^2.1.1" -is-installed-globally@~0.4.0: - version "0.4.0" - resolved "https://registry.yarnpkg.com/is-installed-globally/-/is-installed-globally-0.4.0.tgz#9a0fd407949c30f86eb6959ef1b7994ed0b7b520" - integrity sha512-iwGqO3J21aaSkC7jWnHP/difazwS7SFeIqxv6wEtLU8Y5KlzFTjyqcSIT0d8s4+dDhKytsk9PJZ2BkS5eZwQRQ== - dependencies: - global-dirs "^3.0.0" - is-path-inside "^3.0.2" - is-number@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/is-number/-/is-number-7.0.0.tgz#7535345b896734d5f80c4d06c50955527a14f12b" integrity sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng== -is-path-inside@^3.0.2: - version "3.0.3" - resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.3.tgz#d231362e53a07ff2b0e0ea7fed049161ffd16283" - integrity sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ== - is-plain-obj@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/is-plain-obj/-/is-plain-obj-2.1.0.tgz#45e42e37fccf1f40da8e5f76ee21515840c09287" integrity sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA== -is-stream@^2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.1.tgz#fac1e3d53b97ad5a9d0ae9cef2389f5810a5c077" - integrity sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg== - -is-typedarray@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" - integrity sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA== - -is-unicode-supported@^0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz#3f26c76a809593b52bfa2ecb5710ed2779b522a7" - integrity sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw== - isexe@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" integrity sha1-6PvzdNxVb/iUehDcsFctYz8s+hA= -isstream@~0.1.2: - version "0.1.2" - resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" - integrity sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g== - js-tokens@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" @@ -1704,11 +1182,6 @@ js-yaml@^3.13.1: argparse "^1.0.7" esprima "^4.0.0" -jsbn@~0.1.0: - version "0.1.1" - resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" - integrity sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg== - json-schema-traverse@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" @@ -1719,45 +1192,11 @@ json-schema-traverse@^1.0.0: resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz#ae7bcb3656ab77a73ba5c49bf654f38e6b6860e2" integrity sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug== -json-schema@0.4.0: - version "0.4.0" - resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.4.0.tgz#f7de4cf6efab838ebaeb3236474cbba5a1930ab5" - integrity sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA== - json-stable-stringify-without-jsonify@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651" integrity sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE= -json-stringify-safe@~5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" - integrity sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA== - -jsonfile@^6.0.1: - version "6.1.0" - resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.1.0.tgz#bc55b2634793c679ec6403094eb13698a6ec0aae" - integrity sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ== - dependencies: - universalify "^2.0.0" - optionalDependencies: - graceful-fs "^4.1.6" - -jsprim@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-2.0.2.tgz#77ca23dbcd4135cd364800d22ff82c2185803d4d" - integrity sha512-gqXddjPqQ6G40VdnI6T6yObEC+pDNvyP95wdQhkWkg7crHH3km5qP1FsOXEkzEQwnz6gz5qGTn1c2Y52wP3OyQ== - dependencies: - assert-plus "1.0.0" - extsprintf "1.3.0" - json-schema "0.4.0" - verror "1.10.0" - -lazy-ass@^1.6.0: - version "1.6.0" - resolved "https://registry.yarnpkg.com/lazy-ass/-/lazy-ass-1.6.0.tgz#7999655e8646c17f089fdd187d150d3324d54513" - integrity sha512-cc8oEVoctTvsFZ/Oje/kGnHbpWHYBe8IAJe4C0QNc3t8uM/0Y8+erSz/7Y1ALuXTEZTMvxXwO6YbX1ey3ujiZw== - levn@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/levn/-/levn-0.4.1.tgz#ae4562c007473b932a6200d403268dd2fffc6ade" @@ -1774,20 +1213,6 @@ levn@~0.3.0: prelude-ls "~1.1.2" type-check "~0.3.2" -listr2@^3.8.3: - version "3.14.0" - resolved "https://registry.yarnpkg.com/listr2/-/listr2-3.14.0.tgz#23101cc62e1375fd5836b248276d1d2b51fdbe9e" - integrity sha512-TyWI8G99GX9GjE54cJ+RrNMcIFBfwMPxc3XTFiAYGN4s10hWROGtOg7+O6u6LE3mNkyld7RSLE6nrKBvTfcs3g== - dependencies: - cli-truncate "^2.1.0" - colorette "^2.0.16" - log-update "^4.0.0" - p-map "^4.0.0" - rfdc "^1.3.0" - rxjs "^7.5.1" - through "^2.3.8" - wrap-ansi "^7.0.0" - lodash.clonedeep@^4.5.0: version "4.5.0" resolved "https://registry.yarnpkg.com/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz#e23f3f9c4f8fbdde872529c1071857a086e5ccef" @@ -1798,39 +1223,16 @@ lodash.merge@^4.6.2: resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a" integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== -lodash.once@^4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac" - integrity sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg== - lodash.truncate@^4.4.2: version "4.4.2" resolved "https://registry.yarnpkg.com/lodash.truncate/-/lodash.truncate-4.4.2.tgz#5a350da0b1113b837ecfffd5812cbe58d6eae193" integrity sha1-WjUNoLERO4N+z//VgSy+WNbq4ZM= -lodash@^4.17.21, lodash@^4.7.0: +lodash@^4.7.0: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== -log-symbols@^4.0.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-4.1.0.tgz#3fbdbb95b4683ac9fc785111e792e558d4abd503" - integrity sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg== - dependencies: - chalk "^4.1.0" - is-unicode-supported "^0.1.0" - -log-update@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/log-update/-/log-update-4.0.0.tgz#589ecd352471f2a1c0c570287543a64dfd20e0a1" - integrity sha512-9fkkDevMefjg0mmzWFBW8YkFP91OrizzkW3diF7CpG+S2EYdy4+TVfGwz1zeF8x7hCx1ovSPTOE9Ngib74qqUg== - dependencies: - ansi-escapes "^4.3.0" - cli-cursor "^3.1.0" - slice-ansi "^4.0.0" - wrap-ansi "^6.2.0" - lru-cache@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-6.0.0.tgz#6d6fe6570ebd96aaf90fcad1dafa3b2566db3a94" @@ -1855,11 +1257,6 @@ merge-options@^3.0.4: dependencies: is-plain-obj "^2.1.0" -merge-stream@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/merge-stream/-/merge-stream-2.0.0.tgz#52823629a14dd00c9770fb6ad47dc6310f2c1f60" - integrity sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w== - merge2@^1.3.0: version "1.4.1" resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae" @@ -1873,23 +1270,6 @@ micromatch@^4.0.4: braces "^3.0.1" picomatch "^2.2.3" -mime-db@1.52.0: - version "1.52.0" - resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" - integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== - -mime-types@^2.1.12, mime-types@~2.1.19: - version "2.1.35" - resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" - integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== - dependencies: - mime-db "1.52.0" - -mimic-fn@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" - integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg== - minimatch@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" @@ -1897,21 +1277,11 @@ minimatch@^3.0.4: dependencies: brace-expansion "^1.1.7" -minimist@^1.2.6: - version "1.2.6" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44" - integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q== - ms@2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -ms@^2.1.1: - version "2.1.3" - resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" - integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== - nanoid@^3.3.3: version "3.3.4" resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.4.tgz#730b67e3cd09e2deacf03c027c81c9d9dbc5e8ab" @@ -1930,13 +1300,6 @@ node-html-parser@^4.0.0: css-select "^4.1.3" he "1.2.0" -npm-run-path@^4.0.0: - version "4.0.1" - resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-4.0.1.tgz#b7ecd1e5ed53da8e37a55e1c2269e0b97ed748ea" - integrity sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw== - dependencies: - path-key "^3.0.0" - nth-check@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-2.0.0.tgz#1bb4f6dac70072fc313e8c9cd1417b5074c0a125" @@ -1951,20 +1314,13 @@ off-color@^2.0.0: dependencies: core-js "^3.6.5" -once@^1.3.0, once@^1.3.1, once@^1.4.0: +once@^1.3.0: version "1.4.0" resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" integrity sha1-WDsap3WWHUsROsF9nFC6753Xa9E= dependencies: wrappy "1" -onetime@^5.1.0: - version "5.1.2" - resolved "https://registry.yarnpkg.com/onetime/-/onetime-5.1.2.tgz#d0e96ebb56b07476df1dd9c4806e5237985ca45e" - integrity sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg== - dependencies: - mimic-fn "^2.1.0" - optionator@^0.8.1: version "0.8.3" resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.8.3.tgz#84fa1d036fe9d3c7e21d99884b601167ec8fb495" @@ -1989,18 +1345,6 @@ optionator@^0.9.1: type-check "^0.4.0" word-wrap "^1.2.3" -ospath@^1.2.2: - version "1.2.2" - resolved "https://registry.yarnpkg.com/ospath/-/ospath-1.2.2.tgz#1276639774a3f8ef2572f7fe4280e0ea4550c07b" - integrity sha512-o6E5qJV5zkAbIDNhGSIlyOhScKXgQrSRMilfph0clDfM0nEnBOlKlH4sWDmG95BW/CvwNz0vmm7dJVtU2KlMiA== - -p-map@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/p-map/-/p-map-4.0.0.tgz#bb2f95a5eda2ec168ec9274e06a747c3e2904d2b" - integrity sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ== - dependencies: - aggregate-error "^3.0.0" - parent-module@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/parent-module/-/parent-module-1.0.1.tgz#691d2709e78c79fae3a156622452d00762caaaa2" @@ -2013,7 +1357,7 @@ path-is-absolute@^1.0.0: resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" integrity sha1-F0uSaHNVNP+8es5r9TpanhtcX18= -path-key@^3.0.0, path-key@^3.1.0: +path-key@^3.1.0: version "3.1.1" resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375" integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q== @@ -2028,16 +1372,6 @@ path-type@^4.0.0: resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b" integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw== -pend@~1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50" - integrity sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg== - -performance-now@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" - integrity sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow== - picocolors@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" @@ -2048,11 +1382,6 @@ picomatch@^2.2.3: resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.0.tgz#f1f061de8f6a4bf022892e2d128234fb98302972" integrity sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw== -pify@^2.2.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c" - integrity sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog== - playwright-core@1.27.1: version "1.27.1" resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.27.1.tgz#840ef662e55a3ed759d8b5d3d00a5f885a7184f4" @@ -2096,44 +1425,16 @@ prelude-ls@~1.1.2: resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54" integrity sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ= -pretty-bytes@^5.6.0: - version "5.6.0" - resolved "https://registry.yarnpkg.com/pretty-bytes/-/pretty-bytes-5.6.0.tgz#356256f643804773c82f64723fe78c92c62beaeb" - integrity sha512-FFw039TmrBqFK8ma/7OL3sDz/VytdtJr044/QUJtH0wK9lb9jLq9tJyIxUwtQJHwar2BqtiA4iCWSwo9JLkzFg== - progress@^2.0.0: version "2.0.3" resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== -proxy-from-env@1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.0.0.tgz#33c50398f70ea7eb96d21f7b817630a55791c7ee" - integrity sha512-F2JHgJQ1iqwnHDcQjVBsq3n/uoaFL+iPW/eAeL7kVxy/2RrWaN4WroKjjvbsoRtv0ftelNyC01bjRhn/bhcf4A== - -psl@^1.1.28: - version "1.9.0" - resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" - integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag== - -pump@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64" - integrity sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww== - dependencies: - end-of-stream "^1.1.0" - once "^1.3.1" - punycode@^2.1.0, punycode@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== -qs@~6.5.2: - version "6.5.3" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.3.tgz#3aeeffc91967ef6e35c0e488ef46fb296ab76aad" - integrity sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA== - queue-microtask@^1.2.2: version "1.2.3" resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243" @@ -2159,13 +1460,6 @@ regexpp@^3.1.0: resolved "https://registry.yarnpkg.com/regexpp/-/regexpp-3.1.0.tgz#206d0ad0a5648cffbdb8ae46438f3dc51c9f78e2" integrity sha512-ZOIzd8yVsQQA7j8GCSlPGXwg5PfmA1mrq0JP4nGhh54LaKN3xdai/vHUDu74pKwV8OxseMS65u2NImosQcSD0Q== -request-progress@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/request-progress/-/request-progress-3.0.0.tgz#4ca754081c7fec63f505e4faa825aa06cd669dbe" - integrity sha512-MnWzEHHaxHO2iWiQuHrUPBi/1WeBf5PkxQqNyNvLl9VAYSdXkP8tQ3pBSeCPD+yw0v0Aq1zosWLz0BdeXpWwZg== - dependencies: - throttleit "^1.0.0" - require-from-string@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/require-from-string/-/require-from-string-2.0.2.tgz#89a7fdd938261267318eafe14f9c32e598c36909" @@ -2185,25 +1479,12 @@ resolve@^1.22.0: path-parse "^1.0.7" supports-preserve-symlinks-flag "^1.0.0" -restore-cursor@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/restore-cursor/-/restore-cursor-3.1.0.tgz#39f67c54b3a7a58cea5236d95cf0034239631f7e" - integrity sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA== - dependencies: - onetime "^5.1.0" - signal-exit "^3.0.2" - reusify@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76" integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw== -rfdc@^1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/rfdc/-/rfdc-1.3.0.tgz#d0b7c441ab2720d05dc4cf26e01c89631d9da08b" - integrity sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA== - -rimraf@^3.0.0, rimraf@^3.0.2: +rimraf@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA== @@ -2224,23 +1505,11 @@ run-parallel@^1.1.9: dependencies: queue-microtask "^1.2.2" -rxjs@^7.5.1: - version "7.5.6" - resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-7.5.6.tgz#0446577557862afd6903517ce7cae79ecb9662bc" - integrity sha512-dnyv2/YsXhnm461G+R/Pe5bWP41Nm6LBXEYWI6eiFP4fiwx6WRI/CD0zbdVAudd9xwLEF2IDcKXLHit0FYjUzw== - dependencies: - tslib "^2.1.0" - -safe-buffer@^5.0.1, safe-buffer@^5.1.2: +safe-buffer@^5.0.1: version "5.2.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== -safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: - version "2.1.2" - resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" - integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== - semver@^7.2.1, semver@^7.3.5: version "7.3.5" resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.5.tgz#0b621c879348d8998e4b0e4be94b3f12e6018ef7" @@ -2248,13 +1517,6 @@ semver@^7.2.1, semver@^7.3.5: dependencies: lru-cache "^6.0.0" -semver@^7.3.2: - version "7.3.7" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.7.tgz#12c5b649afdbf9049707796e22a4028814ce523f" - integrity sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g== - dependencies: - lru-cache "^6.0.0" - setimmediate@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/setimmediate/-/setimmediate-1.0.5.tgz#290cbb232e306942d7d7ea9b83732ab7856f8285" @@ -2272,25 +1534,11 @@ shebang-regex@^3.0.0: resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172" integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== -signal-exit@^3.0.2: - version "3.0.7" - resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9" - integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ== - slash@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634" integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q== -slice-ansi@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-3.0.0.tgz#31ddc10930a1b7e0b67b08c96c2f49b77a789787" - integrity sha512-pSyv7bSTC7ig9Dcgbw9AuRNUb5k5V6oDudjZoMBSr13qpLBG7tB+zgCkARjq7xIUgdz5P1Qe8u+rSGdouOOIyQ== - dependencies: - ansi-styles "^4.0.0" - astral-regex "^2.0.0" - is-fullwidth-code-point "^3.0.0" - slice-ansi@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-4.0.0.tgz#500e8dd0fd55b05815086255b3195adf2a45fe6b" @@ -2315,35 +1563,11 @@ sprintf-js@~1.0.2: resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw= -sshpk@^1.14.1: - version "1.17.0" - resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.17.0.tgz#578082d92d4fe612b13007496e543fa0fbcbe4c5" - integrity sha512-/9HIEs1ZXGhSPE8X6Ccm7Nam1z8KcoCqPdI7ecm1N33EzAetWahvQWVqLZtaZQ+IDKX4IyA2o0gBzqIMkAagHQ== - dependencies: - asn1 "~0.2.3" - assert-plus "^1.0.0" - bcrypt-pbkdf "^1.0.0" - dashdash "^1.12.0" - ecc-jsbn "~0.1.1" - getpass "^0.1.1" - jsbn "~0.1.0" - safer-buffer "^2.0.2" - tweetnacl "~0.14.0" - stable@^0.1.8: version "0.1.8" resolved "https://registry.yarnpkg.com/stable/-/stable-0.1.8.tgz#836eb3c8382fe2936feaf544631017ce7d47a3cf" integrity sha512-ji9qxRnOVfcuLDySj9qzhGSEFVobyt1kIOSkj1qZzYLzq7Tos/oUUWvotUPQLlrsidqsK6tBH89Bc9kL5zHA6w== -string-width@^4.1.0: - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - string-width@^4.2.0: version "4.2.2" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.2.tgz#dafd4f9559a7585cfba529c6a0a4f73488ebd4c5" @@ -2360,18 +1584,6 @@ strip-ansi@^6.0.0: dependencies: ansi-regex "^5.0.0" -strip-ansi@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - -strip-final-newline@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/strip-final-newline/-/strip-final-newline-2.0.0.tgz#89b852fb2fcbe936f6f4b3187afb0a12c1ab58ad" - integrity sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA== - strip-json-comments@^3.1.0, strip-json-comments@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006" @@ -2391,13 +1603,6 @@ supports-color@^7.1.0: dependencies: has-flag "^4.0.0" -supports-color@^8.1.1: - version "8.1.1" - resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-8.1.1.tgz#cd6fc17e28500cff56c1b86c0a7fd4a54a73005c" - integrity sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q== - dependencies: - has-flag "^4.0.0" - supports-preserve-symlinks-flag@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09" @@ -2438,23 +1643,6 @@ text-table@^0.2.0: resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" integrity sha1-f17oI66AUgfACvLfSoTsP8+lcLQ= -throttleit@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/throttleit/-/throttleit-1.0.0.tgz#9e785836daf46743145a5984b6268d828528ac6c" - integrity sha512-rkTVqu6IjfQ/6+uNuuc3sZek4CEYxTJom3IktzgdSxcZqdARuebbA/f4QmAxMQIxqq9ZLEUkSYqvuk1I6VKq4g== - -through@^2.3.8: - version "2.3.8" - resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" - integrity sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg== - -tmp@~0.2.1: - version "0.2.1" - resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.2.1.tgz#8457fc3037dcf4719c251367a1af6500ee1ccf14" - integrity sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ== - dependencies: - rimraf "^3.0.0" - to-regex-range@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-5.0.1.tgz#1648c44aae7c8d988a326018ed72f5b4dd0392e4" @@ -2462,14 +1650,6 @@ to-regex-range@^5.0.1: dependencies: is-number "^7.0.0" -tough-cookie@~2.5.0: - version "2.5.0" - resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" - integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== - dependencies: - psl "^1.1.28" - punycode "^2.1.1" - tr46@^2.0.2: version "2.1.0" resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240" @@ -2482,11 +1662,6 @@ tslib@^1.8.1: resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== -tslib@^2.1.0: - version "2.4.0" - resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.4.0.tgz#7cecaa7f073ce680a05847aa77be941098f36dc3" - integrity sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ== - tsutils@^3.21.0: version "3.21.0" resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.21.0.tgz#b48717d394cea6c1e096983eed58e9d61715b623" @@ -2494,18 +1669,6 @@ tsutils@^3.21.0: dependencies: tslib "^1.8.1" -tunnel-agent@^0.6.0: - version "0.6.0" - resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" - integrity sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w== - dependencies: - safe-buffer "^5.0.1" - -tweetnacl@^0.14.3, tweetnacl@~0.14.0: - version "0.14.5" - resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" - integrity sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA== - type-check@^0.4.0, type-check@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1" @@ -2525,11 +1688,6 @@ type-fest@^0.20.2: resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.20.2.tgz#1bf207f4b28f91583666cb5fbd327887301cd5f4" integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ== -type-fest@^0.21.3: - version "0.21.3" - resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.21.3.tgz#d260a24b0198436e133fa26a524a6d65fa3b2e37" - integrity sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w== - typescript@^4.7.0: version "4.7.4" resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.7.4.tgz#1a88596d1cf47d59507a1bcdfb5b9dfe4d488235" @@ -2554,16 +1712,6 @@ typeson@^6.0.0: resolved "https://registry.yarnpkg.com/typeson/-/typeson-6.1.0.tgz#5b2a53705a5f58ff4d6f82f965917cabd0d7448b" integrity sha512-6FTtyGr8ldU0pfbvW/eOZrEtEkczHRUtduBnA90Jh9kMPCiFNnXIon3vF41N0S4tV1HHQt4Hk1j4srpESziCaA== -universalify@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.0.tgz#75a4984efedc4b08975c5aeb73f530d02df25717" - integrity sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ== - -untildify@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" - integrity sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw== - uri-js@^4.2.2: version "4.4.1" resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.4.1.tgz#9b1a52595225859e55f669d928f88c6c57f2a77e" @@ -2571,25 +1719,11 @@ uri-js@^4.2.2: dependencies: punycode "^2.1.0" -uuid@^8.3.2: - version "8.3.2" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" - integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== - v8-compile-cache@^2.0.3: version "2.3.0" resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz#2de19618c66dc247dcfb6f99338035d8245a2cee" integrity sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA== -verror@1.10.0: - version "1.10.0" - resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" - integrity sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw== - dependencies: - assert-plus "^1.0.0" - core-util-is "1.0.2" - extsprintf "^1.2.0" - vite@^2.9.8: version "2.9.8" resolved "https://registry.yarnpkg.com/vite/-/vite-2.9.8.tgz#2c2cb0790beb0fbe4b8c0995b80fe691a91c2545" @@ -2633,24 +1767,6 @@ word-wrap@^1.2.3, word-wrap@~1.2.3: resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ== -wrap-ansi@^6.2.0: - version "6.2.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-6.2.0.tgz#e9393ba07102e6c91a3b221478f0257cd2856e53" - integrity sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrappy@1: version "1.0.2" resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" @@ -2667,11 +1783,3 @@ yallist@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== - -yauzl@^2.10.0: - version "2.10.0" - resolved "https://registry.yarnpkg.com/yauzl/-/yauzl-2.10.0.tgz#c7eb17c93e112cb1086fa6d8e51fb0667b79a5f9" - integrity sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g== - dependencies: - buffer-crc32 "~0.2.3" - fd-slicer "~1.1.0" From 374fb08c98b702908a824ee89962f6158c6f17e9 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 24 Jan 2023 18:08:29 +0530 Subject: [PATCH 326/435] Fix typescript errors --- .../toast/BaseToastNotificationViewModel.ts | 9 ++++---- .../toast/CallToastNotificationViewModel.ts | 22 ++++++++++++------- .../session/toast/ToastCollectionViewModel.ts | 3 ++- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/domain/session/toast/BaseToastNotificationViewModel.ts b/src/domain/session/toast/BaseToastNotificationViewModel.ts index 01ec688b..41e20e42 100644 --- a/src/domain/session/toast/BaseToastNotificationViewModel.ts +++ b/src/domain/session/toast/BaseToastNotificationViewModel.ts @@ -17,14 +17,15 @@ limitations under the License. import {ErrorReportViewModel} from "../../ErrorReportViewModel"; import {Options as BaseOptions} from "../../ViewModel"; import type {Session} from "../../../matrix/Session.js"; +import {SegmentType} from "../../navigation"; -export type BaseClassOptions = { +export type BaseClassOptions = { dismiss: () => void; session: Session; -} & BaseOptions; +} & BaseOptions; -export abstract class BaseToastNotificationViewModel extends ErrorReportViewModel { - constructor(options: T) { +export abstract class BaseToastNotificationViewModel = BaseClassOptions> extends ErrorReportViewModel { + constructor(options: O) { super(options); } diff --git a/src/domain/session/toast/CallToastNotificationViewModel.ts b/src/domain/session/toast/CallToastNotificationViewModel.ts index 4c005cc5..3281b86a 100644 --- a/src/domain/session/toast/CallToastNotificationViewModel.ts +++ b/src/domain/session/toast/CallToastNotificationViewModel.ts @@ -18,15 +18,21 @@ import type {Room} from "../../../matrix/room/Room.js"; import {IAvatarContract, avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../avatar"; import {LocalMedia} from "../../../matrix/calls/LocalMedia"; import {BaseClassOptions, BaseToastNotificationViewModel} from "./BaseToastNotificationViewModel"; +import {SegmentType} from "../../navigation"; -type Options = { +type Options = { call: GroupCall; room: Room; -} & BaseClassOptions; +} & BaseClassOptions; +// Since we access the room segment below, the segment type +// needs to at least contain the room segment! +type MinimumNeededSegmentType = { + "room": string; +}; -export class CallToastNotificationViewModel extends BaseToastNotificationViewModel implements IAvatarContract { - constructor(options: Options) { +export class CallToastNotificationViewModel = Options> extends BaseToastNotificationViewModel implements IAvatarContract { + constructor(options: O) { super(options); this.track( this.call.members.subscribe({ @@ -46,8 +52,8 @@ export class CallToastNotificationViewModel extends BaseToastNotificationViewMod ); // Dismiss the toast if the room is opened manually this.track( - this.navigation.observe("room").subscribe(roomId => { - if (roomId === this.call.roomId) { + this.navigation.observe("room").subscribe((roomId) => { + if ((roomId as unknown as string) === this.call.roomId) { this.dismiss(); } })); @@ -58,8 +64,8 @@ export class CallToastNotificationViewModel extends BaseToastNotificationViewMod const stream = await this.platform.mediaDevices.getMediaTracks(false, true); const localMedia = new LocalMedia().withUserMedia(stream); await this.call.join(localMedia, log); - const url = this.urlCreator.openRoomActionUrl(this.call.roomId); - this.urlCreator.pushUrl(url); + const url = this.urlRouter.openRoomActionUrl(this.call.roomId); + this.urlRouter.pushUrl(url); }); } diff --git a/src/domain/session/toast/ToastCollectionViewModel.ts b/src/domain/session/toast/ToastCollectionViewModel.ts index 30e424ae..18b58b76 100644 --- a/src/domain/session/toast/ToastCollectionViewModel.ts +++ b/src/domain/session/toast/ToastCollectionViewModel.ts @@ -20,12 +20,13 @@ import {ViewModel, Options as BaseOptions} from "../../ViewModel"; import type {GroupCall} from "../../../matrix/calls/group/GroupCall"; import type {Room} from "../../../matrix/room/Room.js"; import type {Session} from "../../../matrix/Session.js"; +import type {SegmentType} from "../../navigation"; type Options = { session: Session; } & BaseOptions; -export class ToastCollectionViewModel extends ViewModel { +export class ToastCollectionViewModel extends ViewModel { public readonly toastViewModels: ObservableArray = new ObservableArray(); constructor(options: Options) { From c2fab59f582943e59f89bce7c8c1afce580c1375 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 24 Jan 2023 23:15:32 +0100 Subject: [PATCH 327/435] ensure call isn't cleared by onCallUpdate when setting up member list --- src/domain/session/room/timeline/tiles/CallTile.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index 8887b834..4583de49 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -29,14 +29,15 @@ export class CallTile extends SimpleTile { constructor(entry, options) { super(entry, options); const calls = this.getOption("session").callHandler.calls; - this._call = calls.get(this._entry.stateKey); this._callSubscription = undefined; - if (this._call) { + const call = calls.get(this._entry.stateKey); + if (call && !call.isTerminated) { + this._call = call; + this.memberViewModels = this._setupMembersList(this._call); this._callSubscription = this.track(this._call.disposableOn("change", () => { this._onCallUpdate(); })); this._onCallUpdate(); - this.memberViewModels = this._setupMembersList(this._call); } } From 98416f8c35bf3e8da6b25628bd06c334de92d02f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 24 Jan 2023 23:17:04 +0100 Subject: [PATCH 328/435] also calculate days in formatDuration --- src/platform/web/dom/TimeFormatter.ts | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/platform/web/dom/TimeFormatter.ts b/src/platform/web/dom/TimeFormatter.ts index cebc6ec4..916b18bb 100644 --- a/src/platform/web/dom/TimeFormatter.ts +++ b/src/platform/web/dom/TimeFormatter.ts @@ -24,6 +24,7 @@ enum TimeScope { const MINUTES_IN_MS = 60 * 1000; const HOURS_IN_MS = MINUTES_IN_MS * 60; +const DAYS_IN_MS = HOURS_IN_MS * 24; export class TimeFormatter implements ITimeFormatter { @@ -80,8 +81,13 @@ export class TimeFormatter implements ITimeFormatter { } formatDuration(milliseconds: number): string { + let days = 0; let hours = 0; let minutes = 0; + if (milliseconds > DAYS_IN_MS) { + days = Math.floor(milliseconds / DAYS_IN_MS); + milliseconds -= days * DAYS_IN_MS; + } if (milliseconds > HOURS_IN_MS) { hours = Math.floor(milliseconds / HOURS_IN_MS); milliseconds -= hours * HOURS_IN_MS; @@ -91,13 +97,18 @@ export class TimeFormatter implements ITimeFormatter { milliseconds -= minutes * MINUTES_IN_MS; } const seconds = Math.floor(milliseconds / 1000); - if (hours) { - return `${hours}h ${minutes}m ${seconds}s`; + let result = ""; + if (days) { + result = `${days}d `; } - if (minutes) { - return `${minutes}m ${seconds}s`; + if (hours || days) { + result += `${hours}h `; } - return `${seconds}s`; + if (minutes || hours || days) { + result += `${minutes}m `; + } + result += `${seconds}s`; + return result; } } From 47d9773fc89c0edb948c371f76df4435ae895334 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 24 Jan 2023 23:17:20 +0100 Subject: [PATCH 329/435] more style changes --- .../web/ui/css/themes/element/timeline.css | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/platform/web/ui/css/themes/element/timeline.css b/src/platform/web/ui/css/themes/element/timeline.css index 5dc86276..16c758a3 100644 --- a/src/platform/web/ui/css/themes/element/timeline.css +++ b/src/platform/web/ui/css/themes/element/timeline.css @@ -449,16 +449,26 @@ only loads when the top comes into view*/ } .CallTileView_members > * { - margin-left: -16px; + margin-right: -16px; } + .CallTileView_members { display: flex; } +.CallTileView_title { + font-weight: bold; +} + +.CallTileView_subtitle { + font-size: 12px; +} + .CallTileView_memberCount::before { content: url('./icons/room-members.svg?primary=text-color'); - background-repeat: no-repeat; - background-size: 24px; - width: 24px; - height: 24px; + width: 16px; + height: 16px; + display: inline-flex; + vertical-align: bottom; + margin-right: 4px; } From e140a4ba64f54a4f897aa8ff8367c9047bffb091 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 23 Jan 2023 15:06:06 +0100 Subject: [PATCH 330/435] element call puts string in terminated, not a boolean --- src/matrix/calls/group/GroupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 0b8c7db5..d323f6b2 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -140,7 +140,7 @@ export class GroupCall extends EventEmitter<{change: never}> { get members(): BaseObservableMap { return this._members; } get isTerminated(): boolean { - return this.callContent?.["m.terminated"] === true; + return !!this.callContent?.["m.terminated"]; } get isRinging(): boolean { From 043ad988669842d8ea71d822714af0263fcb9a62 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 23 Jan 2023 15:06:24 +0100 Subject: [PATCH 331/435] element call also terminates prompt calls, so do so too --- src/matrix/calls/group/GroupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index d323f6b2..6407584a 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -286,7 +286,7 @@ export class GroupCall extends EventEmitter<{change: never}> { const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCallMember, this.options.ownUserId, memberContent, {log}); await request.response(); // our own user isn't included in members, so not in the count - if (this.intent === CallIntent.Ring && this._members.size === 0) { + if ((this.intent === CallIntent.Ring || this.intent === CallIntent.Prompt) && this._members.size === 0) { await this.terminate(log); } } else { From 7f422882dd0ed52e3df418318ed42f999fe252dc Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 23 Jan 2023 15:08:38 +0100 Subject: [PATCH 332/435] make call error view clickable above video elements --- src/platform/web/ui/css/themes/element/call.css | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/platform/web/ui/css/themes/element/call.css b/src/platform/web/ui/css/themes/element/call.css index c0f1550e..3efe3c56 100644 --- a/src/platform/web/ui/css/themes/element/call.css +++ b/src/platform/web/ui/css/themes/element/call.css @@ -25,11 +25,13 @@ limitations under the License. } .CallView_error { - color: red; - font-weight: bold; align-self: start; justify-self: center; margin: 16px; + /** Chrome (v100) requires this to make the buttons clickable + * where they overlap with the video element, even though + * the buttons come later in the DOM. */ + z-index: 1; } .CallView_members { From a278086c3787366cf8738cc015c03629d08d93a4 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 26 Jan 2023 09:58:24 +0100 Subject: [PATCH 333/435] fix error at time unit boundary --- src/platform/web/dom/TimeFormatter.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/platform/web/dom/TimeFormatter.ts b/src/platform/web/dom/TimeFormatter.ts index 916b18bb..090f8076 100644 --- a/src/platform/web/dom/TimeFormatter.ts +++ b/src/platform/web/dom/TimeFormatter.ts @@ -84,15 +84,15 @@ export class TimeFormatter implements ITimeFormatter { let days = 0; let hours = 0; let minutes = 0; - if (milliseconds > DAYS_IN_MS) { + if (milliseconds >= DAYS_IN_MS) { days = Math.floor(milliseconds / DAYS_IN_MS); milliseconds -= days * DAYS_IN_MS; } - if (milliseconds > HOURS_IN_MS) { + if (milliseconds >= HOURS_IN_MS) { hours = Math.floor(milliseconds / HOURS_IN_MS); milliseconds -= hours * HOURS_IN_MS; } - if (milliseconds > MINUTES_IN_MS) { + if (milliseconds >= MINUTES_IN_MS) { minutes = Math.floor(milliseconds / MINUTES_IN_MS); milliseconds -= minutes * MINUTES_IN_MS; } From 0f91f2065cb069dffd6e985dd53adbc88923ef15 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 26 Jan 2023 10:25:22 +0100 Subject: [PATCH 334/435] extract to src/utils as it doesn't assume the DOM --- src/platform/web/dom/TimeFormatter.ts | 39 ++------------------------- src/utils/timeFormatting.ts | 36 +++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 37 deletions(-) create mode 100644 src/utils/timeFormatting.ts diff --git a/src/platform/web/dom/TimeFormatter.ts b/src/platform/web/dom/TimeFormatter.ts index 090f8076..c25e902b 100644 --- a/src/platform/web/dom/TimeFormatter.ts +++ b/src/platform/web/dom/TimeFormatter.ts @@ -16,15 +16,7 @@ limitations under the License. import type { ITimeFormatter } from "../../types/types"; import {Clock} from "./Clock"; - -enum TimeScope { - Minute = 60 * 1000, - Day = 24 * 60 * 60 * 1000, -} - -const MINUTES_IN_MS = 60 * 1000; -const HOURS_IN_MS = MINUTES_IN_MS * 60; -const DAYS_IN_MS = HOURS_IN_MS * 24; +import {formatDuration, TimeScope} from "../../../utils/timeFormatting"; export class TimeFormatter implements ITimeFormatter { @@ -81,34 +73,7 @@ export class TimeFormatter implements ITimeFormatter { } formatDuration(milliseconds: number): string { - let days = 0; - let hours = 0; - let minutes = 0; - if (milliseconds >= DAYS_IN_MS) { - days = Math.floor(milliseconds / DAYS_IN_MS); - milliseconds -= days * DAYS_IN_MS; - } - if (milliseconds >= HOURS_IN_MS) { - hours = Math.floor(milliseconds / HOURS_IN_MS); - milliseconds -= hours * HOURS_IN_MS; - } - if (milliseconds >= MINUTES_IN_MS) { - minutes = Math.floor(milliseconds / MINUTES_IN_MS); - milliseconds -= minutes * MINUTES_IN_MS; - } - const seconds = Math.floor(milliseconds / 1000); - let result = ""; - if (days) { - result = `${days}d `; - } - if (hours || days) { - result += `${hours}h `; - } - if (minutes || hours || days) { - result += `${minutes}m `; - } - result += `${seconds}s`; - return result; + return formatDuration(milliseconds); } } diff --git a/src/utils/timeFormatting.ts b/src/utils/timeFormatting.ts new file mode 100644 index 00000000..8d052e95 --- /dev/null +++ b/src/utils/timeFormatting.ts @@ -0,0 +1,36 @@ +export enum TimeScope { + Minute = 60 * 1000, + Hours = 60 * TimeScope.Minute, + Day = 24 * TimeScope.Hours, +} + +export function formatDuration(milliseconds: number): string { + let days = 0; + let hours = 0; + let minutes = 0; + if (milliseconds >= TimeScope.Day) { + days = Math.floor(milliseconds / TimeScope.Day); + milliseconds -= days * TimeScope.Day; + } + if (milliseconds >= TimeScope.Hours) { + hours = Math.floor(milliseconds / TimeScope.Hours); + milliseconds -= hours * TimeScope.Hours; + } + if (milliseconds >= TimeScope.Minute) { + minutes = Math.floor(milliseconds / TimeScope.Minute); + milliseconds -= minutes * TimeScope.Minute; + } + const seconds = Math.floor(milliseconds / 1000); + let result = ""; + if (days) { + result = `${days}d `; + } + if (hours || days) { + result += `${hours}h `; + } + if (minutes || hours || days) { + result += `${minutes}m `; + } + result += `${seconds}s`; + return result; +} From cd76619953fde153c257b0b6486129eeeafeb859 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 26 Jan 2023 10:30:29 +0100 Subject: [PATCH 335/435] add header --- src/utils/timeFormatting.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/utils/timeFormatting.ts b/src/utils/timeFormatting.ts index 8d052e95..2f2ae8c0 100644 --- a/src/utils/timeFormatting.ts +++ b/src/utils/timeFormatting.ts @@ -1,3 +1,19 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + export enum TimeScope { Minute = 60 * 1000, Hours = 60 * TimeScope.Minute, From e1fc2b46c491cd963de419a15041d2a29fb840e0 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 26 Jan 2023 11:40:40 +0100 Subject: [PATCH 336/435] add observeSize operator on ObservableMap --- src/observable/map/BaseObservableMap.ts | 27 ++++--- .../value/MapSizeObservableValue.ts | 71 +++++++++++++++++++ src/observable/value/index.ts | 1 + 3 files changed, 88 insertions(+), 11 deletions(-) create mode 100644 src/observable/value/MapSizeObservableValue.ts diff --git a/src/observable/map/BaseObservableMap.ts b/src/observable/map/BaseObservableMap.ts index f81a94f8..c82e531c 100644 --- a/src/observable/map/BaseObservableMap.ts +++ b/src/observable/map/BaseObservableMap.ts @@ -18,6 +18,7 @@ import {BaseObservable} from "../BaseObservable"; import {JoinedMap} from "./index"; import {MappedMap} from "./index"; import {FilteredMap} from "./index"; +import {BaseObservableValue, MapSizeObservableValue} from "../value/index"; import {SortedMapList} from "../list/SortedMapList.js"; @@ -66,19 +67,23 @@ export abstract class BaseObservableMap extends BaseObservable>(...otherMaps: Array): JoinedMap { return new JoinedMap([this as BaseObservableMap].concat(otherMaps)); - } + } - mapValues(mapper: Mapper, updater?: Updater): MappedMap { - return new MappedMap(this, mapper, updater); - } + mapValues(mapper: Mapper, updater?: Updater): MappedMap { + return new MappedMap(this, mapper, updater); + } - sortValues(comparator: Comparator): SortedMapList { - return new SortedMapList(this, comparator); - } + sortValues(comparator: Comparator): SortedMapList { + return new SortedMapList(this, comparator); + } - filterValues(filter: Filter): FilteredMap { - return new FilteredMap(this, filter); - } + filterValues(filter: Filter): FilteredMap { + return new FilteredMap(this, filter); + } + + observeSize(): BaseObservableValue { + return new MapSizeObservableValue(this); + } abstract [Symbol.iterator](): Iterator<[K, V]>; abstract get size(): number; @@ -94,4 +99,4 @@ export type Updater = (params: any, mappedValue?: MappedV, value?: V export type Comparator = (a: V, b: V) => number; -export type Filter = (v: V, k: K) => boolean; \ No newline at end of file +export type Filter = (v: V, k: K) => boolean; diff --git a/src/observable/value/MapSizeObservableValue.ts b/src/observable/value/MapSizeObservableValue.ts new file mode 100644 index 00000000..13bac604 --- /dev/null +++ b/src/observable/value/MapSizeObservableValue.ts @@ -0,0 +1,71 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {BaseObservableValue} from "./index"; +import {BaseObservableMap} from "../map/index"; +import type {SubscriptionHandle} from "../BaseObservable"; + +export class MapSizeObservableValue extends BaseObservableValue { + private subscription?: SubscriptionHandle; + + constructor(private readonly map: BaseObservableMap) + { + super(); + } + + onSubscribeFirst(): void { + this.subscription = this.map.subscribe({ + onAdd: (key: K, value: V) => { + this.emit(this.get()); + }, + onRemove: (key: K, value: V) => { + this.emit(this.get()); + }, + onUpdate: (key: K, value: V) => {}, + onReset: () => { + this.emit(this.get()); + }, + }); + } + + onUnsubscribeLast(): void { + this.subscription = this.subscription?.(); + } + + get(): number { + return this.map.size; + } +} + +import {ObservableMap} from "../map/index"; + +export function tests() { + return { + "emits update on add and remove": assert => { + const map = new ObservableMap(); + const size = new MapSizeObservableValue(map); + const updates: number[] = []; + size.subscribe(size => { + updates.push(size); + }); + map.add("hello", 1); + map.add("world", 2); + map.remove("world"); + map.remove("hello"); + assert.deepEqual(updates, [1, 2, 1, 0]); + } + }; +} diff --git a/src/observable/value/index.ts b/src/observable/value/index.ts index ceb53628..1b0f1347 100644 --- a/src/observable/value/index.ts +++ b/src/observable/value/index.ts @@ -12,4 +12,5 @@ export {EventObservableValue} from './EventObservableValue'; export {FlatMapObservableValue} from './FlatMapObservableValue'; export {PickMapObservableValue} from './PickMapObservableValue'; export {RetainedObservableValue} from './RetainedObservableValue'; +export {MapSizeObservableValue} from './MapSizeObservableValue'; export {ObservableValue} from './ObservableValue'; From 59ebcf99fbd1fc1c3f4cfdae50f9e0bf78c01529 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 26 Jan 2023 11:56:30 +0100 Subject: [PATCH 337/435] use observeSize to emit update on memberCount rather than custom handler --- .../toast/CallToastNotificationViewModel.ts | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/src/domain/session/toast/CallToastNotificationViewModel.ts b/src/domain/session/toast/CallToastNotificationViewModel.ts index 3281b86a..5c788334 100644 --- a/src/domain/session/toast/CallToastNotificationViewModel.ts +++ b/src/domain/session/toast/CallToastNotificationViewModel.ts @@ -34,22 +34,9 @@ type MinimumNeededSegmentType = { export class CallToastNotificationViewModel = Options> extends BaseToastNotificationViewModel implements IAvatarContract { constructor(options: O) { super(options); - this.track( - this.call.members.subscribe({ - onAdd: (_, __) => { - this.emitChange("memberCount"); - }, - onUpdate: (_, __) => { - this.emitChange("memberCount"); - }, - onRemove: (_, __) => { - this.emitChange("memberCount"); - }, - onReset: () => { - this.emitChange("memberCount"); - }, - }) - ); + this.track(this.call.members.observeSize().subscribe(() => { + this.emitChange("memberCount"); + })); // Dismiss the toast if the room is opened manually this.track( this.navigation.observe("room").subscribe((roomId) => { From 90ba35da7a19939b967b84bfe5e2e6fb0f34241e Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 26 Jan 2023 14:50:43 +0100 Subject: [PATCH 338/435] listen for members.size changes in CallTile and emit update so memberCount binding updates also be consistent to not emit updates on call object when changing members map --- src/domain/session/room/timeline/tiles/CallTile.js | 4 ++++ src/matrix/calls/group/GroupCall.ts | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index 4583de49..ffb9ef59 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -30,6 +30,7 @@ export class CallTile extends SimpleTile { super(entry, options); const calls = this.getOption("session").callHandler.calls; this._callSubscription = undefined; + this._memberSizeSubscription = undefined; const call = calls.get(this._entry.stateKey); if (call && !call.isTerminated) { this._call = call; @@ -37,6 +38,9 @@ export class CallTile extends SimpleTile { this._callSubscription = this.track(this._call.disposableOn("change", () => { this._onCallUpdate(); })); + this._memberSizeSubscription = this.track(this._call.members.observeSize().subscribe(() => { + this.emitChange("memberCount"); + })); this._onCallUpdate(); } } diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 7f9ba96e..e0099f4e 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -549,7 +549,6 @@ export class GroupCall extends EventEmitter<{change: never}> { member.dispose(); this._members.remove(memberKey); } - this.emitChange(); }); } From d65bbcf168a56ad70ff022cba7f535ac1cdca50f Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Fri, 27 Jan 2023 13:38:45 +0530 Subject: [PATCH 339/435] Fix scaling on chrome --- src/platform/web/ui/css/themes/element/theme.css | 10 ++++++++-- src/platform/web/ui/css/themes/element/timeline.css | 5 ++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/platform/web/ui/css/themes/element/theme.css b/src/platform/web/ui/css/themes/element/theme.css index 3aec2a8a..7bea169e 100644 --- a/src/platform/web/ui/css/themes/element/theme.css +++ b/src/platform/web/ui/css/themes/element/theme.css @@ -1278,7 +1278,10 @@ button.RoomDetailsView_row::after { } .CallToastNotificationView__call-type::before { - content: url("./icons/video-call.svg?primary=light-text-color"); + content: ""; + background-image: url("./icons/video-call.svg?primary=light-text-color"); + background-repeat: no-repeat; + background-size: 20px 20px; display: flex; width: 20px; height: 20px; @@ -1296,7 +1299,10 @@ button.RoomDetailsView_row::after { } .CallToastNotificationView__member-count::before { - content: url("./icons/room-members.svg?primary=light-text-color"); + content: ""; + background-image: url("./icons/room-members.svg?primary=light-text-color"); + background-repeat: no-repeat; + background-size: 20px 20px; display: flex; width: 20px; height: 20px; diff --git a/src/platform/web/ui/css/themes/element/timeline.css b/src/platform/web/ui/css/themes/element/timeline.css index 16c758a3..8606400d 100644 --- a/src/platform/web/ui/css/themes/element/timeline.css +++ b/src/platform/web/ui/css/themes/element/timeline.css @@ -465,7 +465,10 @@ only loads when the top comes into view*/ } .CallTileView_memberCount::before { - content: url('./icons/room-members.svg?primary=text-color'); + content: ""; + background-image: url('./icons/room-members.svg?primary=text-color'); + background-repeat: no-repeat; + background-size: 16px 16px; width: 16px; height: 16px; display: inline-flex; From 43dea3bfdc1af23df1efc43808715c8b5ba733c2 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 27 Jan 2023 10:01:53 +0100 Subject: [PATCH 340/435] detect calls using a foci --- src/matrix/calls/callEventTypes.ts | 6 ++++++ src/matrix/calls/group/GroupCall.ts | 9 +++++++++ src/matrix/calls/group/Member.ts | 5 +++++ 3 files changed, 20 insertions(+) diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index a0eb986c..879d0c11 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -22,11 +22,17 @@ export enum EventType { // TODO: Change to "sdp_stream_metadata" when MSC3077 is merged export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata"; +export interface FocusConfig { + user_id: string, + device_id: string +} + export interface CallDeviceMembership { device_id: string, session_id: string, ["expires_ts"]?: number, feeds?: Array<{purpose: string}> + ["m.foci.active"]?: Array } export interface CallMembership { diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index e0099f4e..16f09dd2 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -145,6 +145,15 @@ export class GroupCall extends EventEmitter<{change: never}> { return !!this.callContent?.["m.terminated"]; } + get usesFoci(): boolean { + for (const member of this._members.values()) { + if (member.usesFoci) { + return true; + } + } + return false; + } + get duration(): number | undefined { if (typeof this.startTime === "number") { return (this.options.clock.now() - this.startTime); diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index e097804c..ab99c6b8 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -117,6 +117,11 @@ export class Member { return this.errorBoundary.error; } + get usesFoci(): boolean { + const activeFoci = this.callDeviceMembership["m.foci.active"]; + return Array.isArray(activeFoci) && activeFoci.length > 0; + } + private _renewExpireTimeout(log: ILogItem) { this.expireTimeout?.dispose(); this.expireTimeout = undefined; From 365157449e9e052c6045f6a8693dadcdc42566be Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 27 Jan 2023 10:02:13 +0100 Subject: [PATCH 341/435] cleanup loops here to not get keys --- src/matrix/calls/group/GroupCall.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 16f09dd2..90c4d982 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -226,7 +226,7 @@ export class GroupCall extends EventEmitter<{change: never}> { this.emitChange(); }); // send invite to all members that are < my userId - for (const [,member] of this._members) { + for (const member of this._members.values()) { this.connectToMember(member, joinedData, log); } }); @@ -530,7 +530,7 @@ export class GroupCall extends EventEmitter<{change: never}> { disconnect(log: ILogItem): boolean { return this.errorBoundary.try(() => { if (this.hasJoined) { - for (const [,member] of this._members) { + for (const member of this._members.values()) { const disconnectLogItem = member.disconnect(true); if (disconnectLogItem) { log.refDetached(disconnectLogItem); From c8bb5fffb0fa7cce87a9d02fd4dbb6f8826af2a6 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 27 Jan 2023 10:12:22 +0100 Subject: [PATCH 342/435] don't allow to join a call using a foci --- src/domain/session/room/timeline/tiles/CallTile.js | 2 +- src/matrix/calls/group/GroupCall.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index ffb9ef59..37be05be 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -94,7 +94,7 @@ export class CallTile extends SimpleTile { } get canJoin() { - return this._call && !this._call.hasJoined; + return this._call && !this._call.hasJoined && !this._call.usesFoci; } get canLeave() { diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 90c4d982..9ef2e068 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -190,7 +190,7 @@ export class GroupCall extends EventEmitter<{change: never}> { join(localMedia: LocalMedia, log?: ILogItem): Promise { return this.options.logger.wrapOrRun(log, "Call.join", async joinLog => { - if (this._state !== GroupCallState.Created || this.joinedData) { + if (this._state !== GroupCallState.Created || this.joinedData || this.usesFoci) { return; } const logItem = this.options.logger.child({ From 825602a04a2d89dd6cb7ec75cab3de821a6b77fc Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 27 Jan 2023 10:12:36 +0100 Subject: [PATCH 343/435] dispose local media here when returning early as join takes ownership --- src/matrix/calls/group/GroupCall.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 9ef2e068..0460c212 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -191,6 +191,7 @@ export class GroupCall extends EventEmitter<{change: never}> { join(localMedia: LocalMedia, log?: ILogItem): Promise { return this.options.logger.wrapOrRun(log, "Call.join", async joinLog => { if (this._state !== GroupCallState.Created || this.joinedData || this.usesFoci) { + localMedia.dispose(); return; } const logItem = this.options.logger.child({ From f5838b21baae8fed048af4e8aa376a00f1c8d50f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 27 Jan 2023 10:12:51 +0100 Subject: [PATCH 344/435] show message in tile when call uses foci, explaining we can't join --- src/domain/session/room/timeline/tiles/CallTile.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index 37be05be..05762bc0 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -118,11 +118,14 @@ export class CallTile extends SimpleTile { } get typeLabel() { + if (this._call && this._call.usesFoci) { + return `This call uses a stream-forwarding unit, which isn't supported yet, so you can't join this call.`; + } if (this.type === CallType.Video) { - return `Video call`; - } else { - return `Voice call`; - } + return `Video call`; + } else { + return `Voice call`; + } } get type() { From 1d7db53f30cb4f6ff438fe86936df84fee797737 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 27 Jan 2023 10:22:01 +0100 Subject: [PATCH 345/435] don't show toast for foci calls --- src/domain/session/toast/ToastCollectionViewModel.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/session/toast/ToastCollectionViewModel.ts b/src/domain/session/toast/ToastCollectionViewModel.ts index 18b58b76..709f5b81 100644 --- a/src/domain/session/toast/ToastCollectionViewModel.ts +++ b/src/domain/session/toast/ToastCollectionViewModel.ts @@ -79,7 +79,7 @@ export class ToastCollectionViewModel extends ViewModel { private _shouldShowNotification(call: GroupCall): boolean { const currentlyOpenedRoomId = this.navigation.path.get("room")?.value; - if (!call.isLoadedFromStorage && call.roomId !== currentlyOpenedRoomId) { + if (!call.isLoadedFromStorage && call.roomId !== currentlyOpenedRoomId && !call.usesFoci) { return true; } return false; From 903a157de2bbf6c61545cccfbb91468199c7f2e5 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 31 Jan 2023 16:48:48 +0530 Subject: [PATCH 346/435] Don't show toast if room is not available --- src/domain/session/toast/ToastCollectionViewModel.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/domain/session/toast/ToastCollectionViewModel.ts b/src/domain/session/toast/ToastCollectionViewModel.ts index 709f5b81..78961029 100644 --- a/src/domain/session/toast/ToastCollectionViewModel.ts +++ b/src/domain/session/toast/ToastCollectionViewModel.ts @@ -39,6 +39,9 @@ export class ToastCollectionViewModel extends ViewModel { onAdd(_, call: GroupCall) { if (this._shouldShowNotification(call)) { const room = this._findRoomForCall(call); + if (!room) { + return; + } const dismiss = () => { const idx = this.toastViewModels.array.findIndex(vm => vm.call === call); if (idx !== -1) { From 9a6d15a72b64c7a93af73319b494833d92fc09c8 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 31 Jan 2023 16:49:58 +0530 Subject: [PATCH 347/435] Fix unwanted toast appearing when opening client --- src/matrix/calls/CallHandler.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index af01beef..b4a1c5cf 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -229,7 +229,8 @@ export class CallHandler implements RoomStateHandler { this._calls.remove(call.id); txn.calls.remove(call.intent, roomId, call.id); } - } else { + } else if(!event.content["m.terminated"]) { + // We don't have this call already and it isn't terminated, so create the call: call = new GroupCall( event.state_key, // id false, // isLoadedFromStorage From de57e0798269b87c899528f78c0375677cde67a3 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Thu, 2 Feb 2023 15:26:38 +0530 Subject: [PATCH 348/435] Wait for room to come through sync --- .../session/toast/ToastCollectionViewModel.ts | 20 ++++++++------ src/matrix/Session.js | 26 ++++++++++++------- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/src/domain/session/toast/ToastCollectionViewModel.ts b/src/domain/session/toast/ToastCollectionViewModel.ts index 78961029..d1595b2d 100644 --- a/src/domain/session/toast/ToastCollectionViewModel.ts +++ b/src/domain/session/toast/ToastCollectionViewModel.ts @@ -21,6 +21,7 @@ import type {GroupCall} from "../../../matrix/calls/group/GroupCall"; import type {Room} from "../../../matrix/room/Room.js"; import type {Session} from "../../../matrix/Session.js"; import type {SegmentType} from "../../navigation"; +import { RoomStatus } from "../../../lib"; type Options = { session: Session; @@ -36,12 +37,9 @@ export class ToastCollectionViewModel extends ViewModel { this.track(callsObservableMap.subscribe(this)); } - onAdd(_, call: GroupCall) { + async onAdd(_, call: GroupCall) { if (this._shouldShowNotification(call)) { - const room = this._findRoomForCall(call); - if (!room) { - return; - } + const room = await this._findRoomForCall(call); const dismiss = () => { const idx = this.toastViewModels.array.findIndex(vm => vm.call === call); if (idx !== -1) { @@ -74,10 +72,16 @@ export class ToastCollectionViewModel extends ViewModel { } } - private _findRoomForCall(call: GroupCall): Room { + private async _findRoomForCall(call: GroupCall): Promise { const id = call.roomId; - const rooms = this.getOption("session").rooms; - return rooms.get(id); + const session = this.getOption("session"); + const rooms = session.rooms; + // Make sure that we know of this room, + // otherwise wait for it to come through sync + const observable = await session.observeRoomStatus(id); + await observable.waitFor(s => s === RoomStatus.Joined).promise; + const room = rooms.get(id); + return room; } private _shouldShowNotification(call: GroupCall): boolean { diff --git a/src/matrix/Session.js b/src/matrix/Session.js index d640e2df..87680a49 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -132,6 +132,7 @@ export class Session { this._createRoomEncryption = this._createRoomEncryption.bind(this); this._forgetArchivedRoom = this._forgetArchivedRoom.bind(this); this.needsKeyBackup = new ObservableValue(false); + this._pendingObserveCalls = []; } get fingerprintKey() { @@ -776,7 +777,7 @@ export class Session { } } - applyRoomCollectionChangesAfterSync(inviteStates, roomStates, archivedRoomStates, log) { + async applyRoomCollectionChangesAfterSync(inviteStates, roomStates, archivedRoomStates, log) { // update the collections after sync for (const rs of roomStates) { if (rs.shouldAdd) { @@ -796,6 +797,8 @@ export class Session { // now all the collections are updated, update the room status // so any listeners to the status will find the collections // completely up to date + await Promise.all(this._pendingObserveCalls); + this._pendingObserveCalls = []; if (this._observedRoomStatus.size !== 0) { for (const ars of archivedRoomStates) { if (ars.shouldAdd) { @@ -938,16 +941,21 @@ export class Session { } async observeRoomStatus(roomId) { - let observable = this._observedRoomStatus.get(roomId); - if (!observable) { - const status = await this.getRoomStatus(roomId); - observable = new RetainedObservableValue(status, () => { - this._observedRoomStatus.delete(roomId); - }); + const op = async () => { + let observable = this._observedRoomStatus.get(roomId); + if (!observable) { + const status = await this.getRoomStatus(roomId); + observable = new RetainedObservableValue(status, () => { + this._observedRoomStatus.delete(roomId); + }); - this._observedRoomStatus.set(roomId, observable); + this._observedRoomStatus.set(roomId, observable); + } + return observable; } - return observable; + const promise = op(); + this._pendingObserveCalls.push(promise); + return await promise; } observeRoomState(roomStateHandler) { From 09e67ec21c42783bc847a8592cd4bd2bd3e618f5 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Thu, 2 Feb 2023 18:39:19 +0530 Subject: [PATCH 349/435] Deal with race in a better way --- src/matrix/Session.js | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 87680a49..f1b273ff 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -132,7 +132,6 @@ export class Session { this._createRoomEncryption = this._createRoomEncryption.bind(this); this._forgetArchivedRoom = this._forgetArchivedRoom.bind(this); this.needsKeyBackup = new ObservableValue(false); - this._pendingObserveCalls = []; } get fingerprintKey() { @@ -797,8 +796,6 @@ export class Session { // now all the collections are updated, update the room status // so any listeners to the status will find the collections // completely up to date - await Promise.all(this._pendingObserveCalls); - this._pendingObserveCalls = []; if (this._observedRoomStatus.size !== 0) { for (const ars of archivedRoomStates) { if (ars.shouldAdd) { @@ -941,21 +938,24 @@ export class Session { } async observeRoomStatus(roomId) { - const op = async () => { - let observable = this._observedRoomStatus.get(roomId); - if (!observable) { - const status = await this.getRoomStatus(roomId); - observable = new RetainedObservableValue(status, () => { - this._observedRoomStatus.delete(roomId); - }); - - this._observedRoomStatus.set(roomId, observable); + let observable = this._observedRoomStatus.get(roomId); + if (!observable) { + let status = undefined; + // Create and set the observable with value = undefined, so that + // we don't loose any sync changes that come in while we are busy + // calculating the current room status. + observable = new RetainedObservableValue(status, () => { + this._observedRoomStatus.delete(roomId); + }); + this._observedRoomStatus.set(roomId, observable); + status = await this.getRoomStatus(roomId); + // If observable.value is not undefined anymore, then some + // change has come through the sync. + if (observable.get() === undefined) { + observable.set(status); } - return observable; } - const promise = op(); - this._pendingObserveCalls.push(promise); - return await promise; + return observable; } observeRoomState(roomStateHandler) { From 7eae171ac9e05a4c06650f2b6b03b9478853b122 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Mon, 6 Feb 2023 17:12:39 +0530 Subject: [PATCH 350/435] Emit after sending signalling message --- src/matrix/calls/PeerCall.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 36136b97..fcb71db1 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -284,11 +284,12 @@ export class PeerCall implements IDisposable { } private async _hangup(errorCode: CallErrorCode, log: ILogItem): Promise { - if (this._state === CallState.Ended) { + if (this._state === CallState.Ended || this._state === CallState.Ending) { return; } - this.terminate(CallParty.Local, errorCode, log); + this.setState(CallState.Ending, log); await this.sendHangupWithCallId(this.callId, errorCode, log); + this.terminate(CallParty.Local, errorCode, log); } getMessageAction(message: SignallingMessage): IncomingMessageAction { @@ -1130,6 +1131,7 @@ export enum CallState { Connecting = 'connecting', Connected = 'connected', Ringing = 'ringing', + Ending = 'ending', Ended = 'ended', } From cadeae98bc69dc35ba4f966e298fc34b5f7e7fdd Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 14:13:49 +0100 Subject: [PATCH 351/435] prevent ignored signaling messages from blocking the queue signaling messages get ignored when they are not for the currently active call id. In that case we currently don't advance the lastProcessedSeqNr counter, as we had a problem before where the counter would be brought out of sync with seq numbers for other call ids. However when we've previously processed a signalling message (e.g. the counter is not undefined) and the first message in the queue is to be ignored, it will prevent the subsequent messages from being dequeued as their seq number is more than 1 away from the last processed seq. This adds an additional counter for ignored seq numbers that is also used to see if the next message is only 1 away from the next seq value. I am adding logging as well here to have a better overview in the future --- src/matrix/calls/group/Member.ts | 51 +++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index ab99c6b8..f82c5ddd 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -59,6 +59,9 @@ class MemberConnection { public retryCount: number = 0; public peerCall?: PeerCall; public lastProcessedSeqNr: number | undefined; + // XXX: Not needed anymore when seq is scoped to call_id + // see https://github.com/matrix-org/matrix-spec-proposals/pull/3401#discussion_r1097482166 + public lastIgnoredSeqNr: number | undefined; public queuedSignallingMessages: SignallingMessage[] = []; public outboundSeqCounter: number = 0; @@ -73,16 +76,24 @@ class MemberConnection { if (this.queuedSignallingMessages.length === 0) { return false; } + const first = this.queuedSignallingMessages[0]; + const firstSeq = first.content.seq; + // prevent not being able to jump over seq values of ignored messages for other call ids + // as they don't increase lastProcessedSeqNr. + if (this.lastIgnoredSeqNr !== undefined && firstSeq === this.lastIgnoredSeqNr + 1) { + return true; + } if (this.lastProcessedSeqNr === undefined) { return true; } - const first = this.queuedSignallingMessages[0]; // allow messages with both a seq we've just seen and // the next one to be dequeued as it can happen // that messages for other callIds (which could repeat seq) // are present in the queue - return first.content.seq === this.lastProcessedSeqNr || - first.content.seq === this.lastProcessedSeqNr + 1; + // XXX: Not needed anymore when seq is scoped to call_id + // see https://github.com/matrix-org/matrix-spec-proposals/pull/3401#discussion_r1097482166 + return firstSeq === this.lastProcessedSeqNr || + firstSeq === this.lastProcessedSeqNr + 1; } dispose() { @@ -382,17 +393,29 @@ export class Member { private dequeueSignallingMessages(connection: MemberConnection, peerCall: PeerCall, newMessage: SignallingMessage, syncLog: ILogItem): boolean { let hasNewMessageBeenDequeued = false; while (connection.canDequeueNextSignallingMessage) { - const message = connection.queuedSignallingMessages.shift()!; - if (message === newMessage) { - hasNewMessageBeenDequeued = true; - } - // ignore items in the queue that should not be handled and prevent - // the lastProcessedSeqNr being corrupted with the `seq` for other call ids - if (peerCall.getMessageAction(message) === IncomingMessageAction.Handle) { - const item = peerCall.handleIncomingSignallingMessage(message, this.deviceId, connection.logItem); - syncLog.refDetached(item); - connection.lastProcessedSeqNr = message.content.seq; - } + syncLog.wrap("dequeue message", log => { + const message = connection.queuedSignallingMessages.shift()!; + if (message === newMessage) { + log.set("isNewMsg", true); + hasNewMessageBeenDequeued = true; + } + const seq = message.content?.seq; + log.set("seq", seq); + // ignore items in the queue that should not be handled and prevent + // the lastProcessedSeqNr being corrupted with the `seq` for other call ids + // XXX: Not needed anymore when seq is scoped to call_id + // see https://github.com/matrix-org/matrix-spec-proposals/pull/3401#discussion_r1097482166 + const action = peerCall.getMessageAction(message); + if (action === IncomingMessageAction.Handle) { + const item = peerCall.handleIncomingSignallingMessage(message, this.deviceId, connection.logItem); + log.refDetached(item); + connection.lastProcessedSeqNr = seq; + } else { + log.set("type", message.type); + log.set("ignored", true); + connection.lastIgnoredSeqNr = seq; + } + }); } return hasNewMessageBeenDequeued; } From 39e9a43a1b365227b5a5bb04e86cfff4763de212 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 14:42:44 +0100 Subject: [PATCH 352/435] be strict about the first seq being 0 otherwise if first 2 messages are delivered in reverse order, the queue gets blocked --- src/matrix/calls/group/Member.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index f82c5ddd..fe6e40a4 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -84,7 +84,7 @@ class MemberConnection { return true; } if (this.lastProcessedSeqNr === undefined) { - return true; + return firstSeq === 0; } // allow messages with both a seq we've just seen and // the next one to be dequeued as it can happen From 5f4ad30d03efaa3f0a1203e9964bac0bdf7f4ef0 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 14:43:28 +0100 Subject: [PATCH 353/435] don't block if it does happen that we have processed a message too early allow dequeueing if the first seq in the queue is actually lower than what we already processed. Normally should not happen, but the bug fixed in the previous commit was aggravated by this behavior, so be more lenient here. --- src/matrix/calls/group/Member.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index fe6e40a4..81b1dc7d 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -92,8 +92,7 @@ class MemberConnection { // are present in the queue // XXX: Not needed anymore when seq is scoped to call_id // see https://github.com/matrix-org/matrix-spec-proposals/pull/3401#discussion_r1097482166 - return firstSeq === this.lastProcessedSeqNr || - firstSeq === this.lastProcessedSeqNr + 1; + return firstSeq <= (this.lastProcessedSeqNr + 1); } dispose() { From 6d800ff35913e2dd494124cda4b2d1d7bd28b00a Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 7 Feb 2023 19:15:36 +0530 Subject: [PATCH 354/435] WIP --- src/matrix/calls/PeerCall.ts | 72 ++++++++++++++++++----------- src/matrix/calls/group/GroupCall.ts | 29 ++++++------ src/matrix/calls/group/Member.ts | 26 +++++++---- src/utils/ErrorBoundary.ts | 4 +- 4 files changed, 76 insertions(+), 55 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index fcb71db1..905756d7 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -44,6 +44,8 @@ import type { SDPStreamMetadata, SignallingMessage } from "./callEventTypes"; +import type { ErrorBoundary } from "../../utils/ErrorBoundary"; +import { AbortError } from "../../utils/error"; export type Options = { webRTC: WebRTC, @@ -51,6 +53,7 @@ export type Options = { turnServer: BaseObservableValue, createTimeout: TimeoutCreator, emitUpdate: (peerCall: PeerCall, params: any, log: ILogItem) => void; + errorBoundary: ErrorBoundary; sendSignallingMessage: (message: SignallingMessage, log: ILogItem) => Promise; }; @@ -125,31 +128,34 @@ export class PeerCall implements IDisposable { this.logItem.log({l: "updating turn server", turnServer}) this.peerConnection.setConfiguration({iceServers: [turnServer]}); })); - const listen = (type: K, listener: (this: PeerConnection, ev: PeerConnectionEventMap[K]) => any, options?: boolean | EventListenerOptions): void => { - this.peerConnection.addEventListener(type, listener); + const listen = (type: K, listener: (ev: PeerConnectionEventMap[K]) => any, options?: boolean | EventListenerOptions): void => { + const newListener = (e) => { + this.options.errorBoundary.try(() => listener(e)); + }; + this.peerConnection.addEventListener(type, newListener); const dispose = () => { - this.peerConnection.removeEventListener(type, listener); + this.peerConnection.removeEventListener(type, newListener); }; this.disposables.track(dispose); }; - listen("iceconnectionstatechange", () => { + listen("iceconnectionstatechange", async () => { const state = this.peerConnection.iceConnectionState; - logItem.wrap({l: "onIceConnectionStateChange", status: state}, log => { - this.onIceConnectionStateChange(state, log); + await logItem.wrap({l: "onIceConnectionStateChange", status: state}, async log => { + await this.onIceConnectionStateChange(state, log); }); }); - listen("icecandidate", event => { - logItem.wrap("onLocalIceCandidate", log => { + listen("icecandidate", async (event) => { + await logItem.wrap("onLocalIceCandidate", async log => { if (event.candidate) { - this.handleLocalIceCandidate(event.candidate, log); + await this.handleLocalIceCandidate(event.candidate, log); } }); }); - listen("icegatheringstatechange", () => { + listen("icegatheringstatechange", async () => { const state = this.peerConnection.iceGatheringState; - logItem.wrap({l: "onIceGatheringStateChange", status: state}, log => { - this.handleIceGatheringState(state, log); + await logItem.wrap({l: "onIceGatheringStateChange", status: state}, async log => { + await this.handleIceGatheringState(state, log); }); }); listen("track", event => { @@ -422,14 +428,18 @@ export class PeerCall implements IDisposable { log.refDetached(timeoutLog); // don't await this, as it would block other negotationneeded events from being processed // as they are processed serially - timeoutLog.run(async log => { - try { await this.delay(CALL_TIMEOUT_MS); } - catch (err) { return; } - // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between - if (this._state === CallState.InviteSent) { - this._hangup(CallErrorCode.InviteTimeout, log); - } - }).catch(err => {}); // prevent error from being unhandled, it will be logged already by run above + try { + await timeoutLog.run(async log => { + await this.delay(CALL_TIMEOUT_MS); + // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between + if (this._state === CallState.InviteSent) { + await this._hangup(CallErrorCode.InviteTimeout, log); + } + }); + } + catch (e) { + // prevent error from being unhandled, it will be logged already by run above + } } }; @@ -579,7 +589,7 @@ export class PeerCall implements IDisposable { } } - private handleIceGatheringState(state: RTCIceGatheringState, log: ILogItem) { + private async handleIceGatheringState(state: RTCIceGatheringState, log: ILogItem) { if (state === 'complete' && !this.sentEndOfCandidates) { // If we didn't get an empty-string candidate to signal the end of candidates, // create one ourselves now gathering has finished. @@ -591,12 +601,12 @@ export class PeerCall implements IDisposable { const c = { candidate: '', } as RTCIceCandidate; - this.queueCandidate(c, log); + await this.queueCandidate(c, log); this.sentEndOfCandidates = true; } } - private handleLocalIceCandidate(candidate: RTCIceCandidate, log: ILogItem) { + private async handleLocalIceCandidate(candidate: RTCIceCandidate, log: ILogItem) { log.set("sdpMid", candidate.sdpMid); log.set("candidate", candidate.candidate); @@ -606,7 +616,7 @@ export class PeerCall implements IDisposable { // As with the offer, note we need to make a copy of this object, not // pass the original: that broke in Chrome ~m43. if (candidate.candidate !== '' || !this.sentEndOfCandidates) { - this.queueCandidate(candidate, log); + await this.queueCandidate(candidate, log); if (candidate.candidate === '') { this.sentEndOfCandidates = true; } @@ -841,13 +851,19 @@ export class PeerCall implements IDisposable { logStats = true; this.iceDisconnectedTimeout?.abort(); this.iceDisconnectedTimeout = undefined; - this._hangup(CallErrorCode.IceFailed, log); + await this._hangup(CallErrorCode.IceFailed, log); } else if (state == 'disconnected') { logStats = true; this.iceDisconnectedTimeout = this.options.createTimeout(30 * 1000); - this.iceDisconnectedTimeout.elapsed().then(() => { - this._hangup(CallErrorCode.IceFailed, log); - }, () => { /* ignore AbortError */ }); + try { + await this.iceDisconnectedTimeout.elapsed() + await this._hangup(CallErrorCode.IceFailed, log); + } + catch (e){ + if (!(e instanceof AbortError)) { + throw e; + } + } } if (logStats) { const stats = await this.peerConnection.getStats(); diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 0460c212..d540d2e0 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -133,7 +133,8 @@ export class GroupCall extends EventEmitter<{change: never}> { }, encryptDeviceMessage: (userId: string, deviceId: string, message: SignallingMessage, log) => { return this.options.encryptDeviceMessage(this.roomId, userId, deviceId, message, log); - } + }, + groupCallErrorBoundary: this.errorBoundary, }); } @@ -392,8 +393,8 @@ export class GroupCall extends EventEmitter<{change: never}> { /** @internal */ updateMembership(userId: string, roomMember: RoomMember, callMembership: CallMembership, syncLog: ILogItem) { - this.errorBoundary.try(() => { - syncLog.wrap({l: "update call membership", t: CALL_LOG_TYPE, id: this.id, userId}, log => { + this.errorBoundary.try(async () => { + await syncLog.wrap({l: "update call membership", t: CALL_LOG_TYPE, id: this.id, userId}, async log => { const now = this.options.clock.now(); const devices = callMembership["m.devices"]; const previousDeviceIds = this.getDeviceIdsForUserId(userId); @@ -415,7 +416,7 @@ export class GroupCall extends EventEmitter<{change: never}> { } }); } else { - log.wrap({l: "update device membership", id: memberKey, sessionId: device.session_id}, log => { + await log.wrap({l: "update device membership", id: memberKey, sessionId: device.session_id}, async log => { if (isMemberExpired(device, now)) { log.set("expired", true); const member = this._members.get(memberKey); @@ -434,7 +435,7 @@ export class GroupCall extends EventEmitter<{change: never}> { } else { if (member && sessionIdChanged) { log.set("removedSessionId", member.sessionId); - const disconnectLogItem = member.disconnect(false); + const disconnectLogItem = await member.disconnect(false); if (disconnectLogItem) { log.refDetached(disconnectLogItem); } @@ -528,11 +529,11 @@ export class GroupCall extends EventEmitter<{change: never}> { } /** @internal */ - disconnect(log: ILogItem): boolean { - return this.errorBoundary.try(() => { + disconnect(log: ILogItem): Promise | true { + return this.errorBoundary.try(async () => { if (this.hasJoined) { for (const member of this._members.values()) { - const disconnectLogItem = member.disconnect(true); + const disconnectLogItem = await member.disconnect(true); if (disconnectLogItem) { log.refDetached(disconnectLogItem); } @@ -546,13 +547,13 @@ export class GroupCall extends EventEmitter<{change: never}> { } /** @internal */ - private removeMemberDevice(userId: string, deviceId: string, log: ILogItem) { + private async removeMemberDevice(userId: string, deviceId: string, log: ILogItem) { const memberKey = getMemberKey(userId, deviceId); - log.wrap({l: "remove device member", id: memberKey}, log => { + await log.wrap({l: "remove device member", id: memberKey}, async log => { const member = this._members.get(memberKey); if (member) { log.set("leave", true); - const disconnectLogItem = member.disconnect(false); + const disconnectLogItem = await member.disconnect(false); if (disconnectLogItem) { log.refDetached(disconnectLogItem); } @@ -634,15 +635,15 @@ export class GroupCall extends EventEmitter<{change: never}> { return stateContent; } - private connectToMember(member: Member, joinedData: JoinedData, log: ILogItem) { + private async connectToMember(member: Member, joinedData: JoinedData, log: ILogItem) { const memberKey = getMemberKey(member.userId, member.deviceId); const logItem = joinedData.membersLogItem.child({ l: "member", id: memberKey, sessionId: member.sessionId }); - log.wrap({l: "connect", id: memberKey}, log => { - const connectItem = member.connect( + await log.wrap({l: "connect", id: memberKey}, async log => { + const connectItem = await member.connect( joinedData.localMedia, joinedData.localMuteSettings, joinedData.turnServer, diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index ab99c6b8..88e0369b 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -42,6 +42,7 @@ export type Options = Omit, log: ILogItem) => Promise, emitUpdate: (participant: Member, params?: any) => void, + groupCallErrorBoundary: ErrorBoundary, clock: Clock } @@ -183,8 +184,8 @@ export class Member { } /** @internal */ - connect(localMedia: LocalMedia, localMuteSettings: MuteSettings, turnServer: BaseObservableValue, memberLogItem: ILogItem): ILogItem | undefined { - return this.errorBoundary.try(() => { + connect(localMedia: LocalMedia, localMuteSettings: MuteSettings, turnServer: BaseObservableValue, memberLogItem: ILogItem): Promise | undefined { + return this.errorBoundary.try(async () => { if (this.connection) { return; } @@ -197,7 +198,7 @@ export class Member { ); this.connection = connection; let connectLogItem: ILogItem | undefined; - connection.logItem.wrap("connect", async log => { + await connection.logItem.wrap("connect", async log => { connectLogItem = log; await this.callIfNeeded(log); }); @@ -230,15 +231,15 @@ export class Member { } /** @internal */ - disconnect(hangup: boolean): ILogItem | undefined { - return this.errorBoundary.try(() => { + disconnect(hangup: boolean): Promise | undefined { + return this.errorBoundary.try(async () => { const {connection} = this; if (!connection) { return; } - let disconnectLogItem; + let disconnectLogItem: ILogItem | undefined; // if if not sending the hangup, still log disconnect - connection.logItem.wrap("disconnect", async log => { + await connection.logItem.wrap("disconnect", async log => { disconnectLogItem = log; if (hangup && connection.peerCall) { await connection.peerCall.hangup(CallErrorCode.UserHangup, log); @@ -269,7 +270,7 @@ export class Member { } /** @internal */ - emitUpdateFromPeerCall = (peerCall: PeerCall, params: any, log: ILogItem): void => { + emitUpdateFromPeerCall = async (peerCall: PeerCall, params: any, log: ILogItem): Promise => { const connection = this.connection!; if (peerCall.state === CallState.Ringing) { connection.logItem.wrap("ringing, answer peercall", answerLog => { @@ -284,12 +285,12 @@ export class Member { if (hangupReason && !errorCodesWithoutRetry.includes(hangupReason)) { connection.retryCount += 1; const {retryCount} = connection; - connection.logItem.wrap({l: "retry connection", retryCount}, async retryLog => { + await connection.logItem.wrap({l: "retry connection", retryCount}, async retryLog => { log.refDetached(retryLog); if (retryCount <= 3) { await this.callIfNeeded(retryLog); } else { - const disconnectLogItem = this.disconnect(false); + const disconnectLogItem = await this.disconnect(false); if (disconnectLogItem) { retryLog.refDetached(disconnectLogItem); } @@ -303,6 +304,10 @@ export class Member { /** @internal */ sendSignallingMessage = async (message: SignallingMessage, log: ILogItem): Promise => { const groupMessage = message as SignallingMessage; + if (this.connection?.peerCall?.state === CallState.CreateOffer) { + // @ts-ignore + this.connection!.foobar.barfpp; + } groupMessage.content.seq = this.connection!.outboundSeqCounter++; groupMessage.content.conf_id = this.options.confId; groupMessage.content.device_id = this.options.ownDeviceId; @@ -421,6 +426,7 @@ export class Member { private _createPeerCall(callId: string): PeerCall { const connection = this.connection!; return new PeerCall(callId, Object.assign({}, this.options, { + errorBoundary: this.options.groupCallErrorBoundary, emitUpdate: this.emitUpdateFromPeerCall, sendSignallingMessage: this.sendSignallingMessage, turnServer: connection.turnServer diff --git a/src/utils/ErrorBoundary.ts b/src/utils/ErrorBoundary.ts index a74da479..a0f6fca7 100644 --- a/src/utils/ErrorBoundary.ts +++ b/src/utils/ErrorBoundary.ts @@ -14,8 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -export const ErrorValue = Symbol("ErrorBoundary:Error"); - export class ErrorBoundary { private _error?: Error; @@ -23,7 +21,7 @@ export class ErrorBoundary { /** * Executes callback() and then runs errorCallback() on error. - * This will never throw but instead return `errorValue` if an error occured. + * This will never throw but instead return `errorValue` if an error occurred. */ try(callback: () => T, errorValue?: E): T | typeof errorValue; try(callback: () => Promise, errorValue?: E): Promise | typeof errorValue { From 496dfee5937ed62cad7a79794ee2e36cc4fc304e Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 7 Feb 2023 19:25:48 +0530 Subject: [PATCH 355/435] Catch error in promise chain --- src/matrix/calls/PeerCall.ts | 3 +++ src/matrix/calls/group/GroupCall.ts | 1 + src/matrix/calls/group/Member.ts | 4 ---- src/utils/ErrorBoundary.ts | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 905756d7..d81bf047 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -176,6 +176,9 @@ export class PeerCall implements IDisposable { }); }; this.responsePromiseChain = this.responsePromiseChain?.then(promiseCreator) ?? promiseCreator(); + this.responsePromiseChain.catch((e) => + this.options.errorBoundary.reportError(e) + ); }); } diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index d540d2e0..d728e7c2 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -99,6 +99,7 @@ export class GroupCall extends EventEmitter<{change: never}> { // in case the error happens in code that does not log, // log it here to make sure it isn't swallowed this.joinedData.logItem.log("error at boundary").catch(err); + console.error(err); } }); diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 88e0369b..0c31bc32 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -304,10 +304,6 @@ export class Member { /** @internal */ sendSignallingMessage = async (message: SignallingMessage, log: ILogItem): Promise => { const groupMessage = message as SignallingMessage; - if (this.connection?.peerCall?.state === CallState.CreateOffer) { - // @ts-ignore - this.connection!.foobar.barfpp; - } groupMessage.content.seq = this.connection!.outboundSeqCounter++; groupMessage.content.conf_id = this.options.confId; groupMessage.content.device_id = this.options.ownDeviceId; diff --git a/src/utils/ErrorBoundary.ts b/src/utils/ErrorBoundary.ts index a0f6fca7..750385c7 100644 --- a/src/utils/ErrorBoundary.ts +++ b/src/utils/ErrorBoundary.ts @@ -42,7 +42,7 @@ export class ErrorBoundary { } } - private reportError(err: Error) { + reportError(err: Error) { try { this.errorCallback(err); } catch (err) { From e39dd176a4b1edb5a583acb22e9ca0ff6e0aa326 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 14:58:32 +0100 Subject: [PATCH 356/435] remove debug logging --- src/matrix/calls/group/Member.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 81b1dc7d..23b35bd8 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -372,6 +372,7 @@ export class Member { } if (action === IncomingMessageAction.Handle) { const idx = sortedIndex(connection.queuedSignallingMessages, message, (a, b) => a.content.seq - b.content.seq); + console.log(`splice ${message.type} at ${idx}`); connection.queuedSignallingMessages.splice(idx, 0, message); if (connection.peerCall) { const hasNewMessageBeenDequeued = this.dequeueSignallingMessages(connection, connection.peerCall, message, syncLog); From f67fb7add6ef0d06c44e9023ceb32254ee368bb1 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 14:58:57 +0100 Subject: [PATCH 357/435] add unit test for this particular error case --- src/matrix/calls/group/Member.ts | 114 ++++++++++++++++++++++++++++++- 1 file changed, 112 insertions(+), 2 deletions(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 23b35bd8..d1463b51 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -21,13 +21,13 @@ import {formatToDeviceMessagesPayload} from "../../common"; import {sortedIndex} from "../../../utils/sortedIndex"; import { ErrorBoundary } from "../../../utils/ErrorBoundary"; -import type {MuteSettings} from "../common"; +import {MuteSettings} from "../common"; import type {Options as PeerCallOptions, RemoteMedia} from "../PeerCall"; import type {LocalMedia} from "../LocalMedia"; import type {HomeServerApi} from "../../net/HomeServerApi"; import type {MCallBase, MGroupCallBase, SignallingMessage, CallDeviceMembership} from "../callEventTypes"; import type {GroupCall} from "./GroupCall"; -import type {RoomMember} from "../../room/members/RoomMember"; +import {RoomMember} from "../../room/members/RoomMember"; import type {EncryptedMessage} from "../../e2ee/olm/Encryption"; import type {ILogItem} from "../../../logging/types"; import type {BaseObservableValue} from "../../../observable/value"; @@ -471,3 +471,113 @@ export function isMemberExpired(callDeviceMembership: CallDeviceMembership, now: const expiresAt = memberExpiresAt(callDeviceMembership); return typeof expiresAt === "number" ? ((expiresAt + margin) <= now) : true; } + +import {ObservableValue} from "../../../observable/value"; +import {Clock as MockClock} from "../../../mocks/Clock"; +import {Instance as NullLoggerInstance} from "../../../logging/NullLogger"; + +export function tests() { + + class MockMedia { + clone(): MockMedia { return this; } + } + + class MockPeerConn { + addEventListener() {} + removeEventListener() {} + setConfiguration() {} + setRemoteDescription() {} + getReceivers() { return [{}]; } // non-empty array + getSenders() { return []; } + addTrack() { return {}; } + removeTrack() {} + close() {} + } + return { + "test queue doesn't get blocked by enqueued, then ignored device message": assert => { + // XXX we might want to refactor the queue code a bit so it's easier to test + // without having to provide so many mocks + const clock = new MockClock(); + // setup logging + const logger = NullLoggerInstance; + // const logger = new Logger({platform: {clock, random: Math.random}}); + // logger.addReporter(new ConsoleReporter()); + + // create member + const callDeviceMembership = { + ["device_id"]: "BVPIHSKXFC", + ["session_id"]: "s1d5863f41ec5a5", + ["expires_ts"]: 123, + feeds: [{purpose: "m.usermedia"}] + }; + const roomMember = RoomMember.fromUserId("!abc", "@bruno4:matrix.org", "join"); + const turnServer = new ObservableValue({}); + const options = { + confId: "conf", + ownUserId: "@foobaraccount2:matrix.org", + ownDeviceId: "CMLEZSARRT", + sessionId: "s1cece7088b9d35", + clock, + emitUpdate: () => {}, + webRTC: { + prepareSenderForPurpose: () => {}, + createPeerConnection() { + return new MockPeerConn(); + } + } + } as Options; + const member = new Member(roomMember, callDeviceMembership, options, logger.child("member")); + member.connect(new MockMedia() as LocalMedia, new MuteSettings(), turnServer, logger.child("connect")); + // pretend we've already received 3 messages + member.connection.lastProcessedSeqNr = 2; + // send hangup with seq=3, this will enqueue the message because there is no peerCall + // as it's up to @bruno4:matrix.org to send the invite + const hangup = { + type: EventType.Hangup, + content: { + "call_id": "c0ac1b0e37afe73", + "version": 1, + "reason": "invite_timeout", + "seq": 3, + "conf_id": "conf-16a120796440a6", + "device_id": "BVPIHSKXFC", + "party_id": "BVPIHSKXFC", + "sender_session_id": "s1d5863f41ec5a5", + "dest_session_id": "s1cece7088b9d35" + } + }; + member.handleDeviceMessage(hangup, logger.child("handle hangup")); + // Send an invite with seq=4, this will create a new peer call with a the call id + // when dequeueing the hangup from before, it'll get ignored because it is + // for the previous call id. + const invite = { + type: EventType.Invite, + content: { + "call_id": "c1175b12d559fb1", + "offer": { + "type": "offer", + "sdp": "..." + }, + "org.matrix.msc3077.sdp_stream_metadata": { + "60087b60-487e-4fa0-8229-b232c18e1332": { + "purpose": "m.usermedia", + "audio_muted": false, + "video_muted": false + } + }, + "version": 1, + "lifetime": 60000, + "seq": 4, + "conf_id": "conf-16a120796440a6", + "device_id": "BVPIHSKXFC", + "party_id": "BVPIHSKXFC", + "sender_session_id": "s1d5863f41ec5a5", + "dest_session_id": "s1cece7088b9d35" + } + }; + member.handleDeviceMessage(invite, logger.child("handle invite")); + assert.equal(member.connection.queuedSignallingMessages.length, 0); + // logger.reporters[0].printOpenItems(); + } + }; +} From 02108c69dc782c2737e4ddde9725c731238a8011 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 15:00:22 +0100 Subject: [PATCH 358/435] remove debug logging --- src/matrix/calls/group/Member.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index d1463b51..e0c9c6ca 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -372,7 +372,6 @@ export class Member { } if (action === IncomingMessageAction.Handle) { const idx = sortedIndex(connection.queuedSignallingMessages, message, (a, b) => a.content.seq - b.content.seq); - console.log(`splice ${message.type} at ${idx}`); connection.queuedSignallingMessages.splice(idx, 0, message); if (connection.peerCall) { const hasNewMessageBeenDequeued = this.dequeueSignallingMessages(connection, connection.peerCall, message, syncLog); From ddb5865ccb0defe5c80238e4565c3e6f45bdd9d6 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 15:00:56 +0100 Subject: [PATCH 359/435] actually forgot to dispose peerCall here when replacing --- src/matrix/calls/group/Member.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index e0c9c6ca..d88a216f 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -362,6 +362,7 @@ export class Member { syncLog.refDetached(log); } if (shouldReplace) { + connection.peerCall.dispose(); connection.peerCall = undefined; action = IncomingMessageAction.Handle; } From 7f9d64c9725c607eaac81a8744c7f95a161a28b5 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 19:42:20 +0100 Subject: [PATCH 360/435] improve logging of arrival of to_device call signalling messages --- src/matrix/calls/group/Member.ts | 88 ++++++++++++++++---------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index d88a216f..daada4af 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -343,64 +343,65 @@ export class Member { /** @internal */ handleDeviceMessage(message: SignallingMessage, syncLog: ILogItem): void { this.errorBoundary.try(() => { - const {connection} = this; - if (connection) { - const destSessionId = message.content.dest_session_id; - if (destSessionId !== this.options.sessionId) { - const logItem = connection.logItem.log({l: "ignoring to_device event with wrong session_id", destSessionId, type: message.type}); - syncLog.refDetached(logItem); - return; - } - // if there is no peerCall, we either create it with an invite and Handle is implied or we'll ignore it - let action = IncomingMessageAction.Handle; - if (connection.peerCall) { - action = connection.peerCall.getMessageAction(message); - // deal with glare and replacing the call before creating new calls - if (action === IncomingMessageAction.InviteGlare) { - const {shouldReplace, log} = connection.peerCall.handleInviteGlare(message, this.deviceId, connection.logItem); - if (log) { - syncLog.refDetached(log); - } - if (shouldReplace) { - connection.peerCall.dispose(); - connection.peerCall = undefined; - action = IncomingMessageAction.Handle; + syncLog.wrap({l: "Member.handleDeviceMessage", type: message.type, seq: message.content?.seq}, log => { + const {connection} = this; + if (connection) { + const destSessionId = message.content.dest_session_id; + if (destSessionId !== this.options.sessionId) { + const logItem = connection.logItem.log({l: "ignoring to_device event with wrong session_id", destSessionId, type: message.type}); + log.refDetached(logItem); + return; + } + // if there is no peerCall, we either create it with an invite and Handle is implied or we'll ignore it + if (connection.peerCall) { + const action = connection.peerCall.getMessageAction(message); + // deal with glare and replacing the call before creating new calls + if (action === IncomingMessageAction.InviteGlare) { + const {shouldReplace, log} = connection.peerCall.handleInviteGlare(message, this.deviceId, connection.logItem); + if (log) { + log.refDetached(log); + } + if (shouldReplace) { + connection.peerCall.dispose(); + connection.peerCall = undefined; + } } } - } - if (message.type === EventType.Invite && !connection.peerCall) { - connection.peerCall = this._createPeerCall(message.content.call_id); - } - if (action === IncomingMessageAction.Handle) { + // create call on invite + if (message.type === EventType.Invite && !connection.peerCall) { + connection.peerCall = this._createPeerCall(message.content.call_id); + } + // enqueue const idx = sortedIndex(connection.queuedSignallingMessages, message, (a, b) => a.content.seq - b.content.seq); connection.queuedSignallingMessages.splice(idx, 0, message); + // dequeue as much as we can + let hasNewMessageBeenDequeued = false; if (connection.peerCall) { - const hasNewMessageBeenDequeued = this.dequeueSignallingMessages(connection, connection.peerCall, message, syncLog); - if (!hasNewMessageBeenDequeued) { - syncLog.refDetached(connection.logItem.log({l: "queued signalling message", type: message.type, seq: message.content.seq})); - } + hasNewMessageBeenDequeued = this.dequeueSignallingMessages(connection, connection.peerCall, message, log); } - } else if (action === IncomingMessageAction.Ignore && connection.peerCall) { - const logItem = connection.logItem.log({l: "ignoring to_device event with wrong call_id", callId: message.content.call_id, type: message.type}); - syncLog.refDetached(logItem); + if (!hasNewMessageBeenDequeued) { + log.refDetached(connection.logItem.log({l: "queued message", type: message.type, seq: message.content.seq, idx})); + } + } else { + // TODO: the right thing to do here would be to at least enqueue the message rather than drop it, + // and if it's up to the other end to send the invite and the type is an invite to actually + // call connect and assume the m.call.member state update is on its way? + syncLog.log({l: "member not connected", userId: this.userId, deviceId: this.deviceId}); } - } else { - syncLog.log({l: "member not connected", userId: this.userId, deviceId: this.deviceId}); - } + }); }); } private dequeueSignallingMessages(connection: MemberConnection, peerCall: PeerCall, newMessage: SignallingMessage, syncLog: ILogItem): boolean { let hasNewMessageBeenDequeued = false; while (connection.canDequeueNextSignallingMessage) { - syncLog.wrap("dequeue message", log => { - const message = connection.queuedSignallingMessages.shift()!; - if (message === newMessage) { - log.set("isNewMsg", true); - hasNewMessageBeenDequeued = true; - } + const message = connection.queuedSignallingMessages.shift()!; + const isNewMsg = message === newMessage; + hasNewMessageBeenDequeued = hasNewMessageBeenDequeued || isNewMsg; + syncLog.wrap(isNewMsg ? "process message" : "dequeue message", log => { const seq = message.content?.seq; log.set("seq", seq); + log.set("type", message.type); // ignore items in the queue that should not be handled and prevent // the lastProcessedSeqNr being corrupted with the `seq` for other call ids // XXX: Not needed anymore when seq is scoped to call_id @@ -411,7 +412,6 @@ export class Member { log.refDetached(item); connection.lastProcessedSeqNr = seq; } else { - log.set("type", message.type); log.set("ignored", true); connection.lastIgnoredSeqNr = seq; } From 0fa9d193d970e5da96018bc56ca691f9bc14d30a Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 19:42:43 +0100 Subject: [PATCH 361/435] fix comment typo --- src/matrix/calls/PeerCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 36136b97..2b28fbef 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -303,7 +303,7 @@ export class PeerCall implements IDisposable { } handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId, log: ILogItem): ILogItem { - // return logItem item immediately so it can be references in sync manner + // return logItem item immediately so it can be referenced by the sync log let logItem; log.wrap({ l: "receive signalling message", From 7114428b234198164359c60e7147923181687d92 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 19:57:42 +0100 Subject: [PATCH 362/435] now that the dom handler uses an error boundary, don't need this anymore --- src/matrix/calls/PeerCall.ts | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index d81bf047..f7af6546 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -431,18 +431,14 @@ export class PeerCall implements IDisposable { log.refDetached(timeoutLog); // don't await this, as it would block other negotationneeded events from being processed // as they are processed serially - try { - await timeoutLog.run(async log => { - await this.delay(CALL_TIMEOUT_MS); - // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between - if (this._state === CallState.InviteSent) { - await this._hangup(CallErrorCode.InviteTimeout, log); - } - }); - } - catch (e) { - // prevent error from being unhandled, it will be logged already by run above - } + await timeoutLog.run(async log => { + try { await this.delay(CALL_TIMEOUT_MS); } + catch (err) { return; } // return when delay is cancelled by throwing an AbortError + // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between + if (this._state === CallState.InviteSent) { + await this._hangup(CallErrorCode.InviteTimeout, log); + } + }); } }; From 5c2889aa5bd3139327ad5d3b23e880f357150e03 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 19:59:13 +0100 Subject: [PATCH 363/435] show DOM errors on the member error boundary rather than one for call gives a bit more context --- src/matrix/calls/group/GroupCall.ts | 3 +-- src/matrix/calls/group/Member.ts | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index d728e7c2..b82b3276 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -134,8 +134,7 @@ export class GroupCall extends EventEmitter<{change: never}> { }, encryptDeviceMessage: (userId: string, deviceId: string, message: SignallingMessage, log) => { return this.options.encryptDeviceMessage(this.roomId, userId, deviceId, message, log); - }, - groupCallErrorBoundary: this.errorBoundary, + } }); } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 0c31bc32..27842672 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -42,7 +42,6 @@ export type Options = Omit, log: ILogItem) => Promise, emitUpdate: (participant: Member, params?: any) => void, - groupCallErrorBoundary: ErrorBoundary, clock: Clock } @@ -422,7 +421,7 @@ export class Member { private _createPeerCall(callId: string): PeerCall { const connection = this.connection!; return new PeerCall(callId, Object.assign({}, this.options, { - errorBoundary: this.options.groupCallErrorBoundary, + errorBoundary: this.errorBoundary, emitUpdate: this.emitUpdateFromPeerCall, sendSignallingMessage: this.sendSignallingMessage, turnServer: connection.turnServer From 3ff91639c682b1f0746cfad0ba4fc4f0bc01416c Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 20:07:16 +0100 Subject: [PATCH 364/435] log signalingState on negotiationneeded --- src/matrix/calls/PeerCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 2b28fbef..a4ea984d 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -165,7 +165,7 @@ export class PeerCall implements IDisposable { }); listen("negotiationneeded", () => { const promiseCreator = () => { - return logItem.wrap("onNegotiationNeeded", log => { + return logItem.wrap({l: "onNegotiationNeeded", signalingState: this.peerConnection.signalingState}, log => { return this.handleNegotiation(log); }); }; From 5ad3f0c6712cac4f53ab7f1305fb38cce450d0d1 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 20:11:35 +0100 Subject: [PATCH 365/435] look at signalingState when even is fired, not later when it may have changed --- src/matrix/calls/PeerCall.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index a4ea984d..dc85084a 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -164,8 +164,9 @@ export class PeerCall implements IDisposable { }); }); listen("negotiationneeded", () => { + const signalingState = this.peerConnection.signalingState; const promiseCreator = () => { - return logItem.wrap({l: "onNegotiationNeeded", signalingState: this.peerConnection.signalingState}, log => { + return logItem.wrap({l: "onNegotiationNeeded", signalingState}, log => { return this.handleNegotiation(log); }); }; From dd89aab516ab8454c0d84eb56e165384e2e07890 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 20:19:06 +0100 Subject: [PATCH 366/435] fix typescript error in unit test --- src/logging/NullLogger.ts | 8 ++++---- src/matrix/calls/group/Member.ts | 13 ++++++++----- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/logging/NullLogger.ts b/src/logging/NullLogger.ts index bf7b3f40..83e5fc00 100644 --- a/src/logging/NullLogger.ts +++ b/src/logging/NullLogger.ts @@ -21,7 +21,7 @@ function noop (): void {} export class NullLogger implements ILogger { public readonly item: ILogItem = new NullLogItem(this); - log(): ILogItem { + log(labelOrValues: LabelOrValues): ILogItem { return this.item; } @@ -37,7 +37,7 @@ export class NullLogger implements ILogger { forceFinish(): void {} - child(): ILogItem { + child(labelOrValues: LabelOrValues): ILogItem { return this.item; } @@ -83,11 +83,11 @@ export class NullLogItem implements ILogItem { } - log(): ILogItem { + log(labelOrValues: LabelOrValues): ILogItem { return this; } - set(): ILogItem { return this; } + set(labelOrValues: LabelOrValues): ILogItem { return this; } runDetached(_: LabelOrValues, callback: LogCallback): ILogItem { new Promise(r => r(callback(this))).then(noop, noop); diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index daada4af..0a0e697c 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -511,7 +511,8 @@ export function tests() { feeds: [{purpose: "m.usermedia"}] }; const roomMember = RoomMember.fromUserId("!abc", "@bruno4:matrix.org", "join"); - const turnServer = new ObservableValue({}); + const turnServer = new ObservableValue({}) as ObservableValue; + // @ts-ignore const options = { confId: "conf", ownUserId: "@foobaraccount2:matrix.org", @@ -529,7 +530,8 @@ export function tests() { const member = new Member(roomMember, callDeviceMembership, options, logger.child("member")); member.connect(new MockMedia() as LocalMedia, new MuteSettings(), turnServer, logger.child("connect")); // pretend we've already received 3 messages - member.connection.lastProcessedSeqNr = 2; + // @ts-ignore + member.connection!.lastProcessedSeqNr = 2; // send hangup with seq=3, this will enqueue the message because there is no peerCall // as it's up to @bruno4:matrix.org to send the invite const hangup = { @@ -545,7 +547,7 @@ export function tests() { "sender_session_id": "s1d5863f41ec5a5", "dest_session_id": "s1cece7088b9d35" } - }; + } as SignallingMessage; member.handleDeviceMessage(hangup, logger.child("handle hangup")); // Send an invite with seq=4, this will create a new peer call with a the call id // when dequeueing the hangup from before, it'll get ignored because it is @@ -574,9 +576,10 @@ export function tests() { "sender_session_id": "s1d5863f41ec5a5", "dest_session_id": "s1cece7088b9d35" } - }; + } as SignallingMessage; member.handleDeviceMessage(invite, logger.child("handle invite")); - assert.equal(member.connection.queuedSignallingMessages.length, 0); + // @ts-ignore + assert.equal(member.connection!.queuedSignallingMessages.length, 0); // logger.reporters[0].printOpenItems(); } }; From 93661690e134e340a7363a330e39828430c11425 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 22:18:13 +0100 Subject: [PATCH 367/435] fix bg color of calls --- src/platform/web/ui/css/themes/element/call.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/platform/web/ui/css/themes/element/call.css b/src/platform/web/ui/css/themes/element/call.css index 3efe3c56..9b525dc2 100644 --- a/src/platform/web/ui/css/themes/element/call.css +++ b/src/platform/web/ui/css/themes/element/call.css @@ -37,7 +37,7 @@ limitations under the License. .CallView_members { display: grid; gap: 12px; - background: var(--background-color-secondary--darker-60); + background: var(--background-color-secondary--darker-5); padding: 12px; margin: 0; min-height: 0; From 928419502e6ce58bb03c1955dea8d7c36246fd43 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 22:44:43 +0100 Subject: [PATCH 368/435] no point in ref'ing an item on its parent --- src/matrix/calls/group/GroupCall.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index b82b3276..a47a3ff2 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -581,7 +581,6 @@ export class GroupCall extends EventEmitter<{change: never}> { sessionId: message.content.sender_session_id, type: message.type }); - syncLog.refDetached(item); // we haven't received the m.call.member yet for this caller (or with this session id). // buffer the device messages or create the member/call as it should arrive in a moment let messages = this.bufferedDeviceMessages.get(key); From fe5794a4bcfaee0b8bdbaa348391464085cdb47e Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Feb 2023 23:26:02 +0100 Subject: [PATCH 369/435] don't clear options as error boundary may fire after dispose --- src/matrix/calls/group/Member.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index db5482ee..5d67bafe 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -457,7 +457,7 @@ export class Member { this.expireTimeout?.dispose(); this.expireTimeout = undefined; // ensure the emitUpdate callback can't be called anymore - this.options = undefined as any as Options; + this.options.emitUpdate = () => {}; } } From c4944599cfe7268e32d43f8322cd113b37a8b52c Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 9 Feb 2023 11:52:00 +0100 Subject: [PATCH 370/435] add feature set to keep track of enabled features already include the calls feature --- src/features.ts | 50 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 src/features.ts diff --git a/src/features.ts b/src/features.ts new file mode 100644 index 00000000..e503f0e0 --- /dev/null +++ b/src/features.ts @@ -0,0 +1,50 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import type {SettingsStorage} from "./platform/web/dom/SettingsStorage"; + +export enum FeatureFlag { + Calls = 1 << 0, +} + +export class FeatureSet { + constructor(public readonly flags: number = 0) {} + + withFeature(flag: FeatureFlag): FeatureSet { + return new FeatureSet(this.flags | flag); + } + + withoutFeature(flag: FeatureFlag): FeatureSet { + return new FeatureSet(this.flags ^ flag); + } + + isFeatureEnabled(flag: FeatureFlag): boolean { + return (this.flags & flag) !== 0; + } + + get calls(): boolean { + return this.isFeatureEnabled(FeatureFlag.Calls); + } + + static async load(settingsStorage: SettingsStorage): Promise { + const flags = await settingsStorage.getInt("enabled_features") || 0; + return new FeatureSet(flags); + } + + async store(settingsStorage: SettingsStorage): Promise { + await settingsStorage.setInt("enabled_features", this.flags); + } +} From f65b43f612f0b3377bd895ed200ff4201739fbdd Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 9 Feb 2023 11:52:39 +0100 Subject: [PATCH 371/435] load features at startup and pass them along in all view models --- src/domain/ViewModel.ts | 6 ++++++ src/platform/web/main.js | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/src/domain/ViewModel.ts b/src/domain/ViewModel.ts index 34a05855..409e61c9 100644 --- a/src/domain/ViewModel.ts +++ b/src/domain/ViewModel.ts @@ -30,6 +30,7 @@ import type {Navigation} from "./navigation/Navigation"; import type {SegmentType} from "./navigation/index"; import type {IURLRouter} from "./navigation/URLRouter"; import type { ITimeFormatter } from "../platform/types/types"; +import type { FeatureSet } from "../features"; export type Options = { platform: Platform; @@ -37,6 +38,7 @@ export type Options = { urlRouter: IURLRouter; navigation: Navigation; emitChange?: (params: any) => void; + features: FeatureSet } @@ -142,6 +144,10 @@ export class ViewModel = Op return this._options.urlRouter; } + get features(): FeatureSet { + return this._options.features; + } + get navigation(): Navigation { // typescript needs a little help here return this._options.navigation as unknown as Navigation; diff --git a/src/platform/web/main.js b/src/platform/web/main.js index 2b28187e..d9c6fe8a 100644 --- a/src/platform/web/main.js +++ b/src/platform/web/main.js @@ -18,6 +18,8 @@ limitations under the License. // import {RecordRequester, ReplayRequester} from "./matrix/net/request/replay"; import {RootViewModel} from "../../domain/RootViewModel.js"; import {createNavigation, createRouter} from "../../domain/navigation/index"; +import {FeatureSet} from "../../features"; + // Don't use a default export here, as we use multiple entries during legacy build, // which does not support default exports, // see https://github.com/rollup/plugins/tree/master/packages/multi-entry @@ -33,6 +35,7 @@ export async function main(platform) { // const request = recorder.request; // window.getBrawlFetchLog = () => recorder.log(); await platform.init(); + const features = await FeatureSet.load(platform.settingsStorage); const navigation = createNavigation(); platform.setNavigation(navigation); const urlRouter = createRouter({navigation, history: platform.history}); @@ -43,6 +46,7 @@ export async function main(platform) { // so we call it that in the view models urlRouter: urlRouter, navigation, + features }); await vm.load(); platform.createAndMountRootView(vm); From d5929d9ebe6dd561637df57e7c450d58133747e9 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 9 Feb 2023 11:53:39 +0100 Subject: [PATCH 372/435] make features available in Client and Session --- src/domain/login/LoginViewModel.ts | 2 +- src/matrix/Client.js | 5 ++++- src/matrix/Session.js | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/domain/login/LoginViewModel.ts b/src/domain/login/LoginViewModel.ts index 75d57880..f43361d0 100644 --- a/src/domain/login/LoginViewModel.ts +++ b/src/domain/login/LoginViewModel.ts @@ -55,7 +55,7 @@ export class LoginViewModel extends ViewModel { const {ready, defaultHomeserver, loginToken} = options; this._ready = ready; this._loginToken = loginToken; - this._client = new Client(this.platform); + this._client = new Client(this.platform, this.features); this._homeserver = defaultHomeserver; this._initViewModels(); } diff --git a/src/matrix/Client.js b/src/matrix/Client.js index 5b53958b..fabb489b 100644 --- a/src/matrix/Client.js +++ b/src/matrix/Client.js @@ -31,6 +31,7 @@ import {TokenLoginMethod} from "./login/TokenLoginMethod"; import {SSOLoginHelper} from "./login/SSOLoginHelper"; import {getDehydratedDevice} from "./e2ee/Dehydration.js"; import {Registration} from "./registration/Registration"; +import {FeatureSet} from "../features"; export const LoadStatus = createEnum( "NotLoading", @@ -53,7 +54,7 @@ export const LoginFailure = createEnum( ); export class Client { - constructor(platform) { + constructor(platform, features = new FeatureSet(0)) { this._platform = platform; this._sessionStartedByReconnector = false; this._status = new ObservableValue(LoadStatus.NotLoading); @@ -68,6 +69,7 @@ export class Client { this._olmPromise = platform.loadOlm(); this._workerPromise = platform.loadOlmWorker(); this._accountSetup = undefined; + this._features = features; } createNewSessionId() { @@ -278,6 +280,7 @@ export class Client { olmWorker, mediaRepository, platform: this._platform, + features: this._features }); await this._session.load(log); if (dehydratedDevice) { diff --git a/src/matrix/Session.js b/src/matrix/Session.js index f1b273ff..3bcb0ff7 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -54,7 +54,7 @@ const PUSHER_KEY = "pusher"; export class Session { // sessionInfo contains deviceId, userId and homeserver - constructor({storage, hsApi, sessionInfo, olm, olmWorker, platform, mediaRepository}) { + constructor({storage, hsApi, sessionInfo, olm, olmWorker, platform, mediaRepository, features}) { this._platform = platform; this._storage = storage; this._hsApi = hsApi; From f86663fe7be2c05305fa2923ad8562529b50c3fd Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 9 Feb 2023 11:56:22 +0100 Subject: [PATCH 373/435] pass features to tilesCreator (and all options really for comfort) --- src/domain/session/room/RoomViewModel.js | 2 +- src/domain/session/room/timeline/TilesCollection.js | 2 +- src/domain/session/room/timeline/tiles/BaseMessageTile.js | 2 +- src/domain/session/room/timeline/tiles/index.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index a741050d..65afd348 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -209,7 +209,7 @@ export class RoomViewModel extends ErrorReportViewModel { _createTile(entry) { if (this._tileOptions) { - const Tile = this._tileOptions.tileClassForEntry(entry); + const Tile = this._tileOptions.tileClassForEntry(entry, this._tileOptions); if (Tile) { return new Tile(entry, this._tileOptions); } diff --git a/src/domain/session/room/timeline/TilesCollection.js b/src/domain/session/room/timeline/TilesCollection.js index 458697ca..c5bddc2c 100644 --- a/src/domain/session/room/timeline/TilesCollection.js +++ b/src/domain/session/room/timeline/TilesCollection.js @@ -34,7 +34,7 @@ export class TilesCollection extends BaseObservableList { } _createTile(entry) { - const Tile = this._tileOptions.tileClassForEntry(entry); + const Tile = this._tileOptions.tileClassForEntry(entry, this._tileOptions); if (Tile) { return new Tile(entry, this._tileOptions); } diff --git a/src/domain/session/room/timeline/tiles/BaseMessageTile.js b/src/domain/session/room/timeline/tiles/BaseMessageTile.js index 62c9632e..da5304ed 100644 --- a/src/domain/session/room/timeline/tiles/BaseMessageTile.js +++ b/src/domain/session/room/timeline/tiles/BaseMessageTile.js @@ -126,7 +126,7 @@ export class BaseMessageTile extends SimpleTile { if (action?.shouldReplace || !this._replyTile) { this.disposeTracked(this._replyTile); const tileClassForEntry = this._options.tileClassForEntry; - const ReplyTile = tileClassForEntry(replyEntry); + const ReplyTile = tileClassForEntry(replyEntry, this._options); if (ReplyTile) { this._replyTile = new ReplyTile(replyEntry, this._options); } diff --git a/src/domain/session/room/timeline/tiles/index.ts b/src/domain/session/room/timeline/tiles/index.ts index b4805635..32bc9c76 100644 --- a/src/domain/session/room/timeline/tiles/index.ts +++ b/src/domain/session/room/timeline/tiles/index.ts @@ -47,7 +47,7 @@ export type Options = ViewModelOptions & { }; export type TileConstructor = new (entry: TimelineEntry, options: Options) => ITile; -export function tileClassForEntry(entry: TimelineEntry): TileConstructor | undefined { +export function tileClassForEntry(entry: TimelineEntry, options: Options): TileConstructor | undefined { if (entry.isGap) { return GapTile; } else if (entry.isPending && entry.pendingEvent.isMissingAttachments) { From 22a81822664ffd81be919ca4e3077587eed5f23a Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 9 Feb 2023 11:57:30 +0100 Subject: [PATCH 374/435] feature-gate calls everywhere in the app --- src/domain/session/SessionViewModel.js | 8 +- src/domain/session/room/RoomViewModel.js | 7 ++ .../session/room/timeline/tiles/index.ts | 2 +- .../session/toast/ToastCollectionViewModel.ts | 6 +- src/matrix/Session.js | 103 ++++++++++-------- src/platform/web/ui/session/room/RoomView.js | 6 +- 6 files changed, 76 insertions(+), 56 deletions(-) diff --git a/src/domain/session/SessionViewModel.js b/src/domain/session/SessionViewModel.js index 072e1b74..7d1dac3c 100644 --- a/src/domain/session/SessionViewModel.js +++ b/src/domain/session/SessionViewModel.js @@ -130,9 +130,11 @@ export class SessionViewModel extends ViewModel { start() { this._sessionStatusViewModel.start(); - this._client.session.callHandler.loadCalls("m.ring"); - // TODO: only do this when opening the room - this._client.session.callHandler.loadCalls("m.prompt"); + if (this.features.calls) { + this._client.session.callHandler.loadCalls("m.ring"); + // TODO: only do this when opening the room + this._client.session.callHandler.loadCalls("m.prompt"); + } } get activeMiddleViewModel() { diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 65afd348..5cd610f2 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -50,6 +50,9 @@ export class RoomViewModel extends ErrorReportViewModel { } _setupCallViewModel() { + if (!this.features.calls) { + return; + } // pick call for this room with lowest key const calls = this.getOption("session").callHandler.calls; this._callObservable = new PickMapObservableValue(calls.filterValues(c => { @@ -421,6 +424,10 @@ export class RoomViewModel extends ErrorReportViewModel { startCall() { return this.logAndCatch("RoomViewModel.startCall", async log => { + if (!this.features.calls) { + log.set("feature_disbled", true); + return; + } log.set("roomId", this._room.id); let localMedia; try { diff --git a/src/domain/session/room/timeline/tiles/index.ts b/src/domain/session/room/timeline/tiles/index.ts index 32bc9c76..e86d61cb 100644 --- a/src/domain/session/room/timeline/tiles/index.ts +++ b/src/domain/session/room/timeline/tiles/index.ts @@ -92,7 +92,7 @@ export function tileClassForEntry(entry: TimelineEntry, options: Options): TileC case "org.matrix.msc3401.call": { // if prevContent is present, it's an update to a call event, which we don't render // as the original event is updated through the call object which receive state event updates - if (entry.stateKey && !entry.prevContent) { + if (options.features.calls && entry.stateKey && !entry.prevContent) { return CallTile; } return undefined; diff --git a/src/domain/session/toast/ToastCollectionViewModel.ts b/src/domain/session/toast/ToastCollectionViewModel.ts index d1595b2d..df4da88f 100644 --- a/src/domain/session/toast/ToastCollectionViewModel.ts +++ b/src/domain/session/toast/ToastCollectionViewModel.ts @@ -33,8 +33,10 @@ export class ToastCollectionViewModel extends ViewModel { constructor(options: Options) { super(options); const session = this.getOption("session"); - const callsObservableMap = session.callHandler.calls; - this.track(callsObservableMap.subscribe(this)); + if (this.features.calls) { + const callsObservableMap = session.callHandler.calls; + this.track(callsObservableMap.subscribe(this)); + } } async onAdd(_, call: GroupCall) { diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 3bcb0ff7..b3bd6f98 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -75,6 +75,61 @@ export class Session { }; this._roomsBeingCreated = new ObservableMap(); this._user = new User(sessionInfo.userId); + this._roomStateHandler = new RoomStateHandlerSet(); + this._deviceMessageHandler = new DeviceMessageHandler({storage, callHandler: this._callHandler}); + this._olm = olm; + this._olmUtil = null; + this._e2eeAccount = null; + this._deviceTracker = null; + this._olmEncryption = null; + this._keyLoader = null; + this._megolmEncryption = null; + this._megolmDecryption = null; + this._getSyncToken = () => this.syncToken; + this._olmWorker = olmWorker; + this._keyBackup = new ObservableValue(undefined); + this._observedRoomStatus = new Map(); + + if (olm) { + this._olmUtil = new olm.Utility(); + this._deviceTracker = new DeviceTracker({ + storage, + getSyncToken: this._getSyncToken, + olmUtil: this._olmUtil, + ownUserId: sessionInfo.userId, + ownDeviceId: sessionInfo.deviceId, + }); + } + this._createRoomEncryption = this._createRoomEncryption.bind(this); + this._forgetArchivedRoom = this._forgetArchivedRoom.bind(this); + this.needsKeyBackup = new ObservableValue(false); + + if (features.calls) { + this._setupCallHandler(); + } + } + + get fingerprintKey() { + return this._e2eeAccount?.identityKeys.ed25519; + } + + get hasSecretStorageKey() { + return this._hasSecretStorageKey; + } + + get deviceId() { + return this._sessionInfo.deviceId; + } + + get userId() { + return this._sessionInfo.userId; + } + + get callHandler() { + return this._callHandler; + } + + _setupCallHandler() { this._callHandler = new CallHandler({ clock: this._platform.clock, random: this._platform.random, @@ -103,55 +158,7 @@ export class Session { logger: this._platform.logger, forceTURN: false, }); - this._roomStateHandler = new RoomStateHandlerSet(); this.observeRoomState(this._callHandler); - this._deviceMessageHandler = new DeviceMessageHandler({storage, callHandler: this._callHandler}); - this._olm = olm; - this._olmUtil = null; - this._e2eeAccount = null; - this._deviceTracker = null; - this._olmEncryption = null; - this._keyLoader = null; - this._megolmEncryption = null; - this._megolmDecryption = null; - this._getSyncToken = () => this.syncToken; - this._olmWorker = olmWorker; - this._keyBackup = new ObservableValue(undefined); - this._observedRoomStatus = new Map(); - - if (olm) { - this._olmUtil = new olm.Utility(); - this._deviceTracker = new DeviceTracker({ - storage, - getSyncToken: this._getSyncToken, - olmUtil: this._olmUtil, - ownUserId: sessionInfo.userId, - ownDeviceId: sessionInfo.deviceId, - }); - } - this._createRoomEncryption = this._createRoomEncryption.bind(this); - this._forgetArchivedRoom = this._forgetArchivedRoom.bind(this); - this.needsKeyBackup = new ObservableValue(false); - } - - get fingerprintKey() { - return this._e2eeAccount?.identityKeys.ed25519; - } - - get hasSecretStorageKey() { - return this._hasSecretStorageKey; - } - - get deviceId() { - return this._sessionInfo.deviceId; - } - - get userId() { - return this._sessionInfo.userId; - } - - get callHandler() { - return this._callHandler; } // called once this._e2eeAccount is assigned diff --git a/src/platform/web/ui/session/room/RoomView.js b/src/platform/web/ui/session/room/RoomView.js index 892cb25e..727fb44d 100644 --- a/src/platform/web/ui/session/room/RoomView.js +++ b/src/platform/web/ui/session/room/RoomView.js @@ -73,8 +73,10 @@ export class RoomView extends TemplateView { } else { const vm = this.value; const options = []; - options.push(Menu.option(vm.i18n`Room details`, () => vm.openDetailsPanel())) - options.push(Menu.option(vm.i18n`Start call`, () => vm.startCall())) + options.push(Menu.option(vm.i18n`Room details`, () => vm.openDetailsPanel())); + if (vm.features.calls) { + options.push(Menu.option(vm.i18n`Start call`, () => vm.startCall())); + } if (vm.canLeave) { options.push(Menu.option(vm.i18n`Leave room`, () => this._confirmToLeaveRoom()).setDestructive()); } From 4a46c98d124239fcab319f470eacdc18fb25071b Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 9 Feb 2023 11:57:45 +0100 Subject: [PATCH 375/435] don't assume the call handler is always set in device message handler --- src/matrix/DeviceMessageHandler.js | 46 ++++++++++++++++-------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index 05276549..f6e7cad7 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -74,29 +74,31 @@ export class DeviceMessageHandler { } async afterSyncCompleted(decryptionResults, deviceTracker, hsApi, log) { - // if we don't have a device, we need to fetch the device keys the message claims - // and check the keys, and we should only do network requests during - // sync processing in the afterSyncCompleted step. - const callMessages = decryptionResults.filter(dr => this._callHandler.handlesDeviceMessageEventType(dr.event?.type)); - if (callMessages.length) { - await log.wrap("process call signalling messages", async log => { - for (const dr of callMessages) { - // serialize device loading, so subsequent messages for the same device take advantage of the cache - const device = await deviceTracker.deviceForId(dr.event.sender, dr.event.content.device_id, hsApi, log); - dr.setDevice(device); - if (dr.isVerified) { - this._callHandler.handleDeviceMessage(dr.event, dr.userId, dr.deviceId, log); - } else { - log.log({ - l: "could not verify olm fingerprint key matches, ignoring", - ed25519Key: dr.device.ed25519Key, - claimedEd25519Key: dr.claimedEd25519Key, - deviceId: device.deviceId, - userId: device.userId, - }); + if (this._callHandler) { + // if we don't have a device, we need to fetch the device keys the message claims + // and check the keys, and we should only do network requests during + // sync processing in the afterSyncCompleted step. + const callMessages = decryptionResults.filter(dr => this._callHandler.handlesDeviceMessageEventType(dr.event?.type)); + if (callMessages.length) { + await log.wrap("process call signalling messages", async log => { + for (const dr of callMessages) { + // serialize device loading, so subsequent messages for the same device take advantage of the cache + const device = await deviceTracker.deviceForId(dr.event.sender, dr.event.content.device_id, hsApi, log); + dr.setDevice(device); + if (dr.isVerified) { + this._callHandler.handleDeviceMessage(dr.event, dr.userId, dr.deviceId, log); + } else { + log.log({ + l: "could not verify olm fingerprint key matches, ignoring", + ed25519Key: dr.device.ed25519Key, + claimedEd25519Key: dr.claimedEd25519Key, + deviceId: device.deviceId, + userId: device.userId, + }); + } } - } - }); + }); + } } } } From f9fa59609f04b334c26ce7d023427575ef07e0ec Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 10 Feb 2023 08:48:10 +0100 Subject: [PATCH 376/435] fix local variable usage after extracting method --- src/matrix/Session.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index b3bd6f98..bd4de880 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -153,8 +153,8 @@ export class Session { }, storage: this._storage, webRTC: this._platform.webRTC, - ownDeviceId: sessionInfo.deviceId, - ownUserId: sessionInfo.userId, + ownDeviceId: this._sessionInfo.deviceId, + ownUserId: this._sessionInfo.userId, logger: this._platform.logger, forceTURN: false, }); From bb477b6aad0005471edaced6e65d9fa4d883263f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 10 Feb 2023 08:48:31 +0100 Subject: [PATCH 377/435] fix not passing features to client construction --- src/domain/RootViewModel.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/RootViewModel.js b/src/domain/RootViewModel.js index 524dfe13..2896fba6 100644 --- a/src/domain/RootViewModel.js +++ b/src/domain/RootViewModel.js @@ -158,7 +158,7 @@ export class RootViewModel extends ViewModel { } _showSessionLoader(sessionId) { - const client = new Client(this.platform); + const client = new Client(this.platform, this.features); client.startWithExistingSession(sessionId); this._setSection(() => { this._sessionLoadViewModel = new SessionLoadViewModel(this.childOptions({ From da1b7d410895e9279c8b4017375a74e2dfe258f2 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 10 Feb 2023 09:27:18 +0100 Subject: [PATCH 378/435] UI in settings for toggling features --- .../session/settings/FeaturesViewModel.ts | 70 +++++++++++++++++++ .../session/settings/SettingsViewModel.js | 6 ++ src/platform/web/Platform.js | 4 ++ .../web/ui/css/themes/element/theme.css | 18 +++++ .../web/ui/session/settings/FeaturesView.ts | 52 ++++++++++++++ .../web/ui/session/settings/SettingsView.js | 7 ++ 6 files changed, 157 insertions(+) create mode 100644 src/domain/session/settings/FeaturesViewModel.ts create mode 100644 src/platform/web/ui/session/settings/FeaturesView.ts diff --git a/src/domain/session/settings/FeaturesViewModel.ts b/src/domain/session/settings/FeaturesViewModel.ts new file mode 100644 index 00000000..69018bd2 --- /dev/null +++ b/src/domain/session/settings/FeaturesViewModel.ts @@ -0,0 +1,70 @@ +/* +Copyright 2020 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {ViewModel} from "../../ViewModel"; +import type {Options as BaseOptions} from "../../ViewModel"; +import {FeatureFlag, FeatureSet} from "../../../features"; +import type {SegmentType} from "../../navigation/index"; + +export class FeaturesViewModel extends ViewModel { + public readonly featureViewModels: ReadonlyArray; + + constructor(options) { + super(options); + this.featureViewModels = [ + new FeatureViewModel(this.childOptions({ + name: this.i18n`Audio/video calls (experimental)`, + description: this.i18n`Allows starting and participating in A/V calls compatible with Element Call (MSC3401). Look for the start call option in the room menu ((...) in the right corner) to start a call.`, + feature: FeatureFlag.Calls + })), + ]; + } +} + +type FeatureOptions = BaseOptions & { + feature: FeatureFlag, + description: string, + name: string +}; + +export class FeatureViewModel extends ViewModel { + get enabled(): boolean { + return this.features.isFeatureEnabled(this.getOption("feature")); + } + + async enableFeature(enabled: boolean): Promise { + let newFeatures; + if (enabled) { + newFeatures = this.features.withFeature(this.getOption("feature")); + } else { + newFeatures = this.features.withoutFeature(this.getOption("feature")); + } + await newFeatures.store(this.platform.settingsStorage); + this.platform.restart(); + } + + get id(): string { + return `${this.getOption("feature")}`; + } + + get name(): string { + return this.getOption("name"); + } + + get description(): string { + return this.getOption("description"); + } +} diff --git a/src/domain/session/settings/SettingsViewModel.js b/src/domain/session/settings/SettingsViewModel.js index 952c910b..f8420a53 100644 --- a/src/domain/session/settings/SettingsViewModel.js +++ b/src/domain/session/settings/SettingsViewModel.js @@ -16,6 +16,7 @@ limitations under the License. import {ViewModel} from "../../ViewModel"; import {KeyBackupViewModel} from "./KeyBackupViewModel.js"; +import {FeaturesViewModel} from "./FeaturesViewModel"; import {submitLogsFromSessionToDefaultServer} from "../../../domain/rageshake"; class PushNotificationStatus { @@ -53,6 +54,7 @@ export class SettingsViewModel extends ViewModel { this.pushNotifications = new PushNotificationStatus(); this._activeTheme = undefined; this._logsFeedbackMessage = undefined; + this._featuresViewModel = new FeaturesViewModel(this.childOptions()); } get _session() { @@ -125,6 +127,10 @@ export class SettingsViewModel extends ViewModel { return this._keyBackupViewModel; } + get featuresViewModel() { + return this._featuresViewModel; + } + get storageQuota() { return this._formatBytes(this._estimate?.quota); } diff --git a/src/platform/web/Platform.js b/src/platform/web/Platform.js index bfb585a3..be8c9970 100644 --- a/src/platform/web/Platform.js +++ b/src/platform/web/Platform.js @@ -283,6 +283,10 @@ export class Platform { } } + restart() { + document.location.reload(); + } + openFile(mimeType = null) { const input = document.createElement("input"); input.setAttribute("type", "file"); diff --git a/src/platform/web/ui/css/themes/element/theme.css b/src/platform/web/ui/css/themes/element/theme.css index 7bea169e..fd8e69b0 100644 --- a/src/platform/web/ui/css/themes/element/theme.css +++ b/src/platform/web/ui/css/themes/element/theme.css @@ -751,6 +751,24 @@ a { margin-bottom: 0; } +.FeatureView { + display: flex; + gap: 8px; +} + +.FeaturesView ul { + list-style: none; + padding: 8px 16px; +} + +.FeaturesView input[type="checkbox"] { + align-self: start; +} + +.FeatureView h4 { + margin: 0; +} + .error { color: var(--error-color); font-weight: 600; diff --git a/src/platform/web/ui/session/settings/FeaturesView.ts b/src/platform/web/ui/session/settings/FeaturesView.ts new file mode 100644 index 00000000..625fc362 --- /dev/null +++ b/src/platform/web/ui/session/settings/FeaturesView.ts @@ -0,0 +1,52 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {TemplateView, TemplateBuilder} from "../../general/TemplateView"; +import {ViewNode} from "../../general/types"; +import {disableTargetCallback} from "../../general/utils"; +import type {FeaturesViewModel, FeatureViewModel} from "../../../../../domain/session/settings/FeaturesViewModel"; + +export class FeaturesView extends TemplateView { + render(t, vm: FeaturesViewModel): ViewNode { + return t.div({ + className: "FeaturesView", + }, [ + t.p("Enable experimental features here that are still in development. These are not yet ready for primetime, so expect bugs."), + // we don't use a binding/ListView because this is a static list + t.ul(vm.featureViewModels.map(vm => { + return t.li(t.view(new FeatureView(vm))); + })) + ]); + } +} + +class FeatureView extends TemplateView { + render(t, vm): ViewNode { + let id = `feature_${vm.id}`; + return t.div({className: "FeatureView"}, [ + t.input({ + type: "checkbox", + id, + checked: vm => vm.enabled, + onChange: evt => vm.enableFeature(evt.target.checked) + }), + t.div({class: "FeatureView_container"}, [ + t.h4(t.label({for: id}, vm.name)), + t.p(vm.description) + ]) + ]); + } +} diff --git a/src/platform/web/ui/session/settings/SettingsView.js b/src/platform/web/ui/session/settings/SettingsView.js index d7c48351..0d0d6941 100644 --- a/src/platform/web/ui/session/settings/SettingsView.js +++ b/src/platform/web/ui/session/settings/SettingsView.js @@ -17,6 +17,7 @@ limitations under the License. import {TemplateView} from "../../general/TemplateView"; import {disableTargetCallback} from "../../general/utils"; import {KeyBackupSettingsView} from "./KeyBackupSettingsView.js" +import {FeaturesView} from "./FeaturesView" export class SettingsView extends TemplateView { render(t, vm) { @@ -110,6 +111,12 @@ export class SettingsView extends TemplateView { logButtons.push(t.button({onClick: disableTargetCallback(() => vm.sendLogsToServer())}, `Submit logs to ${vm.logsServer}`)); } logButtons.push(t.button({onClick: () => vm.exportLogs()}, "Download logs")); + + settingNodes.push( + t.h3("Experimental features"), + t.view(new FeaturesView(vm.featuresViewModel)) + ); + settingNodes.push( t.h3("Application"), row(t, vm.i18n`Version`, version), From 83d1ea05a1c9739d049bcb91e0663e06980c3fde Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 10 Feb 2023 11:22:13 +0100 Subject: [PATCH 379/435] fix unit test --- src/matrix/Session.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index bd4de880..d4c68a8d 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -1015,6 +1015,7 @@ export class Session { } } +import {FeatureSet} from "../features"; export function tests() { function createStorageMock(session, pendingEvents = []) { return { @@ -1058,7 +1059,8 @@ export function tests() { clock: { createTimeout: () => undefined } - } + }, + features: new FeatureSet(0) }); await session.load(); let syncSet = false; From c946319891dde161a64b32812f7c33f7797d098c Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 10 Feb 2023 11:33:19 +0100 Subject: [PATCH 380/435] remove double experimal warning in UI --- src/domain/session/settings/FeaturesViewModel.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/session/settings/FeaturesViewModel.ts b/src/domain/session/settings/FeaturesViewModel.ts index 69018bd2..6017cc6a 100644 --- a/src/domain/session/settings/FeaturesViewModel.ts +++ b/src/domain/session/settings/FeaturesViewModel.ts @@ -26,7 +26,7 @@ export class FeaturesViewModel extends ViewModel { super(options); this.featureViewModels = [ new FeatureViewModel(this.childOptions({ - name: this.i18n`Audio/video calls (experimental)`, + name: this.i18n`Audio/video calls`, description: this.i18n`Allows starting and participating in A/V calls compatible with Element Call (MSC3401). Look for the start call option in the room menu ((...) in the right corner) to start a call.`, feature: FeatureFlag.Calls })), From 7777ef83dd194617e51c975484d09875354d03ac Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 10 Feb 2023 12:34:47 +0100 Subject: [PATCH 381/435] adopt errorViewModel in GapTile to report errors --- .../session/room/timeline/tiles/GapTile.js | 83 +++++++------------ .../web/ui/css/themes/element/error.css | 11 ++- .../web/ui/css/themes/element/timeline.css | 2 +- src/platform/web/ui/css/timeline.css | 4 +- .../web/ui/session/room/timeline/GapView.js | 12 ++- 5 files changed, 49 insertions(+), 63 deletions(-) diff --git a/src/domain/session/room/timeline/tiles/GapTile.js b/src/domain/session/room/timeline/tiles/GapTile.js index 1e6bdd08..58f0b3a6 100644 --- a/src/domain/session/room/timeline/tiles/GapTile.js +++ b/src/domain/session/room/timeline/tiles/GapTile.js @@ -24,47 +24,40 @@ export class GapTile extends SimpleTile { constructor(entry, options) { super(entry, options); this._loading = false; - this._error = null; + this._waitingForConnection = false; this._isAtTop = true; this._siblingChanged = false; - this._showSpinner = false; } get needsDateSeparator() { return false; } - async fill() { + async fill(isRetrying = false) { if (!this._loading && !this._entry.edgeReached) { this._loading = true; - this._error = null; - this._showSpinner = true; this.emitChange("isLoading"); try { await this._room.fillGap(this._entry, 10); } catch (err) { - console.error(`room.fillGap(): ${err.message}:\n${err.stack}`); - this._error = err; if (err instanceof ConnectionError) { - this.emitChange("error"); - /* - We need to wait for reconnection here rather than in - notifyVisible() because when we return/throw here - this._loading is set to false and other queued invocations of - this method will succeed and attempt further room.fillGap() calls - - resulting in multiple error entries in logs and elsewhere! - */ await this._waitForReconnection(); + if (!isRetrying) { + // retry after the connection comes back + // if this wasn't already a retry after coming back online + return await this.fill(true); + } else { + return false; + } + } else { + this.reportError(err); + return false; } - // rethrow so caller of this method - // knows not to keep calling this for now - throw err; } finally { this._loading = false; - this._showSpinner = false; this.emitChange("isLoading"); } - return true; + return true; } return false; } @@ -76,19 +69,7 @@ export class GapTile extends SimpleTile { let canFillMore; this._siblingChanged = false; do { - try { - canFillMore = await this.fill(); - } - catch (e) { - if (e instanceof ConnectionError) { - canFillMore = true; - // Don't increase depth because this gap fill was a noop - continue; - } - else { - canFillMore = false; - } - } + canFillMore = await this.fill(); depth = depth + 1; } while (depth < 10 && !this._siblingChanged && canFillMore && !this.isDisposed); } @@ -124,7 +105,11 @@ export class GapTile extends SimpleTile { } async _waitForReconnection() { + this._waitingForConnection = true; + this.emitUpdate("status"); await this.options.client.reconnector.connectionStatus.waitFor(status => status === ConnectionStatus.Online).promise; + this._waitingForConnection = false; + this.emitUpdate("status"); } get shape() { @@ -136,29 +121,19 @@ export class GapTile extends SimpleTile { } get showSpinner() { - return this._showSpinner; + return this.isLoading || this._waitingForConnection; } - get error() { - if (this._error) { - if (this._error instanceof ConnectionError) { - return "Waiting for reconnection"; - } - const dir = this._entry.prev_batch ? "previous" : "next"; - return `Could not load ${dir} messages: ${this._error.message}`; - } - return null; - } - - get currentAction() { - if (this.error) { - return this.error; - } - else if (this.isLoading) { - return "Loading"; - } - else { - return "Not Loading"; + get status() { + const dir = this._entry.prev_batch ? "previous" : "next"; + if (this._waitingForConnection) { + return "Waiting for connection…"; + } else if (this.errorViewModel) { + return `Could not load ${dir} messages`; + } else if (this.isLoading) { + return "Loading more messages…"; + } else { + return "Gave up loading more messages"; } } } diff --git a/src/platform/web/ui/css/themes/element/error.css b/src/platform/web/ui/css/themes/element/error.css index ce252056..e0b945ca 100644 --- a/src/platform/web/ui/css/themes/element/error.css +++ b/src/platform/web/ui/css/themes/element/error.css @@ -4,9 +4,14 @@ margin: 16px; } -.ErrorView_inline { +.ErrorView.ErrorView_inline { color: var(--error-color); - margin: 4px; + margin: 4px 0; + padding: 4px 0; +} + +.ErrorView.ErrorView_inline > p { + margin: 0; } .ErrorView { @@ -48,4 +53,4 @@ .ErrorView_inline .ErrorView_close { background-image: url('icons/clear.svg?primary=text-color'); -} \ No newline at end of file +} diff --git a/src/platform/web/ui/css/themes/element/timeline.css b/src/platform/web/ui/css/themes/element/timeline.css index 8606400d..4a822605 100644 --- a/src/platform/web/ui/css/themes/element/timeline.css +++ b/src/platform/web/ui/css/themes/element/timeline.css @@ -411,7 +411,7 @@ only loads when the top comes into view*/ border-radius: 10px; } -.GapView > :not(:first-child) { +.GapView_container > :not(:first-child) { margin-left: 12px; } diff --git a/src/platform/web/ui/css/timeline.css b/src/platform/web/ui/css/timeline.css index b6d62c98..0932739c 100644 --- a/src/platform/web/ui/css/timeline.css +++ b/src/platform/web/ui/css/timeline.css @@ -52,11 +52,11 @@ limitations under the License. align-items: center; } -.GapView { +.GapView_container { display: flex; } -.GapView > :nth-child(2) { +.GapView_container > span { flex: 1; } diff --git a/src/platform/web/ui/session/room/timeline/GapView.js b/src/platform/web/ui/session/room/timeline/GapView.js index 4fc0e3d6..91181c09 100644 --- a/src/platform/web/ui/session/room/timeline/GapView.js +++ b/src/platform/web/ui/session/room/timeline/GapView.js @@ -16,6 +16,7 @@ limitations under the License. import {TemplateView} from "../../../general/TemplateView"; import {spinner} from "../../../common.js"; +import {ErrorView} from "../../../general/ErrorView"; export class GapView extends TemplateView { // ignore other argument @@ -23,15 +24,20 @@ export class GapView extends TemplateView { super(vm); } - render(t) { + render(t, vm) { const className = { GapView: true, isLoading: vm => vm.isLoading, isAtTop: vm => vm.isAtTop, }; return t.li({ className }, [ - t.if(vm => vm.showSpinner, (t) => spinner(t)), - t.span(vm => vm.currentAction) + t.div({class: "GapView_container"}, [ + t.if(vm => vm.showSpinner, (t) => spinner(t)), + t.span(vm => vm.status), + ]), + t.if(vm => !!vm.errorViewModel, t => { + return t.view(new ErrorView(vm.errorViewModel, {inline: true})); + }) ]); } From 9e28bdcc88f11ab3c50559a6ac54e5ad444db83b Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 10 Feb 2023 12:35:02 +0100 Subject: [PATCH 382/435] don't try to fill when we had an error before --- src/domain/session/room/timeline/tiles/GapTile.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/domain/session/room/timeline/tiles/GapTile.js b/src/domain/session/room/timeline/tiles/GapTile.js index 58f0b3a6..66e83147 100644 --- a/src/domain/session/room/timeline/tiles/GapTile.js +++ b/src/domain/session/room/timeline/tiles/GapTile.js @@ -63,6 +63,13 @@ export class GapTile extends SimpleTile { } async notifyVisible() { + // if any error happened before (apart from being offline), + // let the user dismiss the error before trying to backfill + // again so we don't try to do backfill the don't succeed + // in quick succession + if (this.errorViewModel) { + return; + } // we do (up to 10) backfills while no new tiles have been added to the timeline // because notifyVisible won't be called again until something gets added to the timeline let depth = 0; From 7c1117ddd441db6970bab1f19be7f317306aaa1e Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 10 Feb 2023 14:08:35 +0100 Subject: [PATCH 383/435] keep token in memory to compare stored token with after /messages and don't look at response.start as it can be different as the format can change after a server upgrade while (still pointing at the same location) --- src/matrix/room/BaseRoom.js | 2 +- src/matrix/room/timeline/persistence/GapWriter.js | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/matrix/room/BaseRoom.js b/src/matrix/room/BaseRoom.js index 7eb73faf..4ea25389 100644 --- a/src/matrix/room/BaseRoom.js +++ b/src/matrix/room/BaseRoom.js @@ -350,7 +350,7 @@ export class BaseRoom extends EventEmitter { fragmentIdComparer: this._fragmentIdComparer, relationWriter }); - gapResult = await gapWriter.writeFragmentFill(fragmentEntry, response, txn, log); + gapResult = await gapWriter.writeFragmentFill(fragmentEntry, response, fragmentEntry.token, txn, log); } catch (err) { txn.abort(); throw err; diff --git a/src/matrix/room/timeline/persistence/GapWriter.js b/src/matrix/room/timeline/persistence/GapWriter.js index 4458e1c5..d9ad5476 100644 --- a/src/matrix/room/timeline/persistence/GapWriter.js +++ b/src/matrix/room/timeline/persistence/GapWriter.js @@ -154,10 +154,13 @@ export class GapWriter { return changedFragments; } - async writeFragmentFill(fragmentEntry, response, txn, log) { + /** + * @param {string} fromToken the token used to call /messages, to ensure it hasn't changed in storage + */ + async writeFragmentFill(fragmentEntry, response, fromToken, txn, log) { const {fragmentId, direction} = fragmentEntry; // chunk is in reverse-chronological order when backwards - const {chunk, start, state} = response; + const {chunk, state} = response; let {end} = response; if (!Array.isArray(chunk)) { @@ -174,8 +177,8 @@ export class GapWriter { } fragmentEntry = fragmentEntry.withUpdatedFragment(fragment); // check that the request was done with the token we are aware of (extra care to avoid timeline corruption) - if (fragmentEntry.token !== start) { - throw new Error("start is not equal to prev_batch or next_batch"); + if (fragmentEntry.token !== fromToken) { + throw new Error("The pagination token has changed locally while fetching messages."); } // begin (or end) of timeline reached @@ -263,7 +266,7 @@ export function tests() { async function backfillAndWrite(mocks, fragmentEntry, limit) { const {txn, timelineMock, gapWriter} = mocks; const messageResponse = timelineMock.messages(fragmentEntry.token, undefined, fragmentEntry.direction.asApiString(), limit); - await gapWriter.writeFragmentFill(fragmentEntry, messageResponse, txn, logger); + await gapWriter.writeFragmentFill(fragmentEntry, messageResponse, fragmentEntry.token, txn, logger); } async function allFragmentEvents(mocks, fragmentId) { From c3da2acfb2c634d13d312c9ae287b1500ff40ceb Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 10 Feb 2023 14:11:45 +0100 Subject: [PATCH 384/435] adjust margin on features UI --- src/platform/web/ui/css/themes/element/theme.css | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/platform/web/ui/css/themes/element/theme.css b/src/platform/web/ui/css/themes/element/theme.css index fd8e69b0..4c617386 100644 --- a/src/platform/web/ui/css/themes/element/theme.css +++ b/src/platform/web/ui/css/themes/element/theme.css @@ -769,6 +769,11 @@ a { margin: 0; } + +.FeatureView p { + margin: 8px 0; +} + .error { color: var(--error-color); font-weight: 600; From 551a9e0bc89bf808944dc005173d3f1b6d1d173f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 10 Feb 2023 14:15:00 +0100 Subject: [PATCH 385/435] release v0.3.7 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c80b757d..697bfa43 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hydrogen-web", - "version": "0.3.6", + "version": "0.3.7", "description": "A javascript matrix client prototype, trying to minize RAM usage by offloading as much as possible to IndexedDB", "directories": { "doc": "doc" From 3bb0d26d3fe230a329fe45f76d6dda4b91c9b262 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 10 Feb 2023 16:17:47 +0100 Subject: [PATCH 386/435] fix missing msgtype on emote --- src/domain/session/room/RoomViewModel.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 5cd610f2..31608a62 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -292,7 +292,7 @@ export class RoomViewModel extends ErrorReportViewModel { this.reportError(new Error(`no command name "${commandName}". To send the message instead of executing, please type "/${message}"`)); message = undefined; } - return {type: msgtype, message: message}; + return {msgtype, message: message}; } _pickAndSendFile() { From ab67a28c74db9fea79b8d03af39df95ec6fdb2d3 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 10 Feb 2023 17:35:45 +0100 Subject: [PATCH 387/435] add feature flag for cross-signing --- src/domain/session/settings/FeaturesViewModel.ts | 7 ++++++- src/features.ts | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/domain/session/settings/FeaturesViewModel.ts b/src/domain/session/settings/FeaturesViewModel.ts index 6017cc6a..552e27d4 100644 --- a/src/domain/session/settings/FeaturesViewModel.ts +++ b/src/domain/session/settings/FeaturesViewModel.ts @@ -29,7 +29,12 @@ export class FeaturesViewModel extends ViewModel { name: this.i18n`Audio/video calls`, description: this.i18n`Allows starting and participating in A/V calls compatible with Element Call (MSC3401). Look for the start call option in the room menu ((...) in the right corner) to start a call.`, feature: FeatureFlag.Calls - })), + })), + new FeatureViewModel(this.childOptions({ + name: this.i18n`Cross-Signing`, + description: this.i18n`Allows.verifying the identity of people you chat with`, + feature: FeatureFlag.CrossSigning + })), ]; } } diff --git a/src/features.ts b/src/features.ts index e503f0e0..6fa5dd43 100644 --- a/src/features.ts +++ b/src/features.ts @@ -18,6 +18,7 @@ import type {SettingsStorage} from "./platform/web/dom/SettingsStorage"; export enum FeatureFlag { Calls = 1 << 0, + CrossSigning = 1 << 1 } export class FeatureSet { @@ -39,6 +40,10 @@ export class FeatureSet { return this.isFeatureEnabled(FeatureFlag.Calls); } + get crossSigning(): boolean { + return this.isFeatureEnabled(FeatureFlag.CrossSigning); + } + static async load(settingsStorage: SettingsStorage): Promise { const flags = await settingsStorage.getInt("enabled_features") || 0; return new FeatureSet(flags); From 2043541f568a6c45efab8ca7429c4ca99a5c225e Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 10 Feb 2023 17:36:14 +0100 Subject: [PATCH 388/435] fix missing free in key backup --- src/matrix/e2ee/megolm/keybackup/Curve25519.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/matrix/e2ee/megolm/keybackup/Curve25519.ts b/src/matrix/e2ee/megolm/keybackup/Curve25519.ts index 7d2ebac7..45cacb3f 100644 --- a/src/matrix/e2ee/megolm/keybackup/Curve25519.ts +++ b/src/matrix/e2ee/megolm/keybackup/Curve25519.ts @@ -57,6 +57,7 @@ export class BackupEncryption { encryption.set_recipient_key(pubKey); } catch(err) { decryption.free(); + encryption.free(); throw err; } return new BackupEncryption(encryption, decryption); From ce5b27f4b8c0d1a01fb0d145bbeb8f178fe6b25d Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 14 Feb 2023 12:11:59 +0100 Subject: [PATCH 389/435] support fetching the master signing key for a user in the device tracker --- src/matrix/e2ee/DeviceTracker.js | 53 ++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/src/matrix/e2ee/DeviceTracker.js b/src/matrix/e2ee/DeviceTracker.js index b256065a..c36d2ec6 100644 --- a/src/matrix/e2ee/DeviceTracker.js +++ b/src/matrix/e2ee/DeviceTracker.js @@ -25,6 +25,7 @@ function createUserIdentity(userId, initialRoomId = undefined) { return { userId: userId, roomIds: initialRoomId ? [initialRoomId] : [], + masterKey: undefined, deviceTrackingStatus: TRACKING_STATUS_OUTDATED, }; } @@ -152,6 +153,26 @@ export class DeviceTracker { } } + async getMasterKeyForUser(userId, hsApi, log) { + return await log.wrap("DeviceTracker.getMasterKeyForUser", async log => { + let txn = await this._storage.readTxn([ + this._storage.storeNames.userIdentities + ]); + let userIdentity = await txn.userIdentities.get(userId); + if (userIdentity && userIdentity.deviceTrackingStatus !== TRACKING_STATUS_OUTDATED) { + return userIdentity.masterKey; + } + // fetch from hs + await this._queryKeys([userId], hsApi, log); + // Retreive from storage now + txn = await this._storage.readTxn([ + this._storage.storeNames.userIdentities + ]); + userIdentity = await txn.userIdentities.get(userId); + return userIdentity?.masterKey; + }); + } + async writeHistoryVisibility(room, historyVisibility, syncTxn, log) { const added = []; const removed = []; @@ -224,6 +245,7 @@ export class DeviceTracker { "token": this._getSyncToken() }, {log}).response(); + const masterKeys = log.wrap("master keys", log => this._filterValidMasterKeys(deviceKeyResponse, log)); const verifiedKeysPerUser = log.wrap("verify", log => this._filterVerifiedDeviceKeys(deviceKeyResponse["device_keys"], log)); const txn = await this._storage.readWriteTxn([ this._storage.storeNames.userIdentities, @@ -233,7 +255,7 @@ export class DeviceTracker { try { const devicesIdentitiesPerUser = await Promise.all(verifiedKeysPerUser.map(async ({userId, verifiedKeys}) => { const deviceIdentities = verifiedKeys.map(deviceKeysAsDeviceIdentity); - return await this._storeQueriedDevicesForUserId(userId, deviceIdentities, txn); + return await this._storeQueriedDevicesForUserId(userId, masterKeys.get(userId), deviceIdentities, txn); })); deviceIdentities = devicesIdentitiesPerUser.reduce((all, devices) => all.concat(devices), []); log.set("devices", deviceIdentities.length); @@ -245,7 +267,7 @@ export class DeviceTracker { return deviceIdentities; } - async _storeQueriedDevicesForUserId(userId, deviceIdentities, txn) { + async _storeQueriedDevicesForUserId(userId, masterKey, deviceIdentities, txn) { const knownDeviceIds = await txn.deviceIdentities.getAllDeviceIds(userId); // delete any devices that we know off but are not in the response anymore. // important this happens before checking if the ed25519 key changed, @@ -286,11 +308,38 @@ export class DeviceTracker { identity = createUserIdentity(userId); } identity.deviceTrackingStatus = TRACKING_STATUS_UPTODATE; + identity.masterKey = masterKey; txn.userIdentities.set(identity); return allDeviceIdentities; } + _filterValidMasterKeys(keyQueryResponse, parentLog) { + const masterKeysResponse = keyQueryResponse["master_keys"]; + if (!masterKeysResponse) { + return []; + } + const validMasterKeyResponses = Object.entries(masterKeysResponse).filter(([userId, keyInfo]) => { + if (keyInfo["user_id"] !== userId) { + return false; + } + if (!Array.isArray(keyInfo.usage) || !keyInfo.usage.includes("master")) { + return false; + } + return true; + }); + const masterKeys = validMasterKeyResponses.reduce((msks, [userId, keyInfo]) => { + const keyIds = Object.keys(keyInfo.keys); + if (keyIds.length !== 1) { + return false; + } + const masterKey = keyInfo.keys[keyIds[0]]; + msks.set(userId, masterKey); + return msks; + }, new Map()); + return masterKeys; + } + /** * @return {Array<{userId, verifiedKeys: Array>} */ From fdce098245c24b409b4f617962d54cf7f7774322 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 14 Feb 2023 12:12:20 +0100 Subject: [PATCH 390/435] create cross-signing class, support deriving msk from 4s stored privkey and check if they match the publicized one and then trust it --- src/matrix/Session.js | 23 +++++++++ src/matrix/verification/CrossSigning.ts | 68 +++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 src/matrix/verification/CrossSigning.ts diff --git a/src/matrix/Session.js b/src/matrix/Session.js index d4c68a8d..222c8ef2 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -31,6 +31,7 @@ import {Encryption as OlmEncryption} from "./e2ee/olm/Encryption"; import {Decryption as MegOlmDecryption} from "./e2ee/megolm/Decryption"; import {KeyLoader as MegOlmKeyLoader} from "./e2ee/megolm/decryption/KeyLoader"; import {KeyBackup} from "./e2ee/megolm/keybackup/KeyBackup"; +import {CrossSigning} from "./verification/CrossSigning"; import {Encryption as MegOlmEncryption} from "./e2ee/megolm/Encryption.js"; import {MEGOLM_ALGORITHM} from "./e2ee/common.js"; import {RoomEncryption} from "./e2ee/RoomEncryption.js"; @@ -59,6 +60,7 @@ export class Session { this._storage = storage; this._hsApi = hsApi; this._mediaRepository = mediaRepository; + this._features = features; this._syncInfo = null; this._sessionInfo = sessionInfo; this._rooms = new ObservableMap(); @@ -88,6 +90,7 @@ export class Session { this._getSyncToken = () => this.syncToken; this._olmWorker = olmWorker; this._keyBackup = new ObservableValue(undefined); + this._crossSigning = undefined; this._observedRoomStatus = new Map(); if (olm) { @@ -330,6 +333,20 @@ export class Session { txn ); if (keyBackup) { + if (this._features.crossSigning) { + this._crossSigning = new CrossSigning({ + storage: this._storage, + secretStorage, + platform: this._platform, + olm: this._olm, + deviceTracker: this._deviceTracker, + hsApi: this._hsApi, + ownUserId: this.userId + }); + await log.wrap("enable cross-signing", log => { + return this._crossSigning.init(log); + }); + } for (const room of this._rooms.values()) { if (room.isEncrypted) { room.enableKeyBackup(keyBackup); @@ -337,6 +354,8 @@ export class Session { } this._keyBackup.set(keyBackup); return true; + } else { + log.set("no_backup", true); } } catch (err) { log.catch(err); @@ -354,6 +373,10 @@ export class Session { return this._keyBackup; } + get crossSigning() { + return this._crossSigning; + } + get hasIdentity() { return !!this._e2eeAccount; } diff --git a/src/matrix/verification/CrossSigning.ts b/src/matrix/verification/CrossSigning.ts new file mode 100644 index 00000000..8c3b6b61 --- /dev/null +++ b/src/matrix/verification/CrossSigning.ts @@ -0,0 +1,68 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import type {SecretStorage} from "../ssss/SecretStorage"; +import type {Storage} from "../storage/idb/Storage"; +import type {Platform} from "../../platform/web/Platform"; +import type {DeviceTracker} from "../e2ee/DeviceTracker"; +import type * as OlmNamespace from "@matrix-org/olm"; +import type {HomeServerApi} from "../net/HomeServerApi"; +type Olm = typeof OlmNamespace; + +export class CrossSigning { + private readonly storage: Storage; + private readonly secretStorage: SecretStorage; + private readonly platform: Platform; + private readonly deviceTracker: DeviceTracker; + private readonly olm: Olm; + private readonly hsApi: HomeServerApi; + private readonly ownUserId: string; + private _isMasterKeyTrusted: boolean = false; + + constructor(options: {storage: Storage, secretStorage: SecretStorage, deviceTracker: DeviceTracker, platform: Platform, olm: Olm, ownUserId: string, hsApi: HomeServerApi}) { + this.storage = options.storage; + this.secretStorage = options.secretStorage; + this.platform = options.platform; + this.deviceTracker = options.deviceTracker; + this.olm = options.olm; + this.hsApi = options.hsApi; + this.ownUserId = options.ownUserId; + } + + async init(log) { + // use errorboundary here + const txn = await this.storage.readTxn([this.storage.storeNames.accountData]); + + const mskSeed = await this.secretStorage.readSecret("m.cross_signing.master", txn); + const signing = new this.olm.PkSigning(); + let derivedPublicKey; + try { + const seed = new Uint8Array(this.platform.encoding.base64.decode(mskSeed)); + derivedPublicKey = signing.init_with_seed(seed); + } finally { + signing.free(); + } + const publishedMasterKey = await this.deviceTracker.getMasterKeyForUser(this.ownUserId, this.hsApi, log); + log.set({publishedMasterKey, derivedPublicKey}); + this._isMasterKeyTrusted = publishedMasterKey === derivedPublicKey; + log.set("isMasterKeyTrusted", this.isMasterKeyTrusted); + } + + get isMasterKeyTrusted(): boolean { + return this._isMasterKeyTrusted; + } +} + From 45d45cb690efc4baac95771620a46cf0fb6dc003 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 14 Feb 2023 12:13:07 +0100 Subject: [PATCH 391/435] show MSK trust status in settings after enabling key backup --- src/domain/session/settings/KeyBackupViewModel.js | 4 ++++ src/platform/web/ui/session/settings/KeyBackupSettingsView.js | 3 +++ src/platform/web/ui/session/settings/SettingsView.js | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/domain/session/settings/KeyBackupViewModel.js b/src/domain/session/settings/KeyBackupViewModel.js index 40dbfd73..56339a4e 100644 --- a/src/domain/session/settings/KeyBackupViewModel.js +++ b/src/domain/session/settings/KeyBackupViewModel.js @@ -88,6 +88,10 @@ export class KeyBackupViewModel extends ViewModel { return this._session.keyBackup.get()?.version; } + get isMasterKeyTrusted() { + return this._session.crossSigning?.isMasterKeyTrusted ?? false; + } + get backupWriteStatus() { const keyBackup = this._session.keyBackup.get(); if (!keyBackup) { diff --git a/src/platform/web/ui/session/settings/KeyBackupSettingsView.js b/src/platform/web/ui/session/settings/KeyBackupSettingsView.js index 3f8812c9..bd1eb4e8 100644 --- a/src/platform/web/ui/session/settings/KeyBackupSettingsView.js +++ b/src/platform/web/ui/session/settings/KeyBackupSettingsView.js @@ -53,6 +53,9 @@ export class KeyBackupSettingsView extends TemplateView { default: return null; } + }), + t.if(vm => vm.isMasterKeyTrusted, t => { + return t.p("Cross-signing master key found and trusted.") }) ]); } diff --git a/src/platform/web/ui/session/settings/SettingsView.js b/src/platform/web/ui/session/settings/SettingsView.js index 0d0d6941..aea1108a 100644 --- a/src/platform/web/ui/session/settings/SettingsView.js +++ b/src/platform/web/ui/session/settings/SettingsView.js @@ -49,7 +49,7 @@ export class SettingsView extends TemplateView { }, vm.i18n`Log out`)), ); settingNodes.push( - t.h3("Key backup"), + t.h3("Key backup & security"), t.view(new KeyBackupSettingsView(vm.keyBackupViewModel)) ); From 103ae1e789dea032161c09b4e90a1645999b3edc Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 14 Feb 2023 13:24:26 +0100 Subject: [PATCH 392/435] fix unit tests --- src/matrix/e2ee/DeviceTracker.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/matrix/e2ee/DeviceTracker.js b/src/matrix/e2ee/DeviceTracker.js index c36d2ec6..810dfd95 100644 --- a/src/matrix/e2ee/DeviceTracker.js +++ b/src/matrix/e2ee/DeviceTracker.js @@ -315,9 +315,10 @@ export class DeviceTracker { } _filterValidMasterKeys(keyQueryResponse, parentLog) { + const masterKeys = new Map(); const masterKeysResponse = keyQueryResponse["master_keys"]; if (!masterKeysResponse) { - return []; + return masterKeys; } const validMasterKeyResponses = Object.entries(masterKeysResponse).filter(([userId, keyInfo]) => { if (keyInfo["user_id"] !== userId) { @@ -328,7 +329,7 @@ export class DeviceTracker { } return true; }); - const masterKeys = validMasterKeyResponses.reduce((msks, [userId, keyInfo]) => { + validMasterKeyResponses.reduce((msks, [userId, keyInfo]) => { const keyIds = Object.keys(keyInfo.keys); if (keyIds.length !== 1) { return false; @@ -336,7 +337,7 @@ export class DeviceTracker { const masterKey = keyInfo.keys[keyIds[0]]; msks.set(userId, masterKey); return msks; - }, new Map()); + }, masterKeys); return masterKeys; } @@ -680,12 +681,14 @@ export function tests() { const txn = await storage.readTxn([storage.storeNames.userIdentities]); assert.deepEqual(await txn.userIdentities.get("@alice:hs.tld"), { userId: "@alice:hs.tld", + masterKey: undefined, roomIds: [roomId], deviceTrackingStatus: TRACKING_STATUS_OUTDATED }); assert.deepEqual(await txn.userIdentities.get("@bob:hs.tld"), { userId: "@bob:hs.tld", roomIds: [roomId], + masterKey: undefined, deviceTrackingStatus: TRACKING_STATUS_OUTDATED }); assert.equal(await txn.userIdentities.get("@charly:hs.tld"), undefined); From 693640d222be176bf9398f8989f6999c149642cc Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 14 Feb 2023 16:25:32 +0100 Subject: [PATCH 393/435] implement signing our own device --- src/matrix/Session.js | 3 +- src/matrix/e2ee/Account.js | 12 +++- src/matrix/net/HomeServerApi.ts | 4 ++ src/matrix/verification/CrossSigning.ts | 78 +++++++++++++++++++------ src/matrix/verification/common.ts | 72 +++++++++++++++++++++++ 5 files changed, 150 insertions(+), 19 deletions(-) create mode 100644 src/matrix/verification/common.ts diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 222c8ef2..35f713f6 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -341,7 +341,8 @@ export class Session { olm: this._olm, deviceTracker: this._deviceTracker, hsApi: this._hsApi, - ownUserId: this.userId + ownUserId: this.userId, + e2eeAccount: this._e2eeAccount }); await log.wrap("enable cross-signing", log => { return this._crossSigning.init(log); diff --git a/src/matrix/e2ee/Account.js b/src/matrix/e2ee/Account.js index b704d7f5..0238f0cf 100644 --- a/src/matrix/e2ee/Account.js +++ b/src/matrix/e2ee/Account.js @@ -246,7 +246,7 @@ export class Account { } } - _deviceKeysPayload(identityKeys) { + _keysAsSignableObject(identityKeys) { const obj = { user_id: this._userId, device_id: this._deviceId, @@ -256,6 +256,16 @@ export class Account { for (const [algorithm, pubKey] of Object.entries(identityKeys)) { obj.keys[`${algorithm}:${this._deviceId}`] = pubKey; } + return obj; + } + + getDeviceKeysToSignWithCrossSigning() { + const identityKeys = JSON.parse(this._account.identity_keys()); + return this._keysAsSignableObject(identityKeys); + } + + _deviceKeysPayload(identityKeys) { + const obj = this._keysAsSignableObject(identityKeys); this.signObject(obj); return obj; } diff --git a/src/matrix/net/HomeServerApi.ts b/src/matrix/net/HomeServerApi.ts index 923f23e3..c5f90555 100644 --- a/src/matrix/net/HomeServerApi.ts +++ b/src/matrix/net/HomeServerApi.ts @@ -223,6 +223,10 @@ export class HomeServerApi { return this._post(path, {}, payload, options); } + uploadSignatures(payload: Record, options?: BaseRequestOptions): IHomeServerRequest { + return this._post("/keys/signatures/upload", {}, payload, options); + } + queryKeys(queryRequest: Record, options?: BaseRequestOptions): IHomeServerRequest { return this._post("/keys/query", {}, queryRequest, options); } diff --git a/src/matrix/verification/CrossSigning.ts b/src/matrix/verification/CrossSigning.ts index 8c3b6b61..db480dd0 100644 --- a/src/matrix/verification/CrossSigning.ts +++ b/src/matrix/verification/CrossSigning.ts @@ -20,6 +20,11 @@ import type {Platform} from "../../platform/web/Platform"; import type {DeviceTracker} from "../e2ee/DeviceTracker"; import type * as OlmNamespace from "@matrix-org/olm"; import type {HomeServerApi} from "../net/HomeServerApi"; +import type {Account} from "../e2ee/Account"; +import { ILogItem } from "../../lib"; +import {pkSign} from "./common"; +import type {ISignatures} from "./common"; + type Olm = typeof OlmNamespace; export class CrossSigning { @@ -30,9 +35,19 @@ export class CrossSigning { private readonly olm: Olm; private readonly hsApi: HomeServerApi; private readonly ownUserId: string; + private readonly e2eeAccount: Account; private _isMasterKeyTrusted: boolean = false; - constructor(options: {storage: Storage, secretStorage: SecretStorage, deviceTracker: DeviceTracker, platform: Platform, olm: Olm, ownUserId: string, hsApi: HomeServerApi}) { + constructor(options: { + storage: Storage, + secretStorage: SecretStorage, + deviceTracker: DeviceTracker, + platform: Platform, + olm: Olm, + ownUserId: string, + hsApi: HomeServerApi, + e2eeAccount: Account + }) { this.storage = options.storage; this.secretStorage = options.secretStorage; this.platform = options.platform; @@ -40,25 +55,54 @@ export class CrossSigning { this.olm = options.olm; this.hsApi = options.hsApi; this.ownUserId = options.ownUserId; + this.e2eeAccount = options.e2eeAccount } - async init(log) { - // use errorboundary here + async init(log: ILogItem) { + log.wrap("CrossSigning.init", async log => { + // TODO: use errorboundary here + const txn = await this.storage.readTxn([this.storage.storeNames.accountData]); + + const mskSeed = await this.secretStorage.readSecret("m.cross_signing.master", txn); + const signing = new this.olm.PkSigning(); + let derivedPublicKey; + try { + const seed = new Uint8Array(this.platform.encoding.base64.decode(mskSeed)); + derivedPublicKey = signing.init_with_seed(seed); + } finally { + signing.free(); + } + const publishedKeys = await this.deviceTracker.getCrossSigningKeysForUser(this.ownUserId, this.hsApi, log); + log.set({publishedMasterKey: publishedKeys.masterKey, derivedPublicKey}); + this._isMasterKeyTrusted = publishedKeys.masterKey === derivedPublicKey; + log.set("isMasterKeyTrusted", this.isMasterKeyTrusted); + }); + } + + async signOwnDevice(log: ILogItem) { + log.wrap("CrossSigning.signOwnDevice", async log => { + if (!this._isMasterKeyTrusted) { + log.set("mskNotTrusted", true); + return; + } + const deviceKey = this.e2eeAccount.getDeviceKeysToSignWithCrossSigning(); + const signedDeviceKey = await this.signDevice(deviceKey); + const payload = { + [signedDeviceKey["user_id"]]: { + [signedDeviceKey["device_id"]]: signedDeviceKey + } + }; + const request = this.hsApi.uploadSignatures(payload, {log}); + await request.response(); + }); + } + + private async signDevice(data: T): Promise { const txn = await this.storage.readTxn([this.storage.storeNames.accountData]); - - const mskSeed = await this.secretStorage.readSecret("m.cross_signing.master", txn); - const signing = new this.olm.PkSigning(); - let derivedPublicKey; - try { - const seed = new Uint8Array(this.platform.encoding.base64.decode(mskSeed)); - derivedPublicKey = signing.init_with_seed(seed); - } finally { - signing.free(); - } - const publishedMasterKey = await this.deviceTracker.getMasterKeyForUser(this.ownUserId, this.hsApi, log); - log.set({publishedMasterKey, derivedPublicKey}); - this._isMasterKeyTrusted = publishedMasterKey === derivedPublicKey; - log.set("isMasterKeyTrusted", this.isMasterKeyTrusted); + const seedStr = await this.secretStorage.readSecret(`m.cross_signing.self_signing`, txn); + const seed = new Uint8Array(this.platform.encoding.base64.decode(seedStr)); + pkSign(this.olm, data, seed, this.ownUserId, ""); + return data as T & { signatures: ISignatures }; } get isMasterKeyTrusted(): boolean { diff --git a/src/matrix/verification/common.ts b/src/matrix/verification/common.ts new file mode 100644 index 00000000..369b5618 --- /dev/null +++ b/src/matrix/verification/common.ts @@ -0,0 +1,72 @@ +/* +Copyright 2016-2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { PkSigning } from "@matrix-org/olm"; +import anotherjson from "another-json"; +import type * as OlmNamespace from "@matrix-org/olm"; +type Olm = typeof OlmNamespace; + +export interface IObject { + unsigned?: object; + signatures?: ISignatures; +} + +export interface ISignatures { + [entity: string]: { + [keyId: string]: string; + }; +} + +export interface ISigned { + signatures?: ISignatures; +} + +// from matrix-js-sdk +/** + * Sign a JSON object using public key cryptography + * @param obj - Object to sign. The object will be modified to include + * the new signature + * @param key - the signing object or the private key + * seed + * @param userId - The user ID who owns the signing key + * @param pubKey - The public key (ignored if key is a seed) + * @returns the signature for the object + */ + export function pkSign(olmUtil: Olm, obj: object & IObject, key: Uint8Array | PkSigning, userId: string, pubKey: string): string { + let createdKey = false; + if (key instanceof Uint8Array) { + const keyObj = new olmUtil.PkSigning(); + pubKey = keyObj.init_with_seed(key); + key = keyObj; + createdKey = true; + } + const sigs = obj.signatures || {}; + delete obj.signatures; + const unsigned = obj.unsigned; + if (obj.unsigned) delete obj.unsigned; + try { + const mysigs = sigs[userId] || {}; + sigs[userId] = mysigs; + + return (mysigs["ed25519:" + pubKey] = key.sign(anotherjson.stringify(obj))); + } finally { + obj.signatures = sigs; + if (unsigned) obj.unsigned = unsigned; + if (createdKey) { + key.free(); + } + } +} \ No newline at end of file From 088fcdc77bd93535e6a923701c10444e264fcbd2 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 14 Feb 2023 16:25:46 +0100 Subject: [PATCH 394/435] logging type definition fix --- src/logging/types.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/logging/types.ts b/src/logging/types.ts index 0f4f6150..54436423 100644 --- a/src/logging/types.ts +++ b/src/logging/types.ts @@ -44,7 +44,8 @@ export interface ILogItem { /*** This is sort of low-level, you probably want to use wrap. If you do use it, it should only be called once. */ run(callback: LogCallback): T; log(labelOrValues: LabelOrValues, logLevel?: LogLevel): ILogItem; - set(key: string | object, value: unknown): ILogItem; + set(key: string, value: unknown): ILogItem; + set(key: object): ILogItem; runDetached(labelOrValues: LabelOrValues, callback: LogCallback, logLevel?: LogLevel, filterCreator?: FilterCreator): ILogItem; wrapDetached(labelOrValues: LabelOrValues, callback: LogCallback, logLevel?: LogLevel, filterCreator?: FilterCreator): void; refDetached(logItem: ILogItem, logLevel?: LogLevel): void; From c7a2b4dc2ef6b590eb7dc600332eca6456297be2 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 14 Feb 2023 16:26:02 +0100 Subject: [PATCH 395/435] expose self-signing in settings UI --- src/domain/session/settings/KeyBackupViewModel.js | 12 ++++++++++++ .../ui/session/settings/KeyBackupSettingsView.js | 13 +++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/domain/session/settings/KeyBackupViewModel.js b/src/domain/session/settings/KeyBackupViewModel.js index 56339a4e..22135e41 100644 --- a/src/domain/session/settings/KeyBackupViewModel.js +++ b/src/domain/session/settings/KeyBackupViewModel.js @@ -92,6 +92,18 @@ export class KeyBackupViewModel extends ViewModel { return this._session.crossSigning?.isMasterKeyTrusted ?? false; } + get canSignOwnDevice() { + return !!this._session.crossSigning; + } + + async signOwnDevice() { + if (this._session.crossSigning) { + await this.logger.run("KeyBackupViewModel.signOwnDevice", async log => { + await this._session.crossSigning.signOwnDevice(log); + }); + } + } + get backupWriteStatus() { const keyBackup = this._session.keyBackup.get(); if (!keyBackup) { diff --git a/src/platform/web/ui/session/settings/KeyBackupSettingsView.js b/src/platform/web/ui/session/settings/KeyBackupSettingsView.js index bd1eb4e8..a68a80b3 100644 --- a/src/platform/web/ui/session/settings/KeyBackupSettingsView.js +++ b/src/platform/web/ui/session/settings/KeyBackupSettingsView.js @@ -15,9 +15,10 @@ limitations under the License. */ import {TemplateView} from "../../general/TemplateView"; +import {disableTargetCallback} from "../../general/utils"; export class KeyBackupSettingsView extends TemplateView { - render(t) { + render(t, vm) { return t.div([ t.map(vm => vm.status, (status, t, vm) => { switch (status) { @@ -56,7 +57,15 @@ export class KeyBackupSettingsView extends TemplateView { }), t.if(vm => vm.isMasterKeyTrusted, t => { return t.p("Cross-signing master key found and trusted.") - }) + }), + t.if(vm => vm.canSignOwnDevice, t => { + return t.button({ + onClick: disableTargetCallback(async evt => { + await vm.signOwnDevice(); + }) + }, "Sign own device"); + }), + ]); } } From 71d7dcb0ca92b4d81178055106346d1bb5a07ba7 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 14 Feb 2023 16:26:55 +0100 Subject: [PATCH 396/435] store self-signing keys on userIdentity --- src/matrix/e2ee/DeviceTracker.js | 55 +++++++++++++++++++++++++------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/src/matrix/e2ee/DeviceTracker.js b/src/matrix/e2ee/DeviceTracker.js index 810dfd95..b669629e 100644 --- a/src/matrix/e2ee/DeviceTracker.js +++ b/src/matrix/e2ee/DeviceTracker.js @@ -25,7 +25,7 @@ function createUserIdentity(userId, initialRoomId = undefined) { return { userId: userId, roomIds: initialRoomId ? [initialRoomId] : [], - masterKey: undefined, + crossSigningKeys: undefined, deviceTrackingStatus: TRACKING_STATUS_OUTDATED, }; } @@ -153,15 +153,15 @@ export class DeviceTracker { } } - async getMasterKeyForUser(userId, hsApi, log) { + async getCrossSigningKeysForUser(userId, hsApi, log) { return await log.wrap("DeviceTracker.getMasterKeyForUser", async log => { let txn = await this._storage.readTxn([ this._storage.storeNames.userIdentities ]); let userIdentity = await txn.userIdentities.get(userId); if (userIdentity && userIdentity.deviceTrackingStatus !== TRACKING_STATUS_OUTDATED) { - return userIdentity.masterKey; - } + return userIdentity.crossSigningKeys; + } // fetch from hs await this._queryKeys([userId], hsApi, log); // Retreive from storage now @@ -169,7 +169,7 @@ export class DeviceTracker { this._storage.storeNames.userIdentities ]); userIdentity = await txn.userIdentities.get(userId); - return userIdentity?.masterKey; + return userIdentity?.crossSigningKeys; }); } @@ -246,6 +246,7 @@ export class DeviceTracker { }, {log}).response(); const masterKeys = log.wrap("master keys", log => this._filterValidMasterKeys(deviceKeyResponse, log)); + const selfSigningKeys = log.wrap("self-signing keys", log => this._filterVerifiedCrossSigningKeys(deviceKeyResponse["self_signing_keys"], "self_signing", masterKeys, log)) const verifiedKeysPerUser = log.wrap("verify", log => this._filterVerifiedDeviceKeys(deviceKeyResponse["device_keys"], log)); const txn = await this._storage.readWriteTxn([ this._storage.storeNames.userIdentities, @@ -255,7 +256,11 @@ export class DeviceTracker { try { const devicesIdentitiesPerUser = await Promise.all(verifiedKeysPerUser.map(async ({userId, verifiedKeys}) => { const deviceIdentities = verifiedKeys.map(deviceKeysAsDeviceIdentity); - return await this._storeQueriedDevicesForUserId(userId, masterKeys.get(userId), deviceIdentities, txn); + const crossSigningKeys = { + masterKey: masterKeys.get(userId), + selfSigningKey: selfSigningKeys.get(userId), + }; + return await this._storeQueriedDevicesForUserId(userId, crossSigningKeys, deviceIdentities, txn); })); deviceIdentities = devicesIdentitiesPerUser.reduce((all, devices) => all.concat(devices), []); log.set("devices", deviceIdentities.length); @@ -267,7 +272,7 @@ export class DeviceTracker { return deviceIdentities; } - async _storeQueriedDevicesForUserId(userId, masterKey, deviceIdentities, txn) { + async _storeQueriedDevicesForUserId(userId, crossSigningKeys, deviceIdentities, txn) { const knownDeviceIds = await txn.deviceIdentities.getAllDeviceIds(userId); // delete any devices that we know off but are not in the response anymore. // important this happens before checking if the ed25519 key changed, @@ -308,13 +313,13 @@ export class DeviceTracker { identity = createUserIdentity(userId); } identity.deviceTrackingStatus = TRACKING_STATUS_UPTODATE; - identity.masterKey = masterKey; + identity.crossSigningKeys = crossSigningKeys; txn.userIdentities.set(identity); return allDeviceIdentities; } - _filterValidMasterKeys(keyQueryResponse, parentLog) { + _filterValidMasterKeys(keyQueryResponse, log) { const masterKeys = new Map(); const masterKeysResponse = keyQueryResponse["master_keys"]; if (!masterKeysResponse) { @@ -341,6 +346,34 @@ export class DeviceTracker { return masterKeys; } + _filterVerifiedCrossSigningKeys(crossSigningKeysResponse, usage, masterKeys, log) { + const keys = new Map(); + if (!crossSigningKeysResponse) { + return keys; + } + const validKeysResponses = Object.entries(crossSigningKeysResponse).filter(([userId, keyInfo]) => { + if (keyInfo["user_id"] !== userId) { + return false; + } + if (!Array.isArray(keyInfo.usage) || !keyInfo.usage.includes(usage)) { + return false; + } + // verify with master key + const masterKey = masterKeys.get(userId); + return verifyEd25519Signature(this._olmUtil, userId, masterKey, masterKey, keyInfo, log); + }); + validKeysResponses.reduce((keys, [userId, keyInfo]) => { + const keyIds = Object.keys(keyInfo.keys); + if (keyIds.length !== 1) { + return false; + } + const key = keyInfo.keys[keyIds[0]]; + keys.set(userId, key); + return keys; + }, keys); + return keys; + } + /** * @return {Array<{userId, verifiedKeys: Array>} */ @@ -681,14 +714,14 @@ export function tests() { const txn = await storage.readTxn([storage.storeNames.userIdentities]); assert.deepEqual(await txn.userIdentities.get("@alice:hs.tld"), { userId: "@alice:hs.tld", - masterKey: undefined, + crossSigningKeys: undefined, roomIds: [roomId], deviceTrackingStatus: TRACKING_STATUS_OUTDATED }); assert.deepEqual(await txn.userIdentities.get("@bob:hs.tld"), { userId: "@bob:hs.tld", roomIds: [roomId], - masterKey: undefined, + crossSigningKeys: undefined, deviceTrackingStatus: TRACKING_STATUS_OUTDATED }); assert.equal(await txn.userIdentities.get("@charly:hs.tld"), undefined); From 751987826441f26107ec05a9ce86dd2ced83de34 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 14 Feb 2023 16:30:25 +0100 Subject: [PATCH 397/435] add stronger warning to enable cross-signing --- src/domain/session/settings/FeaturesViewModel.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/session/settings/FeaturesViewModel.ts b/src/domain/session/settings/FeaturesViewModel.ts index 552e27d4..b0ee3598 100644 --- a/src/domain/session/settings/FeaturesViewModel.ts +++ b/src/domain/session/settings/FeaturesViewModel.ts @@ -32,7 +32,7 @@ export class FeaturesViewModel extends ViewModel { })), new FeatureViewModel(this.childOptions({ name: this.i18n`Cross-Signing`, - description: this.i18n`Allows.verifying the identity of people you chat with`, + description: this.i18n`Allows verifying the identity of people you chat with. This feature is still evolving constantly, expect things to break.`, feature: FeatureFlag.CrossSigning })), ]; From 8c74e54f9d644a6de47b4a7927396911510df80e Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 14 Feb 2023 16:38:29 +0100 Subject: [PATCH 398/435] release v0.3.8 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 697bfa43..e367bc09 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hydrogen-web", - "version": "0.3.7", + "version": "0.3.8", "description": "A javascript matrix client prototype, trying to minize RAM usage by offloading as much as possible to IndexedDB", "directories": { "doc": "doc" From 151090527b909f18620faddf6ded729038dd8872 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 24 Feb 2023 17:45:56 +0100 Subject: [PATCH 399/435] Store cross-signing keys in format as returned from server, in separate store This will make it easier to sign and verify signatures with these keys, as the signed value needs to have the same layout when signing and for every verification. --- src/matrix/e2ee/DeviceTracker.js | 151 ++++++++++-------- src/matrix/e2ee/common.js | 13 +- src/matrix/storage/common.ts | 3 +- src/matrix/storage/idb/Transaction.ts | 5 + src/matrix/storage/idb/schema.ts | 8 +- .../idb/stores/CrossSigningKeyStore.ts | 70 ++++++++ src/matrix/verification/CrossSigning.ts | 51 +++++- 7 files changed, 221 insertions(+), 80 deletions(-) create mode 100644 src/matrix/storage/idb/stores/CrossSigningKeyStore.ts diff --git a/src/matrix/e2ee/DeviceTracker.js b/src/matrix/e2ee/DeviceTracker.js index b669629e..5d991fcb 100644 --- a/src/matrix/e2ee/DeviceTracker.js +++ b/src/matrix/e2ee/DeviceTracker.js @@ -14,9 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {verifyEd25519Signature, SIGNATURE_ALGORITHM} from "./common.js"; +import {verifyEd25519Signature, getEd25519Signature, SIGNATURE_ALGORITHM} from "./common.js"; import {HistoryVisibility, shouldShareKey} from "./common.js"; import {RoomMember} from "../room/members/RoomMember.js"; +import {getKeyUsage, getKeyEd25519Key, getKeyUserId, KeyUsage} from "../verification/CrossSigning"; const TRACKING_STATUS_OUTDATED = 0; const TRACKING_STATUS_UPTODATE = 1; @@ -153,7 +154,7 @@ export class DeviceTracker { } } - async getCrossSigningKeysForUser(userId, hsApi, log) { + async getCrossSigningKeyForUser(userId, usage, hsApi, log) { return await log.wrap("DeviceTracker.getMasterKeyForUser", async log => { let txn = await this._storage.readTxn([ this._storage.storeNames.userIdentities @@ -163,13 +164,16 @@ export class DeviceTracker { return userIdentity.crossSigningKeys; } // fetch from hs - await this._queryKeys([userId], hsApi, log); - // Retreive from storage now - txn = await this._storage.readTxn([ - this._storage.storeNames.userIdentities - ]); - userIdentity = await txn.userIdentities.get(userId); - return userIdentity?.crossSigningKeys; + const keys = await this._queryKeys([userId], hsApi, log); + switch (usage) { + case KeyUsage.Master: + return keys.masterKeys.get(userId); + case KeyUsage.SelfSigning: + return keys.selfSigningKeys.get(userId); + case KeyUsage.UserSigning: + return keys.userSigningKeys.get(userId); + + } }); } @@ -245,22 +249,29 @@ export class DeviceTracker { "token": this._getSyncToken() }, {log}).response(); - const masterKeys = log.wrap("master keys", log => this._filterValidMasterKeys(deviceKeyResponse, log)); - const selfSigningKeys = log.wrap("self-signing keys", log => this._filterVerifiedCrossSigningKeys(deviceKeyResponse["self_signing_keys"], "self_signing", masterKeys, log)) - const verifiedKeysPerUser = log.wrap("verify", log => this._filterVerifiedDeviceKeys(deviceKeyResponse["device_keys"], log)); + const masterKeys = log.wrap("master keys", log => this._filterVerifiedCrossSigningKeys(deviceKeyResponse["master_keys"], KeyUsage.Master, undefined, log)); + const selfSigningKeys = log.wrap("self-signing keys", log => this._filterVerifiedCrossSigningKeys(deviceKeyResponse["self_signing_keys"], KeyUsage.SelfSigning, masterKeys, log)); + const userSigningKeys = log.wrap("user-signing keys", log => this._filterVerifiedCrossSigningKeys(deviceKeyResponse["user_signing_keys"], KeyUsage.UserSigning, masterKeys, log)); + const verifiedKeysPerUser = log.wrap("device keys", log => this._filterVerifiedDeviceKeys(deviceKeyResponse["device_keys"], log)); const txn = await this._storage.readWriteTxn([ this._storage.storeNames.userIdentities, this._storage.storeNames.deviceIdentities, + this._storage.storeNames.crossSigningKeys, ]); let deviceIdentities; try { + for (const key of masterKeys.values()) { + txn.crossSigningKeys.set(key); + } + for (const key of selfSigningKeys.values()) { + txn.crossSigningKeys.set(key); + } + for (const key of userSigningKeys.values()) { + txn.crossSigningKeys.set(key); + } const devicesIdentitiesPerUser = await Promise.all(verifiedKeysPerUser.map(async ({userId, verifiedKeys}) => { const deviceIdentities = verifiedKeys.map(deviceKeysAsDeviceIdentity); - const crossSigningKeys = { - masterKey: masterKeys.get(userId), - selfSigningKey: selfSigningKeys.get(userId), - }; - return await this._storeQueriedDevicesForUserId(userId, crossSigningKeys, deviceIdentities, txn); + return await this._storeQueriedDevicesForUserId(userId, deviceIdentities, txn); })); deviceIdentities = devicesIdentitiesPerUser.reduce((all, devices) => all.concat(devices), []); log.set("devices", deviceIdentities.length); @@ -269,10 +280,15 @@ export class DeviceTracker { throw err; } await txn.complete(); - return deviceIdentities; + return { + deviceIdentities, + masterKeys, + selfSigningKeys, + userSigningKeys + }; } - async _storeQueriedDevicesForUserId(userId, crossSigningKeys, deviceIdentities, txn) { + async _storeQueriedDevicesForUserId(userId, deviceIdentities, txn) { const knownDeviceIds = await txn.deviceIdentities.getAllDeviceIds(userId); // delete any devices that we know off but are not in the response anymore. // important this happens before checking if the ed25519 key changed, @@ -313,67 +329,63 @@ export class DeviceTracker { identity = createUserIdentity(userId); } identity.deviceTrackingStatus = TRACKING_STATUS_UPTODATE; - identity.crossSigningKeys = crossSigningKeys; txn.userIdentities.set(identity); return allDeviceIdentities; } - _filterValidMasterKeys(keyQueryResponse, log) { - const masterKeys = new Map(); - const masterKeysResponse = keyQueryResponse["master_keys"]; - if (!masterKeysResponse) { - return masterKeys; - } - const validMasterKeyResponses = Object.entries(masterKeysResponse).filter(([userId, keyInfo]) => { - if (keyInfo["user_id"] !== userId) { - return false; - } - if (!Array.isArray(keyInfo.usage) || !keyInfo.usage.includes("master")) { - return false; - } - return true; - }); - validMasterKeyResponses.reduce((msks, [userId, keyInfo]) => { - const keyIds = Object.keys(keyInfo.keys); - if (keyIds.length !== 1) { - return false; - } - const masterKey = keyInfo.keys[keyIds[0]]; - msks.set(userId, masterKey); - return msks; - }, masterKeys); - return masterKeys; - } - - _filterVerifiedCrossSigningKeys(crossSigningKeysResponse, usage, masterKeys, log) { + _filterVerifiedCrossSigningKeys(crossSigningKeysResponse, usage, parentKeys, log) { const keys = new Map(); if (!crossSigningKeysResponse) { return keys; } - const validKeysResponses = Object.entries(crossSigningKeysResponse).filter(([userId, keyInfo]) => { - if (keyInfo["user_id"] !== userId) { - return false; - } - if (!Array.isArray(keyInfo.usage) || !keyInfo.usage.includes(usage)) { - return false; - } - // verify with master key - const masterKey = masterKeys.get(userId); - return verifyEd25519Signature(this._olmUtil, userId, masterKey, masterKey, keyInfo, log); - }); - validKeysResponses.reduce((keys, [userId, keyInfo]) => { - const keyIds = Object.keys(keyInfo.keys); - if (keyIds.length !== 1) { - return false; - } - const key = keyInfo.keys[keyIds[0]]; - keys.set(userId, key); - return keys; - }, keys); + for (const [userId, keyInfo] of Object.entries(crossSigningKeysResponse)) { + log.wrap({l: userId}, log => { + const parentKeyInfo = parentKeys?.get(userId); + const parentKey = parentKeyInfo && getKeyEd25519Key(parentKeyInfo); + if (this._validateCrossSigningKey(userId, keyInfo, usage, parentKey, log)) { + keys.set(getKeyUserId(keyInfo), keyInfo); + } + }); + } return keys; } + _validateCrossSigningKey(userId, keyInfo, usage, parentKey, log) { + if (getKeyUserId(keyInfo) !== userId) { + log.log({l: "user_id mismatch", userId: keyInfo["user_id"]}); + return false; + } + if (getKeyUsage(keyInfo) !== usage) { + log.log({l: "usage mismatch", usage: keyInfo.usage}); + return false; + } + const publicKey = getKeyEd25519Key(keyInfo); + if (!publicKey) { + log.log({l: "no ed25519 key", keys: keyInfo.keys}); + return false; + } + const isSelfSigned = usage === "master"; + const keyToVerifyWith = isSelfSigned ? publicKey : parentKey; + if (!keyToVerifyWith) { + log.log("signing_key not found"); + return false; + } + const hasSignature = !!getEd25519Signature(keyInfo, userId, keyToVerifyWith); + // self-signature is optional for now, not all keys seem to have it + if (!hasSignature && keyToVerifyWith !== publicKey) { + log.log({l: "signature not found", key: keyToVerifyWith}); + return false; + } + if (hasSignature) { + if(!verifyEd25519Signature(this._olmUtil, userId, keyToVerifyWith, keyToVerifyWith, keyInfo, log)) { + log.log("signature mismatch"); + return false; + } + } + return true; + } + /** * @return {Array<{userId, verifiedKeys: Array>} */ @@ -580,7 +592,8 @@ export class DeviceTracker { // TODO: ignore the race between /sync and /keys/query for now, // where users could get marked as outdated or added/removed from the room while // querying keys - queriedDevices = await this._queryKeys(outdatedUserIds, hsApi, log); + const {deviceIdentities} = await this._queryKeys(outdatedUserIds, hsApi, log); + queriedDevices = deviceIdentities; } const deviceTxn = await this._storage.readTxn([ diff --git a/src/matrix/e2ee/common.js b/src/matrix/e2ee/common.js index cc3bfff5..9c5fe66c 100644 --- a/src/matrix/e2ee/common.js +++ b/src/matrix/e2ee/common.js @@ -35,16 +35,21 @@ export class DecryptionError extends Error { export const SIGNATURE_ALGORITHM = "ed25519"; +export function getEd25519Signature(signedValue, userId, deviceOrKeyId) { + return signedValue?.signatures?.[userId]?.[`${SIGNATURE_ALGORITHM}:${deviceOrKeyId}`]; +} + export function verifyEd25519Signature(olmUtil, userId, deviceOrKeyId, ed25519Key, value, log = undefined) { + const signature = getEd25519Signature(value, userId, deviceOrKeyId); + if (!signature) { + log?.set("no_signature", true); + return false; + } const clone = Object.assign({}, value); delete clone.unsigned; delete clone.signatures; const canonicalJson = anotherjson.stringify(clone); - const signature = value?.signatures?.[userId]?.[`${SIGNATURE_ALGORITHM}:${deviceOrKeyId}`]; try { - if (!signature) { - throw new Error("no signature"); - } // throws when signature is invalid olmUtil.ed25519_verify(ed25519Key, canonicalJson, signature); return true; diff --git a/src/matrix/storage/common.ts b/src/matrix/storage/common.ts index e1e34917..adebcdd6 100644 --- a/src/matrix/storage/common.ts +++ b/src/matrix/storage/common.ts @@ -33,7 +33,8 @@ export enum StoreNames { groupSessionDecryptions = "groupSessionDecryptions", operations = "operations", accountData = "accountData", - calls = "calls" + calls = "calls", + crossSigningKeys = "crossSigningKeys" } export const STORE_NAMES: Readonly = Object.values(StoreNames); diff --git a/src/matrix/storage/idb/Transaction.ts b/src/matrix/storage/idb/Transaction.ts index 7a8de420..532ffd1d 100644 --- a/src/matrix/storage/idb/Transaction.ts +++ b/src/matrix/storage/idb/Transaction.ts @@ -30,6 +30,7 @@ import {TimelineFragmentStore} from "./stores/TimelineFragmentStore"; import {PendingEventStore} from "./stores/PendingEventStore"; import {UserIdentityStore} from "./stores/UserIdentityStore"; import {DeviceIdentityStore} from "./stores/DeviceIdentityStore"; +import {CrossSigningKeyStore} from "./stores/CrossSigningKeyStore"; import {OlmSessionStore} from "./stores/OlmSessionStore"; import {InboundGroupSessionStore} from "./stores/InboundGroupSessionStore"; import {OutboundGroupSessionStore} from "./stores/OutboundGroupSessionStore"; @@ -145,6 +146,10 @@ export class Transaction { return this._store(StoreNames.deviceIdentities, idbStore => new DeviceIdentityStore(idbStore)); } + get crossSigningKeys(): CrossSigningKeyStore { + return this._store(StoreNames.crossSigningKeys, idbStore => new CrossSigningKeyStore(idbStore)); + } + get olmSessions(): OlmSessionStore { return this._store(StoreNames.olmSessions, idbStore => new OlmSessionStore(idbStore)); } diff --git a/src/matrix/storage/idb/schema.ts b/src/matrix/storage/idb/schema.ts index d88f535e..3d1e714f 100644 --- a/src/matrix/storage/idb/schema.ts +++ b/src/matrix/storage/idb/schema.ts @@ -34,7 +34,8 @@ export const schema: MigrationFunc[] = [ clearAllStores, addInboundSessionBackupIndex, migrateBackupStatus, - createCallStore + createCallStore, + createCrossSigningKeyStore ]; // TODO: how to deal with git merge conflicts of this array? @@ -275,3 +276,8 @@ async function migrateBackupStatus(db: IDBDatabase, txn: IDBTransaction, localSt function createCallStore(db: IDBDatabase) : void { db.createObjectStore("calls", {keyPath: "key"}); } + +//v18 create calls store +function createCrossSigningKeyStore(db: IDBDatabase) : void { + db.createObjectStore("crossSigningKeys", {keyPath: "key"}); +} diff --git a/src/matrix/storage/idb/stores/CrossSigningKeyStore.ts b/src/matrix/storage/idb/stores/CrossSigningKeyStore.ts new file mode 100644 index 00000000..a2fa9ecb --- /dev/null +++ b/src/matrix/storage/idb/stores/CrossSigningKeyStore.ts @@ -0,0 +1,70 @@ +/* +Copyright 2020 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {MAX_UNICODE, MIN_UNICODE} from "./common"; +import {Store} from "../Store"; + +// we store cross-signing keys in the format we get them from the server +// as that is what the signature is calculated on, so to verify, we need +// it in this format anyway. +export type CrossSigningKey = { + readonly user_id: string; + readonly usage: ReadonlyArray; + readonly keys: {[keyId: string]: string}; + readonly signatures: {[userId: string]: {[keyId: string]: string}} +} + +type CrossSigningKeyEntry = CrossSigningKey & { + key: string; // key in storage, not a crypto key +} + +function encodeKey(userId: string, usage: string): string { + return `${userId}|${usage}`; +} + +function decodeKey(key: string): { userId: string, usage: string } { + const [userId, usage] = key.split("|"); + return {userId, usage}; +} + +export class CrossSigningKeyStore { + private _store: Store; + + constructor(store: Store) { + this._store = store; + } + + get(userId: string, deviceId: string): Promise { + return this._store.get(encodeKey(userId, deviceId)); + } + + set(crossSigningKey: CrossSigningKey): void { + const deviceIdentityEntry = crossSigningKey as CrossSigningKeyEntry; + deviceIdentityEntry.key = encodeKey(crossSigningKey["user_id"], crossSigningKey.usage[0]); + this._store.put(deviceIdentityEntry); + } + + remove(userId: string, usage: string): void { + this._store.delete(encodeKey(userId, usage)); + } + + removeAllForUser(userId: string): void { + // exclude both keys as they are theoretical min and max, + // but we should't have a match for just the room id, or room id with max + const range = this._store.IDBKeyRange.bound(encodeKey(userId, MIN_UNICODE), encodeKey(userId, MAX_UNICODE), true, true); + this._store.delete(range); + } +} diff --git a/src/matrix/verification/CrossSigning.ts b/src/matrix/verification/CrossSigning.ts index db480dd0..d3b6bc90 100644 --- a/src/matrix/verification/CrossSigning.ts +++ b/src/matrix/verification/CrossSigning.ts @@ -27,6 +27,12 @@ import type {ISignatures} from "./common"; type Olm = typeof OlmNamespace; +export enum KeyUsage { + Master = "master", + SelfSigning = "self_signing", + UserSigning = "user_signing" +}; + export class CrossSigning { private readonly storage: Storage; private readonly secretStorage: SecretStorage; @@ -72,9 +78,9 @@ export class CrossSigning { } finally { signing.free(); } - const publishedKeys = await this.deviceTracker.getCrossSigningKeysForUser(this.ownUserId, this.hsApi, log); - log.set({publishedMasterKey: publishedKeys.masterKey, derivedPublicKey}); - this._isMasterKeyTrusted = publishedKeys.masterKey === derivedPublicKey; + const masterKey = await this.deviceTracker.getCrossSigningKeyForUser(this.ownUserId, KeyUsage.Master, this.hsApi, log); + log.set({publishedMasterKey: masterKey, derivedPublicKey}); + this._isMasterKeyTrusted = masterKey === derivedPublicKey; log.set("isMasterKeyTrusted", this.isMasterKeyTrusted); }); } @@ -86,7 +92,7 @@ export class CrossSigning { return; } const deviceKey = this.e2eeAccount.getDeviceKeysToSignWithCrossSigning(); - const signedDeviceKey = await this.signDevice(deviceKey); + const signedDeviceKey = await this.signDeviceData(deviceKey); const payload = { [signedDeviceKey["user_id"]]: { [signedDeviceKey["device_id"]]: signedDeviceKey @@ -97,7 +103,15 @@ export class CrossSigning { }); } - private async signDevice(data: T): Promise { + signDevice(deviceId: string) { + // need to get the device key for the device + } + + signUser(userId: string) { + // need to be able to get the msk for the user + } + + private async signDeviceData(data: T): Promise { const txn = await this.storage.readTxn([this.storage.storeNames.accountData]); const seedStr = await this.secretStorage.readSecret(`m.cross_signing.self_signing`, txn); const seed = new Uint8Array(this.platform.encoding.base64.decode(seedStr)); @@ -110,3 +124,30 @@ export class CrossSigning { } } +export function getKeyUsage(keyInfo): KeyUsage | undefined { + if (!Array.isArray(keyInfo.usage) || keyInfo.usage.length !== 1) { + return undefined; + } + const usage = keyInfo.usage[0]; + if (usage !== KeyUsage.Master && usage !== KeyUsage.SelfSigning && usage !== KeyUsage.UserSigning) { + return undefined; + } + return usage; +} + +const algorithm = "ed25519"; +const prefix = `${algorithm}:`; + +export function getKeyEd25519Key(keyInfo): string | undefined { + const ed25519KeyIds = Object.keys(keyInfo.keys).filter(keyId => keyId.startsWith(prefix)); + if (ed25519KeyIds.length !== 1) { + return undefined; + } + const keyId = ed25519KeyIds[0]; + const publicKey = keyInfo.keys[keyId]; + return publicKey; +} + +export function getKeyUserId(keyInfo): string | undefined { + return keyInfo["user_id"]; +} From b8fb2b6df10d5eb5898bd7375f9442d9a7ed9222 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 27 Feb 2023 18:13:53 +0100 Subject: [PATCH 400/435] Store device keys in format needed to sign/verify, convert to TS In order to sign and verify signatures of design keys, we need to have them in the format as they are uploaded and downloaded from the homeserver. So, like the cross-signing keys, we store them in locally in the same format to avoid constant convertions. I also renamed deviceIdentities to deviceKeys, analogue to crossSigningKeys. In order to prevent mistakes in this refactor, I also converted DeviceTracker to typescript. --- src/matrix/Sync.js | 4 +- src/matrix/e2ee/Account.js | 2 +- src/matrix/e2ee/DecryptionResult.ts | 13 +- .../{DeviceTracker.js => DeviceTracker.ts} | 423 ++++++++++-------- src/matrix/e2ee/RoomEncryption.js | 2 +- src/matrix/e2ee/{common.js => common.ts} | 69 ++- src/matrix/e2ee/megolm/keybackup/types.ts | 2 +- src/matrix/e2ee/olm/Encryption.ts | 54 +-- src/matrix/e2ee/olm/types.ts | 4 +- src/matrix/room/BaseRoom.js | 2 +- src/matrix/room/RoomBeingCreated.ts | 4 +- src/matrix/room/common.ts | 2 + src/matrix/storage/common.ts | 2 +- src/matrix/storage/idb/Transaction.ts | 6 +- src/matrix/storage/idb/schema.ts | 9 +- .../idb/stores/CrossSigningKeyStore.ts | 8 +- ...viceIdentityStore.ts => DeviceKeyStore.ts} | 47 +- 17 files changed, 360 insertions(+), 293 deletions(-) rename src/matrix/e2ee/{DeviceTracker.js => DeviceTracker.ts} (71%) rename src/matrix/e2ee/{common.js => common.ts} (57%) rename src/matrix/storage/idb/stores/{DeviceIdentityStore.ts => DeviceKeyStore.ts} (63%) diff --git a/src/matrix/Sync.js b/src/matrix/Sync.js index d335336d..4fb48713 100644 --- a/src/matrix/Sync.js +++ b/src/matrix/Sync.js @@ -218,7 +218,7 @@ export class Sync { _openPrepareSyncTxn() { const storeNames = this._storage.storeNames; return this._storage.readTxn([ - storeNames.deviceIdentities, // to read device from olm messages + storeNames.deviceKeys, // to read device from olm messages storeNames.olmSessions, storeNames.inboundGroupSessions, // to read fragments when loading sync writer when rejoining archived room @@ -329,7 +329,7 @@ export class Sync { storeNames.pendingEvents, storeNames.userIdentities, storeNames.groupSessionDecryptions, - storeNames.deviceIdentities, + storeNames.deviceKeys, // to discard outbound session when somebody leaves a room // and to create room key messages when somebody joins storeNames.outboundGroupSessions, diff --git a/src/matrix/e2ee/Account.js b/src/matrix/e2ee/Account.js index 0238f0cf..b0dd1546 100644 --- a/src/matrix/e2ee/Account.js +++ b/src/matrix/e2ee/Account.js @@ -15,7 +15,7 @@ limitations under the License. */ import anotherjson from "another-json"; -import {SESSION_E2EE_KEY_PREFIX, OLM_ALGORITHM, MEGOLM_ALGORITHM} from "./common.js"; +import {SESSION_E2EE_KEY_PREFIX, OLM_ALGORITHM, MEGOLM_ALGORITHM} from "./common"; // use common prefix so it's easy to clear properties that are not e2ee related during session clear const ACCOUNT_SESSION_KEY = SESSION_E2EE_KEY_PREFIX + "olmAccount"; diff --git a/src/matrix/e2ee/DecryptionResult.ts b/src/matrix/e2ee/DecryptionResult.ts index 83ad7a1e..146a1ad3 100644 --- a/src/matrix/e2ee/DecryptionResult.ts +++ b/src/matrix/e2ee/DecryptionResult.ts @@ -26,7 +26,8 @@ limitations under the License. * see DeviceTracker */ -import type {DeviceIdentity} from "../storage/idb/stores/DeviceIdentityStore"; +import {getDeviceEd25519Key} from "./common"; +import type {DeviceKey} from "./common"; import type {TimelineEvent} from "../storage/types"; type DecryptedEvent = { @@ -35,7 +36,7 @@ type DecryptedEvent = { } export class DecryptionResult { - private device?: DeviceIdentity; + private device?: DeviceKey; constructor( public readonly event: DecryptedEvent, @@ -44,13 +45,13 @@ export class DecryptionResult { public readonly encryptedEvent?: TimelineEvent ) {} - setDevice(device: DeviceIdentity): void { + setDevice(device: DeviceKey): void { this.device = device; } get isVerified(): boolean { if (this.device) { - const comesFromDevice = this.device.ed25519Key === this.claimedEd25519Key; + const comesFromDevice = getDeviceEd25519Key(this.device) === this.claimedEd25519Key; return comesFromDevice; } return false; @@ -65,11 +66,11 @@ export class DecryptionResult { } get userId(): string | undefined { - return this.device?.userId; + return this.device?.user_id; } get deviceId(): string | undefined { - return this.device?.deviceId; + return this.device?.device_id; } get isVerificationUnknown(): boolean { diff --git a/src/matrix/e2ee/DeviceTracker.js b/src/matrix/e2ee/DeviceTracker.ts similarity index 71% rename from src/matrix/e2ee/DeviceTracker.js rename to src/matrix/e2ee/DeviceTracker.ts index 5d991fcb..c8e9df09 100644 --- a/src/matrix/e2ee/DeviceTracker.js +++ b/src/matrix/e2ee/DeviceTracker.ts @@ -15,23 +15,38 @@ limitations under the License. */ import {verifyEd25519Signature, getEd25519Signature, SIGNATURE_ALGORITHM} from "./common.js"; -import {HistoryVisibility, shouldShareKey} from "./common.js"; +import {HistoryVisibility, shouldShareKey, DeviceKey, getDeviceEd25519Key, getDeviceCurve25519Key} from "./common"; import {RoomMember} from "../room/members/RoomMember.js"; import {getKeyUsage, getKeyEd25519Key, getKeyUserId, KeyUsage} from "../verification/CrossSigning"; +import {MemberChange} from "../room/members/RoomMember"; +import type {CrossSigningKey} from "../storage/idb/stores/CrossSigningKeyStore"; +import type {HomeServerApi} from "../net/HomeServerApi"; +import type {ObservableMap} from "../../observable/map"; +import type {Room} from "../room/Room"; +import type {ILogItem} from "../../logging/types"; +import type {Storage} from "../storage/idb/Storage"; +import type {Transaction} from "../storage/idb/Transaction"; +import type * as OlmNamespace from "@matrix-org/olm"; +type Olm = typeof OlmNamespace; const TRACKING_STATUS_OUTDATED = 0; const TRACKING_STATUS_UPTODATE = 1; -function createUserIdentity(userId, initialRoomId = undefined) { +export type UserIdentity = { + userId: string, + roomIds: string[], + deviceTrackingStatus: number, +} + +function createUserIdentity(userId: string, initialRoomId?: string): UserIdentity { return { userId: userId, roomIds: initialRoomId ? [initialRoomId] : [], - crossSigningKeys: undefined, deviceTrackingStatus: TRACKING_STATUS_OUTDATED, }; } -function addRoomToIdentity(identity, userId, roomId) { +function addRoomToIdentity(identity: UserIdentity | undefined, userId: string, roomId: string): UserIdentity | undefined { if (!identity) { identity = createUserIdentity(userId, roomId); return identity; @@ -43,31 +58,22 @@ function addRoomToIdentity(identity, userId, roomId) { } } -// map 1 device from /keys/query response to DeviceIdentity -function deviceKeysAsDeviceIdentity(deviceSection) { - const deviceId = deviceSection["device_id"]; - const userId = deviceSection["user_id"]; - return { - userId, - deviceId, - ed25519Key: deviceSection.keys[`ed25519:${deviceId}`], - curve25519Key: deviceSection.keys[`curve25519:${deviceId}`], - algorithms: deviceSection.algorithms, - displayName: deviceSection.unsigned?.device_display_name, - }; -} - export class DeviceTracker { - constructor({storage, getSyncToken, olmUtil, ownUserId, ownDeviceId}) { - this._storage = storage; - this._getSyncToken = getSyncToken; - this._identityChangedForRoom = null; - this._olmUtil = olmUtil; - this._ownUserId = ownUserId; - this._ownDeviceId = ownDeviceId; + private readonly _storage: Storage; + private readonly _getSyncToken: () => string; + private readonly _olmUtil: Olm.Utility; + private readonly _ownUserId: string; + private readonly _ownDeviceId: string; + + constructor(options: {storage: Storage, getSyncToken: () => string, olmUtil: Olm.Utility, ownUserId: string, ownDeviceId: string}) { + this._storage = options.storage; + this._getSyncToken = options.getSyncToken; + this._olmUtil = options.olmUtil; + this._ownUserId = options.ownUserId; + this._ownDeviceId = options.ownDeviceId; } - async writeDeviceChanges(changed, txn, log) { + async writeDeviceChanges(changedUserIds: ReadonlyArray, txn: Transaction, log: ILogItem): Promise { const {userIdentities} = txn; // TODO: should we also look at left here to handle this?: // the usual problem here is that you share a room with a user, @@ -76,8 +82,8 @@ export class DeviceTracker { // At which point you come online, all of this happens in the gap, // and you don't notice that they ever left, // and so the client doesn't invalidate their device cache for the user - log.set("changed", changed.length); - await Promise.all(changed.map(async userId => { + log.set("changed", changedUserIds.length); + await Promise.all(changedUserIds.map(async userId => { const user = await userIdentities.get(userId); if (user) { log.log({l: "outdated", id: userId}); @@ -90,9 +96,9 @@ export class DeviceTracker { /** @return Promise<{added: string[], removed: string[]}> the user ids for who the room was added or removed to the userIdentity, * and with who a key should be now be shared **/ - async writeMemberChanges(room, memberChanges, historyVisibility, txn) { - const added = []; - const removed = []; + async writeMemberChanges(room: Room, memberChanges: Map, historyVisibility: HistoryVisibility, txn: Transaction): Promise<{added: string[], removed: string[]}> { + const added: string[] = []; + const removed: string[] = []; await Promise.all(Array.from(memberChanges.values()).map(async memberChange => { // keys should now be shared with this member? // add the room to the userIdentity if so @@ -118,7 +124,7 @@ export class DeviceTracker { return {added, removed}; } - async trackRoom(room, historyVisibility, log) { + async trackRoom(room: Room, historyVisibility: HistoryVisibility, log: ILogItem): Promise { if (room.isTrackingMembers || !room.isEncrypted) { return; } @@ -126,13 +132,13 @@ export class DeviceTracker { const txn = await this._storage.readWriteTxn([ this._storage.storeNames.roomSummary, this._storage.storeNames.userIdentities, - this._storage.storeNames.deviceIdentities, // to remove all devices in _removeRoomFromUserIdentity + this._storage.storeNames.deviceKeys, // to remove all devices in _removeRoomFromUserIdentity ]); try { let isTrackingChanges; try { isTrackingChanges = room.writeIsTrackingMembers(true, txn); - const members = Array.from(memberList.members.values()); + const members = Array.from((memberList.members as ObservableMap).values()); log.set("members", members.length); // TODO: should we remove any userIdentities we should not share the key with?? // e.g. as an extra security measure if we had a mistake in other code? @@ -154,14 +160,15 @@ export class DeviceTracker { } } - async getCrossSigningKeyForUser(userId, usage, hsApi, log) { - return await log.wrap("DeviceTracker.getMasterKeyForUser", async log => { + async getCrossSigningKeyForUser(userId: string, usage: KeyUsage, hsApi: HomeServerApi, log: ILogItem) { + return await log.wrap({l: "DeviceTracker.getCrossSigningKeyForUser", id: userId, usage}, async log => { let txn = await this._storage.readTxn([ - this._storage.storeNames.userIdentities + this._storage.storeNames.userIdentities, + this._storage.storeNames.crossSigningKeys, ]); let userIdentity = await txn.userIdentities.get(userId); if (userIdentity && userIdentity.deviceTrackingStatus !== TRACKING_STATUS_OUTDATED) { - return userIdentity.crossSigningKeys; + return await txn.crossSigningKeys.get(userId, usage); } // fetch from hs const keys = await this._queryKeys([userId], hsApi, log); @@ -172,19 +179,19 @@ export class DeviceTracker { return keys.selfSigningKeys.get(userId); case KeyUsage.UserSigning: return keys.userSigningKeys.get(userId); - } }); } - async writeHistoryVisibility(room, historyVisibility, syncTxn, log) { - const added = []; - const removed = []; + async writeHistoryVisibility(room: Room, historyVisibility: HistoryVisibility, syncTxn: Transaction, log: ILogItem): Promise<{added: string[], removed: string[]}> { + const added: string[] = []; + const removed: string[] = []; if (room.isTrackingMembers && room.isEncrypted) { await log.wrap("rewriting userIdentities", async log => { + // TODO: how do we know that we won't fetch the members from the server here and hence close the syncTxn? const memberList = await room.loadMemberList(syncTxn, log); try { - const members = Array.from(memberList.members.values()); + const members = Array.from((memberList.members as ObservableMap).values()); log.set("members", members.length); await Promise.all(members.map(async member => { if (shouldShareKey(member.membership, historyVisibility)) { @@ -205,7 +212,7 @@ export class DeviceTracker { return {added, removed}; } - async _addRoomToUserIdentity(roomId, userId, txn) { + async _addRoomToUserIdentity(roomId: string, userId: string, txn: Transaction): Promise { const {userIdentities} = txn; const identity = await userIdentities.get(userId); const updatedIdentity = addRoomToIdentity(identity, userId, roomId); @@ -216,15 +223,15 @@ export class DeviceTracker { return false; } - async _removeRoomFromUserIdentity(roomId, userId, txn) { - const {userIdentities, deviceIdentities} = txn; + async _removeRoomFromUserIdentity(roomId: string, userId: string, txn: Transaction): Promise { + const {userIdentities, deviceKeys} = txn; const identity = await userIdentities.get(userId); if (identity) { identity.roomIds = identity.roomIds.filter(id => id !== roomId); // no more encrypted rooms with this user, remove if (identity.roomIds.length === 0) { userIdentities.remove(userId); - deviceIdentities.removeAllForUser(userId); + deviceKeys.removeAllForUser(userId); } else { userIdentities.set(identity); } @@ -233,7 +240,12 @@ export class DeviceTracker { return false; } - async _queryKeys(userIds, hsApi, log) { + async _queryKeys(userIds: string[], hsApi: HomeServerApi, log: ILogItem): Promise<{ + deviceKeys: Map, + masterKeys: Map, + selfSigningKeys: Map, + userSigningKeys: Map + }> { // TODO: we need to handle the race here between /sync and /keys/query just like we need to do for the member list ... // there are multiple requests going out for /keys/query though and only one for /members // So, while doing /keys/query, writeDeviceChanges should add userIds marked as outdated to a list @@ -252,10 +264,10 @@ export class DeviceTracker { const masterKeys = log.wrap("master keys", log => this._filterVerifiedCrossSigningKeys(deviceKeyResponse["master_keys"], KeyUsage.Master, undefined, log)); const selfSigningKeys = log.wrap("self-signing keys", log => this._filterVerifiedCrossSigningKeys(deviceKeyResponse["self_signing_keys"], KeyUsage.SelfSigning, masterKeys, log)); const userSigningKeys = log.wrap("user-signing keys", log => this._filterVerifiedCrossSigningKeys(deviceKeyResponse["user_signing_keys"], KeyUsage.UserSigning, masterKeys, log)); - const verifiedKeysPerUser = log.wrap("device keys", log => this._filterVerifiedDeviceKeys(deviceKeyResponse["device_keys"], log)); + const deviceKeys = log.wrap("device keys", log => this._filterVerifiedDeviceKeys(deviceKeyResponse["device_keys"], log)); const txn = await this._storage.readWriteTxn([ this._storage.storeNames.userIdentities, - this._storage.storeNames.deviceIdentities, + this._storage.storeNames.deviceKeys, this._storage.storeNames.crossSigningKeys, ]); let deviceIdentities; @@ -269,54 +281,59 @@ export class DeviceTracker { for (const key of userSigningKeys.values()) { txn.crossSigningKeys.set(key); } - const devicesIdentitiesPerUser = await Promise.all(verifiedKeysPerUser.map(async ({userId, verifiedKeys}) => { - const deviceIdentities = verifiedKeys.map(deviceKeysAsDeviceIdentity); - return await this._storeQueriedDevicesForUserId(userId, deviceIdentities, txn); + let totalCount = 0; + await Promise.all(Array.from(deviceKeys.keys()).map(async (userId) => { + let deviceKeysForUser = deviceKeys.get(userId)!; + totalCount += deviceKeysForUser.length; + // check for devices that changed their keys and keep the old key + deviceKeysForUser = await this._storeQueriedDevicesForUserId(userId, deviceKeysForUser, txn); + deviceKeys.set(userId, deviceKeysForUser); })); - deviceIdentities = devicesIdentitiesPerUser.reduce((all, devices) => all.concat(devices), []); - log.set("devices", deviceIdentities.length); + log.set("devices", totalCount); } catch (err) { txn.abort(); throw err; } await txn.complete(); return { - deviceIdentities, + deviceKeys, masterKeys, selfSigningKeys, userSigningKeys }; } - async _storeQueriedDevicesForUserId(userId, deviceIdentities, txn) { - const knownDeviceIds = await txn.deviceIdentities.getAllDeviceIds(userId); + async _storeQueriedDevicesForUserId(userId: string, deviceKeys: DeviceKey[], txn: Transaction): Promise { + // TODO: we should obsolete (flag) the device keys that have been removed, + // but keep them to verify messages encrypted with it? + const knownDeviceIds = await txn.deviceKeys.getAllDeviceIds(userId); // delete any devices that we know off but are not in the response anymore. // important this happens before checking if the ed25519 key changed, // otherwise we would end up deleting existing devices with changed keys. for (const deviceId of knownDeviceIds) { - if (deviceIdentities.every(di => di.deviceId !== deviceId)) { - txn.deviceIdentities.remove(userId, deviceId); + if (deviceKeys.every(di => di.device_id !== deviceId)) { + txn.deviceKeys.remove(userId, deviceId); } } // all the device identities as we will have them in storage - const allDeviceIdentities = []; - const deviceIdentitiesToStore = []; + const allDeviceKeys: DeviceKey[] = []; + const deviceKeysToStore: DeviceKey[] = []; // filter out devices that have changed their ed25519 key since last time we queried them - await Promise.all(deviceIdentities.map(async deviceIdentity => { - if (knownDeviceIds.includes(deviceIdentity.deviceId)) { - const existingDevice = await txn.deviceIdentities.get(deviceIdentity.userId, deviceIdentity.deviceId); - if (existingDevice.ed25519Key !== deviceIdentity.ed25519Key) { - allDeviceIdentities.push(existingDevice); + await Promise.all(deviceKeys.map(async deviceKey => { + if (knownDeviceIds.includes(deviceKey.device_id)) { + const existingDevice = await txn.deviceKeys.get(deviceKey.user_id, deviceKey.device_id); + if (existingDevice && getDeviceEd25519Key(existingDevice) !== getDeviceEd25519Key(deviceKey)) { + allDeviceKeys.push(existingDevice); return; } } - allDeviceIdentities.push(deviceIdentity); - deviceIdentitiesToStore.push(deviceIdentity); + allDeviceKeys.push(deviceKey); + deviceKeysToStore.push(deviceKey); })); // store devices - for (const deviceIdentity of deviceIdentitiesToStore) { - txn.deviceIdentities.set(deviceIdentity); + for (const deviceKey of deviceKeysToStore) { + txn.deviceKeys.set(deviceKey); } // mark user identities as up to date let identity = await txn.userIdentities.get(userId); @@ -331,11 +348,11 @@ export class DeviceTracker { identity.deviceTrackingStatus = TRACKING_STATUS_UPTODATE; txn.userIdentities.set(identity); - return allDeviceIdentities; + return allDeviceKeys; } - _filterVerifiedCrossSigningKeys(crossSigningKeysResponse, usage, parentKeys, log) { - const keys = new Map(); + _filterVerifiedCrossSigningKeys(crossSigningKeysResponse: {[userId: string]: CrossSigningKey}, usage, parentKeys: Map | undefined, log): Map { + const keys: Map = new Map(); if (!crossSigningKeysResponse) { return keys; } @@ -344,14 +361,14 @@ export class DeviceTracker { const parentKeyInfo = parentKeys?.get(userId); const parentKey = parentKeyInfo && getKeyEd25519Key(parentKeyInfo); if (this._validateCrossSigningKey(userId, keyInfo, usage, parentKey, log)) { - keys.set(getKeyUserId(keyInfo), keyInfo); + keys.set(getKeyUserId(keyInfo)!, keyInfo); } }); } return keys; } - _validateCrossSigningKey(userId, keyInfo, usage, parentKey, log) { + _validateCrossSigningKey(userId: string, keyInfo: CrossSigningKey, usage: KeyUsage, parentKey: string | undefined, log: ILogItem): boolean { if (getKeyUserId(keyInfo) !== userId) { log.log({l: "user_id mismatch", userId: keyInfo["user_id"]}); return false; @@ -389,51 +406,67 @@ export class DeviceTracker { /** * @return {Array<{userId, verifiedKeys: Array>} */ - _filterVerifiedDeviceKeys(keyQueryDeviceKeysResponse, parentLog) { - const curve25519Keys = new Set(); - const verifiedKeys = Object.entries(keyQueryDeviceKeysResponse).map(([userId, keysByDevice]) => { - const verifiedEntries = Object.entries(keysByDevice).filter(([deviceId, deviceKeys]) => { - const deviceIdOnKeys = deviceKeys["device_id"]; - const userIdOnKeys = deviceKeys["user_id"]; - if (userIdOnKeys !== userId) { - return false; - } - if (deviceIdOnKeys !== deviceId) { - return false; - } - const ed25519Key = deviceKeys.keys?.[`ed25519:${deviceId}`]; - const curve25519Key = deviceKeys.keys?.[`curve25519:${deviceId}`]; - if (typeof ed25519Key !== "string" || typeof curve25519Key !== "string") { - return false; - } - if (curve25519Keys.has(curve25519Key)) { - parentLog.log({ - l: "ignore device with duplicate curve25519 key", - keys: deviceKeys - }, parentLog.level.Warn); - return false; - } - curve25519Keys.add(curve25519Key); - const isValid = this._hasValidSignature(deviceKeys, parentLog); - if (!isValid) { - parentLog.log({ - l: "ignore device with invalid signature", - keys: deviceKeys - }, parentLog.level.Warn); - } - return isValid; + _filterVerifiedDeviceKeys( + keyQueryDeviceKeysResponse: {[userId: string]: {[deviceId: string]: DeviceKey}}, + parentLog: ILogItem + ): Map { + const curve25519Keys: Set = new Set(); + const keys: Map = new Map(); + if (!keyQueryDeviceKeysResponse) { + return keys; + } + for (const [userId, keysByDevice] of Object.entries(keyQueryDeviceKeysResponse)) { + parentLog.wrap(userId, log => { + const verifiedEntries = Object.entries(keysByDevice).filter(([deviceId, deviceKey]) => { + return log.wrap(deviceId, log => { + if (this._validateDeviceKey(userId, deviceId, deviceKey, log)) { + const curve25519Key = getDeviceCurve25519Key(deviceKey); + if (curve25519Keys.has(curve25519Key)) { + parentLog.log({ + l: "ignore device with duplicate curve25519 key", + keys: deviceKey + }, parentLog.level.Warn); + return false; + } + curve25519Keys.add(curve25519Key); + return true; + } else { + return false; + } + }); + }); + const verifiedKeys = verifiedEntries.map(([, deviceKeys]) => deviceKeys); + keys.set(userId, verifiedKeys); }); - const verifiedKeys = verifiedEntries.map(([, deviceKeys]) => deviceKeys); - return {userId, verifiedKeys}; - }); - return verifiedKeys; + } + return keys; } - _hasValidSignature(deviceSection, parentLog) { - const deviceId = deviceSection["device_id"]; - const userId = deviceSection["user_id"]; - const ed25519Key = deviceSection?.keys?.[`${SIGNATURE_ALGORITHM}:${deviceId}`]; - return verifyEd25519Signature(this._olmUtil, userId, deviceId, ed25519Key, deviceSection, parentLog); + _validateDeviceKey(userIdFromServer: string, deviceIdFromServer: string, deviceKey: DeviceKey, log: ILogItem): boolean { + const deviceId = deviceKey["device_id"]; + const userId = deviceKey["user_id"]; + if (userId !== userIdFromServer) { + log.log("user_id mismatch"); + return false; + } + if (deviceId !== deviceIdFromServer) { + log.log("device_id mismatch"); + return false; + } + const ed25519Key = getDeviceEd25519Key(deviceKey); + const curve25519Key = getDeviceCurve25519Key(deviceKey); + if (typeof ed25519Key !== "string" || typeof curve25519Key !== "string") { + log.log("ed25519 and/or curve25519 key invalid").set({deviceKey}); + return false; + } + const isValid = verifyEd25519Signature(this._olmUtil, userId, deviceId, ed25519Key, deviceKey, log); + if (!isValid) { + log.log({ + l: "ignore device with invalid signature", + keys: deviceKey + }, log.level.Warn); + } + return isValid; } /** @@ -443,7 +476,7 @@ export class DeviceTracker { * @param {String} roomId [description] * @return {[type]} [description] */ - async devicesForTrackedRoom(roomId, hsApi, log) { + async devicesForTrackedRoom(roomId: string, hsApi: HomeServerApi, log: ILogItem): Promise { const txn = await this._storage.readTxn([ this._storage.storeNames.roomMembers, this._storage.storeNames.userIdentities, @@ -463,7 +496,7 @@ export class DeviceTracker { * Can be used to decide which users to share keys with. * Assumes room is already tracked. Call `trackRoom` first if unsure. */ - async devicesForRoomMembers(roomId, userIds, hsApi, log) { + async devicesForRoomMembers(roomId: string, userIds: string[], hsApi: HomeServerApi, log: ILogItem): Promise { const txn = await this._storage.readTxn([ this._storage.storeNames.userIdentities, ]); @@ -474,13 +507,13 @@ export class DeviceTracker { * Cannot be used to decide which users to share keys with. * Does not assume membership to any room or whether any room is tracked. */ - async devicesForUsers(userIds, hsApi, log) { + async devicesForUsers(userIds: string[], hsApi: HomeServerApi, log: ILogItem): Promise { const txn = await this._storage.readTxn([ this._storage.storeNames.userIdentities, ]); - const upToDateIdentities = []; - const outdatedUserIds = []; + const upToDateIdentities: UserIdentity[] = []; + const outdatedUserIds: string[] = []; await Promise.all(userIds.map(async userId => { const i = await txn.userIdentities.get(userId); if (i && i.deviceTrackingStatus === TRACKING_STATUS_UPTODATE) { @@ -495,12 +528,12 @@ export class DeviceTracker { } /** gets a single device */ - async deviceForId(userId, deviceId, hsApi, log) { + async deviceForId(userId: string, deviceId: string, hsApi: HomeServerApi, log: ILogItem) { const txn = await this._storage.readTxn([ - this._storage.storeNames.deviceIdentities, + this._storage.storeNames.deviceKeys, ]); - let device = await txn.deviceIdentities.get(userId, deviceId); - if (device) { + let deviceKey = await txn.deviceKeys.get(userId, deviceId); + if (deviceKey) { log.set("existingDevice", true); } else { //// BEGIN EXTRACT (deviceKeysMap) @@ -514,29 +547,26 @@ export class DeviceTracker { // verify signature const verifiedKeysPerUser = log.wrap("verify", log => this._filterVerifiedDeviceKeys(deviceKeyResponse["device_keys"], log)); //// END EXTRACT - // TODO: what if verifiedKeysPerUser is empty or does not contain userId? - const verifiedKeys = verifiedKeysPerUser - .find(vkpu => vkpu.userId === userId).verifiedKeys - .find(vk => vk["device_id"] === deviceId); + const verifiedKey = verifiedKeysPerUser.get(userId)?.find(d => d.device_id === deviceId); // user hasn't uploaded keys for device? - if (!verifiedKeys) { + if (!verifiedKey) { return undefined; } - device = deviceKeysAsDeviceIdentity(verifiedKeys); const txn = await this._storage.readWriteTxn([ - this._storage.storeNames.deviceIdentities, + this._storage.storeNames.deviceKeys, ]); // check again we don't have the device already. // when updating all keys for a user we allow updating the // device when the key hasn't changed so the device display name // can be updated, but here we don't. - const existingDevice = await txn.deviceIdentities.get(userId, deviceId); + const existingDevice = await txn.deviceKeys.get(userId, deviceId); if (existingDevice) { - device = existingDevice; + deviceKey = existingDevice; log.set("existingDeviceAfterFetch", true); } else { try { - txn.deviceIdentities.set(device); + txn.deviceKeys.set(verifiedKey); + deviceKey = verifiedKey; log.set("newDevice", true); } catch (err) { txn.abort(); @@ -545,7 +575,7 @@ export class DeviceTracker { await txn.complete(); } } - return device; + return deviceKey; } /** @@ -555,9 +585,9 @@ export class DeviceTracker { * @param {Array} userIds a set of user ids to try and find the identity for. * @param {Transaction} userIdentityTxn to read the user identities * @param {HomeServerApi} hsApi - * @return {Array} all devices identities for the given users we should share keys with. + * @return {Array} all devices identities for the given users we should share keys with. */ - async _devicesForUserIdsInTrackedRoom(roomId, userIds, userIdentityTxn, hsApi, log) { + async _devicesForUserIdsInTrackedRoom(roomId: string, userIds: string[], userIdentityTxn: Transaction, hsApi: HomeServerApi, log: ILogItem): Promise { const allMemberIdentities = await Promise.all(userIds.map(userId => userIdentityTxn.userIdentities.get(userId))); const identities = allMemberIdentities.filter(identity => { // we use roomIds to decide with whom we should share keys for a given room, @@ -566,7 +596,7 @@ export class DeviceTracker { // Given we assume the room is tracked, // also exclude any userId which doesn't have a userIdentity yet. return identity && identity.roomIds.includes(roomId); - }); + }) as UserIdentity[]; // undefined has been filter out const upToDateIdentities = identities.filter(i => i.deviceTrackingStatus === TRACKING_STATUS_UPTODATE); const outdatedUserIds = identities .filter(i => i.deviceTrackingStatus === TRACKING_STATUS_OUTDATED) @@ -574,7 +604,7 @@ export class DeviceTracker { let devices = await this._devicesForUserIdentities(upToDateIdentities, outdatedUserIds, hsApi, log); // filter out our own device as we should never share keys with it. devices = devices.filter(device => { - const isOwnDevice = device.userId === this._ownUserId && device.deviceId === this._ownDeviceId; + const isOwnDevice = device.user_id === this._ownUserId && device.device_id === this._ownDeviceId; return !isOwnDevice; }); return devices; @@ -584,43 +614,44 @@ export class DeviceTracker { * are known to be up to date, and a set of userIds that are known * to be absent from our store our outdated. The outdated user ids * will have their keys fetched from the homeserver. */ - async _devicesForUserIdentities(upToDateIdentities, outdatedUserIds, hsApi, log) { + async _devicesForUserIdentities(upToDateIdentities: UserIdentity[], outdatedUserIds: string[], hsApi: HomeServerApi, log: ILogItem): Promise { log.set("uptodate", upToDateIdentities.length); log.set("outdated", outdatedUserIds.length); - let queriedDevices; + let queriedDeviceKeys: Map | undefined; if (outdatedUserIds.length) { // TODO: ignore the race between /sync and /keys/query for now, // where users could get marked as outdated or added/removed from the room while // querying keys - const {deviceIdentities} = await this._queryKeys(outdatedUserIds, hsApi, log); - queriedDevices = deviceIdentities; + const {deviceKeys} = await this._queryKeys(outdatedUserIds, hsApi, log); + queriedDeviceKeys = deviceKeys; } const deviceTxn = await this._storage.readTxn([ - this._storage.storeNames.deviceIdentities, + this._storage.storeNames.deviceKeys, ]); const devicesPerUser = await Promise.all(upToDateIdentities.map(identity => { - return deviceTxn.deviceIdentities.getAllForUserId(identity.userId); + return deviceTxn.deviceKeys.getAllForUserId(identity.userId); })); let flattenedDevices = devicesPerUser.reduce((all, devicesForUser) => all.concat(devicesForUser), []); - if (queriedDevices && queriedDevices.length) { - flattenedDevices = flattenedDevices.concat(queriedDevices); + if (queriedDeviceKeys && queriedDeviceKeys.size) { + for (const deviceKeysForUser of queriedDeviceKeys.values()) { + flattenedDevices = flattenedDevices.concat(deviceKeysForUser); + } } return flattenedDevices; } - async getDeviceByCurve25519Key(curve25519Key, txn) { - return await txn.deviceIdentities.getByCurve25519Key(curve25519Key); + async getDeviceByCurve25519Key(curve25519Key, txn: Transaction): Promise { + return await txn.deviceKeys.getByCurve25519Key(curve25519Key); } } import {createMockStorage} from "../../mocks/Storage"; import {Instance as NullLoggerInstance} from "../../logging/NullLogger"; -import {MemberChange} from "../room/members/RoomMember"; export function tests() { - function createUntrackedRoomMock(roomId, joinedUserIds, invitedUserIds = []) { + function createUntrackedRoomMock(roomId: string, joinedUserIds: string[], invitedUserIds: string[] = []) { return { id: roomId, isTrackingMembers: false, @@ -649,11 +680,11 @@ export function tests() { } } - function createQueryKeysHSApiMock(createKey = (algorithm, userId, deviceId) => `${algorithm}:${userId}:${deviceId}:key`) { + function createQueryKeysHSApiMock(createKey = (algorithm, userId, deviceId) => `${algorithm}:${userId}:${deviceId}:key`): HomeServerApi { return { queryKeys(payload) { const {device_keys: deviceKeys} = payload; - const userKeys = Object.entries(deviceKeys).reduce((userKeys, [userId, deviceIds]) => { + const userKeys = Object.entries(deviceKeys as {[userId: string]: string[]}).reduce((userKeys, [userId, deviceIds]) => { if (deviceIds.length === 0) { deviceIds = ["device1"]; } @@ -689,7 +720,7 @@ export function tests() { } }; } - }; + } as unknown as HomeServerApi; } async function writeMemberListToStorage(room, storage) { @@ -718,7 +749,7 @@ export function tests() { const tracker = new DeviceTracker({ storage, getSyncToken: () => "token", - olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw + olmUtil: {ed25519_verify: () => {}} as unknown as Olm.Utility, // valid if it does not throw ownUserId: "@alice:hs.tld", ownDeviceId: "ABCD", }); @@ -727,14 +758,12 @@ export function tests() { const txn = await storage.readTxn([storage.storeNames.userIdentities]); assert.deepEqual(await txn.userIdentities.get("@alice:hs.tld"), { userId: "@alice:hs.tld", - crossSigningKeys: undefined, roomIds: [roomId], deviceTrackingStatus: TRACKING_STATUS_OUTDATED }); assert.deepEqual(await txn.userIdentities.get("@bob:hs.tld"), { userId: "@bob:hs.tld", roomIds: [roomId], - crossSigningKeys: undefined, deviceTrackingStatus: TRACKING_STATUS_OUTDATED }); assert.equal(await txn.userIdentities.get("@charly:hs.tld"), undefined); @@ -744,7 +773,7 @@ export function tests() { const tracker = new DeviceTracker({ storage, getSyncToken: () => "token", - olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw + olmUtil: {ed25519_verify: () => {}} as unknown as Olm.Utility, // valid if it does not throw ownUserId: "@alice:hs.tld", ownDeviceId: "ABCD", }); @@ -753,15 +782,15 @@ export function tests() { const hsApi = createQueryKeysHSApiMock(); const devices = await tracker.devicesForRoomMembers(roomId, ["@alice:hs.tld", "@bob:hs.tld"], hsApi, NullLoggerInstance.item); assert.equal(devices.length, 2); - assert.equal(devices.find(d => d.userId === "@alice:hs.tld").ed25519Key, "ed25519:@alice:hs.tld:device1:key"); - assert.equal(devices.find(d => d.userId === "@bob:hs.tld").ed25519Key, "ed25519:@bob:hs.tld:device1:key"); + assert.equal(getDeviceEd25519Key(devices.find(d => d.user_id === "@alice:hs.tld")!), "ed25519:@alice:hs.tld:device1:key"); + assert.equal(getDeviceEd25519Key(devices.find(d => d.user_id === "@bob:hs.tld")!), "ed25519:@bob:hs.tld:device1:key"); }, "device with changed key is ignored": async assert => { const storage = await createMockStorage(); const tracker = new DeviceTracker({ storage, getSyncToken: () => "token", - olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw + olmUtil: {ed25519_verify: () => {}} as unknown as Olm.Utility, // valid if it does not throw ownUserId: "@alice:hs.tld", ownDeviceId: "ABCD", }); @@ -779,18 +808,18 @@ export function tests() { }); const devices = await tracker.devicesForRoomMembers(roomId, ["@alice:hs.tld", "@bob:hs.tld"], hsApiWithChangedAliceKey, NullLoggerInstance.item); assert.equal(devices.length, 2); - assert.equal(devices.find(d => d.userId === "@alice:hs.tld").ed25519Key, "ed25519:@alice:hs.tld:device1:key"); - assert.equal(devices.find(d => d.userId === "@bob:hs.tld").ed25519Key, "ed25519:@bob:hs.tld:device1:key"); - const txn2 = await storage.readTxn([storage.storeNames.deviceIdentities]); + assert.equal(getDeviceEd25519Key(devices.find(d => d.user_id === "@alice:hs.tld")!), "ed25519:@alice:hs.tld:device1:key"); + assert.equal(getDeviceEd25519Key(devices.find(d => d.user_id === "@bob:hs.tld")!), "ed25519:@bob:hs.tld:device1:key"); + const txn2 = await storage.readTxn([storage.storeNames.deviceKeys]); // also check the modified key was not stored - assert.equal((await txn2.deviceIdentities.get("@alice:hs.tld", "device1")).ed25519Key, "ed25519:@alice:hs.tld:device1:key"); + assert.equal(getDeviceEd25519Key((await txn2.deviceKeys.get("@alice:hs.tld", "device1"))!), "ed25519:@alice:hs.tld:device1:key"); }, "change history visibility from joined to invited adds invitees": async assert => { const storage = await createMockStorage(); const tracker = new DeviceTracker({ storage, getSyncToken: () => "token", - olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw + olmUtil: {ed25519_verify: () => {}} as unknown as Olm.Utility, // valid if it does not throw ownUserId: "@alice:hs.tld", ownDeviceId: "ABCD", }); @@ -798,10 +827,10 @@ export function tests() { const room = await createUntrackedRoomMock(roomId, ["@alice:hs.tld"], ["@bob:hs.tld"]); await tracker.trackRoom(room, HistoryVisibility.Joined, NullLoggerInstance.item); - const txn = await storage.readWriteTxn([storage.storeNames.userIdentities, storage.storeNames.deviceIdentities]); + const txn = await storage.readWriteTxn([storage.storeNames.userIdentities, storage.storeNames.deviceKeys]); assert.equal(await txn.userIdentities.get("@bob:hs.tld"), undefined); const {added, removed} = await tracker.writeHistoryVisibility(room, HistoryVisibility.Invited, txn, NullLoggerInstance.item); - assert.equal((await txn.userIdentities.get("@bob:hs.tld")).userId, "@bob:hs.tld"); + assert.equal((await txn.userIdentities.get("@bob:hs.tld"))!.userId, "@bob:hs.tld"); assert.deepEqual(added, ["@bob:hs.tld"]); assert.deepEqual(removed, []); }, @@ -810,7 +839,7 @@ export function tests() { const tracker = new DeviceTracker({ storage, getSyncToken: () => "token", - olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw + olmUtil: {ed25519_verify: () => {}} as unknown as Olm.Utility, // valid if it does not throw ownUserId: "@alice:hs.tld", ownDeviceId: "ABCD", }); @@ -818,8 +847,8 @@ export function tests() { const room = await createUntrackedRoomMock(roomId, ["@alice:hs.tld"], ["@bob:hs.tld"]); await tracker.trackRoom(room, HistoryVisibility.Invited, NullLoggerInstance.item); - const txn = await storage.readWriteTxn([storage.storeNames.userIdentities, storage.storeNames.deviceIdentities]); - assert.equal((await txn.userIdentities.get("@bob:hs.tld")).userId, "@bob:hs.tld"); + const txn = await storage.readWriteTxn([storage.storeNames.userIdentities, storage.storeNames.deviceKeys]); + assert.equal((await txn.userIdentities.get("@bob:hs.tld"))!.userId, "@bob:hs.tld"); const {added, removed} = await tracker.writeHistoryVisibility(room, HistoryVisibility.Joined, txn, NullLoggerInstance.item); assert.equal(await txn.userIdentities.get("@bob:hs.tld"), undefined); assert.deepEqual(added, []); @@ -830,32 +859,32 @@ export function tests() { const tracker = new DeviceTracker({ storage, getSyncToken: () => "token", - olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw + olmUtil: {ed25519_verify: () => {}} as unknown as Olm.Utility, // valid if it does not throw ownUserId: "@alice:hs.tld", ownDeviceId: "ABCD", }); const room = await createUntrackedRoomMock(roomId, ["@alice:hs.tld"]); await tracker.trackRoom(room, HistoryVisibility.Invited, NullLoggerInstance.item); - const txn = await storage.readWriteTxn([storage.storeNames.userIdentities, storage.storeNames.deviceIdentities]); + const txn = await storage.readWriteTxn([storage.storeNames.userIdentities, storage.storeNames.deviceKeys]); // inviting a new member const inviteChange = new MemberChange(RoomMember.fromUserId(roomId, "@bob:hs.tld", "invite")); - const {added, removed} = await tracker.writeMemberChanges(room, [inviteChange], HistoryVisibility.Invited, txn); + const {added, removed} = await tracker.writeMemberChanges(room, new Map([[inviteChange.userId, inviteChange]]), HistoryVisibility.Invited, txn); assert.deepEqual(added, ["@bob:hs.tld"]); assert.deepEqual(removed, []); - assert.equal((await txn.userIdentities.get("@bob:hs.tld")).userId, "@bob:hs.tld"); + assert.equal((await txn.userIdentities.get("@bob:hs.tld"))!.userId, "@bob:hs.tld"); }, "adding invitee with history visibility of joined doesn't add room": async assert => { const storage = await createMockStorage(); const tracker = new DeviceTracker({ storage, getSyncToken: () => "token", - olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw + olmUtil: {ed25519_verify: () => {}} as unknown as Olm.Utility, // valid if it does not throw ownUserId: "@alice:hs.tld", ownDeviceId: "ABCD", }); const room = await createUntrackedRoomMock(roomId, ["@alice:hs.tld"]); await tracker.trackRoom(room, HistoryVisibility.Joined, NullLoggerInstance.item); - const txn = await storage.readWriteTxn([storage.storeNames.userIdentities, storage.storeNames.deviceIdentities]); + const txn = await storage.readWriteTxn([storage.storeNames.userIdentities, storage.storeNames.deviceKeys]); // inviting a new member const inviteChange = new MemberChange(RoomMember.fromUserId(roomId, "@bob:hs.tld", "invite")); const memberChanges = new Map([[inviteChange.userId, inviteChange]]); @@ -869,7 +898,7 @@ export function tests() { const tracker = new DeviceTracker({ storage, getSyncToken: () => "token", - olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw + olmUtil: {ed25519_verify: () => {}} as unknown as Olm.Utility, // valid if it does not throw ownUserId: "@alice:hs.tld", ownDeviceId: "ABCD", }); @@ -881,22 +910,22 @@ export function tests() { await writeMemberListToStorage(room, storage); const devices = await tracker.devicesForTrackedRoom(roomId, hsApi, NullLoggerInstance.item); assert.equal(devices.length, 2); - assert.equal(devices.find(d => d.userId === "@alice:hs.tld").ed25519Key, "ed25519:@alice:hs.tld:device1:key"); - assert.equal(devices.find(d => d.userId === "@bob:hs.tld").ed25519Key, "ed25519:@bob:hs.tld:device1:key"); + assert.equal(getDeviceEd25519Key(devices.find(d => d.user_id === "@alice:hs.tld")!), "ed25519:@alice:hs.tld:device1:key"); + assert.equal(getDeviceEd25519Key(devices.find(d => d.user_id === "@bob:hs.tld")!), "ed25519:@bob:hs.tld:device1:key"); }, "rejecting invite with history visibility of invited removes room from user identity": async assert => { const storage = await createMockStorage(); const tracker = new DeviceTracker({ storage, getSyncToken: () => "token", - olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw + olmUtil: {ed25519_verify: () => {}} as unknown as Olm.Utility, // valid if it does not throw ownUserId: "@alice:hs.tld", ownDeviceId: "ABCD", }); // alice is joined, bob is invited const room = await createUntrackedRoomMock(roomId, ["@alice:hs.tld"], ["@bob:hs.tld"]); await tracker.trackRoom(room, HistoryVisibility.Invited, NullLoggerInstance.item); - const txn = await storage.readWriteTxn([storage.storeNames.userIdentities, storage.storeNames.deviceIdentities]); + const txn = await storage.readWriteTxn([storage.storeNames.userIdentities, storage.storeNames.deviceKeys]); // reject invite const inviteChange = new MemberChange(RoomMember.fromUserId(roomId, "@bob:hs.tld", "leave"), "invite"); const memberChanges = new Map([[inviteChange.userId, inviteChange]]); @@ -910,7 +939,7 @@ export function tests() { const tracker = new DeviceTracker({ storage, getSyncToken: () => "token", - olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw + olmUtil: {ed25519_verify: () => {}} as unknown as Olm.Utility, // valid if it does not throw ownUserId: "@alice:hs.tld", ownDeviceId: "ABCD", }); @@ -920,21 +949,21 @@ export function tests() { await tracker.trackRoom(room1, HistoryVisibility.Joined, NullLoggerInstance.item); await tracker.trackRoom(room2, HistoryVisibility.Joined, NullLoggerInstance.item); const txn1 = await storage.readTxn([storage.storeNames.userIdentities]); - assert.deepEqual((await txn1.userIdentities.get("@bob:hs.tld")).roomIds, ["!abc:hs.tld", "!def:hs.tld"]); + assert.deepEqual((await txn1.userIdentities.get("@bob:hs.tld"))!.roomIds, ["!abc:hs.tld", "!def:hs.tld"]); const leaveChange = new MemberChange(RoomMember.fromUserId(room2.id, "@bob:hs.tld", "leave"), "join"); const memberChanges = new Map([[leaveChange.userId, leaveChange]]); - const txn2 = await storage.readWriteTxn([storage.storeNames.userIdentities, storage.storeNames.deviceIdentities]); + const txn2 = await storage.readWriteTxn([storage.storeNames.userIdentities, storage.storeNames.deviceKeys]); await tracker.writeMemberChanges(room2, memberChanges, HistoryVisibility.Joined, txn2); await txn2.complete(); const txn3 = await storage.readTxn([storage.storeNames.userIdentities]); - assert.deepEqual((await txn3.userIdentities.get("@bob:hs.tld")).roomIds, ["!abc:hs.tld"]); + assert.deepEqual((await txn3.userIdentities.get("@bob:hs.tld"))!.roomIds, ["!abc:hs.tld"]); }, "add room to user identity sharing multiple rooms with us preserves other room": async assert => { const storage = await createMockStorage(); const tracker = new DeviceTracker({ storage, getSyncToken: () => "token", - olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw + olmUtil: {ed25519_verify: () => {}} as unknown as Olm.Utility, // valid if it does not throw ownUserId: "@alice:hs.tld", ownDeviceId: "ABCD", }); @@ -943,40 +972,40 @@ export function tests() { const room2 = await createUntrackedRoomMock("!def:hs.tld", ["@alice:hs.tld", "@bob:hs.tld"]); await tracker.trackRoom(room1, HistoryVisibility.Joined, NullLoggerInstance.item); const txn1 = await storage.readTxn([storage.storeNames.userIdentities]); - assert.deepEqual((await txn1.userIdentities.get("@bob:hs.tld")).roomIds, ["!abc:hs.tld"]); + assert.deepEqual((await txn1.userIdentities.get("@bob:hs.tld"))!.roomIds, ["!abc:hs.tld"]); await tracker.trackRoom(room2, HistoryVisibility.Joined, NullLoggerInstance.item); const txn2 = await storage.readTxn([storage.storeNames.userIdentities]); - assert.deepEqual((await txn2.userIdentities.get("@bob:hs.tld")).roomIds, ["!abc:hs.tld", "!def:hs.tld"]); + assert.deepEqual((await txn2.userIdentities.get("@bob:hs.tld"))!.roomIds, ["!abc:hs.tld", "!def:hs.tld"]); }, "devicesForUsers fetches users even though they aren't in any tracked room": async assert => { const storage = await createMockStorage(); const tracker = new DeviceTracker({ storage, getSyncToken: () => "token", - olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw + olmUtil: {ed25519_verify: () => {}} as unknown as Olm.Utility, // valid if it does not throw ownUserId: "@alice:hs.tld", ownDeviceId: "ABCD", }); const hsApi = createQueryKeysHSApiMock(); const devices = await tracker.devicesForUsers(["@bob:hs.tld"], hsApi, NullLoggerInstance.item); assert.equal(devices.length, 1); - assert.equal(devices[0].curve25519Key, "curve25519:@bob:hs.tld:device1:key"); + assert.equal(getDeviceCurve25519Key(devices[0]), "curve25519:@bob:hs.tld:device1:key"); const txn1 = await storage.readTxn([storage.storeNames.userIdentities]); - assert.deepEqual((await txn1.userIdentities.get("@bob:hs.tld")).roomIds, []); + assert.deepEqual((await txn1.userIdentities.get("@bob:hs.tld"))!.roomIds, []); }, "devicesForUsers doesn't add any roomId when creating userIdentity": async assert => { const storage = await createMockStorage(); const tracker = new DeviceTracker({ storage, getSyncToken: () => "token", - olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw + olmUtil: {ed25519_verify: () => {}} as unknown as Olm.Utility, // valid if it does not throw ownUserId: "@alice:hs.tld", ownDeviceId: "ABCD", }); const hsApi = createQueryKeysHSApiMock(); await tracker.devicesForUsers(["@bob:hs.tld"], hsApi, NullLoggerInstance.item); const txn1 = await storage.readTxn([storage.storeNames.userIdentities]); - assert.deepEqual((await txn1.userIdentities.get("@bob:hs.tld")).roomIds, []); + assert.deepEqual((await txn1.userIdentities.get("@bob:hs.tld"))!.roomIds, []); } } } diff --git a/src/matrix/e2ee/RoomEncryption.js b/src/matrix/e2ee/RoomEncryption.js index b74dc710..47112892 100644 --- a/src/matrix/e2ee/RoomEncryption.js +++ b/src/matrix/e2ee/RoomEncryption.js @@ -235,7 +235,7 @@ export class RoomEncryption { // Use devicesForUsers rather than devicesForRoomMembers as the room might not be tracked yet await this._deviceTracker.devicesForUsers(sendersWithoutDevice, hsApi, log); // now that we've fetched the missing devices, try verifying the results again - const txn = await this._storage.readTxn([this._storage.storeNames.deviceIdentities]); + const txn = await this._storage.readTxn([this._storage.storeNames.deviceKeys]); await this._verifyDecryptionResults(resultsWithoutDevice, txn); const resultsWithFoundDevice = resultsWithoutDevice.filter(r => !r.isVerificationUnknown); const resultsToEventIdMap = resultsWithFoundDevice.reduce((map, r) => { diff --git a/src/matrix/e2ee/common.js b/src/matrix/e2ee/common.ts similarity index 57% rename from src/matrix/e2ee/common.js rename to src/matrix/e2ee/common.ts index 9c5fe66c..63cb389d 100644 --- a/src/matrix/e2ee/common.js +++ b/src/matrix/e2ee/common.ts @@ -15,9 +15,15 @@ limitations under the License. */ import anotherjson from "another-json"; -import {createEnum} from "../../utils/enum"; -export const DecryptionSource = createEnum("Sync", "Timeline", "Retry"); +import type {UnsentStateEvent} from "../room/common"; +import type {ILogItem} from "../../logging/types"; +import type * as OlmNamespace from "@matrix-org/olm"; +type Olm = typeof OlmNamespace; + +export enum DecryptionSource { + Sync, Timeline, Retry +}; // use common prefix so it's easy to clear properties that are not e2ee related during session clear export const SESSION_E2EE_KEY_PREFIX = "e2ee:"; @@ -25,29 +31,52 @@ export const OLM_ALGORITHM = "m.olm.v1.curve25519-aes-sha2"; export const MEGOLM_ALGORITHM = "m.megolm.v1.aes-sha2"; export class DecryptionError extends Error { - constructor(code, event, detailsObj = null) { + constructor(private readonly code: string, private readonly event: object, private readonly detailsObj?: object) { super(`Decryption error ${code}${detailsObj ? ": "+JSON.stringify(detailsObj) : ""}`); - this.code = code; - this.event = event; - this.details = detailsObj; } } export const SIGNATURE_ALGORITHM = "ed25519"; -export function getEd25519Signature(signedValue, userId, deviceOrKeyId) { +export type SignedValue = { + signatures: {[userId: string]: {[keyId: string]: string}} + unsigned?: object +} + +// we store device keys (and cross-signing) in the format we get them from the server +// as that is what the signature is calculated on, so to verify and sign, we need +// it in this format anyway. +export type DeviceKey = SignedValue & { + readonly user_id: string; + readonly device_id: string; + readonly algorithms: ReadonlyArray; + readonly keys: {[keyId: string]: string}; + readonly unsigned: { + device_display_name?: string + } +} + +export function getDeviceEd25519Key(deviceKey: DeviceKey): string { + return deviceKey.keys[`ed25519:${deviceKey.device_id}`]; +} + +export function getDeviceCurve25519Key(deviceKey: DeviceKey): string { + return deviceKey.keys[`curve25519:${deviceKey.device_id}`]; +} + +export function getEd25519Signature(signedValue: SignedValue, userId: string, deviceOrKeyId: string) { return signedValue?.signatures?.[userId]?.[`${SIGNATURE_ALGORITHM}:${deviceOrKeyId}`]; } -export function verifyEd25519Signature(olmUtil, userId, deviceOrKeyId, ed25519Key, value, log = undefined) { +export function verifyEd25519Signature(olmUtil: Olm.Utility, userId: string, deviceOrKeyId: string, ed25519Key: string, value: SignedValue, log?: ILogItem) { const signature = getEd25519Signature(value, userId, deviceOrKeyId); if (!signature) { log?.set("no_signature", true); return false; } - const clone = Object.assign({}, value); - delete clone.unsigned; - delete clone.signatures; + const clone = Object.assign({}, value) as object; + delete clone["unsigned"]; + delete clone["signatures"]; const canonicalJson = anotherjson.stringify(clone); try { // throws when signature is invalid @@ -63,7 +92,7 @@ export function verifyEd25519Signature(olmUtil, userId, deviceOrKeyId, ed25519Ke } } -export function createRoomEncryptionEvent() { +export function createRoomEncryptionEvent(): UnsentStateEvent { return { "type": "m.room.encryption", "state_key": "", @@ -75,16 +104,14 @@ export function createRoomEncryptionEvent() { } } +export enum HistoryVisibility { + Joined = "joined", + Invited = "invited", + WorldReadable = "world_readable", + Shared = "shared", +}; -// Use enum when converting to TS -export const HistoryVisibility = Object.freeze({ - Joined: "joined", - Invited: "invited", - WorldReadable: "world_readable", - Shared: "shared", -}); - -export function shouldShareKey(membership, historyVisibility) { +export function shouldShareKey(membership: string, historyVisibility: HistoryVisibility) { switch (historyVisibility) { case HistoryVisibility.WorldReadable: return true; diff --git a/src/matrix/e2ee/megolm/keybackup/types.ts b/src/matrix/e2ee/megolm/keybackup/types.ts index ce56cca7..f433a7d1 100644 --- a/src/matrix/e2ee/megolm/keybackup/types.ts +++ b/src/matrix/e2ee/megolm/keybackup/types.ts @@ -42,7 +42,7 @@ export type SessionInfo = { } export type MegOlmSessionKeyInfo = { - algorithm: MEGOLM_ALGORITHM, + algorithm: typeof MEGOLM_ALGORITHM, sender_key: string, sender_claimed_keys: {[algorithm: string]: string}, forwarding_curve25519_key_chain: string[], diff --git a/src/matrix/e2ee/olm/Encryption.ts b/src/matrix/e2ee/olm/Encryption.ts index 5fd1f25b..0b552387 100644 --- a/src/matrix/e2ee/olm/Encryption.ts +++ b/src/matrix/e2ee/olm/Encryption.ts @@ -15,7 +15,7 @@ limitations under the License. */ import {groupByWithCreator} from "../../../utils/groupBy"; -import {verifyEd25519Signature, OLM_ALGORITHM} from "../common.js"; +import {verifyEd25519Signature, OLM_ALGORITHM, getDeviceCurve25519Key, getDeviceEd25519Key} from "../common.js"; import {createSessionEntry} from "./Session"; import type {OlmMessage, OlmPayload, OlmEncryptedMessageContent} from "./types"; @@ -24,7 +24,7 @@ import type {LockMap} from "../../../utils/LockMap"; import {Lock, MultiLock, ILock} from "../../../utils/Lock"; import type {Storage} from "../../storage/idb/Storage"; import type {Transaction} from "../../storage/idb/Transaction"; -import type {DeviceIdentity} from "../../storage/idb/stores/DeviceIdentityStore"; +import type {DeviceKey} from "../common"; import type {HomeServerApi} from "../../net/HomeServerApi"; import type {ILogItem} from "../../../logging/types"; import type * as OlmNamespace from "@matrix-org/olm"; @@ -99,7 +99,7 @@ export class Encryption { return new MultiLock(locks); } - async encrypt(type: string, content: Record, devices: DeviceIdentity[], hsApi: HomeServerApi, log: ILogItem): Promise { + async encrypt(type: string, content: Record, devices: DeviceKey[], hsApi: HomeServerApi, log: ILogItem): Promise { let messages: EncryptedMessage[] = []; for (let i = 0; i < devices.length ; i += MAX_BATCH_SIZE) { const batchDevices = devices.slice(i, i + MAX_BATCH_SIZE); @@ -115,12 +115,12 @@ export class Encryption { return messages; } - async _encryptForMaxDevices(type: string, content: Record, devices: DeviceIdentity[], hsApi: HomeServerApi, log: ILogItem): Promise { + async _encryptForMaxDevices(type: string, content: Record, devices: DeviceKey[], hsApi: HomeServerApi, log: ILogItem): Promise { // TODO: see if we can only hold some of the locks until after the /keys/claim call (if needed) // take a lock on all senderKeys so decryption and other calls to encrypt (should not happen) // don't modify the sessions at the same time const locks = await Promise.all(devices.map(device => { - return this.senderKeyLock.takeLock(device.curve25519Key); + return this.senderKeyLock.takeLock(getDeviceCurve25519Key(device)); })); try { const { @@ -158,10 +158,10 @@ export class Encryption { } } - async _findExistingSessions(devices: DeviceIdentity[]): Promise<{devicesWithoutSession: DeviceIdentity[], existingEncryptionTargets: EncryptionTarget[]}> { + async _findExistingSessions(devices: DeviceKey[]): Promise<{devicesWithoutSession: DeviceKey[], existingEncryptionTargets: EncryptionTarget[]}> { const txn = await this.storage.readTxn([this.storage.storeNames.olmSessions]); const sessionIdsForDevice = await Promise.all(devices.map(async device => { - return await txn.olmSessions.getSessionIds(device.curve25519Key); + return await txn.olmSessions.getSessionIds(getDeviceCurve25519Key(device)); })); const devicesWithoutSession = devices.filter((_, i) => { const sessionIds = sessionIdsForDevice[i]; @@ -184,36 +184,36 @@ export class Encryption { const plaintext = JSON.stringify(this._buildPlainTextMessageForDevice(type, content, device)); const message = session!.encrypt(plaintext); const encryptedContent = { - algorithm: OLM_ALGORITHM, + algorithm: OLM_ALGORITHM as typeof OLM_ALGORITHM, sender_key: this.account.identityKeys.curve25519, ciphertext: { - [device.curve25519Key]: message + [getDeviceCurve25519Key(device)]: message } }; return encryptedContent; } - _buildPlainTextMessageForDevice(type: string, content: Record, device: DeviceIdentity): OlmPayload { + _buildPlainTextMessageForDevice(type: string, content: Record, device: DeviceKey): OlmPayload { return { keys: { "ed25519": this.account.identityKeys.ed25519 }, recipient_keys: { - "ed25519": device.ed25519Key + "ed25519": getDeviceEd25519Key(device) }, - recipient: device.userId, + recipient: device.user_id, sender: this.ownUserId, content, type } } - async _createNewSessions(devicesWithoutSession: DeviceIdentity[], hsApi: HomeServerApi, timestamp: number, log: ILogItem): Promise { + async _createNewSessions(devicesWithoutSession: DeviceKey[], hsApi: HomeServerApi, timestamp: number, log: ILogItem): Promise { const newEncryptionTargets = await log.wrap("claim", log => this._claimOneTimeKeys(hsApi, devicesWithoutSession, log)); try { for (const target of newEncryptionTargets) { const {device, oneTimeKey} = target; - target.session = await this.account.createOutboundOlmSession(device.curve25519Key, oneTimeKey); + target.session = await this.account.createOutboundOlmSession(getDeviceCurve25519Key(device), oneTimeKey); } await this._storeSessions(newEncryptionTargets, timestamp); } catch (err) { @@ -225,16 +225,16 @@ export class Encryption { return newEncryptionTargets; } - async _claimOneTimeKeys(hsApi: HomeServerApi, deviceIdentities: DeviceIdentity[], log: ILogItem): Promise { + async _claimOneTimeKeys(hsApi: HomeServerApi, deviceIdentities: DeviceKey[], log: ILogItem): Promise { // create a Map> const devicesByUser = groupByWithCreator(deviceIdentities, - (device: DeviceIdentity) => device.userId, - (): Map => new Map(), - (deviceMap: Map, device: DeviceIdentity) => deviceMap.set(device.deviceId, device) + (device: DeviceKey) => device.user_id, + (): Map => new Map(), + (deviceMap: Map, device: DeviceKey) => deviceMap.set(device.device_id, device) ); const oneTimeKeys = Array.from(devicesByUser.entries()).reduce((usersObj, [userId, deviceMap]) => { usersObj[userId] = Array.from(deviceMap.values()).reduce((devicesObj, device) => { - devicesObj[device.deviceId] = OTK_ALGORITHM; + devicesObj[device.device_id] = OTK_ALGORITHM; return devicesObj; }, {}); return usersObj; @@ -250,7 +250,7 @@ export class Encryption { return this._verifyAndCreateOTKTargets(userKeyMap, devicesByUser, log); } - _verifyAndCreateOTKTargets(userKeyMap: ClaimedOTKResponse, devicesByUser: Map>, log: ILogItem): EncryptionTarget[] { + _verifyAndCreateOTKTargets(userKeyMap: ClaimedOTKResponse, devicesByUser: Map>, log: ILogItem): EncryptionTarget[] { const verifiedEncryptionTargets: EncryptionTarget[] = []; for (const [userId, userSection] of Object.entries(userKeyMap)) { for (const [deviceId, deviceSection] of Object.entries(userSection)) { @@ -260,7 +260,7 @@ export class Encryption { const device = devicesByUser.get(userId)?.get(deviceId); if (device) { const isValidSignature = verifyEd25519Signature( - this.olmUtil, userId, deviceId, device.ed25519Key, keySection, log); + this.olmUtil, userId, deviceId, getDeviceEd25519Key(device), keySection, log); if (isValidSignature) { const target = EncryptionTarget.fromOTK(device, keySection.key); verifiedEncryptionTargets.push(target); @@ -281,7 +281,7 @@ export class Encryption { try { await Promise.all(encryptionTargets.map(async encryptionTarget => { const sessionEntry = await txn.olmSessions.get( - encryptionTarget.device.curve25519Key, encryptionTarget.sessionId!); + getDeviceCurve25519Key(encryptionTarget.device), encryptionTarget.sessionId!); if (sessionEntry && !failed) { const olmSession = new this.olm.Session(); olmSession.unpickle(this.pickleKey, sessionEntry.session); @@ -303,7 +303,7 @@ export class Encryption { try { for (const target of encryptionTargets) { const sessionEntry = createSessionEntry( - target.session!, target.device.curve25519Key, timestamp, this.pickleKey); + target.session!, getDeviceCurve25519Key(target.device), timestamp, this.pickleKey); txn.olmSessions.set(sessionEntry); } } catch (err) { @@ -323,16 +323,16 @@ class EncryptionTarget { public session: Olm.Session | null = null; constructor( - public readonly device: DeviceIdentity, + public readonly device: DeviceKey, public readonly oneTimeKey: string | null, public readonly sessionId: string | null ) {} - static fromOTK(device: DeviceIdentity, oneTimeKey: string): EncryptionTarget { + static fromOTK(device: DeviceKey, oneTimeKey: string): EncryptionTarget { return new EncryptionTarget(device, oneTimeKey, null); } - static fromSessionId(device: DeviceIdentity, sessionId: string): EncryptionTarget { + static fromSessionId(device: DeviceKey, sessionId: string): EncryptionTarget { return new EncryptionTarget(device, null, sessionId); } @@ -346,6 +346,6 @@ class EncryptionTarget { export class EncryptedMessage { constructor( public readonly content: OlmEncryptedMessageContent, - public readonly device: DeviceIdentity + public readonly device: DeviceKey ) {} } diff --git a/src/matrix/e2ee/olm/types.ts b/src/matrix/e2ee/olm/types.ts index 5302dad8..164854ad 100644 --- a/src/matrix/e2ee/olm/types.ts +++ b/src/matrix/e2ee/olm/types.ts @@ -14,6 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ +import type {OLM_ALGORITHM} from "../common"; + export const enum OlmPayloadType { PreKey = 0, Normal = 1 @@ -25,7 +27,7 @@ export type OlmMessage = { } export type OlmEncryptedMessageContent = { - algorithm?: "m.olm.v1.curve25519-aes-sha2" + algorithm?: typeof OLM_ALGORITHM sender_key?: string, ciphertext?: { [deviceCurve25519Key: string]: OlmMessage diff --git a/src/matrix/room/BaseRoom.js b/src/matrix/room/BaseRoom.js index 4ea25389..9931be83 100644 --- a/src/matrix/room/BaseRoom.js +++ b/src/matrix/room/BaseRoom.js @@ -173,7 +173,7 @@ export class BaseRoom extends EventEmitter { const isTimelineOpen = this._isTimelineOpen; if (isTimelineOpen) { // read to fetch devices if timeline is open - stores.push(this._storage.storeNames.deviceIdentities); + stores.push(this._storage.storeNames.deviceKeys); } const writeTxn = await this._storage.readWriteTxn(stores); let decryption; diff --git a/src/matrix/room/RoomBeingCreated.ts b/src/matrix/room/RoomBeingCreated.ts index b2c9dafb..4e908aa2 100644 --- a/src/matrix/room/RoomBeingCreated.ts +++ b/src/matrix/room/RoomBeingCreated.ts @@ -20,7 +20,7 @@ import {MediaRepository} from "../net/MediaRepository"; import {EventEmitter} from "../../utils/EventEmitter"; import {AttachmentUpload} from "./AttachmentUpload"; import {loadProfiles, Profile, UserIdProfile} from "../profile"; -import {RoomType} from "./common"; +import {RoomType, UnsentStateEvent} from "./common"; import type {HomeServerApi} from "../net/HomeServerApi"; import type {ILogItem} from "../../logging/types"; @@ -37,7 +37,7 @@ type CreateRoomPayload = { invite?: string[]; room_alias_name?: string; creation_content?: {"m.federate": boolean}; - initial_state: { type: string; state_key: string; content: Record }[]; + initial_state: UnsentStateEvent[]; power_level_content_override?: Record; } diff --git a/src/matrix/room/common.ts b/src/matrix/room/common.ts index 2ce8b5dd..1174d09d 100644 --- a/src/matrix/room/common.ts +++ b/src/matrix/room/common.ts @@ -28,6 +28,8 @@ export function isRedacted(event) { return !!event?.unsigned?.redacted_because; } +export type UnsentStateEvent = { type: string; state_key: string; content: Record }; + export enum RoomStatus { None = 1 << 0, BeingCreated = 1 << 1, diff --git a/src/matrix/storage/common.ts b/src/matrix/storage/common.ts index adebcdd6..bf9ce39b 100644 --- a/src/matrix/storage/common.ts +++ b/src/matrix/storage/common.ts @@ -26,7 +26,7 @@ export enum StoreNames { timelineFragments = "timelineFragments", pendingEvents = "pendingEvents", userIdentities = "userIdentities", - deviceIdentities = "deviceIdentities", + deviceKeys = "deviceKeys", olmSessions = "olmSessions", inboundGroupSessions = "inboundGroupSessions", outboundGroupSessions = "outboundGroupSessions", diff --git a/src/matrix/storage/idb/Transaction.ts b/src/matrix/storage/idb/Transaction.ts index 532ffd1d..4c76608c 100644 --- a/src/matrix/storage/idb/Transaction.ts +++ b/src/matrix/storage/idb/Transaction.ts @@ -29,7 +29,7 @@ import {RoomMemberStore} from "./stores/RoomMemberStore"; import {TimelineFragmentStore} from "./stores/TimelineFragmentStore"; import {PendingEventStore} from "./stores/PendingEventStore"; import {UserIdentityStore} from "./stores/UserIdentityStore"; -import {DeviceIdentityStore} from "./stores/DeviceIdentityStore"; +import {DeviceKeyStore} from "./stores/DeviceKeyStore"; import {CrossSigningKeyStore} from "./stores/CrossSigningKeyStore"; import {OlmSessionStore} from "./stores/OlmSessionStore"; import {InboundGroupSessionStore} from "./stores/InboundGroupSessionStore"; @@ -142,8 +142,8 @@ export class Transaction { return this._store(StoreNames.userIdentities, idbStore => new UserIdentityStore(idbStore)); } - get deviceIdentities(): DeviceIdentityStore { - return this._store(StoreNames.deviceIdentities, idbStore => new DeviceIdentityStore(idbStore)); + get deviceKeys(): DeviceKeyStore { + return this._store(StoreNames.deviceKeys, idbStore => new DeviceKeyStore(idbStore)); } get crossSigningKeys(): CrossSigningKeyStore { diff --git a/src/matrix/storage/idb/schema.ts b/src/matrix/storage/idb/schema.ts index 3d1e714f..200f4089 100644 --- a/src/matrix/storage/idb/schema.ts +++ b/src/matrix/storage/idb/schema.ts @@ -35,7 +35,7 @@ export const schema: MigrationFunc[] = [ addInboundSessionBackupIndex, migrateBackupStatus, createCallStore, - createCrossSigningKeyStore + createCrossSigningKeyStoreAndRenameDeviceIdentities ]; // TODO: how to deal with git merge conflicts of this array? @@ -277,7 +277,10 @@ function createCallStore(db: IDBDatabase) : void { db.createObjectStore("calls", {keyPath: "key"}); } -//v18 create calls store -function createCrossSigningKeyStore(db: IDBDatabase) : void { +//v18 create calls store and rename deviceIdentities to deviceKeys +function createCrossSigningKeyStoreAndRenameDeviceIdentities(db: IDBDatabase) : void { db.createObjectStore("crossSigningKeys", {keyPath: "key"}); + db.deleteObjectStore("deviceIdentities"); + const deviceKeys = db.createObjectStore("deviceKeys", {keyPath: "key"}); + deviceKeys.createIndex("byCurve25519Key", "curve25519Key", {unique: true}); } diff --git a/src/matrix/storage/idb/stores/CrossSigningKeyStore.ts b/src/matrix/storage/idb/stores/CrossSigningKeyStore.ts index a2fa9ecb..dc5804ef 100644 --- a/src/matrix/storage/idb/stores/CrossSigningKeyStore.ts +++ b/src/matrix/storage/idb/stores/CrossSigningKeyStore.ts @@ -16,15 +16,15 @@ limitations under the License. import {MAX_UNICODE, MIN_UNICODE} from "./common"; import {Store} from "../Store"; +import type {SignedValue} from "../../../e2ee/common"; -// we store cross-signing keys in the format we get them from the server -// as that is what the signature is calculated on, so to verify, we need +// we store cross-signing (and device) keys in the format we get them from the server +// as that is what the signature is calculated on, so to verify and sign, we need // it in this format anyway. -export type CrossSigningKey = { +export type CrossSigningKey = SignedValue & { readonly user_id: string; readonly usage: ReadonlyArray; readonly keys: {[keyId: string]: string}; - readonly signatures: {[userId: string]: {[keyId: string]: string}} } type CrossSigningKeyEntry = CrossSigningKey & { diff --git a/src/matrix/storage/idb/stores/DeviceIdentityStore.ts b/src/matrix/storage/idb/stores/DeviceKeyStore.ts similarity index 63% rename from src/matrix/storage/idb/stores/DeviceIdentityStore.ts rename to src/matrix/storage/idb/stores/DeviceKeyStore.ts index 2936f079..897d6453 100644 --- a/src/matrix/storage/idb/stores/DeviceIdentityStore.ts +++ b/src/matrix/storage/idb/stores/DeviceKeyStore.ts @@ -16,15 +16,13 @@ limitations under the License. import {MAX_UNICODE, MIN_UNICODE} from "./common"; import {Store} from "../Store"; +import {getDeviceCurve25519Key} from "../../../e2ee/common"; +import type {DeviceKey} from "../../../e2ee/common"; -export interface DeviceIdentity { - userId: string; - deviceId: string; - ed25519Key: string; +type DeviceKeyEntry = { + key: string; // key in storage, not a crypto key curve25519Key: string; - algorithms: string[]; - displayName: string; - key: string; + deviceKey: DeviceKey } function encodeKey(userId: string, deviceId: string): string { @@ -36,23 +34,24 @@ function decodeKey(key: string): { userId: string, deviceId: string } { return {userId, deviceId}; } -export class DeviceIdentityStore { - private _store: Store; +export class DeviceKeyStore { + private _store: Store; - constructor(store: Store) { + constructor(store: Store) { this._store = store; } - getAllForUserId(userId: string): Promise { - const range = this._store.IDBKeyRange.lowerBound(encodeKey(userId, "")); - return this._store.selectWhile(range, device => { - return device.userId === userId; + async getAllForUserId(userId: string): Promise { + const range = this._store.IDBKeyRange.lowerBound(encodeKey(userId, MIN_UNICODE)); + const entries = await this._store.selectWhile(range, device => { + return device.deviceKey.user_id === userId; }); + return entries.map(e => e.deviceKey); } async getAllDeviceIds(userId: string): Promise { const deviceIds: string[] = []; - const range = this._store.IDBKeyRange.lowerBound(encodeKey(userId, "")); + const range = this._store.IDBKeyRange.lowerBound(encodeKey(userId, MIN_UNICODE)); await this._store.iterateKeys(range, key => { const decodedKey = decodeKey(key as string); // prevent running into the next room @@ -65,17 +64,21 @@ export class DeviceIdentityStore { return deviceIds; } - get(userId: string, deviceId: string): Promise { - return this._store.get(encodeKey(userId, deviceId)); + async get(userId: string, deviceId: string): Promise { + return (await this._store.get(encodeKey(userId, deviceId)))?.deviceKey; } - set(deviceIdentity: DeviceIdentity): void { - deviceIdentity.key = encodeKey(deviceIdentity.userId, deviceIdentity.deviceId); - this._store.put(deviceIdentity); + set(deviceKey: DeviceKey): void { + this._store.put({ + key: encodeKey(deviceKey.user_id, deviceKey.device_id), + curve25519Key: getDeviceCurve25519Key(deviceKey)!, + deviceKey + }); } - getByCurve25519Key(curve25519Key: string): Promise { - return this._store.index("byCurve25519Key").get(curve25519Key); + async getByCurve25519Key(curve25519Key: string): Promise { + const entry = await this._store.index("byCurve25519Key").get(curve25519Key); + return entry?.deviceKey; } remove(userId: string, deviceId: string): void { From daf66e1d6c742df080bc269019c2cf649c1ef1ea Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Mar 2023 15:02:42 +0100 Subject: [PATCH 401/435] implement signing users and other devices --- src/matrix/e2ee/Account.js | 2 +- src/matrix/e2ee/DeviceTracker.ts | 8 +- src/matrix/e2ee/common.ts | 4 +- .../idb/stores/CrossSigningKeyStore.ts | 11 +- src/matrix/verification/CrossSigning.ts | 152 ++++++++++++------ src/matrix/verification/common.ts | 20 +-- 6 files changed, 115 insertions(+), 82 deletions(-) diff --git a/src/matrix/e2ee/Account.js b/src/matrix/e2ee/Account.js index b0dd1546..8fa2db02 100644 --- a/src/matrix/e2ee/Account.js +++ b/src/matrix/e2ee/Account.js @@ -259,7 +259,7 @@ export class Account { return obj; } - getDeviceKeysToSignWithCrossSigning() { + getUnsignedDeviceKey() { const identityKeys = JSON.parse(this._account.identity_keys()); return this._keysAsSignableObject(identityKeys); } diff --git a/src/matrix/e2ee/DeviceTracker.ts b/src/matrix/e2ee/DeviceTracker.ts index c8e9df09..2b2728e1 100644 --- a/src/matrix/e2ee/DeviceTracker.ts +++ b/src/matrix/e2ee/DeviceTracker.ts @@ -19,7 +19,7 @@ import {HistoryVisibility, shouldShareKey, DeviceKey, getDeviceEd25519Key, getDe import {RoomMember} from "../room/members/RoomMember.js"; import {getKeyUsage, getKeyEd25519Key, getKeyUserId, KeyUsage} from "../verification/CrossSigning"; import {MemberChange} from "../room/members/RoomMember"; -import type {CrossSigningKey} from "../storage/idb/stores/CrossSigningKeyStore"; +import type {CrossSigningKey} from "../verification/CrossSigning"; import type {HomeServerApi} from "../net/HomeServerApi"; import type {ObservableMap} from "../../observable/map"; import type {Room} from "../room/Room"; @@ -160,7 +160,7 @@ export class DeviceTracker { } } - async getCrossSigningKeyForUser(userId: string, usage: KeyUsage, hsApi: HomeServerApi, log: ILogItem) { + async getCrossSigningKeyForUser(userId: string, usage: KeyUsage, hsApi: HomeServerApi, log: ILogItem): Promise { return await log.wrap({l: "DeviceTracker.getCrossSigningKeyForUser", id: userId, usage}, async log => { let txn = await this._storage.readTxn([ this._storage.storeNames.userIdentities, @@ -495,6 +495,7 @@ export class DeviceTracker { /** * Can be used to decide which users to share keys with. * Assumes room is already tracked. Call `trackRoom` first if unsure. + * This will not return the device key for our own user, as we don't need to share keys with ourselves. */ async devicesForRoomMembers(roomId: string, userIds: string[], hsApi: HomeServerApi, log: ILogItem): Promise { const txn = await this._storage.readTxn([ @@ -506,6 +507,7 @@ export class DeviceTracker { /** * Cannot be used to decide which users to share keys with. * Does not assume membership to any room or whether any room is tracked. + * This will return device keys for our own user, including our own device. */ async devicesForUsers(userIds: string[], hsApi: HomeServerApi, log: ILogItem): Promise { const txn = await this._storage.readTxn([ @@ -527,7 +529,7 @@ export class DeviceTracker { return this._devicesForUserIdentities(upToDateIdentities, outdatedUserIds, hsApi, log); } - /** gets a single device */ + /** Gets a single device */ async deviceForId(userId: string, deviceId: string, hsApi: HomeServerApi, log: ILogItem) { const txn = await this._storage.readTxn([ this._storage.storeNames.deviceKeys, diff --git a/src/matrix/e2ee/common.ts b/src/matrix/e2ee/common.ts index 63cb389d..27078135 100644 --- a/src/matrix/e2ee/common.ts +++ b/src/matrix/e2ee/common.ts @@ -39,7 +39,7 @@ export class DecryptionError extends Error { export const SIGNATURE_ALGORITHM = "ed25519"; export type SignedValue = { - signatures: {[userId: string]: {[keyId: string]: string}} + signatures?: {[userId: string]: {[keyId: string]: string}} unsigned?: object } @@ -64,7 +64,7 @@ export function getDeviceCurve25519Key(deviceKey: DeviceKey): string { return deviceKey.keys[`curve25519:${deviceKey.device_id}`]; } -export function getEd25519Signature(signedValue: SignedValue, userId: string, deviceOrKeyId: string) { +export function getEd25519Signature(signedValue: SignedValue, userId: string, deviceOrKeyId: string): string | undefined { return signedValue?.signatures?.[userId]?.[`${SIGNATURE_ALGORITHM}:${deviceOrKeyId}`]; } diff --git a/src/matrix/storage/idb/stores/CrossSigningKeyStore.ts b/src/matrix/storage/idb/stores/CrossSigningKeyStore.ts index dc5804ef..32100aca 100644 --- a/src/matrix/storage/idb/stores/CrossSigningKeyStore.ts +++ b/src/matrix/storage/idb/stores/CrossSigningKeyStore.ts @@ -16,16 +16,7 @@ limitations under the License. import {MAX_UNICODE, MIN_UNICODE} from "./common"; import {Store} from "../Store"; -import type {SignedValue} from "../../../e2ee/common"; - -// we store cross-signing (and device) keys in the format we get them from the server -// as that is what the signature is calculated on, so to verify and sign, we need -// it in this format anyway. -export type CrossSigningKey = SignedValue & { - readonly user_id: string; - readonly usage: ReadonlyArray; - readonly keys: {[keyId: string]: string}; -} +import type {CrossSigningKey} from "../../../verification/CrossSigning"; type CrossSigningKeyEntry = CrossSigningKey & { key: string; // key in storage, not a crypto key diff --git a/src/matrix/verification/CrossSigning.ts b/src/matrix/verification/CrossSigning.ts index d3b6bc90..e58c89e8 100644 --- a/src/matrix/verification/CrossSigning.ts +++ b/src/matrix/verification/CrossSigning.ts @@ -14,19 +14,28 @@ See the License for the specific language governing permissions and limitations under the License. */ +import { ILogItem } from "../../lib"; +import {pkSign} from "./common"; + import type {SecretStorage} from "../ssss/SecretStorage"; import type {Storage} from "../storage/idb/Storage"; import type {Platform} from "../../platform/web/Platform"; import type {DeviceTracker} from "../e2ee/DeviceTracker"; -import type * as OlmNamespace from "@matrix-org/olm"; import type {HomeServerApi} from "../net/HomeServerApi"; import type {Account} from "../e2ee/Account"; -import { ILogItem } from "../../lib"; -import {pkSign} from "./common"; -import type {ISignatures} from "./common"; - +import type {SignedValue, DeviceKey} from "../e2ee/common"; +import type * as OlmNamespace from "@matrix-org/olm"; type Olm = typeof OlmNamespace; +// we store cross-signing (and device) keys in the format we get them from the server +// as that is what the signature is calculated on, so to verify and sign, we need +// it in this format anyway. +export type CrossSigningKey = SignedValue & { + readonly user_id: string; + readonly usage: ReadonlyArray; + readonly keys: {[keyId: string]: string}; +} + export enum KeyUsage { Master = "master", SelfSigning = "self_signing", @@ -68,63 +77,108 @@ export class CrossSigning { log.wrap("CrossSigning.init", async log => { // TODO: use errorboundary here const txn = await this.storage.readTxn([this.storage.storeNames.accountData]); - - const mskSeed = await this.secretStorage.readSecret("m.cross_signing.master", txn); + const privateMasterKey = await this.getSigningKey(KeyUsage.Master); const signing = new this.olm.PkSigning(); let derivedPublicKey; try { - const seed = new Uint8Array(this.platform.encoding.base64.decode(mskSeed)); - derivedPublicKey = signing.init_with_seed(seed); + derivedPublicKey = signing.init_with_seed(privateMasterKey); } finally { signing.free(); } - const masterKey = await this.deviceTracker.getCrossSigningKeyForUser(this.ownUserId, KeyUsage.Master, this.hsApi, log); - log.set({publishedMasterKey: masterKey, derivedPublicKey}); - this._isMasterKeyTrusted = masterKey === derivedPublicKey; + const publishedMasterKey = await this.deviceTracker.getCrossSigningKeyForUser(this.ownUserId, KeyUsage.Master, this.hsApi, log); + const publisedEd25519Key = publishedMasterKey && getKeyEd25519Key(publishedMasterKey); + log.set({publishedMasterKey: publisedEd25519Key, derivedPublicKey}); + this._isMasterKeyTrusted = !!publisedEd25519Key && publisedEd25519Key === derivedPublicKey; log.set("isMasterKeyTrusted", this.isMasterKeyTrusted); }); } - async signOwnDevice(log: ILogItem) { - log.wrap("CrossSigning.signOwnDevice", async log => { - if (!this._isMasterKeyTrusted) { - log.set("mskNotTrusted", true); - return; - } - const deviceKey = this.e2eeAccount.getDeviceKeysToSignWithCrossSigning(); - const signedDeviceKey = await this.signDeviceData(deviceKey); - const payload = { - [signedDeviceKey["user_id"]]: { - [signedDeviceKey["device_id"]]: signedDeviceKey - } - }; - const request = this.hsApi.uploadSignatures(payload, {log}); - await request.response(); - }); - } - - signDevice(deviceId: string) { - // need to get the device key for the device - } - - signUser(userId: string) { - // need to be able to get the msk for the user - } - - private async signDeviceData(data: T): Promise { - const txn = await this.storage.readTxn([this.storage.storeNames.accountData]); - const seedStr = await this.secretStorage.readSecret(`m.cross_signing.self_signing`, txn); - const seed = new Uint8Array(this.platform.encoding.base64.decode(seedStr)); - pkSign(this.olm, data, seed, this.ownUserId, ""); - return data as T & { signatures: ISignatures }; - } - get isMasterKeyTrusted(): boolean { return this._isMasterKeyTrusted; } + + /** returns our own device key signed by our self-signing key. Other signatures will be missing. */ + async signOwnDevice(log: ILogItem): Promise { + return log.wrap("CrossSigning.signOwnDevice", async log => { + if (!this._isMasterKeyTrusted) { + log.set("mskNotTrusted", true); + return; + } + const ownDeviceKey = this.e2eeAccount.getUnsignedDeviceKey() as DeviceKey; + return this.signDeviceKey(ownDeviceKey, log); + }); + } + + /** @return the signed device key for the given device id */ + async signDevice(deviceId: string, log: ILogItem): Promise { + return log.wrap("CrossSigning.signDevice", async log => { + log.set("id", deviceId); + if (!this._isMasterKeyTrusted) { + log.set("mskNotTrusted", true); + return; + } + // need to be able to get the msk for the user + const keyToSign = await this.deviceTracker.deviceForId(this.ownUserId, deviceId, this.hsApi, log); + if (!keyToSign) { + return undefined; + } + return this.signDeviceKey(keyToSign, log); + }); + } + + /** @return the signed MSK for the given user id */ + async signUser(userId: string, log: ILogItem): Promise { + return log.wrap("CrossSigning.signUser", async log => { + log.set("id", userId); + if (!this._isMasterKeyTrusted) { + log.set("mskNotTrusted", true); + return; + } + // need to be able to get the msk for the user + const keyToSign = await this.deviceTracker.getCrossSigningKeyForUser(userId, KeyUsage.Master, this.hsApi, log); + if (!keyToSign) { + return undefined; + } + const signingKey = await this.getSigningKey(KeyUsage.UserSigning); + // add signature to keyToSign + pkSign(this.olm, keyToSign, signingKey, userId, ""); + const payload = { + [keyToSign.user_id]: { + [getKeyEd25519Key(keyToSign)!]: keyToSign + } + }; + const request = this.hsApi.uploadSignatures(payload, {log}); + await request.response(); + return keyToSign; + }); + } + + private async signDeviceKey(keyToSign: DeviceKey, log: ILogItem): Promise { + const signingKey = await this.getSigningKey(KeyUsage.SelfSigning); + // add signature to keyToSign + pkSign(this.olm, keyToSign, signingKey, this.ownUserId, ""); + // so the payload format of a signature is a map from userid to key id of the signed key + // (without the algoritm prefix though according to example, e.g. just device id or base 64 public key) + // to the complete signed key with the signature of the signing key in the signatures section. + const payload = { + [keyToSign.user_id]: { + [keyToSign.device_id]: keyToSign + } + }; + const request = this.hsApi.uploadSignatures(payload, {log}); + await request.response(); + return keyToSign; + } + + private async getSigningKey(usage: KeyUsage): Promise { + const txn = await this.storage.readTxn([this.storage.storeNames.accountData]); + const seedStr = await this.secretStorage.readSecret(`m.cross_signing.${usage}`, txn); + const seed = new Uint8Array(this.platform.encoding.base64.decode(seedStr)); + return seed; + } } -export function getKeyUsage(keyInfo): KeyUsage | undefined { +export function getKeyUsage(keyInfo: CrossSigningKey): KeyUsage | undefined { if (!Array.isArray(keyInfo.usage) || keyInfo.usage.length !== 1) { return undefined; } @@ -138,7 +192,7 @@ export function getKeyUsage(keyInfo): KeyUsage | undefined { const algorithm = "ed25519"; const prefix = `${algorithm}:`; -export function getKeyEd25519Key(keyInfo): string | undefined { +export function getKeyEd25519Key(keyInfo: CrossSigningKey): string | undefined { const ed25519KeyIds = Object.keys(keyInfo.keys).filter(keyId => keyId.startsWith(prefix)); if (ed25519KeyIds.length !== 1) { return undefined; @@ -148,6 +202,6 @@ export function getKeyEd25519Key(keyInfo): string | undefined { return publicKey; } -export function getKeyUserId(keyInfo): string | undefined { +export function getKeyUserId(keyInfo: CrossSigningKey): string | undefined { return keyInfo["user_id"]; } diff --git a/src/matrix/verification/common.ts b/src/matrix/verification/common.ts index 369b5618..de9b1b1b 100644 --- a/src/matrix/verification/common.ts +++ b/src/matrix/verification/common.ts @@ -16,24 +16,10 @@ limitations under the License. import { PkSigning } from "@matrix-org/olm"; import anotherjson from "another-json"; +import type {SignedValue} from "../e2ee/common"; import type * as OlmNamespace from "@matrix-org/olm"; type Olm = typeof OlmNamespace; -export interface IObject { - unsigned?: object; - signatures?: ISignatures; -} - -export interface ISignatures { - [entity: string]: { - [keyId: string]: string; - }; -} - -export interface ISigned { - signatures?: ISignatures; -} - // from matrix-js-sdk /** * Sign a JSON object using public key cryptography @@ -45,7 +31,7 @@ export interface ISigned { * @param pubKey - The public key (ignored if key is a seed) * @returns the signature for the object */ - export function pkSign(olmUtil: Olm, obj: object & IObject, key: Uint8Array | PkSigning, userId: string, pubKey: string): string { + export function pkSign(olmUtil: Olm, obj: SignedValue, key: Uint8Array | PkSigning, userId: string, pubKey: string): string { let createdKey = false; if (key instanceof Uint8Array) { const keyObj = new olmUtil.PkSigning(); @@ -69,4 +55,4 @@ export interface ISigned { key.free(); } } -} \ No newline at end of file +} From a9412aa57c3b35780202b297576444dac2fe5b04 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Mar 2023 17:12:56 +0100 Subject: [PATCH 402/435] fix import paths after TS conversion --- src/matrix/DeviceMessageHandler.js | 2 +- src/matrix/Session.js | 4 ++-- src/matrix/e2ee/DeviceTracker.ts | 2 +- src/matrix/e2ee/RoomEncryption.js | 2 +- src/matrix/e2ee/megolm/Decryption.ts | 3 +-- src/matrix/e2ee/megolm/Encryption.js | 2 +- src/matrix/e2ee/megolm/decryption/DecryptionChanges.js | 2 +- src/matrix/e2ee/megolm/decryption/SessionDecryption.ts | 2 +- src/matrix/e2ee/olm/Decryption.ts | 2 +- src/matrix/e2ee/olm/Encryption.ts | 2 +- src/matrix/room/BaseRoom.js | 2 +- src/matrix/room/Room.js | 2 +- src/matrix/room/RoomSummary.js | 2 +- src/matrix/ssss/index.ts | 2 +- src/matrix/storage/idb/schema.ts | 2 +- src/matrix/storage/idb/stores/SessionStore.ts | 2 +- 16 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index f6e7cad7..78b384ab 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {OLM_ALGORITHM} from "./e2ee/common.js"; +import {OLM_ALGORITHM} from "./e2ee/common"; import {countBy, groupBy} from "../utils/groupBy"; import {LRUCache} from "../utils/LRUCache"; diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 35f713f6..9aa0494d 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -33,9 +33,9 @@ import {KeyLoader as MegOlmKeyLoader} from "./e2ee/megolm/decryption/KeyLoader"; import {KeyBackup} from "./e2ee/megolm/keybackup/KeyBackup"; import {CrossSigning} from "./verification/CrossSigning"; import {Encryption as MegOlmEncryption} from "./e2ee/megolm/Encryption.js"; -import {MEGOLM_ALGORITHM} from "./e2ee/common.js"; +import {MEGOLM_ALGORITHM} from "./e2ee/common"; import {RoomEncryption} from "./e2ee/RoomEncryption.js"; -import {DeviceTracker} from "./e2ee/DeviceTracker.js"; +import {DeviceTracker} from "./e2ee/DeviceTracker"; import {LockMap} from "../utils/LockMap"; import {groupBy} from "../utils/groupBy"; import { diff --git a/src/matrix/e2ee/DeviceTracker.ts b/src/matrix/e2ee/DeviceTracker.ts index 2b2728e1..0464377a 100644 --- a/src/matrix/e2ee/DeviceTracker.ts +++ b/src/matrix/e2ee/DeviceTracker.ts @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {verifyEd25519Signature, getEd25519Signature, SIGNATURE_ALGORITHM} from "./common.js"; +import {verifyEd25519Signature, getEd25519Signature, SIGNATURE_ALGORITHM} from "./common"; import {HistoryVisibility, shouldShareKey, DeviceKey, getDeviceEd25519Key, getDeviceCurve25519Key} from "./common"; import {RoomMember} from "../room/members/RoomMember.js"; import {getKeyUsage, getKeyEd25519Key, getKeyUserId, KeyUsage} from "../verification/CrossSigning"; diff --git a/src/matrix/e2ee/RoomEncryption.js b/src/matrix/e2ee/RoomEncryption.js index 47112892..bd0defac 100644 --- a/src/matrix/e2ee/RoomEncryption.js +++ b/src/matrix/e2ee/RoomEncryption.js @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {MEGOLM_ALGORITHM, DecryptionSource} from "./common.js"; +import {MEGOLM_ALGORITHM, DecryptionSource} from "./common"; import {groupEventsBySession} from "./megolm/decryption/utils"; import {mergeMap} from "../../utils/mergeMap"; import {groupBy} from "../../utils/groupBy"; diff --git a/src/matrix/e2ee/megolm/Decryption.ts b/src/matrix/e2ee/megolm/Decryption.ts index e139e8c9..c2d56207 100644 --- a/src/matrix/e2ee/megolm/Decryption.ts +++ b/src/matrix/e2ee/megolm/Decryption.ts @@ -14,10 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {DecryptionError} from "../common.js"; import {DecryptionPreparation} from "./decryption/DecryptionPreparation.js"; import {SessionDecryption} from "./decryption/SessionDecryption"; -import {MEGOLM_ALGORITHM} from "../common.js"; +import {DecryptionError, MEGOLM_ALGORITHM} from "../common"; import {validateEvent, groupEventsBySession} from "./decryption/utils"; import {keyFromStorage, keyFromDeviceMessage, keyFromBackup} from "./decryption/RoomKey"; import type {RoomKey, IncomingRoomKey} from "./decryption/RoomKey"; diff --git a/src/matrix/e2ee/megolm/Encryption.js b/src/matrix/e2ee/megolm/Encryption.js index eb5f68d3..681344fe 100644 --- a/src/matrix/e2ee/megolm/Encryption.js +++ b/src/matrix/e2ee/megolm/Encryption.js @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {MEGOLM_ALGORITHM} from "../common.js"; +import {MEGOLM_ALGORITHM} from "../common"; import {OutboundRoomKey} from "./decryption/RoomKey"; export class Encryption { diff --git a/src/matrix/e2ee/megolm/decryption/DecryptionChanges.js b/src/matrix/e2ee/megolm/decryption/DecryptionChanges.js index b45ab6dd..24226e25 100644 --- a/src/matrix/e2ee/megolm/decryption/DecryptionChanges.js +++ b/src/matrix/e2ee/megolm/decryption/DecryptionChanges.js @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {DecryptionError} from "../../common.js"; +import {DecryptionError} from "../../common"; export class DecryptionChanges { constructor(roomId, results, errors, replayEntries) { diff --git a/src/matrix/e2ee/megolm/decryption/SessionDecryption.ts b/src/matrix/e2ee/megolm/decryption/SessionDecryption.ts index ca294460..72af718c 100644 --- a/src/matrix/e2ee/megolm/decryption/SessionDecryption.ts +++ b/src/matrix/e2ee/megolm/decryption/SessionDecryption.ts @@ -15,7 +15,7 @@ limitations under the License. */ import {DecryptionResult} from "../../DecryptionResult"; -import {DecryptionError} from "../../common.js"; +import {DecryptionError} from "../../common"; import {ReplayDetectionEntry} from "./ReplayDetectionEntry"; import type {RoomKey} from "./RoomKey"; import type {KeyLoader, OlmDecryptionResult} from "./KeyLoader"; diff --git a/src/matrix/e2ee/olm/Decryption.ts b/src/matrix/e2ee/olm/Decryption.ts index 0f96f2fc..e1546b0b 100644 --- a/src/matrix/e2ee/olm/Decryption.ts +++ b/src/matrix/e2ee/olm/Decryption.ts @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {DecryptionError} from "../common.js"; +import {DecryptionError} from "../common"; import {groupBy} from "../../../utils/groupBy"; import {MultiLock, ILock} from "../../../utils/Lock"; import {Session} from "./Session"; diff --git a/src/matrix/e2ee/olm/Encryption.ts b/src/matrix/e2ee/olm/Encryption.ts index 0b552387..c4fee911 100644 --- a/src/matrix/e2ee/olm/Encryption.ts +++ b/src/matrix/e2ee/olm/Encryption.ts @@ -15,7 +15,7 @@ limitations under the License. */ import {groupByWithCreator} from "../../../utils/groupBy"; -import {verifyEd25519Signature, OLM_ALGORITHM, getDeviceCurve25519Key, getDeviceEd25519Key} from "../common.js"; +import {verifyEd25519Signature, OLM_ALGORITHM, getDeviceCurve25519Key, getDeviceEd25519Key} from "../common"; import {createSessionEntry} from "./Session"; import type {OlmMessage, OlmPayload, OlmEncryptedMessageContent} from "./types"; diff --git a/src/matrix/room/BaseRoom.js b/src/matrix/room/BaseRoom.js index 9931be83..7ab8a209 100644 --- a/src/matrix/room/BaseRoom.js +++ b/src/matrix/room/BaseRoom.js @@ -26,7 +26,7 @@ import {MemberList} from "./members/MemberList.js"; import {Heroes} from "./members/Heroes.js"; import {EventEntry} from "./timeline/entries/EventEntry.js"; import {ObservedEventMap} from "./ObservedEventMap.js"; -import {DecryptionSource} from "../e2ee/common.js"; +import {DecryptionSource} from "../e2ee/common"; import {ensureLogItem} from "../../logging/utils"; import {PowerLevels} from "./PowerLevels.js"; import {RetainedObservableValue} from "../../observable/value"; diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js index b87d7a88..47da3c03 100644 --- a/src/matrix/room/Room.js +++ b/src/matrix/room/Room.js @@ -22,7 +22,7 @@ import {SendQueue} from "./sending/SendQueue.js"; import {WrappedError} from "../error.js" import {Heroes} from "./members/Heroes.js"; import {AttachmentUpload} from "./AttachmentUpload.js"; -import {DecryptionSource} from "../e2ee/common.js"; +import {DecryptionSource} from "../e2ee/common"; import {iterateResponseStateEvents} from "./common"; import {PowerLevels, EVENT_TYPE as POWERLEVELS_EVENT_TYPE } from "./PowerLevels.js"; diff --git a/src/matrix/room/RoomSummary.js b/src/matrix/room/RoomSummary.js index 62608683..8e1619ca 100644 --- a/src/matrix/room/RoomSummary.js +++ b/src/matrix/room/RoomSummary.js @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {MEGOLM_ALGORITHM} from "../e2ee/common.js"; +import {MEGOLM_ALGORITHM} from "../e2ee/common"; import {iterateResponseStateEvents} from "./common"; function applyTimelineEntries(data, timelineEntries, isInitialSync, canMarkUnread, ownUserId) { diff --git a/src/matrix/ssss/index.ts b/src/matrix/ssss/index.ts index fd4c2245..02f3290e 100644 --- a/src/matrix/ssss/index.ts +++ b/src/matrix/ssss/index.ts @@ -17,7 +17,7 @@ limitations under the License. import {KeyDescription, Key} from "./common"; import {keyFromPassphrase} from "./passphrase"; import {keyFromRecoveryKey} from "./recoveryKey"; -import {SESSION_E2EE_KEY_PREFIX} from "../e2ee/common.js"; +import {SESSION_E2EE_KEY_PREFIX} from "../e2ee/common"; import type {Storage} from "../storage/idb/Storage"; import type {Transaction} from "../storage/idb/Transaction"; import type {KeyDescriptionData} from "./common"; diff --git a/src/matrix/storage/idb/schema.ts b/src/matrix/storage/idb/schema.ts index 200f4089..c8d260cd 100644 --- a/src/matrix/storage/idb/schema.ts +++ b/src/matrix/storage/idb/schema.ts @@ -2,7 +2,7 @@ import {IDOMStorage} from "./types"; import {ITransaction} from "./QueryTarget"; import {iterateCursor, NOT_DONE, reqAsPromise} from "./utils"; import {RoomMember, EVENT_TYPE as MEMBER_EVENT_TYPE} from "../../room/members/RoomMember.js"; -import {SESSION_E2EE_KEY_PREFIX} from "../../e2ee/common.js"; +import {SESSION_E2EE_KEY_PREFIX} from "../../e2ee/common"; import {SummaryData} from "../../room/RoomSummary"; import {RoomMemberStore, MemberData} from "./stores/RoomMemberStore"; import {InboundGroupSessionStore, InboundGroupSessionEntry, BackupStatus, KeySource} from "./stores/InboundGroupSessionStore"; diff --git a/src/matrix/storage/idb/stores/SessionStore.ts b/src/matrix/storage/idb/stores/SessionStore.ts index 9ae9bb7e..24b7099a 100644 --- a/src/matrix/storage/idb/stores/SessionStore.ts +++ b/src/matrix/storage/idb/stores/SessionStore.ts @@ -15,7 +15,7 @@ limitations under the License. */ import {Store} from "../Store"; import {IDOMStorage} from "../types"; -import {SESSION_E2EE_KEY_PREFIX} from "../../../e2ee/common.js"; +import {SESSION_E2EE_KEY_PREFIX} from "../../../e2ee/common"; import {parse, stringify} from "../../../../utils/typedJSON"; import type {ILogItem} from "../../../../logging/types"; From 4dce93e5ef5dcecce31556acdfd3bb4ab417d3cf Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Mar 2023 17:13:15 +0100 Subject: [PATCH 403/435] make sure the key property doesn't leak out of the storage layer as it ends up in the value we're signing and uploading, corrupting the signature --- .../storage/idb/stores/CrossSigningKeyStore.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/matrix/storage/idb/stores/CrossSigningKeyStore.ts b/src/matrix/storage/idb/stores/CrossSigningKeyStore.ts index 32100aca..bbda15c0 100644 --- a/src/matrix/storage/idb/stores/CrossSigningKeyStore.ts +++ b/src/matrix/storage/idb/stores/CrossSigningKeyStore.ts @@ -18,7 +18,8 @@ import {MAX_UNICODE, MIN_UNICODE} from "./common"; import {Store} from "../Store"; import type {CrossSigningKey} from "../../../verification/CrossSigning"; -type CrossSigningKeyEntry = CrossSigningKey & { +type CrossSigningKeyEntry = { + crossSigningKey: CrossSigningKey key: string; // key in storage, not a crypto key } @@ -38,14 +39,15 @@ export class CrossSigningKeyStore { this._store = store; } - get(userId: string, deviceId: string): Promise { - return this._store.get(encodeKey(userId, deviceId)); + async get(userId: string, deviceId: string): Promise { + return (await this._store.get(encodeKey(userId, deviceId)))?.crossSigningKey; } set(crossSigningKey: CrossSigningKey): void { - const deviceIdentityEntry = crossSigningKey as CrossSigningKeyEntry; - deviceIdentityEntry.key = encodeKey(crossSigningKey["user_id"], crossSigningKey.usage[0]); - this._store.put(deviceIdentityEntry); + this._store.put({ + key:encodeKey(crossSigningKey["user_id"], crossSigningKey.usage[0]), + crossSigningKey + }); } remove(userId: string, usage: string): void { From 20a6fcda72e65f42f9d46846535351134e54b466 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Mar 2023 17:14:05 +0100 Subject: [PATCH 404/435] don't allow signing own user --- src/matrix/verification/CrossSigning.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/matrix/verification/CrossSigning.ts b/src/matrix/verification/CrossSigning.ts index e58c89e8..a3bd5f47 100644 --- a/src/matrix/verification/CrossSigning.ts +++ b/src/matrix/verification/CrossSigning.ts @@ -135,6 +135,10 @@ export class CrossSigning { return; } // need to be able to get the msk for the user + // can't sign own user + if (userId === this.ownUserId) { + return; + } const keyToSign = await this.deviceTracker.getCrossSigningKeyForUser(userId, KeyUsage.Master, this.hsApi, log); if (!keyToSign) { return undefined; From 504d869b385160620d04ea05000af7ddaf7d99f7 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Mar 2023 17:14:27 +0100 Subject: [PATCH 405/435] provide correct user id for signing key owner when signing other user --- src/matrix/verification/CrossSigning.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/verification/CrossSigning.ts b/src/matrix/verification/CrossSigning.ts index a3bd5f47..6430c0a7 100644 --- a/src/matrix/verification/CrossSigning.ts +++ b/src/matrix/verification/CrossSigning.ts @@ -145,7 +145,7 @@ export class CrossSigning { } const signingKey = await this.getSigningKey(KeyUsage.UserSigning); // add signature to keyToSign - pkSign(this.olm, keyToSign, signingKey, userId, ""); + pkSign(this.olm, keyToSign, signingKey, this.ownUserId, ""); const payload = { [keyToSign.user_id]: { [getKeyEd25519Key(keyToSign)!]: keyToSign From 34b113b26eb6eb6053f7b1a2bb5b648535310922 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Mar 2023 17:14:50 +0100 Subject: [PATCH 406/435] don't upload pre-existing signatures when signing --- src/matrix/verification/CrossSigning.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/matrix/verification/CrossSigning.ts b/src/matrix/verification/CrossSigning.ts index 6430c0a7..67a5616a 100644 --- a/src/matrix/verification/CrossSigning.ts +++ b/src/matrix/verification/CrossSigning.ts @@ -122,6 +122,7 @@ export class CrossSigning { if (!keyToSign) { return undefined; } + delete keyToSign.signatures; return this.signDeviceKey(keyToSign, log); }); } @@ -143,6 +144,7 @@ export class CrossSigning { if (!keyToSign) { return undefined; } + delete keyToSign.signatures; const signingKey = await this.getSigningKey(KeyUsage.UserSigning); // add signature to keyToSign pkSign(this.olm, keyToSign, signingKey, this.ownUserId, ""); From 3a303ff84d9b6ea160ed8b2d587ad07da9b20c0c Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Mar 2023 17:15:05 +0100 Subject: [PATCH 407/435] cleanup comments --- src/matrix/verification/CrossSigning.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/matrix/verification/CrossSigning.ts b/src/matrix/verification/CrossSigning.ts index 67a5616a..77b489d2 100644 --- a/src/matrix/verification/CrossSigning.ts +++ b/src/matrix/verification/CrossSigning.ts @@ -117,7 +117,6 @@ export class CrossSigning { log.set("mskNotTrusted", true); return; } - // need to be able to get the msk for the user const keyToSign = await this.deviceTracker.deviceForId(this.ownUserId, deviceId, this.hsApi, log); if (!keyToSign) { return undefined; @@ -135,7 +134,6 @@ export class CrossSigning { log.set("mskNotTrusted", true); return; } - // need to be able to get the msk for the user // can't sign own user if (userId === this.ownUserId) { return; From fa662db70b0c34b54cf1fa4c5d2ea86ee679046b Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Mar 2023 17:16:53 +0100 Subject: [PATCH 408/435] show cross-sign user option in right panel --- .../rightpanel/MemberDetailsViewModel.js | 8 ++++++++ .../web/ui/css/themes/element/theme.css | 1 + .../ui/session/rightpanel/MemberDetailsView.js | 18 +++++++++++++----- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/domain/session/rightpanel/MemberDetailsViewModel.js b/src/domain/session/rightpanel/MemberDetailsViewModel.js index b3c8278c..774f64af 100644 --- a/src/domain/session/rightpanel/MemberDetailsViewModel.js +++ b/src/domain/session/rightpanel/MemberDetailsViewModel.js @@ -54,6 +54,14 @@ export class MemberDetailsViewModel extends ViewModel { this.emitChange("role"); } + async signUser() { + if (this._session.crossSigning) { + await this.logger.run("MemberDetailsViewModel.signUser", async log => { + await this._session.crossSigning.signUser(this.userId, log); + }); + } + } + get avatarLetter() { return avatarInitials(this.name); } diff --git a/src/platform/web/ui/css/themes/element/theme.css b/src/platform/web/ui/css/themes/element/theme.css index 4c617386..ca64e15a 100644 --- a/src/platform/web/ui/css/themes/element/theme.css +++ b/src/platform/web/ui/css/themes/element/theme.css @@ -1182,6 +1182,7 @@ button.RoomDetailsView_row::after { border: none; background: none; cursor: pointer; + text-align: left; } .LazyListParent { diff --git a/src/platform/web/ui/session/rightpanel/MemberDetailsView.js b/src/platform/web/ui/session/rightpanel/MemberDetailsView.js index 5d2f9387..72bd9e37 100644 --- a/src/platform/web/ui/session/rightpanel/MemberDetailsView.js +++ b/src/platform/web/ui/session/rightpanel/MemberDetailsView.js @@ -41,14 +41,22 @@ export class MemberDetailsView extends TemplateView { } _createOptions(t, vm) { + const options = [ + t.a({href: vm.linkToUser, target: "_blank", rel: "noopener"}, vm.i18n`Open Link to User`), + t.button({className: "text", onClick: () => vm.openDirectMessage()}, vm.i18n`Open direct message`) + ]; + if (vm.features.crossSigning) { + const onClick = () => { + if (confirm("You don't want to do this with any account but a test account. This will cross-sign this user without verifying their keys first. You won't be able to undo this apart from resetting your cross-signing keys.")) { + vm.signUser(); + } + } + options.push(t.button({className: "text", onClick}, vm.i18n`Cross-sign user (DO NOT USE, TESTING ONLY)`)) + } return t.div({ className: "MemberDetailsView_section" }, [ t.div({className: "MemberDetailsView_label"}, vm.i18n`Options`), - t.div({className: "MemberDetailsView_options"}, - [ - t.a({href: vm.linkToUser, target: "_blank", rel: "noopener"}, vm.i18n`Open Link to User`), - t.button({className: "text", onClick: () => vm.openDirectMessage()}, vm.i18n`Open direct message`) - ]) + t.div({className: "MemberDetailsView_options"}, options) ]); } } From 9789e5881d9165dbeb598fef60c22492861afd58 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Mar 2023 17:29:30 +0100 Subject: [PATCH 409/435] cleanup --- src/platform/web/ui/session/rightpanel/MemberDetailsView.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/platform/web/ui/session/rightpanel/MemberDetailsView.js b/src/platform/web/ui/session/rightpanel/MemberDetailsView.js index 72bd9e37..caa8037f 100644 --- a/src/platform/web/ui/session/rightpanel/MemberDetailsView.js +++ b/src/platform/web/ui/session/rightpanel/MemberDetailsView.js @@ -47,10 +47,10 @@ export class MemberDetailsView extends TemplateView { ]; if (vm.features.crossSigning) { const onClick = () => { - if (confirm("You don't want to do this with any account but a test account. This will cross-sign this user without verifying their keys first. You won't be able to undo this apart from resetting your cross-signing keys.")) { - vm.signUser(); - } + if (confirm("You don't want to do this with any account but a test account. This will cross-sign this user without verifying their keys first. You won't be able to undo this apart from resetting your cross-signing keys.")) { + vm.signUser(); } + }; options.push(t.button({className: "text", onClick}, vm.i18n`Cross-sign user (DO NOT USE, TESTING ONLY)`)) } return t.div({ className: "MemberDetailsView_section" }, From 1dc3acad036e01f5b8880ebaccaf65d16e7b2c2b Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 2 Mar 2023 17:32:46 +0100 Subject: [PATCH 410/435] use enum for device tracking status --- src/matrix/e2ee/DeviceTracker.ts | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/matrix/e2ee/DeviceTracker.ts b/src/matrix/e2ee/DeviceTracker.ts index 0464377a..bbfec900 100644 --- a/src/matrix/e2ee/DeviceTracker.ts +++ b/src/matrix/e2ee/DeviceTracker.ts @@ -29,20 +29,22 @@ import type {Transaction} from "../storage/idb/Transaction"; import type * as OlmNamespace from "@matrix-org/olm"; type Olm = typeof OlmNamespace; -const TRACKING_STATUS_OUTDATED = 0; -const TRACKING_STATUS_UPTODATE = 1; +enum DeviceTrackingStatus { + Outdated = 0, + UpToDate = 1 +} export type UserIdentity = { userId: string, roomIds: string[], - deviceTrackingStatus: number, + deviceTrackingStatus: DeviceTrackingStatus, } function createUserIdentity(userId: string, initialRoomId?: string): UserIdentity { return { userId: userId, roomIds: initialRoomId ? [initialRoomId] : [], - deviceTrackingStatus: TRACKING_STATUS_OUTDATED, + deviceTrackingStatus: DeviceTrackingStatus.Outdated, }; } @@ -87,7 +89,7 @@ export class DeviceTracker { const user = await userIdentities.get(userId); if (user) { log.log({l: "outdated", id: userId}); - user.deviceTrackingStatus = TRACKING_STATUS_OUTDATED; + user.deviceTrackingStatus = DeviceTrackingStatus.Outdated; userIdentities.set(user); } })); @@ -167,7 +169,7 @@ export class DeviceTracker { this._storage.storeNames.crossSigningKeys, ]); let userIdentity = await txn.userIdentities.get(userId); - if (userIdentity && userIdentity.deviceTrackingStatus !== TRACKING_STATUS_OUTDATED) { + if (userIdentity && userIdentity.deviceTrackingStatus !== DeviceTrackingStatus.Outdated) { return await txn.crossSigningKeys.get(userId, usage); } // fetch from hs @@ -345,7 +347,7 @@ export class DeviceTracker { // checked, we could share keys with that user without them being in the room identity = createUserIdentity(userId); } - identity.deviceTrackingStatus = TRACKING_STATUS_UPTODATE; + identity.deviceTrackingStatus = DeviceTrackingStatus.UpToDate; txn.userIdentities.set(identity); return allDeviceKeys; @@ -518,9 +520,9 @@ export class DeviceTracker { const outdatedUserIds: string[] = []; await Promise.all(userIds.map(async userId => { const i = await txn.userIdentities.get(userId); - if (i && i.deviceTrackingStatus === TRACKING_STATUS_UPTODATE) { + if (i && i.deviceTrackingStatus === DeviceTrackingStatus.UpToDate) { upToDateIdentities.push(i); - } else if (!i || i.deviceTrackingStatus === TRACKING_STATUS_OUTDATED) { + } else if (!i || i.deviceTrackingStatus === DeviceTrackingStatus.Outdated) { // allow fetching for userIdentities we don't know about yet, // as we don't assume the room is tracked here. outdatedUserIds.push(userId); @@ -599,9 +601,9 @@ export class DeviceTracker { // also exclude any userId which doesn't have a userIdentity yet. return identity && identity.roomIds.includes(roomId); }) as UserIdentity[]; // undefined has been filter out - const upToDateIdentities = identities.filter(i => i.deviceTrackingStatus === TRACKING_STATUS_UPTODATE); + const upToDateIdentities = identities.filter(i => i.deviceTrackingStatus === DeviceTrackingStatus.UpToDate); const outdatedUserIds = identities - .filter(i => i.deviceTrackingStatus === TRACKING_STATUS_OUTDATED) + .filter(i => i.deviceTrackingStatus === DeviceTrackingStatus.Outdated) .map(i => i.userId); let devices = await this._devicesForUserIdentities(upToDateIdentities, outdatedUserIds, hsApi, log); // filter out our own device as we should never share keys with it. @@ -761,12 +763,12 @@ export function tests() { assert.deepEqual(await txn.userIdentities.get("@alice:hs.tld"), { userId: "@alice:hs.tld", roomIds: [roomId], - deviceTrackingStatus: TRACKING_STATUS_OUTDATED + deviceTrackingStatus: DeviceTrackingStatus.Outdated }); assert.deepEqual(await txn.userIdentities.get("@bob:hs.tld"), { userId: "@bob:hs.tld", roomIds: [roomId], - deviceTrackingStatus: TRACKING_STATUS_OUTDATED + deviceTrackingStatus: DeviceTrackingStatus.Outdated }); assert.equal(await txn.userIdentities.get("@charly:hs.tld"), undefined); }, From 7d806b03b37da83311a02b49cf3b4607d0684ebd Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 3 Mar 2023 11:33:19 +0100 Subject: [PATCH 411/435] mark all existing user identities outdated as cross-signing keys missing --- src/matrix/storage/idb/schema.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/matrix/storage/idb/schema.ts b/src/matrix/storage/idb/schema.ts index c8d260cd..ce825edb 100644 --- a/src/matrix/storage/idb/schema.ts +++ b/src/matrix/storage/idb/schema.ts @@ -13,6 +13,7 @@ import {encodeScopeTypeKey} from "./stores/OperationStore"; import {MAX_UNICODE} from "./stores/common"; import {ILogItem} from "../../../logging/types"; +import type {UserIdentity} from "../../e2ee/DeviceTracker"; export type MigrationFunc = (db: IDBDatabase, txn: IDBTransaction, localStorage: IDOMStorage, log: ILogItem) => Promise | void; // FUNCTIONS SHOULD ONLY BE APPENDED!! @@ -35,7 +36,7 @@ export const schema: MigrationFunc[] = [ addInboundSessionBackupIndex, migrateBackupStatus, createCallStore, - createCrossSigningKeyStoreAndRenameDeviceIdentities + applyCrossSigningChanges ]; // TODO: how to deal with git merge conflicts of this array? @@ -277,10 +278,16 @@ function createCallStore(db: IDBDatabase) : void { db.createObjectStore("calls", {keyPath: "key"}); } -//v18 create calls store and rename deviceIdentities to deviceKeys -function createCrossSigningKeyStoreAndRenameDeviceIdentities(db: IDBDatabase) : void { +//v18 add crossSigningKeys store, rename deviceIdentities to deviceKeys and empties userIdentities +async function applyCrossSigningChanges(db: IDBDatabase, txn: IDBTransaction) : Promise { db.createObjectStore("crossSigningKeys", {keyPath: "key"}); db.deleteObjectStore("deviceIdentities"); const deviceKeys = db.createObjectStore("deviceKeys", {keyPath: "key"}); deviceKeys.createIndex("byCurve25519Key", "curve25519Key", {unique: true}); + // mark all userIdentities as outdated as cross-signing keys won't be stored + const userIdentities = txn.objectStore("userIdentities"); + await iterateCursor(userIdentities.openCursor(), (value, key, cursor) => { + value.deviceTrackingStatus = 0 // outdated; + return NOT_DONE; + }); } From c747d5f22828ec8397f97308506492acd1ed81c5 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 3 Mar 2023 11:34:09 +0100 Subject: [PATCH 412/435] rename deviceTrackingStatus to keysTrackingStatus as this field also reflects the tracking status of the cross-signing keys for a given user. --- src/matrix/e2ee/DeviceTracker.ts | 25 ++++++++++--------- src/matrix/storage/idb/schema.ts | 5 +++- .../storage/idb/stores/UserIdentityStore.ts | 7 +----- 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/matrix/e2ee/DeviceTracker.ts b/src/matrix/e2ee/DeviceTracker.ts index bbfec900..98221523 100644 --- a/src/matrix/e2ee/DeviceTracker.ts +++ b/src/matrix/e2ee/DeviceTracker.ts @@ -29,7 +29,8 @@ import type {Transaction} from "../storage/idb/Transaction"; import type * as OlmNamespace from "@matrix-org/olm"; type Olm = typeof OlmNamespace; -enum DeviceTrackingStatus { +// tracking status for cross-signing and device keys +export enum KeysTrackingStatus { Outdated = 0, UpToDate = 1 } @@ -37,14 +38,14 @@ enum DeviceTrackingStatus { export type UserIdentity = { userId: string, roomIds: string[], - deviceTrackingStatus: DeviceTrackingStatus, + keysTrackingStatus: KeysTrackingStatus, } function createUserIdentity(userId: string, initialRoomId?: string): UserIdentity { return { userId: userId, roomIds: initialRoomId ? [initialRoomId] : [], - deviceTrackingStatus: DeviceTrackingStatus.Outdated, + keysTrackingStatus: KeysTrackingStatus.Outdated, }; } @@ -89,7 +90,7 @@ export class DeviceTracker { const user = await userIdentities.get(userId); if (user) { log.log({l: "outdated", id: userId}); - user.deviceTrackingStatus = DeviceTrackingStatus.Outdated; + user.keysTrackingStatus = KeysTrackingStatus.Outdated; userIdentities.set(user); } })); @@ -169,7 +170,7 @@ export class DeviceTracker { this._storage.storeNames.crossSigningKeys, ]); let userIdentity = await txn.userIdentities.get(userId); - if (userIdentity && userIdentity.deviceTrackingStatus !== DeviceTrackingStatus.Outdated) { + if (userIdentity && userIdentity.keysTrackingStatus !== KeysTrackingStatus.Outdated) { return await txn.crossSigningKeys.get(userId, usage); } // fetch from hs @@ -347,7 +348,7 @@ export class DeviceTracker { // checked, we could share keys with that user without them being in the room identity = createUserIdentity(userId); } - identity.deviceTrackingStatus = DeviceTrackingStatus.UpToDate; + identity.keysTrackingStatus = KeysTrackingStatus.UpToDate; txn.userIdentities.set(identity); return allDeviceKeys; @@ -520,9 +521,9 @@ export class DeviceTracker { const outdatedUserIds: string[] = []; await Promise.all(userIds.map(async userId => { const i = await txn.userIdentities.get(userId); - if (i && i.deviceTrackingStatus === DeviceTrackingStatus.UpToDate) { + if (i && i.keysTrackingStatus === KeysTrackingStatus.UpToDate) { upToDateIdentities.push(i); - } else if (!i || i.deviceTrackingStatus === DeviceTrackingStatus.Outdated) { + } else if (!i || i.keysTrackingStatus === KeysTrackingStatus.Outdated) { // allow fetching for userIdentities we don't know about yet, // as we don't assume the room is tracked here. outdatedUserIds.push(userId); @@ -601,9 +602,9 @@ export class DeviceTracker { // also exclude any userId which doesn't have a userIdentity yet. return identity && identity.roomIds.includes(roomId); }) as UserIdentity[]; // undefined has been filter out - const upToDateIdentities = identities.filter(i => i.deviceTrackingStatus === DeviceTrackingStatus.UpToDate); + const upToDateIdentities = identities.filter(i => i.keysTrackingStatus === KeysTrackingStatus.UpToDate); const outdatedUserIds = identities - .filter(i => i.deviceTrackingStatus === DeviceTrackingStatus.Outdated) + .filter(i => i.keysTrackingStatus === KeysTrackingStatus.Outdated) .map(i => i.userId); let devices = await this._devicesForUserIdentities(upToDateIdentities, outdatedUserIds, hsApi, log); // filter out our own device as we should never share keys with it. @@ -763,12 +764,12 @@ export function tests() { assert.deepEqual(await txn.userIdentities.get("@alice:hs.tld"), { userId: "@alice:hs.tld", roomIds: [roomId], - deviceTrackingStatus: DeviceTrackingStatus.Outdated + keysTrackingStatus: KeysTrackingStatus.Outdated }); assert.deepEqual(await txn.userIdentities.get("@bob:hs.tld"), { userId: "@bob:hs.tld", roomIds: [roomId], - deviceTrackingStatus: DeviceTrackingStatus.Outdated + keysTrackingStatus: KeysTrackingStatus.Outdated }); assert.equal(await txn.userIdentities.get("@charly:hs.tld"), undefined); }, diff --git a/src/matrix/storage/idb/schema.ts b/src/matrix/storage/idb/schema.ts index ce825edb..c4c4bd61 100644 --- a/src/matrix/storage/idb/schema.ts +++ b/src/matrix/storage/idb/schema.ts @@ -14,6 +14,7 @@ import {MAX_UNICODE} from "./stores/common"; import {ILogItem} from "../../../logging/types"; import type {UserIdentity} from "../../e2ee/DeviceTracker"; +import {KeysTrackingStatus} from "../../e2ee/DeviceTracker"; export type MigrationFunc = (db: IDBDatabase, txn: IDBTransaction, localStorage: IDOMStorage, log: ILogItem) => Promise | void; // FUNCTIONS SHOULD ONLY BE APPENDED!! @@ -285,9 +286,11 @@ async function applyCrossSigningChanges(db: IDBDatabase, txn: IDBTransaction) : const deviceKeys = db.createObjectStore("deviceKeys", {keyPath: "key"}); deviceKeys.createIndex("byCurve25519Key", "curve25519Key", {unique: true}); // mark all userIdentities as outdated as cross-signing keys won't be stored + // also rename the deviceTrackingStatus field to keysTrackingStatus const userIdentities = txn.objectStore("userIdentities"); await iterateCursor(userIdentities.openCursor(), (value, key, cursor) => { - value.deviceTrackingStatus = 0 // outdated; + delete value["deviceTrackingStatus"]; + value.keysTrackingStatus = KeysTrackingStatus.Outdated; return NOT_DONE; }); } diff --git a/src/matrix/storage/idb/stores/UserIdentityStore.ts b/src/matrix/storage/idb/stores/UserIdentityStore.ts index 1c55baf0..76bb2080 100644 --- a/src/matrix/storage/idb/stores/UserIdentityStore.ts +++ b/src/matrix/storage/idb/stores/UserIdentityStore.ts @@ -14,12 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ import {Store} from "../Store"; - -interface UserIdentity { - userId: string; - roomIds: string[]; - deviceTrackingStatus: number; -} +import type {UserIdentity} from "../../../e2ee/DeviceTracker"; export class UserIdentityStore { private _store: Store; From 2563aa23e13ba8ae677283891c2dd9744430cc68 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 3 Mar 2023 11:56:51 +0100 Subject: [PATCH 413/435] actually write modified values in migration --- src/matrix/storage/idb/schema.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/matrix/storage/idb/schema.ts b/src/matrix/storage/idb/schema.ts index c4c4bd61..191d3c1d 100644 --- a/src/matrix/storage/idb/schema.ts +++ b/src/matrix/storage/idb/schema.ts @@ -291,6 +291,7 @@ async function applyCrossSigningChanges(db: IDBDatabase, txn: IDBTransaction) : await iterateCursor(userIdentities.openCursor(), (value, key, cursor) => { delete value["deviceTrackingStatus"]; value.keysTrackingStatus = KeysTrackingStatus.Outdated; + cursor.update(value); return NOT_DONE; }); } From 08984ad1bc0b06903c095ba31632ed3f8bf3e272 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 3 Mar 2023 11:57:15 +0100 Subject: [PATCH 414/435] log amount of marked user identities in migration --- src/matrix/storage/idb/schema.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/matrix/storage/idb/schema.ts b/src/matrix/storage/idb/schema.ts index 191d3c1d..fe7cd900 100644 --- a/src/matrix/storage/idb/schema.ts +++ b/src/matrix/storage/idb/schema.ts @@ -279,8 +279,8 @@ function createCallStore(db: IDBDatabase) : void { db.createObjectStore("calls", {keyPath: "key"}); } -//v18 add crossSigningKeys store, rename deviceIdentities to deviceKeys and empties userIdentities -async function applyCrossSigningChanges(db: IDBDatabase, txn: IDBTransaction) : Promise { +//v18 add crossSigningKeys store, rename deviceIdentities to deviceKeys and empties userIdentities +async function applyCrossSigningChanges(db: IDBDatabase, txn: IDBTransaction, localStorage: IDOMStorage, log: ILogItem) : Promise { db.createObjectStore("crossSigningKeys", {keyPath: "key"}); db.deleteObjectStore("deviceIdentities"); const deviceKeys = db.createObjectStore("deviceKeys", {keyPath: "key"}); @@ -288,10 +288,13 @@ async function applyCrossSigningChanges(db: IDBDatabase, txn: IDBTransaction) : // mark all userIdentities as outdated as cross-signing keys won't be stored // also rename the deviceTrackingStatus field to keysTrackingStatus const userIdentities = txn.objectStore("userIdentities"); + let counter = 0; await iterateCursor(userIdentities.openCursor(), (value, key, cursor) => { delete value["deviceTrackingStatus"]; value.keysTrackingStatus = KeysTrackingStatus.Outdated; cursor.update(value); + counter += 1; return NOT_DONE; }); + log.set("marked_outdated", counter); } From eff495c36d825e7bd7079fe6806595766fcf53e1 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 3 Mar 2023 11:57:29 +0100 Subject: [PATCH 415/435] also delete old crossSigningKeys field on userIdentities --- src/matrix/storage/idb/schema.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/matrix/storage/idb/schema.ts b/src/matrix/storage/idb/schema.ts index fe7cd900..9b4d5547 100644 --- a/src/matrix/storage/idb/schema.ts +++ b/src/matrix/storage/idb/schema.ts @@ -291,6 +291,7 @@ async function applyCrossSigningChanges(db: IDBDatabase, txn: IDBTransaction, lo let counter = 0; await iterateCursor(userIdentities.openCursor(), (value, key, cursor) => { delete value["deviceTrackingStatus"]; + delete value["crossSigningKeys"]; value.keysTrackingStatus = KeysTrackingStatus.Outdated; cursor.update(value); counter += 1; From c2ee824c1c9882546c074dfeb5bf133d5f3c2c68 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 3 Mar 2023 12:03:31 +0100 Subject: [PATCH 416/435] fix lint warning from previous cross-signing PR --- src/platform/web/ui/session/settings/KeyBackupSettingsView.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/platform/web/ui/session/settings/KeyBackupSettingsView.js b/src/platform/web/ui/session/settings/KeyBackupSettingsView.js index a68a80b3..6a886e3a 100644 --- a/src/platform/web/ui/session/settings/KeyBackupSettingsView.js +++ b/src/platform/web/ui/session/settings/KeyBackupSettingsView.js @@ -60,7 +60,7 @@ export class KeyBackupSettingsView extends TemplateView { }), t.if(vm => vm.canSignOwnDevice, t => { return t.button({ - onClick: disableTargetCallback(async evt => { + onClick: disableTargetCallback(async () => { await vm.signOwnDevice(); }) }, "Sign own device"); From 774efc17d9eb989a9743c61c219abe051660ad5e Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 3 Mar 2023 12:15:54 +0100 Subject: [PATCH 417/435] extract method to sign key, as most params are always the same --- src/matrix/verification/CrossSigning.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/matrix/verification/CrossSigning.ts b/src/matrix/verification/CrossSigning.ts index 77b489d2..f98c469a 100644 --- a/src/matrix/verification/CrossSigning.ts +++ b/src/matrix/verification/CrossSigning.ts @@ -145,7 +145,7 @@ export class CrossSigning { delete keyToSign.signatures; const signingKey = await this.getSigningKey(KeyUsage.UserSigning); // add signature to keyToSign - pkSign(this.olm, keyToSign, signingKey, this.ownUserId, ""); + this.signKey(keyToSign, signingKey); const payload = { [keyToSign.user_id]: { [getKeyEd25519Key(keyToSign)!]: keyToSign @@ -160,7 +160,7 @@ export class CrossSigning { private async signDeviceKey(keyToSign: DeviceKey, log: ILogItem): Promise { const signingKey = await this.getSigningKey(KeyUsage.SelfSigning); // add signature to keyToSign - pkSign(this.olm, keyToSign, signingKey, this.ownUserId, ""); + this.signKey(keyToSign, signingKey); // so the payload format of a signature is a map from userid to key id of the signed key // (without the algoritm prefix though according to example, e.g. just device id or base 64 public key) // to the complete signed key with the signature of the signing key in the signatures section. @@ -180,6 +180,10 @@ export class CrossSigning { const seed = new Uint8Array(this.platform.encoding.base64.decode(seedStr)); return seed; } + + private signKey(keyToSign: DeviceKey | CrossSigningKey, signingKey: Uint8Array) { + pkSign(this.olm, keyToSign, signingKey, this.ownUserId, ""); + } } export function getKeyUsage(keyInfo: CrossSigningKey): KeyUsage | undefined { From 4c7f7849115510233ba7b6d8497dbc12264be567 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 3 Mar 2023 15:21:37 +0100 Subject: [PATCH 418/435] implement verifying signaturs for user trust (green shield/red shield) --- src/matrix/Session.js | 1 + src/matrix/verification/CrossSigning.ts | 41 ++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 9aa0494d..82eeba68 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -339,6 +339,7 @@ export class Session { secretStorage, platform: this._platform, olm: this._olm, + olmUtil: this._olmUtil, deviceTracker: this._deviceTracker, hsApi: this._hsApi, ownUserId: this.userId, diff --git a/src/matrix/verification/CrossSigning.ts b/src/matrix/verification/CrossSigning.ts index f98c469a..3d2fb930 100644 --- a/src/matrix/verification/CrossSigning.ts +++ b/src/matrix/verification/CrossSigning.ts @@ -16,6 +16,7 @@ limitations under the License. import { ILogItem } from "../../lib"; import {pkSign} from "./common"; +import {verifyEd25519Signature} from "../e2ee/common"; import type {SecretStorage} from "../ssss/SecretStorage"; import type {Storage} from "../storage/idb/Storage"; @@ -48,6 +49,7 @@ export class CrossSigning { private readonly platform: Platform; private readonly deviceTracker: DeviceTracker; private readonly olm: Olm; + private readonly olmUtil: Olm.Utility; private readonly hsApi: HomeServerApi; private readonly ownUserId: string; private readonly e2eeAccount: Account; @@ -68,13 +70,14 @@ export class CrossSigning { this.platform = options.platform; this.deviceTracker = options.deviceTracker; this.olm = options.olm; + this.olmUtil = options.olmUtil; this.hsApi = options.hsApi; this.ownUserId = options.ownUserId; this.e2eeAccount = options.e2eeAccount } async init(log: ILogItem) { - log.wrap("CrossSigning.init", async log => { + await log.wrap("CrossSigning.init", async log => { // TODO: use errorboundary here const txn = await this.storage.readTxn([this.storage.storeNames.accountData]); const privateMasterKey = await this.getSigningKey(KeyUsage.Master); @@ -157,6 +160,34 @@ export class CrossSigning { }); } + async isUserTrusted(userId: string, log: ILogItem): Promise { + return log.wrap("isUserTrusted", async log => { + log.set("id", userId); + if (!this.isMasterKeyTrusted) { + return false; + } + const theirDeviceKeys = await log.wrap("get their devices", log => this.deviceTracker.devicesForUsers([userId], this.hsApi, log)); + const theirSSK = await log.wrap("get their ssk", log => this.deviceTracker.getCrossSigningKeyForUser(userId, KeyUsage.SelfSigning, this.hsApi, log)); + if (!theirSSK) { + return false; + } + const hasUnsignedDevice = theirDeviceKeys.some(dk => log.wrap({l: "verify device", id: dk.device_id}, log => !this.hasValidSignatureFrom(dk, theirSSK, log))); + if (hasUnsignedDevice) { + return false; + } + const theirMSK = await log.wrap("get their msk", log => this.deviceTracker.getCrossSigningKeyForUser(userId, KeyUsage.Master, this.hsApi, log)); + if (!theirMSK || !log.wrap("verify their ssk", log => this.hasValidSignatureFrom(theirSSK, theirMSK, log))) { + return false; + } + const ourUSK = await log.wrap("get our usk", log => this.deviceTracker.getCrossSigningKeyForUser(this.ownUserId, KeyUsage.UserSigning, this.hsApi, log)); + if (!ourUSK || !log.wrap("verify their msk", log => this.hasValidSignatureFrom(theirMSK, ourUSK, log))) { + return false; + } + const ourMSK = await log.wrap("get our msk", log => this.deviceTracker.getCrossSigningKeyForUser(this.ownUserId, KeyUsage.Master, this.hsApi, log)); + return !!ourMSK && log.wrap("verify our usk", log => this.hasValidSignatureFrom(ourUSK, ourMSK, log)); + }); + } + private async signDeviceKey(keyToSign: DeviceKey, log: ILogItem): Promise { const signingKey = await this.getSigningKey(KeyUsage.SelfSigning); // add signature to keyToSign @@ -184,6 +215,14 @@ export class CrossSigning { private signKey(keyToSign: DeviceKey | CrossSigningKey, signingKey: Uint8Array) { pkSign(this.olm, keyToSign, signingKey, this.ownUserId, ""); } + + private hasValidSignatureFrom(key: DeviceKey | CrossSigningKey, signingKey: CrossSigningKey, log: ILogItem): boolean { + const pubKey = getKeyEd25519Key(signingKey); + if (!pubKey) { + return false; + } + return verifyEd25519Signature(this.olmUtil, signingKey.user_id, pubKey, pubKey, key, log); + } } export function getKeyUsage(keyInfo: CrossSigningKey): KeyUsage | undefined { From 149f18790464fcb68df58ac352c8a372c0addf04 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 3 Mar 2023 15:22:02 +0100 Subject: [PATCH 419/435] expose user trust in member panel --- .../session/rightpanel/MemberDetailsViewModel.js | 12 ++++++++++++ .../ui/session/rightpanel/MemberDetailsView.js | 15 +++++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/domain/session/rightpanel/MemberDetailsViewModel.js b/src/domain/session/rightpanel/MemberDetailsViewModel.js index 774f64af..df622aae 100644 --- a/src/domain/session/rightpanel/MemberDetailsViewModel.js +++ b/src/domain/session/rightpanel/MemberDetailsViewModel.js @@ -29,10 +29,22 @@ export class MemberDetailsViewModel extends ViewModel { this._session = options.session; this.track(this._powerLevelsObservable.subscribe(() => this._onPowerLevelsChange())); this.track(this._observableMember.subscribe( () => this._onMemberChange())); + this._isTrusted = false; + this.init(); // TODO: call this from parent view model and do something smart with error view model if it fails async? + } + + async init() { + if (this.features.crossSigning) { + this._isTrusted = await this.logger.run({l: "MemberDetailsViewModel.verify user", id: this._member.userId}, log => { + return this._session.crossSigning.isUserTrusted(this._member.userId, log); + }); + this.emitChange("isTrusted"); + } } get name() { return this._member.name; } get userId() { return this._member.userId; } + get isTrusted() { return this._isTrusted; } get type() { return "member-details"; } get shouldShowBackButton() { return true; } diff --git a/src/platform/web/ui/session/rightpanel/MemberDetailsView.js b/src/platform/web/ui/session/rightpanel/MemberDetailsView.js index caa8037f..45504a74 100644 --- a/src/platform/web/ui/session/rightpanel/MemberDetailsView.js +++ b/src/platform/web/ui/session/rightpanel/MemberDetailsView.js @@ -19,15 +19,22 @@ import {TemplateView} from "../../general/TemplateView"; export class MemberDetailsView extends TemplateView { render(t, vm) { + const securityNodes = [ + t.p(vm.isEncrypted ? + vm.i18n`Messages in this room are end-to-end encrypted.` : + vm.i18n`Messages in this room are not end-to-end encrypted.`), + ] + + if (vm.features.crossSigning) { + securityNodes.push(t.p(vm => vm.isTrusted ? vm.i18n`This user is trusted` : vm.i18n`This user is not trusted`)); + } + return t.div({className: "MemberDetailsView"}, [ t.view(new AvatarView(vm, 128)), t.div({className: "MemberDetailsView_name"}, t.h2(vm => vm.name)), t.div({className: "MemberDetailsView_id"}, vm.userId), this._createSection(t, vm.i18n`Role`, vm => vm.role), - this._createSection(t, vm.i18n`Security`, vm.isEncrypted ? - vm.i18n`Messages in this room are end-to-end encrypted.` : - vm.i18n`Messages in this room are not end-to-end encrypted.` - ), + this._createSection(t, vm.i18n`Security`, securityNodes), this._createOptions(t, vm) ]); } From e00d02a599f2067cdc448745e4be1433a17a5d22 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 3 Mar 2023 16:18:30 +0100 Subject: [PATCH 420/435] fix ts error --- src/matrix/verification/CrossSigning.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/matrix/verification/CrossSigning.ts b/src/matrix/verification/CrossSigning.ts index 3d2fb930..975f0d43 100644 --- a/src/matrix/verification/CrossSigning.ts +++ b/src/matrix/verification/CrossSigning.ts @@ -61,6 +61,7 @@ export class CrossSigning { deviceTracker: DeviceTracker, platform: Platform, olm: Olm, + olmUtil: Olm.Utility, ownUserId: string, hsApi: HomeServerApi, e2eeAccount: Account From bae18c037f3103a4163e8283556255605daf5a3a Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Mar 2023 10:53:32 +0100 Subject: [PATCH 421/435] return enum explaining user trust level rather than boolean --- src/matrix/e2ee/DeviceTracker.ts | 4 +- src/matrix/e2ee/common.ts | 14 +++- src/matrix/e2ee/olm/Encryption.ts | 4 +- src/matrix/verification/CrossSigning.ts | 107 ++++++++++++++++++------ 4 files changed, 97 insertions(+), 32 deletions(-) diff --git a/src/matrix/e2ee/DeviceTracker.ts b/src/matrix/e2ee/DeviceTracker.ts index 98221523..ae00b1e0 100644 --- a/src/matrix/e2ee/DeviceTracker.ts +++ b/src/matrix/e2ee/DeviceTracker.ts @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {verifyEd25519Signature, getEd25519Signature, SIGNATURE_ALGORITHM} from "./common"; +import {verifyEd25519Signature, getEd25519Signature, SIGNATURE_ALGORITHM, SignatureVerification} from "./common"; import {HistoryVisibility, shouldShareKey, DeviceKey, getDeviceEd25519Key, getDeviceCurve25519Key} from "./common"; import {RoomMember} from "../room/members/RoomMember.js"; import {getKeyUsage, getKeyEd25519Key, getKeyUserId, KeyUsage} from "../verification/CrossSigning"; @@ -462,7 +462,7 @@ export class DeviceTracker { log.log("ed25519 and/or curve25519 key invalid").set({deviceKey}); return false; } - const isValid = verifyEd25519Signature(this._olmUtil, userId, deviceId, ed25519Key, deviceKey, log); + const isValid = verifyEd25519Signature(this._olmUtil, userId, deviceId, ed25519Key, deviceKey, log) === SignatureVerification.Valid; if (!isValid) { log.log({ l: "ignore device with invalid signature", diff --git a/src/matrix/e2ee/common.ts b/src/matrix/e2ee/common.ts index 27078135..c8a5ec0f 100644 --- a/src/matrix/e2ee/common.ts +++ b/src/matrix/e2ee/common.ts @@ -68,11 +68,17 @@ export function getEd25519Signature(signedValue: SignedValue, userId: string, de return signedValue?.signatures?.[userId]?.[`${SIGNATURE_ALGORITHM}:${deviceOrKeyId}`]; } -export function verifyEd25519Signature(olmUtil: Olm.Utility, userId: string, deviceOrKeyId: string, ed25519Key: string, value: SignedValue, log?: ILogItem) { +export enum SignatureVerification { + Valid, + Invalid, + NotSigned, +} + +export function verifyEd25519Signature(olmUtil: Olm.Utility, userId: string, deviceOrKeyId: string, ed25519Key: string, value: SignedValue, log?: ILogItem): SignatureVerification { const signature = getEd25519Signature(value, userId, deviceOrKeyId); if (!signature) { log?.set("no_signature", true); - return false; + return SignatureVerification.NotSigned; } const clone = Object.assign({}, value) as object; delete clone["unsigned"]; @@ -81,14 +87,14 @@ export function verifyEd25519Signature(olmUtil: Olm.Utility, userId: string, dev try { // throws when signature is invalid olmUtil.ed25519_verify(ed25519Key, canonicalJson, signature); - return true; + return SignatureVerification.Valid; } catch (err) { if (log) { const logItem = log.log({l: "Invalid signature, ignoring.", ed25519Key, canonicalJson, signature}); logItem.error = err; logItem.logLevel = log.level.Warn; } - return false; + return SignatureVerification.Invalid; } } diff --git a/src/matrix/e2ee/olm/Encryption.ts b/src/matrix/e2ee/olm/Encryption.ts index c4fee911..ef16ba45 100644 --- a/src/matrix/e2ee/olm/Encryption.ts +++ b/src/matrix/e2ee/olm/Encryption.ts @@ -15,7 +15,7 @@ limitations under the License. */ import {groupByWithCreator} from "../../../utils/groupBy"; -import {verifyEd25519Signature, OLM_ALGORITHM, getDeviceCurve25519Key, getDeviceEd25519Key} from "../common"; +import {verifyEd25519Signature, OLM_ALGORITHM, getDeviceCurve25519Key, getDeviceEd25519Key, SignatureVerification} from "../common"; import {createSessionEntry} from "./Session"; import type {OlmMessage, OlmPayload, OlmEncryptedMessageContent} from "./types"; @@ -260,7 +260,7 @@ export class Encryption { const device = devicesByUser.get(userId)?.get(deviceId); if (device) { const isValidSignature = verifyEd25519Signature( - this.olmUtil, userId, deviceId, getDeviceEd25519Key(device), keySection, log); + this.olmUtil, userId, deviceId, getDeviceEd25519Key(device), keySection, log) === SignatureVerification.Valid; if (isValidSignature) { const target = EncryptionTarget.fromOTK(device, keySection.key); verifiedEncryptionTargets.push(target); diff --git a/src/matrix/verification/CrossSigning.ts b/src/matrix/verification/CrossSigning.ts index 975f0d43..c35914bb 100644 --- a/src/matrix/verification/CrossSigning.ts +++ b/src/matrix/verification/CrossSigning.ts @@ -16,7 +16,7 @@ limitations under the License. import { ILogItem } from "../../lib"; import {pkSign} from "./common"; -import {verifyEd25519Signature} from "../e2ee/common"; +import {verifyEd25519Signature, SignatureVerification} from "../e2ee/common"; import type {SecretStorage} from "../ssss/SecretStorage"; import type {Storage} from "../storage/idb/Storage"; @@ -43,6 +43,27 @@ export enum KeyUsage { UserSigning = "user_signing" }; +export enum UserTrust { + /** We trust the user, the whole signature chain checks out from our MSK to all of their device keys. */ + Trusted = 1, + /** We haven't signed this user's identity yet. Verify this user first to sign it. */ + UserNotSigned, + /** We have signed the user already, but the signature isn't valid. + One possible cause could be that an attacker is uploading signatures in our name. */ + UserSignatureMismatch, + /** We trust the user, but they don't trust one of their devices. */ + UserDeviceNotSigned, + /** We trust the user, but the signatures of one of their devices is invalid. + * One possible cause could be that an attacker is uploading signatures in their name. */ + UserDeviceSignatureMismatch, + /** The user doesn't have a valid signature for the SSK with their MSK, or the SSK is missing. + * This likely means bootstrapping cross-signing on their end didn't finish correctly. */ + UserSetupError, + /** We don't have a valid signature for our SSK with our MSK, the SSK is missing, or we don't trust our own MSK. + * This likely means bootstrapping cross-signing on our end didn't finish correctly. */ + OwnSetupError +} + export class CrossSigning { private readonly storage: Storage; private readonly secretStorage: SecretStorage; @@ -161,31 +182,69 @@ export class CrossSigning { }); } - async isUserTrusted(userId: string, log: ILogItem): Promise { - return log.wrap("isUserTrusted", async log => { + async getUserTrust(userId: string, log: ILogItem): Promise { + return log.wrap("getUserTrust", async log => { log.set("id", userId); if (!this.isMasterKeyTrusted) { - return false; - } - const theirDeviceKeys = await log.wrap("get their devices", log => this.deviceTracker.devicesForUsers([userId], this.hsApi, log)); - const theirSSK = await log.wrap("get their ssk", log => this.deviceTracker.getCrossSigningKeyForUser(userId, KeyUsage.SelfSigning, this.hsApi, log)); - if (!theirSSK) { - return false; - } - const hasUnsignedDevice = theirDeviceKeys.some(dk => log.wrap({l: "verify device", id: dk.device_id}, log => !this.hasValidSignatureFrom(dk, theirSSK, log))); - if (hasUnsignedDevice) { - return false; - } - const theirMSK = await log.wrap("get their msk", log => this.deviceTracker.getCrossSigningKeyForUser(userId, KeyUsage.Master, this.hsApi, log)); - if (!theirMSK || !log.wrap("verify their ssk", log => this.hasValidSignatureFrom(theirSSK, theirMSK, log))) { - return false; - } - const ourUSK = await log.wrap("get our usk", log => this.deviceTracker.getCrossSigningKeyForUser(this.ownUserId, KeyUsage.UserSigning, this.hsApi, log)); - if (!ourUSK || !log.wrap("verify their msk", log => this.hasValidSignatureFrom(theirMSK, ourUSK, log))) { - return false; + return UserTrust.OwnSetupError; } const ourMSK = await log.wrap("get our msk", log => this.deviceTracker.getCrossSigningKeyForUser(this.ownUserId, KeyUsage.Master, this.hsApi, log)); - return !!ourMSK && log.wrap("verify our usk", log => this.hasValidSignatureFrom(ourUSK, ourMSK, log)); + if (!ourMSK) { + return UserTrust.OwnSetupError; + } + const ourUSK = await log.wrap("get our usk", log => this.deviceTracker.getCrossSigningKeyForUser(this.ownUserId, KeyUsage.UserSigning, this.hsApi, log)); + if (!ourUSK) { + return UserTrust.OwnSetupError; + } + const ourUSKVerification = log.wrap("verify our usk", log => this.hasValidSignatureFrom(ourUSK, ourMSK, log)); + if (ourUSKVerification !== SignatureVerification.Valid) { + return UserTrust.OwnSetupError; + } + const theirMSK = await log.wrap("get their msk", log => this.deviceTracker.getCrossSigningKeyForUser(userId, KeyUsage.Master, this.hsApi, log)); + if (!theirMSK) { + /* assume that when they don't have an MSK, they've never enabled cross-signing on their client + (or it's not supported) rather than assuming a setup error on their side. + Later on, for their SSK, we _do_ assume it's a setup error as it doesn't make sense to have an MSK without a SSK */ + return UserTrust.UserNotSigned; + } + const theirMSKVerification = log.wrap("verify their msk", log => this.hasValidSignatureFrom(theirMSK, ourUSK, log)); + if (theirMSKVerification !== SignatureVerification.Valid) { + if (theirMSKVerification === SignatureVerification.NotSigned) { + return UserTrust.UserNotSigned; + } else { /* SignatureVerification.Invalid */ + return UserTrust.UserSignatureMismatch; + } + } + const theirSSK = await log.wrap("get their ssk", log => this.deviceTracker.getCrossSigningKeyForUser(userId, KeyUsage.SelfSigning, this.hsApi, log)); + if (!theirSSK) { + return UserTrust.UserSetupError; + } + const theirSSKVerification = log.wrap("verify their ssk", log => this.hasValidSignatureFrom(theirSSK, theirMSK, log)); + if (theirSSKVerification !== SignatureVerification.Valid) { + return UserTrust.UserSetupError; + } + const theirDeviceKeys = await log.wrap("get their devices", log => this.deviceTracker.devicesForUsers([userId], this.hsApi, log)); + const lowestDeviceVerification = theirDeviceKeys.reduce((lowest, dk) => log.wrap({l: "verify device", id: dk.device_id}, log => { + const verification = this.hasValidSignatureFrom(dk, theirSSK, log); + // first Invalid, then NotSigned, then Valid + if (lowest === SignatureVerification.Invalid || verification === SignatureVerification.Invalid) { + return SignatureVerification.Invalid; + } else if (lowest === SignatureVerification.NotSigned || verification === SignatureVerification.NotSigned) { + return SignatureVerification.NotSigned; + } else if (lowest === SignatureVerification.Valid || verification === SignatureVerification.Valid) { + return SignatureVerification.Valid; + } + // should never happen as we went over all the enum options + return SignatureVerification.Invalid; + }), SignatureVerification.Valid); + if (lowestDeviceVerification !== SignatureVerification.Valid) { + if (lowestDeviceVerification === SignatureVerification.NotSigned) { + return UserTrust.UserDeviceNotSigned; + } else { /* SignatureVerification.Invalid */ + return UserTrust.UserDeviceSignatureMismatch; + } + } + return UserTrust.Trusted; }); } @@ -217,10 +276,10 @@ export class CrossSigning { pkSign(this.olm, keyToSign, signingKey, this.ownUserId, ""); } - private hasValidSignatureFrom(key: DeviceKey | CrossSigningKey, signingKey: CrossSigningKey, log: ILogItem): boolean { + private hasValidSignatureFrom(key: DeviceKey | CrossSigningKey, signingKey: CrossSigningKey, log: ILogItem): SignatureVerification { const pubKey = getKeyEd25519Key(signingKey); if (!pubKey) { - return false; + return SignatureVerification.NotSigned; } return verifyEd25519Signature(this.olmUtil, signingKey.user_id, pubKey, pubKey, key, log); } From f1ecad5b58bd2e7450926111fb073ca91cee9851 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Mar 2023 10:54:07 +0100 Subject: [PATCH 422/435] adjust UI to more detailed trust level --- .../rightpanel/MemberDetailsViewModel.js | 37 +++++++++++++++++-- .../session/rightpanel/MemberDetailsView.js | 2 + 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/domain/session/rightpanel/MemberDetailsViewModel.js b/src/domain/session/rightpanel/MemberDetailsViewModel.js index df622aae..0eabf33a 100644 --- a/src/domain/session/rightpanel/MemberDetailsViewModel.js +++ b/src/domain/session/rightpanel/MemberDetailsViewModel.js @@ -17,6 +17,7 @@ limitations under the License. import {ViewModel} from "../../ViewModel"; import {RoomType} from "../../../matrix/room/common"; import {avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../avatar"; +import {UserTrust} from "../../../matrix/verification/CrossSigning"; export class MemberDetailsViewModel extends ViewModel { constructor(options) { @@ -29,14 +30,14 @@ export class MemberDetailsViewModel extends ViewModel { this._session = options.session; this.track(this._powerLevelsObservable.subscribe(() => this._onPowerLevelsChange())); this.track(this._observableMember.subscribe( () => this._onMemberChange())); - this._isTrusted = false; + this._userTrust = undefined; this.init(); // TODO: call this from parent view model and do something smart with error view model if it fails async? } async init() { if (this.features.crossSigning) { - this._isTrusted = await this.logger.run({l: "MemberDetailsViewModel.verify user", id: this._member.userId}, log => { - return this._session.crossSigning.isUserTrusted(this._member.userId, log); + this._userTrust = await this.logger.run({l: "MemberDetailsViewModel.get user trust", id: this._member.userId}, log => { + return this._session.crossSigning.getUserTrust(this._member.userId, log); }); this.emitChange("isTrusted"); } @@ -44,7 +45,35 @@ export class MemberDetailsViewModel extends ViewModel { get name() { return this._member.name; } get userId() { return this._member.userId; } - get isTrusted() { return this._isTrusted; } + get isTrusted() { return this._userTrust === UserTrust.Trusted; } + get trustDescription() { + switch (this._userTrust) { + case UserTrust.Trusted: return this.i18n`You have verified this user. This user has verified all of their sessions.`; + case UserTrust.UserNotSigned: return this.i18n`You have not verified this user.`; + case UserTrust.UserSignatureMismatch: return this.i18n`You appear to have signed this user, but the signature is invalid.`; + case UserTrust.UserDeviceNotSigned: return this.i18n`You have verified this user, but they have one or more unverified sessions.`; + case UserTrust.UserDeviceSignatureMismatch: return this.i18n`This user has a session signature that is invalid.`; + case UserTrust.UserSetupError: return this.i18n`This user hasn't set up cross-signing correctly`; + case UserTrust.OwnSetupError: return this.i18n`Cross-signing wasn't set up correctly on your side.`; + default: return this.i18n`Pending…`; + } + } + get trustShieldColor() { + if (!this._isEncrypted) { + return undefined; + } + switch (this._userTrust) { + case undefined: + case UserTrust.OwnSetupError: + return undefined; + case UserTrust.Trusted: + return "green"; + case UserTrust.UserNotSigned: + return "black"; + default: + return "red"; + } + } get type() { return "member-details"; } get shouldShowBackButton() { return true; } diff --git a/src/platform/web/ui/session/rightpanel/MemberDetailsView.js b/src/platform/web/ui/session/rightpanel/MemberDetailsView.js index 45504a74..aa70d5b4 100644 --- a/src/platform/web/ui/session/rightpanel/MemberDetailsView.js +++ b/src/platform/web/ui/session/rightpanel/MemberDetailsView.js @@ -27,6 +27,8 @@ export class MemberDetailsView extends TemplateView { if (vm.features.crossSigning) { securityNodes.push(t.p(vm => vm.isTrusted ? vm.i18n`This user is trusted` : vm.i18n`This user is not trusted`)); + securityNodes.push(t.p(vm => vm.trustDescription)); + securityNodes.push(t.p(["Shield color: ", vm => vm.trustShieldColor])); } return t.div({className: "MemberDetailsView"}, From a065189836e0c07769b7260e0d583e762d81143a Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Mar 2023 11:00:52 +0100 Subject: [PATCH 423/435] delay signature validation of cross-signing keys until calculating trust always store them, if not we'll think that the user hasn't uploaded the cross-signing keys if we don't store them in spite of invalid or missing signature. --- src/matrix/e2ee/DeviceTracker.ts | 32 ++++++-------------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/src/matrix/e2ee/DeviceTracker.ts b/src/matrix/e2ee/DeviceTracker.ts index ae00b1e0..8a6e351b 100644 --- a/src/matrix/e2ee/DeviceTracker.ts +++ b/src/matrix/e2ee/DeviceTracker.ts @@ -264,9 +264,9 @@ export class DeviceTracker { "token": this._getSyncToken() }, {log}).response(); - const masterKeys = log.wrap("master keys", log => this._filterVerifiedCrossSigningKeys(deviceKeyResponse["master_keys"], KeyUsage.Master, undefined, log)); - const selfSigningKeys = log.wrap("self-signing keys", log => this._filterVerifiedCrossSigningKeys(deviceKeyResponse["self_signing_keys"], KeyUsage.SelfSigning, masterKeys, log)); - const userSigningKeys = log.wrap("user-signing keys", log => this._filterVerifiedCrossSigningKeys(deviceKeyResponse["user_signing_keys"], KeyUsage.UserSigning, masterKeys, log)); + const masterKeys = log.wrap("master keys", log => this._filterVerifiedCrossSigningKeys(deviceKeyResponse["master_keys"], KeyUsage.Master, log)); + const selfSigningKeys = log.wrap("self-signing keys", log => this._filterVerifiedCrossSigningKeys(deviceKeyResponse["self_signing_keys"], KeyUsage.SelfSigning, log)); + const userSigningKeys = log.wrap("user-signing keys", log => this._filterVerifiedCrossSigningKeys(deviceKeyResponse["user_signing_keys"], KeyUsage.UserSigning, log)); const deviceKeys = log.wrap("device keys", log => this._filterVerifiedDeviceKeys(deviceKeyResponse["device_keys"], log)); const txn = await this._storage.readWriteTxn([ this._storage.storeNames.userIdentities, @@ -354,16 +354,14 @@ export class DeviceTracker { return allDeviceKeys; } - _filterVerifiedCrossSigningKeys(crossSigningKeysResponse: {[userId: string]: CrossSigningKey}, usage, parentKeys: Map | undefined, log): Map { + _filterVerifiedCrossSigningKeys(crossSigningKeysResponse: {[userId: string]: CrossSigningKey}, usage: KeyUsage, log: ILogItem): Map { const keys: Map = new Map(); if (!crossSigningKeysResponse) { return keys; } for (const [userId, keyInfo] of Object.entries(crossSigningKeysResponse)) { log.wrap({l: userId}, log => { - const parentKeyInfo = parentKeys?.get(userId); - const parentKey = parentKeyInfo && getKeyEd25519Key(parentKeyInfo); - if (this._validateCrossSigningKey(userId, keyInfo, usage, parentKey, log)) { + if (this._validateCrossSigningKey(userId, keyInfo, usage, log)) { keys.set(getKeyUserId(keyInfo)!, keyInfo); } }); @@ -371,7 +369,7 @@ export class DeviceTracker { return keys; } - _validateCrossSigningKey(userId: string, keyInfo: CrossSigningKey, usage: KeyUsage, parentKey: string | undefined, log: ILogItem): boolean { + _validateCrossSigningKey(userId: string, keyInfo: CrossSigningKey, usage: KeyUsage, log: ILogItem): boolean { if (getKeyUserId(keyInfo) !== userId) { log.log({l: "user_id mismatch", userId: keyInfo["user_id"]}); return false; @@ -385,24 +383,6 @@ export class DeviceTracker { log.log({l: "no ed25519 key", keys: keyInfo.keys}); return false; } - const isSelfSigned = usage === "master"; - const keyToVerifyWith = isSelfSigned ? publicKey : parentKey; - if (!keyToVerifyWith) { - log.log("signing_key not found"); - return false; - } - const hasSignature = !!getEd25519Signature(keyInfo, userId, keyToVerifyWith); - // self-signature is optional for now, not all keys seem to have it - if (!hasSignature && keyToVerifyWith !== publicKey) { - log.log({l: "signature not found", key: keyToVerifyWith}); - return false; - } - if (hasSignature) { - if(!verifyEd25519Signature(this._olmUtil, userId, keyToVerifyWith, keyToVerifyWith, keyInfo, log)) { - log.log("signature mismatch"); - return false; - } - } return true; } From a69246fb5ad32e2b241e415f8c73f813bb9f57ad Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 7 Mar 2023 14:40:11 +0100 Subject: [PATCH 424/435] return undefined if we don't have the signing key --- src/matrix/verification/CrossSigning.ts | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/matrix/verification/CrossSigning.ts b/src/matrix/verification/CrossSigning.ts index c35914bb..05be9ba1 100644 --- a/src/matrix/verification/CrossSigning.ts +++ b/src/matrix/verification/CrossSigning.ts @@ -165,10 +165,13 @@ export class CrossSigning { } const keyToSign = await this.deviceTracker.getCrossSigningKeyForUser(userId, KeyUsage.Master, this.hsApi, log); if (!keyToSign) { - return undefined; + return; + } + const signingKey = await this.getSigningKey(KeyUsage.UserSigning); + if (!signingKey) { + return; } delete keyToSign.signatures; - const signingKey = await this.getSigningKey(KeyUsage.UserSigning); // add signature to keyToSign this.signKey(keyToSign, signingKey); const payload = { @@ -248,8 +251,11 @@ export class CrossSigning { }); } - private async signDeviceKey(keyToSign: DeviceKey, log: ILogItem): Promise { + private async signDeviceKey(keyToSign: DeviceKey, log: ILogItem): Promise { const signingKey = await this.getSigningKey(KeyUsage.SelfSigning); + if (!signingKey) { + return undefined; + } // add signature to keyToSign this.signKey(keyToSign, signingKey); // so the payload format of a signature is a map from userid to key id of the signed key @@ -265,11 +271,12 @@ export class CrossSigning { return keyToSign; } - private async getSigningKey(usage: KeyUsage): Promise { + private async getSigningKey(usage: KeyUsage): Promise { const txn = await this.storage.readTxn([this.storage.storeNames.accountData]); const seedStr = await this.secretStorage.readSecret(`m.cross_signing.${usage}`, txn); - const seed = new Uint8Array(this.platform.encoding.base64.decode(seedStr)); - return seed; + if (seedStr) { + return new Uint8Array(this.platform.encoding.base64.decode(seedStr)); + } } private signKey(keyToSign: DeviceKey | CrossSigningKey, signingKey: Uint8Array) { From 760da6277a1ee8e8648ab32f33911956d9d82012 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 9 Mar 2023 09:08:01 +0100 Subject: [PATCH 425/435] remove unused transaction --- src/matrix/verification/CrossSigning.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/matrix/verification/CrossSigning.ts b/src/matrix/verification/CrossSigning.ts index 05be9ba1..e4915129 100644 --- a/src/matrix/verification/CrossSigning.ts +++ b/src/matrix/verification/CrossSigning.ts @@ -101,7 +101,6 @@ export class CrossSigning { async init(log: ILogItem) { await log.wrap("CrossSigning.init", async log => { // TODO: use errorboundary here - const txn = await this.storage.readTxn([this.storage.storeNames.accountData]); const privateMasterKey = await this.getSigningKey(KeyUsage.Master); const signing = new this.olm.PkSigning(); let derivedPublicKey; From 780dfeb199c24a2e52c3fd813c172ded11bbfe6d Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 13 Mar 2023 09:15:49 +0100 Subject: [PATCH 426/435] WIP --- src/matrix/Session.js | 123 ++++++++++-------- src/matrix/e2ee/DeviceTracker.ts | 10 +- src/matrix/e2ee/megolm/keybackup/KeyBackup.ts | 16 ++- src/matrix/ssss/SecretStorage.ts | 37 +++++- .../storage/idb/stores/AccountDataStore.ts | 6 +- src/matrix/verification/CrossSigning.ts | 44 +++++-- 6 files changed, 160 insertions(+), 76 deletions(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 82eeba68..0b5b8577 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -252,16 +252,14 @@ export class Session { this._keyBackup.get().dispose(); this._keyBackup.set(null); } + // TODO: stop cross-signing const key = await ssssKeyFromCredential(type, credential, this._storage, this._platform, this._olm); - // and create key backup, which needs to read from accountData - const readTxn = await this._storage.readTxn([ - this._storage.storeNames.accountData, - ]); - if (await this._createKeyBackup(key, readTxn, log)) { + if (await this._tryLoadSecretStorage(key, undefined, log)) { // only after having read a secret, write the key // as we only find out if it was good if the MAC verification succeeds await this._writeSSSSKey(key, log); - this._keyBackup.get().flush(log); + await this._keyBackup?.start(log); + await this._crossSigning?.start(log); return key; } else { throw new Error("Could not read key backup with the given key"); @@ -317,12 +315,35 @@ export class Session { this._keyBackup.get().dispose(); this._keyBackup.set(null); } + // TODO: stop cross-signing } - _createKeyBackup(ssssKey, txn, log) { - return log.wrap("enable key backup", async log => { - try { - const secretStorage = new SecretStorage({key: ssssKey, platform: this._platform}); + _tryLoadSecretStorage(ssssKey, existingTxn, log) { + return log.wrap("enable secret storage", async log => { + const txn = existingTxn ?? await this._storage.readTxn([ + this._storage.storeNames.accountData, + this._storage.storeNames.crossSigningKeys, + this._storage.storeNames.userIdentities, + ]); + const secretStorage = new SecretStorage({key: ssssKey, platform: this._platform}); + const isValid = await secretStorage.hasValidKeyForAnyAccountData(txn); + log.set("isValid", isValid); + if (isValid) { + await this._loadSecretStorageServices(secretStorage, txn, log); + } + if (!this._keyBackup.get()) { + // null means key backup isn't configured yet + // as opposed to undefined, which means we're still checking + this._keyBackup.set(null); + } + return isValid; + }); + } + + _loadSecretStorageServices(secretStorage, txn, log) { + try { + await log.wrap("enable key backup", async log => { + // TODO: delay network request here until start() const keyBackup = await KeyBackup.fromSecretStorage( this._platform, this._olm, @@ -333,22 +354,6 @@ export class Session { txn ); if (keyBackup) { - if (this._features.crossSigning) { - this._crossSigning = new CrossSigning({ - storage: this._storage, - secretStorage, - platform: this._platform, - olm: this._olm, - olmUtil: this._olmUtil, - deviceTracker: this._deviceTracker, - hsApi: this._hsApi, - ownUserId: this.userId, - e2eeAccount: this._e2eeAccount - }); - await log.wrap("enable cross-signing", log => { - return this._crossSigning.init(log); - }); - } for (const room of this._rooms.values()) { if (room.isEncrypted) { room.enableKeyBackup(keyBackup); @@ -359,11 +364,28 @@ export class Session { } else { log.set("no_backup", true); } - } catch (err) { - log.catch(err); + }); + if (this._features.crossSigning) { + await log.wrap("enable cross-signing", async log => { + const crossSigning = new CrossSigning({ + storage: this._storage, + secretStorage, + platform: this._platform, + olm: this._olm, + olmUtil: this._olmUtil, + deviceTracker: this._deviceTracker, + hsApi: this._hsApi, + ownUserId: this.userId, + e2eeAccount: this._e2eeAccount + }); + if (crossSigning.load(txn, log)) { + this._crossSigning = crossSigning; + } + }); } - return false; - }); + } catch (err) { + log.catch(err); + } } /** @@ -467,6 +489,8 @@ export class Session { this._storage.storeNames.timelineEvents, this._storage.storeNames.timelineFragments, this._storage.storeNames.pendingEvents, + this._storage.storeNames.accountData, + this._storage.storeNames.crossSigningKeys, ]); // restore session object this._syncInfo = await txn.session.get("sync"); @@ -484,6 +508,11 @@ export class Session { if (this._e2eeAccount) { log.set("keys", this._e2eeAccount.identityKeys); this._setupEncryption(); + // try set up session backup if we stored the ssss key + const ssssKey = await ssssReadKey(txn); + if (ssssKey) { + await this._tryLoadSecretStorage(ssssKey, txn, log); + } } } const pendingEventsByRoomId = await this._getPendingEventsByRoom(txn); @@ -544,35 +573,21 @@ export class Session { // TODO: what can we do if this throws? await txn.complete(); } - // enable session backup, this requests the latest backup version - if (!this._keyBackup.get()) { - if (dehydratedDevice) { - await log.wrap("SSSSKeyFromDehydratedDeviceKey", async log => { - const ssssKey = await createSSSSKeyFromDehydratedDeviceKey(dehydratedDevice.key, this._storage, this._platform); - if (ssssKey) { + // try if the key used to decrypt the dehydrated device also fits for secret storage + if (dehydratedDevice) { + await log.wrap("SSSSKeyFromDehydratedDeviceKey", async log => { + const ssssKey = await createSSSSKeyFromDehydratedDeviceKey(dehydratedDevice.key, this._storage, this._platform); + if (ssssKey) { + if (await this._tryLoadSecretStorage(ssssKey, undefined, log)) { log.set("success", true); await this._writeSSSSKey(ssssKey); } - }); - } - const txn = await this._storage.readTxn([ - this._storage.storeNames.session, - this._storage.storeNames.accountData, - ]); - // try set up session backup if we stored the ssss key - const ssssKey = await ssssReadKey(txn); - if (ssssKey) { - // txn will end here as this does a network request - if (await this._createKeyBackup(ssssKey, txn, log)) { - this._keyBackup.get()?.flush(log); } - } - if (!this._keyBackup.get()) { - // null means key backup isn't configured yet - // as opposed to undefined, which means we're still checking - this._keyBackup.set(null); - } + }); } + this._keyBackup?.start(log); + this._crossSigning?.start(log); + // restore unfinished operations, like sending out room keys const opsTxn = await this._storage.readWriteTxn([ this._storage.storeNames.operations diff --git a/src/matrix/e2ee/DeviceTracker.ts b/src/matrix/e2ee/DeviceTracker.ts index 8a6e351b..23bdc31e 100644 --- a/src/matrix/e2ee/DeviceTracker.ts +++ b/src/matrix/e2ee/DeviceTracker.ts @@ -163,16 +163,20 @@ export class DeviceTracker { } } - async getCrossSigningKeyForUser(userId: string, usage: KeyUsage, hsApi: HomeServerApi, log: ILogItem): Promise { + async getCrossSigningKeyForUser(userId: string, usage: KeyUsage, hsApi: HomeServerApi | undefined, existingTxn: Transaction | undefined, log: ILogItem): Promise { return await log.wrap({l: "DeviceTracker.getCrossSigningKeyForUser", id: userId, usage}, async log => { - let txn = await this._storage.readTxn([ + const txn = existingTxn ?? await this._storage.readTxn([ this._storage.storeNames.userIdentities, this._storage.storeNames.crossSigningKeys, ]); - let userIdentity = await txn.userIdentities.get(userId); + const userIdentity = await txn.userIdentities.get(userId); if (userIdentity && userIdentity.keysTrackingStatus !== KeysTrackingStatus.Outdated) { return await txn.crossSigningKeys.get(userId, usage); } + // not allowed to access the network, bail out + if (!hsApi) { + return undefined; + } // fetch from hs const keys = await this._queryKeys([userId], hsApi, log); switch (usage) { diff --git a/src/matrix/e2ee/megolm/keybackup/KeyBackup.ts b/src/matrix/e2ee/megolm/keybackup/KeyBackup.ts index bcfbf85a..0ef610ff 100644 --- a/src/matrix/e2ee/megolm/keybackup/KeyBackup.ts +++ b/src/matrix/e2ee/megolm/keybackup/KeyBackup.ts @@ -38,15 +38,14 @@ const KEYS_PER_REQUEST = 200; export class KeyBackup { public readonly operationInProgress = new ObservableValue, Progress> | undefined>(undefined); - private _stopped = false; private _needsNewKey = false; private _hasBackedUpAllKeys = false; private _error?: Error; + private crypto?: Curve25519.BackupEncryption; constructor( private readonly backupInfo: BackupInfo, - private readonly crypto: Curve25519.BackupEncryption, private readonly hsApi: HomeServerApi, private readonly keyLoader: KeyLoader, private readonly storage: Storage, @@ -61,6 +60,9 @@ export class KeyBackup { get hasBackedUpAllKeys(): boolean { return this._hasBackedUpAllKeys; } async getRoomKey(roomId: string, sessionId: string, log: ILogItem): Promise { + if (this.needsNewKey || !this.crypto) { + return; + } const sessionResponse = await this.hsApi.roomKeyForRoomAndSession(this.backupInfo.version, roomId, sessionId, {log}).response(); if (!sessionResponse.session_data) { return; @@ -77,6 +79,12 @@ export class KeyBackup { return txn.inboundGroupSessions.markAllAsNotBackedUp(); } + start(log: ILogItem) { + + // fetch latest version + this.flush(log); + } + flush(log: ILogItem): void { if (!this.operationInProgress.get()) { log.wrapDetached("flush key backup", async log => { @@ -184,7 +192,7 @@ export class KeyBackup { } dispose() { - this.crypto.dispose(); + this.crypto?.dispose(); } static async fromSecretStorage(platform: Platform, olm: Olm, secretStorage: SecretStorage, hsApi: HomeServerApi, keyLoader: KeyLoader, storage: Storage, txn: Transaction): Promise { @@ -194,7 +202,7 @@ export class KeyBackup { const backupInfo = await hsApi.roomKeysVersion().response() as BackupInfo; if (backupInfo.algorithm === Curve25519.Algorithm) { const crypto = Curve25519.BackupEncryption.fromAuthData(backupInfo.auth_data, privateKey, olm); - return new KeyBackup(backupInfo, crypto, hsApi, keyLoader, storage, platform); + return new KeyBackup(backupInfo, privateKey, hsApi, keyLoader, storage, platform); } else { throw new Error(`Unknown backup algorithm: ${backupInfo.algorithm}`); } diff --git a/src/matrix/ssss/SecretStorage.ts b/src/matrix/ssss/SecretStorage.ts index c026b453..ebdcd13a 100644 --- a/src/matrix/ssss/SecretStorage.ts +++ b/src/matrix/ssss/SecretStorage.ts @@ -16,6 +16,8 @@ limitations under the License. import type {Key} from "./common"; import type {Platform} from "../../platform/web/Platform.js"; import type {Transaction} from "../storage/idb/Transaction"; +import type {Storage} from "../storage/idb/Storage"; +import type {AccountDataEntry} from "../storage/idb/stores/AccountDataStore"; type EncryptedData = { iv: string; @@ -23,6 +25,18 @@ type EncryptedData = { mac: string; } +export enum DecryptionFailure { + NotEncryptedWithKey, + BadMAC, + UnsupportedAlgorithm, +} + +class DecryptionError extends Error { + constructor(msg: string, public readonly reason: DecryptionFailure) { + super(msg); + } +} + export class SecretStorage { private readonly _key: Key; private readonly _platform: Platform; @@ -32,20 +46,37 @@ export class SecretStorage { this._platform = platform; } + async hasValidKeyForAnyAccountData(txn: Transaction) { + const allAccountData = await txn.accountData.getAll(); + for (const accountData of allAccountData) { + try { + const secret = await this._decryptAccountData(accountData); + return true; // decryption succeeded + } catch (err) { + continue; + } + } + return false; + } + async readSecret(name: string, txn: Transaction): Promise { const accountData = await txn.accountData.get(name); if (!accountData) { return; } + return await this._decryptAccountData(accountData); + } + + async _decryptAccountData(accountData: AccountDataEntry): Promise { const encryptedData = accountData?.content?.encrypted?.[this._key.id] as EncryptedData; if (!encryptedData) { - throw new Error(`Secret ${accountData.type} is not encrypted for key ${this._key.id}`); + throw new DecryptionError(`Secret ${accountData.type} is not encrypted for key ${this._key.id}`, DecryptionFailure.NotEncryptedWithKey); } if (this._key.algorithm === "m.secret_storage.v1.aes-hmac-sha2") { return await this._decryptAESSecret(accountData.type, encryptedData); } else { - throw new Error(`Unsupported algorithm for key ${this._key.id}: ${this._key.algorithm}`); + throw new DecryptionError(`Unsupported algorithm for key ${this._key.id}: ${this._key.algorithm}`, DecryptionFailure.UnsupportedAlgorithm); } } @@ -68,7 +99,7 @@ export class SecretStorage { ciphertextBytes, "SHA-256"); if (!isVerified) { - throw new Error("Bad MAC"); + throw new DecryptionError("Bad MAC", DecryptionFailure.BadMAC); } const plaintextBytes = await this._platform.crypto.aes.decryptCTR({ diff --git a/src/matrix/storage/idb/stores/AccountDataStore.ts b/src/matrix/storage/idb/stores/AccountDataStore.ts index 2081ad8f..33c8a162 100644 --- a/src/matrix/storage/idb/stores/AccountDataStore.ts +++ b/src/matrix/storage/idb/stores/AccountDataStore.ts @@ -16,7 +16,7 @@ limitations under the License. import {Store} from "../Store"; import {Content} from "../../types"; -interface AccountDataEntry { +export interface AccountDataEntry { type: string; content: Content; } @@ -35,4 +35,8 @@ export class AccountDataStore { set(event: AccountDataEntry): void { this._store.put(event); } + + async getAll(): Promise> { + return await this._store.selectAll(); + } } diff --git a/src/matrix/verification/CrossSigning.ts b/src/matrix/verification/CrossSigning.ts index e4915129..1abc3702 100644 --- a/src/matrix/verification/CrossSigning.ts +++ b/src/matrix/verification/CrossSigning.ts @@ -20,6 +20,7 @@ import {verifyEd25519Signature, SignatureVerification} from "../e2ee/common"; import type {SecretStorage} from "../ssss/SecretStorage"; import type {Storage} from "../storage/idb/Storage"; +import type {Transaction} from "../storage/idb/Transaction"; import type {Platform} from "../../platform/web/Platform"; import type {DeviceTracker} from "../e2ee/DeviceTracker"; import type {HomeServerApi} from "../net/HomeServerApi"; @@ -98,10 +99,25 @@ export class CrossSigning { this.e2eeAccount = options.e2eeAccount } - async init(log: ILogItem) { - await log.wrap("CrossSigning.init", async log => { + async load(txn: Transaction, log: ILogItem) { + // try to verify the msk without accessing the network + return await this.verifyMSKFrom4S(undefined, txn, log); + } + + async start(log: ILogItem) { + if (!this.isMasterKeyTrusted) { + // try to verify the msk _with_ access to the network + return await this.verifyMSKFrom4S(this.hsApi, undefined, log); + } + } + + private async verifyMSKFrom4S(hsApi: HomeServerApi | undefined, txn: Transaction | undefined, log: ILogItem): Promise { + return await log.wrap("CrossSigning.verifyMSKFrom4S", async log => { // TODO: use errorboundary here - const privateMasterKey = await this.getSigningKey(KeyUsage.Master); + const privateMasterKey = await this.getSigningKey(KeyUsage.Master, txn); + if (!privateMasterKey) { + return false; + } const signing = new this.olm.PkSigning(); let derivedPublicKey; try { @@ -109,11 +125,15 @@ export class CrossSigning { } finally { signing.free(); } - const publishedMasterKey = await this.deviceTracker.getCrossSigningKeyForUser(this.ownUserId, KeyUsage.Master, this.hsApi, log); + const publishedMasterKey = await this.deviceTracker.getCrossSigningKeyForUser(this.ownUserId, KeyUsage.Master, hsApi, txn, log); + if (!publishedMasterKey) { + return false; + } const publisedEd25519Key = publishedMasterKey && getKeyEd25519Key(publishedMasterKey); log.set({publishedMasterKey: publisedEd25519Key, derivedPublicKey}); this._isMasterKeyTrusted = !!publisedEd25519Key && publisedEd25519Key === derivedPublicKey; log.set("isMasterKeyTrusted", this.isMasterKeyTrusted); + return this.isMasterKeyTrusted; }); } @@ -162,7 +182,7 @@ export class CrossSigning { if (userId === this.ownUserId) { return; } - const keyToSign = await this.deviceTracker.getCrossSigningKeyForUser(userId, KeyUsage.Master, this.hsApi, log); + const keyToSign = await this.deviceTracker.getCrossSigningKeyForUser(userId, KeyUsage.Master, this.hsApi, undefined, log); if (!keyToSign) { return; } @@ -190,11 +210,11 @@ export class CrossSigning { if (!this.isMasterKeyTrusted) { return UserTrust.OwnSetupError; } - const ourMSK = await log.wrap("get our msk", log => this.deviceTracker.getCrossSigningKeyForUser(this.ownUserId, KeyUsage.Master, this.hsApi, log)); + const ourMSK = await log.wrap("get our msk", log => this.deviceTracker.getCrossSigningKeyForUser(this.ownUserId, KeyUsage.Master, this.hsApi, txn, log)); if (!ourMSK) { return UserTrust.OwnSetupError; } - const ourUSK = await log.wrap("get our usk", log => this.deviceTracker.getCrossSigningKeyForUser(this.ownUserId, KeyUsage.UserSigning, this.hsApi, log)); + const ourUSK = await log.wrap("get our usk", log => this.deviceTracker.getCrossSigningKeyForUser(this.ownUserId, KeyUsage.UserSigning, this.hsApi, txn, log)); if (!ourUSK) { return UserTrust.OwnSetupError; } @@ -202,7 +222,7 @@ export class CrossSigning { if (ourUSKVerification !== SignatureVerification.Valid) { return UserTrust.OwnSetupError; } - const theirMSK = await log.wrap("get their msk", log => this.deviceTracker.getCrossSigningKeyForUser(userId, KeyUsage.Master, this.hsApi, log)); + const theirMSK = await log.wrap("get their msk", log => this.deviceTracker.getCrossSigningKeyForUser(userId, KeyUsage.Master, this.hsApi, txn, log)); if (!theirMSK) { /* assume that when they don't have an MSK, they've never enabled cross-signing on their client (or it's not supported) rather than assuming a setup error on their side. @@ -217,7 +237,7 @@ export class CrossSigning { return UserTrust.UserSignatureMismatch; } } - const theirSSK = await log.wrap("get their ssk", log => this.deviceTracker.getCrossSigningKeyForUser(userId, KeyUsage.SelfSigning, this.hsApi, log)); + const theirSSK = await log.wrap("get their ssk", log => this.deviceTracker.getCrossSigningKeyForUser(userId, KeyUsage.SelfSigning, this.hsApi, txn, log)); if (!theirSSK) { return UserTrust.UserSetupError; } @@ -270,8 +290,10 @@ export class CrossSigning { return keyToSign; } - private async getSigningKey(usage: KeyUsage): Promise { - const txn = await this.storage.readTxn([this.storage.storeNames.accountData]); + private async getSigningKey(usage: KeyUsage, existingTxn?: Transaction): Promise { + const txn = existingTxn ?? await this.storage.readTxn([ + this.storage.storeNames.accountData, + ]); const seedStr = await this.secretStorage.readSecret(`m.cross_signing.${usage}`, txn); if (seedStr) { return new Uint8Array(this.platform.encoding.base64.decode(seedStr)); From dd59f37dce0eba80f807ea973d79c0a66c64c6db Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 21 Mar 2023 18:24:46 +0100 Subject: [PATCH 427/435] WIP2 --- src/domain/AccountSetupViewModel.js | 2 +- .../session/settings/KeyBackupViewModel.js | 223 --------------- .../session/settings/KeyBackupViewModel.ts | 270 ++++++++++++++++++ .../session/settings/SettingsViewModel.js | 2 +- src/matrix/Session.js | 34 +-- src/matrix/e2ee/megolm/keybackup/KeyBackup.ts | 130 ++++++--- src/matrix/ssss/SecretStorage.ts | 12 +- src/platform/web/ui/login/AccountSetupView.js | 2 +- ...ttingsView.js => KeyBackupSettingsView.ts} | 53 ++-- .../web/ui/session/settings/SettingsView.js | 2 +- src/utils/AbortableOperation.ts | 14 +- src/utils/Deferred.ts | 41 +++ 12 files changed, 473 insertions(+), 312 deletions(-) delete mode 100644 src/domain/session/settings/KeyBackupViewModel.js create mode 100644 src/domain/session/settings/KeyBackupViewModel.ts rename src/platform/web/ui/session/settings/{KeyBackupSettingsView.js => KeyBackupSettingsView.ts} (73%) create mode 100644 src/utils/Deferred.ts diff --git a/src/domain/AccountSetupViewModel.js b/src/domain/AccountSetupViewModel.js index e7c1301f..3e643582 100644 --- a/src/domain/AccountSetupViewModel.js +++ b/src/domain/AccountSetupViewModel.js @@ -16,7 +16,7 @@ limitations under the License. import {ViewModel} from "./ViewModel"; import {KeyType} from "../matrix/ssss/index"; -import {Status} from "./session/settings/KeyBackupViewModel.js"; +import {Status} from "./session/settings/KeyBackupViewModel"; export class AccountSetupViewModel extends ViewModel { constructor(options) { diff --git a/src/domain/session/settings/KeyBackupViewModel.js b/src/domain/session/settings/KeyBackupViewModel.js deleted file mode 100644 index 22135e41..00000000 --- a/src/domain/session/settings/KeyBackupViewModel.js +++ /dev/null @@ -1,223 +0,0 @@ -/* -Copyright 2020 The Matrix.org Foundation C.I.C. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import {ViewModel} from "../../ViewModel"; -import {KeyType} from "../../../matrix/ssss/index"; -import {createEnum} from "../../../utils/enum"; -import {FlatMapObservableValue} from "../../../observable/value"; - -export const Status = createEnum("Enabled", "SetupKey", "SetupPhrase", "Pending", "NewVersionAvailable"); -export const BackupWriteStatus = createEnum("Writing", "Stopped", "Done", "Pending"); - -export class KeyBackupViewModel extends ViewModel { - constructor(options) { - super(options); - this._session = options.session; - this._error = null; - this._isBusy = false; - this._dehydratedDeviceId = undefined; - this._status = undefined; - this._backupOperation = new FlatMapObservableValue(this._session.keyBackup, keyBackup => keyBackup.operationInProgress); - this._progress = new FlatMapObservableValue(this._backupOperation, op => op.progress); - this.track(this._backupOperation.subscribe(() => { - // see if needsNewKey might be set - this._reevaluateStatus(); - this.emitChange("isBackingUp"); - })); - this.track(this._progress.subscribe(() => this.emitChange("backupPercentage"))); - this._reevaluateStatus(); - this.track(this._session.keyBackup.subscribe(() => { - if (this._reevaluateStatus()) { - this.emitChange("status"); - } - })); - } - - _reevaluateStatus() { - if (this._isBusy) { - return false; - } - let status; - const keyBackup = this._session.keyBackup.get(); - if (keyBackup) { - status = keyBackup.needsNewKey ? Status.NewVersionAvailable : Status.Enabled; - } else if (keyBackup === null) { - status = this.showPhraseSetup() ? Status.SetupPhrase : Status.SetupKey; - } else { - status = Status.Pending; - } - const changed = status !== this._status; - this._status = status; - return changed; - } - - get decryptAction() { - return this.i18n`Set up`; - } - - get purpose() { - return this.i18n`set up key backup`; - } - - offerDehydratedDeviceSetup() { - return true; - } - - get dehydratedDeviceId() { - return this._dehydratedDeviceId; - } - - get isBusy() { - return this._isBusy; - } - - get backupVersion() { - return this._session.keyBackup.get()?.version; - } - - get isMasterKeyTrusted() { - return this._session.crossSigning?.isMasterKeyTrusted ?? false; - } - - get canSignOwnDevice() { - return !!this._session.crossSigning; - } - - async signOwnDevice() { - if (this._session.crossSigning) { - await this.logger.run("KeyBackupViewModel.signOwnDevice", async log => { - await this._session.crossSigning.signOwnDevice(log); - }); - } - } - - get backupWriteStatus() { - const keyBackup = this._session.keyBackup.get(); - if (!keyBackup) { - return BackupWriteStatus.Pending; - } else if (keyBackup.hasStopped) { - return BackupWriteStatus.Stopped; - } - const operation = keyBackup.operationInProgress.get(); - if (operation) { - return BackupWriteStatus.Writing; - } else if (keyBackup.hasBackedUpAllKeys) { - return BackupWriteStatus.Done; - } else { - return BackupWriteStatus.Pending; - } - } - - get backupError() { - return this._session.keyBackup.get()?.error?.message; - } - - get status() { - return this._status; - } - - get error() { - return this._error?.message; - } - - showPhraseSetup() { - if (this._status === Status.SetupKey) { - this._status = Status.SetupPhrase; - this.emitChange("status"); - } - } - - showKeySetup() { - if (this._status === Status.SetupPhrase) { - this._status = Status.SetupKey; - this.emitChange("status"); - } - } - - async _enterCredentials(keyType, credential, setupDehydratedDevice) { - if (credential) { - try { - this._isBusy = true; - this.emitChange("isBusy"); - const key = await this._session.enableSecretStorage(keyType, credential); - if (setupDehydratedDevice) { - this._dehydratedDeviceId = await this._session.setupDehydratedDevice(key); - } - } catch (err) { - console.error(err); - this._error = err; - this.emitChange("error"); - } finally { - this._isBusy = false; - this._reevaluateStatus(); - this.emitChange(""); - } - } - } - - enterSecurityPhrase(passphrase, setupDehydratedDevice) { - this._enterCredentials(KeyType.Passphrase, passphrase, setupDehydratedDevice); - } - - enterSecurityKey(securityKey, setupDehydratedDevice) { - this._enterCredentials(KeyType.RecoveryKey, securityKey, setupDehydratedDevice); - } - - async disable() { - try { - this._isBusy = true; - this.emitChange("isBusy"); - await this._session.disableSecretStorage(); - } catch (err) { - console.error(err); - this._error = err; - this.emitChange("error"); - } finally { - this._isBusy = false; - this._reevaluateStatus(); - this.emitChange(""); - } - } - - get isBackingUp() { - return !!this._backupOperation.get(); - } - - get backupPercentage() { - const progress = this._progress.get(); - if (progress) { - return Math.round((progress.finished / progress.total) * 100); - } - return 0; - } - - get backupInProgressLabel() { - const progress = this._progress.get(); - if (progress) { - return this.i18n`${progress.finished} of ${progress.total}`; - } - return this.i18n`…`; - } - - cancelBackup() { - this._backupOperation.get()?.abort(); - } - - startBackup() { - this._session.keyBackup.get()?.flush(); - } -} - diff --git a/src/domain/session/settings/KeyBackupViewModel.ts b/src/domain/session/settings/KeyBackupViewModel.ts new file mode 100644 index 00000000..cdfd4081 --- /dev/null +++ b/src/domain/session/settings/KeyBackupViewModel.ts @@ -0,0 +1,270 @@ +/* +Copyright 2020 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {ViewModel} from "../../ViewModel"; +import {SegmentType} from "../../navigation/index"; +import {KeyType} from "../../../matrix/ssss/index"; + +import type {Options as BaseOptions} from "../../ViewModel"; +import type {Session} from "../../../matrix/Session"; +import type {Disposable} from "../../../utils/Disposables"; +import type {KeyBackup, Progress} from "../../../matrix/e2ee/megolm/keybackup/KeyBackup"; +import type {CrossSigning} from "../../../matrix/verification/CrossSigning"; + +export enum Status { + Enabled, + Setup, + Pending, + NewVersionAvailable +}; + +export enum BackupWriteStatus { + Writing, + Stopped, + Done, + Pending +}; + +type Options = { + session: Session, +} & BaseOptions; + +export class KeyBackupViewModel extends ViewModel { + private _error?: Error = undefined; + private _isBusy = false; + private _dehydratedDeviceId?: string = undefined; + private _status = Status.Pending; + private _backupOperationSubscription?: Disposable = undefined; + private _keyBackupSubscription?: Disposable = undefined; + private _progress?: Progress = undefined; + private _setupKeyType = KeyType.RecoveryKey; + + constructor(options) { + super(options); + const onKeyBackupSet = (keyBackup: KeyBackup | undefined) => { + if (keyBackup && !this._keyBackupSubscription) { + this._keyBackupSubscription = this.track(this._session.keyBackup.disposableOn("change", () => { + this._onKeyBackupChange(); + })); + } else if (!keyBackup && this._keyBackupSubscription) { + this._keyBackupSubscription = this.disposeTracked(this._keyBackupSubscription); + } + this._onKeyBackupChange(); // update status + }; + this.track(this._session.keyBackup.subscribe(onKeyBackupSet)); + onKeyBackupSet(this._keyBackup); + } + + private get _session(): Session { + return this.getOption("session"); + } + + private get _keyBackup(): KeyBackup | undefined { + return this._session.keyBackup.get(); + } + + private get _crossSigning(): CrossSigning | undefined { + return this._session.crossSigning.get(); + } + + private _onKeyBackupChange() { + const keyBackup = this._keyBackup; + if (keyBackup) { + const {operationInProgress} = keyBackup; + if (operationInProgress && !this._backupOperationSubscription) { + this._backupOperationSubscription = this.track(operationInProgress.disposableOn("change", () => { + this._progress = operationInProgress.progress; + this.emitChange("backupPercentage"); + })); + } else if (this._backupOperationSubscription && !operationInProgress) { + this._backupOperationSubscription = this.disposeTracked(this._backupOperationSubscription); + this._progress = undefined; + } + } + this.emitChange("status"); + } + + get status(): Status { + const keyBackup = this._keyBackup; + if (keyBackup) { + if (keyBackup.needsNewKey) { + return Status.NewVersionAvailable; + } else if (keyBackup.version === undefined) { + return Status.Pending; + } else { + return keyBackup.needsNewKey ? Status.NewVersionAvailable : Status.Enabled; + } + } else { + return Status.Setup; + } + } + + get decryptAction(): string { + return this.i18n`Set up`; + } + + get purpose(): string { + return this.i18n`set up key backup`; + } + + offerDehydratedDeviceSetup(): boolean { + return true; + } + + get dehydratedDeviceId(): string | undefined { + return this._dehydratedDeviceId; + } + + get isBusy(): boolean { + return this._isBusy; + } + + get backupVersion(): string { + return this._keyBackup?.version ?? ""; + } + + get isMasterKeyTrusted(): boolean { + return this._crossSigning?.isMasterKeyTrusted ?? false; + } + + get canSignOwnDevice(): boolean { + return !!this._crossSigning; + } + + async signOwnDevice(): Promise { + const crossSigning = this._crossSigning; + if (crossSigning) { + await this.logger.run("KeyBackupViewModel.signOwnDevice", async log => { + await crossSigning.signOwnDevice(log); + }); + } + } + + get backupWriteStatus(): BackupWriteStatus { + const keyBackup = this._keyBackup; + if (!keyBackup || keyBackup.version === undefined) { + return BackupWriteStatus.Pending; + } else if (keyBackup.hasStopped) { + return BackupWriteStatus.Stopped; + } + const operation = keyBackup.operationInProgress; + if (operation) { + return BackupWriteStatus.Writing; + } else if (keyBackup.hasBackedUpAllKeys) { + return BackupWriteStatus.Done; + } else { + return BackupWriteStatus.Pending; + } + } + + get backupError(): string | undefined { + return this._keyBackup?.error?.message; + } + + get error(): string | undefined { + return this._error?.message; + } + + showPhraseSetup(): void { + if (this._status === Status.Setup) { + this._setupKeyType = KeyType.Passphrase; + this.emitChange("setupKeyType"); + } + } + + showKeySetup(): void { + if (this._status === Status.Setup) { + this._setupKeyType = KeyType.Passphrase; + this.emitChange("setupKeyType"); + } + } + + get setupKeyType(): KeyType { + return this._setupKeyType; + } + + private async _enterCredentials(keyType, credential, setupDehydratedDevice): Promise { + if (credential) { + try { + this._isBusy = true; + this.emitChange("isBusy"); + const key = await this._session.enableSecretStorage(keyType, credential); + if (setupDehydratedDevice) { + this._dehydratedDeviceId = await this._session.setupDehydratedDevice(key); + } + } catch (err) { + console.error(err); + this._error = err; + this.emitChange("error"); + } finally { + this._isBusy = false; + this.emitChange(); + } + } + } + + enterSecurityPhrase(passphrase, setupDehydratedDevice): Promise { + return this._enterCredentials(KeyType.Passphrase, passphrase, setupDehydratedDevice); + } + + enterSecurityKey(securityKey, setupDehydratedDevice): Promise { + return this._enterCredentials(KeyType.RecoveryKey, securityKey, setupDehydratedDevice); + } + + async disable(): Promise { + try { + this._isBusy = true; + this.emitChange("isBusy"); + await this._session.disableSecretStorage(); + } catch (err) { + console.error(err); + this._error = err; + this.emitChange("error"); + } finally { + this._isBusy = false; + this.emitChange(); + } + } + + get isBackingUp(): boolean { + return this._keyBackup?.operationInProgress !== undefined; + } + + get backupPercentage(): number { + if (this._progress) { + return Math.round((this._progress.finished / this._progress.total) * 100); + } + return 0; + } + + get backupInProgressLabel(): string { + if (this._progress) { + return this.i18n`${this._progress.finished} of ${this._progress.total}`; + } + return this.i18n`…`; + } + + cancelBackup(): void { + this._keyBackup?.operationInProgress?.abort(); + } + + startBackup(): void { + this.logger.run("KeyBackupViewModel.startBackup", log => { + this._keyBackup?.flush(log); + }); + } +} + diff --git a/src/domain/session/settings/SettingsViewModel.js b/src/domain/session/settings/SettingsViewModel.js index f8420a53..7f4cab59 100644 --- a/src/domain/session/settings/SettingsViewModel.js +++ b/src/domain/session/settings/SettingsViewModel.js @@ -15,7 +15,7 @@ limitations under the License. */ import {ViewModel} from "../../ViewModel"; -import {KeyBackupViewModel} from "./KeyBackupViewModel.js"; +import {KeyBackupViewModel} from "./KeyBackupViewModel"; import {FeaturesViewModel} from "./FeaturesViewModel"; import {submitLogsFromSessionToDefaultServer} from "../../../domain/rageshake"; diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 0b5b8577..b10b2824 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -90,7 +90,7 @@ export class Session { this._getSyncToken = () => this.syncToken; this._olmWorker = olmWorker; this._keyBackup = new ObservableValue(undefined); - this._crossSigning = undefined; + this._crossSigning = new ObservableValue(undefined); this._observedRoomStatus = new Map(); if (olm) { @@ -250,7 +250,7 @@ export class Session { } if (this._keyBackup.get()) { this._keyBackup.get().dispose(); - this._keyBackup.set(null); + this._keyBackup.set(undefined); } // TODO: stop cross-signing const key = await ssssKeyFromCredential(type, credential, this._storage, this._platform, this._olm); @@ -258,8 +258,8 @@ export class Session { // only after having read a secret, write the key // as we only find out if it was good if the MAC verification succeeds await this._writeSSSSKey(key, log); - await this._keyBackup?.start(log); - await this._crossSigning?.start(log); + await this._keyBackup.get()?.start(log); + await this._crossSigning.get()?.start(log); return key; } else { throw new Error("Could not read key backup with the given key"); @@ -331,29 +331,21 @@ export class Session { if (isValid) { await this._loadSecretStorageServices(secretStorage, txn, log); } - if (!this._keyBackup.get()) { - // null means key backup isn't configured yet - // as opposed to undefined, which means we're still checking - this._keyBackup.set(null); - } return isValid; }); } - _loadSecretStorageServices(secretStorage, txn, log) { + async _loadSecretStorageServices(secretStorage, txn, log) { try { await log.wrap("enable key backup", async log => { - // TODO: delay network request here until start() - const keyBackup = await KeyBackup.fromSecretStorage( - this._platform, - this._olm, - secretStorage, + const keyBackup = new KeyBackup( this._hsApi, + this._olm, this._keyLoader, this._storage, - txn + this._platform, ); - if (keyBackup) { + if (await keyBackup.load(secretStorage, txn)) { for (const room of this._rooms.values()) { if (room.isEncrypted) { room.enableKeyBackup(keyBackup); @@ -378,8 +370,8 @@ export class Session { ownUserId: this.userId, e2eeAccount: this._e2eeAccount }); - if (crossSigning.load(txn, log)) { - this._crossSigning = crossSigning; + if (await crossSigning.load(txn, log)) { + this._crossSigning.set(crossSigning); } }); } @@ -585,8 +577,8 @@ export class Session { } }); } - this._keyBackup?.start(log); - this._crossSigning?.start(log); + this._keyBackup.get()?.start(log); + this._crossSigning.get()?.start(log); // restore unfinished operations, like sending out room keys const opsTxn = await this._storage.readWriteTxn([ diff --git a/src/matrix/e2ee/megolm/keybackup/KeyBackup.ts b/src/matrix/e2ee/megolm/keybackup/KeyBackup.ts index 0ef610ff..8e9a4a81 100644 --- a/src/matrix/e2ee/megolm/keybackup/KeyBackup.ts +++ b/src/matrix/e2ee/megolm/keybackup/KeyBackup.ts @@ -20,6 +20,8 @@ import {MEGOLM_ALGORITHM} from "../../common"; import * as Curve25519 from "./Curve25519"; import {AbortableOperation} from "../../../../utils/AbortableOperation"; import {ObservableValue} from "../../../../observable/value"; +import {Deferred} from "../../../../utils/Deferred"; +import {EventEmitter} from "../../../../utils/EventEmitter"; import {SetAbortableFn} from "../../../../utils/AbortableOperation"; import type {BackupInfo, SessionData, SessionKeyInfo, SessionInfo, KeyBackupPayload} from "./types"; @@ -31,43 +33,69 @@ import type {Storage} from "../../../storage/idb/Storage"; import type {ILogItem} from "../../../../logging/types"; import type {Platform} from "../../../../platform/web/Platform"; import type {Transaction} from "../../../storage/idb/Transaction"; +import type {IHomeServerRequest} from "../../../net/HomeServerRequest"; import type * as OlmNamespace from "@matrix-org/olm"; type Olm = typeof OlmNamespace; const KEYS_PER_REQUEST = 200; -export class KeyBackup { - public readonly operationInProgress = new ObservableValue, Progress> | undefined>(undefined); +// a set of fields we need to store once we've fetched +// the backup info from the homeserver, which happens in start() +class BackupConfig { + constructor( + public readonly info: BackupInfo, + public readonly crypto: Curve25519.BackupEncryption + ) {} +} + +export class KeyBackup extends EventEmitter<{change: never}> { + private _operationInProgress?: AbortableOperation, Progress>; private _stopped = false; private _needsNewKey = false; private _hasBackedUpAllKeys = false; private _error?: Error; private crypto?: Curve25519.BackupEncryption; + private backupInfo?: BackupInfo; + private privateKey?: Uint8Array; + private backupConfigDeferred: Deferred = new Deferred(); + private backupInfoRequest?: IHomeServerRequest; constructor( - private readonly backupInfo: BackupInfo, private readonly hsApi: HomeServerApi, + private readonly olm: Olm, private readonly keyLoader: KeyLoader, private readonly storage: Storage, private readonly platform: Platform, private readonly maxDelay: number = 10000 - ) {} + ) { + super(); + // doing the network request for getting the backup info + // and hence creating the crypto instance depending on the chose algorithm + // is delayed until start() is called, but we want to already take requests + // for fetching the room keys, so put the crypto and backupInfo in a deferred. + this.backupConfigDeferred = new Deferred(); + } get hasStopped(): boolean { return this._stopped; } get error(): Error | undefined { return this._error; } - get version(): string { return this.backupInfo.version; } + get version(): string | undefined { return this.backupConfigDeferred.value?.info?.version; } get needsNewKey(): boolean { return this._needsNewKey; } get hasBackedUpAllKeys(): boolean { return this._hasBackedUpAllKeys; } + get operationInProgress(): AbortableOperation, Progress> | undefined { return this._operationInProgress; } async getRoomKey(roomId: string, sessionId: string, log: ILogItem): Promise { - if (this.needsNewKey || !this.crypto) { + if (this.needsNewKey) { return; } - const sessionResponse = await this.hsApi.roomKeyForRoomAndSession(this.backupInfo.version, roomId, sessionId, {log}).response(); + const backupConfig = await this.backupConfigDeferred.promise; + if (!backupConfig) { + return; + } + const sessionResponse = await this.hsApi.roomKeyForRoomAndSession(backupConfig.info.version, roomId, sessionId, {log}).response(); if (!sessionResponse.session_data) { return; } - const sessionKeyInfo = this.crypto.decryptRoomKey(sessionResponse.session_data as SessionData); + const sessionKeyInfo = backupConfig.crypto.decryptRoomKey(sessionResponse.session_data as SessionData); if (sessionKeyInfo?.algorithm === MEGOLM_ALGORITHM) { return keyFromBackup(roomId, sessionId, sessionKeyInfo); } else if (sessionKeyInfo?.algorithm) { @@ -79,14 +107,53 @@ export class KeyBackup { return txn.inboundGroupSessions.markAllAsNotBackedUp(); } - start(log: ILogItem) { + async load(secretStorage: SecretStorage, txn: Transaction) { + // TODO: no exception here we should anticipate? + const base64PrivateKey = await secretStorage.readSecret("m.megolm_backup.v1", txn); + if (base64PrivateKey) { + this.privateKey = new Uint8Array(this.platform.encoding.base64.decode(base64PrivateKey)); + return true; + } else { + this.backupConfigDeferred.resolve(undefined); + return false; + } + } - // fetch latest version - this.flush(log); + async start(log: ILogItem) { + await log.wrap("KeyBackup.start", async log => { + if (this.privateKey && !this.backupInfoRequest) { + let backupInfo: BackupInfo; + try { + this.backupInfoRequest = this.hsApi.roomKeysVersion(undefined, {log}); + backupInfo = await this.backupInfoRequest.response() as BackupInfo; + } catch (err) { + if (err.name === "AbortError") { + log.set("aborted", true); + return; + } else { + throw err; + } + } finally { + this.backupInfoRequest = undefined; + } + // TODO: what if backupInfo is undefined or we get 404 or something? + if (backupInfo.algorithm === Curve25519.Algorithm) { + const crypto = Curve25519.BackupEncryption.fromAuthData(backupInfo.auth_data, this.privateKey, this.olm); + this.backupConfigDeferred.resolve(new BackupConfig(backupInfo, crypto)); + this.emit("change"); + } else { + this.backupConfigDeferred.resolve(undefined); + log.log({l: `Unknown backup algorithm`, algorithm: backupInfo.algorithm}); + } + this.privateKey = undefined; + } + // fetch latest version + this.flush(log); + }); } flush(log: ILogItem): void { - if (!this.operationInProgress.get()) { + if (!this._operationInProgress) { log.wrapDetached("flush key backup", async log => { if (this._needsNewKey) { log.set("needsNewKey", this._needsNewKey); @@ -96,7 +163,8 @@ export class KeyBackup { this._error = undefined; this._hasBackedUpAllKeys = false; const operation = this._runFlushOperation(log); - this.operationInProgress.set(operation); + this._operationInProgress = operation; + this.emit("change"); try { await operation.result; this._hasBackedUpAllKeys = true; @@ -113,13 +181,18 @@ export class KeyBackup { } log.catch(err); } - this.operationInProgress.set(undefined); + this._operationInProgress = undefined; + this.emit("change"); }); } } private _runFlushOperation(log: ILogItem): AbortableOperation, Progress> { return new AbortableOperation(async (setAbortable, setProgress) => { + const backupConfig = await this.backupConfigDeferred.promise; + if (!backupConfig) { + return; + } let total = 0; let amountFinished = 0; while (true) { @@ -138,8 +211,8 @@ export class KeyBackup { log.set("total", total); return; } - const payload = await this.encodeKeysForBackup(keysNeedingBackup); - const uploadRequest = this.hsApi.uploadRoomKeysToBackup(this.backupInfo.version, payload, {log}); + const payload = await this.encodeKeysForBackup(keysNeedingBackup, backupConfig.crypto); + const uploadRequest = this.hsApi.uploadRoomKeysToBackup(backupConfig.info.version, payload, {log}); setAbortable(uploadRequest); await uploadRequest.response(); await this.markKeysAsBackedUp(keysNeedingBackup, setAbortable); @@ -149,7 +222,7 @@ export class KeyBackup { }); } - private async encodeKeysForBackup(roomKeys: RoomKey[]): Promise { + private async encodeKeysForBackup(roomKeys: RoomKey[], crypto: Curve25519.BackupEncryption): Promise { const payload: KeyBackupPayload = { rooms: {} }; const payloadRooms = payload.rooms; for (const key of roomKeys) { @@ -157,7 +230,7 @@ export class KeyBackup { if (!roomPayload) { roomPayload = payloadRooms[key.roomId] = { sessions: {} }; } - roomPayload.sessions[key.sessionId] = await this.encodeRoomKey(key); + roomPayload.sessions[key.sessionId] = await this.encodeRoomKey(key, crypto); } return payload; } @@ -178,7 +251,7 @@ export class KeyBackup { await txn.complete(); } - private async encodeRoomKey(roomKey: RoomKey): Promise { + private async encodeRoomKey(roomKey: RoomKey, crypto: Curve25519.BackupEncryption): Promise { return await this.keyLoader.useKey(roomKey, session => { const firstMessageIndex = session.first_known_index(); const sessionKey = session.export_session(firstMessageIndex); @@ -186,27 +259,14 @@ export class KeyBackup { first_message_index: firstMessageIndex, forwarded_count: 0, is_verified: false, - session_data: this.crypto.encryptRoomKey(roomKey, sessionKey) + session_data: crypto.encryptRoomKey(roomKey, sessionKey) }; }); } dispose() { - this.crypto?.dispose(); - } - - static async fromSecretStorage(platform: Platform, olm: Olm, secretStorage: SecretStorage, hsApi: HomeServerApi, keyLoader: KeyLoader, storage: Storage, txn: Transaction): Promise { - const base64PrivateKey = await secretStorage.readSecret("m.megolm_backup.v1", txn); - if (base64PrivateKey) { - const privateKey = new Uint8Array(platform.encoding.base64.decode(base64PrivateKey)); - const backupInfo = await hsApi.roomKeysVersion().response() as BackupInfo; - if (backupInfo.algorithm === Curve25519.Algorithm) { - const crypto = Curve25519.BackupEncryption.fromAuthData(backupInfo.auth_data, privateKey, olm); - return new KeyBackup(backupInfo, privateKey, hsApi, keyLoader, storage, platform); - } else { - throw new Error(`Unknown backup algorithm: ${backupInfo.algorithm}`); - } - } + this.backupInfoRequest?.abort(); + this.backupConfigDeferred.value?.crypto?.dispose(); } } diff --git a/src/matrix/ssss/SecretStorage.ts b/src/matrix/ssss/SecretStorage.ts index ebdcd13a..093382a8 100644 --- a/src/matrix/ssss/SecretStorage.ts +++ b/src/matrix/ssss/SecretStorage.ts @@ -50,10 +50,20 @@ export class SecretStorage { const allAccountData = await txn.accountData.getAll(); for (const accountData of allAccountData) { try { + // TODO: fix this, using the webcrypto api closes the transaction + if (accountData.type === "m.megolm_backup.v1") { + return true; + } else { + continue; + } const secret = await this._decryptAccountData(accountData); return true; // decryption succeeded } catch (err) { - continue; + if (err instanceof DecryptionError && err.reason !== DecryptionFailure.NotEncryptedWithKey) { + throw err; + } else { + continue; + } } } return false; diff --git a/src/platform/web/ui/login/AccountSetupView.js b/src/platform/web/ui/login/AccountSetupView.js index e0d41693..cf2b544f 100644 --- a/src/platform/web/ui/login/AccountSetupView.js +++ b/src/platform/web/ui/login/AccountSetupView.js @@ -15,7 +15,7 @@ limitations under the License. */ import {TemplateView} from "../general/TemplateView"; -import {KeyBackupSettingsView} from "../session/settings/KeyBackupSettingsView.js"; +import {KeyBackupSettingsView} from "../session/settings/KeyBackupSettingsView"; export class AccountSetupView extends TemplateView { render(t, vm) { diff --git a/src/platform/web/ui/session/settings/KeyBackupSettingsView.js b/src/platform/web/ui/session/settings/KeyBackupSettingsView.ts similarity index 73% rename from src/platform/web/ui/session/settings/KeyBackupSettingsView.js rename to src/platform/web/ui/session/settings/KeyBackupSettingsView.ts index 6a886e3a..28c4febf 100644 --- a/src/platform/web/ui/session/settings/KeyBackupSettingsView.js +++ b/src/platform/web/ui/session/settings/KeyBackupSettingsView.ts @@ -14,32 +14,41 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {TemplateView} from "../../general/TemplateView"; +import {TemplateView, Builder} from "../../general/TemplateView"; import {disableTargetCallback} from "../../general/utils"; +import {ViewNode} from "../../general/types"; +import {KeyBackupViewModel, Status, BackupWriteStatus} from "../../../../../domain/session/settings/KeyBackupViewModel"; +import {KeyType} from "../../../../../matrix/ssss/index"; -export class KeyBackupSettingsView extends TemplateView { - render(t, vm) { +export class KeyBackupSettingsView extends TemplateView { + render(t: Builder, vm: KeyBackupViewModel): ViewNode { return t.div([ t.map(vm => vm.status, (status, t, vm) => { switch (status) { - case "Enabled": return renderEnabled(t, vm); - case "NewVersionAvailable": return renderNewVersionAvailable(t, vm); - case "SetupKey": return renderEnableFromKey(t, vm); - case "SetupPhrase": return renderEnableFromPhrase(t, vm); - case "Pending": return t.p(vm.i18n`Waiting to go online…`); + case Status.Enabled: return renderEnabled(t, vm); + case Status.NewVersionAvailable: return renderNewVersionAvailable(t, vm); + case Status.Setup: { + if (vm.setupKeyType === KeyType.Passphrase) { + return renderEnableFromPhrase(t, vm); + } else { + return renderEnableFromKey(t, vm); + } + break; + } + case Status.Pending: return t.p(vm.i18n`Waiting to go online…`); } }), t.map(vm => vm.backupWriteStatus, (status, t, vm) => { switch (status) { - case "Writing": { + case BackupWriteStatus.Writing: { const progress = t.progress({ - min: 0, - max: 100, + min: 0+"", + max: 100+"", value: vm => vm.backupPercentage, }); return t.div([`Backup in progress `, progress, " ", vm => vm.backupInProgressLabel]); } - case "Stopped": { + case BackupWriteStatus.Stopped: { let label; const error = vm.backupError; if (error) { @@ -47,12 +56,12 @@ export class KeyBackupSettingsView extends TemplateView { } else { label = `Backup has stopped`; } - return t.p(label, " ", t.button({onClick: () => vm.startBackup()}, `Backup now`)); + return t.p([label, " ", t.button({onClick: () => vm.startBackup()}, `Backup now`)]); } - case "Done": + case BackupWriteStatus.Done: return t.p(`All keys are backed up.`); default: - return null; + return undefined; } }), t.if(vm => vm.isMasterKeyTrusted, t => { @@ -70,7 +79,7 @@ export class KeyBackupSettingsView extends TemplateView { } } -function renderEnabled(t, vm) { +function renderEnabled(t: Builder, vm: KeyBackupViewModel): ViewNode { const items = [ t.p([vm.i18n`Key backup is enabled, using backup version ${vm.backupVersion}. `, t.button({onClick: () => vm.disable()}, vm.i18n`Disable`)]) ]; @@ -80,14 +89,14 @@ function renderEnabled(t, vm) { return t.div(items); } -function renderNewVersionAvailable(t, vm) { +function renderNewVersionAvailable(t: Builder, vm: KeyBackupViewModel): ViewNode { const items = [ t.p([vm.i18n`A new backup version has been created from another device. Disable key backup and enable it again with the new key.`, t.button({onClick: () => vm.disable()}, vm.i18n`Disable`)]) ]; return t.div(items); } -function renderEnableFromKey(t, vm) { +function renderEnableFromKey(t: Builder, vm: KeyBackupViewModel): ViewNode { const useASecurityPhrase = t.button({className: "link", onClick: () => vm.showPhraseSetup()}, vm.i18n`use a security phrase`); return t.div([ t.p(vm.i18n`Enter your secret storage security key below to ${vm.purpose}, which will enable you to decrypt messages received before you logged into this session. The security key is a code of 12 groups of 4 characters separated by a space that Element created for you when setting up security.`), @@ -97,7 +106,7 @@ function renderEnableFromKey(t, vm) { ]); } -function renderEnableFromPhrase(t, vm) { +function renderEnableFromPhrase(t: Builder, vm: KeyBackupViewModel): ViewNode { const useASecurityKey = t.button({className: "link", onClick: () => vm.showKeySetup()}, vm.i18n`use your security key`); return t.div([ t.p(vm.i18n`Enter your secret storage security phrase below to ${vm.purpose}, which will enable you to decrypt messages received before you logged into this session. The security phrase is a freeform secret phrase you optionally chose when setting up security in Element. It is different from your password to login, unless you chose to set them to the same value.`), @@ -107,7 +116,7 @@ function renderEnableFromPhrase(t, vm) { ]); } -function renderEnableFieldRow(t, vm, label, callback) { +function renderEnableFieldRow(t, vm, label, callback): ViewNode { let setupDehydrationCheck; const eventHandler = () => callback(input.value, setupDehydrationCheck?.checked || false); const input = t.input({type: "password", disabled: vm => vm.isBusy, placeholder: label}); @@ -131,8 +140,8 @@ function renderEnableFieldRow(t, vm, label, callback) { ]); } -function renderError(t) { - return t.if(vm => vm.error, (t, vm) => { +function renderError(t: Builder): ViewNode { + return t.if(vm => vm.error !== undefined, (t, vm) => { return t.div([ t.p({className: "error"}, vm => vm.i18n`Could not enable key backup: ${vm.error}.`), t.p(vm.i18n`Try double checking that you did not mix up your security key, security phrase and login password as explained above.`) diff --git a/src/platform/web/ui/session/settings/SettingsView.js b/src/platform/web/ui/session/settings/SettingsView.js index aea1108a..4035281f 100644 --- a/src/platform/web/ui/session/settings/SettingsView.js +++ b/src/platform/web/ui/session/settings/SettingsView.js @@ -16,7 +16,7 @@ limitations under the License. import {TemplateView} from "../../general/TemplateView"; import {disableTargetCallback} from "../../general/utils"; -import {KeyBackupSettingsView} from "./KeyBackupSettingsView.js" +import {KeyBackupSettingsView} from "./KeyBackupSettingsView" import {FeaturesView} from "./FeaturesView" export class SettingsView extends TemplateView { diff --git a/src/utils/AbortableOperation.ts b/src/utils/AbortableOperation.ts index e0afecd3..3592c951 100644 --- a/src/utils/AbortableOperation.ts +++ b/src/utils/AbortableOperation.ts @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {BaseObservableValue, ObservableValue} from "../observable/value"; +import {EventEmitter} from "../utils/EventEmitter"; export interface IAbortable { abort(); @@ -24,25 +24,27 @@ export type SetAbortableFn = (a: IAbortable) => typeof a; export type SetProgressFn

= (progress: P) => void; type RunFn = (setAbortable: SetAbortableFn, setProgress: SetProgressFn

) => T; -export class AbortableOperation implements IAbortable { +export class AbortableOperation extends EventEmitter<{change: keyof AbortableOperation}> implements IAbortable { public readonly result: T; private _abortable?: IAbortable; - private _progress: ObservableValue

; + private _progress?: P; constructor(run: RunFn) { + super(); this._abortable = undefined; const setAbortable: SetAbortableFn = abortable => { this._abortable = abortable; return abortable; }; - this._progress = new ObservableValue

(undefined); + this._progress = undefined; const setProgress: SetProgressFn

= (progress: P) => { - this._progress.set(progress); + this._progress = progress; + this.emit("change", "progress"); }; this.result = run(setAbortable, setProgress); } - get progress(): BaseObservableValue

{ + get progress(): P | undefined { return this._progress; } diff --git a/src/utils/Deferred.ts b/src/utils/Deferred.ts new file mode 100644 index 00000000..430fe996 --- /dev/null +++ b/src/utils/Deferred.ts @@ -0,0 +1,41 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + + +export class Deferred { + public readonly promise: Promise; + public readonly resolve: (value: T) => void; + public readonly reject: (err: Error) => void; + private _value?: T; + + constructor() { + let resolve; + let reject; + this.promise = new Promise((_resolve, _reject) => { + resolve = _resolve; + reject = _reject; + }) + this.resolve = (value: T) => { + this._value = value; + resolve(value); + }; + this.reject = reject; + } + + get value(): T | undefined { + return this._value; + } +} From 762a91bd162846608534f96ce9229804ed74e296 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 24 Mar 2023 13:42:19 +0100 Subject: [PATCH 428/435] don't reuse existing transaction to read from 4S, as webcrypto terminates idb transactions --- src/domain/SessionLoadViewModel.js | 2 +- .../session/settings/KeyBackupViewModel.ts | 2 +- src/matrix/Session.js | 46 +++++++++---------- src/matrix/e2ee/DeviceTracker.ts | 4 +- src/matrix/e2ee/megolm/keybackup/KeyBackup.ts | 5 +- src/matrix/ssss/SecretStorage.ts | 22 +++++---- src/matrix/verification/CrossSigning.ts | 27 +++++------ 7 files changed, 52 insertions(+), 56 deletions(-) diff --git a/src/domain/SessionLoadViewModel.js b/src/domain/SessionLoadViewModel.js index 6a63145f..75ecfec1 100644 --- a/src/domain/SessionLoadViewModel.js +++ b/src/domain/SessionLoadViewModel.js @@ -78,7 +78,7 @@ export class SessionLoadViewModel extends ViewModel { this._ready(client); } if (loadError) { - console.error("session load error", loadError); + console.error("session load error", loadError.stack); } } catch (err) { this._error = err; diff --git a/src/domain/session/settings/KeyBackupViewModel.ts b/src/domain/session/settings/KeyBackupViewModel.ts index cdfd4081..3426191b 100644 --- a/src/domain/session/settings/KeyBackupViewModel.ts +++ b/src/domain/session/settings/KeyBackupViewModel.ts @@ -56,7 +56,7 @@ export class KeyBackupViewModel extends ViewModel { super(options); const onKeyBackupSet = (keyBackup: KeyBackup | undefined) => { if (keyBackup && !this._keyBackupSubscription) { - this._keyBackupSubscription = this.track(this._session.keyBackup.disposableOn("change", () => { + this._keyBackupSubscription = this.track(this._session.keyBackup.get().disposableOn("change", () => { this._onKeyBackupChange(); })); } else if (!keyBackup && this._keyBackupSubscription) { diff --git a/src/matrix/Session.js b/src/matrix/Session.js index b10b2824..65555286 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -254,7 +254,7 @@ export class Session { } // TODO: stop cross-signing const key = await ssssKeyFromCredential(type, credential, this._storage, this._platform, this._olm); - if (await this._tryLoadSecretStorage(key, undefined, log)) { + if (await this._tryLoadSecretStorage(key, log)) { // only after having read a secret, write the key // as we only find out if it was good if the MAC verification succeeds await this._writeSSSSKey(key, log); @@ -318,24 +318,19 @@ export class Session { // TODO: stop cross-signing } - _tryLoadSecretStorage(ssssKey, existingTxn, log) { + _tryLoadSecretStorage(ssssKey, log) { return log.wrap("enable secret storage", async log => { - const txn = existingTxn ?? await this._storage.readTxn([ - this._storage.storeNames.accountData, - this._storage.storeNames.crossSigningKeys, - this._storage.storeNames.userIdentities, - ]); - const secretStorage = new SecretStorage({key: ssssKey, platform: this._platform}); - const isValid = await secretStorage.hasValidKeyForAnyAccountData(txn); + const secretStorage = new SecretStorage({key: ssssKey, platform: this._platform, storage: this._storage}); + const isValid = await secretStorage.hasValidKeyForAnyAccountData(); log.set("isValid", isValid); if (isValid) { - await this._loadSecretStorageServices(secretStorage, txn, log); + await this._loadSecretStorageServices(secretStorage, log); } return isValid; }); } - async _loadSecretStorageServices(secretStorage, txn, log) { + async _loadSecretStorageServices(secretStorage, log) { try { await log.wrap("enable key backup", async log => { const keyBackup = new KeyBackup( @@ -345,7 +340,7 @@ export class Session { this._storage, this._platform, ); - if (await keyBackup.load(secretStorage, txn)) { + if (await keyBackup.load(secretStorage, log)) { for (const room of this._rooms.values()) { if (room.isEncrypted) { room.enableKeyBackup(keyBackup); @@ -370,7 +365,7 @@ export class Session { ownUserId: this.userId, e2eeAccount: this._e2eeAccount }); - if (await crossSigning.load(txn, log)) { + if (await crossSigning.load(log)) { this._crossSigning.set(crossSigning); } }); @@ -497,15 +492,8 @@ export class Session { olmWorker: this._olmWorker, txn }); - if (this._e2eeAccount) { - log.set("keys", this._e2eeAccount.identityKeys); - this._setupEncryption(); - // try set up session backup if we stored the ssss key - const ssssKey = await ssssReadKey(txn); - if (ssssKey) { - await this._tryLoadSecretStorage(ssssKey, txn, log); - } - } + log.set("keys", this._e2eeAccount.identityKeys); + this._setupEncryption(); } const pendingEventsByRoomId = await this._getPendingEventsByRoom(txn); // load invites @@ -530,6 +518,14 @@ export class Session { room.setInvite(invite); } } + if (this._olm && this._e2eeAccount) { + // try set up session backup and cross-signing if we stored the ssss key + const ssssKey = await ssssReadKey(txn); + if (ssssKey) { + // this will close the txn above, so we do it last + await this._tryLoadSecretStorage(ssssKey, log); + } + } } dispose() { @@ -570,15 +566,15 @@ export class Session { await log.wrap("SSSSKeyFromDehydratedDeviceKey", async log => { const ssssKey = await createSSSSKeyFromDehydratedDeviceKey(dehydratedDevice.key, this._storage, this._platform); if (ssssKey) { - if (await this._tryLoadSecretStorage(ssssKey, undefined, log)) { + if (await this._tryLoadSecretStorage(ssssKey, log)) { log.set("success", true); await this._writeSSSSKey(ssssKey); } } }); } - this._keyBackup.get()?.start(log); - this._crossSigning.get()?.start(log); + await this._keyBackup.get()?.start(log); + await this._crossSigning.get()?.start(log); // restore unfinished operations, like sending out room keys const opsTxn = await this._storage.readWriteTxn([ diff --git a/src/matrix/e2ee/DeviceTracker.ts b/src/matrix/e2ee/DeviceTracker.ts index 23bdc31e..3a50a890 100644 --- a/src/matrix/e2ee/DeviceTracker.ts +++ b/src/matrix/e2ee/DeviceTracker.ts @@ -163,9 +163,9 @@ export class DeviceTracker { } } - async getCrossSigningKeyForUser(userId: string, usage: KeyUsage, hsApi: HomeServerApi | undefined, existingTxn: Transaction | undefined, log: ILogItem): Promise { + async getCrossSigningKeyForUser(userId: string, usage: KeyUsage, hsApi: HomeServerApi | undefined, log: ILogItem): Promise { return await log.wrap({l: "DeviceTracker.getCrossSigningKeyForUser", id: userId, usage}, async log => { - const txn = existingTxn ?? await this._storage.readTxn([ + const txn = await this._storage.readTxn([ this._storage.storeNames.userIdentities, this._storage.storeNames.crossSigningKeys, ]); diff --git a/src/matrix/e2ee/megolm/keybackup/KeyBackup.ts b/src/matrix/e2ee/megolm/keybackup/KeyBackup.ts index 8e9a4a81..da107502 100644 --- a/src/matrix/e2ee/megolm/keybackup/KeyBackup.ts +++ b/src/matrix/e2ee/megolm/keybackup/KeyBackup.ts @@ -107,9 +107,8 @@ export class KeyBackup extends EventEmitter<{change: never}> { return txn.inboundGroupSessions.markAllAsNotBackedUp(); } - async load(secretStorage: SecretStorage, txn: Transaction) { - // TODO: no exception here we should anticipate? - const base64PrivateKey = await secretStorage.readSecret("m.megolm_backup.v1", txn); + async load(secretStorage: SecretStorage, log: ILogItem) { + const base64PrivateKey = await secretStorage.readSecret("m.megolm_backup.v1"); if (base64PrivateKey) { this.privateKey = new Uint8Array(this.platform.encoding.base64.decode(base64PrivateKey)); return true; diff --git a/src/matrix/ssss/SecretStorage.ts b/src/matrix/ssss/SecretStorage.ts index 093382a8..4c767bbb 100644 --- a/src/matrix/ssss/SecretStorage.ts +++ b/src/matrix/ssss/SecretStorage.ts @@ -40,22 +40,22 @@ class DecryptionError extends Error { export class SecretStorage { private readonly _key: Key; private readonly _platform: Platform; + private readonly _storage: Storage; - constructor({key, platform}: {key: Key, platform: Platform}) { + constructor({key, platform, storage}: {key: Key, platform: Platform, storage: Storage}) { this._key = key; this._platform = platform; + this._storage = storage; } - async hasValidKeyForAnyAccountData(txn: Transaction) { + /** this method will auto-commit any indexeddb transaction because of its use of the webcrypto api */ + async hasValidKeyForAnyAccountData() { + const txn = await this._storage.readTxn([ + this._storage.storeNames.accountData, + ]); const allAccountData = await txn.accountData.getAll(); for (const accountData of allAccountData) { try { - // TODO: fix this, using the webcrypto api closes the transaction - if (accountData.type === "m.megolm_backup.v1") { - return true; - } else { - continue; - } const secret = await this._decryptAccountData(accountData); return true; // decryption succeeded } catch (err) { @@ -69,7 +69,11 @@ export class SecretStorage { return false; } - async readSecret(name: string, txn: Transaction): Promise { + /** this method will auto-commit any indexeddb transaction because of its use of the webcrypto api */ + async readSecret(name: string): Promise { + const txn = await this._storage.readTxn([ + this._storage.storeNames.accountData, + ]); const accountData = await txn.accountData.get(name); if (!accountData) { return; diff --git a/src/matrix/verification/CrossSigning.ts b/src/matrix/verification/CrossSigning.ts index 1abc3702..196190ae 100644 --- a/src/matrix/verification/CrossSigning.ts +++ b/src/matrix/verification/CrossSigning.ts @@ -99,22 +99,22 @@ export class CrossSigning { this.e2eeAccount = options.e2eeAccount } - async load(txn: Transaction, log: ILogItem) { + async load(log: ILogItem) { // try to verify the msk without accessing the network - return await this.verifyMSKFrom4S(undefined, txn, log); + return await this.verifyMSKFrom4S(false, log); } async start(log: ILogItem) { if (!this.isMasterKeyTrusted) { // try to verify the msk _with_ access to the network - return await this.verifyMSKFrom4S(this.hsApi, undefined, log); + return await this.verifyMSKFrom4S(true, log); } } - private async verifyMSKFrom4S(hsApi: HomeServerApi | undefined, txn: Transaction | undefined, log: ILogItem): Promise { + private async verifyMSKFrom4S(allowNetwork: boolean, log: ILogItem): Promise { return await log.wrap("CrossSigning.verifyMSKFrom4S", async log => { // TODO: use errorboundary here - const privateMasterKey = await this.getSigningKey(KeyUsage.Master, txn); + const privateMasterKey = await this.getSigningKey(KeyUsage.Master); if (!privateMasterKey) { return false; } @@ -125,7 +125,7 @@ export class CrossSigning { } finally { signing.free(); } - const publishedMasterKey = await this.deviceTracker.getCrossSigningKeyForUser(this.ownUserId, KeyUsage.Master, hsApi, txn, log); + const publishedMasterKey = await this.deviceTracker.getCrossSigningKeyForUser(this.ownUserId, KeyUsage.Master, allowNetwork ? this.hsApi : undefined, log); if (!publishedMasterKey) { return false; } @@ -210,11 +210,11 @@ export class CrossSigning { if (!this.isMasterKeyTrusted) { return UserTrust.OwnSetupError; } - const ourMSK = await log.wrap("get our msk", log => this.deviceTracker.getCrossSigningKeyForUser(this.ownUserId, KeyUsage.Master, this.hsApi, txn, log)); + const ourMSK = await log.wrap("get our msk", log => this.deviceTracker.getCrossSigningKeyForUser(this.ownUserId, KeyUsage.Master, this.hsApi, undefined, log)); if (!ourMSK) { return UserTrust.OwnSetupError; } - const ourUSK = await log.wrap("get our usk", log => this.deviceTracker.getCrossSigningKeyForUser(this.ownUserId, KeyUsage.UserSigning, this.hsApi, txn, log)); + const ourUSK = await log.wrap("get our usk", log => this.deviceTracker.getCrossSigningKeyForUser(this.ownUserId, KeyUsage.UserSigning, this.hsApi, undefined, log)); if (!ourUSK) { return UserTrust.OwnSetupError; } @@ -222,7 +222,7 @@ export class CrossSigning { if (ourUSKVerification !== SignatureVerification.Valid) { return UserTrust.OwnSetupError; } - const theirMSK = await log.wrap("get their msk", log => this.deviceTracker.getCrossSigningKeyForUser(userId, KeyUsage.Master, this.hsApi, txn, log)); + const theirMSK = await log.wrap("get their msk", log => this.deviceTracker.getCrossSigningKeyForUser(userId, KeyUsage.Master, this.hsApi, undefined, log)); if (!theirMSK) { /* assume that when they don't have an MSK, they've never enabled cross-signing on their client (or it's not supported) rather than assuming a setup error on their side. @@ -237,7 +237,7 @@ export class CrossSigning { return UserTrust.UserSignatureMismatch; } } - const theirSSK = await log.wrap("get their ssk", log => this.deviceTracker.getCrossSigningKeyForUser(userId, KeyUsage.SelfSigning, this.hsApi, txn, log)); + const theirSSK = await log.wrap("get their ssk", log => this.deviceTracker.getCrossSigningKeyForUser(userId, KeyUsage.SelfSigning, this.hsApi, undefined, log)); if (!theirSSK) { return UserTrust.UserSetupError; } @@ -290,11 +290,8 @@ export class CrossSigning { return keyToSign; } - private async getSigningKey(usage: KeyUsage, existingTxn?: Transaction): Promise { - const txn = existingTxn ?? await this.storage.readTxn([ - this.storage.storeNames.accountData, - ]); - const seedStr = await this.secretStorage.readSecret(`m.cross_signing.${usage}`, txn); + private async getSigningKey(usage: KeyUsage): Promise { + const seedStr = await this.secretStorage.readSecret(`m.cross_signing.${usage}`); if (seedStr) { return new Uint8Array(this.platform.encoding.base64.decode(seedStr)); } From 90faad551a5b67ace68f7917d7aa38d212e31dac Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 24 Mar 2023 23:07:22 +0100 Subject: [PATCH 429/435] remove txn argument that was removed in previous commit --- src/matrix/verification/CrossSigning.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/matrix/verification/CrossSigning.ts b/src/matrix/verification/CrossSigning.ts index 196190ae..bd661e7d 100644 --- a/src/matrix/verification/CrossSigning.ts +++ b/src/matrix/verification/CrossSigning.ts @@ -210,11 +210,11 @@ export class CrossSigning { if (!this.isMasterKeyTrusted) { return UserTrust.OwnSetupError; } - const ourMSK = await log.wrap("get our msk", log => this.deviceTracker.getCrossSigningKeyForUser(this.ownUserId, KeyUsage.Master, this.hsApi, undefined, log)); + const ourMSK = await log.wrap("get our msk", log => this.deviceTracker.getCrossSigningKeyForUser(this.ownUserId, KeyUsage.Master, this.hsApi, log)); if (!ourMSK) { return UserTrust.OwnSetupError; } - const ourUSK = await log.wrap("get our usk", log => this.deviceTracker.getCrossSigningKeyForUser(this.ownUserId, KeyUsage.UserSigning, this.hsApi, undefined, log)); + const ourUSK = await log.wrap("get our usk", log => this.deviceTracker.getCrossSigningKeyForUser(this.ownUserId, KeyUsage.UserSigning, this.hsApi, log)); if (!ourUSK) { return UserTrust.OwnSetupError; } @@ -222,7 +222,7 @@ export class CrossSigning { if (ourUSKVerification !== SignatureVerification.Valid) { return UserTrust.OwnSetupError; } - const theirMSK = await log.wrap("get their msk", log => this.deviceTracker.getCrossSigningKeyForUser(userId, KeyUsage.Master, this.hsApi, undefined, log)); + const theirMSK = await log.wrap("get their msk", log => this.deviceTracker.getCrossSigningKeyForUser(userId, KeyUsage.Master, this.hsApi, log)); if (!theirMSK) { /* assume that when they don't have an MSK, they've never enabled cross-signing on their client (or it's not supported) rather than assuming a setup error on their side. @@ -237,7 +237,7 @@ export class CrossSigning { return UserTrust.UserSignatureMismatch; } } - const theirSSK = await log.wrap("get their ssk", log => this.deviceTracker.getCrossSigningKeyForUser(userId, KeyUsage.SelfSigning, this.hsApi, undefined, log)); + const theirSSK = await log.wrap("get their ssk", log => this.deviceTracker.getCrossSigningKeyForUser(userId, KeyUsage.SelfSigning, this.hsApi, log)); if (!theirSSK) { return UserTrust.UserSetupError; } From d170c6f7871cf86f504fe26e0f01799164eae01d Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 24 Mar 2023 23:10:54 +0100 Subject: [PATCH 430/435] crossSigning is an observable value now --- src/domain/session/rightpanel/MemberDetailsViewModel.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/domain/session/rightpanel/MemberDetailsViewModel.js b/src/domain/session/rightpanel/MemberDetailsViewModel.js index 0eabf33a..1a5dabd8 100644 --- a/src/domain/session/rightpanel/MemberDetailsViewModel.js +++ b/src/domain/session/rightpanel/MemberDetailsViewModel.js @@ -30,6 +30,9 @@ export class MemberDetailsViewModel extends ViewModel { this._session = options.session; this.track(this._powerLevelsObservable.subscribe(() => this._onPowerLevelsChange())); this.track(this._observableMember.subscribe( () => this._onMemberChange())); + this.track(this._session.crossSigning.subscribe(() => { + this.emitChange("isTrusted"); + })); this._userTrust = undefined; this.init(); // TODO: call this from parent view model and do something smart with error view model if it fails async? } @@ -37,7 +40,7 @@ export class MemberDetailsViewModel extends ViewModel { async init() { if (this.features.crossSigning) { this._userTrust = await this.logger.run({l: "MemberDetailsViewModel.get user trust", id: this._member.userId}, log => { - return this._session.crossSigning.getUserTrust(this._member.userId, log); + return this._session.crossSigning.get()?.getUserTrust(this._member.userId, log); }); this.emitChange("isTrusted"); } From 9383246f8de6627a49a1e4651cb375203e78d507 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 24 Mar 2023 23:14:30 +0100 Subject: [PATCH 431/435] remove obsolete parameter here as well --- src/matrix/verification/CrossSigning.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/verification/CrossSigning.ts b/src/matrix/verification/CrossSigning.ts index bd661e7d..2be443e5 100644 --- a/src/matrix/verification/CrossSigning.ts +++ b/src/matrix/verification/CrossSigning.ts @@ -182,7 +182,7 @@ export class CrossSigning { if (userId === this.ownUserId) { return; } - const keyToSign = await this.deviceTracker.getCrossSigningKeyForUser(userId, KeyUsage.Master, this.hsApi, undefined, log); + const keyToSign = await this.deviceTracker.getCrossSigningKeyForUser(userId, KeyUsage.Master, this.hsApi, log); if (!keyToSign) { return; } From eaa7de8a551d77946c86288cbc540b27f0631231 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 24 Mar 2023 23:16:54 +0100 Subject: [PATCH 432/435] fix import --- src/matrix/verification/CrossSigning.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/verification/CrossSigning.ts b/src/matrix/verification/CrossSigning.ts index f98c469a..2a4d21f2 100644 --- a/src/matrix/verification/CrossSigning.ts +++ b/src/matrix/verification/CrossSigning.ts @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { ILogItem } from "../../lib"; +import {ILogItem} from "../../logging/types"; import {pkSign} from "./common"; import type {SecretStorage} from "../ssss/SecretStorage"; From 6abc918ce89c17cbc4d2654727250e55ab38c704 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 27 Mar 2023 10:54:44 +0200 Subject: [PATCH 433/435] show shield as icon --- .../rightpanel/MemberDetailsViewModel.js | 5 ++- src/platform/web/ui/css/right-panel.css | 31 +++++++++++++++++++ .../element/icons/verification-error.svg | 3 ++ .../ui/css/themes/element/icons/verified.svg | 3 ++ .../session/rightpanel/MemberDetailsView.js | 7 +++-- 5 files changed, 43 insertions(+), 6 deletions(-) create mode 100644 src/platform/web/ui/css/themes/element/icons/verification-error.svg create mode 100644 src/platform/web/ui/css/themes/element/icons/verified.svg diff --git a/src/domain/session/rightpanel/MemberDetailsViewModel.js b/src/domain/session/rightpanel/MemberDetailsViewModel.js index 1a5dabd8..d4e1151f 100644 --- a/src/domain/session/rightpanel/MemberDetailsViewModel.js +++ b/src/domain/session/rightpanel/MemberDetailsViewModel.js @@ -31,7 +31,7 @@ export class MemberDetailsViewModel extends ViewModel { this.track(this._powerLevelsObservable.subscribe(() => this._onPowerLevelsChange())); this.track(this._observableMember.subscribe( () => this._onMemberChange())); this.track(this._session.crossSigning.subscribe(() => { - this.emitChange("isTrusted"); + this.emitChange("trustShieldColor"); })); this._userTrust = undefined; this.init(); // TODO: call this from parent view model and do something smart with error view model if it fails async? @@ -42,13 +42,12 @@ export class MemberDetailsViewModel extends ViewModel { this._userTrust = await this.logger.run({l: "MemberDetailsViewModel.get user trust", id: this._member.userId}, log => { return this._session.crossSigning.get()?.getUserTrust(this._member.userId, log); }); - this.emitChange("isTrusted"); + this.emitChange("trustShieldColor"); } } get name() { return this._member.name; } get userId() { return this._member.userId; } - get isTrusted() { return this._userTrust === UserTrust.Trusted; } get trustDescription() { switch (this._userTrust) { case UserTrust.Trusted: return this.i18n`You have verified this user. This user has verified all of their sessions.`; diff --git a/src/platform/web/ui/css/right-panel.css b/src/platform/web/ui/css/right-panel.css index 92a89c0a..5af0e6a0 100644 --- a/src/platform/web/ui/css/right-panel.css +++ b/src/platform/web/ui/css/right-panel.css @@ -19,6 +19,37 @@ text-align: center; } +.MemberDetailsView_shield_container { + display: flex; + gap: 4px; +} + +.MemberDetailsView_shield_red, .MemberDetailsView_shield_green, .MemberDetailsView_shield_black { + background-size: contain; + background-repeat: no-repeat; + width: 24px; + height: 24px; + display: block; + flex-shrink: 0; +} + +.MemberDetailsView_shield_description { + flex-grow: 1; + margin: 0; +} + +.MemberDetailsView_shield_red { + background-image: url("./icons/verification-error.svg?primary=error-color"); +} + +.MemberDetailsView_shield_green { + background-image: url("./icons/verified.svg?primary=accent-color"); +} + +.MemberDetailsView_shield_black { + background-image: url("./icons/encryption-status.svg?primary=text-color"); +} + .RoomDetailsView_label, .RoomDetailsView_row, .RoomDetailsView, .MemberDetailsView, .EncryptionIconView { display: flex; align-items: center; diff --git a/src/platform/web/ui/css/themes/element/icons/verification-error.svg b/src/platform/web/ui/css/themes/element/icons/verification-error.svg new file mode 100644 index 00000000..9733f563 --- /dev/null +++ b/src/platform/web/ui/css/themes/element/icons/verification-error.svg @@ -0,0 +1,3 @@ + + + diff --git a/src/platform/web/ui/css/themes/element/icons/verified.svg b/src/platform/web/ui/css/themes/element/icons/verified.svg new file mode 100644 index 00000000..340891f1 --- /dev/null +++ b/src/platform/web/ui/css/themes/element/icons/verified.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/src/platform/web/ui/session/rightpanel/MemberDetailsView.js b/src/platform/web/ui/session/rightpanel/MemberDetailsView.js index aa70d5b4..c02d8d73 100644 --- a/src/platform/web/ui/session/rightpanel/MemberDetailsView.js +++ b/src/platform/web/ui/session/rightpanel/MemberDetailsView.js @@ -26,9 +26,10 @@ export class MemberDetailsView extends TemplateView { ] if (vm.features.crossSigning) { - securityNodes.push(t.p(vm => vm.isTrusted ? vm.i18n`This user is trusted` : vm.i18n`This user is not trusted`)); - securityNodes.push(t.p(vm => vm.trustDescription)); - securityNodes.push(t.p(["Shield color: ", vm => vm.trustShieldColor])); + securityNodes.push(t.div({className: "MemberDetailsView_shield_container"}, [ + t.span({className: vm => `MemberDetailsView_shield_${vm.trustShieldColor}`}), + t.p({className: "MemberDetailsView_shield_description"}, vm => vm.trustDescription) + ])); } return t.div({className: "MemberDetailsView"}, From 21729a60490af4debebe0903d1078503a043bdb1 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 27 Mar 2023 10:57:26 +0200 Subject: [PATCH 434/435] add newlines between getters --- src/domain/session/rightpanel/MemberDetailsViewModel.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/domain/session/rightpanel/MemberDetailsViewModel.js b/src/domain/session/rightpanel/MemberDetailsViewModel.js index d4e1151f..b73bf4bb 100644 --- a/src/domain/session/rightpanel/MemberDetailsViewModel.js +++ b/src/domain/session/rightpanel/MemberDetailsViewModel.js @@ -47,7 +47,9 @@ export class MemberDetailsViewModel extends ViewModel { } get name() { return this._member.name; } + get userId() { return this._member.userId; } + get trustDescription() { switch (this._userTrust) { case UserTrust.Trusted: return this.i18n`You have verified this user. This user has verified all of their sessions.`; @@ -60,6 +62,7 @@ export class MemberDetailsViewModel extends ViewModel { default: return this.i18n`Pending…`; } } + get trustShieldColor() { if (!this._isEncrypted) { return undefined; @@ -78,7 +81,9 @@ export class MemberDetailsViewModel extends ViewModel { } get type() { return "member-details"; } + get shouldShowBackButton() { return true; } + get previousSegmentName() { return "members"; } get role() { From 22140614ec3ec864bba6e23e030b50d19f3482dd Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 27 Mar 2023 11:06:30 +0200 Subject: [PATCH 435/435] clear cross-signing object when disabling 4s --- src/matrix/Session.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 65555286..4acdd56a 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -252,7 +252,9 @@ export class Session { this._keyBackup.get().dispose(); this._keyBackup.set(undefined); } - // TODO: stop cross-signing + if (this._crossSigning.get()) { + this._crossSigning.set(undefined); + } const key = await ssssKeyFromCredential(type, credential, this._storage, this._platform, this._olm); if (await this._tryLoadSecretStorage(key, log)) { // only after having read a secret, write the key @@ -313,9 +315,11 @@ export class Session { } } this._keyBackup.get().dispose(); - this._keyBackup.set(null); + this._keyBackup.set(undefined); + } + if (this._crossSigning.get()) { + this._crossSigning.set(undefined); } - // TODO: stop cross-signing } _tryLoadSecretStorage(ssssKey, log) {