/* Copyright 2022 The Matrix.org Foundation C.I.C. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ import {ObservableMap} from "../../observable/map/ObservableMap"; import {recursivelyAssign} from "../../utils/recursivelyAssign"; import {AsyncQueue} from "../../utils/AsyncQueue"; import type {Room} from "../room/Room"; import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; import {WebRTC, PeerConnection, PeerConnectionHandler, StreamPurpose} from "../../platform/types/WebRTC"; import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; import type {LocalMedia} from "./LocalMedia"; // when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already // do for sharing keys will be best as that already deals with room tracking. /** * Does WebRTC signalling for a single PeerConnection, and deals with WebRTC wrappers from platform * */ /** Implements a call between two peers with the signalling state keeping, while still delegating the signalling message sending. Used by GroupCall.*/ class PeerCall { private readonly peerConnection: PeerConnection; private state = CallState.Fledgling; private direction: CallDirection; // A queue for candidates waiting to go out. // We try to amalgamate candidates into a single candidate message where // possible private candidateSendQueue: Array = []; // If candidates arrive before we've picked an opponent (which, in particular, // will happen if the opponent sends candidates eagerly before the user answers // the call) we buffer them up here so we can then add the ones from the party we pick private remoteCandidateBuffer? = new Map(); private logger: any; private remoteSDPStreamMetadata?: SDPStreamMetadata; private responsePromiseChain?: Promise; private opponentPartyId?: PartyId; constructor( private readonly handler: PeerCallHandler, private localMedia: LocalMedia, webRTC: WebRTC ) { const outer = this; this.peerConnection = webRTC.createPeerConnection({ onIceConnectionStateChange(state: RTCIceConnectionState) {}, onLocalIceCandidate(candidate: RTCIceCandidate) {}, onIceGatheringStateChange(state: RTCIceGatheringState) {}, onRemoteTracksChanged(tracks: Track[]) {}, onDataChannelChanged(dataChannel: DataChannel | undefined) {}, onNegotiationNeeded() { const promiseCreator = () => outer.handleNegotiation(); outer.responsePromiseChain = outer.responsePromiseChain?.then(promiseCreator) ?? promiseCreator(); }, getPurposeForStreamId(streamId: string): SDPStreamMetadataPurpose { return outer.remoteSDPStreamMetadata?.[streamId]?.purpose ?? SDPStreamMetadataPurpose.Usermedia; } }); this.logger = { debug(...args) { console.log.apply(console, ["WebRTC debug:", ...args])}, log(...args) { console.log.apply(console, ["WebRTC log:", ...args])}, warn(...args) { console.log.apply(console, ["WebRTC warn:", ...args])}, error(...args) { console.error.apply(console, ["WebRTC error:", ...args])}, } } handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId) { switch (message.type) { case EventType.Invite: this.handleInvite(message.content); break; case EventType.Answer: this.handleAnswer(message.content); break; case EventType.Candidates: this.handleRemoteIceCandidates(message.content); break; case EventType.Hangup: } } async call(localMediaPromise: Promise): Promise { if (this.state !== CallState.Fledgling) { return; } this.direction = CallDirection.Outbound; this.setState(CallState.WaitLocalMedia); try { this.localMedia = await localMediaPromise; } catch (err) { this.setState(CallState.Ended); return; } this.setState(CallState.CreateOffer); // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called for (const t of this.localMedia.tracks) { this.peerConnection.addTrack(t); } await this.waitForState(CallState.InviteSent); } async answer() { } async hangup() { } async updateLocalMedia(localMediaPromise: Promise) { const oldMedia = this.localMedia; this.localMedia = await localMediaPromise; const applyTrack = (selectTrack: (media: LocalMedia) => Track | undefined) => { const oldTrack = selectTrack(oldMedia); const newTrack = selectTrack(this.localMedia); if (oldTrack && newTrack) { this.peerConnection.replaceTrack(oldTrack, newTrack); } else if (oldTrack) { this.peerConnection.removeTrack(oldTrack); } else if (newTrack) { this.peerConnection.addTrack(newTrack); } }; // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called applyTrack(m => m.microphoneTrack); applyTrack(m => m.cameraTrack); applyTrack(m => m.screenShareTrack); } // calls are serialized and deduplicated by negotiationQueue private handleNegotiation = async (): Promise => { try { await this.peerConnection.setLocalDescription(); } catch (err) { this.logger.debug(`Call ${this.callId} Error setting local description!`, err); this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); return; } if (this.peerConnection.iceGatheringState === 'gathering') { // Allow a short time for initial candidates to be gathered await new Promise(resolve => setTimeout(resolve, 200)); } if (this.state === CallState.Ended) { return; } const offer = this.peerConnection.localDescription!; // Get rid of any candidates waiting to be sent: they'll be included in the local // description we just got and will send in the offer. this.logger.info(`Call ${this.callId} Discarding ${ this.candidateSendQueue.length} candidates that will be sent in offer`); this.candidateSendQueue = []; // need to queue this const content = { offer, [SDPStreamMetadataKey]: this.localMedia.getSDPMetadata(), version: 1, lifetime: CALL_TIMEOUT_MS }; if (this.state === CallState.CreateOffer) { await this.handler.sendSignallingMessage({type: EventType.Invite, content}); this.setState(CallState.InviteSent); } }; private async handleInvite(content: InviteContent, partyId: PartyId): Promise { if (this.state !== CallState.Fledgling || this.opponentPartyId !== undefined) { // TODO: hangup or ignore? return; } // we must set the party ID before await-ing on anything: the call event // handler will start giving us more call events (eg. candidates) so if // we haven't set the party ID, we'll ignore them. this.opponentPartyId = partyId; this.direction = CallDirection.Inbound; const sdpStreamMetadata = content[SDPStreamMetadataKey]; if (sdpStreamMetadata) { this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); } else { this.logger.debug(`Call ${ this.callId} did not get any SDPStreamMetadata! Can not send/receive multiple streams`); } try { await this.peerConnection.setRemoteDescription(content.offer); await this.addBufferedIceCandidates(); } catch (e) { this.logger.debug(`Call ${this.callId} failed to set remote description`, e); this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); return; } // According to previous comments in this file, firefox at some point did not // add streams until media started arriving on them. Testing latest firefox // (81 at time of writing), this is no longer a problem, so let's do it the correct way. if (this.peerConnection.remoteTracks.length === 0) { this.logger.error(`Call ${this.callId} no remote stream or no tracks after setting remote description!`); this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); return; } this.setState(CallState.Ringing); setTimeout(() => { if (this.state == CallState.Ringing) { this.logger.debug(`Call ${this.callId} invite has expired. Hanging up.`); this.hangupParty = CallParty.Remote; // effectively this.setState(CallState.Ended); this.stopAllMedia(); if (this.peerConnection.signalingState != 'closed') { this.peerConnection.close(); } this.emit(CallEvent.Hangup); } }, content.lifetime ?? CALL_TIMEOUT_MS /* - event.getLocalAge() */ ); } private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { this.remoteSDPStreamMetadata = recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); // will rerequest stream purpose for all tracks and set track.type accordingly this.peerConnection.notifyStreamPurposeChanged(); for (const track of this.peerConnection.remoteTracks) { const streamMetaData = this.remoteSDPStreamMetadata?.[track.streamId]; if (streamMetaData) { if (track.type === TrackType.Microphone) { track.setMuted(streamMetaData.audio_muted); } else { // Camera or ScreenShare track.setMuted(streamMetaData.video_muted); } } } } private async addBufferedIceCandidates(): Promise { const bufferedCandidates = this.remoteCandidateBuffer!.get(this.opponentPartyId!); if (bufferedCandidates) { this.logger.info(`Call ${this.callId} Adding ${ bufferedCandidates.length} buffered candidates for opponent ${this.opponentPartyId}`); await this.addIceCandidates(bufferedCandidates); } this.remoteCandidateBuffer = undefined; } private async addIceCandidates(candidates: RTCIceCandidate[]): Promise { for (const candidate of candidates) { if ( (candidate.sdpMid === null || candidate.sdpMid === undefined) && (candidate.sdpMLineIndex === null || candidate.sdpMLineIndex === undefined) ) { this.logger.debug(`Call ${this.callId} ignoring remote ICE candidate with no sdpMid or sdpMLineIndex`); continue; } this.logger.debug(`Call ${this.callId} got remote ICE ${candidate.sdpMid} candidate: ${candidate.candidate}`); try { await this.peerConnection.addIceCandidate(candidate); } catch (err) { if (!this.ignoreOffer) { this.logger.info(`Call ${this.callId} failed to add remote ICE candidate`, err); } } } } private setState(state: CallState): void { const oldState = this.state; this.state = state; this.handler.emitUpdate(); } private waitForState(state: CallState): Promise { } private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean): Promise { } } //import { randomString } from '../randomstring'; import { MCallReplacesEvent, MCallAnswer, MCallInviteNegotiate, CallCapabilities, SDPStreamMetadataPurpose, SDPStreamMetadata, SDPStreamMetadataKey, MCallSDPStreamMetadataChanged, MCallSelectAnswer, MCAllAssertedIdentity, MCallCandidates, MCallBase, MCallHangupReject, } from './callEventTypes'; // null is used as a special value meaning that the we're in a legacy 1:1 call // without MSC2746 that doesn't provide an id which device sent the message. type PartyId = string | null; export enum CallParty { Local = 'local', Remote = 'remote', } export enum CallState { Fledgling = 'fledgling', InviteSent = 'invite_sent', WaitLocalMedia = 'wait_local_media', CreateOffer = 'create_offer', CreateAnswer = 'create_answer', Connecting = 'connecting', Connected = 'connected', Ringing = 'ringing', Ended = 'ended', } export enum CallDirection { Inbound = 'inbound', Outbound = 'outbound', } export enum EventType { Invite = "m.call.invite", Candidates = "m.call.candidates", Answer = "m.call.answer", Hangup = "m.call.hangup", Reject = "m.call.reject", SelectAnswer = "m.call.select_answer", Negotiate = "m.call.negotiate", SDPStreamMetadataChanged = "m.call.sdp_stream_metadata_changed", SDPStreamMetadataChangedPrefix = "org.matrix.call.sdp_stream_metadata_changed", Replaces = "m.call.replaces", AssertedIdentity = "m.call.asserted_identity", AssertedIdentityPrefix = "org.matrix.call.asserted_identity", } export enum CallErrorCode { /** The user chose to end the call */ UserHangup = 'user_hangup', /** An error code when the local client failed to create an offer. */ LocalOfferFailed = 'local_offer_failed', /** * An error code when there is no local mic/camera to use. This may be because * the hardware isn't plugged in, or the user has explicitly denied access. */ NoUserMedia = 'no_user_media', /** * Error code used when a call event failed to send * because unknown devices were present in the room */ UnknownDevices = 'unknown_devices', /** * Error code used when we fail to send the invite * for some reason other than there being unknown devices */ SendInvite = 'send_invite', /** * An answer could not be created */ CreateAnswer = 'create_answer', /** * Error code used when we fail to send the answer * for some reason other than there being unknown devices */ SendAnswer = 'send_answer', /** * The session description from the other side could not be set */ SetRemoteDescription = 'set_remote_description', /** * The session description from this side could not be set */ SetLocalDescription = 'set_local_description', /** * A different device answered the call */ AnsweredElsewhere = 'answered_elsewhere', /** * No media connection could be established to the other party */ IceFailed = 'ice_failed', /** * The invite timed out whilst waiting for an answer */ InviteTimeout = 'invite_timeout', /** * The call was replaced by another call */ Replaced = 'replaced', /** * Signalling for the call could not be sent (other than the initial invite) */ SignallingFailed = 'signalling_timeout', /** * The remote party is busy */ UserBusy = 'user_busy', /** * We transferred the call off to somewhere else */ Transfered = 'transferred', /** * A call from the same user was found with a new session id */ NewSession = 'new_session', } /** * The version field that we set in m.call.* events */ const VOIP_PROTO_VERSION = 1; /** The fallback ICE server to use for STUN or TURN protocols. */ const FALLBACK_ICE_SERVER = 'stun:turn.matrix.org'; /** The length of time a call can be ringing for. */ const CALL_TIMEOUT_MS = 60000; const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); export class CallError extends Error { code: string; constructor(code: CallErrorCode, msg: string, err: Error) { // Still don't think there's any way to have proper nested errors super(msg + ": " + err); this.code = code; } } type InviteContent = { offer: RTCSessionDescriptionInit, [SDPStreamMetadataKey]: SDPStreamMetadata, version?: number, lifetime?: number } export type InviteMessage = { type: EventType.Invite, content: InviteContent } export type SignallingMessage = InviteMessage; export interface PeerCallHandler { emitUpdate(peerCall: PeerCall, params: any); sendSignallingMessage(message: InviteMessage); }