diff --git a/doc/SDK.md b/doc/SDK.md index 8ce0b304..54e37cca 100644 --- a/doc/SDK.md +++ b/doc/SDK.md @@ -47,7 +47,8 @@ const assetPaths = { wasmBundle: olmJsPath } }; -import "hydrogen-view-sdk/style.css"; +import "hydrogen-view-sdk/theme-element-light.css"; +// OR import "hydrogen-view-sdk/theme-element-dark.css"; async function main() { const app = document.querySelector('#app')! diff --git a/scripts/postcss/css-url-processor.js b/scripts/postcss/css-url-processor.js index 3ae7c60d..f58818f1 100644 --- a/scripts/postcss/css-url-processor.js +++ b/scripts/postcss/css-url-processor.js @@ -39,7 +39,7 @@ function colorsFromURL(url, colorMap) { function processURL(decl, replacer, colorMap) { const value = decl.value; const parsed = valueParser(value); - parsed.walk(async node => { + parsed.walk(node => { if (node.type !== "function" || node.value !== "url") { return; } diff --git a/scripts/postcss/svg-colorizer.js b/scripts/postcss/svg-colorizer.js index 95355ea8..06b7b14b 100644 --- a/scripts/postcss/svg-colorizer.js +++ b/scripts/postcss/svg-colorizer.js @@ -37,7 +37,7 @@ module.exports.buildColorizedSVG = function (svgLocation, primaryColor, secondar if (svgCode === coloredSVGCode) { throw new Error("svg-colorizer made no color replacements! The input svg should only contain colors #ff00ff (primary, case-sensitive) and #00ffff (secondary, case-sensitive)."); } - const fileName = svgLocation.match(/.+\/(.+\.svg)/)[1]; + const fileName = svgLocation.match(/.+[/\\](.+\.svg)/)[1]; const outputName = `${fileName.substring(0, fileName.length - 4)}-${createHash(coloredSVGCode)}.svg`; const outputPath = path.resolve(__dirname, "../../.tmp"); try { diff --git a/scripts/sdk/build.sh b/scripts/sdk/build.sh index eb916a5d..0e7ee70a 100755 --- a/scripts/sdk/build.sh +++ b/scripts/sdk/build.sh @@ -1,4 +1,8 @@ #!/bin/bash +# Exit whenever one of the commands fail with a non-zero exit code +set -e +set -o pipefail + rm -rf target yarn run vite build -c vite.sdk-assets-config.js yarn run vite build -c vite.sdk-lib-config.js diff --git a/src/domain/AvatarSource.ts b/src/domain/AvatarSource.ts new file mode 100644 index 00000000..8673b6e6 --- /dev/null +++ b/src/domain/AvatarSource.ts @@ -0,0 +1,23 @@ +/* +Copyright 2020 Bruno Windels +Copyright 2020, 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +export interface AvatarSource { + get avatarLetter(): string; + get avatarColorNumber(): number; + avatarUrl(size: number): string | undefined; + get avatarTitle(): string; +} diff --git a/src/domain/SessionPickerViewModel.js b/src/domain/SessionPickerViewModel.js index e486c64f..f65fd2e6 100644 --- a/src/domain/SessionPickerViewModel.js +++ b/src/domain/SessionPickerViewModel.js @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {SortedArray} from "../observable/index.js"; +import {SortedArray} from "../observable/index"; import {ViewModel} from "./ViewModel"; import {avatarInitials, getIdentifierColorNumber} from "./avatar"; diff --git a/src/domain/avatar.ts b/src/domain/avatar.ts index 6f1ef8b0..ecaccebe 100644 --- a/src/domain/avatar.ts +++ b/src/domain/avatar.ts @@ -51,10 +51,10 @@ export function getIdentifierColorNumber(id: string): number { return (hashCode(id) % 8) + 1; } -export function getAvatarHttpUrl(avatarUrl: string, cssSize: number, platform: Platform, mediaRepository: MediaRepository): string | null { +export function getAvatarHttpUrl(avatarUrl: string | undefined, cssSize: number, platform: Platform, mediaRepository: MediaRepository): string | undefined { if (avatarUrl) { const imageSize = cssSize * platform.devicePixelRatio; return mediaRepository.mxcUrlThumbnail(avatarUrl, imageSize, imageSize, "crop"); } - return null; + return undefined; } diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index bc7b54ff..3db1feb8 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -14,23 +14,34 @@ See the License for the specific language governing permissions and limitations under the License. */ +import {AvatarSource} from "../../AvatarSource"; import {ViewModel, Options as BaseOptions} from "../../ViewModel"; +import {getStreamVideoTrack, getStreamAudioTrack} from "../../../matrix/calls/common"; +import {avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../avatar"; +import {EventObservableValue} from "../../../observable/value/EventObservableValue"; +import {ObservableValueMap} from "../../../observable/map/ObservableValueMap"; import type {GroupCall} from "../../../matrix/calls/group/GroupCall"; import type {Member} from "../../../matrix/calls/group/Member"; import type {BaseObservableList} from "../../../observable/list/BaseObservableList"; import type {Stream} from "../../../platform/types/MediaDevices"; +import type {MediaRepository} from "../../../matrix/net/MediaRepository"; -type Options = BaseOptions & {call: GroupCall}; +type Options = BaseOptions & { + call: GroupCall, + mediaRepository: MediaRepository +}; export class CallViewModel extends ViewModel { - - public readonly memberViewModels: BaseObservableList; + public readonly memberViewModels: BaseObservableList; constructor(options: Options) { super(options); - this.memberViewModels = this.getOption("call").members + const ownMemberViewModelMap = new ObservableValueMap("self", new EventObservableValue(this.call, "change")) + .mapValues(call => new OwnMemberViewModel(this.childOptions({call: this.call, mediaRepository: this.getOption("mediaRepository")})), () => {}); + this.memberViewModels = this.call.members .filterValues(member => member.isConnected) - .mapValues(member => new CallMemberViewModel(this.childOptions({member}))) + .mapValues(member => new CallMemberViewModel(this.childOptions({member, mediaRepository: this.getOption("mediaRepository")}))) + .join(ownMemberViewModelMap) .sortValues((a, b) => a.compare(b)); } @@ -46,7 +57,7 @@ export class CallViewModel extends ViewModel { return this.call.id; } - get localStream(): Stream | undefined { + get stream(): Stream | undefined { return this.call.localMedia?.userMedia; } @@ -55,11 +66,66 @@ export class CallViewModel extends ViewModel { this.call.leave(); } } + + get isCameraMuted(): boolean { + return this.call.muteSettings.camera; + } + + get isMicrophoneMuted(): boolean { + return this.call.muteSettings.microphone; + } + + async toggleVideo() { + this.call.setMuted(this.call.muteSettings.toggleCamera()); + } } -type MemberOptions = BaseOptions & {member: Member}; +type OwnMemberOptions = BaseOptions & { + call: GroupCall, + mediaRepository: MediaRepository +} -export class CallMemberViewModel extends ViewModel { +class OwnMemberViewModel extends ViewModel implements IStreamViewModel { + get stream(): Stream | undefined { + return this.call.localMedia?.userMedia; + } + + private get call(): GroupCall { + return this.getOption("call"); + } + + get isCameraMuted(): boolean { + return this.call.muteSettings.camera ?? !!getStreamVideoTrack(this.stream); + } + + get isMicrophoneMuted(): boolean { + return this.call.muteSettings.microphone ?? !!getStreamAudioTrack(this.stream); + } + + get avatarLetter(): string { + return "I"; + } + + get avatarColorNumber(): number { + return 3; + } + + avatarUrl(size: number): string | undefined { + return undefined; + } + + get avatarTitle(): string { + return "Me"; + } + + compare(other: OwnMemberViewModel | CallMemberViewModel): number { + return -1; + } +} + +type MemberOptions = BaseOptions & {member: Member, mediaRepository: MediaRepository}; + +export class CallMemberViewModel extends ViewModel implements IStreamViewModel { get stream(): Stream | undefined { return this.member.remoteMedia?.userMedia; } @@ -68,7 +134,36 @@ export class CallMemberViewModel extends ViewModel { return this.getOption("member"); } - compare(other: CallMemberViewModel): number { + get isCameraMuted(): boolean { + return this.member.remoteMuteSettings?.camera ?? !getStreamVideoTrack(this.stream); + } + + get isMicrophoneMuted(): boolean { + return this.member.remoteMuteSettings?.microphone ?? !getStreamAudioTrack(this.stream); + } + + get avatarLetter(): string { + return avatarInitials(this.member.member.name); + } + + get avatarColorNumber(): number { + return getIdentifierColorNumber(this.member.userId); + } + + avatarUrl(size: number): string | undefined { + const {avatarUrl} = this.member.member; + const mediaRepository = this.getOption("mediaRepository"); + return getAvatarHttpUrl(avatarUrl, size, this.platform, mediaRepository); + } + + get avatarTitle(): string { + return this.member.member.name; + } + + compare(other: OwnMemberViewModel | CallMemberViewModel): number { + if (other instanceof OwnMemberViewModel) { + return -other.compare(this); + } const myUserId = this.member.member.userId; const otherUserId = other.member.member.userId; if(myUserId === otherUserId) { @@ -77,3 +172,9 @@ export class CallMemberViewModel extends ViewModel { return myUserId < otherUserId ? -1 : 1; } } + +export interface IStreamViewModel extends AvatarSource, ViewModel { + get stream(): Stream | undefined; + get isCameraMuted(): boolean; + get isMicrophoneMuted(): boolean; +} diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 868ca189..c60ce649 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -62,7 +62,7 @@ export class RoomViewModel extends ViewModel { } this._callViewModel = this.disposeTracked(this._callViewModel); if (call) { - this._callViewModel = this.track(new CallViewModel(this.childOptions({call}))); + this._callViewModel = this.track(new CallViewModel(this.childOptions({call, mediaRepository: this._room.mediaRepository}))); } this.emitChange("callViewModel"); })); @@ -367,9 +367,8 @@ export class RoomViewModel extends ViewModel { const session = this.getOption("session"); const stream = await this.platform.mediaDevices.getMediaTracks(false, true); const localMedia = new LocalMedia().withUserMedia(stream); - await this._call.join(localMedia); // this will set the callViewModel above as a call will be added to callHandler.calls - const call = await session.callHandler.createCall(this._room.id, localMedia, "A call " + Math.round(this.platform.random() * 100)); + const call = await session.callHandler.createCall(this._room.id, "m.video", "A call " + Math.round(this.platform.random() * 100)); await call.join(localMedia); } catch (err) { console.error(err.stack); diff --git a/src/matrix/Session.js b/src/matrix/Session.js index bd361434..9d63d335 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -21,7 +21,7 @@ import {RoomStatus} from "./room/common"; import {RoomBeingCreated} from "./room/RoomBeingCreated"; import {Invite} from "./room/Invite.js"; import {Pusher} from "./push/Pusher"; -import { ObservableMap } from "../observable/index.js"; +import { ObservableMap } from "../observable/index"; import {User} from "./User.js"; import {DeviceMessageHandler} from "./DeviceMessageHandler.js"; import {Account as E2EEAccount} from "./e2ee/Account.js"; diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index c33f24eb..d1634320 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -15,8 +15,8 @@ limitations under the License. */ import {ObservableMap} from "../../observable/map/ObservableMap"; -import {WebRTC, PeerConnection, PeerConnectionHandler} from "../../platform/types/WebRTC"; -import {MediaDevices, Track, AudioTrack} from "../../platform/types/MediaDevices"; +import {WebRTC, PeerConnection} from "../../platform/types/WebRTC"; +import {MediaDevices, Track} from "../../platform/types/MediaDevices"; import {handlesEventType} from "./PeerCall"; import {EventType, CallIntent} from "./callEventTypes"; import {GroupCall} from "./group/GroupCall"; @@ -107,7 +107,7 @@ export class CallHandler { }); } - async createCall(roomId: string, callType: "m.video" | "m.voice", name: string, intent: CallIntent = CallIntent.Ring): Promise { + async createCall(roomId: string, type: "m.video" | "m.voice", name: string, intent: CallIntent = CallIntent.Ring): Promise { const logItem = this.options.logger.child({l: "call", incoming: false}); const call = new GroupCall(makeId("conf-"), true, { "m.name": name, @@ -116,7 +116,7 @@ export class CallHandler { this._calls.set(call.id, call); try { - await call.create(callType); + await call.create(type); // store call info so it will ring again when reopening the app const txn = await this.options.storage.readWriteTxn([this.options.storage.storeNames.calls]); txn.calls.add({ diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 6564adac..b79fe098 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -17,6 +17,7 @@ limitations under the License. import {SDPStreamMetadataPurpose} from "./callEventTypes"; import {Stream} from "../../platform/types/MediaDevices"; import {SDPStreamMetadata} from "./callEventTypes"; +import {getStreamVideoTrack, getStreamAudioTrack} from "./common"; export class LocalMedia { constructor( @@ -37,13 +38,44 @@ export class LocalMedia { return new LocalMedia(this.userMedia, this.screenShare, options); } + /** @internal */ + replaceClone(oldClone: LocalMedia | undefined, oldOriginal: LocalMedia | undefined): LocalMedia { + let userMedia; + let screenShare; + const cloneOrAdoptStream = (oldOriginalStream: Stream | undefined, oldCloneStream: Stream | undefined, newStream: Stream | undefined): Stream | undefined => { + let stream; + if (oldOriginalStream?.id === newStream?.id) { + stream = oldCloneStream; + } else { + stream = newStream?.clone(); + getStreamAudioTrack(oldCloneStream)?.stop(); + getStreamVideoTrack(oldCloneStream)?.stop(); + } + return stream; + } + return new LocalMedia( + cloneOrAdoptStream(oldOriginal?.userMedia, oldClone?.userMedia, this.userMedia), + cloneOrAdoptStream(oldOriginal?.screenShare, oldClone?.screenShare, this.screenShare), + this.dataChannelOptions + ); + } + + /** @internal */ clone(): LocalMedia { - return new LocalMedia(this.userMedia?.clone(), this.screenShare?.clone(), this.dataChannelOptions); + return new LocalMedia(this.userMedia?.clone(),this.screenShare?.clone(), this.dataChannelOptions); } dispose() { - this.userMedia?.audioTrack?.stop(); - this.userMedia?.videoTrack?.stop(); - this.screenShare?.videoTrack?.stop(); + this.stopExcept(undefined); + } + + stopExcept(newMedia: LocalMedia | undefined) { + if(newMedia?.userMedia?.id !== this.userMedia?.id) { + getStreamAudioTrack(this.userMedia)?.stop(); + getStreamVideoTrack(this.userMedia)?.stop(); + } + if(newMedia?.screenShare?.id !== this.screenShare?.id) { + getStreamVideoTrack(this.screenShare)?.stop(); + } } } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 5b6e2ca8..6aff72c1 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -16,25 +16,26 @@ limitations under the License. import {ObservableMap} from "../../observable/map/ObservableMap"; import {recursivelyAssign} from "../../utils/recursivelyAssign"; -import {AsyncQueue} from "../../utils/AsyncQueue"; -import {Disposables, IDisposable} from "../../utils/Disposables"; -import type {Room} from "../room/Room"; -import type {StateEvent} from "../storage/types"; -import type {ILogItem} from "../../logging/types"; - -import type {TimeoutCreator, Timeout} from "../../platform/types/types"; -import {WebRTC, PeerConnection, PeerConnectionHandler, TrackSender, TrackReceiver} from "../../platform/types/WebRTC"; -import {MediaDevices, Track, AudioTrack, Stream} from "../../platform/types/MediaDevices"; -import type {LocalMedia} from "./LocalMedia"; - +import {Disposables, Disposable, IDisposable} from "../../utils/Disposables"; +import {WebRTC, PeerConnection, Transceiver, TransceiverDirection, Sender, Receiver, PeerConnectionEventMap} from "../../platform/types/WebRTC"; +import {MediaDevices, Track, TrackKind, Stream, StreamTrackEvent} from "../../platform/types/MediaDevices"; +import {getStreamVideoTrack, getStreamAudioTrack, MuteSettings} from "./common"; import { SDPStreamMetadataKey, SDPStreamMetadataPurpose, EventType, + CallErrorCode, } from "./callEventTypes"; + +import type {Room} from "../room/Room"; +import type {StateEvent} from "../storage/types"; +import type {ILogItem} from "../../logging/types"; +import type {TimeoutCreator, Timeout} from "../../platform/types/types"; +import type {LocalMedia} from "./LocalMedia"; import type { MCallBase, MCallInvite, + MCallNegotiate, MCallAnswer, MCallSDPStreamMetadataChanged, MCallCandidates, @@ -66,7 +67,9 @@ export class PeerCall implements IDisposable { private readonly peerConnection: PeerConnection; private _state = CallState.Fledgling; private direction: CallDirection; + // we don't own localMedia and should hence not call dispose on it from here private localMedia?: LocalMedia; + private localMuteSettings?: MuteSettings; private seq: number = 0; // A queue for candidates waiting to go out. // We try to amalgamate candidates into a single candidate message where @@ -83,7 +86,8 @@ export class PeerCall implements IDisposable { private hangupParty: CallParty; private disposables = new Disposables(); private statePromiseMap = new Map void, promise: Promise}>(); - + private _remoteTrackToStreamId = new Map(); + private _remoteStreams = new Map(); // perfect negotiation flags private makingOffer: boolean = false; private ignoreOffer: boolean = false; @@ -94,55 +98,62 @@ export class PeerCall implements IDisposable { private _dataChannel?: any; private _hangupReason?: CallErrorCode; private _remoteMedia: RemoteMedia; + private _remoteMuteSettings = new MuteSettings(); constructor( private callId: string, private readonly options: Options, private readonly logItem: ILogItem, ) { - const outer = this; this._remoteMedia = new RemoteMedia(); - this.peerConnection = options.webRTC.createPeerConnection({ - onIceConnectionStateChange(state: RTCIceConnectionState) { - outer.logItem.wrap({l: "onIceConnectionStateChange", status: state}, log => { - outer.onIceConnectionStateChange(state, log); + this.peerConnection = options.webRTC.createPeerConnection(this.options.forceTURN, this.options.turnServers, 0); + + const listen = (type: K, listener: (this: PeerConnection, ev: PeerConnectionEventMap[K]) => any, options?: boolean | EventListenerOptions): void => { + this.peerConnection.addEventListener(type, listener); + const dispose = () => { + this.peerConnection.removeEventListener(type, listener); + }; + this.disposables.track(dispose); + }; + + listen("iceconnectionstatechange", () => { + const state = this.peerConnection.iceConnectionState; + this.logItem.wrap({l: "onIceConnectionStateChange", status: state}, log => { + this.onIceConnectionStateChange(state, log); + }); + }); + listen("icecandidate", event => { + this.logItem.wrap("onLocalIceCandidate", log => { + if (event.candidate) { + this.handleLocalIceCandidate(event.candidate, log); + } + }); + }); + listen("icegatheringstatechange", () => { + const state = this.peerConnection.iceGatheringState; + this.logItem.wrap({l: "onIceGatheringStateChange", status: state}, log => { + this.handleIceGatheringState(state, log); + }); + }); + listen("track", event => { + this.logItem.wrap("onRemoteTrack", log => { + this.onRemoteTrack(event.track, event.streams, log); + }); + }); + listen("datachannel", event => { + this.logItem.wrap("onRemoteDataChannel", log => { + this._dataChannel = event.channel; + this.options.emitUpdate(this, undefined); + }); + }); + listen("negotiationneeded", () => { + const promiseCreator = () => { + return this.logItem.wrap("onNegotiationNeeded", log => { + return this.handleNegotiation(log); }); - }, - onLocalIceCandidate(candidate: RTCIceCandidate) { - outer.logItem.wrap("onLocalIceCandidate", log => { - outer.handleLocalIceCandidate(candidate, log); - }); - }, - onIceGatheringStateChange(state: RTCIceGatheringState) { - outer.logItem.wrap({l: "onIceGatheringStateChange", status: state}, log => { - outer.handleIceGatheringState(state, log); - }); - }, - onRemoteStreamRemoved(stream: Stream) { - outer.logItem.wrap("onRemoteStreamRemoved", log => { - outer.updateRemoteMedia(log); - }); - }, - onRemoteTracksAdded(trackReceiver: TrackReceiver) { - outer.logItem.wrap("onRemoteTracksAdded", log => { - outer.updateRemoteMedia(log); - }); - }, - onRemoteDataChannel(dataChannel: any | undefined) { - outer.logItem.wrap("onRemoteDataChannel", log => { - outer._dataChannel = dataChannel; - outer.options.emitUpdate(outer, undefined); - }); - }, - onNegotiationNeeded() { - const promiseCreator = () => { - return outer.logItem.wrap("onNegotiationNeeded", log => { - return outer.handleNegotiation(log); - }); - }; - outer.responsePromiseChain = outer.responsePromiseChain?.then(promiseCreator) ?? promiseCreator(); - } - }, this.options.forceTURN, this.options.turnServers, 0); + }; + this.responsePromiseChain = this.responsePromiseChain?.then(promiseCreator) ?? promiseCreator(); + }); } get dataChannel(): any | undefined { return this._dataChannel; } @@ -151,21 +162,25 @@ export class PeerCall implements IDisposable { get hangupReason(): CallErrorCode | undefined { return this._hangupReason; } - // we should keep an object with streams by purpose ... e.g. RemoteMedia? get remoteMedia(): Readonly { return this._remoteMedia; } - call(localMedia: LocalMedia): Promise { + get remoteMuteSettings(): MuteSettings { + return this._remoteMuteSettings; + } + + call(localMedia: LocalMedia, localMuteSettings: MuteSettings): Promise { return this.logItem.wrap("call", async log => { if (this._state !== CallState.Fledgling) { return; } this.direction = CallDirection.Outbound; this.setState(CallState.CreateOffer, log); - this.setMedia(localMedia); + this.localMuteSettings = localMuteSettings; + await this.updateLocalMedia(localMedia, log); if (this.localMedia?.dataChannelOptions) { - this._dataChannel = this.peerConnection.createDataChannel(this.localMedia.dataChannelOptions); + this._dataChannel = this.peerConnection.createDataChannel("channel", this.localMedia.dataChannelOptions); } // after adding the local tracks, and wait for handleNegotiation to be called, // or invite glare where we give up our invite and answer instead @@ -173,13 +188,14 @@ export class PeerCall implements IDisposable { }); } - answer(localMedia: LocalMedia): Promise { + answer(localMedia: LocalMedia, localMuteSettings: MuteSettings): Promise { return this.logItem.wrap("answer", async log => { if (this._state !== CallState.Ringing) { return; } this.setState(CallState.CreateAnswer, log); - this.setMedia(localMedia, log); + this.localMuteSettings = localMuteSettings; + await this.updateLocalMedia(localMedia, log); let myAnswer: RTCSessionDescriptionInit; try { myAnswer = await this.peerConnection.createAnswer(); @@ -208,46 +224,47 @@ export class PeerCall implements IDisposable { }); } - setMedia(localMedia: LocalMedia, logItem: ILogItem = this.logItem): Promise { - return logItem.wrap("setMedia", async log => { - const oldMedia = this.localMedia; - this.localMedia = localMedia; - const applyStream = (oldStream: Stream | undefined, stream: Stream | undefined, logLabel: string) => { - const streamSender = oldMedia ? this.peerConnection.localStreams.get(oldStream!.id) : undefined; - - const applyTrack = (oldTrack: Track | undefined, sender: TrackSender | undefined, track: Track | undefined) => { - if (track) { - if (oldTrack && sender) { - log.wrap(`replacing ${logLabel} ${track.kind} track`, log => { - sender.replaceTrack(track); - }); - } else { - log.wrap(`adding ${logLabel} ${track.kind} track`, log => { - this.peerConnection.addTrack(track); - }); - } - } else { - if (sender) { - log.wrap(`replacing ${logLabel} ${sender.track.kind} track`, log => { - this.peerConnection.removeTrack(sender); - }); - } - } - } - - applyTrack(oldStream?.audioTrack, streamSender?.audioSender, stream?.audioTrack); - applyTrack(oldStream?.videoTrack, streamSender?.videoSender, stream?.videoTrack); - } - - applyStream(oldMedia?.userMedia, localMedia?.userMedia, "userMedia"); - applyStream(oldMedia?.screenShare, localMedia?.screenShare, "screenShare"); - // TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method + setMedia(localMedia: LocalMedia): Promise { + return this.logItem.wrap("setMedia", async log => { + log.set("userMedia_audio", !!getStreamAudioTrack(localMedia.userMedia)); + log.set("userMedia_video", !!getStreamVideoTrack(localMedia.userMedia)); + log.set("screenShare_video", !!getStreamVideoTrack(localMedia.screenShare)); + log.set("datachannel", !!localMedia.dataChannelOptions); + await this.updateLocalMedia(localMedia, log); + const content: MCallSDPStreamMetadataChanged = { + call_id: this.callId, + version: 1, + seq: this.seq++, + [SDPStreamMetadataKey]: this.getSDPMetadata() + }; + await this.sendSignallingMessage({type: EventType.SDPStreamMetadataChangedPrefix, content}, log); }); } - /** group calls would handle reject at the group call level, not at the peer call level */ - async reject() { + setMuted(localMuteSettings: MuteSettings) { + return this.logItem.wrap("setMuted", async log => { + this.localMuteSettings = localMuteSettings; + log.set("cameraMuted", localMuteSettings.camera); + log.set("microphoneMuted", localMuteSettings.microphone); + if (this.localMedia) { + const userMediaAudio = getStreamAudioTrack(this.localMedia.userMedia); + if (userMediaAudio) { + this.muteTrack(userMediaAudio, this.localMuteSettings.microphone, log); + } + const userMediaVideo = getStreamVideoTrack(this.localMedia.userMedia); + if (userMediaVideo) { + this.muteTrack(userMediaVideo, this.localMuteSettings.camera, log); + } + const content: MCallSDPStreamMetadataChanged = { + call_id: this.callId, + version: 1, + seq: this.seq++, + [SDPStreamMetadataKey]: this.getSDPMetadata() + }; + await this.sendSignallingMessage({type: EventType.SDPStreamMetadataChangedPrefix, content}, log); + } + }); } hangup(errorCode: CallErrorCode): Promise { @@ -256,6 +273,22 @@ export class PeerCall implements IDisposable { }); } + private muteTrack(track: Track, muted: boolean, log: ILogItem): void { + log.wrap({l: "track", kind: track.kind, id: track.id}, log => { + const enabled = !muted; + log.set("enabled", enabled); + const transceiver = this.findTransceiverForTrack(track); + if (transceiver) { + if (transceiver.sender.track) { + transceiver.sender.track.enabled = enabled; + } + log.set("fromDirection", transceiver.direction); + // enableSenderOnTransceiver(transceiver, enabled); + log.set("toDirection", transceiver.direction); + } + }); + } + private async _hangup(errorCode: CallErrorCode, log: ILogItem): Promise { if (this._state === CallState.Ended) { return; @@ -277,10 +310,20 @@ export class PeerCall implements IDisposable { case EventType.Answer: await this.handleAnswer(message.content, partyId, log); break; + case EventType.Negotiate: + await this.onNegotiateReceived(message.content, log); + break; case EventType.Candidates: await this.handleRemoteIceCandidates(message.content, partyId, log); break; + case EventType.SDPStreamMetadataChanged: + case EventType.SDPStreamMetadataChangedPrefix: + this.updateRemoteSDPStreamMetadata(message.content[SDPStreamMetadataKey], log); + break; case EventType.Hangup: + // TODO: this is a bit hacky, double check its what we need + this.terminate(CallParty.Remote, message.content.reason ?? CallErrorCode.UserHangup, log); + break; default: log.log(`Unknown event type for call: ${message.type}`); break; @@ -305,7 +348,7 @@ export class PeerCall implements IDisposable { } // calls are serialized and deduplicated by responsePromiseChain - private handleNegotiation = async (log: ILogItem): Promise => { + private async handleNegotiation(log: ILogItem): Promise { this.makingOffer = true; try { try { @@ -334,22 +377,27 @@ export class PeerCall implements IDisposable { this.candidateSendQueue = []; // need to queue this - const content = { - call_id: this.callId, - offer, - [SDPStreamMetadataKey]: this.getSDPMetadata(), - version: 1, - seq: this.seq++, - lifetime: CALL_TIMEOUT_MS - }; if (this._state === CallState.CreateOffer) { + const content = { + call_id: this.callId, + offer, + [SDPStreamMetadataKey]: this.getSDPMetadata(), + version: 1, + seq: this.seq++, + lifetime: CALL_TIMEOUT_MS + }; await this.sendSignallingMessage({type: EventType.Invite, content}, log); this.setState(CallState.InviteSent, log); } else if (this._state === CallState.Connected || this._state === CallState.Connecting) { - log.log("would send renegotiation now but not implemented"); - // send Negotiate message - //await this.sendSignallingMessage({type: EventType.Invite, content}); - //this.setState(CallState.InviteSent); + const content = { + call_id: this.callId, + description: offer, + [SDPStreamMetadataKey]: this.getSDPMetadata(), + version: 1, + seq: this.seq++, + lifetime: CALL_TIMEOUT_MS + }; + await this.sendSignallingMessage({type: EventType.Negotiate, content}, log); } } finally { this.makingOffer = false; @@ -385,7 +433,7 @@ export class PeerCall implements IDisposable { } await this.handleInvite(content, partyId, log); // TODO: need to skip state check - await this.answer(this.localMedia!); + await this.answer(this.localMedia!, this.localMuteSettings!); } else { log.log( "Glare detected: rejecting incoming call " + newCallId + @@ -444,12 +492,12 @@ export class PeerCall implements IDisposable { // According to previous comments in this file, firefox at some point did not // add streams until media started arriving on them. Testing latest firefox // (81 at time of writing), this is no longer a problem, so let's do it the correct way. - // if (this.peerConnection.remoteTracks.length === 0) { - // await log.wrap(`Call no remote stream or no tracks after setting remote description!`, async log => { - // return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); - // }); - // return; - // } + if (this.peerConnection.getReceivers().length === 0) { + await log.wrap(`Call no remote stream or no tracks after setting remote description!`, async log => { + return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); + }); + return; + } this.setState(CallState.Ringing, log); @@ -571,55 +619,55 @@ export class PeerCall implements IDisposable { await this.addIceCandidates(candidates, log); } - // private async onNegotiateReceived(event: MatrixEvent): Promise { - // const content = event.getContent(); - // const description = content.description; - // if (!description || !description.sdp || !description.type) { - // this.logger.info(`Ignoring invalid m.call.negotiate event`); - // return; - // } - // // Politeness always follows the direction of the call: in a glare situation, - // // we pick either the inbound or outbound call, so one side will always be - // // inbound and one outbound - // const polite = this.direction === CallDirection.Inbound; + private async onNegotiateReceived(content: MCallNegotiate, log: ILogItem): Promise { + const description = content.description; + if (!description || !description.sdp || !description.type) { + log.log(`Ignoring invalid m.call.negotiate event`); + return; + } + // Politeness always follows the direction of the call: in a glare situation, + // we pick either the inbound or outbound call, so one side will always be + // inbound and one outbound + const polite = this.direction === CallDirection.Inbound; - // // Here we follow the perfect negotiation logic from - // // https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation - // const offerCollision = ( - // (description.type === 'offer') && - // (this.makingOffer || this.peerConnection.signalingState !== 'stable') - // ); + // Here we follow the perfect negotiation logic from + // https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation + const offerCollision = ( + (description.type === 'offer') && + (this.makingOffer || this.peerConnection.signalingState !== 'stable') + ); - // this.ignoreOffer = !polite && offerCollision; - // if (this.ignoreOffer) { - // this.logger.info(`Ignoring colliding negotiate event because we're impolite`); - // return; - // } + this.ignoreOffer = !polite && offerCollision; + if (this.ignoreOffer) { + log.log(`Ignoring colliding negotiate event because we're impolite`); + return; + } - // const sdpStreamMetadata = content[SDPStreamMetadataKey]; - // if (sdpStreamMetadata) { - // this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); - // } else { - // this.logger.warn(`Received negotiation event without SDPStreamMetadata!`); - // } + const sdpStreamMetadata = content[SDPStreamMetadataKey]; + if (sdpStreamMetadata) { + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata, log); + } else { + log.log(`Received negotiation event without SDPStreamMetadata!`); + } - // try { - // await this.peerConnection.setRemoteDescription(description); - - // if (description.type === 'offer') { - // await this.peerConnection.setLocalDescription(); - // await this.sendSignallingMessage({ - // type: EventType.CallNegotiate, - // content: { - // description: this.peerConnection.localDescription!, - // [SDPStreamMetadataKey]: this.getSDPMetadata(), - // } - // }); - // } - // } catch (err) { - // this.logger.warn(`Failed to complete negotiation`, err); - // } - // } + try { + await this.peerConnection.setRemoteDescription(description); + if (description.type === 'offer') { + await this.peerConnection.setLocalDescription(); + const content = { + call_id: this.callId, + description: this.peerConnection.localDescription!, + [SDPStreamMetadataKey]: this.getSDPMetadata(), + version: 1, + seq: this.seq++, + lifetime: CALL_TIMEOUT_MS + }; + await this.sendSignallingMessage({type: EventType.Negotiate, content}, log); + } + } catch (err) { + log.log(`Failed to complete negotiation`, err); + } + } private async sendAnswer(log: ILogItem): Promise { const localDescription = this.peerConnection.localDescription!; @@ -719,7 +767,7 @@ export class PeerCall implements IDisposable { // this will accumulate all updates into one object, so we still have the old stream info when we change stream id this.remoteSDPStreamMetadata = recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); this.updateRemoteMedia(log); - // TODO: apply muting + } private async addBufferedIceCandidates(log: ILogItem): Promise { @@ -825,13 +873,11 @@ export class PeerCall implements IDisposable { const metadata = {}; if (this.localMedia?.userMedia) { const streamId = this.localMedia.userMedia.id; - const streamSender = this.peerConnection.localStreams.get(streamId); metadata[streamId] = { purpose: SDPStreamMetadataPurpose.Usermedia, - audio_muted: !(streamSender?.audioSender?.enabled), - video_muted: !(streamSender?.videoSender?.enabled), + audio_muted: this.localMuteSettings?.microphone || !getStreamAudioTrack(this.localMedia.userMedia), + video_muted: this.localMuteSettings?.camera || !getStreamVideoTrack(this.localMedia.userMedia), }; - console.log("video_muted", streamSender?.videoSender?.enabled, streamSender?.videoSender?.transceiver?.direction, streamSender?.videoSender?.transceiver?.currentDirection, JSON.stringify(metadata)); } if (this.localMedia?.screenShare) { const streamId = this.localMedia.screenShare.id; @@ -842,17 +888,78 @@ export class PeerCall implements IDisposable { return metadata; } - private updateRemoteMedia(log: ILogItem) { + private findReceiverForStream(kind: TrackKind, streamId: string): Receiver | undefined { + return this.peerConnection.getReceivers().find(r => { + return r.track.kind === kind && this._remoteTrackToStreamId.get(r.track.id) === streamId; + }); + } + + private findTransceiverForTrack(track: Track): Transceiver | undefined { + return this.peerConnection.getTransceivers().find(t => { + return t.sender.track?.id === track.id; + }); + } + + + private onRemoteTrack(track: Track, streams: ReadonlyArray, log: ILogItem) { + if (streams.length === 0) { + log.log({l: `ignoring ${track.kind} streamless track`, id: track.id}); + return; + } + const stream = streams[0]; + this._remoteTrackToStreamId.set(track.id, stream.id); + if (!this._remoteStreams.has(stream.id)) { + const listener = (event: StreamTrackEvent): void => { + this.logItem.wrap({l: "removetrack", id: event.track.id}, log => { + const streamId = this._remoteTrackToStreamId.get(event.track.id); + if (streamId) { + this._remoteTrackToStreamId.delete(event.track.id); + const streamDetails = this._remoteStreams.get(streamId); + if (streamDetails && streamDetails.stream.getTracks().length === 0) { + this.disposables.disposeTracked(disposeListener); + this._remoteStreams.delete(stream.id); + this.updateRemoteMedia(log); + } + } + }) + }; + stream.addEventListener("removetrack", listener); + const disposeListener = () => { + stream.removeEventListener("removetrack", listener); + }; + this.disposables.track(disposeListener); + this._remoteStreams.set(stream.id, { + disposeListener, + stream + }); + this.updateRemoteMedia(log); + } + } + + private updateRemoteMedia(log: ILogItem): void { this._remoteMedia.userMedia = undefined; this._remoteMedia.screenShare = undefined; if (this.remoteSDPStreamMetadata) { - for (const [streamId, streamReceiver] of this.peerConnection.remoteStreams.entries()) { - const metaData = this.remoteSDPStreamMetadata[streamId]; + for (const streamDetails of this._remoteStreams.values()) { + const {stream} = streamDetails; + const metaData = this.remoteSDPStreamMetadata[stream.id]; if (metaData) { if (metaData.purpose === SDPStreamMetadataPurpose.Usermedia) { - this._remoteMedia.userMedia = streamReceiver.stream; + this._remoteMedia.userMedia = stream; + const audioReceiver = this.findReceiverForStream(TrackKind.Audio, stream.id); + if (audioReceiver) { + audioReceiver.track.enabled = !metaData.audio_muted; + } + const videoReceiver = this.findReceiverForStream(TrackKind.Video, stream.id); + if (videoReceiver) { + videoReceiver.track.enabled = !metaData.video_muted; + } + this._remoteMuteSettings = new MuteSettings( + metaData.audio_muted || !audioReceiver?.track, + metaData.video_muted || !videoReceiver?.track + ); } else if (metaData.purpose === SDPStreamMetadataPurpose.Screenshare) { - this._remoteMedia.screenShare = streamReceiver.stream; + this._remoteMedia.screenShare = stream; } } } @@ -860,6 +967,55 @@ export class PeerCall implements IDisposable { this.options.emitUpdate(this, undefined); } + private updateLocalMedia(localMedia: LocalMedia, logItem: ILogItem): Promise { + return logItem.wrap("updateLocalMedia", async log => { + const oldMedia = this.localMedia; + this.localMedia = localMedia; + const applyStream = async (oldStream: Stream | undefined, stream: Stream | undefined, streamPurpose: SDPStreamMetadataPurpose) => { + const applyTrack = async (oldTrack: Track | undefined, newTrack: Track | undefined) => { + if (!oldTrack && newTrack) { + log.wrap(`adding ${streamPurpose} ${newTrack.kind} track`, log => { + const sender = this.peerConnection.addTrack(newTrack, stream!); + this.options.webRTC.prepareSenderForPurpose(this.peerConnection, sender, streamPurpose); + }); + } else if (oldTrack) { + const sender = this.peerConnection.getSenders().find(s => s.track && s.track.id === oldTrack.id); + if (sender) { + if (newTrack && oldTrack.id !== newTrack.id) { + try { + await log.wrap(`replacing ${streamPurpose} ${newTrack.kind} track`, log => { + return sender.replaceTrack(newTrack); + }); + } catch (err) { + // can't replace the track without renegotiating{ + log.wrap(`adding and removing ${streamPurpose} ${newTrack.kind} track`, log => { + this.peerConnection.removeTrack(sender); + const newSender = this.peerConnection.addTrack(newTrack); + this.options.webRTC.prepareSenderForPurpose(this.peerConnection, newSender, streamPurpose); + }); + } + } else if (!newTrack) { + log.wrap(`removing ${streamPurpose} ${sender.track!.kind} track`, log => { + this.peerConnection.removeTrack(sender); + }); + } else { + log.log(`${streamPurpose} ${oldTrack.kind} track hasn't changed`); + } + } + // TODO: should we do something if we didn't find the sender? e.g. some other code already removed the sender but didn't update localMedia + } + } + + await applyTrack(getStreamAudioTrack(oldStream), getStreamAudioTrack(stream)); + await applyTrack(getStreamVideoTrack(oldStream), getStreamVideoTrack(stream)); + }; + + await applyStream(oldMedia?.userMedia, localMedia?.userMedia, SDPStreamMetadataPurpose.Usermedia); + await applyStream(oldMedia?.screenShare, localMedia?.screenShare, SDPStreamMetadataPurpose.Screenshare); + // TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method + }); + } + private async delay(timeoutMs: number): Promise { // Allow a short time for initial candidates to be gathered const timeout = this.disposables.track(this.options.createTimeout(timeoutMs)); @@ -875,7 +1031,7 @@ export class PeerCall implements IDisposable { public dispose(): void { this.disposables.dispose(); - this.peerConnection.dispose(); + this.peerConnection.close(); } public close(reason: CallErrorCode | undefined, log: ILogItem): void { @@ -915,100 +1071,11 @@ export enum CallDirection { Outbound = 'outbound', } -export enum CallErrorCode { - /** The user chose to end the call */ - UserHangup = 'user_hangup', - - /** An error code when the local client failed to create an offer. */ - LocalOfferFailed = 'local_offer_failed', - /** - * An error code when there is no local mic/camera to use. This may be because - * the hardware isn't plugged in, or the user has explicitly denied access. - */ - NoUserMedia = 'no_user_media', - - /** - * Error code used when a call event failed to send - * because unknown devices were present in the room - */ - UnknownDevices = 'unknown_devices', - - /** - * Error code used when we fail to send the invite - * for some reason other than there being unknown devices - */ - SendInvite = 'send_invite', - - /** - * An answer could not be created - */ - CreateAnswer = 'create_answer', - - /** - * Error code used when we fail to send the answer - * for some reason other than there being unknown devices - */ - SendAnswer = 'send_answer', - - /** - * The session description from the other side could not be set - */ - SetRemoteDescription = 'set_remote_description', - - /** - * The session description from this side could not be set - */ - SetLocalDescription = 'set_local_description', - - /** - * A different device answered the call - */ - AnsweredElsewhere = 'answered_elsewhere', - - /** - * No media connection could be established to the other party - */ - IceFailed = 'ice_failed', - - /** - * The invite timed out whilst waiting for an answer - */ - InviteTimeout = 'invite_timeout', - - /** - * The call was replaced by another call - */ - Replaced = 'replaced', - - /** - * Signalling for the call could not be sent (other than the initial invite) - */ - SignallingFailed = 'signalling_timeout', - - /** - * The remote party is busy - */ - UserBusy = 'user_busy', - - /** - * We transferred the call off to somewhere else - */ - Transfered = 'transferred', - - /** - * A call from the same user was found with a new session id - */ - NewSession = 'new_session', -} - /** * The version field that we set in m.call.* events */ const VOIP_PROTO_VERSION = 1; -/** The fallback ICE server to use for STUN or TURN protocols. */ -const FALLBACK_ICE_SERVER = 'stun:turn.matrix.org'; - /** The length of time a call can be ringing for. */ const CALL_TIMEOUT_MS = 60000; @@ -1029,9 +1096,28 @@ export function handlesEventType(eventType: string): boolean { return eventType === EventType.Invite || eventType === EventType.Candidates || eventType === EventType.Answer || - eventType === EventType.Hangup; + eventType === EventType.Hangup || + eventType === EventType.SDPStreamMetadataChanged || + eventType === EventType.SDPStreamMetadataChangedPrefix || + eventType === EventType.Negotiate; } -export function tests() { - +function enableSenderOnTransceiver(transceiver: Transceiver, enabled: boolean) { + return enableTransceiver(transceiver, enabled, "sendonly", "recvonly"); +} + +function enableTransceiver(transceiver: Transceiver, enabled: boolean, exclusiveValue: TransceiverDirection, excludedValue: TransceiverDirection) { + if (enabled) { + if (transceiver.direction === "inactive") { + transceiver.direction = exclusiveValue; + } else { + transceiver.direction = "sendrecv"; + } + } else { + if (transceiver.direction === "sendrecv") { + transceiver.direction = excludedValue; + } else { + transceiver.direction = "inactive"; + } + } } diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index 8c041fcc..a7f4e82c 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -9,14 +9,14 @@ ## TODO - DONE: implement receiving hangup - - making logging better + - DONE: implement cloning the localMedia so it works in safari? + - DONE: implement 3 retries per peer + - implement muting tracks with m.call.sdp_stream_metadata_changed - implement renegotiation + - making logging better - finish session id support - call peers are essentially identified by (userid, deviceid, sessionid). If see a new session id, we first disconnect from the current member so we're ready to connect with a clean slate again (in a member event, also in to_device? no harm I suppose, given olm encryption ensures you can't spoof the deviceid). - implement to_device messages arriving before m.call(.member) state event - - implement muting tracks with m.call.sdp_stream_metadata_changed - - implement cloning the localMedia so it works in safari? - - DONE: implement 3 retries per peer - reeable crypto & implement fetching olm keys before sending encrypted signalling message - local echo for join/leave buttons? - make UI pretsy diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index 4726175c..0490b44f 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -1,7 +1,7 @@ // allow non-camelcase as these are events type that go onto the wire /* eslint-disable camelcase */ import type {StateEvent} from "../storage/types"; - +import type {SessionDescription} from "../../platform/types/WebRTC"; export enum EventType { GroupCall = "org.matrix.msc3401.call", GroupCallMember = "org.matrix.msc3401.call.member", @@ -36,11 +36,6 @@ export interface CallMemberContent { ["m.calls"]: CallMembership[]; } -export interface SessionDescription { - sdp?: string; - type: RTCSdpType -} - export enum SDPStreamMetadataPurpose { Usermedia = "m.usermedia", Screenshare = "m.screenshare", @@ -97,6 +92,12 @@ export type MCallInvite = Base & { [SDPStreamMetadataKey]: SDPStreamMetadata; } +export type MCallNegotiate = Base & { + description: SessionDescription; + lifetime: number; + [SDPStreamMetadataKey]: SDPStreamMetadata; +} + export type MCallSDPStreamMetadataChanged = Base & { [SDPStreamMetadataKey]: SDPStreamMetadata; } @@ -213,6 +214,7 @@ export enum CallErrorCode { export type SignallingMessage = {type: EventType.Invite, content: MCallInvite} | + {type: EventType.Negotiate, content: MCallNegotiate} | {type: EventType.Answer, content: MCallAnswer} | {type: EventType.SDPStreamMetadataChanged | EventType.SDPStreamMetadataChangedPrefix, content: MCallSDPStreamMetadataChanged} | {type: EventType.Candidates, content: MCallCandidates} | diff --git a/src/matrix/calls/common.ts b/src/matrix/calls/common.ts new file mode 100644 index 00000000..6e3dc7ab --- /dev/null +++ b/src/matrix/calls/common.ts @@ -0,0 +1,37 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import type {Track, Stream} from "../../platform/types/MediaDevices"; + +export function getStreamAudioTrack(stream: Stream | undefined): Track | undefined { + return stream?.getAudioTracks()[0]; +} + +export function getStreamVideoTrack(stream: Stream | undefined): Track | undefined { + return stream?.getVideoTracks()[0]; +} + +export class MuteSettings { + constructor (public readonly microphone: boolean = false, public readonly camera: boolean = false) {} + + toggleCamera(): MuteSettings { + return new MuteSettings(this.microphone, !this.camera); + } + + toggleMicrophone(): MuteSettings { + return new MuteSettings(!this.microphone, this.camera); + } +} diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index dc48a692..9b5df521 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -17,6 +17,7 @@ limitations under the License. import {ObservableMap} from "../../../observable/map/ObservableMap"; import {Member} from "./Member"; import {LocalMedia} from "../LocalMedia"; +import {MuteSettings} from "../common"; import {RoomMember} from "../../room/members/RoomMember"; import {EventEmitter} from "../../../utils/EventEmitter"; import {EventType, CallIntent} from "../callEventTypes"; @@ -63,6 +64,7 @@ export class GroupCall extends EventEmitter<{change: never}> { private _localMedia?: LocalMedia = undefined; private _memberOptions: MemberOptions; private _state: GroupCallState; + private localMuteSettings: MuteSettings = new MuteSettings(false, false); private _deviceIndex?: number; private _eventTimestamp?: number; @@ -129,11 +131,36 @@ export class GroupCall extends EventEmitter<{change: never}> { this.emitChange(); // send invite to all members that are < my userId for (const [,member] of this._members) { - member.connect(this._localMedia!.clone()); + member.connect(this._localMedia!.clone(), this.localMuteSettings); } }); } + async setMedia(localMedia: LocalMedia): Promise { + if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined && this._localMedia) { + const oldMedia = this._localMedia!; + this._localMedia = localMedia; + await Promise.all(Array.from(this._members.values()).map(m => { + return m.setMedia(localMedia, oldMedia); + })); + oldMedia?.stopExcept(localMedia); + } + } + + setMuted(muteSettings: MuteSettings) { + this.localMuteSettings = muteSettings; + if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { + for (const [,member] of this._members) { + member.setMuted(this.localMuteSettings); + } + } + this.emitChange(); + } + + get muteSettings(): MuteSettings { + return this.localMuteSettings; + } + get hasJoined() { return this._state === GroupCallState.Joining || this._state === GroupCallState.Joined; } @@ -168,7 +195,7 @@ export class GroupCall extends EventEmitter<{change: never}> { } /** @internal */ - create(callType: "m.video" | "m.voice"): Promise { + create(type: "m.video" | "m.voice"): Promise { return this.logItem.wrap("create", async log => { if (this._state !== GroupCallState.Fledgling) { return; @@ -176,7 +203,7 @@ export class GroupCall extends EventEmitter<{change: never}> { this._state = GroupCallState.Creating; this.emitChange(); this.callContent = Object.assign({ - "m.type": callType, + "m.type": type, }, this.callContent); const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCall, this.id, this.callContent!, {log}); await request.response(); @@ -235,7 +262,7 @@ export class GroupCall extends EventEmitter<{change: never}> { this._members.add(memberKey, member); if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { // Safari can't send a MediaStream to multiple sources, so clone it - member.connect(this._localMedia!.clone()); + member.connect(this._localMedia!.clone(), this.localMuteSettings); } } } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 7e6281aa..47db7674 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -19,6 +19,7 @@ import {makeTxnId, makeId} from "../../common"; import {EventType, CallErrorCode} from "../callEventTypes"; import {formatToDeviceMessagesPayload} from "../../common"; +import type {MuteSettings} from "../common"; import type {Options as PeerCallOptions, RemoteMedia} from "../PeerCall"; import type {LocalMedia} from "../LocalMedia"; import type {HomeServerApi} from "../../net/HomeServerApi"; @@ -50,6 +51,7 @@ const errorCodesWithoutRetry = [ export class Member { private peerCall?: PeerCall; private localMedia?: LocalMedia; + private localMuteSettings?: MuteSettings; private retryCount: number = 0; constructor( @@ -65,6 +67,10 @@ export class Member { return this.peerCall?.remoteMedia; } + get remoteMuteSettings(): MuteSettings | undefined { + return this.peerCall?.remoteMuteSettings; + } + get isConnected(): boolean { return this.peerCall?.state === CallState.Connected; } @@ -90,9 +96,10 @@ export class Member { } /** @internal */ - connect(localMedia: LocalMedia) { + connect(localMedia: LocalMedia, localMuteSettings: MuteSettings) { this.logItem.wrap("connect", () => { this.localMedia = localMedia; + this.localMuteSettings = localMuteSettings; // otherwise wait for it to connect let shouldInitiateCall; // the lexicographically lower side initiates the call @@ -103,7 +110,7 @@ export class Member { } if (shouldInitiateCall) { this.peerCall = this._createPeerCall(makeId("c")); - this.peerCall.call(localMedia); + this.peerCall.call(localMedia, localMuteSettings); } }); } @@ -133,7 +140,7 @@ export class Member { /** @internal */ emitUpdate = (peerCall: PeerCall, params: any) => { if (peerCall.state === CallState.Ringing) { - peerCall.answer(this.localMedia!); + peerCall.answer(this.localMedia!, this.localMuteSettings!); } else if (peerCall.state === CallState.Ended) { const hangupReason = peerCall.hangupReason; @@ -142,7 +149,7 @@ export class Member { if (hangupReason && !errorCodesWithoutRetry.includes(hangupReason)) { this.retryCount += 1; if (this.retryCount <= 3) { - this.connect(this.localMedia!); + this.connect(this.localMedia!, this.localMuteSettings!); } } } @@ -166,6 +173,8 @@ export class Member { } } }; + // TODO: remove this for release + log.set("payload", groupMessage.content); const request = this.options.hsApi.sendToDevice( message.type, //"m.room.encrypted", @@ -194,6 +203,17 @@ export class Member { } } + /** @internal */ + async setMedia(localMedia: LocalMedia, previousMedia: LocalMedia): Promise { + this.localMedia = localMedia.replaceClone(this.localMedia, previousMedia); + await this.peerCall?.setMedia(this.localMedia); + } + + setMuted(muteSettings: MuteSettings) { + this.localMuteSettings = muteSettings; + this.peerCall?.setMuted(muteSettings); + } + private _createPeerCall(callId: string): PeerCall { return new PeerCall(callId, Object.assign({}, this.options, { emitUpdate: this.emitUpdate, diff --git a/src/matrix/net/MediaRepository.ts b/src/matrix/net/MediaRepository.ts index 357b17c6..e95ed60c 100644 --- a/src/matrix/net/MediaRepository.ts +++ b/src/matrix/net/MediaRepository.ts @@ -29,32 +29,31 @@ export class MediaRepository { this._platform = platform; } - mxcUrlThumbnail(url: string, width: number, height: number, method: "crop" | "scale"): string | null { + mxcUrlThumbnail(url: string, width: number, height: number, method: "crop" | "scale"): string | undefined { const parts = this._parseMxcUrl(url); if (parts) { const [serverName, mediaId] = parts; const httpUrl = `${this._homeserver}/_matrix/media/r0/thumbnail/${encodeURIComponent(serverName)}/${encodeURIComponent(mediaId)}`; return httpUrl + "?" + encodeQueryParams({width: Math.round(width), height: Math.round(height), method}); } - return null; + return undefined; } - mxcUrl(url: string): string | null { + mxcUrl(url: string): string | undefined { const parts = this._parseMxcUrl(url); if (parts) { const [serverName, mediaId] = parts; return `${this._homeserver}/_matrix/media/r0/download/${encodeURIComponent(serverName)}/${encodeURIComponent(mediaId)}`; - } else { - return null; } + return undefined; } - private _parseMxcUrl(url: string): string[] | null { + private _parseMxcUrl(url: string): string[] | undefined { const prefix = "mxc://"; if (url.startsWith(prefix)) { return url.substr(prefix.length).split("/", 2); } else { - return null; + return undefined; } } diff --git a/src/matrix/room/timeline/Timeline.js b/src/matrix/room/timeline/Timeline.js index 3332a5b0..de17fce2 100644 --- a/src/matrix/room/timeline/Timeline.js +++ b/src/matrix/room/timeline/Timeline.js @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {SortedArray, AsyncMappedList, ConcatList, ObservableArray} from "../../../observable/index.js"; +import {SortedArray, AsyncMappedList, ConcatList, ObservableArray} from "../../../observable/index"; import {Disposables} from "../../../utils/Disposables"; import {Direction} from "./Direction"; import {TimelineReader} from "./persistence/TimelineReader.js"; diff --git a/src/observable/index.js b/src/observable/index.ts similarity index 78% rename from src/observable/index.js rename to src/observable/index.ts index 6057174b..040ec761 100644 --- a/src/observable/index.js +++ b/src/observable/index.ts @@ -46,3 +46,12 @@ Object.assign(BaseObservableMap.prototype, { return new JoinedMap([this].concat(otherMaps)); } }); + +declare module "./map/BaseObservableMap" { + interface BaseObservableMap { + sortValues(comparator: (a: V, b: V) => number): SortedMapList; + mapValues(mapper: (V, emitSpontaneousUpdate: (params: any) => void) => M, updater: (mappedValue: M, params: any, value: V) => void): MappedMap; + filterValues(filter: (V, K) => boolean): FilteredMap; + join(...otherMaps: BaseObservableMap[]): JoinedMap; + } +} diff --git a/src/observable/map/ObservableValueMap.ts b/src/observable/map/ObservableValueMap.ts new file mode 100644 index 00000000..2108d8ce --- /dev/null +++ b/src/observable/map/ObservableValueMap.ts @@ -0,0 +1,53 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {BaseObservableMap} from "./BaseObservableMap"; +import {BaseObservableValue} from "../value/BaseObservableValue"; +import {SubscriptionHandle} from "../BaseObservable"; + +export class ObservableValueMap extends BaseObservableMap { + private subscription?: SubscriptionHandle; + + constructor(private readonly key: K, private readonly observableValue: BaseObservableValue) { + super(); + } + + onSubscribeFirst() { + this.subscription = this.observableValue.subscribe(value => { + this.emitUpdate(this.key, value, undefined); + }); + super.onSubscribeFirst(); + } + + onUnsubscribeLast() { + this.subscription!(); + super.onUnsubscribeLast(); + } + + *[Symbol.iterator](): Iterator<[K, V]> { + yield [this.key, this.observableValue.get()]; + } + + get size(): number { + return 1; + } + + get(key: K): V | undefined { + if (key == this.key) { + return this.observableValue.get(); + } + } +} diff --git a/src/observable/value/EventObservableValue.ts b/src/observable/value/EventObservableValue.ts new file mode 100644 index 00000000..0f5fd524 --- /dev/null +++ b/src/observable/value/EventObservableValue.ts @@ -0,0 +1,45 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {BaseObservableValue} from "./BaseObservableValue"; +import {EventEmitter} from "../../utils/EventEmitter"; + +export class EventObservableValue> extends BaseObservableValue { + private eventSubscription: () => void; + + constructor( + private readonly value: V, + private readonly eventName: keyof T + ) { + super(); + } + + onSubscribeFirst(): void { + this.eventSubscription = this.value.disposableOn(this.eventName, () => { + this.emit(this.value); + }); + super.onSubscribeFirst(); + } + + onUnsubscribeLast(): void { + this.eventSubscription!(); + super.onUnsubscribeLast(); + } + + get(): V { + return this.value; + } +} diff --git a/src/platform/types/MediaDevices.ts b/src/platform/types/MediaDevices.ts index d0edbbea..1b5f7afd 100644 --- a/src/platform/types/MediaDevices.ts +++ b/src/platform/types/MediaDevices.ts @@ -14,19 +14,48 @@ See the License for the specific language governing permissions and limitations under the License. */ +export interface Event {} + export interface MediaDevices { // filter out audiooutput enumerate(): Promise; // to assign to a video element, we downcast to WrappedTrack and use the stream property. getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise; getScreenShareTrack(): Promise; + createVolumeMeasurer(stream: Stream, callback: () => void): VolumeMeasurer; +} + +// Typescript definitions derived from https://github.com/microsoft/TypeScript/blob/main/lib/lib.dom.d.ts +/*! ***************************************************************************** +Copyright (c) Microsoft Corporation. All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at http://www.apache.org/licenses/LICENSE-2.0 +THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +MERCHANTABLITY OR NON-INFRINGEMENT. +See the Apache Version 2.0 License for specific language governing permissions +and limitations under the License. +***************************************************************************** */ + +export interface StreamTrackEvent extends Event { + readonly track: Track; +} + +export interface StreamEventMap { + "addtrack": StreamTrackEvent; + "removetrack": StreamTrackEvent; } export interface Stream { - readonly audioTrack: AudioTrack | undefined; - readonly videoTrack: Track | undefined; + getTracks(): ReadonlyArray; + getAudioTracks(): ReadonlyArray; + getVideoTracks(): ReadonlyArray; readonly id: string; clone(): Stream; + addEventListener(type: K, listener: (this: Stream, ev: StreamEventMap[K]) => any, options?: boolean | AddEventListenerOptions): void; + removeEventListener(type: K, listener: (this: Stream, ev: StreamEventMap[K]) => any, options?: boolean | EventListenerOptions): void; } export enum TrackKind { @@ -38,12 +67,13 @@ export interface Track { readonly kind: TrackKind; readonly label: string; readonly id: string; - readonly settings: MediaTrackSettings; + enabled: boolean; + // getSettings(): MediaTrackSettings; stop(): void; } -export interface AudioTrack extends Track { - // TODO: how to emit updates on this? +export interface VolumeMeasurer { get isSpeaking(): boolean; + setSpeakingThreshold(threshold: number): void; + stop(); } - diff --git a/src/platform/types/WebRTC.ts b/src/platform/types/WebRTC.ts index edb26c0a..9f2e1e0e 100644 --- a/src/platform/types/WebRTC.ts +++ b/src/platform/types/WebRTC.ts @@ -14,63 +14,155 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {Track, Stream} from "./MediaDevices"; +import {Track, Stream, Event} from "./MediaDevices"; import {SDPStreamMetadataPurpose} from "../../matrix/calls/callEventTypes"; export interface WebRTC { - createPeerConnection(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize: number): PeerConnection; + createPeerConnection(forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize: number): PeerConnection; + prepareSenderForPurpose(peerConnection: PeerConnection, sender: Sender, purpose: SDPStreamMetadataPurpose): void; } -export interface StreamSender { - get stream(): Stream; - get audioSender(): TrackSender | undefined; - get videoSender(): TrackSender | undefined; +// Typescript definitions derived from https://github.com/microsoft/TypeScript/blob/main/lib/lib.dom.d.ts +/*! ***************************************************************************** +Copyright (c) Microsoft Corporation. All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at http://www.apache.org/licenses/LICENSE-2.0 +THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +MERCHANTABLITY OR NON-INFRINGEMENT. +See the Apache Version 2.0 License for specific language governing permissions +and limitations under the License. +***************************************************************************** */ + +export interface DataChannelEventMap { + "bufferedamountlow": Event; + "close": Event; + "error": Event; + "message": MessageEvent; + "open": Event; } -export interface StreamReceiver { - get stream(): Stream; - get audioReceiver(): TrackReceiver | undefined; - get videoReceiver(): TrackReceiver | undefined; -} - -export interface TrackReceiver { - get track(): Track; - get enabled(): boolean; - enable(enabled: boolean); // this modifies the transceiver direction -} - -export interface TrackSender extends TrackReceiver { - /** replaces the track if possible without renegotiation. Can throw. */ - replaceTrack(track: Track | undefined): Promise; - /** make any needed adjustments to the sender or transceiver settings - * depending on the purpose, after adding the track to the connection */ - prepareForPurpose(purpose: SDPStreamMetadataPurpose): void; -} - -export interface PeerConnectionHandler { - onIceConnectionStateChange(state: RTCIceConnectionState); - onLocalIceCandidate(candidate: RTCIceCandidate); - onIceGatheringStateChange(state: RTCIceGatheringState); - onRemoteStreamRemoved(stream: Stream); - onRemoteTracksAdded(receiver: TrackReceiver); - onRemoteDataChannel(dataChannel: any | undefined); - onNegotiationNeeded(); -} - -export interface PeerConnection { - get iceGatheringState(): RTCIceGatheringState; - get signalingState(): RTCSignalingState; - get localDescription(): RTCSessionDescription | undefined; - get localStreams(): ReadonlyMap; - get remoteStreams(): ReadonlyMap; - createOffer(): Promise; - createAnswer(): Promise; - setLocalDescription(description?: RTCSessionDescriptionInit): Promise; - setRemoteDescription(description: RTCSessionDescriptionInit): Promise; - addIceCandidate(candidate: RTCIceCandidate): Promise; - addTrack(track: Track): TrackSender | undefined; - removeTrack(track: TrackSender): void; - createDataChannel(options: RTCDataChannelInit): any; - dispose(): void; +export interface DataChannel { + binaryType: BinaryType; + readonly id: number | null; + readonly label: string; + readonly negotiated: boolean; + readonly readyState: DataChannelState; close(): void; + send(data: string): void; + send(data: Blob): void; + send(data: ArrayBuffer): void; + send(data: ArrayBufferView): void; + addEventListener(type: K, listener: (this: DataChannel, ev: DataChannelEventMap[K]) => any, options?: boolean | AddEventListenerOptions): void; + removeEventListener(type: K, listener: (this: DataChannel, ev: DataChannelEventMap[K]) => any, options?: boolean | EventListenerOptions): void; +} + +export interface DataChannelInit { + id?: number; + maxPacketLifeTime?: number; + maxRetransmits?: number; + negotiated?: boolean; + ordered?: boolean; + protocol?: string; +} + +export interface DataChannelEvent extends Event { + readonly channel: DataChannel; +} + +export interface PeerConnectionIceEvent extends Event { + readonly candidate: RTCIceCandidate | null; +} + +export interface TrackEvent extends Event { + readonly receiver: Receiver; + readonly streams: ReadonlyArray; + readonly track: Track; + readonly transceiver: Transceiver; +} + +export interface PeerConnectionEventMap { + "connectionstatechange": Event; + "datachannel": DataChannelEvent; + "icecandidate": PeerConnectionIceEvent; + "iceconnectionstatechange": Event; + "icegatheringstatechange": Event; + "negotiationneeded": Event; + "signalingstatechange": Event; + "track": TrackEvent; +} + +export type DataChannelState = "closed" | "closing" | "connecting" | "open"; +export type IceConnectionState = "checking" | "closed" | "completed" | "connected" | "disconnected" | "failed" | "new"; +export type PeerConnectionState = "closed" | "connected" | "connecting" | "disconnected" | "failed" | "new"; +export type SignalingState = "closed" | "have-local-offer" | "have-local-pranswer" | "have-remote-offer" | "have-remote-pranswer" | "stable"; +export type IceGatheringState = "complete" | "gathering" | "new"; +export type SdpType = "answer" | "offer" | "pranswer" | "rollback"; +export type TransceiverDirection = "inactive" | "recvonly" | "sendonly" | "sendrecv" | "stopped"; +export interface SessionDescription { + readonly sdp: string; + readonly type: SdpType; +} + +export interface AnswerOptions {} + +export interface OfferOptions { + iceRestart?: boolean; + offerToReceiveAudio?: boolean; + offerToReceiveVideo?: boolean; +} + +export interface SessionDescriptionInit { + sdp?: string; + type: SdpType; +} + +export interface LocalSessionDescriptionInit { + sdp?: string; + type?: SdpType; +} + +/** A WebRTC connection between the local computer and a remote peer. It provides methods to connect to a remote peer, maintain and monitor the connection, and close the connection once it's no longer needed. */ +export interface PeerConnection { + readonly connectionState: PeerConnectionState; + readonly iceConnectionState: IceConnectionState; + readonly iceGatheringState: IceGatheringState; + readonly localDescription: SessionDescription | null; + readonly remoteDescription: SessionDescription | null; + readonly signalingState: SignalingState; + addIceCandidate(candidate?: RTCIceCandidateInit): Promise; + addTrack(track: Track, ...streams: Stream[]): Sender; + close(): void; + createAnswer(options?: AnswerOptions): Promise; + createDataChannel(label: string, dataChannelDict?: DataChannelInit): DataChannel; + createOffer(options?: OfferOptions): Promise; + getReceivers(): Receiver[]; + getSenders(): Sender[]; + getTransceivers(): Transceiver[]; + removeTrack(sender: Sender): void; + restartIce(): void; + setLocalDescription(description?: LocalSessionDescriptionInit): Promise; + setRemoteDescription(description: SessionDescriptionInit): Promise; + addEventListener(type: K, listener: (this: PeerConnection, ev: PeerConnectionEventMap[K]) => any, options?: boolean | AddEventListenerOptions): void; + removeEventListener(type: K, listener: (this: PeerConnection, ev: PeerConnectionEventMap[K]) => any, options?: boolean | EventListenerOptions): void; +} + +export interface Receiver { + readonly track: Track; +} + +export interface Sender { + readonly track: Track | null; + replaceTrack(withTrack: Track | null): Promise; +} + +export interface Transceiver { + readonly currentDirection: TransceiverDirection | null; + direction: TransceiverDirection; + readonly mid: string | null; + readonly receiver: Receiver; + readonly sender: Sender; + stop(): void; } diff --git a/src/platform/web/LegacyPlatform.js b/src/platform/web/LegacyPlatform.js index 85632bf2..b8a6d7e7 100644 --- a/src/platform/web/LegacyPlatform.js +++ b/src/platform/web/LegacyPlatform.js @@ -19,6 +19,6 @@ import {hkdf} from "../../utils/crypto/hkdf"; import {Platform as ModernPlatform} from "./Platform.js"; -export function Platform(container, assetPaths, config, options = null) { - return new ModernPlatform(container, assetPaths, config, options, {aesjs, hkdf}); +export function Platform({ container, assetPaths, config, configURL, options = null }) { + return new ModernPlatform({ container, assetPaths, config, configURL, options, cryptoExtras: { aesjs, hkdf }}); } diff --git a/src/platform/web/Platform.js b/src/platform/web/Platform.js index 7426b138..9a8a19c9 100644 --- a/src/platform/web/Platform.js +++ b/src/platform/web/Platform.js @@ -128,10 +128,11 @@ function adaptUIOnVisualViewportResize(container) { } export class Platform { - constructor(container, assetPaths, config, options = null, cryptoExtras = null) { + constructor({ container, assetPaths, config, configURL, options = null, cryptoExtras = null }) { this._container = container; this._assetPaths = assetPaths; this._config = config; + this._configURL = configURL; this.settingsStorage = new SettingsStorage("hydrogen_setting_v1_"); this.clock = new Clock(); this.encoding = new Encoding(); @@ -144,7 +145,7 @@ export class Platform { this._serviceWorkerHandler = new ServiceWorkerHandler(); this._serviceWorkerHandler.registerAndStart(assetPaths.serviceWorker); } - this.notificationService = new NotificationService(this._serviceWorkerHandler, config.push); + this.notificationService = undefined; // Only try to use crypto when olm is provided if(this._assetPaths.olm) { this.crypto = new Crypto(cryptoExtras); @@ -169,6 +170,20 @@ export class Platform { this.webRTC = new DOMWebRTC(); } + async init() { + if (!this._config) { + if (!this._configURL) { + throw new Error("Neither config nor configURL was provided!"); + } + const {body}= await this.request(this._configURL, {method: "GET", format: "json", cache: true}).response(); + this._config = body; + } + this.notificationService = new NotificationService( + this._serviceWorkerHandler, + this._config.push + ); + } + _createLogger(isDevelopment) { // Make sure that loginToken does not end up in the logs const transformer = (item) => { diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index 3723162a..c34ab85b 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {MediaDevices as IMediaDevices, Stream, Track, TrackKind, AudioTrack} from "../../types/MediaDevices"; +import {MediaDevices as IMediaDevices, Stream, Track, TrackKind, VolumeMeasurer} from "../../types/MediaDevices"; const POLLING_INTERVAL = 200; // ms export const SPEAKING_THRESHOLD = -60; // dB @@ -30,12 +30,12 @@ export class MediaDevicesWrapper implements IMediaDevices { async getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise { const stream = await this.mediaDevices.getUserMedia(this.getUserMediaContraints(audio, video)); - return new StreamWrapper(stream); + return stream as Stream; } async getScreenShareTrack(): Promise { const stream = await this.mediaDevices.getDisplayMedia(this.getScreenshareContraints()); - return new StreamWrapper(stream); + return stream as Stream; } private getUserMediaContraints(audio: boolean | MediaDeviceInfo, video: boolean | MediaDeviceInfo): MediaStreamConstraints { @@ -68,55 +68,13 @@ export class MediaDevicesWrapper implements IMediaDevices { video: true, }; } -} -export class StreamWrapper implements Stream { - - public audioTrack: AudioTrackWrapper | undefined = undefined; - public videoTrack: TrackWrapper | undefined = undefined; - - constructor(public readonly stream: MediaStream) { - for (const track of stream.getTracks()) { - this.update(track); - } - } - - get id(): string { return this.stream.id; } - - clone(): Stream { - return new StreamWrapper(this.stream.clone()); - } - - update(track: MediaStreamTrack): TrackWrapper | undefined { - if (track.kind === "video") { - if (!this.videoTrack || track.id !== this.videoTrack.track.id) { - this.videoTrack = new TrackWrapper(track, this.stream); - } - return this.videoTrack; - } else if (track.kind === "audio") { - if (!this.audioTrack || track.id !== this.audioTrack.track.id) { - this.audioTrack = new AudioTrackWrapper(track, this.stream); - } - return this.audioTrack; - } + createVolumeMeasurer(stream: Stream, callback: () => void): VolumeMeasurer { + return new WebAudioVolumeMeasurer(stream as MediaStream, callback); } } -export class TrackWrapper implements Track { - constructor( - public readonly track: MediaStreamTrack, - public readonly stream: MediaStream - ) {} - - get kind(): TrackKind { return this.track.kind as TrackKind; } - get label(): string { return this.track.label; } - get id(): string { return this.track.id; } - get settings(): MediaTrackSettings { return this.track.getSettings(); } - - stop() { this.track.stop(); } -} - -export class AudioTrackWrapper extends TrackWrapper { +export class WebAudioVolumeMeasurer implements VolumeMeasurer { private measuringVolumeActivity = false; private audioContext?: AudioContext; private analyser: AnalyserNode; @@ -125,9 +83,12 @@ export class AudioTrackWrapper extends TrackWrapper { private speaking = false; private volumeLooperTimeout: number; private speakingVolumeSamples: number[]; + private callback: () => void; + private stream: MediaStream; - constructor(track: MediaStreamTrack, stream: MediaStream) { - super(track, stream); + constructor(stream: MediaStream, callback: () => void) { + this.stream = stream; + this.callback = callback; this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); this.initVolumeMeasuring(); this.measureVolumeActivity(true); @@ -147,6 +108,7 @@ export class AudioTrackWrapper extends TrackWrapper { } else { this.measuringVolumeActivity = false; this.speakingVolumeSamples.fill(-Infinity); + this.callback(); // this.emit(CallFeedEvent.VolumeChanged, -Infinity); } } @@ -167,7 +129,6 @@ export class AudioTrackWrapper extends TrackWrapper { this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount); } - public setSpeakingThreshold(threshold: number) { this.speakingThreshold = threshold; } @@ -189,6 +150,7 @@ export class AudioTrackWrapper extends TrackWrapper { this.speakingVolumeSamples.shift(); this.speakingVolumeSamples.push(maxVolume); + this.callback(); // this.emit(CallFeedEvent.VolumeChanged, maxVolume); let newSpeaking = false; @@ -204,267 +166,16 @@ export class AudioTrackWrapper extends TrackWrapper { if (this.speaking !== newSpeaking) { this.speaking = newSpeaking; + this.callback(); // this.emit(CallFeedEvent.Speaking, this.speaking); } this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL) as unknown as number; }; - public dispose(): void { + public stop(): void { clearTimeout(this.volumeLooperTimeout); + this.analyser.disconnect(); + this.audioContext?.close(); } } - -// export interface ICallFeedOpts { -// client: MatrixClient; -// roomId: string; -// userId: string; -// stream: MediaStream; -// purpose: SDPStreamMetadataPurpose; -// audioMuted: boolean; -// videoMuted: boolean; -// } - -// export enum CallFeedEvent { -// NewStream = "new_stream", -// MuteStateChanged = "mute_state_changed", -// VolumeChanged = "volume_changed", -// Speaking = "speaking", -// } - -// export class CallFeed extends EventEmitter { -// public stream: MediaStream; -// public sdpMetadataStreamId: string; -// public userId: string; -// public purpose: SDPStreamMetadataPurpose; -// public speakingVolumeSamples: number[]; - -// private client: MatrixClient; -// private roomId: string; -// private audioMuted: boolean; -// private videoMuted: boolean; -// private measuringVolumeActivity = false; -// private audioContext: AudioContext; -// private analyser: AnalyserNode; -// private frequencyBinCount: Float32Array; -// private speakingThreshold = SPEAKING_THRESHOLD; -// private speaking = false; -// private volumeLooperTimeout: number; - -// constructor(opts: ICallFeedOpts) { -// super(); - -// this.client = opts.client; -// this.roomId = opts.roomId; -// this.userId = opts.userId; -// this.purpose = opts.purpose; -// this.audioMuted = opts.audioMuted; -// this.videoMuted = opts.videoMuted; -// this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); -// this.sdpMetadataStreamId = opts.stream.id; - -// this.updateStream(null, opts.stream); - -// if (this.hasAudioTrack) { -// this.initVolumeMeasuring(); -// } -// } - -// private get hasAudioTrack(): boolean { -// return this.stream.getAudioTracks().length > 0; -// } - -// private updateStream(oldStream: MediaStream, newStream: MediaStream): void { -// if (newStream === oldStream) return; - -// if (oldStream) { -// oldStream.removeEventListener("addtrack", this.onAddTrack); -// this.measureVolumeActivity(false); -// } -// if (newStream) { -// this.stream = newStream; -// newStream.addEventListener("addtrack", this.onAddTrack); - -// if (this.hasAudioTrack) { -// this.initVolumeMeasuring(); -// } else { -// this.measureVolumeActivity(false); -// } -// } - -// this.emit(CallFeedEvent.NewStream, this.stream); -// } - -// private initVolumeMeasuring(): void { -// const AudioContext = window.AudioContext || window.webkitAudioContext; -// if (!this.hasAudioTrack || !AudioContext) return; - -// this.audioContext = new AudioContext(); - -// this.analyser = this.audioContext.createAnalyser(); -// this.analyser.fftSize = 512; -// this.analyser.smoothingTimeConstant = 0.1; - -// const mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.stream); -// mediaStreamAudioSourceNode.connect(this.analyser); - -// this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount); -// } - -// private onAddTrack = (): void => { -// this.emit(CallFeedEvent.NewStream, this.stream); -// }; - -// /** -// * Returns callRoom member -// * @returns member of the callRoom -// */ -// public getMember(): RoomMember { -// const callRoom = this.client.getRoom(this.roomId); -// return callRoom.getMember(this.userId); -// } - -// /** -// * Returns true if CallFeed is local, otherwise returns false -// * @returns {boolean} is local? -// */ -// public isLocal(): boolean { -// return this.userId === this.client.getUserId(); -// } - -// /** -// * Returns true if audio is muted or if there are no audio -// * tracks, otherwise returns false -// * @returns {boolean} is audio muted? -// */ -// public isAudioMuted(): boolean { -// return this.stream.getAudioTracks().length === 0 || this.audioMuted; -// } - -// * -// * Returns true video is muted or if there are no video -// * tracks, otherwise returns false -// * @returns {boolean} is video muted? - -// public isVideoMuted(): boolean { -// // We assume only one video track -// return this.stream.getVideoTracks().length === 0 || this.videoMuted; -// } - -// public isSpeaking(): boolean { -// return this.speaking; -// } - -// /** -// * Replaces the current MediaStream with a new one. -// * This method should be only used by MatrixCall. -// * @param newStream new stream with which to replace the current one -// */ -// public setNewStream(newStream: MediaStream): void { -// this.updateStream(this.stream, newStream); -// } - -// /** -// * Set feed's internal audio mute state -// * @param muted is the feed's audio muted? -// */ -// public setAudioMuted(muted: boolean): void { -// this.audioMuted = muted; -// this.speakingVolumeSamples.fill(-Infinity); -// this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); -// } - -// /** -// * Set feed's internal video mute state -// * @param muted is the feed's video muted? -// */ -// public setVideoMuted(muted: boolean): void { -// this.videoMuted = muted; -// this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); -// } - -// /** -// * Starts emitting volume_changed events where the emitter value is in decibels -// * @param enabled emit volume changes -// */ -// public measureVolumeActivity(enabled: boolean): void { -// if (enabled) { -// if (!this.audioContext || !this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return; - -// this.measuringVolumeActivity = true; -// this.volumeLooper(); -// } else { -// this.measuringVolumeActivity = false; -// this.speakingVolumeSamples.fill(-Infinity); -// this.emit(CallFeedEvent.VolumeChanged, -Infinity); -// } -// } - -// public setSpeakingThreshold(threshold: number) { -// this.speakingThreshold = threshold; -// } - -// private volumeLooper = () => { -// if (!this.analyser) return; - -// if (!this.measuringVolumeActivity) return; - -// this.analyser.getFloatFrequencyData(this.frequencyBinCount); - -// let maxVolume = -Infinity; -// for (let i = 0; i < this.frequencyBinCount.length; i++) { -// if (this.frequencyBinCount[i] > maxVolume) { -// maxVolume = this.frequencyBinCount[i]; -// } -// } - -// this.speakingVolumeSamples.shift(); -// this.speakingVolumeSamples.push(maxVolume); - -// this.emit(CallFeedEvent.VolumeChanged, maxVolume); - -// let newSpeaking = false; - -// for (let i = 0; i < this.speakingVolumeSamples.length; i++) { -// const volume = this.speakingVolumeSamples[i]; - -// if (volume > this.speakingThreshold) { -// newSpeaking = true; -// break; -// } -// } - -// if (this.speaking !== newSpeaking) { -// this.speaking = newSpeaking; -// this.emit(CallFeedEvent.Speaking, this.speaking); -// } - -// this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL); -// }; - -// public clone(): CallFeed { -// const mediaHandler = this.client.getMediaHandler(); -// const stream = this.stream.clone(); - -// if (this.purpose === SDPStreamMetadataPurpose.Usermedia) { -// mediaHandler.userMediaStreams.push(stream); -// } else { -// mediaHandler.screensharingStreams.push(stream); -// } - -// return new CallFeed({ -// client: this.client, -// roomId: this.roomId, -// userId: this.userId, -// stream, -// purpose: this.purpose, -// audioMuted: this.audioMuted, -// videoMuted: this.videoMuted, -// }); -// } - -// public dispose(): void { -// clearTimeout(this.volumeLooperTimeout); -// this.measureVolumeActivity(false); -// } -// } diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 672c14d4..61d481d4 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -14,9 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {StreamWrapper, TrackWrapper, AudioTrackWrapper} from "./MediaDevices"; -import {Stream, Track, AudioTrack, TrackKind} from "../../types/MediaDevices"; -import {WebRTC, PeerConnectionHandler, StreamSender, TrackSender, StreamReceiver, TrackReceiver, PeerConnection} from "../../types/WebRTC"; +import {Stream, Track, TrackKind} from "../../types/MediaDevices"; +import {WebRTC, Sender, PeerConnection} from "../../types/WebRTC"; import {SDPStreamMetadataPurpose} from "../../../matrix/calls/callEventTypes"; const POLLING_INTERVAL = 200; // ms @@ -24,151 +23,21 @@ export const SPEAKING_THRESHOLD = -60; // dB const SPEAKING_SAMPLE_COUNT = 8; // samples export class DOMWebRTC implements WebRTC { - createPeerConnection(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize): PeerConnection { - return new DOMPeerConnection(handler, forceTURN, turnServers, iceCandidatePoolSize); - } -} - -export class RemoteStreamWrapper extends StreamWrapper { - constructor(stream: MediaStream, private readonly emptyCallback: (stream: RemoteStreamWrapper) => void) { - super(stream); - this.stream.addEventListener("removetrack", this.onTrackRemoved); + createPeerConnection(forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize): PeerConnection { + return new RTCPeerConnection({ + iceTransportPolicy: forceTURN ? 'relay' : undefined, + iceServers: turnServers, + iceCandidatePoolSize: iceCandidatePoolSize, + }) as PeerConnection; } - onTrackRemoved = (evt: MediaStreamTrackEvent) => { - if (evt.track.id === this.audioTrack?.track.id) { - this.audioTrack = undefined; - } else if (evt.track.id === this.videoTrack?.track.id) { - this.videoTrack = undefined; - } - if (!this.audioTrack && !this.videoTrack) { - this.emptyCallback(this); - } - }; - - dispose() { - this.stream.removeEventListener("removetrack", this.onTrackRemoved); - } -} - -export class DOMStreamSender implements StreamSender { - public audioSender: DOMTrackSender | undefined; - public videoSender: DOMTrackSender | undefined; - - constructor(public readonly stream: StreamWrapper) {} - - update(transceivers: ReadonlyArray, sender: RTCRtpSender): DOMTrackSender | undefined { - const transceiver = transceivers.find(t => t.sender === sender); - if (transceiver && sender.track) { - const trackWrapper = this.stream.update(sender.track); - if (trackWrapper) { - if (trackWrapper.kind === TrackKind.Video && (!this.videoSender || this.videoSender.track.id !== trackWrapper.id)) { - this.videoSender = new DOMTrackSender(trackWrapper, transceiver); - return this.videoSender; - } else if (trackWrapper.kind === TrackKind.Audio && (!this.audioSender || this.audioSender.track.id !== trackWrapper.id)) { - this.audioSender = new DOMTrackSender(trackWrapper, transceiver); - return this.audioSender; - } - } - } - } -} - -export class DOMStreamReceiver implements StreamReceiver { - public audioReceiver: DOMTrackReceiver | undefined; - public videoReceiver: DOMTrackReceiver | undefined; - - constructor(public readonly stream: RemoteStreamWrapper) {} - - update(event: RTCTrackEvent): DOMTrackReceiver | undefined { - const {receiver} = event; - const {track} = receiver; - const trackWrapper = this.stream.update(track); - if (trackWrapper) { - if (trackWrapper.kind === TrackKind.Video) { - this.videoReceiver = new DOMTrackReceiver(trackWrapper, event.transceiver); - return this.videoReceiver; - } else { - this.audioReceiver = new DOMTrackReceiver(trackWrapper, event.transceiver); - return this.audioReceiver; - } - } - } -} - -export class DOMTrackSenderOrReceiver implements TrackReceiver { - constructor( - public readonly track: TrackWrapper, - public readonly transceiver: RTCRtpTransceiver, - private readonly exclusiveValue: RTCRtpTransceiverDirection, - private readonly excludedValue: RTCRtpTransceiverDirection - ) {} - - get enabled(): boolean { - return this.transceiver.direction === "sendrecv" || - this.transceiver.direction === this.exclusiveValue; - } - - enable(enabled: boolean) { - if (enabled !== this.enabled) { - if (enabled) { - if (this.transceiver.direction === "inactive") { - this.transceiver.direction = this.exclusiveValue; - } else { - this.transceiver.direction = "sendrecv"; - } - } else { - if (this.transceiver.direction === "sendrecv") { - this.transceiver.direction = this.excludedValue; - } else { - this.transceiver.direction = "inactive"; - } - } - } - } -} - -export class DOMTrackReceiver extends DOMTrackSenderOrReceiver { - constructor( - track: TrackWrapper, - transceiver: RTCRtpTransceiver, - ) { - super(track, transceiver, "recvonly", "sendonly"); - } -} - -export class DOMTrackSender extends DOMTrackSenderOrReceiver { - constructor( - track: TrackWrapper, - transceiver: RTCRtpTransceiver, - ) { - super(track, transceiver, "sendonly", "recvonly"); - } - /** replaces the track if possible without renegotiation. Can throw. */ - replaceTrack(track: Track | undefined): Promise { - return this.transceiver.sender.replaceTrack(track ? (track as TrackWrapper).track : null); - } - - prepareForPurpose(purpose: SDPStreamMetadataPurpose): void { + prepareSenderForPurpose(peerConnection: PeerConnection, sender: Sender, purpose: SDPStreamMetadataPurpose): void { if (purpose === SDPStreamMetadataPurpose.Screenshare) { - this.getRidOfRTXCodecs(); + this.getRidOfRTXCodecs(peerConnection as RTCPeerConnection, sender as RTCRtpSender); } } - /** - * This method removes all video/rtx codecs from screensharing video - * transceivers. This is necessary since they can cause problems. Without - * this the following steps should produce an error: - * Chromium calls Firefox - * Firefox answers - * Firefox starts screen-sharing - * Chromium starts screen-sharing - * Call crashes for Chromium with: - * [96685:23:0518/162603.933321:ERROR:webrtc_video_engine.cc(3296)] RTX codec (PT=97) mapped to PT=96 which is not in the codec list. - * [96685:23:0518/162603.933377:ERROR:webrtc_video_engine.cc(1171)] GetChangedRecvParameters called without any video codecs. - * [96685:23:0518/162603.933430:ERROR:sdp_offer_answer.cc(4302)] Failed to set local video description recv parameters for m-section with mid='2'. (INVALID_PARAMETER) - */ - private getRidOfRTXCodecs(): void { + private getRidOfRTXCodecs(peerConnection: RTCPeerConnection, sender: RTCRtpSender): void { // RTCRtpReceiver.getCapabilities and RTCRtpSender.getCapabilities don't seem to be supported on FF if (!RTCRtpReceiver.getCapabilities || !RTCRtpSender.getCapabilities) return; @@ -182,172 +51,14 @@ export class DOMTrackSender extends DOMTrackSenderOrReceiver { codecs.splice(rtxCodecIndex, 1); } } - if (this.transceiver.sender.track?.kind === "video" || - this.transceiver.receiver.track?.kind === "video") { - this.transceiver.setCodecPreferences(codecs); - } - } -} - -class DOMPeerConnection implements PeerConnection { - private readonly peerConnection: RTCPeerConnection; - private readonly handler: PeerConnectionHandler; - public readonly localStreams: Map = new Map(); - public readonly remoteStreams: Map = new Map(); - - constructor(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize) { - this.handler = handler; - this.peerConnection = new RTCPeerConnection({ - iceTransportPolicy: forceTURN ? 'relay' : undefined, - iceServers: turnServers, - iceCandidatePoolSize: iceCandidatePoolSize, - }); - this.registerHandler(); - } - - get iceGatheringState(): RTCIceGatheringState { return this.peerConnection.iceGatheringState; } - get localDescription(): RTCSessionDescription | undefined { return this.peerConnection.localDescription ?? undefined; } - get signalingState(): RTCSignalingState { return this.peerConnection.signalingState; } - - createOffer(): Promise { - return this.peerConnection.createOffer(); - } - - createAnswer(): Promise { - return this.peerConnection.createAnswer(); - } - - setLocalDescription(description?: RTCSessionDescriptionInit): Promise { - return this.peerConnection.setLocalDescription(description); - } - - setRemoteDescription(description: RTCSessionDescriptionInit): Promise { - return this.peerConnection.setRemoteDescription(description); - } - - addIceCandidate(candidate: RTCIceCandidate): Promise { - return this.peerConnection.addIceCandidate(candidate); - } - - close(): void { - return this.peerConnection.close(); - } - - addTrack(track: Track): DOMTrackSender | undefined { - if (!(track instanceof TrackWrapper)) { - throw new Error("Not a TrackWrapper"); - } - const sender = this.peerConnection.addTrack(track.track, track.stream); - let streamSender = this.localStreams.get(track.stream.id); - if (!streamSender) { - // TODO: reuse existing stream wrapper here? - streamSender = new DOMStreamSender(new StreamWrapper(track.stream)); - this.localStreams.set(track.stream.id, streamSender); - } - const trackSender = streamSender.update(this.peerConnection.getTransceivers(), sender); - return trackSender; - } - - removeTrack(sender: TrackSender): void { - if (!(sender instanceof DOMTrackSender)) { - throw new Error("Not a DOMTrackSender"); - } - this.peerConnection.removeTrack((sender as DOMTrackSender).transceiver.sender); - // TODO: update localStreams - } - - createDataChannel(options: RTCDataChannelInit): any { - return this.peerConnection.createDataChannel("channel", options); - } - - private registerHandler() { - const pc = this.peerConnection; - pc.addEventListener('negotiationneeded', this); - pc.addEventListener('icecandidate', this); - pc.addEventListener('iceconnectionstatechange', this); - pc.addEventListener('icegatheringstatechange', this); - pc.addEventListener('signalingstatechange', this); - pc.addEventListener('track', this); - pc.addEventListener('datachannel', this); - } - - private deregisterHandler() { - const pc = this.peerConnection; - pc.removeEventListener('negotiationneeded', this); - pc.removeEventListener('icecandidate', this); - pc.removeEventListener('iceconnectionstatechange', this); - pc.removeEventListener('icegatheringstatechange', this); - pc.removeEventListener('signalingstatechange', this); - pc.removeEventListener('track', this); - pc.removeEventListener('datachannel', this); - } - - /** @internal */ - handleEvent(evt: Event) { - switch (evt.type) { - case "iceconnectionstatechange": - this.handleIceConnectionStateChange(); - break; - case "icecandidate": - this.handleLocalIceCandidate(evt as RTCPeerConnectionIceEvent); - break; - case "icegatheringstatechange": - this.handler.onIceGatheringStateChange(this.peerConnection.iceGatheringState); - break; - case "track": - this.handleRemoteTrack(evt as RTCTrackEvent); - break; - case "negotiationneeded": - this.handler.onNegotiationNeeded(); - break; - case "datachannel": - this.handler.onRemoteDataChannel((evt as RTCDataChannelEvent).channel); - break; - } - } - - dispose(): void { - this.deregisterHandler(); - for (const r of this.remoteStreams.values()) { - r.stream.dispose(); - } - } - - private handleLocalIceCandidate(event: RTCPeerConnectionIceEvent) { - if (event.candidate) { - this.handler.onLocalIceCandidate(event.candidate); - } - }; - - private handleIceConnectionStateChange() { - const {iceConnectionState} = this.peerConnection; - if (iceConnectionState === "failed" && this.peerConnection.restartIce) { - this.peerConnection.restartIce(); - } else { - this.handler.onIceConnectionStateChange(iceConnectionState); - } - } - - onRemoteStreamEmpty = (stream: RemoteStreamWrapper): void => { - if (this.remoteStreams.delete(stream.id)) { - this.handler.onRemoteStreamRemoved(stream); - } - } - - private handleRemoteTrack(evt: RTCTrackEvent) { - if (evt.streams.length !== 1) { - throw new Error("track in multiple streams is not supported"); - } - const stream = evt.streams[0]; - const transceivers = this.peerConnection.getTransceivers(); - let streamReceiver: DOMStreamReceiver | undefined = this.remoteStreams.get(stream.id); - if (!streamReceiver) { - streamReceiver = new DOMStreamReceiver(new RemoteStreamWrapper(stream, this.onRemoteStreamEmpty)); - this.remoteStreams.set(stream.id, streamReceiver); - } - const trackReceiver = streamReceiver.update(evt); - if (trackReceiver) { - this.handler.onRemoteTracksAdded(trackReceiver); + + const transceiver = peerConnection.getTransceivers().find(t => t.sender === sender); + if (transceiver && ( + transceiver.sender.track?.kind === "video" || + transceiver.receiver.track?.kind === "video" + ) + ) { + transceiver.setCodecPreferences(codecs); } } } diff --git a/src/platform/web/index.html b/src/platform/web/index.html index 5950d89f..16418699 100644 --- a/src/platform/web/index.html +++ b/src/platform/web/index.html @@ -17,17 +17,17 @@ diff --git a/src/platform/web/main.js b/src/platform/web/main.js index 1729c17c..edc2cf14 100644 --- a/src/platform/web/main.js +++ b/src/platform/web/main.js @@ -32,6 +32,7 @@ export async function main(platform) { // const recorder = new RecordRequester(createFetchRequest(clock.createTimeout)); // const request = recorder.request; // window.getBrawlFetchLog = () => recorder.log(); + await platform.init(); const navigation = createNavigation(); platform.setNavigation(navigation); const urlRouter = createRouter({navigation, history: platform.history}); diff --git a/src/platform/web/sw.js b/src/platform/web/sw.js index c5f69438..a9a92979 100644 --- a/src/platform/web/sw.js +++ b/src/platform/web/sw.js @@ -92,8 +92,12 @@ function isCacheableThumbnail(url) { const baseURL = new URL(self.registration.scope); let pendingFetchAbortController = new AbortController(); + async function handleRequest(request) { try { + if (request.url.includes("config.json")) { + return handleConfigRequest(request); + } const url = new URL(request.url); // rewrite / to /index.html so it hits the cache if (url.origin === baseURL.origin && url.pathname === baseURL.pathname) { @@ -119,6 +123,27 @@ async function handleRequest(request) { } } +async function handleConfigRequest(request) { + let response = await readCache(request); + const networkResponsePromise = fetchAndUpdateConfig(request); + if (response) { + return response; + } else { + return await networkResponsePromise; + } +} + +async function fetchAndUpdateConfig(request) { + const response = await fetch(request, { + signal: pendingFetchAbortController.signal, + headers: { + "Cache-Control": "no-cache", + }, + }); + updateCache(request, response.clone()); + return response; +} + async function updateCache(request, response) { // don't write error responses to the cache if (response.status >= 400) { diff --git a/src/platform/web/ui/css/themes/element/theme.css b/src/platform/web/ui/css/themes/element/theme.css index 113ea254..a0ad1ef0 100644 --- a/src/platform/web/ui/css/themes/element/theme.css +++ b/src/platform/web/ui/css/themes/element/theme.css @@ -1155,3 +1155,55 @@ button.RoomDetailsView_row::after { background-position: center; background-size: 36px; } + +.CallView { + max-height: 50vh; + overflow-y: auto; +} + +.CallView ul { + display: flex; + margin: 0; + gap: 12px; + padding: 0; + flex-wrap: wrap; + justify-content: center; +} + +.StreamView { + width: 360px; + min-height: 200px; + border: 2px var(--accent-color) solid; + display: grid; + border-radius: 8px; + overflow: hidden; + background-color: black; +} + +.StreamView > * { + grid-column: 1; + grid-row: 1; +} + +.StreamView video { + width: 100%; +} + +.StreamView_avatar { + align-self: center; + justify-self: center; +} + +.StreamView_muteStatus { + align-self: end; + justify-self: start; +} + +.StreamView_muteStatus.microphoneMuted::before { + content: "mic muted"; +} + +.StreamView_muteStatus.cameraMuted::before { + content: "cam muted"; +} + diff --git a/src/platform/web/ui/general/TemplateView.ts b/src/platform/web/ui/general/TemplateView.ts index ce593f75..3b65ed9c 100644 --- a/src/platform/web/ui/general/TemplateView.ts +++ b/src/platform/web/ui/general/TemplateView.ts @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { setAttribute, text, isChildren, classNames, TAG_NAMES, HTML_NS, ClassNames, Child} from "./html"; +import { setAttribute, text, isChildren, classNames, TAG_NAMES, HTML_NS, ClassNames, Child as NonBoundChild} from "./html"; import {mountView} from "./utils"; import {BaseUpdateView, IObservableValue} from "./BaseUpdateView"; import {IMountArgs, ViewNode, IView} from "./types"; @@ -30,12 +30,15 @@ function objHasFns(obj: ClassNames): obj is { [className: string]: bool } export type RenderFn = (t: Builder, vm: T) => ViewNode; +type TextBinding = (T) => string | number | boolean | undefined | null; +type Child = NonBoundChild | TextBinding; +type Children = Child | Child[]; type EventHandler = ((event: Event) => void); type AttributeStaticValue = string | boolean; type AttributeBinding = (value: T) => AttributeStaticValue; export type AttrValue = AttributeStaticValue | AttributeBinding | EventHandler | ClassNames; export type Attributes = { [attribute: string]: AttrValue }; -type ElementFn = (attributes?: Attributes | Child | Child[], children?: Child | Child[]) => Element; +type ElementFn = (attributes?: Attributes | Children, children?: Children) => Element; export type Builder = TemplateBuilder & { [tagName in typeof TAG_NAMES[string][number]]: ElementFn }; /** @@ -178,7 +181,7 @@ export class TemplateBuilder { this._templateView._addEventListener(node, name, fn, useCapture); } - _addAttributeBinding(node: Element, name: string, fn: (value: T) => boolean | string): void { + _addAttributeBinding(node: Element, name: string, fn: AttributeBinding): void { let prevValue: string | boolean | undefined = undefined; const binding = () => { const newValue = fn(this._value); @@ -195,15 +198,15 @@ export class TemplateBuilder { this._addAttributeBinding(node, "className", value => classNames(obj, value)); } - _addTextBinding(fn: (value: T) => string): Text { - const initialValue = fn(this._value); + _addTextBinding(fn: (value: T) => ReturnType>): Text { + const initialValue = fn(this._value)+""; const node = text(initialValue); let prevValue = initialValue; const binding = () => { - const newValue = fn(this._value); + const newValue = fn(this._value)+""; if (prevValue !== newValue) { prevValue = newValue; - node.textContent = newValue+""; + node.textContent = newValue; } }; @@ -242,7 +245,7 @@ export class TemplateBuilder { } } - _setNodeChildren(node: Element, children: Child | Child[]): void{ + _setNodeChildren(node: Element, children: Children): void{ if (!Array.isArray(children)) { children = [children]; } @@ -276,14 +279,18 @@ export class TemplateBuilder { return node; } - el(name: string, attributes?: Attributes | Child | Child[], children?: Child | Child[]): ViewNode { + el(name: string, attributes?: Attributes | Children, children?: Children): ViewNode { return this.elNS(HTML_NS, name, attributes, children); } - elNS(ns: string, name: string, attributes?: Attributes | Child | Child[], children?: Child | Child[]): ViewNode { - if (attributes !== undefined && isChildren(attributes)) { - children = attributes; - attributes = undefined; + elNS(ns: string, name: string, attributesOrChildren?: Attributes | Children, children?: Children): ViewNode { + let attributes: Attributes | undefined; + if (attributesOrChildren) { + if (isChildren(attributesOrChildren)) { + children = attributesOrChildren as Children; + } else { + attributes = attributesOrChildren as Attributes; + } } const node = document.createElementNS(ns, name); @@ -330,7 +337,7 @@ export class TemplateBuilder { // Special case of mapView for a TemplateView. // Always creates a TemplateView, if this is optional depending // on mappedValue, use `if` or `mapView` - map(mapFn: (value: T) => R, renderFn: (mapped: R, t: Builder, vm: T) => ViewNode): ViewNode { + map(mapFn: (value: T) => R, renderFn: (mapped: R, t: Builder, vm: T) => ViewNode | undefined): ViewNode { return this.mapView(mapFn, mappedValue => { return new InlineTemplateView(this._value, (t, vm) => { const rootNode = renderFn(mappedValue, t, vm); @@ -364,17 +371,17 @@ export class TemplateBuilder { event handlers, ... You should not call the TemplateBuilder (e.g. `t.xxx()`) at all from the side effect, instead use tags from html.ts to help you construct any DOM you need. */ - mapSideEffect(mapFn: (value: T) => R, sideEffect: (newV: R, oldV: R | undefined) => void) { + mapSideEffect(mapFn: (value: T) => R, sideEffect: (newV: R, oldV: R | undefined, value: T) => void) { let prevValue = mapFn(this._value); const binding = () => { const newValue = mapFn(this._value); if (prevValue !== newValue) { - sideEffect(newValue, prevValue); + sideEffect(newValue, prevValue, this._value); prevValue = newValue; } }; this._addBinding(binding); - sideEffect(prevValue, undefined); + sideEffect(prevValue, undefined, this._value); } } diff --git a/src/platform/web/ui/session/room/CallView.ts b/src/platform/web/ui/session/room/CallView.ts index 115e6035..92dfe6b9 100644 --- a/src/platform/web/ui/session/room/CallView.ts +++ b/src/platform/web/ui/session/room/CallView.ts @@ -14,36 +14,50 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {TemplateView, TemplateBuilder} from "../../general/TemplateView"; +import {TemplateView, Builder} from "../../general/TemplateView"; +import {AvatarView} from "../../AvatarView"; import {ListView} from "../../general/ListView"; import {Stream} from "../../../../types/MediaDevices"; -import type {StreamWrapper} from "../../../dom/MediaDevices"; -import type {CallViewModel, CallMemberViewModel} from "../../../../../domain/session/room/CallViewModel"; - -function bindVideoTracks(t: TemplateBuilder, video: HTMLVideoElement, propSelector: (vm: T) => Stream | undefined) { - t.mapSideEffect(propSelector, stream => { - if (stream) { - video.srcObject = (stream as StreamWrapper).stream; - } - }); - return video; -} +import type {CallViewModel, CallMemberViewModel, IStreamViewModel} from "../../../../../domain/session/room/CallViewModel"; export class CallView extends TemplateView { - render(t: TemplateBuilder, vm: CallViewModel): HTMLElement { + render(t: Builder, vm: CallViewModel): Element { return t.div({class: "CallView"}, [ t.p(vm => `Call ${vm.name} (${vm.id})`), - t.div({class: "CallView_me"}, bindVideoTracks(t, t.video({autoplay: true, width: 240}), vm => vm.localStream)), - t.view(new ListView({list: vm.memberViewModels}, vm => new MemberView(vm))), + t.view(new ListView({list: vm.memberViewModels}, vm => new StreamView(vm))), t.div({class: "buttons"}, [ - t.button({onClick: () => vm.leave()}, "Leave") + t.button({onClick: () => vm.leave()}, "Leave"), + t.button({onClick: () => vm.toggleVideo()}, "Toggle video"), ]) ]); } } -class MemberView extends TemplateView { - render(t: TemplateBuilder, vm: CallMemberViewModel) { - return bindVideoTracks(t, t.video({autoplay: true, width: 360}), vm => vm.stream); +class StreamView extends TemplateView { + render(t: Builder, vm: IStreamViewModel): Element { + const video = t.video({ + autoplay: true, + className: { + hidden: vm => vm.isCameraMuted + } + }) as HTMLVideoElement; + t.mapSideEffect(vm => vm.stream, stream => { + video.srcObject = stream as MediaStream; + }); + return t.div({className: "StreamView"}, [ + video, + t.div({className: { + StreamView_avatar: true, + hidden: vm => !vm.isCameraMuted + }}, t.view(new AvatarView(vm, 64), {parentProvidesUpdates: true})), + t.div({ + className: { + StreamView_muteStatus: true, + hidden: vm => !vm.isCameraMuted && !vm.isMicrophoneMuted, + microphoneMuted: vm => vm.isMicrophoneMuted && !vm.isCameraMuted, + cameraMuted: vm => vm.isCameraMuted, + } + }) + ]); } } diff --git a/src/utils/AsyncQueue.ts b/src/utils/AsyncQueue.ts deleted file mode 100644 index 0686314c..00000000 --- a/src/utils/AsyncQueue.ts +++ /dev/null @@ -1,52 +0,0 @@ -/* -Copyright 2022 The Matrix.org Foundation C.I.C. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -export class AsyncQueue { - private isRunning = false; - private queue: T[] = []; - private error?: Error; - - constructor( - private readonly reducer: (v: V, t: T) => Promise, - private value: V, - private readonly contains: (t: T, queue: T[]) => boolean = (t, queue) => queue.includes(t) - ) {} - - push(t: T) { - if (this.contains(t, this.queue)) { - return; - } - this.queue.push(t); - this.runLoopIfNeeded(); - } - - private async runLoopIfNeeded() { - if (this.isRunning || this.error) { - return; - } - this.isRunning = true; - try { - let item: T | undefined; - while (item = this.queue.shift()) { - this.value = await this.reducer(this.value, item); - } - } catch (err) { - this.error = err; - } finally { - this.isRunning = false; - } - } -} diff --git a/vite.common-config.js b/vite.common-config.js index f5a90154..8a82a9da 100644 --- a/vite.common-config.js +++ b/vite.common-config.js @@ -31,6 +31,7 @@ const commonOptions = { assetsInlineLimit: 0, polyfillModulePreload: false, }, + assetsInclude: ['**/config.json'], define: { DEFINE_VERSION: JSON.stringify(version), DEFINE_GLOBAL_HASH: JSON.stringify(null), diff --git a/vite.config.js b/vite.config.js index 4dd35af2..87e3d063 100644 --- a/vite.config.js +++ b/vite.config.js @@ -14,6 +14,11 @@ export default defineConfig(({mode}) => { outDir: "../../../target", minify: true, sourcemap: true, + rollupOptions: { + output: { + assetFileNames: (asset) => asset.name.includes("config.json") ? "assets/[name][extname]": "assets/[name].[hash][extname]", + }, + }, }, plugins: [ themeBuilder({