more improvements, make hangup work

This commit is contained in:
Bruno Windels 2022-03-24 13:52:19 +01:00
parent 0a37fd561e
commit a0a07355d4
16 changed files with 182 additions and 56 deletions

View File

@ -33,16 +33,26 @@ export class CallViewModel extends ViewModel<Options> {
.sortValues((a, b) => a.compare(b)); .sortValues((a, b) => a.compare(b));
} }
private get call(): GroupCall {
return this.getOption("call");
}
get name(): string { get name(): string {
return this.getOption("call").name; return this.call.name;
} }
get id(): string { get id(): string {
return this.getOption("call").id; return this.call.id;
} }
get localTracks(): Track[] { get localTracks(): Track[] {
return this.getOption("call").localMedia?.tracks ?? []; return this.call.localMedia?.tracks ?? [];
}
leave() {
if (this.call.hasJoined) {
this.call.leave();
}
} }
} }
@ -50,12 +60,16 @@ type MemberOptions = BaseOptions & {member: Member};
export class CallMemberViewModel extends ViewModel<MemberOptions> { export class CallMemberViewModel extends ViewModel<MemberOptions> {
get tracks(): Track[] { get tracks(): Track[] {
return this.getOption("member").remoteTracks; return this.member.remoteTracks;
}
private get member(): Member {
return this.getOption("member");
} }
compare(other: CallMemberViewModel): number { compare(other: CallMemberViewModel): number {
const myUserId = this.getOption("member").member.userId; const myUserId = this.member.member.userId;
const otherUserId = other.getOption("member").member.userId; const otherUserId = other.member.member.userId;
if(myUserId === otherUserId) { if(myUserId === otherUserId) {
return 0; return 0;
} }

View File

@ -49,12 +49,17 @@ export class RoomViewModel extends ViewModel {
_setupCallViewModel() { _setupCallViewModel() {
// pick call for this room with lowest key // pick call for this room with lowest key
const calls = this.getOption("session").callHandler.calls; const calls = this.getOption("session").callHandler.calls;
this._callObservable = new PickMapObservableValue(calls.filterValues(c => c.roomId === this._room.id && c.hasJoined)); this._callObservable = new PickMapObservableValue(calls.filterValues(c => {
return c.roomId === this._room.id && c.hasJoined;
}));
this._callViewModel = undefined; this._callViewModel = undefined;
this.track(this._callObservable.subscribe(call => { this.track(this._callObservable.subscribe(call => {
if (call && this._callViewModel && call.id === this._callViewModel.id) {
return;
}
this._callViewModel = this.disposeTracked(this._callViewModel); this._callViewModel = this.disposeTracked(this._callViewModel);
if (call) { if (call) {
this._callViewModel = new CallViewModel(this.childOptions({call})); this._callViewModel = this.track(new CallViewModel(this.childOptions({call})));
} }
this.emitChange("callViewModel"); this.emitChange("callViewModel");
})); }));

View File

@ -49,14 +49,6 @@ export class BaseMessageTile extends SimpleTile {
return `https://matrix.to/#/${encodeURIComponent(this.sender)}`; return `https://matrix.to/#/${encodeURIComponent(this.sender)}`;
} }
get displayName() {
return this._entry.displayName || this.sender;
}
get sender() {
return this._entry.sender;
}
get memberPanelLink() { get memberPanelLink() {
return `${this.urlCreator.urlUntilSegment("room")}/member/${this.sender}`; return `${this.urlCreator.urlUntilSegment("room")}/member/${this.sender}`;
} }

View File

@ -23,6 +23,27 @@ import {LocalMedia} from "../../../../../matrix/calls/LocalMedia";
// alternatively, we could just subscribe to the GroupCall and spontanously emit an update when it updates // alternatively, we could just subscribe to the GroupCall and spontanously emit an update when it updates
export class CallTile extends SimpleTile { export class CallTile extends SimpleTile {
constructor(options) {
super(options);
const calls = this.getOption("session").callHandler.calls;
this._call = calls.get(this._entry.stateKey);
this._callSubscription = undefined;
if (this._call) {
this._callSubscription = this._call.disposableOn("change", () => {
// unsubscribe when terminated
if (this._call.isTerminated) {
this._callSubscription = this._callSubscription();
this._call = undefined;
}
this.emitChange();
});
}
}
get confId() {
return this._entry.stateKey;
}
get shape() { get shape() {
return "call"; return "call";
@ -32,17 +53,43 @@ export class CallTile extends SimpleTile {
return this._entry.content["m.name"]; return this._entry.content["m.name"];
} }
get _call() { get canJoin() {
const calls = this.getOption("session").callHandler.calls; return this._call && !this._call.hasJoined;
return calls.get(this._entry.stateKey); }
get canLeave() {
return this._call && this._call.hasJoined;
}
get label() {
if (this._call) {
if (this._call.hasJoined) {
return `Ongoing call (${this.name}, ${this.confId})`;
} else {
return `${this.displayName} started a call (${this.name}, ${this.confId})`;
}
} else {
return `Call finished, started by ${this.displayName} (${this.name}, ${this.confId})`;
}
} }
async join() { async join() {
const call = this._call; if (this.canJoin) {
if (call) {
const mediaTracks = await this.platform.mediaDevices.getMediaTracks(false, true); const mediaTracks = await this.platform.mediaDevices.getMediaTracks(false, true);
const localMedia = new LocalMedia().withTracks(mediaTracks); const localMedia = new LocalMedia().withTracks(mediaTracks);
await call.join(localMedia); await this._call.join(localMedia);
}
}
async leave() {
if (this.canLeave) {
this._call.leave();
}
}
dispose() {
if (this._callSubscription) {
this._callSubscription = this._callSubscription();
} }
} }
} }

View File

@ -154,4 +154,12 @@ export class SimpleTile extends ViewModel {
get _ownMember() { get _ownMember() {
return this._options.timeline.me; return this._options.timeline.me;
} }
get displayName() {
return this._entry.displayName || this.sender;
}
get sender() {
return this._entry.sender;
}
} }

View File

@ -73,7 +73,9 @@ export function tilesCreator(baseOptions) {
case "m.room.encryption": case "m.room.encryption":
return new EncryptionEnabledTile(options); return new EncryptionEnabledTile(options);
case "m.call": case "m.call":
return entry.stateKey ? new CallTile(options) : null; // if prevContent is present, it's an update to a call event, which we don't render
// as the original event is updated through the call object which receive state event updates
return entry.stateKey && !entry.prevContent ? new CallTile(options) : null;
default: default:
// unknown type not rendered // unknown type not rendered
return null; return null;

View File

@ -86,7 +86,6 @@ export class Session {
// although we probably already fetched all devices to send messages in the likely e2ee room // although we probably already fetched all devices to send messages in the likely e2ee room
await this._deviceTracker.trackRoom(this.rooms.get(roomId), log); await this._deviceTracker.trackRoom(this.rooms.get(roomId), log);
const devices = await this._deviceTracker.devicesForRoomMembers(roomId, [userId], this._hsApi, log); const devices = await this._deviceTracker.devicesForRoomMembers(roomId, [userId], this._hsApi, log);
console.log("devices", devices);
const encryptedMessage = await this._olmEncryption.encrypt(message.type, message.content, devices, this._hsApi, log); const encryptedMessage = await this._olmEncryption.encrypt(message.type, message.content, devices, this._hsApi, log);
return encryptedMessage; return encryptedMessage;
}, },

View File

@ -112,7 +112,7 @@ export class CallHandler {
const callId = event.state_key; const callId = event.state_key;
let call = this._calls.get(callId); let call = this._calls.get(callId);
if (call) { if (call) {
call.updateCallEvent(event); call.updateCallEvent(event.content);
if (call.isTerminated) { if (call.isTerminated) {
this._calls.remove(call.id); this._calls.remove(call.id);
} }
@ -125,13 +125,13 @@ export class CallHandler {
private handleCallMemberEvent(event: StateEvent) { private handleCallMemberEvent(event: StateEvent) {
const userId = event.state_key; const userId = event.state_key;
const calls = event.content["m.calls"] ?? []; const calls = event.content["m.calls"] ?? [];
const newCallIdsMemberOf = new Set<string>(calls.map(call => { for (const call of calls) {
const callId = call["m.call_id"]; const callId = call["m.call_id"];
const groupCall = this._calls.get(callId); const groupCall = this._calls.get(callId);
// TODO: also check the member when receiving the m.call event // TODO: also check the member when receiving the m.call event
groupCall?.addMember(userId, call); groupCall?.addMember(userId, call);
return callId; };
})); const newCallIdsMemberOf = new Set<string>(calls.map(call => call["m.call_id"]));
let previousCallIdsMemberOf = this.memberToCallIds.get(userId); let previousCallIdsMemberOf = this.memberToCallIds.get(userId);
// remove user as member of any calls not present anymore // remove user as member of any calls not present anymore
if (previousCallIdsMemberOf) { if (previousCallIdsMemberOf) {

View File

@ -60,4 +60,10 @@ export class LocalMedia {
} }
return metadata; return metadata;
} }
dispose() {
this.cameraTrack?.stop();
this.microphoneTrack?.stop();
this.screenShareTrack?.stop();
}
} }

View File

@ -748,6 +748,10 @@ export class PeerCall implements IDisposable {
this.disposables.dispose(); this.disposables.dispose();
this.peerConnection.dispose(); this.peerConnection.dispose();
} }
public close(): void {
this.peerConnection.close();
}
} }

View File

@ -19,6 +19,7 @@ import {Member} from "./Member";
import {LocalMedia} from "../LocalMedia"; import {LocalMedia} from "../LocalMedia";
import {RoomMember} from "../../room/members/RoomMember"; import {RoomMember} from "../../room/members/RoomMember";
import {makeId} from "../../common"; import {makeId} from "../../common";
import {EventEmitter} from "../../../utils/EventEmitter";
import type {Options as MemberOptions} from "./Member"; import type {Options as MemberOptions} from "./Member";
import type {BaseObservableMap} from "../../../observable/map/BaseObservableMap"; import type {BaseObservableMap} from "../../../observable/map/BaseObservableMap";
@ -31,6 +32,9 @@ import type {EncryptedMessage} from "../../e2ee/olm/Encryption";
import type {ILogItem} from "../../../logging/types"; import type {ILogItem} from "../../../logging/types";
import type {Storage} from "../../storage/idb/Storage"; import type {Storage} from "../../storage/idb/Storage";
const CALL_TYPE = "m.call";
const CALL_MEMBER_TYPE = "m.call.member";
export enum GroupCallState { export enum GroupCallState {
Fledgling = "fledgling", Fledgling = "fledgling",
Creating = "creating", Creating = "creating",
@ -46,7 +50,7 @@ export type Options = Omit<MemberOptions, "emitUpdate" | "confId" | "encryptDevi
ownDeviceId: string ownDeviceId: string
}; };
export class GroupCall { export class GroupCall extends EventEmitter<{change: never}> {
public readonly id: string; public readonly id: string;
private readonly _members: ObservableMap<string, Member> = new ObservableMap(); private readonly _members: ObservableMap<string, Member> = new ObservableMap();
private _localMedia?: LocalMedia = undefined; private _localMedia?: LocalMedia = undefined;
@ -59,6 +63,7 @@ export class GroupCall {
public readonly roomId: string, public readonly roomId: string,
private readonly options: Options private readonly options: Options
) { ) {
super();
this.id = id ?? makeId("conf-"); this.id = id ?? makeId("conf-");
this._state = id ? GroupCallState.Created : GroupCallState.Fledgling; this._state = id ? GroupCallState.Created : GroupCallState.Fledgling;
this._memberOptions = Object.assign({}, options, { this._memberOptions = Object.assign({}, options, {
@ -87,12 +92,12 @@ export class GroupCall {
} }
this._state = GroupCallState.Joining; this._state = GroupCallState.Joining;
this._localMedia = localMedia; this._localMedia = localMedia;
this.options.emitUpdate(this); this.emitChange();
const memberContent = await this._joinCallMemberContent(); const memberContent = await this._createJoinPayload();
// send m.call.member state event // send m.call.member state event
const request = this.options.hsApi.sendState(this.roomId, "m.call.member", this.options.ownUserId, memberContent); const request = this.options.hsApi.sendState(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId, memberContent);
await request.response(); await request.response();
this.options.emitUpdate(this); this.emitChange();
// send invite to all members that are < my userId // send invite to all members that are < my userId
for (const [,member] of this._members) { for (const [,member] of this._members) {
member.connect(this._localMedia); member.connect(this._localMedia);
@ -107,25 +112,41 @@ export class GroupCall {
const memberContent = await this._leaveCallMemberContent(); const memberContent = await this._leaveCallMemberContent();
// send m.call.member state event // send m.call.member state event
if (memberContent) { if (memberContent) {
const request = this.options.hsApi.sendState(this.roomId, "m.call.member", this.options.ownUserId, memberContent); const request = this.options.hsApi.sendState(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId, memberContent);
await request.response(); await request.response();
// our own user isn't included in members, so not in the count
if (this._members.size === 0) {
this.terminate();
}
} }
} }
async terminate() {
if (this._state === GroupCallState.Fledgling) {
return;
}
const request = this.options.hsApi.sendState(this.roomId, CALL_TYPE, this.id, Object.assign({}, this.callContent, {
"m.terminated": true
}));
await request.response();
}
/** @internal */ /** @internal */
async create(localMedia: LocalMedia, name: string) { async create(localMedia: LocalMedia, name: string) {
if (this._state !== GroupCallState.Fledgling) { if (this._state !== GroupCallState.Fledgling) {
return; return;
} }
this._state = GroupCallState.Creating; this._state = GroupCallState.Creating;
this.emitChange();
this.callContent = { this.callContent = {
"m.type": localMedia.cameraTrack ? "m.video" : "m.voice", "m.type": localMedia.cameraTrack ? "m.video" : "m.voice",
"m.name": name, "m.name": name,
"m.intent": "m.ring" "m.intent": "m.ring"
}; };
const request = this.options.hsApi.sendState(this.roomId, "m.call", this.id, this.callContent); const request = this.options.hsApi.sendState(this.roomId, CALL_TYPE, this.id, this.callContent);
await request.response(); await request.response();
this._state = GroupCallState.Created; this._state = GroupCallState.Created;
this.emitChange();
} }
/** @internal */ /** @internal */
@ -134,6 +155,7 @@ export class GroupCall {
if (this._state === GroupCallState.Creating) { if (this._state === GroupCallState.Creating) {
this._state = GroupCallState.Created; this._state = GroupCallState.Created;
} }
this.emitChange();
} }
/** @internal */ /** @internal */
@ -141,6 +163,7 @@ export class GroupCall {
if (userId === this.options.ownUserId) { if (userId === this.options.ownUserId) {
if (this._state === GroupCallState.Joining) { if (this._state === GroupCallState.Joining) {
this._state = GroupCallState.Joined; this._state = GroupCallState.Joined;
this.emitChange();
} }
return; return;
} }
@ -160,11 +183,21 @@ export class GroupCall {
removeMember(userId) { removeMember(userId) {
if (userId === this.options.ownUserId) { if (userId === this.options.ownUserId) {
if (this._state === GroupCallState.Joined) { if (this._state === GroupCallState.Joined) {
this._localMedia?.dispose();
this._localMedia = undefined;
for (const [,member] of this._members) {
member.disconnect();
}
this._state = GroupCallState.Created; this._state = GroupCallState.Created;
} }
return; } else {
const member = this._members.get(userId);
if (member) {
this._members.remove(userId);
member.disconnect();
}
} }
this._members.remove(userId); this.emitChange();
} }
/** @internal */ /** @internal */
@ -179,10 +212,10 @@ export class GroupCall {
} }
} }
private async _joinCallMemberContent() { private async _createJoinPayload() {
const {storage} = this.options; const {storage} = this.options;
const txn = await storage.readTxn([storage.storeNames.roomState]); const txn = await storage.readTxn([storage.storeNames.roomState]);
const stateEvent = await txn.roomState.get(this.roomId, "m.call.member", this.options.ownUserId); const stateEvent = await txn.roomState.get(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId);
const stateContent = stateEvent?.event?.content ?? { const stateContent = stateEvent?.event?.content ?? {
["m.calls"]: [] ["m.calls"]: []
}; };
@ -209,9 +242,18 @@ export class GroupCall {
private async _leaveCallMemberContent(): Promise<Record<string, any> | undefined> { private async _leaveCallMemberContent(): Promise<Record<string, any> | undefined> {
const {storage} = this.options; const {storage} = this.options;
const txn = await storage.readTxn([storage.storeNames.roomState]); const txn = await storage.readTxn([storage.storeNames.roomState]);
const stateEvent = await txn.roomState.get(this.roomId, "m.call.member", this.options.ownUserId); const stateEvent = await txn.roomState.get(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId);
const callsInfo = stateEvent?.event?.content?.["m.calls"]; if (stateEvent) {
callsInfo?.filter(c => c["m.call_id"] === this.id); const content = stateEvent.event.content;
return stateEvent?.event.content; const callsInfo = content["m.calls"];
content["m.calls"] = callsInfo?.filter(c => c["m.call_id"] !== this.id);
return content;
}
}
protected emitChange() {
this.emit("change");
this.options.emitUpdate(this);
} }
} }

View File

@ -65,6 +65,14 @@ export class Member {
} }
} }
/** @internal */
disconnect() {
this.peerCall?.close();
this.peerCall?.dispose();
this.peerCall = undefined;
this.localMedia = undefined;
}
/** @internal */ /** @internal */
updateCallInfo(memberCallInfo) { updateCallInfo(memberCallInfo) {
// m.calls object from the m.call.member event // m.calls object from the m.call.member event

View File

@ -451,7 +451,7 @@ export class Room extends BaseRoom {
_updateCallHandler(roomResponse, log) { _updateCallHandler(roomResponse, log) {
if (this._callHandler) { if (this._callHandler) {
const stateEvents = roomResponse.state?.events; const stateEvents = roomResponse.state?.events;
if (stateEvents) { if (stateEvents?.length) {
this._callHandler.handleRoomState(this, stateEvents, log); this._callHandler.handleRoomState(this, stateEvents, log);
} }
let timelineEvents = roomResponse.timeline?.events; let timelineEvents = roomResponse.timeline?.events;

View File

@ -60,13 +60,11 @@ export class PickMapObservableValue<K, V> extends BaseObservableValue<V | undefi
onRemove(key: K, value: V): void { onRemove(key: K, value: V): void {
if (key === this.key) { if (key === this.key) {
this.key = undefined; this.key = undefined;
let changed = false; // try to see if there is another key that fullfills pickKey
for (const [key] of this.map) { for (const [key] of this.map) {
changed = this.updateKey(key) || changed; this.updateKey(key) || changed;
}
if (changed) {
this.emit(this.get());
} }
this.emit(this.get());
} }
} }

View File

@ -22,7 +22,6 @@ import type {CallViewModel, CallMemberViewModel} from "../../../../../domain/ses
function bindVideoTracks<T>(t: TemplateBuilder<T>, video: HTMLVideoElement, propSelector: (vm: T) => Track[]) { function bindVideoTracks<T>(t: TemplateBuilder<T>, video: HTMLVideoElement, propSelector: (vm: T) => Track[]) {
t.mapSideEffect(propSelector, tracks => { t.mapSideEffect(propSelector, tracks => {
console.log("tracks", tracks);
if (tracks.length) { if (tracks.length) {
video.srcObject = (tracks[0] as TrackWrapper).stream; video.srcObject = (tracks[0] as TrackWrapper).stream;
} }
@ -33,8 +32,8 @@ function bindVideoTracks<T>(t: TemplateBuilder<T>, video: HTMLVideoElement, prop
export class CallView extends TemplateView<CallViewModel> { export class CallView extends TemplateView<CallViewModel> {
render(t: TemplateBuilder<CallViewModel>, vm: CallViewModel): HTMLElement { render(t: TemplateBuilder<CallViewModel>, vm: CallViewModel): HTMLElement {
return t.div({class: "CallView"}, [ return t.div({class: "CallView"}, [
t.p(["Call ", vm => vm.name, vm => ` (${vm.id})`]), t.p(vm => `Call ${vm.name} (${vm.id})`),
t.div({class: "CallView_me"}, bindVideoTracks(t, t.video({autoplay: true}), vm => vm.localTracks)), t.div({class: "CallView_me"}, bindVideoTracks(t, t.video({autoplay: true, width: 240}), vm => vm.localTracks)),
t.view(new ListView({list: vm.memberViewModels}, vm => new MemberView(vm))) t.view(new ListView({list: vm.memberViewModels}, vm => new MemberView(vm)))
]); ]);
} }
@ -42,6 +41,6 @@ export class CallView extends TemplateView<CallViewModel> {
class MemberView extends TemplateView<CallMemberViewModel> { class MemberView extends TemplateView<CallMemberViewModel> {
render(t: TemplateBuilder<CallMemberViewModel>, vm: CallMemberViewModel) { render(t: TemplateBuilder<CallMemberViewModel>, vm: CallMemberViewModel) {
return bindVideoTracks(t, t.video({autoplay: true}), vm => vm.tracks); return bindVideoTracks(t, t.video({autoplay: true, width: 360}), vm => vm.tracks);
} }
} }

View File

@ -22,9 +22,9 @@ export class CallTileView extends TemplateView<CallTile> {
return t.li( return t.li(
{className: "AnnouncementView"}, {className: "AnnouncementView"},
t.div([ t.div([
"Call ", vm => vm.label,
vm => vm.name, t.button({className: "CallTileView_join", hidden: vm => !vm.canJoin}, "Join"),
t.button({className: "CallTileView_join"}, "Join") t.button({className: "CallTileView_leave", hidden: vm => !vm.canLeave}, "Leave")
]) ])
); );
} }
@ -33,6 +33,8 @@ export class CallTileView extends TemplateView<CallTile> {
onClick(evt) { onClick(evt) {
if (evt.target.className === "CallTileView_join") { if (evt.target.className === "CallTileView_join") {
this.value.join(); this.value.join();
} else if (evt.target.className === "CallTileView_leave") {
this.value.leave();
} }
} }
} }