Merge pull request #82 from vector-im/bwindels/megolm-encrypt

Implement megolm encryption

src/matrix/SendScheduler.js

@@ -121,7 +121,7 @@ export class SendScheduler {
                     }
                     this._sendRequests = [];
                 }
-                console.error("error for request", request);
+                console.error("error for request", err);
                 request.reject(err);
                 break;
             }

src/matrix/Session.js

@@ -18,11 +18,14 @@ import {Room} from "./room/Room.js";
 import { ObservableMap } from "../observable/index.js";
 import { SendScheduler, RateLimitingBackoff } from "./SendScheduler.js";
 import {User} from "./User.js";
-import {Account as E2EEAccount} from "./e2ee/Account.js";
 import {DeviceMessageHandler} from "./DeviceMessageHandler.js";
+import {Account as E2EEAccount} from "./e2ee/Account.js";
 import {Decryption as OlmDecryption} from "./e2ee/olm/Decryption.js";
 import {Encryption as OlmEncryption} from "./e2ee/olm/Encryption.js";
 import {Decryption as MegOlmDecryption} from "./e2ee/megolm/Decryption.js";
+import {Encryption as MegOlmEncryption} from "./e2ee/megolm/Encryption.js";
+import {MEGOLM_ALGORITHM} from "./e2ee/common.js";
+import {RoomEncryption} from "./e2ee/RoomEncryption.js";
 import {DeviceTracker} from "./e2ee/DeviceTracker.js";
 import {LockMap} from "../utils/LockMap.js";
 
@@ -56,6 +59,7 @@ export class Session {
                 ownDeviceId: sessionInfo.deviceId,
             });
         }
+        this._createRoomEncryption = this._createRoomEncryption.bind(this);
     }
 
     // called once this._e2eeAccount is assigned
@@ -65,26 +69,59 @@ export class Session {
         const olmDecryption = new OlmDecryption({
             account: this._e2eeAccount,
             pickleKey: PICKLE_KEY,
+            olm: this._olm,
+            storage: this._storage,
             now: this._clock.now,
             ownUserId: this._user.id,
-            storage: this._storage,
-            olm: this._olm,
             senderKeyLock
         });
         this._olmEncryption = new OlmEncryption({
             account: this._e2eeAccount,
             pickleKey: PICKLE_KEY,
+            olm: this._olm,
+            storage: this._storage,
             now: this._clock.now,
             ownUserId: this._user.id,
-            storage: this._storage,
-            olm: this._olm,
             olmUtil: this._olmUtil,
             senderKeyLock
         });
+        this._megolmEncryption = new MegOlmEncryption({
+            account: this._e2eeAccount,
+            pickleKey: PICKLE_KEY,
+            olm: this._olm,
+            storage: this._storage,
+            now: this._clock.now,
+            ownDeviceId: this._sessionInfo.deviceId,
+        })
         const megolmDecryption = new MegOlmDecryption({pickleKey: PICKLE_KEY, olm: this._olm});
         this._deviceMessageHandler.enableEncryption({olmDecryption, megolmDecryption});
     }
 
+    _createRoomEncryption(room, encryptionParams) {
+        // TODO: this will actually happen when users start using the e2ee version for the first time
+
+        // this should never happen because either a session was already synced once
+        // and thus an e2ee account was created as well and _setupEncryption is called from load
+        // OR
+        // this is a new session and loading it will load zero rooms, thus not calling this method.
+        // in this case _setupEncryption is called from beforeFirstSync, right after load,
+        // so any incoming synced rooms won't be there yet
+        if (!this._olmEncryption) {
+            throw new Error("creating room encryption before encryption got globally enabled");
+        }
+        // only support megolm
+        if (encryptionParams.algorithm !== MEGOLM_ALGORITHM) {
+            return null;
+        }
+        return new RoomEncryption({
+            room,
+            deviceTracker: this._deviceTracker,
+            olmEncryption: this._olmEncryption,
+            megolmEncryption: this._megolmEncryption,
+            encryptionParams
+        });
+    }
+
     // called after load
     async beforeFirstSync(isNewLogin) {
         if (this._olm) {
@@ -202,6 +239,7 @@ export class Session {
             sendScheduler: this._sendScheduler,
             pendingEvents,
             user: this._user,
+            createRoomEncryption: this._createRoomEncryption
         });
         this._rooms.add(roomId, room);
         return room;
@@ -222,12 +260,6 @@ export class Session {
             changes.syncInfo = syncInfo;
         }
         if (this._deviceTracker) {
-            for (const {room, changes} of roomChanges) {
-                // TODO: move this so the room passes this to it's "encryption" object in its own writeSync method?
-                if (room.isTrackingMembers && changes.memberChanges?.size) {
-                    await this._deviceTracker.writeMemberChanges(room, changes.memberChanges, txn);
-                }
-            } 
             const deviceLists = syncResponse.device_lists;
             if (deviceLists) {
                 await this._deviceTracker.writeDeviceChanges(deviceLists, txn);

src/matrix/common.js

@@ -0,0 +1,22 @@
+/*
+Copyright 2020 Bruno Windels <bruno@windels.cloud>
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export function makeTxnId() {
+    const n = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER);
+    const str = n.toString(16);
+    return "t" + "0".repeat(14 - str.length) + str;
+}

src/matrix/e2ee/RoomEncryption.js

@@ -0,0 +1,66 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {groupBy} from "../../utils/groupBy.js";
+import {makeTxnId} from "../common.js";
+
+const ENCRYPTED_TYPE = "m.room.encrypted";
+
+export class RoomEncryption {
+    constructor({room, deviceTracker, olmEncryption, megolmEncryption, encryptionParams}) {
+        this._room = room;
+        this._deviceTracker = deviceTracker;
+        this._olmEncryption = olmEncryption;
+        this._megolmEncryption = megolmEncryption;
+        // content of the m.room.encryption event
+        this._encryptionParams = encryptionParams;
+    }
+
+    async writeMemberChanges(memberChanges, txn) {
+        return await this._deviceTracker.writeMemberChanges(this._room, memberChanges, txn);
+    }
+
+    async encrypt(type, content, hsApi) {
+        const megolmResult = await this._megolmEncryption.encrypt(this._room.id, type, content, this._encryptionParams);
+        // share the new megolm session if needed
+        if (megolmResult.roomKeyMessage) {
+            await this._deviceTracker.trackRoom(this._room);
+            const devices = await this._deviceTracker.deviceIdentitiesForTrackedRoom(this._room.id, hsApi);
+            const messages = await this._olmEncryption.encrypt(
+                "m.room_key", megolmResult.roomKeyMessage, devices, hsApi);
+            await this._sendMessagesToDevices(ENCRYPTED_TYPE, messages, hsApi);
+        }
+        return {
+            type: ENCRYPTED_TYPE,
+            content: megolmResult.content
+        };
+    }
+
+    async _sendMessagesToDevices(type, messages, hsApi) {
+        const messagesByUser = groupBy(messages, message => message.device.userId);
+        const payload = {
+            messages: Array.from(messagesByUser.entries()).reduce((userMap, [userId, messages]) => {
+                userMap[userId] = messages.reduce((deviceMap, message) => {
+                    deviceMap[message.device.deviceId] = message.content;
+                    return deviceMap;
+                }, {});
+                return userMap;
+            }, {})
+        };
+        const txnId = makeTxnId();
+        await hsApi.sendToDevice(type, payload, txnId).response();
+    }
+}

src/matrix/e2ee/megolm/Decryption.js

@@ -14,7 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// senderKey is a curve25519 key
 export class Decryption {
     constructor({pickleKey, olm}) {
         this._pickleKey = pickleKey;

src/matrix/e2ee/megolm/Encryption.js

@@ -0,0 +1,147 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {MEGOLM_ALGORITHM} from "../common.js";
+
+export class Encryption {
+    constructor({pickleKey, olm, account, storage, now, ownDeviceId}) {
+        this._pickleKey = pickleKey;
+        this._olm = olm;
+        this._account = account;
+        this._storage = storage;
+        this._now = now;
+        this._ownDeviceId = ownDeviceId;
+    }
+
+    async encrypt(roomId, type, content, encryptionParams) {
+        let session = new this._olm.OutboundGroupSession();
+        try {
+            const txn = await this._storage.readWriteTxn([
+                this._storage.storeNames.inboundGroupSessions,
+                this._storage.storeNames.outboundGroupSessions,
+            ]);
+            let roomKeyMessage;
+            let encryptedContent;
+            try {
+                let sessionEntry = await txn.outboundGroupSessions.get(roomId);
+                if (sessionEntry) {
+                    session.unpickle(this._pickleKey, sessionEntry.session);
+                }
+                if (!sessionEntry || this._needsToRotate(session, sessionEntry.createdAt, encryptionParams)) {
+                    // in the case of rotating, recreate a session as we already unpickled into it
+                    if (session) {
+                        session.free();
+                        session = new this._olm.OutboundGroupSession();
+                    }
+                    session.create();
+                    roomKeyMessage = this._createRoomKeyMessage(session, roomId);
+                    this._storeAsInboundSession(session, roomId, txn);
+                    // TODO: we could tell the Decryption here that we have a new session so it can add it to its cache
+                }
+                encryptedContent = this._encryptContent(roomId, session, type, content);
+                txn.outboundGroupSessions.set({
+                    roomId,
+                    session: session.pickle(this._pickleKey),
+                    createdAt: sessionEntry?.createdAt || this._now(),
+                });
+
+            } catch (err) {
+                txn.abort();
+                throw err;
+            }
+            await txn.complete();
+            return new EncryptionResult(encryptedContent, roomKeyMessage);
+        } finally {
+            if (session) {
+                session.free();
+            }
+        }
+    }
+
+    _needsToRotate(session, createdAt, encryptionParams) {
+        let rotationPeriodMs = 604800000; // default
+        if (Number.isSafeInteger(encryptionParams?.rotation_period_ms)) {
+            rotationPeriodMs = encryptionParams?.rotation_period_ms;
+        }
+        let rotationPeriodMsgs = 100; // default
+        if (Number.isSafeInteger(encryptionParams?.rotation_period_msgs)) {
+            rotationPeriodMsgs = encryptionParams?.rotation_period_msgs;
+        }
+
+        if (this._now() > (createdAt + rotationPeriodMs)) {
+            return true;
+        }
+        if (session.message_index() >= rotationPeriodMsgs) {
+            return true;
+        }  
+    }
+
+    _encryptContent(roomId, session, type, content) {
+        const plaintext = JSON.stringify({
+            room_id: roomId,
+            type,
+            content
+        });
+        const ciphertext = session.encrypt(plaintext);
+
+        const encryptedContent = {
+            algorithm: MEGOLM_ALGORITHM,
+            sender_key: this._account.identityKeys.curve25519,
+            ciphertext,
+            session_id: session.session_id(),
+            device_id: this._ownDeviceId
+        };
+
+        return encryptedContent;
+    }
+
+    _createRoomKeyMessage(session, roomId) {
+        return {
+            room_id: roomId,
+            session_id: session.session_id(),
+            session_key: session.session_key(),
+            algorithm: MEGOLM_ALGORITHM,
+            // chain_index: session.message_index()
+        }
+    }
+
+    _storeAsInboundSession(outboundSession, roomId, txn) {
+        const {identityKeys} = this._account;
+        const claimedKeys = {ed25519: identityKeys.ed25519};
+        const session = new this._olm.InboundGroupSession();
+        try {
+            session.create(outboundSession.session_key());
+            const sessionEntry = {
+                roomId,
+                senderKey: identityKeys.curve25519,
+                sessionId: session.session_id(),
+                session: session.pickle(this._pickleKey),
+                claimedKeys,
+            };
+            txn.inboundGroupSessions.set(sessionEntry);
+            return sessionEntry;
+        } finally {
+            session.free();
+        }
+    }
+}
+
+class EncryptionResult {
+    constructor(content, roomKeyMessage) {
+        this.content = content;
+        this.roomKeyMessage = roomKeyMessage;
+    }
+}

src/matrix/net/HomeServerApi.js

@@ -172,6 +172,10 @@ export class HomeServerApi {
         return this._post("/keys/claim", null, payload, options);
     }
 
+    sendToDevice(type, payload, txnId, options = null) {
+        return this._put(`/sendToDevice/${encodeURIComponent(type)}/${encodeURIComponent(txnId)}`, null, payload, options);
+    }
+
     get mediaRepository() {
         return this._mediaRepository;
     }

src/matrix/room/Room.js

@@ -27,7 +27,7 @@ import {MemberList} from "./members/MemberList.js";
 import {Heroes} from "./members/Heroes.js";
 
 export class Room extends EventEmitter {
-	constructor({roomId, storage, hsApi, emitCollectionChange, sendScheduler, pendingEvents, user}) {
+	constructor({roomId, storage, hsApi, emitCollectionChange, sendScheduler, pendingEvents, user, createRoomEncryption}) {
         super();
         this._roomId = roomId;
         this._storage = storage;
@@ -41,6 +41,8 @@ export class Room extends EventEmitter {
         this._user = user;
         this._changedMembersDuringSync = null;
         this._memberList = null;
+        this._createRoomEncryption = createRoomEncryption;
+        this._roomEncryption = null;
 	}
 
     /** @package */
@@ -62,6 +64,10 @@ export class Room extends EventEmitter {
             }
             heroChanges = await this._heroes.calculateChanges(summaryChanges.heroes, memberChanges, txn);
         }
+        // pass member changes to device tracker
+        if (this._roomEncryption && this.isTrackingMembers && memberChanges?.size) {
+            await this._roomEncryption.writeMemberChanges(memberChanges, txn);
+        }
         let removedPendingEvents;
         if (roomResponse.timeline && roomResponse.timeline.events) {
             removedPendingEvents = this._sendQueue.removeRemoteEchos(roomResponse.timeline.events, txn);
@@ -79,6 +85,13 @@ export class Room extends EventEmitter {
     /** @package */
     afterSync({summaryChanges, newTimelineEntries, newLiveKey, removedPendingEvents, memberChanges, heroChanges}) {
         this._syncWriter.afterSync(newLiveKey);
+        // encryption got enabled
+        if (!this._summary.encryption && summaryChanges.encryption && !this._roomEncryption) {
+            this._roomEncryption = this._createRoomEncryption(this, summaryChanges.encryption);
+            if (this._roomEncryption) {
+                this._sendQueue.enableEncryption(this._roomEncryption);
+            }
+        }
         if (memberChanges.size) {
             if (this._changedMembersDuringSync) {
                 for (const [userId, memberChange] of memberChanges.entries()) {
@@ -125,6 +138,12 @@ export class Room extends EventEmitter {
 	async load(summary, txn) {
         try {
             this._summary.load(summary);
+            if (this._summary.encryption) {
+                this._roomEncryption = this._createRoomEncryption(this, this._summary.encryption);
+                if (this._roomEncryption) {
+                    this._sendQueue.enableEncryption(this._roomEncryption);
+                }
+            }
             // need to load members for name?
             if (this._summary.needsHeroes) {
                 this._heroes = new Heroes(this._roomId);

src/matrix/room/sending/PendingEvent.js

@@ -26,5 +26,12 @@ export class PendingEvent {
     get remoteId() { return this._data.remoteId; }
     set remoteId(value) { this._data.remoteId = value; }
     get content() { return this._data.content; }
+    get needsEncryption() { return this._data.needsEncryption; }
     get data() { return this._data; }
+
+    setEncrypted(type, content) {
+        this._data.eventType = type;
+        this._data.content = content;
+        this._data.needsEncryption = false;
+    }
 }

src/matrix/room/sending/SendQueue.js

@@ -17,12 +17,7 @@ limitations under the License.
 import {SortedArray} from "../../../observable/list/SortedArray.js";
 import {ConnectionError} from "../../error.js";
 import {PendingEvent} from "./PendingEvent.js";
-
+import {makeTxnId} from "../../common.js";
-function makeTxnId() {
-    const n = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER);
-    const str = n.toString(16);
-    return "t" + "0".repeat(14 - str.length) + str;
-}
 
 export class SendQueue {
     constructor({roomId, storage, sendScheduler, pendingEvents}) {
@@ -38,6 +33,11 @@ export class SendQueue {
         this._isSending = false;
         this._offline = false;
         this._amountSent = 0;
+        this._roomEncryption = null;
+    }
+
+    enableEncryption(roomEncryption) {
+        this._roomEncryption = roomEncryption;
     }
 
     async _sendLoop() {
@@ -50,6 +50,13 @@ export class SendQueue {
                 if (pendingEvent.remoteId) {
                     continue;
                 }
+                if (pendingEvent.needsEncryption) {
+                    const {type, content} = await this._sendScheduler.request(async hsApi => {
+                        return await this._roomEncryption.encrypt(pendingEvent.eventType, pendingEvent.content, hsApi);
+                    });
+                    pendingEvent.setEncrypted(type, content);
+                    await this._tryUpdateEvent(pendingEvent);
+                }
                 console.log("really sending now");
                 const response = await this._sendScheduler.request(hsApi => {
                     console.log("got sendScheduler slot");
@@ -161,7 +168,8 @@ export class SendQueue {
                 queueIndex,
                 eventType,
                 content,
-                txnId: makeTxnId()
+                txnId: makeTxnId(),
+                needsEncryption: !!this._roomEncryption
             });
             console.log("_createAndStoreEvent: adding to pendingEventsStore");
             pendingEventsStore.add(pendingEvent.data);

src/matrix/storage/common.js

@@ -26,6 +26,7 @@ export const STORE_NAMES = Object.freeze([
     "deviceIdentities",
     "olmSessions",
     "inboundGroupSessions",
+    "outboundGroupSessions",
 ]);
 
 export const STORE_MAP = Object.freeze(STORE_NAMES.reduce((nameMap, name) => {

src/matrix/storage/idb/Transaction.js

@@ -28,6 +28,7 @@ import {UserIdentityStore} from "./stores/UserIdentityStore.js";
 import {DeviceIdentityStore} from "./stores/DeviceIdentityStore.js";
 import {OlmSessionStore} from "./stores/OlmSessionStore.js";
 import {InboundGroupSessionStore} from "./stores/InboundGroupSessionStore.js";
+import {OutboundGroupSessionStore} from "./stores/OutboundGroupSessionStore.js";
 
 export class Transaction {
     constructor(txn, allowedStoreNames) {
@@ -101,6 +102,9 @@ export class Transaction {
         return this._store("inboundGroupSessions", idbStore => new InboundGroupSessionStore(idbStore));
     }
     
+    get outboundGroupSessions() {
+        return this._store("outboundGroupSessions", idbStore => new OutboundGroupSessionStore(idbStore));
+    }
     complete() {
         return txnAsPromise(this._txn);
     }

src/matrix/storage/idb/schema.js

@@ -12,6 +12,7 @@ export const schema = [
     createIdentityStores,
     createOlmSessionStore,
     createInboundGroupSessionsStore,
+    createOutboundGroupSessionsStore,
 ];
 // TODO: how to deal with git merge conflicts of this array?
 
@@ -82,3 +83,9 @@ function createOlmSessionStore(db) {
 function createInboundGroupSessionsStore(db) {
     db.createObjectStore("inboundGroupSessions", {keyPath: "key"});
 }
+
+//v7
+function createOutboundGroupSessionsStore(db) {
+    db.createObjectStore("outboundGroupSessions", {keyPath: "roomId"});
+}
+

src/matrix/storage/idb/stores/OutboundGroupSessionStore.js

@@ -0,0 +1,29 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export class OutboundGroupSessionStore {
+    constructor(store) {
+        this._store = store;
+    }
+
+    get(roomId) {
+        return this._store.get(roomId);
+    }
+
+    set(session) {
+        this._store.put(session);
+    }
+}