From cfd2fd98629678c805f3386c7b0bba1a50a07669 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 26 Aug 2020 14:49:16 +0200 Subject: [PATCH 001/173] add olm as a dependency --- package.json | 3 +++ yarn.lock | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/package.json b/package.json index 9786ce4b..2034f6b9 100644 --- a/package.json +++ b/package.json @@ -44,5 +44,8 @@ "rollup-plugin-cleanup": "^3.1.1", "serve-static": "^1.13.2", "xxhash": "^0.3.0" + }, + "dependencies": { + "olm": "https://packages.matrix.org/npm/olm/olm-3.1.4.tgz" } } diff --git a/yarn.lock b/yarn.lock index 1f3e276b..9e556624 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1496,6 +1496,10 @@ object.assign@^4.1.0: has-symbols "^1.0.0" object-keys "^1.0.11" +"olm@https://packages.matrix.org/npm/olm/olm-3.1.4.tgz": + version "3.1.4" + resolved "https://packages.matrix.org/npm/olm/olm-3.1.4.tgz#0f03128b7d3b2f614d2216409a1dfccca765fdb3" + on-finished@~2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.3.0.tgz#20f1336481b083cd75337992a16971aa2d906947" From 6edbec45eb46ac0d1e3d2481ca96ec00d3a0fd12 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 26 Aug 2020 15:43:08 +0200 Subject: [PATCH 002/173] move olm into own lib directory --- lib/olm.js | 1 - lib/olm.wasm | 1 - lib/olm/olm.js | 1 + lib/olm/olm.wasm | 1 + lib/olm/olm_legacy.js | 1 + prototypes/olmtest.html | 4 ++-- 6 files changed, 5 insertions(+), 4 deletions(-) delete mode 120000 lib/olm.js delete mode 120000 lib/olm.wasm create mode 120000 lib/olm/olm.js create mode 120000 lib/olm/olm.wasm create mode 120000 lib/olm/olm_legacy.js diff --git a/lib/olm.js b/lib/olm.js deleted file mode 120000 index 9f16c77b..00000000 --- a/lib/olm.js +++ /dev/null @@ -1 +0,0 @@ -../node_modules/olm/olm.js \ No newline at end of file diff --git a/lib/olm.wasm b/lib/olm.wasm deleted file mode 120000 index 8d848e89..00000000 --- a/lib/olm.wasm +++ /dev/null @@ -1 +0,0 @@ -../node_modules/olm/olm.wasm \ No newline at end of file diff --git a/lib/olm/olm.js b/lib/olm/olm.js new file mode 120000 index 00000000..8bedac52 --- /dev/null +++ b/lib/olm/olm.js @@ -0,0 +1 @@ +../../node_modules/olm/olm.js \ No newline at end of file diff --git a/lib/olm/olm.wasm b/lib/olm/olm.wasm new file mode 120000 index 00000000..39293356 --- /dev/null +++ b/lib/olm/olm.wasm @@ -0,0 +1 @@ +../../node_modules/olm/olm.wasm \ No newline at end of file diff --git a/lib/olm/olm_legacy.js b/lib/olm/olm_legacy.js new file mode 120000 index 00000000..140fffb8 --- /dev/null +++ b/lib/olm/olm_legacy.js @@ -0,0 +1 @@ +../../node_modules/olm/olm_legacy.js \ No newline at end of file diff --git a/prototypes/olmtest.html b/prototypes/olmtest.html index 9a17a189..839fb5ee 100644 --- a/prototypes/olmtest.html +++ b/prototypes/olmtest.html @@ -12,13 +12,13 @@ - + + + + + + + From 08b12eace5e167bab59d0272eb6c6b688de5cb94 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 26 Aug 2020 16:30:32 +0100 Subject: [PATCH 004/173] add a bit of metrics to ie11 olm prototype --- prototypes/olmtest-ie11.html | 74 ++++++++++++++++++++++++++---------- 1 file changed, 54 insertions(+), 20 deletions(-) diff --git a/prototypes/olmtest-ie11.html b/prototypes/olmtest-ie11.html index 316f5423..dec102ff 100644 --- a/prototypes/olmtest-ie11.html +++ b/prototypes/olmtest-ie11.html @@ -41,45 +41,79 @@ + 

+    
 
 
  

From 8098f9d646c22de76481cb259a82c31e6a6c1959 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Wed, 26 Aug 2020 17:42:29 +0200
Subject: [PATCH 005/173] try faster imul that might break

---
 prototypes/olmtest-ie11.html | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/prototypes/olmtest-ie11.html b/prototypes/olmtest-ie11.html
index dec102ff..13d906b9 100644
--- a/prototypes/olmtest-ie11.html
+++ b/prototypes/olmtest-ie11.html
@@ -14,7 +14,7 @@
 
 
     
     

     

From 92fdbe15df179fe6f1d4015e266b6046ad644319 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 27 Aug 2020 13:24:04 +0200
Subject: [PATCH 006/173] pass olm paths to main fn

so build can adjust the file paths,
and we can prevent olm from loading by not passing them
---
 index.html        |  6 +++++-
 scripts/build.mjs | 14 +++++++++++---
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/index.html b/index.html
index cdf0ad4f..b09286a0 100644
--- a/index.html
+++ b/index.html
@@ -18,7 +18,11 @@
         
 		
         ` +
+        `` +
         `` +
-        ``);
+        ``);
     removeOrEnableScript(doc("script#service-worker"), offline);
 
     const versionScript = doc("script#version");
@@ -338,7 +346,7 @@ async function copyFolder(srcRoot, dstRoot, filter) {
         if (dirEnt.isDirectory()) {
             await fs.mkdir(dstPath);
             Object.assign(assetPaths, await copyFolder(srcPath, dstPath, filter));
-        } else if (dirEnt.isFile() && filter(srcPath)) {
+        } else if ((dirEnt.isFile() || dirEnt.isSymbolicLink()) && (!filter || filter(srcPath))) {
             const content = await fs.readFile(srcPath);
             const hashedDstPath = resource(dstPath, content);
             await fs.writeFile(hashedDstPath, content);

From fe0257bca10b3d1a08c70c143f9c6094a15e04c1 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 27 Aug 2020 13:24:55 +0200
Subject: [PATCH 007/173] load olm and pass it to session

---
 src/main.js                    | 27 ++++++++++++++++++++++++++-
 src/matrix/Session.js          |  3 ++-
 src/matrix/SessionContainer.js |  6 ++++--
 3 files changed, 32 insertions(+), 4 deletions(-)

diff --git a/src/main.js b/src/main.js
index 2c86b9d9..85274948 100644
--- a/src/main.js
+++ b/src/main.js
@@ -26,10 +26,34 @@ import {BrawlView} from "./ui/web/BrawlView.js";
 import {Clock} from "./ui/web/dom/Clock.js";
 import {OnlineStatus} from "./ui/web/dom/OnlineStatus.js";
 
+function addScript(src) {
+    return new Promise(function (resolve, reject) {
+        var s = document.createElement("script");
+        s.setAttribute("src", src );
+        s.onload=resolve;
+        s.onerror=reject;
+        document.body.appendChild(s);
+    });
+}
+
+async function loadOlm(olmPaths) {
+    if (olmPaths) {
+        if (window.WebAssembly) {
+            await addScript(olmPaths.wasmBundle);
+            await window.Olm.init({locateFile: () => olmPaths.wasm});
+        } else {
+            await addScript(olmPaths.legacyBundle);
+            await window.Olm.init();
+        }
+        return window.Olm;
+    }
+    return null;
+}
+
 // Don't use a default export here, as we use multiple entries during legacy build,
 // which does not support default exports,
 // see https://github.com/rollup/plugins/tree/master/packages/multi-entry
-export async function main(container) {
+export async function main(container, olmPaths) {
     try {
         // to replay:
         // const fetchLog = await (await fetch("/fetchlogs/constrainterror.json")).json();
@@ -59,6 +83,7 @@ export async function main(container) {
                     sessionInfoStorage,
                     request,
                     clock,
+                    olmPromise: loadOlm(olmPaths),
                 });
             },
             sessionInfoStorage,
diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index 7a7dea52..f6db9c97 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -21,7 +21,7 @@ import {User} from "./User.js";
 
 export class Session {
     // sessionInfo contains deviceId, userId and homeServer
-    constructor({storage, hsApi, sessionInfo}) {
+    constructor({storage, hsApi, sessionInfo, olm}) {
         this._storage = storage;
         this._hsApi = hsApi;
         this._session = null;
@@ -30,6 +30,7 @@ export class Session {
         this._sendScheduler = new SendScheduler({hsApi, backoff: new RateLimitingBackoff()});
         this._roomUpdateCallback = (room, params) => this._rooms.update(room.id, params);
         this._user = new User(sessionInfo.userId);
+        this._olm = olm;
     }
 
     async load() {
diff --git a/src/matrix/SessionContainer.js b/src/matrix/SessionContainer.js
index 74184a0a..eb025ec6 100644
--- a/src/matrix/SessionContainer.js
+++ b/src/matrix/SessionContainer.js
@@ -41,7 +41,7 @@ export const LoginFailure = createEnum(
 );
 
 export class SessionContainer {
-    constructor({clock, random, onlineStatus, request, storageFactory, sessionInfoStorage}) {
+    constructor({clock, random, onlineStatus, request, storageFactory, sessionInfoStorage, olmPromise}) {
         this._random = random;
         this._clock = clock;
         this._onlineStatus = onlineStatus;
@@ -57,6 +57,7 @@ export class SessionContainer {
         this._sync = null;
         this._sessionId = null;
         this._storage = null;
+        this._olmPromise = olmPromise;
     }
 
     createNewSessionId() {
@@ -149,7 +150,8 @@ export class SessionContainer {
             userId: sessionInfo.userId,
             homeServer: sessionInfo.homeServer,
         };
-        this._session = new Session({storage: this._storage, sessionInfo: filteredSessionInfo, hsApi});
+        const olm = await this._olmPromise;
+        this._session = new Session({storage: this._storage, sessionInfo: filteredSessionInfo, hsApi, olm});
         await this._session.load();
         
         this._sync = new Sync({hsApi, storage: this._storage, session: this._session});

From 87aabb30579136593bbce306cdd61aac5edd2621 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 27 Aug 2020 13:31:14 +0200
Subject: [PATCH 008/173] make crypto.getRandomValues available on IE11 without
 a prefix

olm needs this to work on IE11
---
 src/main.js | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/main.js b/src/main.js
index 85274948..79f5698d 100644
--- a/src/main.js
+++ b/src/main.js
@@ -37,6 +37,11 @@ function addScript(src) {
 }
 
 async function loadOlm(olmPaths) {
+    // make crypto.getRandomValues available without
+    // a prefix on IE11, needed by olm to work
+    if (window.msCrypto && !window.crypto) {
+        window.crypto = window.msCrypto;
+    }
     if (olmPaths) {
         if (window.WebAssembly) {
             await addScript(olmPaths.wasmBundle);

From 7bf2a3929c6cdd64cf5a6abaac3841fcf46f3f84 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 27 Aug 2020 18:45:54 +0200
Subject: [PATCH 009/173] add another-json as a dependency

also add a postinstall script to turn it into an ES module,
so it works with our setup
---
 package.json             |  4 ++-
 scripts/post-install.mjs | 53 ++++++++++++++++++++++++++++++++++++++++
 yarn.lock                |  5 ++++
 3 files changed, 61 insertions(+), 1 deletion(-)
 create mode 100644 scripts/post-install.mjs

diff --git a/package.json b/package.json
index 3935c174..a4f4c082 100644
--- a/package.json
+++ b/package.json
@@ -9,7 +9,8 @@
   "scripts": {
     "test": "node_modules/.bin/impunity --entry-point src/main.js --force-esm",
     "start": "node scripts/serve-local.js",
-    "build": "node --experimental-modules scripts/build.mjs"
+    "build": "node --experimental-modules scripts/build.mjs",
+    "postinstall": "node ./scripts/post-install.mjs"
   },
   "repository": {
     "type": "git",
@@ -46,6 +47,7 @@
     "xxhash": "^0.3.0"
   },
   "dependencies": {
+    "another-json": "^0.2.0",
     "olm": "https://packages.matrix.org/npm/olm/olm-3.1.4.tgz"
   }
 }
diff --git a/scripts/post-install.mjs b/scripts/post-install.mjs
new file mode 100644
index 00000000..3b287fd9
--- /dev/null
+++ b/scripts/post-install.mjs
@@ -0,0 +1,53 @@
+/*
+Copyright 2020 Bruno Windels 
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import fsRoot from "fs";
+const fs = fsRoot.promises;
+import path from "path";
+import { rollup } from 'rollup';
+import { fileURLToPath } from 'url';
+import { dirname } from 'path';
+// needed to translate commonjs modules to esm
+import commonjs from '@rollup/plugin-commonjs';
+// multi-entry plugin so we can add polyfill file to main
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const projectDir = path.join(__dirname, "../");
+
+async function commonjsToESM(src, dst) {
+    // create js bundle
+    const bundle = await rollup({
+        input: src,
+        plugins: [commonjs()]
+    });
+    const {output} = await bundle.generate({
+        format: 'es'
+    });
+    const code = output[0].code;
+    await fs.writeFile(dst, code, "utf8");
+}
+
+async function transpile() {
+    await fs.mkdir(path.join(projectDir, "lib/another-json/"));
+    await commonjsToESM(
+        path.join(projectDir, 'node_modules/another-json/another-json.js'),
+        path.join(projectDir, "lib/another-json/index.js")
+    );
+}
+
+transpile();
diff --git a/yarn.lock b/yarn.lock
index 9e556624..ea3c6ba5 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -907,6 +907,11 @@
   dependencies:
     "@types/node" "*"
 
+another-json@^0.2.0:
+  version "0.2.0"
+  resolved "https://registry.yarnpkg.com/another-json/-/another-json-0.2.0.tgz#b5f4019c973b6dd5c6506a2d93469cb6d32aeedc"
+  integrity sha1-tfQBnJc7bdXGUGotk0acttMq7tw=
+
 ansi-styles@^3.2.1:
   version "3.2.1"
   resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d"

From f98b3dd5fab313ee7a9f3cabe51924a8eae55273 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 27 Aug 2020 19:12:06 +0200
Subject: [PATCH 010/173] create/load olm account before first sync

---
 src/matrix/Session.js                         | 41 ++++++++++++++
 src/matrix/SessionContainer.js                |  7 +--
 src/matrix/e2ee/Account.js                    | 54 +++++++++++++++++++
 src/matrix/storage/idb/stores/SessionStore.js |  4 ++
 4 files changed, 103 insertions(+), 3 deletions(-)
 create mode 100644 src/matrix/e2ee/Account.js

diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index 90516c5f..907a4c4b 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -18,6 +18,8 @@ import {Room} from "./room/Room.js";
 import { ObservableMap } from "../observable/index.js";
 import { SendScheduler, RateLimitingBackoff } from "./SendScheduler.js";
 import {User} from "./User.js";
+import {Account as E2EEAccount} from "./e2ee/Account.js";
+const PICKLE_KEY = "DEFAULT_KEY";
 
 export class Session {
     // sessionInfo contains deviceId, userId and homeServer
@@ -31,6 +33,34 @@ export class Session {
         this._roomUpdateCallback = (room, params) => this._rooms.update(room.id, params);
         this._user = new User(sessionInfo.userId);
         this._olm = olm;
+        this._e2eeAccount = null;
+    }
+
+    async beforeFirstSync(isNewLogin) {
+        if (this._olm) {
+            if (isNewLogin && this._e2eeAccount) {
+                throw new Error("there should not be an e2ee account already on a fresh login");
+            }
+            if (!this._e2eeAccount) {
+                const txn = await this._storage.readWriteTxn([
+                    this._storage.storeNames.session
+                ]);
+                try {
+                    this._e2eeAccount = await E2EEAccount.create({
+                        hsApi: this._hsApi,
+                        olm: this._olm,
+                        pickleKey: PICKLE_KEY,
+                        userId: this._sessionInfo.userId,
+                        deviceId: this._sessionInfo.deviceId,
+                        txn
+                    });
+                } catch (err) {
+                    txn.abort();
+                    throw err;
+                }
+                await txn.complete();
+            }
+        }
     }
 
     async load() {
@@ -44,6 +74,17 @@ export class Session {
         ]);
         // restore session object
         this._syncInfo = await txn.session.get("sync");
+        // restore e2ee account, if any
+        if (this._olm) {
+            this._e2eeAccount = await E2EEAccount.load({
+                hsApi: this._hsApi,
+                olm: this._olm,
+                pickleKey: PICKLE_KEY,
+                userId: this._sessionInfo.userId,
+                deviceId: this._sessionInfo.deviceId,
+                txn
+            });
+        }
         const pendingEventsByRoomId = await this._getPendingEventsByRoom(txn);
         // load rooms
         const rooms = await txn.roomSummary.getAll();
diff --git a/src/matrix/SessionContainer.js b/src/matrix/SessionContainer.js
index eb025ec6..ae9572a9 100644
--- a/src/matrix/SessionContainer.js
+++ b/src/matrix/SessionContainer.js
@@ -74,7 +74,7 @@ export class SessionContainer {
             if (!sessionInfo) {
                 throw new Error("Invalid session id: " + sessionId);
             }
-            await this._loadSessionInfo(sessionInfo);
+            await this._loadSessionInfo(sessionInfo, false);
         } catch (err) {
             this._error = err;
             this._status.set(LoadStatus.Error);
@@ -121,14 +121,14 @@ export class SessionContainer {
         // LoadStatus.Error in case of an error,
         // so separate try/catch
         try {
-            await this._loadSessionInfo(sessionInfo);
+            await this._loadSessionInfo(sessionInfo, true);
         } catch (err) {
             this._error = err;
             this._status.set(LoadStatus.Error);
         }
     }
 
-    async _loadSessionInfo(sessionInfo) {
+    async _loadSessionInfo(sessionInfo, isNewLogin) {
         this._status.set(LoadStatus.Loading);
         this._reconnector = new Reconnector({
             onlineStatus: this._onlineStatus,
@@ -153,6 +153,7 @@ export class SessionContainer {
         const olm = await this._olmPromise;
         this._session = new Session({storage: this._storage, sessionInfo: filteredSessionInfo, hsApi, olm});
         await this._session.load();
+        await this._session.beforeFirstSync(isNewLogin);
         
         this._sync = new Sync({hsApi, storage: this._storage, session: this._session});
         // notify sync and session when back online
diff --git a/src/matrix/e2ee/Account.js b/src/matrix/e2ee/Account.js
new file mode 100644
index 00000000..a159fa64
--- /dev/null
+++ b/src/matrix/e2ee/Account.js
@@ -0,0 +1,54 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import anotherjson from "../../../lib/another-json/index.js";
+
+const ACCOUNT_SESSION_KEY = "olmAccount";
+const DEVICE_KEY_FLAG_SESSION_KEY = "areDeviceKeysUploaded";
+
+export class Account {
+    static async load({olm, pickleKey, hsApi, userId, deviceId, txn}) {
+        const pickledAccount = await txn.session.get(ACCOUNT_SESSION_KEY);
+        if (pickledAccount) {
+            const account = new olm.Account();
+            const areDeviceKeysUploaded = await txn.session.get(DEVICE_KEY_FLAG_SESSION_KEY);
+            account.unpickle(pickleKey, pickledAccount);
+            return new Account({pickleKey, hsApi, account, userId, deviceId, areDeviceKeysUploaded});
+        }
+    }
+
+    static async create({olm, pickleKey, hsApi, userId, deviceId, txn}) {
+        const account = new olm.Account();
+        account.create();
+        account.generate_one_time_keys(account.max_number_of_one_time_keys());
+        const pickledAccount = account.pickle(pickleKey);
+        // add will throw if the key already exists
+        // we would not want to overwrite olmAccount here
+        const areDeviceKeysUploaded = false;
+        await txn.session.add(ACCOUNT_SESSION_KEY, pickledAccount);
+        await txn.session.add(DEVICE_KEY_FLAG_SESSION_KEY, areDeviceKeysUploaded);
+        return new Account({pickleKey, hsApi, account, userId, deviceId, areDeviceKeysUploaded});
+    }
+
+    constructor({pickleKey, hsApi, account, userId, deviceId, areDeviceKeysUploaded}) {
+        this._pickleKey = pickleKey;
+        this._hsApi = hsApi;
+        this._account = account;
+        this._userId = userId;
+        this._deviceId = deviceId;
+        this._areDeviceKeysUploaded = areDeviceKeysUploaded;
+    }
+}
diff --git a/src/matrix/storage/idb/stores/SessionStore.js b/src/matrix/storage/idb/stores/SessionStore.js
index 2e74b9df..f64a8299 100644
--- a/src/matrix/storage/idb/stores/SessionStore.js
+++ b/src/matrix/storage/idb/stores/SessionStore.js
@@ -45,4 +45,8 @@ export class SessionStore {
 	set(key, value) {
 		return this._sessionStore.put({key, value});
 	}
+
+    add(key, value) {
+        return this._sessionStore.put({key, value});
+    }
 }

From 4c290f0394fdfcf98f2dfcd7367b23446eda83ac Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 27 Aug 2020 19:13:24 +0200
Subject: [PATCH 011/173] upload identity and one-time keys

---
 src/matrix/Session.js           |  1 +
 src/matrix/e2ee/Account.js      | 86 +++++++++++++++++++++++++++++++++
 src/matrix/net/HomeServerApi.js |  4 ++
 3 files changed, 91 insertions(+)

diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index 907a4c4b..f5d9021c 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -60,6 +60,7 @@ export class Session {
                 }
                 await txn.complete();
             }
+            await this._e2eeAccount.uploadKeys(this._storage);
         }
     }
 
diff --git a/src/matrix/e2ee/Account.js b/src/matrix/e2ee/Account.js
index a159fa64..ef342d49 100644
--- a/src/matrix/e2ee/Account.js
+++ b/src/matrix/e2ee/Account.js
@@ -51,4 +51,90 @@ export class Account {
         this._deviceId = deviceId;
         this._areDeviceKeysUploaded = areDeviceKeysUploaded;
     }
+
+    async uploadKeys(storage) {
+        const oneTimeKeys = JSON.parse(this._account.one_time_keys());
+        // only one algorithm supported by olm atm, so hardcode its name
+        const oneTimeKeysEntries = Object.entries(oneTimeKeys.curve25519);
+        if (oneTimeKeysEntries.length || !this._areDeviceKeysUploaded) {
+            const payload = {};
+            if (!this._areDeviceKeysUploaded) {
+                const identityKeys = JSON.parse(this._account.identity_keys());
+                payload.device_keys = this._deviceKeysPayload(identityKeys);
+            }
+            if (oneTimeKeysEntries.length) {
+                payload.one_time_keys = this._oneTimeKeysPayload(oneTimeKeysEntries);
+            }
+            await this._hsApi.uploadKeys(payload);
+
+            await this._updateSessionStorage(storage, sessionStore => {
+                if (oneTimeKeysEntries.length) {
+                    this._account.mark_keys_as_published();
+                    sessionStore.set(ACCOUNT_SESSION_KEY, this._account.pickle(this._pickleKey));
+                }
+                if (!this._areDeviceKeysUploaded) {
+                    this._areDeviceKeysUploaded = true;
+                    sessionStore.set(DEVICE_KEY_FLAG_SESSION_KEY, this._areDeviceKeysUploaded);
+                }
+            });
+        }
+    }
+
+    _deviceKeysPayload(identityKeys) {
+        const obj = {
+            user_id: this._userId,
+            device_id: this._deviceId,
+            algorithms: [
+                "m.olm.v1.curve25519-aes-sha2",
+                "m.megolm.v1.aes-sha2"
+            ],
+            keys: {}
+        };
+        for (const [algorithm, pubKey] of Object.entries(identityKeys)) {
+            obj.keys[`${algorithm}:${this._deviceId}`] = pubKey;
+        }
+        this.signObject(obj);
+        return obj;
+    }
+
+    _oneTimeKeysPayload(oneTimeKeysEntries) {
+        const obj = {};
+        for (const [keyId, pubKey] of oneTimeKeysEntries) {
+            const keyObj = {
+                key: pubKey  
+            };
+            this.signObject(keyObj);
+            obj[`signed_curve25519:${keyId}`] = keyObj;
+        }
+        return obj;
+    }
+
+    async _updateSessionStorage(storage, callback) {
+        const txn = await storage.readWriteTxn([
+            storage.storeNames.session
+        ]);
+        try {
+            callback(txn.session);
+        } catch (err) {
+            txn.abort();
+            throw err;
+        }
+        await txn.complete();
+    }
+
+    signObject(obj) {
+        const sigs = obj.signatures || {};
+        const unsigned = obj.unsigned;
+
+        delete obj.signatures;
+        delete obj.unsigned;
+
+        sigs[this._userId] = sigs[this._userId] || {};
+        sigs[this._userId]["ed25519:" + this._deviceId] = 
+            this._account.sign(anotherjson.stringify(obj));
+        obj.signatures = sigs;
+        if (unsigned !== undefined) {
+            obj.unsigned = unsigned;
+        }
+    }
 }
diff --git a/src/matrix/net/HomeServerApi.js b/src/matrix/net/HomeServerApi.js
index 234c8bc3..ac55f0a5 100644
--- a/src/matrix/net/HomeServerApi.js
+++ b/src/matrix/net/HomeServerApi.js
@@ -160,6 +160,10 @@ export class HomeServerApi {
         return this._request("GET", `${this._homeserver}/_matrix/client/versions`, null, null, options);
     }
 
+    uploadKeys(payload, options = null) {
+        return this._post("/keys/upload", null, payload, options);
+    }
+
     get mediaRepository() {
         return this._mediaRepository;
     }

From cdb83dd3c988c7edf6a39f078b3678f305b066d7 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 27 Aug 2020 19:15:31 +0200
Subject: [PATCH 012/173] adjust copyright

---
 scripts/post-install.mjs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/post-install.mjs b/scripts/post-install.mjs
index 3b287fd9..8c00d2a0 100644
--- a/scripts/post-install.mjs
+++ b/scripts/post-install.mjs
@@ -1,5 +1,4 @@
 /*
-Copyright 2020 Bruno Windels 
 Copyright 2020 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");

From 68a3e8867bc18d470d12510e550cfe210a0badb1 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 27 Aug 2020 19:51:04 +0200
Subject: [PATCH 013/173] populate lib dir entirely in postinstall script

---
 lib/olm/olm.js           |  1 -
 lib/olm/olm.wasm         |  1 -
 lib/olm/olm_legacy.js    |  1 -
 scripts/common.mjs       | 12 ++++++++++++
 scripts/post-install.mjs | 22 +++++++++++++++++-----
 5 files changed, 29 insertions(+), 8 deletions(-)
 delete mode 120000 lib/olm/olm.js
 delete mode 120000 lib/olm/olm.wasm
 delete mode 120000 lib/olm/olm_legacy.js
 create mode 100644 scripts/common.mjs

diff --git a/lib/olm/olm.js b/lib/olm/olm.js
deleted file mode 120000
index 8bedac52..00000000
--- a/lib/olm/olm.js
+++ /dev/null
@@ -1 +0,0 @@
-../../node_modules/olm/olm.js
\ No newline at end of file
diff --git a/lib/olm/olm.wasm b/lib/olm/olm.wasm
deleted file mode 120000
index 39293356..00000000
--- a/lib/olm/olm.wasm
+++ /dev/null
@@ -1 +0,0 @@
-../../node_modules/olm/olm.wasm
\ No newline at end of file
diff --git a/lib/olm/olm_legacy.js b/lib/olm/olm_legacy.js
deleted file mode 120000
index 140fffb8..00000000
--- a/lib/olm/olm_legacy.js
+++ /dev/null
@@ -1 +0,0 @@
-../../node_modules/olm/olm_legacy.js
\ No newline at end of file
diff --git a/scripts/common.mjs b/scripts/common.mjs
new file mode 100644
index 00000000..e135022a
--- /dev/null
+++ b/scripts/common.mjs
@@ -0,0 +1,12 @@
+import fsRoot from "fs";
+const fs = fsRoot.promises;
+
+export async function removeDirIfExists(targetDir) {
+    try {
+        await fs.rmdir(targetDir, {recursive: true});
+    } catch (err) {
+        if (err.code !== "ENOENT") {
+            throw err;
+        }
+    }
+}
diff --git a/scripts/post-install.mjs b/scripts/post-install.mjs
index 8c00d2a0..328f8c7c 100644
--- a/scripts/post-install.mjs
+++ b/scripts/post-install.mjs
@@ -23,6 +23,7 @@ import { dirname } from 'path';
 // needed to translate commonjs modules to esm
 import commonjs from '@rollup/plugin-commonjs';
 // multi-entry plugin so we can add polyfill file to main
+import {removeDirIfExists} from "./common.mjs";
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
@@ -41,12 +42,23 @@ async function commonjsToESM(src, dst) {
     await fs.writeFile(dst, code, "utf8");
 }
 
-async function transpile() {
-    await fs.mkdir(path.join(projectDir, "lib/another-json/"));
+async function populateLib() {
+    const libDir = path.join(projectDir, "lib/");
+    const modulesDir = path.join(projectDir, "node_modules/");
+    await removeDirIfExists(libDir);
+    await fs.mkdir(libDir);
+    const olmSrcDir = path.join(modulesDir, "olm/");
+    const olmDstDir = path.join(libDir, "olm/");
+    await fs.mkdir(olmDstDir);
+    for (const file of ["olm.js", "olm.wasm", "olm_legacy.js"]) {
+        await fs.symlink(path.join(olmSrcDir, file), path.join(olmDstDir, file));
+    }
+    // transpile another-json to esm
+    await fs.mkdir(path.join(libDir, "another-json/"));
     await commonjsToESM(
-        path.join(projectDir, 'node_modules/another-json/another-json.js'),
-        path.join(projectDir, "lib/another-json/index.js")
+        path.join(modulesDir, 'another-json/another-json.js'),
+        path.join(libDir, "another-json/index.js")
     );
 }
 
-transpile();
+populateLib();

From 16b681a79a3d6205ce4256cae093953520266599 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 27 Aug 2020 19:51:32 +0200
Subject: [PATCH 014/173] don't commit lib dir

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index e38531b4..00e360eb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,3 +5,4 @@ fetchlogs
 sessionexports
 bundle.js
 target
+lib

From d24be7ee552195721d694fed67d5d379a3252a72 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 28 Aug 2020 13:51:58 +0200
Subject: [PATCH 015/173] extract constants out

---
 src/matrix/e2ee/Account.js | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/matrix/e2ee/Account.js b/src/matrix/e2ee/Account.js
index ef342d49..829e7595 100644
--- a/src/matrix/e2ee/Account.js
+++ b/src/matrix/e2ee/Account.js
@@ -18,6 +18,8 @@ import anotherjson from "../../../lib/another-json/index.js";
 
 const ACCOUNT_SESSION_KEY = "olmAccount";
 const DEVICE_KEY_FLAG_SESSION_KEY = "areDeviceKeysUploaded";
+const OLM_ALGORITHM = "m.olm.v1.curve25519-aes-sha2";
+const MEGOLM_ALGORITHM = "m.megolm.v1.aes-sha2";
 
 export class Account {
     static async load({olm, pickleKey, hsApi, userId, deviceId, txn}) {
@@ -84,10 +86,7 @@ export class Account {
         const obj = {
             user_id: this._userId,
             device_id: this._deviceId,
-            algorithms: [
-                "m.olm.v1.curve25519-aes-sha2",
-                "m.megolm.v1.aes-sha2"
-            ],
+            algorithms: [OLM_ALGORITHM, MEGOLM_ALGORITHM],
             keys: {}
         };
         for (const [algorithm, pubKey] of Object.entries(identityKeys)) {

From 3ab5a7222160551e9143a7182f374d462633677f Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 28 Aug 2020 13:52:27 +0200
Subject: [PATCH 016/173] give e2ee account values a prefix so we can prevent
 from clearing them

---
 src/matrix/e2ee/Account.js | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/matrix/e2ee/Account.js b/src/matrix/e2ee/Account.js
index 829e7595..f53532f1 100644
--- a/src/matrix/e2ee/Account.js
+++ b/src/matrix/e2ee/Account.js
@@ -16,8 +16,10 @@ limitations under the License.
 
 import anotherjson from "../../../lib/another-json/index.js";
 
-const ACCOUNT_SESSION_KEY = "olmAccount";
-const DEVICE_KEY_FLAG_SESSION_KEY = "areDeviceKeysUploaded";
+// use common prefix so it's easy to clear properties that are not e2ee related during session clear
+export const SESSION_KEY_PREFIX = "e2ee:";
+const ACCOUNT_SESSION_KEY = SESSION_KEY_PREFIX + "olmAccount";
+const DEVICE_KEY_FLAG_SESSION_KEY = SESSION_KEY_PREFIX + "areDeviceKeysUploaded";
 const OLM_ALGORITHM = "m.olm.v1.curve25519-aes-sha2";
 const MEGOLM_ALGORITHM = "m.megolm.v1.aes-sha2";
 

From d64db185bd1a32726af2e05edd19f9f0a3363a46 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 28 Aug 2020 13:54:42 +0200
Subject: [PATCH 017/173] await callback in case we need to read, then write
 from it

---
 src/matrix/e2ee/Account.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/matrix/e2ee/Account.js b/src/matrix/e2ee/Account.js
index f53532f1..92478156 100644
--- a/src/matrix/e2ee/Account.js
+++ b/src/matrix/e2ee/Account.js
@@ -115,7 +115,7 @@ export class Account {
             storage.storeNames.session
         ]);
         try {
-            callback(txn.session);
+            await callback(txn.session);
         } catch (err) {
             txn.abort();
             throw err;

From 681dfdf62ba39c955bfdf3e4305932917067e070 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 28 Aug 2020 13:56:44 +0200
Subject: [PATCH 018/173] sync otk count to e2ee account

---
 src/matrix/Session.js      | 17 ++++++++++++++---
 src/matrix/Sync.js         |  2 +-
 src/matrix/e2ee/Account.js | 37 ++++++++++++++++++++++++++++++++-----
 3 files changed, 47 insertions(+), 9 deletions(-)

diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index f5d9021c..90d29b61 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -151,20 +151,31 @@ export class Session {
         return room;
     }
 
-    writeSync(syncToken, syncFilterId, accountData, txn) {
+    writeSync(syncResponse, syncFilterId, txn) {
+        const changes = {};
+        const syncToken = syncResponse.next_batch;
+        const deviceOneTimeKeysCount = syncResponse.device_one_time_keys_count;
+
+        if (this._e2eeAccount && deviceOneTimeKeysCount) {
+            changes.e2eeAccountChanges = this._e2eeAccount.writeSync(deviceOneTimeKeysCount, txn);
+        }
         if (syncToken !== this.syncToken) {
             const syncInfo = {token: syncToken, filterId: syncFilterId};
             // don't modify `this` because transaction might still fail
             txn.session.set("sync", syncInfo);
-            return syncInfo;
+            changes.syncInfo = syncInfo;
         }
+        return changes;
     }
 
-    afterSync(syncInfo) {
+    afterSync({syncInfo, e2eeAccountChanges}) {
         if (syncInfo) {
             // sync transaction succeeded, modify object state now
             this._syncInfo = syncInfo;
         }
+        if (this._e2eeAccount && e2eeAccountChanges) {
+            this._e2eeAccount.afterSync(e2eeAccountChanges);
+        }
     }
 
     get syncToken() {
diff --git a/src/matrix/Sync.js b/src/matrix/Sync.js
index e6de7146..e14451a9 100644
--- a/src/matrix/Sync.js
+++ b/src/matrix/Sync.js
@@ -127,7 +127,7 @@ export class Sync {
         const roomChanges = [];
         let sessionChanges;
         try {
-            sessionChanges = this._session.writeSync(syncToken, syncFilterId, response.account_data,  syncTxn);
+            sessionChanges = this._session.writeSync(response, syncFilterId, syncTxn);
             // to_device
             // presence
             if (response.rooms) {
diff --git a/src/matrix/e2ee/Account.js b/src/matrix/e2ee/Account.js
index 92478156..90fabc72 100644
--- a/src/matrix/e2ee/Account.js
+++ b/src/matrix/e2ee/Account.js
@@ -20,6 +20,7 @@ import anotherjson from "../../../lib/another-json/index.js";
 export const SESSION_KEY_PREFIX = "e2ee:";
 const ACCOUNT_SESSION_KEY = SESSION_KEY_PREFIX + "olmAccount";
 const DEVICE_KEY_FLAG_SESSION_KEY = SESSION_KEY_PREFIX + "areDeviceKeysUploaded";
+const SERVER_OTK_COUNT_SESSION_KEY = SESSION_KEY_PREFIX + "serverOTKCount";
 const OLM_ALGORITHM = "m.olm.v1.curve25519-aes-sha2";
 const MEGOLM_ALGORITHM = "m.megolm.v1.aes-sha2";
 
@@ -30,7 +31,9 @@ export class Account {
             const account = new olm.Account();
             const areDeviceKeysUploaded = await txn.session.get(DEVICE_KEY_FLAG_SESSION_KEY);
             account.unpickle(pickleKey, pickledAccount);
-            return new Account({pickleKey, hsApi, account, userId, deviceId, areDeviceKeysUploaded});
+            const serverOTKCount = await txn.session.get(SERVER_OTK_COUNT_SESSION_KEY);
+            return new Account({pickleKey, hsApi, account, userId,
+                deviceId, areDeviceKeysUploaded, serverOTKCount});
         }
     }
 
@@ -44,16 +47,19 @@ export class Account {
         const areDeviceKeysUploaded = false;
         await txn.session.add(ACCOUNT_SESSION_KEY, pickledAccount);
         await txn.session.add(DEVICE_KEY_FLAG_SESSION_KEY, areDeviceKeysUploaded);
-        return new Account({pickleKey, hsApi, account, userId, deviceId, areDeviceKeysUploaded});
+        await txn.session.add(SERVER_OTK_COUNT_SESSION_KEY, 0);
+        return new Account({pickleKey, hsApi, account, userId,
+            deviceId, areDeviceKeysUploaded, serverOTKCount: 0});
     }
 
-    constructor({pickleKey, hsApi, account, userId, deviceId, areDeviceKeysUploaded}) {
+    constructor({pickleKey, hsApi, account, userId, deviceId, areDeviceKeysUploaded, serverOTKCount}) {
         this._pickleKey = pickleKey;
         this._hsApi = hsApi;
         this._account = account;
         this._userId = userId;
         this._deviceId = deviceId;
         this._areDeviceKeysUploaded = areDeviceKeysUploaded;
+        this._serverOTKCount = serverOTKCount;
     }
 
     async uploadKeys(storage) {
@@ -69,12 +75,17 @@ export class Account {
             if (oneTimeKeysEntries.length) {
                 payload.one_time_keys = this._oneTimeKeysPayload(oneTimeKeysEntries);
             }
-            await this._hsApi.uploadKeys(payload);
-
+            const response = await this._hsApi.uploadKeys(payload).response();
+            this._serverOTKCount = response?.one_time_key_counts?.signed_curve25519;
+            // TODO: should we not modify this in the txn like we do elsewhere?
+            // we'd have to pickle and unpickle the account to clone it though ...
+            // and the upload has succeed at this point, so in-memory would be correct
+            // but in-storage not if the txn fails. 
             await this._updateSessionStorage(storage, sessionStore => {
                 if (oneTimeKeysEntries.length) {
                     this._account.mark_keys_as_published();
                     sessionStore.set(ACCOUNT_SESSION_KEY, this._account.pickle(this._pickleKey));
+                    sessionStore.set(SERVER_OTK_COUNT_SESSION_KEY, this._serverOTKCount);
                 }
                 if (!this._areDeviceKeysUploaded) {
                     this._areDeviceKeysUploaded = true;
@@ -84,6 +95,22 @@ export class Account {
         }
     }
 
+    writeSync(deviceOneTimeKeysCount, txn) {
+        // we only upload signed_curve25519 otks
+        const otkCount = deviceOneTimeKeysCount.signed_curve25519;
+        if (Number.isSafeInteger(otkCount) && otkCount !== this._serverOTKCount) {
+            txn.session.set(SERVER_OTK_COUNT_SESSION_KEY, otkCount);
+            return otkCount;
+        }
+    }
+
+    afterSync(otkCount) {
+        // could also be undefined
+        if (Number.isSafeInteger(otkCount)) {
+            this._serverOTKCount = otkCount;
+        }
+    }
+
     _deviceKeysPayload(identityKeys) {
         const obj = {
             user_id: this._userId,

From a1ba5d7dba9ef7e3db4bc24cafe450c6d513fe34 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 28 Aug 2020 13:58:17 +0200
Subject: [PATCH 019/173] between syncs, generate more otks if needed, and
 upload them

---
 src/matrix/Session.js      | 10 ++++++++++
 src/matrix/Sync.js         |  6 ++++++
 src/matrix/e2ee/Account.js | 25 +++++++++++++++++++++++++
 3 files changed, 41 insertions(+)

diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index 90d29b61..bc52227a 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -60,6 +60,7 @@ export class Session {
                 }
                 await txn.complete();
             }
+            await this._e2eeAccount.generateOTKsIfNeeded(this._storage);
             await this._e2eeAccount.uploadKeys(this._storage);
         }
     }
@@ -178,6 +179,15 @@ export class Session {
         }
     }
 
+    async afterSyncCompleted() {
+        const needsToUploadOTKs = await this._e2eeAccount.generateOTKsIfNeeded(this._storage);
+        if (needsToUploadOTKs) {
+            // TODO: we could do this in parallel with sync if it proves to be too slow
+            // but I'm not sure how to not swallow errors in that case
+            await this._e2eeAccount.uploadKeys(this._storage);
+        }
+    }
+
     get syncToken() {
         return this._syncInfo?.token;
     }
diff --git a/src/matrix/Sync.js b/src/matrix/Sync.js
index e14451a9..c7aaaa99 100644
--- a/src/matrix/Sync.js
+++ b/src/matrix/Sync.js
@@ -100,6 +100,12 @@ export class Sync {
                     this._status.set(SyncStatus.Stopped);
                 }
             }
+            try {
+                await this._session.afterSyncCompleted();
+            } catch (err) {
+                console.err("error during after sync completed, continuing to sync.",  err.stack);
+                // swallowing error here apart from logging
+            }
         }
     }
 
diff --git a/src/matrix/e2ee/Account.js b/src/matrix/e2ee/Account.js
index 90fabc72..b8c39826 100644
--- a/src/matrix/e2ee/Account.js
+++ b/src/matrix/e2ee/Account.js
@@ -95,6 +95,31 @@ export class Account {
         }
     }
 
+    async generateOTKsIfNeeded(storage) {
+        const maxOTKs = this._account.max_number_of_one_time_keys();
+        const limit = maxOTKs / 2;
+        if (this._serverOTKCount < limit) {
+            // TODO: cache unpublishedOTKCount, so we don't have to parse this JSON on every sync iteration
+            // for now, we only determine it when serverOTKCount is sufficiently low, which is should rarely be,
+            // and recheck
+            const oneTimeKeys = JSON.parse(this._account.one_time_keys());
+            const oneTimeKeysEntries = Object.entries(oneTimeKeys.curve25519);
+            const unpublishedOTKCount = oneTimeKeysEntries.length;
+            const totalOTKCount = this._serverOTKCount + unpublishedOTKCount;
+            if (totalOTKCount < limit) {
+                // we could in theory also generated the keys and store them in
+                // writeSync, but then we would have to clone the account to avoid side-effects.
+                await this._updateSessionStorage(storage, sessionStore => {
+                    const newKeyCount = maxOTKs - totalOTKCount;
+                    this._account.generate_one_time_keys(newKeyCount);
+                    sessionStore.set(ACCOUNT_SESSION_KEY, this._account.pickle(this._pickleKey));
+                });
+                return true;
+            }
+        }
+        return false;
+    }
+
     writeSync(deviceOneTimeKeysCount, txn) {
         // we only upload signed_curve25519 otks
         const otkCount = deviceOneTimeKeysCount.signed_curve25519;

From e751333bbdf2978af811b1c4876eb05b95d277fd Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 28 Aug 2020 13:58:42 +0200
Subject: [PATCH 020/173] don't assume setting up a session went all the way
 through when stopping

---
 src/matrix/SessionContainer.js | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/matrix/SessionContainer.js b/src/matrix/SessionContainer.js
index ae9572a9..1b6e21d8 100644
--- a/src/matrix/SessionContainer.js
+++ b/src/matrix/SessionContainer.js
@@ -237,10 +237,16 @@ export class SessionContainer {
     }
 
     stop() {
-        this._reconnectSubscription();
-        this._reconnectSubscription = null;
-        this._sync.stop();
-        this._session.stop();
+        if (this._reconnectSubscription) {
+            this._reconnectSubscription();
+            this._reconnectSubscription = null;
+        }
+        if (this._sync) {
+            this._sync.stop();
+        }
+        if (this._session) {
+            this._session.stop();
+        }
         if (this._waitForFirstSyncHandle) {
             this._waitForFirstSyncHandle.dispose();
             this._waitForFirstSyncHandle = null;

From 693682f360d6c1bade10da4b3dd57314cfb0da2a Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 28 Aug 2020 14:35:47 +0200
Subject: [PATCH 021/173] move e2ee constants to common file

---
 src/matrix/e2ee/Account.js |  4 +---
 src/matrix/e2ee/common.js  | 20 ++++++++++++++++++++
 2 files changed, 21 insertions(+), 3 deletions(-)
 create mode 100644 src/matrix/e2ee/common.js

diff --git a/src/matrix/e2ee/Account.js b/src/matrix/e2ee/Account.js
index b8c39826..4905bbb6 100644
--- a/src/matrix/e2ee/Account.js
+++ b/src/matrix/e2ee/Account.js
@@ -15,14 +15,12 @@ limitations under the License.
 */
 
 import anotherjson from "../../../lib/another-json/index.js";
+import {SESSION_KEY_PREFIX, OLM_ALGORITHM, MEGOLM_ALGORITHM} from "./common.js";
 
 // use common prefix so it's easy to clear properties that are not e2ee related during session clear
-export const SESSION_KEY_PREFIX = "e2ee:";
 const ACCOUNT_SESSION_KEY = SESSION_KEY_PREFIX + "olmAccount";
 const DEVICE_KEY_FLAG_SESSION_KEY = SESSION_KEY_PREFIX + "areDeviceKeysUploaded";
 const SERVER_OTK_COUNT_SESSION_KEY = SESSION_KEY_PREFIX + "serverOTKCount";
-const OLM_ALGORITHM = "m.olm.v1.curve25519-aes-sha2";
-const MEGOLM_ALGORITHM = "m.megolm.v1.aes-sha2";
 
 export class Account {
     static async load({olm, pickleKey, hsApi, userId, deviceId, txn}) {
diff --git a/src/matrix/e2ee/common.js b/src/matrix/e2ee/common.js
new file mode 100644
index 00000000..82709051
--- /dev/null
+++ b/src/matrix/e2ee/common.js
@@ -0,0 +1,20 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// use common prefix so it's easy to clear properties that are not e2ee related during session clear
+export const SESSION_KEY_PREFIX = "e2ee:";
+export const OLM_ALGORITHM = "m.olm.v1.curve25519-aes-sha2";
+export const MEGOLM_ALGORITHM = "m.megolm.v1.aes-sha2";

From d813e6d9321e14c96a409a1de73b4cc9be1b8266 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 28 Aug 2020 14:36:00 +0200
Subject: [PATCH 022/173] store encryption event content rather than just flag
 in room summary

---
 src/matrix/room/Room.js        |  4 ++++
 src/matrix/room/RoomSummary.js | 14 +++++++++++---
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index 27031203..c7b5e736 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -256,6 +256,10 @@ export class Room extends EventEmitter {
         return !!(tags && tags['m.lowpriority']);
     }
 
+    get isEncrypted() {
+        return !!this._summary.encryption;
+    }
+
     async _getLastEventId() {
         const lastKey = this._syncWriter.lastMessageKey;
         if (lastKey) {
diff --git a/src/matrix/room/RoomSummary.js b/src/matrix/room/RoomSummary.js
index e910c4bc..803aff49 100644
--- a/src/matrix/room/RoomSummary.js
+++ b/src/matrix/room/RoomSummary.js
@@ -14,6 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+import {MEGOLM_ALGORITHM} from "../e2ee/common.js";
+
 function applySyncResponse(data, roomResponse, membership, isInitialSync, isTimelineOpen, ownUserId) {
     if (roomResponse.summary) {
         data = updateSummary(data, roomResponse.summary);
@@ -68,9 +70,10 @@ function processRoomAccountData(data, event) {
 
 function processStateEvent(data, event) {
     if (event.type === "m.room.encryption") {
-        if (!data.isEncrypted) {
+        const algorithm = event.content?.algorithm;
+        if (!data.encryption && algorithm === MEGOLM_ALGORITHM) {
             data = data.cloneIfNeeded();
-            data.isEncrypted = true;
+            data.encryption = event.content;
         }
     } else if (event.type === "m.room.name") {
         const newName = event.content?.name;
@@ -136,7 +139,7 @@ class SummaryData {
         this.lastMessageBody = copy ? copy.lastMessageBody : null;
         this.lastMessageTimestamp = copy ? copy.lastMessageTimestamp : null;
         this.isUnread = copy ? copy.isUnread : false;
-        this.isEncrypted = copy ? copy.isEncrypted : false;
+        this.encryption = copy ? copy.encryption : null;
         this.isDirectMessage = copy ? copy.isDirectMessage : false;
         this.membership = copy ? copy.membership : null;
         this.inviteCount = copy ? copy.inviteCount : 0;
@@ -190,6 +193,11 @@ export class RoomSummary {
         return this._data.heroes;
     }
 
+    get encryption() {
+        return this._data.encryption;
+    }
+
+    // whether the room name should be determined with Heroes
     get needsHeroes() {
         return needsHeroes(this._data);
     }

From 8da00f9a030bfeca021ece14f70c7c5c29812efd Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Mon, 31 Aug 2020 08:53:47 +0200
Subject: [PATCH 023/173] add isTrackingMembers flag to know if EncryptionUsers
 have been written

for this room
---
 src/matrix/room/Room.js        |  4 ++++
 src/matrix/room/RoomSummary.js | 12 ++++++++++++
 2 files changed, 16 insertions(+)

diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index c7b5e736..17214c3a 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -260,6 +260,10 @@ export class Room extends EventEmitter {
         return !!this._summary.encryption;
     }
 
+    get isTrackingMembers() {
+        return this._summary.isTrackingMembers;
+    }
+
     async _getLastEventId() {
         const lastKey = this._syncWriter.lastMessageKey;
         if (lastKey) {
diff --git a/src/matrix/room/RoomSummary.js b/src/matrix/room/RoomSummary.js
index 803aff49..51a31d21 100644
--- a/src/matrix/room/RoomSummary.js
+++ b/src/matrix/room/RoomSummary.js
@@ -147,6 +147,7 @@ class SummaryData {
         this.heroes = copy ? copy.heroes : null;
         this.canonicalAlias = copy ? copy.canonicalAlias : null;
         this.hasFetchedMembers = copy ? copy.hasFetchedMembers : false;
+        this.isTrackingMembers = copy ? copy.isTrackingMembers : false;
         this.lastPaginationToken = copy ? copy.lastPaginationToken : null;
         this.avatarUrl = copy ? copy.avatarUrl : null;
         this.notificationCount = copy ? copy.notificationCount : 0;
@@ -238,6 +239,10 @@ export class RoomSummary {
         return this._data.hasFetchedMembers;
     }
 
+    get isTrackingMembers() {
+        return this._data.isTrackingMembers;
+    }
+
     get lastPaginationToken() {
         return this._data.lastPaginationToken;
     }
@@ -262,6 +267,13 @@ export class RoomSummary {
         return data;
     }
 
+    writeIsTrackingMembers(value, txn) {
+        const data = new SummaryData(this._data);
+        data.isTrackingMembers = value;
+        txn.roomSummary.set(data.serialize());
+        return data;
+    }
+
 	writeSync(roomResponse, membership, isInitialSync, isTimelineOpen, txn) {
         // clear cloned flag, so cloneIfNeeded makes a copy and
         // this._data is not modified if any field is changed.

From 164384f312d83a39865ed7eef014f3131b85d5c9 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Mon, 31 Aug 2020 08:54:27 +0200
Subject: [PATCH 024/173] forgot memberlist member

---
 src/matrix/room/Room.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index 17214c3a..86ab6c35 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -40,6 +40,7 @@ export class Room extends EventEmitter {
         this._timeline = null;
         this._user = user;
         this._changedMembersDuringSync = null;
+        this._memberList = null;
 	}
 
     /** @package */

From 8482bc95ec946cf09d400c3c64d6fa5bc5be7707 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Mon, 31 Aug 2020 09:50:57 +0200
Subject: [PATCH 025/173] pass memberchanges around instead of members

so we can easily tell how their membership changes, (e.g. join -> left)
which we'll need for device tracking.

Not adding this to RoomMember because RoomMember also needs to be
able to represent a member loaded from storage which doesn't contain
this error. A MemberChange exists only within a sync.
---
 src/matrix/room/Room.js                       | 16 ++++----
 src/matrix/room/members/Heroes.js             | 10 ++---
 src/matrix/room/members/MemberList.js         |  6 +--
 src/matrix/room/members/RoomMember.js         | 27 +++++++++++++
 .../room/timeline/persistence/SyncWriter.js   | 40 ++++++++++---------
 5 files changed, 64 insertions(+), 35 deletions(-)

diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index 86ab6c35..f50b221b 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -51,7 +51,7 @@ export class Room extends EventEmitter {
             membership,
             isInitialSync, isTimelineOpen,
             txn);
-		const {entries, newLiveKey, changedMembers} = await this._syncWriter.writeSync(roomResponse, txn);
+		const {entries, newLiveKey, memberChanges} = await this._syncWriter.writeSync(roomResponse, txn);
         // fetch new members while we have txn open,
         // but don't make any in-memory changes yet
         let heroChanges;
@@ -60,7 +60,7 @@ export class Room extends EventEmitter {
             if (!this._heroes) {
                 this._heroes = new Heroes(this._roomId);
             }
-            heroChanges = await this._heroes.calculateChanges(summaryChanges.heroes, changedMembers, txn);
+            heroChanges = await this._heroes.calculateChanges(summaryChanges.heroes, memberChanges, txn);
         }
         let removedPendingEvents;
         if (roomResponse.timeline && roomResponse.timeline.events) {
@@ -71,22 +71,22 @@ export class Room extends EventEmitter {
             newTimelineEntries: entries,
             newLiveKey,
             removedPendingEvents,
-            changedMembers,
+            memberChanges,
             heroChanges
         };
     }
 
     /** @package */
-    afterSync({summaryChanges, newTimelineEntries, newLiveKey, removedPendingEvents, changedMembers, heroChanges}) {
+    afterSync({summaryChanges, newTimelineEntries, newLiveKey, removedPendingEvents, memberChanges, heroChanges}) {
         this._syncWriter.afterSync(newLiveKey);
-        if (changedMembers.length) {
+        if (memberChanges.size) {
             if (this._changedMembersDuringSync) {
-                for (const member of changedMembers) {
-                    this._changedMembersDuringSync.set(member.userId, member);
+                for (const [userId, memberChange] of memberChanges.entries()) {
+                    this._changedMembersDuringSync.set(userId, memberChange.member);
                 }
             }
             if (this._memberList) {
-                this._memberList.afterSync(changedMembers);
+                this._memberList.afterSync(memberChanges);
             }
         }
         let emitChange = false;
diff --git a/src/matrix/room/members/Heroes.js b/src/matrix/room/members/Heroes.js
index f7ccb2df..809b61c2 100644
--- a/src/matrix/room/members/Heroes.js
+++ b/src/matrix/room/members/Heroes.js
@@ -42,11 +42,11 @@ export class Heroes {
 
     /**
      * @param  {string[]} newHeroes      array of user ids
-     * @param  {RoomMember[]} changedMembers array of changed members in this sync
+     * @param  {Map} memberChanges map of changed memberships
      * @param  {Transaction} txn
      * @return {Promise}
      */
-    async calculateChanges(newHeroes, changedMembers, txn) {
+    async calculateChanges(newHeroes, memberChanges, txn) {
         const updatedHeroMembers = new Map();
         const removedUserIds = [];
         // remove non-present members
@@ -56,9 +56,9 @@ export class Heroes {
             }
         }
         // update heroes with synced member changes
-        for (const member of changedMembers) {
-            if (this._members.has(member.userId) || newHeroes.indexOf(member.userId) !== -1) {
-                updatedHeroMembers.set(member.userId, member);
+        for (const [userId, memberChange] of memberChanges.entries()) {
+            if (this._members.has(userId) || newHeroes.indexOf(userId) !== -1) {
+                updatedHeroMembers.set(userId, memberChange.member);
             }
         }
         // load member for new heroes from storage
diff --git a/src/matrix/room/members/MemberList.js b/src/matrix/room/members/MemberList.js
index f428ed6c..734887fd 100644
--- a/src/matrix/room/members/MemberList.js
+++ b/src/matrix/room/members/MemberList.js
@@ -26,9 +26,9 @@ export class MemberList {
         this._retentionCount = 1;
     }
 
-    afterSync(updatedMembers) {
-        for (const member of updatedMembers) {
-            this._members.add(member.userId, member);
+    afterSync(memberChanges) {
+        for (const [userId, memberChange] of memberChanges.entries()) {
+            this._members.add(userId, memberChange.member);
         }
     }
 
diff --git a/src/matrix/room/members/RoomMember.js b/src/matrix/room/members/RoomMember.js
index 02c3c292..27a2e59f 100644
--- a/src/matrix/room/members/RoomMember.js
+++ b/src/matrix/room/members/RoomMember.js
@@ -99,3 +99,30 @@ export class RoomMember {
         return this._data;
     }
 }
+
+export class MemberChange {
+    constructor(roomId, memberEvent) {
+        this._roomId = roomId;
+        this._memberEvent = memberEvent;
+        this._member = null;
+    }
+
+    get member() {
+        if (!this._member) {
+            this._member = RoomMember.fromMemberEvent(this._roomId, this._memberEvent);
+        }
+        return this._member;
+    }
+
+    userId() {
+        return this._memberEvent.state_key;
+    }
+
+    previousMembership() {
+        return this._memberEvent.unsigned?.prev_content?.membership;
+    }
+
+    membership() {
+        return this._memberEvent.content?.membership;
+    }
+}
diff --git a/src/matrix/room/timeline/persistence/SyncWriter.js b/src/matrix/room/timeline/persistence/SyncWriter.js
index 84e8a18f..fdc4035b 100644
--- a/src/matrix/room/timeline/persistence/SyncWriter.js
+++ b/src/matrix/room/timeline/persistence/SyncWriter.js
@@ -18,7 +18,7 @@ import {EventKey} from "../EventKey.js";
 import {EventEntry} from "../entries/EventEntry.js";
 import {FragmentBoundaryEntry} from "../entries/FragmentBoundaryEntry.js";
 import {createEventEntry} from "./common.js";
-import {RoomMember, EVENT_TYPE as MEMBER_EVENT_TYPE} from "../../members/RoomMember.js";
+import {MemberChange, RoomMember, EVENT_TYPE as MEMBER_EVENT_TYPE} from "../../members/RoomMember.js";
 
 // Synapse bug? where the m.room.create event appears twice in sync response
 // when first syncing the room
@@ -102,13 +102,13 @@ export class SyncWriter {
         if (event.type === MEMBER_EVENT_TYPE) {
             const userId = event.state_key;
             if (userId) {
-                const member = RoomMember.fromMemberEvent(this._roomId, event);
-                if (member) {
+                const memberChange = new MemberChange(this._roomId, event);
+                if (memberChange.member) {
                     // as this is sync, we can just replace the member
                     // if it is there already
-                    txn.roomMembers.set(member.serialize());
+                    txn.roomMembers.set(memberChange.member.serialize());
+                    return memberChange;
                 }
-                return member;
             }
         } else {
             txn.roomState.set(this._roomId, event);
@@ -116,22 +116,22 @@ export class SyncWriter {
     }
 
     _writeStateEvents(roomResponse, txn) {
-        const changedMembers = [];
+        const memberChanges = new Map();
         // persist state
         const {state} = roomResponse;
         if (Array.isArray(state?.events)) {
             for (const event of state.events) {
-                const member = this._writeStateEvent(event, txn);
-                if (member) {
-                    changedMembers.push(member);
+                const memberChange = this._writeStateEvent(event, txn);
+                if (memberChange) {
+                    memberChanges.set(memberChange.userId, memberChange);
                 }
             }
         }
-        return changedMembers;
+        return memberChanges;
     }
 
     async _writeTimeline(entries, timeline, currentKey, txn) {
-        const changedMembers = [];
+        const memberChanges = new Map();
         if (timeline.events) {
             const events = deduplicateEvents(timeline.events);
             for(const event of events) {
@@ -148,14 +148,14 @@ export class SyncWriter {
                 
                 // process live state events first, so new member info is available
                 if (typeof event.state_key === "string") {
-                    const member = this._writeStateEvent(event, txn);
-                    if (member) {
-                        changedMembers.push(member);
+                    const memberChange = this._writeStateEvent(event, txn);
+                    if (memberChange) {
+                        memberChanges.set(memberChange.userId, memberChange);
                     }
                 }
             }
         }
-        return {currentKey, changedMembers};
+        return {currentKey, memberChanges};
     }
 
     async _findMemberData(userId, events, txn) {
@@ -198,12 +198,14 @@ export class SyncWriter {
         }
         // important this happens before _writeTimeline so
         // members are available in the transaction
-        const changedMembers = this._writeStateEvents(roomResponse, txn);
+        const memberChanges = this._writeStateEvents(roomResponse, txn);
         const timelineResult = await this._writeTimeline(entries, timeline, currentKey, txn);
         currentKey = timelineResult.currentKey;
-        changedMembers.push(...timelineResult.changedMembers);
-
-        return {entries, newLiveKey: currentKey, changedMembers};
+        // merge member changes from state and timeline, giving precedence to the latter
+        for (const [userId, memberChange] of timelineResult.memberChanges.entries()) {
+            memberChanges.set(userId, memberChange);
+        }
+        return {entries, newLiveKey: currentKey, memberChanges};
     }
 
     afterSync(newLiveKey) {

From 8b358379e85fa4c47f2c27ce5cf4e71380c001b0 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Mon, 31 Aug 2020 14:11:08 +0200
Subject: [PATCH 026/173] first draft of device tracker

mainly missing race protection with /sync and actually running the code
---
 src/matrix/e2ee/DeviceTracker.js              | 275 ++++++++++++++++++
 src/matrix/room/Room.js                       |  10 +
 src/matrix/room/members/RoomMember.js         |   6 +-
 src/matrix/storage/idb/QueryTarget.js         |   7 +
 .../storage/idb/stores/RoomMemberStore.js     |  20 ++
 src/observable/map/ObservableMap.js           |   4 +
 6 files changed, 321 insertions(+), 1 deletion(-)
 create mode 100644 src/matrix/e2ee/DeviceTracker.js

diff --git a/src/matrix/e2ee/DeviceTracker.js b/src/matrix/e2ee/DeviceTracker.js
new file mode 100644
index 00000000..e13b3a5c
--- /dev/null
+++ b/src/matrix/e2ee/DeviceTracker.js
@@ -0,0 +1,275 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import anotherjson from "../../../lib/another-json/index.js";
+
+const TRACKING_STATUS_OUTDATED = 0;
+const TRACKING_STATUS_UPTODATE = 1;
+
+const DEVICE_KEYS_SIGNATURE_ALGORITHM = "ed25519";
+
+// map 1 device from /keys/query response to DeviceIdentity
+function deviceKeysAsDeviceIdentity(deviceSection) {
+    const deviceId = deviceSection["device_id"];
+    const userId = deviceSection["userId_id"];
+    return {
+        userId,
+        deviceId,
+        ed25519Key: deviceSection.keys?.[`ed25519:${deviceId}`],
+        curve25519Key: deviceSection.keys?.[`curve25519:${deviceId}`],
+        algorithms: deviceSection.algorithms,
+        displayName: deviceSection.unsigned?.device_display_name,
+    };
+}
+
+export class DeviceTracker {
+    constructor({storage, getSyncToken, olm}) {
+        this._storage = storage;
+        this._getSyncToken = getSyncToken;
+        this._identityChangedForRoom = null;
+        this._olm = olm;
+    }
+
+    async writeDeviceChanges(deviceLists, txn) {
+        const {userIdentities} = txn;
+        if (Array.isArray(deviceLists.changed) && deviceLists.changed.length) {
+            await Promise.all(deviceLists.changed.map(async userId => {
+                const user = await userIdentities.get(userId)
+                user.deviceTrackingStatus = TRACKING_STATUS_OUTDATED;
+                userIdentities.set(user);
+            }));
+        }
+    }
+
+    writeMemberChanges(room, memberChanges, txn) {
+        return Promise.all(Array.from(memberChanges.values()).map(async memberChange => {
+            return this._applyMemberChange(memberChange, txn);
+        }));
+    }
+
+    async trackRoom(room) {
+        if (room.isTrackingMembers) {
+            return;
+        }
+        const memberList = await room.loadMemberList();
+        try {
+            const txn = await this._storage.readWriteTxn([
+                this._storage.storeNames.roomSummary,
+                this._storage.storeNames.userIdentities,
+            ]);
+            let isTrackingChanges;
+            try {
+                isTrackingChanges = room.writeIsTrackingMembers(true, txn);
+                const members = Array.from(memberList.members.values());
+                await this._writeJoinedMembers(members, txn);
+            } catch (err) {
+                txn.abort();
+                throw err;
+            }
+            await txn.complete();
+            room.applyIsTrackingMembersChanges(isTrackingChanges);
+        } finally {
+            memberList.release();
+        }
+    }
+
+    async _writeJoinedMembers(members, txn) {
+        await Promise.all(members.map(async member => {
+            if (member.membership === "join") {
+                await this._writeMember(member, txn);
+            }
+        }));
+    }
+
+    async _writeMember(member, txn) {
+        const {userIdentities} = txn;
+        const identity = await userIdentities.get(member.userId);
+        if (!identity) {
+            userIdentities.set({
+                userId: member.userId,
+                roomIds: [member.roomId],
+                deviceTrackingStatus: TRACKING_STATUS_OUTDATED,
+            });
+        } else {
+            if (!identity.roomIds.includes(member.roomId)) {
+                identity.roomIds.push(member.roomId);
+                userIdentities.set(identity);
+            }
+        }
+    }
+
+    async _applyMemberChange(memberChange, txn) {
+        // TODO: depends whether we encrypt for invited users??
+        // add room
+        if (memberChange.previousMembership !== "join" && memberChange.membership === "join") {
+            await this._writeMember(memberChange.member, txn);
+        }
+        // remove room
+        else if (memberChange.previousMembership === "join" && memberChange.membership !== "join") {
+            const {userIdentities} = txn;
+            const identity = await userIdentities.get(memberChange.userId);
+            if (identity) {
+                identity.roomIds = identity.roomIds.filter(roomId => roomId !== memberChange.roomId);
+                // no more encrypted rooms with this user, remove
+                if (identity.roomIds.length === 0) {
+                    userIdentities.remove(identity.userId);
+                } else {
+                    userIdentities.set(identity);
+                }
+            }
+        }
+    }
+
+    async _queryKeys(userIds, hsApi) {
+        // TODO: we need to handle the race here between /sync and /keys/query just like we need to do for the member list ...
+        // there are multiple requests going out for /keys/query though and only one for /members
+
+        const deviceKeyResponse = await hsApi.keysQuery({
+            "timeout": 10000,
+            "device_keys": userIds.reduce((deviceKeysMap, userId) => {
+                deviceKeysMap[userId] = [];
+                return deviceKeysMap;
+            }, {}),
+            "token": this._getSyncToken()
+        }).response();
+
+        const verifiedKeysPerUser = this._filterVerifiedDeviceKeys(deviceKeyResponse["device_keys"]);
+        const flattenedVerifiedKeysPerUser = verifiedKeysPerUser.reduce((all, {verifiedKeys}) => all.concat(verifiedKeys), []);
+        const deviceIdentitiesWithPossibleChangedKeys = flattenedVerifiedKeysPerUser.map(deviceKeysAsDeviceIdentity);
+
+        const txn = await this._storage.readWriteTxn([
+            this._storage.storeNames.userIdentities,
+            this._storage.storeNames.deviceIdentities,
+        ]);
+        let deviceIdentities;
+        try {
+            // check ed25519 key has not changed if we've seen the device before
+            deviceIdentities = await Promise.all(deviceIdentitiesWithPossibleChangedKeys.map(async (deviceIdentity) => {
+                const existingDevice = await txn.deviceIdentities.get(deviceIdentity.userId, deviceIdentity.deviceId);
+                if (!existingDevice || existingDevice.ed25519Key === deviceIdentity.ed25519Key) {
+                    return deviceIdentity;
+                }
+                // ignore devices where the keys have changed
+                return null;
+            }));
+            // filter out nulls
+            deviceIdentities = deviceIdentities.filter(di => !!di);
+            // store devices
+            for (const deviceIdentity of deviceIdentities) {
+                txn.deviceIdentities.set(deviceIdentity);
+            }
+            // mark user identities as up to date
+            await Promise.all(verifiedKeysPerUser.map(async ({userId}) => {
+                const identity = await txn.userIdentities.get(userId);
+                identity.deviceTrackingStatus = TRACKING_STATUS_UPTODATE;
+                txn.userIdentities.set(identity);
+            }));
+        } catch (err) {
+            txn.abort();
+            throw err;
+        }
+        await txn.complete();
+        return deviceIdentities;
+    }
+
+    _filterVerifiedDeviceKeys(keyQueryDeviceKeysResponse) {
+        const verifiedKeys = Object.entries(keyQueryDeviceKeysResponse).map((userId, keysByDevice) => {
+            const verifiedKeys = Object.entries(keysByDevice).filter((deviceId, deviceKeys) => {
+                const deviceIdOnKeys = deviceKeys["device_id"];
+                const userIdOnKeys = deviceKeys["user_id"];
+                if (userIdOnKeys !== userId) {
+                    return false;
+                }
+                if (deviceIdOnKeys !== deviceId) {
+                    return false;
+                }
+                return this._verifyUserDeviceKeys(deviceKeys);
+            });
+            return {userId, verifiedKeys};
+        });
+        return verifiedKeys;
+    }
+
+    _verifyUserDeviceKeys(deviceSection) {
+        const deviceId = deviceSection["device_id"];
+        const userId = deviceSection["user_id"];
+        const clone = Object.assign({}, deviceSection);
+        delete clone.unsigned;
+        delete clone.signatures;
+        const canonicalJson = anotherjson.stringify(clone);
+        const key = deviceSection?.keys?.[`${DEVICE_KEYS_SIGNATURE_ALGORITHM}:${deviceId}`];
+        const signature = deviceSection?.signatures?.[userId]?.[`${DEVICE_KEYS_SIGNATURE_ALGORITHM}:${deviceId}`];
+        try {
+            if (!signature) {
+                throw new Error("no signature");
+            }
+            // throws when signature is invalid
+            this._olm.Utility.ed25519_verify(key, canonicalJson, signature);
+            return true;
+        } catch (err) {
+            console.warn("Invalid device signature, ignoring device.", key, canonicalJson, signature, err);
+            return false;
+        }
+    }
+
+    /**
+     * Gives all the device identities for a room that is already tracked.
+     * Assumes room is already tracked. Call `trackRoom` first if unsure.
+     * @param  {String} roomId [description]
+     * @return {[type]}        [description]
+     */
+    async deviceIdentitiesForTrackedRoom(roomId, hsApi) {
+        let identities;
+        const txn = await this._storage.readTxn([
+            this._storage.storeNames.roomMembers,
+            this._storage.storeNames.userIdentities,
+        ]);
+
+        // because we don't have multiEntry support in IE11, we get a set of userIds that is pretty close to what we
+        // need as a good first filter (given that non-join memberships will be in there). After fetching the identities,
+        // we check which ones have the roomId for the room we're looking at.
+        
+        // So, this will also contain non-joined memberships
+        const userIds = await txn.roomMembers.getAllUserIds();
+        const allMemberIdentities = await Promise.all(userIds.map(userId => txn.userIdentities.get(userId)));
+        identities = allMemberIdentities.filter(identity => {
+            // identity will be missing for any userIds that don't have 
+            // membership join in any of your encrypted rooms
+            return identity && identity.roomIds.includes(roomId);
+        });
+        const upToDateIdentities = identities.filter(i => i.deviceTrackingStatus === TRACKING_STATUS_UPTODATE);
+        const outdatedIdentities = identities.filter(i => i.deviceTrackingStatus === TRACKING_STATUS_OUTDATED);
+        let queriedDevices;
+        if (outdatedIdentities.length) {
+            // TODO: ignore the race between /sync and /keys/query for now,
+            // where users could get marked as outdated or added/removed from the room while
+            // querying keys
+            queriedDevices = await this._queryKeys(outdatedIdentities.map(i => i.userId), hsApi);
+        }
+
+        const deviceTxn = await this._storage.readTxn([
+            this._storage.storeNames.deviceIdentities,
+        ]);
+        const devicesPerUser = await Promise.all(upToDateIdentities.map(identity => {
+            return deviceTxn.deviceIdentities.getAllForUserId(identity.userId);
+        }));
+        let flattenedDevices = devicesPerUser.reduce((all, devicesForUser) => all.concat(devicesForUser), []);
+        if (queriedDevices && queriedDevices.length) {
+            flattenedDevices = flattenedDevices.concat(queriedDevices);
+        }
+        return flattenedDevices;
+    }
+}
diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index f50b221b..6b23ad35 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -331,5 +331,15 @@ export class Room extends EventEmitter {
     get mediaRepository() {
         return this._hsApi.mediaRepository;
     }
+
+    /** @package */
+    writeIsTrackingMembers(value, txn) {
+        return this._summary.writeIsTrackingMembers(value, txn);
+    }
+
+    /** @package */
+    applyIsTrackingMembersChanges(changes) {
+        this._summary.applyChanges(changes);
+    }
 }
 
diff --git a/src/matrix/room/members/RoomMember.js b/src/matrix/room/members/RoomMember.js
index 27a2e59f..0c05b4cb 100644
--- a/src/matrix/room/members/RoomMember.js
+++ b/src/matrix/room/members/RoomMember.js
@@ -114,7 +114,11 @@ export class MemberChange {
         return this._member;
     }
 
-    userId() {
+    get roomId() {
+        return this._roomId;
+    }
+
+    get userId() {
         return this._memberEvent.state_key;
     }
 
diff --git a/src/matrix/storage/idb/QueryTarget.js b/src/matrix/storage/idb/QueryTarget.js
index 0738df60..fd3050bd 100644
--- a/src/matrix/storage/idb/QueryTarget.js
+++ b/src/matrix/storage/idb/QueryTarget.js
@@ -105,6 +105,13 @@ export class QueryTarget {
         return maxKey;
     }
 
+    async iterateKeys(range, callback) {
+        const cursor = this._target.openKeyCursor(range, "next");
+        await iterateCursor(cursor, (_, key) => {
+            return {done: callback(key)};
+        });
+    }
+
     /**
      * Checks if a given set of keys exist.
      * Calls `callback(key, found)` for each key in `keys`, in key sorting order (or reversed if backwards=true).
diff --git a/src/matrix/storage/idb/stores/RoomMemberStore.js b/src/matrix/storage/idb/stores/RoomMemberStore.js
index aa979056..75649935 100644
--- a/src/matrix/storage/idb/stores/RoomMemberStore.js
+++ b/src/matrix/storage/idb/stores/RoomMemberStore.js
@@ -19,6 +19,11 @@ function encodeKey(roomId, userId) {
     return `${roomId}|${userId}`;
 }
 
+function decodeKey(key) {
+    const [roomId, userId] = key.split("|");
+    return {roomId, userId};
+}
+
 // no historical members
 export class RoomMemberStore {
     constructor(roomMembersStore) {
@@ -40,4 +45,19 @@ export class RoomMemberStore {
             return member.roomId === roomId;
         });
     }
+
+    async getAllUserIds(roomId) {
+        const userIds = [];
+        const range = IDBKeyRange.lowerBound(encodeKey(roomId, ""));
+        await this._roomMembersStore.iterateKeys(range, key => {
+            const decodedKey = decodedKey(key);
+            // prevent running into the next room
+            if (decodedKey.roomId === roomId) {
+                userIds.push(decodedKey.userId);
+                return false;   // fetch more
+            }
+            return true; // done
+        });
+        return userIds;
+    }
 }
diff --git a/src/observable/map/ObservableMap.js b/src/observable/map/ObservableMap.js
index 68e64c89..7fe10d95 100644
--- a/src/observable/map/ObservableMap.js
+++ b/src/observable/map/ObservableMap.js
@@ -70,6 +70,10 @@ export class ObservableMap extends BaseObservableMap {
     [Symbol.iterator]() {
         return this._values.entries();
     }
+
+    values() {
+        return this._values.values();
+    }
 }
 
 export function tests() {

From afb9ae439167958590a2a32aa30a2e0b056f0cea Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Mon, 31 Aug 2020 14:13:21 +0200
Subject: [PATCH 027/173] hook up device tracker with sync

---
 src/matrix/Session.js | 19 ++++++++++++++++++-
 src/matrix/Sync.js    |  2 +-
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index bc52227a..6cf9225c 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -19,6 +19,7 @@ import { ObservableMap } from "../observable/index.js";
 import { SendScheduler, RateLimitingBackoff } from "./SendScheduler.js";
 import {User} from "./User.js";
 import {Account as E2EEAccount} from "./e2ee/Account.js";
+import {DeviceTracker} from "./e2ee/DeviceTracker.js";
 const PICKLE_KEY = "DEFAULT_KEY";
 
 export class Session {
@@ -34,6 +35,11 @@ export class Session {
         this._user = new User(sessionInfo.userId);
         this._olm = olm;
         this._e2eeAccount = null;
+        this._deviceTracker = olm ? new DeviceTracker({
+            storage,
+            getSyncToken: () => this.syncToken,
+            olm,
+        }) : null;
     }
 
     async beforeFirstSync(isNewLogin) {
@@ -152,7 +158,7 @@ export class Session {
         return room;
     }
 
-    writeSync(syncResponse, syncFilterId, txn) {
+    async writeSync(syncResponse, syncFilterId, roomChanges, txn) {
         const changes = {};
         const syncToken = syncResponse.next_batch;
         const deviceOneTimeKeysCount = syncResponse.device_one_time_keys_count;
@@ -166,6 +172,17 @@ export class Session {
             txn.session.set("sync", syncInfo);
             changes.syncInfo = syncInfo;
         }
+        if (this._deviceTracker) {
+            const deviceLists = syncResponse.device_lists;
+            if (deviceLists) {
+                await this._deviceTracker.writeDeviceChanges(deviceLists, txn);
+            }
+            for (const {room, changes} of roomChanges) {
+                if (room.isTrackingMembers && changes.memberChanges?.size) {
+                    await this._deviceTracker.writeMemberChanges(room, changes.memberChanges, txn);
+                }
+            } 
+        }
         return changes;
     }
 
diff --git a/src/matrix/Sync.js b/src/matrix/Sync.js
index c7aaaa99..4da24ba6 100644
--- a/src/matrix/Sync.js
+++ b/src/matrix/Sync.js
@@ -133,7 +133,6 @@ export class Sync {
         const roomChanges = [];
         let sessionChanges;
         try {
-            sessionChanges = this._session.writeSync(response, syncFilterId, syncTxn);
             // to_device
             // presence
             if (response.rooms) {
@@ -153,6 +152,7 @@ export class Sync {
                 });
                 await Promise.all(promises);
             }
+            sessionChanges = await this._session.writeSync(response, syncFilterId, roomChanges, syncTxn);
         } catch(err) {
             console.warn("aborting syncTxn because of error");
             console.error(err);

From 09cb39b5538da89372b0e7c206ef920af31973f7 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Mon, 31 Aug 2020 14:13:36 +0200
Subject: [PATCH 028/173] don't run afterSyncCompleted when there was an error

---
 src/matrix/Sync.js | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/matrix/Sync.js b/src/matrix/Sync.js
index 4da24ba6..5587d2b0 100644
--- a/src/matrix/Sync.js
+++ b/src/matrix/Sync.js
@@ -100,11 +100,13 @@ export class Sync {
                     this._status.set(SyncStatus.Stopped);
                 }
             }
-            try {
-                await this._session.afterSyncCompleted();
-            } catch (err) {
-                console.err("error during after sync completed, continuing to sync.",  err.stack);
-                // swallowing error here apart from logging
+            if (!this._error) {
+                try {
+                    await this._session.afterSyncCompleted();
+                } catch (err) {
+                    console.err("error during after sync completed, continuing to sync.",  err.stack);
+                    // swallowing error here apart from logging
+                }
             }
         }
     }

From 2e67b2b6b8ae1d82609369702df624bc1a8ec589 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Mon, 31 Aug 2020 14:21:18 +0200
Subject: [PATCH 029/173] handle prev_content location ambiguity

---
 src/matrix/room/common.js                     | 21 +++++++++++++++++++
 src/matrix/room/members/RoomMember.js         |  9 ++++----
 .../room/timeline/entries/EventEntry.js       |  3 ++-
 3 files changed, 28 insertions(+), 5 deletions(-)
 create mode 100644 src/matrix/room/common.js

diff --git a/src/matrix/room/common.js b/src/matrix/room/common.js
new file mode 100644
index 00000000..922ca115
--- /dev/null
+++ b/src/matrix/room/common.js
@@ -0,0 +1,21 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export function getPrevContentFromStateEvent(event) {
+    // where to look for prev_content is a bit of a mess,
+    // see https://matrix.to/#/!NasysSDfxKxZBzJJoE:matrix.org/$DvrAbZJiILkOmOIuRsNoHmh2v7UO5CWp_rYhlGk34fQ?via=matrix.org&via=pixie.town&via=amorgan.xyz
+    return event.unsigned?.prev_content || event.prev_content;
+}
diff --git a/src/matrix/room/members/RoomMember.js b/src/matrix/room/members/RoomMember.js
index 0c05b4cb..c1c9f93b 100644
--- a/src/matrix/room/members/RoomMember.js
+++ b/src/matrix/room/members/RoomMember.js
@@ -1,5 +1,4 @@
 /*
-Copyright 2020 Bruno Windels 
 Copyright 2020 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,6 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+import {getPrevContentFromStateEvent} from "../common.js";
+
 export const EVENT_TYPE = "m.room.member";
 
 export class RoomMember {
@@ -28,7 +29,7 @@ export class RoomMember {
             return;
         }
         const content = memberEvent.content;
-        const prevContent = memberEvent.unsigned?.prev_content;
+        const prevContent = getPrevContentFromStateEvent(memberEvent);
         const membership = content?.membership;
         // fall back to prev_content for these as synapse doesn't (always?)
         // put them on content for "leave" memberships
@@ -45,7 +46,7 @@ export class RoomMember {
         if (typeof userId !== "string") {
             return;
         }
-        const content = memberEvent.unsigned?.prev_content
+        const content = getPrevContentFromStateEvent(memberEvent);
         return this._validateAndCreateMember(roomId, userId,
             content?.membership,
             content?.displayname,
@@ -123,7 +124,7 @@ export class MemberChange {
     }
 
     previousMembership() {
-        return this._memberEvent.unsigned?.prev_content?.membership;
+        return getPrevContentFromStateEvent(this._memberEvent)?.membership;
     }
 
     membership() {
diff --git a/src/matrix/room/timeline/entries/EventEntry.js b/src/matrix/room/timeline/entries/EventEntry.js
index d1d5b64c..4dce9834 100644
--- a/src/matrix/room/timeline/entries/EventEntry.js
+++ b/src/matrix/room/timeline/entries/EventEntry.js
@@ -15,6 +15,7 @@ limitations under the License.
 */
 
 import {BaseEntry} from "./BaseEntry.js";
+import {getPrevContentFromStateEvent} from "../../common.js";
 
 export class EventEntry extends BaseEntry {
     constructor(eventEntry, fragmentIdComparer) {
@@ -35,7 +36,7 @@ export class EventEntry extends BaseEntry {
     }
 
     get prevContent() {
-        return this._eventEntry.event.unsigned?.prev_content;
+        return getPrevContentFromStateEvent(this._eventEntry.event);
     }
 
     get eventType() {

From 4ef5d4b3b844e9a90c35d1b35e1b03e3bf4230e7 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Mon, 31 Aug 2020 14:24:09 +0200
Subject: [PATCH 030/173] implement hsapi /keys/query method

---
 src/matrix/e2ee/DeviceTracker.js | 2 +-
 src/matrix/net/HomeServerApi.js  | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/matrix/e2ee/DeviceTracker.js b/src/matrix/e2ee/DeviceTracker.js
index e13b3a5c..032f5648 100644
--- a/src/matrix/e2ee/DeviceTracker.js
+++ b/src/matrix/e2ee/DeviceTracker.js
@@ -137,7 +137,7 @@ export class DeviceTracker {
         // TODO: we need to handle the race here between /sync and /keys/query just like we need to do for the member list ...
         // there are multiple requests going out for /keys/query though and only one for /members
 
-        const deviceKeyResponse = await hsApi.keysQuery({
+        const deviceKeyResponse = await hsApi.queryKeys({
             "timeout": 10000,
             "device_keys": userIds.reduce((deviceKeysMap, userId) => {
                 deviceKeysMap[userId] = [];
diff --git a/src/matrix/net/HomeServerApi.js b/src/matrix/net/HomeServerApi.js
index ac55f0a5..42d1b0e0 100644
--- a/src/matrix/net/HomeServerApi.js
+++ b/src/matrix/net/HomeServerApi.js
@@ -164,6 +164,10 @@ export class HomeServerApi {
         return this._post("/keys/upload", null, payload, options);
     }
 
+    queryKeys(queryRequest, options = null) {
+        return this._post("/keys/query", null, queryRequest, options);
+    }
+
     get mediaRepository() {
         return this._mediaRepository;
     }

From 8b7fdb2c61787dda69c2d4b7572f3fbf879ae11d Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Mon, 31 Aug 2020 14:38:03 +0200
Subject: [PATCH 031/173] create user & device identity stores

---
 src/matrix/storage/common.js                  |  2 +
 src/matrix/storage/idb/Transaction.js         | 10 +++++
 src/matrix/storage/idb/schema.js              |  8 +++-
 .../storage/idb/stores/DeviceIdentityStore.js | 41 +++++++++++++++++++
 .../storage/idb/stores/UserIdentityStore.js   | 33 +++++++++++++++
 5 files changed, 93 insertions(+), 1 deletion(-)
 create mode 100644 src/matrix/storage/idb/stores/DeviceIdentityStore.js
 create mode 100644 src/matrix/storage/idb/stores/UserIdentityStore.js

diff --git a/src/matrix/storage/common.js b/src/matrix/storage/common.js
index 0cf5b9b0..7d6fae09 100644
--- a/src/matrix/storage/common.js
+++ b/src/matrix/storage/common.js
@@ -22,6 +22,8 @@ export const STORE_NAMES = Object.freeze([
     "timelineEvents",
     "timelineFragments",
     "pendingEvents",
+    "userIdentities",
+    "deviceIdentities",
 ]);
 
 export const STORE_MAP = Object.freeze(STORE_NAMES.reduce((nameMap, name) => {
diff --git a/src/matrix/storage/idb/Transaction.js b/src/matrix/storage/idb/Transaction.js
index 4f5e3af5..921c23e2 100644
--- a/src/matrix/storage/idb/Transaction.js
+++ b/src/matrix/storage/idb/Transaction.js
@@ -24,6 +24,8 @@ import {RoomStateStore} from "./stores/RoomStateStore.js";
 import {RoomMemberStore} from "./stores/RoomMemberStore.js";
 import {TimelineFragmentStore} from "./stores/TimelineFragmentStore.js";
 import {PendingEventStore} from "./stores/PendingEventStore.js";
+import {UserIdentityStore} from "./stores/UserIdentityStore.js";
+import {DeviceIdentityStore} from "./stores/DeviceIdentityStore.js";
 
 export class Transaction {
     constructor(txn, allowedStoreNames) {
@@ -81,6 +83,14 @@ export class Transaction {
         return this._store("pendingEvents", idbStore => new PendingEventStore(idbStore));
     }
 
+    get userIdentities() {
+        return this._store("userIdentities", idbStore => new UserIdentityStore(idbStore));
+    }
+
+    get deviceIdentities() {
+        return this._store("deviceIdentities", idbStore => new DeviceIdentityStore(idbStore));
+    }
+    
     complete() {
         return txnAsPromise(this._txn);
     }
diff --git a/src/matrix/storage/idb/schema.js b/src/matrix/storage/idb/schema.js
index 0ab5707a..d8aa81cc 100644
--- a/src/matrix/storage/idb/schema.js
+++ b/src/matrix/storage/idb/schema.js
@@ -9,6 +9,7 @@ export const schema = [
     createInitialStores,
     createMemberStore,
     migrateSession,
+    createIdentityStores,
 ];
 // TODO: how to deal with git merge conflicts of this array?
 
@@ -46,7 +47,7 @@ async function createMemberStore(db, txn) {
         }
     });
 }
-
+//v3
 async function migrateSession(db, txn) {
     const session = txn.objectStore("session");
     try {
@@ -64,3 +65,8 @@ async function migrateSession(db, txn) {
         console.error("could not migrate session", err.stack);
     }
 }
+//v4
+function createIdentityStores(db) {
+    db.createObjectStore("userIdentities", {keyPath: "userId"});
+    db.createObjectStore("deviceIdentities", {keyPath: "key"});
+}
diff --git a/src/matrix/storage/idb/stores/DeviceIdentityStore.js b/src/matrix/storage/idb/stores/DeviceIdentityStore.js
new file mode 100644
index 00000000..8bbab1b7
--- /dev/null
+++ b/src/matrix/storage/idb/stores/DeviceIdentityStore.js
@@ -0,0 +1,41 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+function encodeKey(userId, deviceId) {
+    return `${userId}|${deviceId}`;
+}
+
+export class DeviceIdentityStore {
+    constructor(store) {
+        this._store = store;
+    }
+
+    getAllForUserId(userId) {
+        const range = IDBKeyRange.lowerBound(encodeKey(userId, ""));
+        return this._store.selectWhile(range, device => {
+            return device.userId === userId;
+        });
+    }
+
+    get(userId, deviceId) {
+        return this._store.get(encodeKey(userId, deviceId));
+    }
+
+    set(deviceIdentity) {
+        deviceIdentity.key = encodeKey(deviceIdentity.userId, deviceIdentity.deviceId);
+        return this._store.set(deviceIdentity);
+    }
+}
diff --git a/src/matrix/storage/idb/stores/UserIdentityStore.js b/src/matrix/storage/idb/stores/UserIdentityStore.js
new file mode 100644
index 00000000..2eadefec
--- /dev/null
+++ b/src/matrix/storage/idb/stores/UserIdentityStore.js
@@ -0,0 +1,33 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export class UserIdentityStore {
+    constructor(store) {
+        this._store = store;
+    }
+
+    get(userId) {
+        return this._store.get(userId);
+    }
+
+    set(userIdentity) {
+        return this._store.set(userIdentity);
+    }
+
+    remove(userId) {
+        return this._eventStore.delete(userId);
+    }
+}

From 03995623409f800d15423fabc01d4a14be243dcc Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Mon, 31 Aug 2020 16:05:04 +0200
Subject: [PATCH 032/173] fix typo

---
 src/matrix/e2ee/DeviceTracker.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/matrix/e2ee/DeviceTracker.js b/src/matrix/e2ee/DeviceTracker.js
index 032f5648..070a93fa 100644
--- a/src/matrix/e2ee/DeviceTracker.js
+++ b/src/matrix/e2ee/DeviceTracker.js
@@ -24,7 +24,7 @@ const DEVICE_KEYS_SIGNATURE_ALGORITHM = "ed25519";
 // map 1 device from /keys/query response to DeviceIdentity
 function deviceKeysAsDeviceIdentity(deviceSection) {
     const deviceId = deviceSection["device_id"];
-    const userId = deviceSection["userId_id"];
+    const userId = deviceSection["user_id"];
     return {
         userId,
         deviceId,

From 561df456413251a615e73c8405924a6037493183 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Mon, 31 Aug 2020 16:05:21 +0200
Subject: [PATCH 033/173] olm.Utility should be instanciated

---
 src/matrix/e2ee/DeviceTracker.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/matrix/e2ee/DeviceTracker.js b/src/matrix/e2ee/DeviceTracker.js
index 070a93fa..50b9f5e8 100644
--- a/src/matrix/e2ee/DeviceTracker.js
+++ b/src/matrix/e2ee/DeviceTracker.js
@@ -40,7 +40,7 @@ export class DeviceTracker {
         this._storage = storage;
         this._getSyncToken = getSyncToken;
         this._identityChangedForRoom = null;
-        this._olm = olm;
+        this._olmUtil = new olm.Utility();
     }
 
     async writeDeviceChanges(deviceLists, txn) {
@@ -217,7 +217,7 @@ export class DeviceTracker {
                 throw new Error("no signature");
             }
             // throws when signature is invalid
-            this._olm.Utility.ed25519_verify(key, canonicalJson, signature);
+            this._olmUtil.ed25519_verify(key, canonicalJson, signature);
             return true;
         } catch (err) {
             console.warn("Invalid device signature, ignoring device.", key, canonicalJson, signature, err);

From 78c3157a5f9d295632e82381d893321c57f3afcf Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Mon, 31 Aug 2020 16:05:57 +0200
Subject: [PATCH 034/173] fix not taking into account Object.entries yields
 arrays for the pairs

---
 src/matrix/e2ee/DeviceTracker.js | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/matrix/e2ee/DeviceTracker.js b/src/matrix/e2ee/DeviceTracker.js
index 50b9f5e8..41e9d231 100644
--- a/src/matrix/e2ee/DeviceTracker.js
+++ b/src/matrix/e2ee/DeviceTracker.js
@@ -186,8 +186,8 @@ export class DeviceTracker {
     }
 
     _filterVerifiedDeviceKeys(keyQueryDeviceKeysResponse) {
-        const verifiedKeys = Object.entries(keyQueryDeviceKeysResponse).map((userId, keysByDevice) => {
-            const verifiedKeys = Object.entries(keysByDevice).filter((deviceId, deviceKeys) => {
+        const verifiedKeys = Object.entries(keyQueryDeviceKeysResponse).map(([userId, keysByDevice]) => {
+            const verifiedEntries = Object.entries(keysByDevice).filter(([deviceId, deviceKeys]) => {
                 const deviceIdOnKeys = deviceKeys["device_id"];
                 const userIdOnKeys = deviceKeys["user_id"];
                 if (userIdOnKeys !== userId) {
@@ -198,6 +198,7 @@ export class DeviceTracker {
                 }
                 return this._verifyUserDeviceKeys(deviceKeys);
             });
+            const verifiedKeys = verifiedEntries.map(([, deviceKeys]) => deviceKeys);
             return {userId, verifiedKeys};
         });
         return verifiedKeys;

From fef6586e5b8b1c837c4489ffde97c80c39b3504d Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Mon, 31 Aug 2020 16:06:31 +0200
Subject: [PATCH 035/173] actually pass the room id

---
 src/matrix/e2ee/DeviceTracker.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/matrix/e2ee/DeviceTracker.js b/src/matrix/e2ee/DeviceTracker.js
index 41e9d231..3c64f81f 100644
--- a/src/matrix/e2ee/DeviceTracker.js
+++ b/src/matrix/e2ee/DeviceTracker.js
@@ -244,7 +244,7 @@ export class DeviceTracker {
         // we check which ones have the roomId for the room we're looking at.
         
         // So, this will also contain non-joined memberships
-        const userIds = await txn.roomMembers.getAllUserIds();
+        const userIds = await txn.roomMembers.getAllUserIds(roomId);
         const allMemberIdentities = await Promise.all(userIds.map(userId => txn.userIdentities.get(userId)));
         identities = allMemberIdentities.filter(identity => {
             // identity will be missing for any userIds that don't have 

From d43cdfd8896dd052fd199e6aa7119f8d8a94b61c Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Mon, 31 Aug 2020 16:06:40 +0200
Subject: [PATCH 036/173] don't crash when tracked user is not there

---
 src/matrix/e2ee/DeviceTracker.js | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/matrix/e2ee/DeviceTracker.js b/src/matrix/e2ee/DeviceTracker.js
index 3c64f81f..5eaa23b7 100644
--- a/src/matrix/e2ee/DeviceTracker.js
+++ b/src/matrix/e2ee/DeviceTracker.js
@@ -47,9 +47,13 @@ export class DeviceTracker {
         const {userIdentities} = txn;
         if (Array.isArray(deviceLists.changed) && deviceLists.changed.length) {
             await Promise.all(deviceLists.changed.map(async userId => {
-                const user = await userIdentities.get(userId)
-                user.deviceTrackingStatus = TRACKING_STATUS_OUTDATED;
-                userIdentities.set(user);
+                const user = await userIdentities.get(userId);
+                if (user) {
+                    user.deviceTrackingStatus = TRACKING_STATUS_OUTDATED;
+                    userIdentities.set(user);
+                } else {
+                    console.warn("changed device userid not found", userId);
+                }
             }));
         }
     }

From 4fd3e2ab20e59f5b9aac7c01b76b3cd7a2ea99a3 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Mon, 31 Aug 2020 16:07:04 +0200
Subject: [PATCH 037/173] response is a method

---
 src/matrix/room/members/load.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/matrix/room/members/load.js b/src/matrix/room/members/load.js
index 54d7c3dc..667bec96 100644
--- a/src/matrix/room/members/load.js
+++ b/src/matrix/room/members/load.js
@@ -31,7 +31,7 @@ async function fetchMembers({summary, roomId, hsApi, storage, setChangedMembersM
     const changedMembersDuringSync = new Map();
     setChangedMembersMap(changedMembersDuringSync);
     
-    const memberResponse = await hsApi.members(roomId, {at: summary.lastPaginationToken}).response;
+    const memberResponse = await hsApi.members(roomId, {at: summary.lastPaginationToken}).response();
 
     const txn = await storage.readWriteTxn([
         storage.storeNames.roomSummary,

From 374dce638d9f6a6eced928ada32814c80c65487a Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Mon, 31 Aug 2020 16:07:18 +0200
Subject: [PATCH 038/173] these are assumed to be getters

---
 src/matrix/room/members/RoomMember.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/matrix/room/members/RoomMember.js b/src/matrix/room/members/RoomMember.js
index c1c9f93b..0ce205a5 100644
--- a/src/matrix/room/members/RoomMember.js
+++ b/src/matrix/room/members/RoomMember.js
@@ -123,11 +123,11 @@ export class MemberChange {
         return this._memberEvent.state_key;
     }
 
-    previousMembership() {
+    get previousMembership() {
         return getPrevContentFromStateEvent(this._memberEvent)?.membership;
     }
 
-    membership() {
+    get membership() {
         return this._memberEvent.content?.membership;
     }
 }

From 703c89e27628d29dc75e0919f7088cd92f989c41 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Mon, 31 Aug 2020 16:07:29 +0200
Subject: [PATCH 039/173] make membership available on member

---
 src/matrix/room/members/RoomMember.js | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/matrix/room/members/RoomMember.js b/src/matrix/room/members/RoomMember.js
index 0ce205a5..954de0a4 100644
--- a/src/matrix/room/members/RoomMember.js
+++ b/src/matrix/room/members/RoomMember.js
@@ -67,6 +67,10 @@ export class RoomMember {
         });
     }
 
+    get membership() {
+        return this._data.membership;
+    }
+
     /**
      * @return {String?} the display name, if any
      */

From aeb2f5402a1c55ddae353c0e251e5913dea2aea5 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Mon, 31 Aug 2020 16:08:47 +0200
Subject: [PATCH 040/173] process own membership changes before device lists

---
 src/matrix/Session.js | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index 6cf9225c..90db1b68 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -173,15 +173,15 @@ export class Session {
             changes.syncInfo = syncInfo;
         }
         if (this._deviceTracker) {
-            const deviceLists = syncResponse.device_lists;
-            if (deviceLists) {
-                await this._deviceTracker.writeDeviceChanges(deviceLists, txn);
-            }
             for (const {room, changes} of roomChanges) {
                 if (room.isTrackingMembers && changes.memberChanges?.size) {
                     await this._deviceTracker.writeMemberChanges(room, changes.memberChanges, txn);
                 }
             } 
+            const deviceLists = syncResponse.device_lists;
+            if (deviceLists) {
+                await this._deviceTracker.writeDeviceChanges(deviceLists, txn);
+            }
         }
         return changes;
     }

From 4077f57afb63bcdbd40c7cc68d8d58ea16db0c7c Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Mon, 31 Aug 2020 16:09:13 +0200
Subject: [PATCH 041/173] fix typos in stores

---
 src/matrix/storage/idb/stores/DeviceIdentityStore.js | 2 +-
 src/matrix/storage/idb/stores/RoomMemberStore.js     | 2 +-
 src/matrix/storage/idb/stores/UserIdentityStore.js   | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/matrix/storage/idb/stores/DeviceIdentityStore.js b/src/matrix/storage/idb/stores/DeviceIdentityStore.js
index 8bbab1b7..aec337fc 100644
--- a/src/matrix/storage/idb/stores/DeviceIdentityStore.js
+++ b/src/matrix/storage/idb/stores/DeviceIdentityStore.js
@@ -36,6 +36,6 @@ export class DeviceIdentityStore {
 
     set(deviceIdentity) {
         deviceIdentity.key = encodeKey(deviceIdentity.userId, deviceIdentity.deviceId);
-        return this._store.set(deviceIdentity);
+        return this._store.put(deviceIdentity);
     }
 }
diff --git a/src/matrix/storage/idb/stores/RoomMemberStore.js b/src/matrix/storage/idb/stores/RoomMemberStore.js
index 75649935..be2b16ec 100644
--- a/src/matrix/storage/idb/stores/RoomMemberStore.js
+++ b/src/matrix/storage/idb/stores/RoomMemberStore.js
@@ -50,7 +50,7 @@ export class RoomMemberStore {
         const userIds = [];
         const range = IDBKeyRange.lowerBound(encodeKey(roomId, ""));
         await this._roomMembersStore.iterateKeys(range, key => {
-            const decodedKey = decodedKey(key);
+            const decodedKey = decodeKey(key);
             // prevent running into the next room
             if (decodedKey.roomId === roomId) {
                 userIds.push(decodedKey.userId);
diff --git a/src/matrix/storage/idb/stores/UserIdentityStore.js b/src/matrix/storage/idb/stores/UserIdentityStore.js
index 2eadefec..1cf6d636 100644
--- a/src/matrix/storage/idb/stores/UserIdentityStore.js
+++ b/src/matrix/storage/idb/stores/UserIdentityStore.js
@@ -24,10 +24,10 @@ export class UserIdentityStore {
     }
 
     set(userIdentity) {
-        return this._store.set(userIdentity);
+        this._store.put(userIdentity);
     }
 
     remove(userId) {
-        return this._eventStore.delete(userId);
+        return this._store.delete(userId);
     }
 }

From 6580219b09dadf66ec37ea10adf9d08923d28fa6 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Mon, 31 Aug 2020 16:09:24 +0200
Subject: [PATCH 042/173] add userIdentities to sync txn

---
 src/matrix/Sync.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/matrix/Sync.js b/src/matrix/Sync.js
index 5587d2b0..e3d519bd 100644
--- a/src/matrix/Sync.js
+++ b/src/matrix/Sync.js
@@ -131,6 +131,7 @@ export class Sync {
             storeNames.timelineEvents,
             storeNames.timelineFragments,
             storeNames.pendingEvents,
+            storeNames.userIdentities,
         ]);
         const roomChanges = [];
         let sessionChanges;

From 15ae35bbbc93113b914b53bd912c6f5cbcf88020 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Mon, 31 Aug 2020 16:09:38 +0200
Subject: [PATCH 043/173] add future todo

---
 src/matrix/room/RoomSummary.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/matrix/room/RoomSummary.js b/src/matrix/room/RoomSummary.js
index 51a31d21..3b550527 100644
--- a/src/matrix/room/RoomSummary.js
+++ b/src/matrix/room/RoomSummary.js
@@ -116,7 +116,9 @@ function updateSummary(data, summary) {
     const heroes = summary["m.heroes"];
     const joinCount = summary["m.joined_member_count"];
     const inviteCount = summary["m.invited_member_count"];
-
+    // TODO: we could easily calculate if all members are available here and set hasFetchedMembers?
+    // so we can avoid calling /members...
+    // we'd need to do a count query in the roomMembers store though ...
     if (heroes && Array.isArray(heroes)) {
         data = data.cloneIfNeeded();
         data.heroes = heroes;

From 007333628ae870f835575fbb8a3415f23dabadf8 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Mon, 31 Aug 2020 16:10:18 +0200
Subject: [PATCH 044/173] add todo for /sync <-> /members race

---
 src/matrix/room/Room.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index 6b23ad35..98cde3a5 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -145,6 +145,7 @@ export class Room extends EventEmitter {
     /** @public */
     async loadMemberList() {
         if (this._memberList) {
+            // TODO: also await fetchOrLoadMembers promise here
             this._memberList.retain();
             return this._memberList;
         } else {

From 6c4243eac7d4d1f3d37c98098f593f7377e81977 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Mon, 31 Aug 2020 16:19:15 +0200
Subject: [PATCH 045/173] early start of code style

---
 codestyle.md | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 codestyle.md

diff --git a/codestyle.md b/codestyle.md
new file mode 100644
index 00000000..1db6ad0f
--- /dev/null
+++ b/codestyle.md
@@ -0,0 +1,7 @@
+
+# Code-style
+
+ - methods that return a promise should always use async/await
+    otherwise synchronous errors can get swallowed
+ - only named exports, no default exports
+    otherwise it becomes hard to remember what was a default/named export

From 9870483121c909d3ab68e61ba564c8b310f50cea Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Mon, 31 Aug 2020 16:21:12 +0200
Subject: [PATCH 046/173] remove room list sorting logging, works well now

---
 .../session/roomlist/RoomTileViewModel.js     | 39 +++++--------------
 1 file changed, 9 insertions(+), 30 deletions(-)

diff --git a/src/domain/session/roomlist/RoomTileViewModel.js b/src/domain/session/roomlist/RoomTileViewModel.js
index 1882438f..76cab067 100644
--- a/src/domain/session/roomlist/RoomTileViewModel.js
+++ b/src/domain/session/roomlist/RoomTileViewModel.js
@@ -57,35 +57,17 @@ export class RoomTileViewModel extends ViewModel {
         const myRoom = this._room;
         const theirRoom = other._room;
 
-        let buf = "";
-        function log(...args) {
-            buf = buf + args.map(a => a+"").join(" ") + "\n";
-        }
-        function logResult(result) {
-            if (result === 0) {
-                log("rooms are equal (should not happen)", result);
-            } else if (result > 0) {
-                log(`${theirRoom.name || theirRoom.id} comes first`, result);
-            } else {
-                log(`${myRoom.name || myRoom.id} comes first`, result);
-            }
-            console.info(buf);
-            return result;
-        }
-        log(`comparing ${myRoom.name || theirRoom.id} and ${theirRoom.name || theirRoom.id} ...`);
-        log("comparing isLowPriority...");
         if (myRoom.isLowPriority !== theirRoom.isLowPriority) {
             if (myRoom.isLowPriority) {
-                return logResult(1);
+                return 1;
             }
-            return logResult(-1);
+            return -1;
         }
-        log("comparing isUnread...");
         if (isSortedAsUnread(this) !== isSortedAsUnread(other)) {
             if (isSortedAsUnread(this)) {
-                return logResult(-1);
+                return -1;
             }
-            return logResult(1);
+            return 1;
         }
         const myTimestamp = myRoom.lastMessageTimestamp;
         const theirTimestamp = theirRoom.lastMessageTimestamp;
@@ -93,24 +75,21 @@ export class RoomTileViewModel extends ViewModel {
         const theirTimestampValid = Number.isSafeInteger(theirTimestamp);
         // if either does not have a timestamp, put the one with a timestamp first
         if (myTimestampValid !== theirTimestampValid) {
-            log("checking if either does not have lastMessageTimestamp ...", myTimestamp, theirTimestamp);
             if (!theirTimestampValid) {
-                return logResult(-1);
+                return -1;
             }
-            return logResult(1);
+            return 1;
         }
         const timeDiff = theirTimestamp - myTimestamp;
         if (timeDiff === 0 || !theirTimestampValid || !myTimestampValid) {
-            log("checking name ...", myTimestamp, theirTimestamp);
             // sort alphabetically
             const nameCmp = this.name.localeCompare(other.name);
             if (nameCmp === 0) {
-                return logResult(this._room.id.localeCompare(other._room.id));
+                return this._room.id.localeCompare(other._room.id);
             }
-            return logResult(nameCmp);
+            return nameCmp;
         }
-        log("checking timestamp ...");
-        return logResult(timeDiff);
+        return timeDiff;
     }
 
     get isOpen() {

From 81a1573e3bc457257f9f70c28d02854d78122395 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 1 Sep 2020 17:57:59 +0200
Subject: [PATCH 047/173] make a shared olm util for the whole session

---
 src/matrix/Session.js            | 3 ++-
 src/matrix/e2ee/DeviceTracker.js | 4 ++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index 90db1b68..4d30516a 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -35,10 +35,11 @@ export class Session {
         this._user = new User(sessionInfo.userId);
         this._olm = olm;
         this._e2eeAccount = null;
+        const olmUtil = olm ? new olm.Utility() : null;
         this._deviceTracker = olm ? new DeviceTracker({
             storage,
             getSyncToken: () => this.syncToken,
-            olm,
+            olmUtil,
         }) : null;
     }
 
diff --git a/src/matrix/e2ee/DeviceTracker.js b/src/matrix/e2ee/DeviceTracker.js
index 5eaa23b7..b085be80 100644
--- a/src/matrix/e2ee/DeviceTracker.js
+++ b/src/matrix/e2ee/DeviceTracker.js
@@ -36,11 +36,11 @@ function deviceKeysAsDeviceIdentity(deviceSection) {
 }
 
 export class DeviceTracker {
-    constructor({storage, getSyncToken, olm}) {
+    constructor({storage, getSyncToken, olmUtil}) {
         this._storage = storage;
         this._getSyncToken = getSyncToken;
         this._identityChangedForRoom = null;
-        this._olmUtil = new olm.Utility();
+        this._olmUtil = olmUtil;
     }
 
     async writeDeviceChanges(deviceLists, txn) {

From 5fee7fedc3dab376194cc585b8b912acc39b1f86 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 1 Sep 2020 17:59:39 +0200
Subject: [PATCH 048/173] implement olm decryption algorithm

---
 src/matrix/e2ee/Account.js        |  27 ++++-
 src/matrix/e2ee/common.js         |   8 ++
 src/matrix/e2ee/olm/Decryption.js | 187 ++++++++++++++++++++++++++++++
 3 files changed, 219 insertions(+), 3 deletions(-)
 create mode 100644 src/matrix/e2ee/olm/Decryption.js

diff --git a/src/matrix/e2ee/Account.js b/src/matrix/e2ee/Account.js
index 4905bbb6..0478112b 100644
--- a/src/matrix/e2ee/Account.js
+++ b/src/matrix/e2ee/Account.js
@@ -31,7 +31,7 @@ export class Account {
             account.unpickle(pickleKey, pickledAccount);
             const serverOTKCount = await txn.session.get(SERVER_OTK_COUNT_SESSION_KEY);
             return new Account({pickleKey, hsApi, account, userId,
-                deviceId, areDeviceKeysUploaded, serverOTKCount});
+                deviceId, areDeviceKeysUploaded, serverOTKCount, olm});
         }
     }
 
@@ -47,10 +47,11 @@ export class Account {
         await txn.session.add(DEVICE_KEY_FLAG_SESSION_KEY, areDeviceKeysUploaded);
         await txn.session.add(SERVER_OTK_COUNT_SESSION_KEY, 0);
         return new Account({pickleKey, hsApi, account, userId,
-            deviceId, areDeviceKeysUploaded, serverOTKCount: 0});
+            deviceId, areDeviceKeysUploaded, serverOTKCount: 0, olm});
     }
 
-    constructor({pickleKey, hsApi, account, userId, deviceId, areDeviceKeysUploaded, serverOTKCount}) {
+    constructor({pickleKey, hsApi, account, userId, deviceId, areDeviceKeysUploaded, serverOTKCount, olm}) {
+        this._olm = olm;
         this._pickleKey = pickleKey;
         this._hsApi = hsApi;
         this._account = account;
@@ -58,6 +59,11 @@ export class Account {
         this._deviceId = deviceId;
         this._areDeviceKeysUploaded = areDeviceKeysUploaded;
         this._serverOTKCount = serverOTKCount;
+        this._identityKeys = JSON.parse(this._account.identity_keys());
+    }
+
+    get identityKeys() {
+        return this._identityKeys;
     }
 
     async uploadKeys(storage) {
@@ -118,6 +124,21 @@ export class Account {
         return false;
     }
 
+    createInboundOlmSession(senderKey, body) {
+        const newSession = new this._olm.Session();
+        newSession.create_inbound_from(this._account, senderKey, body);
+        return newSession;
+    }
+
+    writeRemoveOneTimeKey(session, txn) {
+        // this is side-effecty and will have applied the change if the txn fails,
+        // but don't want to clone the account for now
+        // and it is not the worst thing to think we have used a OTK when
+        // decrypting the message that actually used it threw for some reason.
+        this._account.remove_one_time_keys(session);
+        txn.session.set(ACCOUNT_SESSION_KEY, this._account.pickle(this._pickleKey));
+    }
+
     writeSync(deviceOneTimeKeysCount, txn) {
         // we only upload signed_curve25519 otks
         const otkCount = deviceOneTimeKeysCount.signed_curve25519;
diff --git a/src/matrix/e2ee/common.js b/src/matrix/e2ee/common.js
index 82709051..ef758feb 100644
--- a/src/matrix/e2ee/common.js
+++ b/src/matrix/e2ee/common.js
@@ -18,3 +18,11 @@ limitations under the License.
 export const SESSION_KEY_PREFIX = "e2ee:";
 export const OLM_ALGORITHM = "m.olm.v1.curve25519-aes-sha2";
 export const MEGOLM_ALGORITHM = "m.megolm.v1.aes-sha2";
+
+export class DecryptionError extends Error {
+    constructor(code, detailsObj = null) {
+        super(`Decryption error ${code}${detailsObj ? ": "+JSON.stringify(detailsObj) : ""}`);
+        this.code = code;
+        this.details = detailsObj;
+    }
+}
diff --git a/src/matrix/e2ee/olm/Decryption.js b/src/matrix/e2ee/olm/Decryption.js
new file mode 100644
index 00000000..582f96d2
--- /dev/null
+++ b/src/matrix/e2ee/olm/Decryption.js
@@ -0,0 +1,187 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {DecryptionError} from "../common.js";
+
+const SESSION_LIMIT_PER_SENDER_KEY = 4;
+
+function isPreKeyMessage(message) {
+    return message.type === 0;
+}
+
+export class Decryption {
+    constructor({account, pickleKey, now, ownUserId, storage, olm}) {
+        this._account = account;
+        this._pickleKey = pickleKey;
+        this._now = now;
+        this._ownUserId = ownUserId;
+        this._storage = storage;
+        this._olm = olm;
+        this._createOutboundSessionPromise = null;
+    }
+
+    // we can't run this in the sync txn because decryption will be async ...
+    // should we store the encrypted events in the sync loop and then pop them from there?
+    // it would be good in any case to run the (next) sync request in parallel with decryption
+    async decrypt(event) {
+        const senderKey = event.content?.["sender_key"];
+        const ciphertext = event.content?.ciphertext;
+        if (!ciphertext) {
+            throw new DecryptionError("OLM_MISSING_CIPHERTEXT");
+        }
+        const message = ciphertext?.[this._account.identityKeys.curve25519];
+        if (!message) {
+            // TODO: use same error messages as element-web
+            throw new DecryptionError("OLM_NOT_INCLUDED_IN_RECIPIENTS");
+        }
+        const sortedSessionIds = await this._getSortedSessionIds(senderKey);
+        let plaintext;
+        for (const sessionId of sortedSessionIds) {
+            try {
+                plaintext = await this._attemptDecryption(senderKey, sessionId, message);
+            } catch (err) {
+                throw new DecryptionError("OLM_BAD_ENCRYPTED_MESSAGE", {senderKey, error: err.message});
+            }
+            if (typeof plaintext === "string") {
+                break;
+            }
+        }
+        if (typeof plaintext !== "string" && isPreKeyMessage(message)) {
+            plaintext = await this._createOutboundSessionAndDecrypt(senderKey, message, sortedSessionIds);
+        }
+        if (typeof plaintext === "string") {
+            return this._parseAndValidatePayload(plaintext, event);
+        }
+    }
+
+    async _getSortedSessionIds(senderKey) {
+        const readTxn = await this._storage.readTxn([this._storage.storeNames.olmSessions]);
+        const sortedSessions = await readTxn.olmSessions.getAll(senderKey);
+        // sort most recent used sessions first
+        sortedSessions.sort((a, b) => {
+            return b.lastUsed - a.lastUsed;
+        });
+        return sortedSessions.map(s => s.sessionId);
+    }
+
+    async _createOutboundSessionAndDecrypt(senderKey, message, sortedSessionIds) {
+        // serialize calls so the account isn't written from multiple
+        // sessions at once
+        while (this._createOutboundSessionPromise) {
+            await this._createOutboundSessionPromise;
+        }
+        this._createOutboundSessionPromise = (async () => {
+            try {
+                return await this._createOutboundSessionAndDecryptImpl(senderKey, message, sortedSessionIds);
+            } finally {
+                this._createOutboundSessionPromise = null;
+            }
+        })();
+        return await this._createOutboundSessionPromise;
+    }
+
+    // this could internally dispatch to a web-worker
+    async _createOutboundSessionAndDecryptImpl(senderKey, message, sortedSessionIds) {
+        let plaintext;
+        const session = this._account.createInboundOlmSession(senderKey, message.body);
+        try {
+            const txn = await this._storage.readWriteTxn([
+                this._storage.storeNames.session,
+                this._storage.storeNames.olmSessions,
+            ]);
+            try {
+                // do this before removing the OTK removal, so we know decryption succeeded beforehand,
+                // as we don't have a way of undoing the OTK removal atm.
+                plaintext = session.decrypt(message.type, message.body);
+                this._account.writeRemoveOneTimeKey(session, txn);
+                // remove oldest session if we reach the limit including the new session
+                if (sortedSessionIds.length >= SESSION_LIMIT_PER_SENDER_KEY) {
+                    // given they are sorted, the oldest one is the last one
+                    const oldestSessionId = sortedSessionIds[sortedSessionIds.length - 1];
+                    txn.olmSessions.remove(senderKey, oldestSessionId);
+                }
+                txn.olmSessions.set({
+                    session: session.pickle(this._pickleKey),
+                    sessionId: session.session_id(),
+                    senderKey,
+                    lastUsed: this._now(),
+                });
+            } catch (err) {
+                txn.abort();
+                throw err;
+            }
+            await txn.complete();
+        } finally {
+            session.free();
+        }
+        return plaintext;
+    }
+
+    // this could internally dispatch to a web-worker
+    async _attemptDecryption(senderKey, sessionId, message) {
+        const txn = await this._storage.readWriteTxn([this._storage.storeNames.olmSessions]);
+        const session = new this._olm.Session();
+        let plaintext;
+        try {
+            const sessionEntry = await txn.olmSessions.get(senderKey, sessionId);
+            session.unpickle(this._pickleKey, sessionEntry.session);
+            if (isPreKeyMessage(message) && !session.matches_inbound(message.body)) {
+                return;
+            }
+            try {
+                plaintext = session.decrypt(message.type, message.body);
+            } catch (err) {
+                if (isPreKeyMessage(message)) {
+                    throw new Error(`Error decrypting prekey message with existing session id ${sessionId}: ${err.message}`);
+                }
+                // decryption failed, bail out
+                return;
+            }
+            sessionEntry.session = session.pickle(this._pickleKey);
+            sessionEntry.lastUsed = this._now();
+            txn.olmSessions.set(sessionEntry);
+        } catch(err) {
+            txn.abort();
+            throw err;
+        } finally {
+            session.free();
+        }
+        await txn.complete();
+        return plaintext;
+    }
+
+    _parseAndValidatePayload(plaintext, event) {
+        const payload = JSON.parse(plaintext);
+
+        if (payload.sender !== event.sender) {
+            throw new DecryptionError("OLM_FORWARDED_MESSAGE", {sentBy: event.sender, encryptedBy: payload.sender});
+        }
+        if (payload.recipient !== this._ownUserId) {
+            throw new DecryptionError("OLM_BAD_RECIPIENT", {recipient: payload.recipient});
+        }
+        if (payload.recipient_keys?.ed25519 !== this._account.identityKeys.ed25519) {
+            throw new DecryptionError("OLM_BAD_RECIPIENT_KEY", {key: payload.recipient_keys?.ed25519});
+        }
+        // TODO: check room_id
+        if (!payload.type) {
+            throw new Error("missing type on payload");
+        }
+        if (!payload.content) {
+            throw new Error("missing content on payload");
+        }
+        return payload;
+    }
+}

From 6788a612fc2b0ee5c157bfbbbb0afd1f8e563802 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 1 Sep 2020 17:59:59 +0200
Subject: [PATCH 049/173] implement olm session storage

---
 src/matrix/storage/common.js                  |  1 +
 src/matrix/storage/idb/Transaction.js         |  5 +++
 src/matrix/storage/idb/schema.js              |  6 +++
 .../storage/idb/stores/OlmSessionStore.js     | 45 +++++++++++++++++++
 4 files changed, 57 insertions(+)
 create mode 100644 src/matrix/storage/idb/stores/OlmSessionStore.js

diff --git a/src/matrix/storage/common.js b/src/matrix/storage/common.js
index 7d6fae09..73900af3 100644
--- a/src/matrix/storage/common.js
+++ b/src/matrix/storage/common.js
@@ -24,6 +24,7 @@ export const STORE_NAMES = Object.freeze([
     "pendingEvents",
     "userIdentities",
     "deviceIdentities",
+    "olmSessions",
 ]);
 
 export const STORE_MAP = Object.freeze(STORE_NAMES.reduce((nameMap, name) => {
diff --git a/src/matrix/storage/idb/Transaction.js b/src/matrix/storage/idb/Transaction.js
index 921c23e2..370a5563 100644
--- a/src/matrix/storage/idb/Transaction.js
+++ b/src/matrix/storage/idb/Transaction.js
@@ -26,6 +26,7 @@ import {TimelineFragmentStore} from "./stores/TimelineFragmentStore.js";
 import {PendingEventStore} from "./stores/PendingEventStore.js";
 import {UserIdentityStore} from "./stores/UserIdentityStore.js";
 import {DeviceIdentityStore} from "./stores/DeviceIdentityStore.js";
+import {OlmSessionStore} from "./stores/OlmSessionStore.js";
 
 export class Transaction {
     constructor(txn, allowedStoreNames) {
@@ -91,6 +92,10 @@ export class Transaction {
         return this._store("deviceIdentities", idbStore => new DeviceIdentityStore(idbStore));
     }
     
+    get olmSessions() {
+        return this._store("olmSessions", idbStore => new OlmSessionStore(idbStore));
+    }
+
     complete() {
         return txnAsPromise(this._txn);
     }
diff --git a/src/matrix/storage/idb/schema.js b/src/matrix/storage/idb/schema.js
index d8aa81cc..8e34ac27 100644
--- a/src/matrix/storage/idb/schema.js
+++ b/src/matrix/storage/idb/schema.js
@@ -10,6 +10,7 @@ export const schema = [
     createMemberStore,
     migrateSession,
     createIdentityStores,
+    createOlmSessionStore,
 ];
 // TODO: how to deal with git merge conflicts of this array?
 
@@ -70,3 +71,8 @@ function createIdentityStores(db) {
     db.createObjectStore("userIdentities", {keyPath: "userId"});
     db.createObjectStore("deviceIdentities", {keyPath: "key"});
 }
+
+//v5
+function createOlmSessionStore(db) {
+    db.createObjectStore("olmSessions", {keyPath: "key"});
+}
diff --git a/src/matrix/storage/idb/stores/OlmSessionStore.js b/src/matrix/storage/idb/stores/OlmSessionStore.js
new file mode 100644
index 00000000..c94b3bfd
--- /dev/null
+++ b/src/matrix/storage/idb/stores/OlmSessionStore.js
@@ -0,0 +1,45 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+function encodeKey(senderKey, sessionId) {
+    return `${senderKey}|${sessionId}`;
+}
+
+export class OlmSessionStore {
+    constructor(store) {
+        this._store = store;
+    }
+
+    getAll(senderKey) {
+        const range = IDBKeyRange.lowerBound(encodeKey(senderKey, ""));
+        return this._store.selectWhile(range, session => {
+            return session.senderKey === senderKey;
+        });
+    }
+
+    get(senderKey, sessionId) {
+        return this._store.get(encodeKey(senderKey, sessionId));
+    }
+
+    set(session) {
+        session.key = encodeKey(session.senderKey, session.sessionId);
+        return this._store.put(session);
+    }
+
+    remove(senderKey, sessionId) {
+        return this._store.delete(encodeKey(senderKey, sessionId));
+    }
+}

From dc29956e027bf7e88e8f36dea3722a7e0d713b4d Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 1 Sep 2020 18:00:15 +0200
Subject: [PATCH 050/173] extend ie11 benchmark with pickle/unpickle roundtrip

---
 prototypes/olmtest-ie11.html | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/prototypes/olmtest-ie11.html b/prototypes/olmtest-ie11.html
index 13d906b9..2ea3eeb8 100644
--- a/prototypes/olmtest-ie11.html
+++ b/prototypes/olmtest-ie11.html
@@ -61,6 +61,16 @@
                     JSON.parse(bob.identity_keys()).curve25519,
                     bobOneTimeKey
                 );
+                log("alice outbound session created");
+                var aliceSessionPickled = aliceSession.pickle("secret");
+                log("aliceSession pickled", aliceSessionPickled);
+                try {
+                    var tmp = new Olm.Session();
+                    tmp.unpickle("secret", aliceSessionPickled);
+                    log("aliceSession unpickled");
+                } finally {
+                    tmp.free();
+                }
                 var message = aliceSession.encrypt("hello secret world");
                 log("message", message);
                 // decrypt

From 44e9f91d4c3ae6ca591b9f688de9bef23c5a593c Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Wed, 2 Sep 2020 13:33:27 +0200
Subject: [PATCH 051/173] to_device handler for encrypted messages

changes the api of the olm decryption to decrypt in batch
so we can isolate side-effects until we have a write-txn open
and we can parallelize the decryption of different sender keys.
---
 src/matrix/DeviceMessageHandler.js            |  87 ++++
 src/matrix/e2ee/common.js                     |   3 +-
 src/matrix/e2ee/olm/Decryption.js             | 374 ++++++++++++------
 src/matrix/storage/idb/stores/SessionStore.js |   4 +
 4 files changed, 345 insertions(+), 123 deletions(-)
 create mode 100644 src/matrix/DeviceMessageHandler.js

diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js
new file mode 100644
index 00000000..a26bfe33
--- /dev/null
+++ b/src/matrix/DeviceMessageHandler.js
@@ -0,0 +1,87 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {OLM_ALGORITHM, MEGOLM_ALGORITHM} from "./e2ee/common.js";
+
+// key to store in session store
+const PENDING_ENCRYPTED_EVENTS = "pendingEncryptedDeviceEvents";
+
+export class DeviceMessageHandler {
+    constructor({storage, olmDecryption, megolmEncryption}) {
+        this._storage = storage;
+        this._olmDecryption = olmDecryption;
+        this._megolmEncryption = megolmEncryption;
+    }
+
+    async writeSync(toDeviceEvents, txn) {
+        const encryptedEvents = toDeviceEvents.filter(e => e.type === "m.room.encrypted");
+        // store encryptedEvents
+        let pendingEvents = this._getPendingEvents(txn);
+        pendingEvents = pendingEvents.concat(encryptedEvents);
+        txn.session.set(PENDING_ENCRYPTED_EVENTS, pendingEvents);
+        // we don't handle anything other for now
+    }
+
+    async _handleDecryptedEvents(payloads, txn) {
+        const megOlmRoomKeysPayloads = payloads.filter(p => {
+            return p.event.type === "m.room_key" && p.event.content?.algorithm === MEGOLM_ALGORITHM;
+        });
+        let megolmChanges;
+        if (megOlmRoomKeysPayloads.length) {
+            megolmChanges = await this._megolmEncryption.addRoomKeys(megOlmRoomKeysPayloads, txn);
+        }
+        return {megolmChanges};
+    }
+
+    applyChanges({megolmChanges}) {
+        if (megolmChanges) {
+            this._megolmEncryption.applyRoomKeyChanges(megolmChanges);
+        }
+    }
+
+    // not safe to call multiple times without awaiting first call
+    async decryptPending() {
+        const readTxn = await this._storage.readTxn([this._storage.storeNames.session]);
+        const pendingEvents = this._getPendingEvents(readTxn);
+        // only know olm for now
+        const olmEvents = pendingEvents.filter(e => e.content?.algorithm === OLM_ALGORITHM);
+        const decryptChanges = await this._olmDecryption.decryptAll(olmEvents);
+        for (const err of decryptChanges.errors) {
+            console.warn("decryption failed for event", err, err.event);
+        }
+        const txn = await this._storage.readWriteTxn([
+            // both to remove the pending events and to modify the olm account
+            this._storage.storeNames.session,
+            this._storage.storeNames.olmSessions,
+            // this._storage.storeNames.megolmInboundSessions,
+        ]);
+        let changes;
+        try {
+            changes = await this._handleDecryptedEvent(decryptChanges.payloads, txn);
+            decryptChanges.write(txn);
+            txn.session.remove(PENDING_ENCRYPTED_EVENTS);
+        } catch (err) {
+            txn.abort();
+            throw err;
+        }
+        await txn.complete();
+        this._applyChanges(changes);
+    }
+
+    async _getPendingEvents(txn) {
+        return (await txn.session.get(PENDING_ENCRYPTED_EVENTS)) || [];
+    }
+}
diff --git a/src/matrix/e2ee/common.js b/src/matrix/e2ee/common.js
index ef758feb..c5e7399f 100644
--- a/src/matrix/e2ee/common.js
+++ b/src/matrix/e2ee/common.js
@@ -20,9 +20,10 @@ export const OLM_ALGORITHM = "m.olm.v1.curve25519-aes-sha2";
 export const MEGOLM_ALGORITHM = "m.megolm.v1.aes-sha2";
 
 export class DecryptionError extends Error {
-    constructor(code, detailsObj = null) {
+    constructor(code, event, detailsObj = null) {
         super(`Decryption error ${code}${detailsObj ? ": "+JSON.stringify(detailsObj) : ""}`);
         this.code = code;
+        this.event = event;
         this.details = detailsObj;
     }
 }
diff --git a/src/matrix/e2ee/olm/Decryption.js b/src/matrix/e2ee/olm/Decryption.js
index 582f96d2..f701f4df 100644
--- a/src/matrix/e2ee/olm/Decryption.js
+++ b/src/matrix/e2ee/olm/Decryption.js
@@ -22,6 +22,12 @@ function isPreKeyMessage(message) {
     return message.type === 0;
 }
 
+function sortSessions(sessions) {
+    sessions.sort((a, b) => {
+        return b.data.lastUsed - a.data.lastUsed;
+    });
+}
+
 export class Decryption {
     constructor({account, pickleKey, now, ownUserId, storage, olm}) {
         this._account = account;
@@ -33,155 +39,279 @@ export class Decryption {
         this._createOutboundSessionPromise = null;
     }
 
-    // we can't run this in the sync txn because decryption will be async ...
-    // should we store the encrypted events in the sync loop and then pop them from there?
-    // it would be good in any case to run the (next) sync request in parallel with decryption
-    async decrypt(event) {
-        const senderKey = event.content?.["sender_key"];
+    // we need decryptAll because there is some parallelization we can do for decrypting different sender keys at once
+    // but for the same sender key we need to do one by one
+    // 
+    // also we want to store the room key, etc ... in the same txn as we remove the pending encrypted event
+    // 
+    // so we need to decrypt events in a batch (so we can decide which ones can run in parallel and which one one by one)
+    // and also can avoid side-effects before all can be stored this way
+    // 
+    // doing it one by one would be possible, but we would lose the opportunity for parallelization
+    async decryptAll(events) {
+        const eventsPerSenderKey = events.reduce((map, event) => {
+            const senderKey = event.content?.["sender_key"];
+            let list = map.get(senderKey);
+            if (!list) {
+                list = [];
+                map.set(senderKey, list);
+            }
+            list.push(event);
+            return map;
+        }, new Map());
+        const timestamp = this._now();
+        const readSessionsTxn = await this._storage.readTxn([this._storage.storeNames.olmSessions]);
+        // decrypt events for different sender keys in parallel
+        const results = await Promise.all(Array.from(eventsPerSenderKey.entries()).map(([senderKey, events]) => {
+            return this._decryptAllForSenderKey(senderKey, events, timestamp, readSessionsTxn);
+        }));
+        const payloads = results.reduce((all, r) => all.concat(r.payloads), []);
+        const errors = results.reduce((all, r) => all.concat(r.errors), []);
+        const senderKeyDecryptions = results.map(r => r.senderKeyDecryption);
+        return new DecryptionChanges(senderKeyDecryptions, payloads, errors);
+    }
+
+    async _decryptAllForSenderKey(senderKey, events, timestamp, readSessionsTxn) {
+        const sessions = await this._getSessions(senderKey, readSessionsTxn);
+        const senderKeyDecryption = new SenderKeyDecryption(senderKey, sessions, this._olm, timestamp);
+        const payloads = [];
+        const errors = [];
+        // events for a single senderKey need to be decrypted one by one
+        for (const event of events) {
+            try {
+                const payload = this._decryptForSenderKey(senderKeyDecryption, event, timestamp);
+                payloads.push(payload);
+            } catch (err) {
+                errors.push(err);
+            }
+        }
+        return {payloads, errors, senderKeyDecryption};
+    }
+
+    _decryptForSenderKey(senderKeyDecryption, event, timestamp) {
+        const senderKey = senderKeyDecryption.senderKey;
+        const message = this._getMessageAndValidateEvent(event);
+        let plaintext;
+        try {
+            plaintext = senderKeyDecryption.decrypt(message);
+        } catch (err) {
+            // TODO: is it ok that an error on one session prevents other sessions from being attempted?
+            throw new DecryptionError("OLM_BAD_ENCRYPTED_MESSAGE", event, {senderKey, error: err.message});
+        }
+        // could not decrypt with any existing session
+        if (typeof plaintext !== "string" && isPreKeyMessage(message)) {
+            const createResult = this._createSessionAndDecrypt(senderKey, message, timestamp);
+            senderKeyDecryption.addNewSession(createResult.session);
+            plaintext = createResult.plaintext;
+        }
+        if (typeof plaintext === "string") {
+            const payload = JSON.parse(plaintext);
+            this._validatePayload(payload, event);
+            return {event: payload, senderKey};
+        } else {
+            throw new DecryptionError("Didn't find any session to decrypt with", event,
+                {sessionIds: senderKeyDecryption.sessions.map(s => s.id)});
+        }
+    }
+
+    // only for pre-key messages after having attempted decryption with existing sessions
+    _createSessionAndDecrypt(senderKey, message, timestamp) {
+        let plaintext;
+        // if we have multiple messages encrypted with the same new session,
+        // this could create multiple sessions as the OTK isn't removed yet
+        // (this only happens in DecryptionChanges.write)
+        // This should be ok though as we'll first try to decrypt with the new session
+        const olmSession = this._account.createInboundOlmSession(senderKey, message.body);
+        try {
+            plaintext = olmSession.decrypt(message.type, message.body);
+            const session = Session.create(senderKey, olmSession, this._olm, this._pickleKey, timestamp);
+            session.unload(olmSession);
+            return {session, plaintext};
+        } catch (err) {
+            olmSession.free();
+            throw err;
+        }
+    }
+
+    _getMessageAndValidateEvent(event) {
         const ciphertext = event.content?.ciphertext;
         if (!ciphertext) {
-            throw new DecryptionError("OLM_MISSING_CIPHERTEXT");
+            throw new DecryptionError("OLM_MISSING_CIPHERTEXT", event);
         }
         const message = ciphertext?.[this._account.identityKeys.curve25519];
         if (!message) {
-            // TODO: use same error messages as element-web
-            throw new DecryptionError("OLM_NOT_INCLUDED_IN_RECIPIENTS");
-        }
-        const sortedSessionIds = await this._getSortedSessionIds(senderKey);
-        let plaintext;
-        for (const sessionId of sortedSessionIds) {
-            try {
-                plaintext = await this._attemptDecryption(senderKey, sessionId, message);
-            } catch (err) {
-                throw new DecryptionError("OLM_BAD_ENCRYPTED_MESSAGE", {senderKey, error: err.message});
-            }
-            if (typeof plaintext === "string") {
-                break;
-            }
-        }
-        if (typeof plaintext !== "string" && isPreKeyMessage(message)) {
-            plaintext = await this._createOutboundSessionAndDecrypt(senderKey, message, sortedSessionIds);
-        }
-        if (typeof plaintext === "string") {
-            return this._parseAndValidatePayload(plaintext, event);
+            throw new DecryptionError("OLM_NOT_INCLUDED_IN_RECIPIENTS", event);
         }
+
+        return message;
     }
 
-    async _getSortedSessionIds(senderKey) {
-        const readTxn = await this._storage.readTxn([this._storage.storeNames.olmSessions]);
-        const sortedSessions = await readTxn.olmSessions.getAll(senderKey);
+    async _getSessions(senderKey, txn) {
+        const sessionEntries = await txn.olmSessions.getAll(senderKey);
         // sort most recent used sessions first
-        sortedSessions.sort((a, b) => {
-            return b.lastUsed - a.lastUsed;
-        });
-        return sortedSessions.map(s => s.sessionId);
+        const sessions = sessionEntries.map(s => new Session(s, this._pickleKey, this._olm));
+        sortSessions(sessions);
+        return sessions;
     }
 
-    async _createOutboundSessionAndDecrypt(senderKey, message, sortedSessionIds) {
-        // serialize calls so the account isn't written from multiple
-        // sessions at once
-        while (this._createOutboundSessionPromise) {
-            await this._createOutboundSessionPromise;
+    _validatePayload(payload, event) {
+        if (payload.sender !== event.sender) {
+            throw new DecryptionError("OLM_FORWARDED_MESSAGE", event, {sentBy: event.sender, encryptedBy: payload.sender});
         }
-        this._createOutboundSessionPromise = (async () => {
-            try {
-                return await this._createOutboundSessionAndDecryptImpl(senderKey, message, sortedSessionIds);
-            } finally {
-                this._createOutboundSessionPromise = null;
-            }
-        })();
-        return await this._createOutboundSessionPromise;
-    }
-
-    // this could internally dispatch to a web-worker
-    async _createOutboundSessionAndDecryptImpl(senderKey, message, sortedSessionIds) {
-        let plaintext;
-        const session = this._account.createInboundOlmSession(senderKey, message.body);
-        try {
-            const txn = await this._storage.readWriteTxn([
-                this._storage.storeNames.session,
-                this._storage.storeNames.olmSessions,
-            ]);
-            try {
-                // do this before removing the OTK removal, so we know decryption succeeded beforehand,
-                // as we don't have a way of undoing the OTK removal atm.
-                plaintext = session.decrypt(message.type, message.body);
-                this._account.writeRemoveOneTimeKey(session, txn);
-                // remove oldest session if we reach the limit including the new session
-                if (sortedSessionIds.length >= SESSION_LIMIT_PER_SENDER_KEY) {
-                    // given they are sorted, the oldest one is the last one
-                    const oldestSessionId = sortedSessionIds[sortedSessionIds.length - 1];
-                    txn.olmSessions.remove(senderKey, oldestSessionId);
-                }
-                txn.olmSessions.set({
-                    session: session.pickle(this._pickleKey),
-                    sessionId: session.session_id(),
-                    senderKey,
-                    lastUsed: this._now(),
-                });
-            } catch (err) {
-                txn.abort();
-                throw err;
-            }
-            await txn.complete();
-        } finally {
-            session.free();
+        if (payload.recipient !== this._ownUserId) {
+            throw new DecryptionError("OLM_BAD_RECIPIENT", event, {recipient: payload.recipient});
         }
-        return plaintext;
+        if (payload.recipient_keys?.ed25519 !== this._account.identityKeys.ed25519) {
+            throw new DecryptionError("OLM_BAD_RECIPIENT_KEY", event, {key: payload.recipient_keys?.ed25519});
+        }
+        // TODO: check room_id
+        if (!payload.type) {
+            throw new DecryptionError("missing type on payload", event, {payload});
+        }
+        if (!payload.content) {
+            throw new DecryptionError("missing content on payload", event, {payload});
+        }
+        // TODO: how important is it to verify the message?
+        // we should look at payload.keys.ed25519 for that... and compare it to the key we have fetched
+        // from /keys/query, which we might not have done yet at this point.
+    }
+}
+
+class Session {
+    constructor(data, pickleKey, olm, isNew = false) {
+        this.data = data;
+        this._olm = olm;
+        this._pickleKey = pickleKey;
+        this.isNew = isNew;
+        this.isModified = isNew;
     }
 
-    // this could internally dispatch to a web-worker
-    async _attemptDecryption(senderKey, sessionId, message) {
-        const txn = await this._storage.readWriteTxn([this._storage.storeNames.olmSessions]);
+    static create(senderKey, olmSession, olm, pickleKey, timestamp) {
+        return new Session({
+            session: olmSession.pickle(pickleKey),
+            sessionId: olmSession.session_id(),
+            senderKey,
+            lastUsed: timestamp,
+        }, pickleKey, olm, true);
+    }
+
+    get id() {
+        return this.data.sessionId;
+    }
+
+    load() {
         const session = new this._olm.Session();
-        let plaintext;
+        session.unpickle(this._pickleKey, this.data.session);
+        return session;
+    }
+
+    unload(olmSession) {
+        olmSession.free();
+    }
+
+    save(olmSession) {
+        this.data.session = olmSession.pickle(this._pickleKey);
+        this.isModified = true;
+    }
+}
+
+// decryption helper for a single senderKey
+class SenderKeyDecryption {
+    constructor(senderKey, sessions, olm, timestamp) {
+        this.senderKey = senderKey;
+        this.sessions = sessions;
+        this._olm = olm;
+        this._timestamp = timestamp;
+    }
+
+    addNewSession(session) {
+        // add at top as it is most recent
+        this.sessions.unshift(session);
+    }
+
+    decrypt(message) {
+        for (const session of this.sessions) {
+            const plaintext = this._decryptWithSession(session, message);
+            if (typeof plaintext === "string") {
+                // keep them sorted so will try the same session first for other messages
+                // and so we can assume the excess ones are at the end
+                // if they grow too large
+                sortSessions(this.sessions);
+                return plaintext;
+            }
+        }
+    }
+
+    getModifiedSessions() {
+        return this.sessions.filter(session => session.isModified);
+    }
+
+    get hasNewSessions() {
+        return this.sessions.some(session => session.isNew);
+    }
+
+    // this could internally dispatch to a web-worker
+    // and is why we unpickle/pickle on each iteration
+    // if this turns out to be a real cost for IE11,
+    // we could look into adding a less expensive serialization mechanism
+    // for olm sessions to libolm
+    _decryptWithSession(session, message) {
+        const olmSession = session.load();
         try {
-            const sessionEntry = await txn.olmSessions.get(senderKey, sessionId);
-            session.unpickle(this._pickleKey, sessionEntry.session);
-            if (isPreKeyMessage(message) && !session.matches_inbound(message.body)) {
+            if (isPreKeyMessage(message) && !olmSession.matches_inbound(message.body)) {
                 return;
             }
             try {
-                plaintext = session.decrypt(message.type, message.body);
+                const plaintext = olmSession.decrypt(message.type, message.body);
+                session.save(olmSession);
+                session.lastUsed = this._timestamp;
+                return plaintext;
             } catch (err) {
                 if (isPreKeyMessage(message)) {
-                    throw new Error(`Error decrypting prekey message with existing session id ${sessionId}: ${err.message}`);
+                    throw new Error(`Error decrypting prekey message with existing session id ${session.id}: ${err.message}`);
                 }
                 // decryption failed, bail out
                 return;
             }
-            sessionEntry.session = session.pickle(this._pickleKey);
-            sessionEntry.lastUsed = this._now();
-            txn.olmSessions.set(sessionEntry);
-        } catch(err) {
-            txn.abort();
-            throw err;
         } finally {
-            session.free();
+            session.unload(olmSession);
+        }
+    }
+}
+
+class DecryptionChanges {
+    constructor(senderKeyDecryptions, payloads, errors, account) {
+        this._senderKeyDecryptions = senderKeyDecryptions;
+        this._account = account;    
+        this.payloads = payloads;
+        this.errors = errors;
+    }
+
+    get hasNewSessions() {
+        return this._senderKeyDecryptions.some(skd => skd.hasNewSessions);
+    }
+
+    write(txn) {
+        for (const senderKeyDecryption of this._senderKeyDecryptions) {
+            for (const session of senderKeyDecryption.getModifiedSessions()) {
+                txn.olmSessions.set(session.data);
+                if (session.isNew) {
+                    const olmSession = session.load();
+                    try {
+                        this._account.writeRemoveOneTimeKey(olmSession, txn);
+                    } finally {
+                        session.unload(olmSession);
+                    }
+                }
+            }
+            if (senderKeyDecryption.sessions.length > SESSION_LIMIT_PER_SENDER_KEY) {
+                const {senderKey, sessions} = senderKeyDecryption;
+                // >= because index is zero-based
+                for (let i = sessions.length - 1; i >= SESSION_LIMIT_PER_SENDER_KEY ; i -= 1) {
+                    const session = sessions[i];
+                    txn.olmSessions.remove(senderKey, session.id);
+                }
+            }
         }
-        await txn.complete();
-        return plaintext;
-    }
-
-    _parseAndValidatePayload(plaintext, event) {
-        const payload = JSON.parse(plaintext);
-
-        if (payload.sender !== event.sender) {
-            throw new DecryptionError("OLM_FORWARDED_MESSAGE", {sentBy: event.sender, encryptedBy: payload.sender});
-        }
-        if (payload.recipient !== this._ownUserId) {
-            throw new DecryptionError("OLM_BAD_RECIPIENT", {recipient: payload.recipient});
-        }
-        if (payload.recipient_keys?.ed25519 !== this._account.identityKeys.ed25519) {
-            throw new DecryptionError("OLM_BAD_RECIPIENT_KEY", {key: payload.recipient_keys?.ed25519});
-        }
-        // TODO: check room_id
-        if (!payload.type) {
-            throw new Error("missing type on payload");
-        }
-        if (!payload.content) {
-            throw new Error("missing content on payload");
-        }
-        return payload;
     }
 }
diff --git a/src/matrix/storage/idb/stores/SessionStore.js b/src/matrix/storage/idb/stores/SessionStore.js
index f64a8299..25ea2351 100644
--- a/src/matrix/storage/idb/stores/SessionStore.js
+++ b/src/matrix/storage/idb/stores/SessionStore.js
@@ -49,4 +49,8 @@ export class SessionStore {
     add(key, value) {
         return this._sessionStore.put({key, value});
     }
+
+    remove(key) {
+        this._sessionStore.delete(key);
+    }
 }

From 6aad75161173ae7b1ac033d3cef9a44642f7747f Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Wed, 2 Sep 2020 13:35:08 +0200
Subject: [PATCH 052/173] fix wrong idb method used in session store

---
 src/matrix/storage/idb/stores/SessionStore.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/matrix/storage/idb/stores/SessionStore.js b/src/matrix/storage/idb/stores/SessionStore.js
index 25ea2351..6246169b 100644
--- a/src/matrix/storage/idb/stores/SessionStore.js
+++ b/src/matrix/storage/idb/stores/SessionStore.js
@@ -47,7 +47,7 @@ export class SessionStore {
 	}
 
     add(key, value) {
-        return this._sessionStore.put({key, value});
+        return this._sessionStore.add({key, value});
     }
 
     remove(key) {

From f5c7b1b3ecb3d9f6609dbb1fe48ec5cf93885ff6 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Wed, 2 Sep 2020 13:35:25 +0200
Subject: [PATCH 053/173] remove obsolete comment

---
 src/matrix/storage/idb/stores/SessionStore.js | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/src/matrix/storage/idb/stores/SessionStore.js b/src/matrix/storage/idb/stores/SessionStore.js
index 6246169b..c6486651 100644
--- a/src/matrix/storage/idb/stores/SessionStore.js
+++ b/src/matrix/storage/idb/stores/SessionStore.js
@@ -14,22 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-/**
-store contains:
-	loginData {
-		device_id
-		home_server
-		access_token
-		user_id
-	}
-	// flags {
-	// 	lazyLoading?
-	// }
-	syncToken
-	displayName
-	avatarUrl
-	lastSynced
-*/
 export class SessionStore {
 	constructor(sessionStore) {
 		this._sessionStore = sessionStore;

From 6d3aa219fa6bbf7deb44af3c6ab97a5d6b117c1b Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Wed, 2 Sep 2020 14:24:38 +0200
Subject: [PATCH 054/173] implement storing room keys

---
 src/matrix/DeviceMessageHandler.js            | 24 ++++---
 src/matrix/e2ee/megolm/Decryption.js          | 68 +++++++++++++++++++
 src/matrix/storage/common.js                  |  1 +
 src/matrix/storage/idb/Transaction.js         |  5 ++
 src/matrix/storage/idb/schema.js              |  6 ++
 .../idb/stores/InboundGroupSessionStore.js    | 36 ++++++++++
 6 files changed, 132 insertions(+), 8 deletions(-)
 create mode 100644 src/matrix/e2ee/megolm/Decryption.js
 create mode 100644 src/matrix/storage/idb/stores/InboundGroupSessionStore.js

diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js
index a26bfe33..7154d192 100644
--- a/src/matrix/DeviceMessageHandler.js
+++ b/src/matrix/DeviceMessageHandler.js
@@ -20,10 +20,15 @@ import {OLM_ALGORITHM, MEGOLM_ALGORITHM} from "./e2ee/common.js";
 const PENDING_ENCRYPTED_EVENTS = "pendingEncryptedDeviceEvents";
 
 export class DeviceMessageHandler {
-    constructor({storage, olmDecryption, megolmEncryption}) {
+    constructor({storage}) {
         this._storage = storage;
+        this._olmDecryption = null;
+        this._megolmDecryption = null;
+    }
+
+    enableEncryption({olmDecryption, megolmDecryption}) {
         this._olmDecryption = olmDecryption;
-        this._megolmEncryption = megolmEncryption;
+        this._megolmDecryption = megolmDecryption;
     }
 
     async writeSync(toDeviceEvents, txn) {
@@ -35,25 +40,28 @@ export class DeviceMessageHandler {
         // we don't handle anything other for now
     }
 
-    async _handleDecryptedEvents(payloads, txn) {
+    async _writeDecryptedEvents(payloads, txn) {
         const megOlmRoomKeysPayloads = payloads.filter(p => {
-            return p.event.type === "m.room_key" && p.event.content?.algorithm === MEGOLM_ALGORITHM;
+            return p.event?.type === "m.room_key" && p.event.content?.algorithm === MEGOLM_ALGORITHM;
         });
         let megolmChanges;
         if (megOlmRoomKeysPayloads.length) {
-            megolmChanges = await this._megolmEncryption.addRoomKeys(megOlmRoomKeysPayloads, txn);
+            megolmChanges = await this._megolmDecryption.addRoomKeys(megOlmRoomKeysPayloads, txn);
         }
         return {megolmChanges};
     }
 
     applyChanges({megolmChanges}) {
         if (megolmChanges) {
-            this._megolmEncryption.applyRoomKeyChanges(megolmChanges);
+            this._megolmDecryption.applyRoomKeyChanges(megolmChanges);
         }
     }
 
     // not safe to call multiple times without awaiting first call
     async decryptPending() {
+        if (!this._olmDecryption) {
+            return;
+        }
         const readTxn = await this._storage.readTxn([this._storage.storeNames.session]);
         const pendingEvents = this._getPendingEvents(readTxn);
         // only know olm for now
@@ -66,11 +74,11 @@ export class DeviceMessageHandler {
             // both to remove the pending events and to modify the olm account
             this._storage.storeNames.session,
             this._storage.storeNames.olmSessions,
-            // this._storage.storeNames.megolmInboundSessions,
+            this._storage.storeNames.inboundGroupSessions,
         ]);
         let changes;
         try {
-            changes = await this._handleDecryptedEvent(decryptChanges.payloads, txn);
+            changes = await this._writeDecryptedEvents(decryptChanges.payloads, txn);
             decryptChanges.write(txn);
             txn.session.remove(PENDING_ENCRYPTED_EVENTS);
         } catch (err) {
diff --git a/src/matrix/e2ee/megolm/Decryption.js b/src/matrix/e2ee/megolm/Decryption.js
new file mode 100644
index 00000000..1627564d
--- /dev/null
+++ b/src/matrix/e2ee/megolm/Decryption.js
@@ -0,0 +1,68 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// senderKey is a curve25519 key
+export class Decryption {
+    constructor({pickleKey}) {
+        this._pickleKey = pickleKey;
+    }
+
+    async addRoomKeys(payloads, txn) {
+        const newSessions = [];
+        for (const {senderKey, event} of payloads) {
+            const roomId = event.content?.["room_id"];
+            const sessionId = event.content?.["session_id"];
+            const sessionKey = event.content?.["session_key"];
+
+            if (
+                typeof roomId !== "string" || 
+                typeof sessionId !== "string" || 
+                typeof senderKey !== "string" ||
+                typeof sessionKey !== "string"
+            ) {
+                return;
+            }
+
+            const hasSession = await txn.inboundGroupSessions.has(roomId, senderKey, sessionId);
+            if (!hasSession) {
+                const session = new this._olm.InboundGroupSession();
+                try {
+                    session.create(sessionKey);
+                    const sessionEntry = {
+                        roomId,
+                        senderKey,
+                        sessionId,
+                        session: session.pickle(this._pickleKey),
+                        claimedKeys: event.keys,
+                    };
+                    txn.megOlmInboundSessions.set(sessionEntry);
+                    newSessions.push(sessionEntry);
+                } finally {
+                    session.free();
+                }
+            }
+
+        }
+        return newSessions;
+    }
+
+    applyRoomKeyChanges(newSessions) {
+        // retry decryption with the new sessions
+        if (newSessions.length) {
+            console.log(`I have ${newSessions.length} new inbound group sessions`, newSessions)
+        }
+    }
+}
diff --git a/src/matrix/storage/common.js b/src/matrix/storage/common.js
index 73900af3..76a60e66 100644
--- a/src/matrix/storage/common.js
+++ b/src/matrix/storage/common.js
@@ -25,6 +25,7 @@ export const STORE_NAMES = Object.freeze([
     "userIdentities",
     "deviceIdentities",
     "olmSessions",
+    "inboundGroupSessions",
 ]);
 
 export const STORE_MAP = Object.freeze(STORE_NAMES.reduce((nameMap, name) => {
diff --git a/src/matrix/storage/idb/Transaction.js b/src/matrix/storage/idb/Transaction.js
index 370a5563..fa862c08 100644
--- a/src/matrix/storage/idb/Transaction.js
+++ b/src/matrix/storage/idb/Transaction.js
@@ -27,6 +27,7 @@ import {PendingEventStore} from "./stores/PendingEventStore.js";
 import {UserIdentityStore} from "./stores/UserIdentityStore.js";
 import {DeviceIdentityStore} from "./stores/DeviceIdentityStore.js";
 import {OlmSessionStore} from "./stores/OlmSessionStore.js";
+import {InboundGroupSessionStore} from "./stores/InboundGroupSessionStore.js";
 
 export class Transaction {
     constructor(txn, allowedStoreNames) {
@@ -95,6 +96,10 @@ export class Transaction {
     get olmSessions() {
         return this._store("olmSessions", idbStore => new OlmSessionStore(idbStore));
     }
+    
+    get inboundGroupSessions() {
+        return this._store("inboundGroupSessions", idbStore => new InboundGroupSessionStore(idbStore));
+    }
 
     complete() {
         return txnAsPromise(this._txn);
diff --git a/src/matrix/storage/idb/schema.js b/src/matrix/storage/idb/schema.js
index 8e34ac27..81a56991 100644
--- a/src/matrix/storage/idb/schema.js
+++ b/src/matrix/storage/idb/schema.js
@@ -11,6 +11,7 @@ export const schema = [
     migrateSession,
     createIdentityStores,
     createOlmSessionStore,
+    createInboundGroupSessionsStore,
 ];
 // TODO: how to deal with git merge conflicts of this array?
 
@@ -76,3 +77,8 @@ function createIdentityStores(db) {
 function createOlmSessionStore(db) {
     db.createObjectStore("olmSessions", {keyPath: "key"});
 }
+
+//v6
+function createInboundGroupSessionsStore(db) {
+    db.createObjectStore("inboundGroupSessions", {keyPath: "key"});
+}
diff --git a/src/matrix/storage/idb/stores/InboundGroupSessionStore.js b/src/matrix/storage/idb/stores/InboundGroupSessionStore.js
new file mode 100644
index 00000000..3de5a103
--- /dev/null
+++ b/src/matrix/storage/idb/stores/InboundGroupSessionStore.js
@@ -0,0 +1,36 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+function encodeKey(roomId, senderKey, sessionId) {
+    return `${roomId}|${senderKey}|${sessionId}`;
+}
+
+export class InboundGroupSessionStore {
+    constructor(store) {
+        this._store = store;
+    }
+
+    async has(roomId, senderKey, sessionId) {
+        const key = encodeKey(roomId, senderKey, sessionId);
+        const fetchedKey = await this._store.getKey(key);
+        return key === fetchedKey;
+    }
+
+    set(session) {
+        session.key = encodeKey(session.roomId, session.senderKey, session.sessionId);
+        this._store.put(session);
+    }
+}

From 0219932f5095b6e2f80ba485585bc5a539e074e0 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Wed, 2 Sep 2020 14:29:18 +0200
Subject: [PATCH 055/173] typo

---
 src/matrix/DeviceMessageHandler.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js
index 7154d192..43c084ef 100644
--- a/src/matrix/DeviceMessageHandler.js
+++ b/src/matrix/DeviceMessageHandler.js
@@ -51,7 +51,7 @@ export class DeviceMessageHandler {
         return {megolmChanges};
     }
 
-    applyChanges({megolmChanges}) {
+    _applyDecryptChanges({megolmChanges}) {
         if (megolmChanges) {
             this._megolmDecryption.applyRoomKeyChanges(megolmChanges);
         }
@@ -86,7 +86,7 @@ export class DeviceMessageHandler {
             throw err;
         }
         await txn.complete();
-        this._applyChanges(changes);
+        this._applyDecryptChanges(changes);
     }
 
     async _getPendingEvents(txn) {

From 7d517eb700ee05b04b493ce2dfab603dc457a80a Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Wed, 2 Sep 2020 14:30:18 +0200
Subject: [PATCH 056/173] wire up the olm decryption,megolm room key handler
 and to_device handler

---
 src/matrix/Session.js          | 55 +++++++++++++++++++++++++++++-----
 src/matrix/SessionContainer.js |  3 +-
 2 files changed, 49 insertions(+), 9 deletions(-)

diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index 4d30516a..18a0af2c 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -19,12 +19,15 @@ import { ObservableMap } from "../observable/index.js";
 import { SendScheduler, RateLimitingBackoff } from "./SendScheduler.js";
 import {User} from "./User.js";
 import {Account as E2EEAccount} from "./e2ee/Account.js";
+import {DeviceMessageHandler} from "./DeviceMessageHandler.js";
+import {Decryption as OlmDecryption} from "./e2ee/olm/Decryption.js";
+import {Decryption as MegOlmDecryption} from "./e2ee/megolm/Decryption.js";
 import {DeviceTracker} from "./e2ee/DeviceTracker.js";
 const PICKLE_KEY = "DEFAULT_KEY";
 
 export class Session {
     // sessionInfo contains deviceId, userId and homeServer
-    constructor({storage, hsApi, sessionInfo, olm}) {
+    constructor({storage, hsApi, sessionInfo, olm, clock}) {
         this._storage = storage;
         this._hsApi = hsApi;
         this._syncInfo = null;
@@ -33,16 +36,38 @@ export class Session {
         this._sendScheduler = new SendScheduler({hsApi, backoff: new RateLimitingBackoff()});
         this._roomUpdateCallback = (room, params) => this._rooms.update(room.id, params);
         this._user = new User(sessionInfo.userId);
+        this._clock = clock;
         this._olm = olm;
+        this._olmUtil = null;
         this._e2eeAccount = null;
-        const olmUtil = olm ? new olm.Utility() : null;
-        this._deviceTracker = olm ? new DeviceTracker({
-            storage,
-            getSyncToken: () => this.syncToken,
-            olmUtil,
-        }) : null;
+        this._deviceTracker = null;
+        this._olmDecryption = null;
+        this._deviceMessageHandler = new DeviceMessageHandler({storage});
+        if (olm) {
+            this._olmUtil = new olm.Utility();
+            this._deviceTracker = new DeviceTracker({
+                storage,
+                getSyncToken: () => this.syncToken,
+                olmUtil: this._olmUtil,
+            });
+        }
     }
 
+    // called once this._e2eeAccount is assigned
+    _setupEncryption() {
+        const olmDecryption = new OlmDecryption({
+            account: this._e2eeAccount,
+            pickleKey: PICKLE_KEY,
+            now: this._clock.now,
+            ownUserId: this._user.id,
+            storage: this._storage,
+            olm: this._olm,
+        });
+        const megolmDecryption = new MegOlmDecryption({pickleKey: PICKLE_KEY});
+        this._deviceMessageHandler.enableEncryption({olmDecryption, megolmDecryption});
+    }
+
+    // called after load
     async beforeFirstSync(isNewLogin) {
         if (this._olm) {
             if (isNewLogin && this._e2eeAccount) {
@@ -66,9 +91,11 @@ export class Session {
                     throw err;
                 }
                 await txn.complete();
+                this._setupEncryption();
             }
             await this._e2eeAccount.generateOTKsIfNeeded(this._storage);
             await this._e2eeAccount.uploadKeys(this._storage);
+            await this._deviceMessageHandler.decryptPending();
         }
     }
 
@@ -93,6 +120,9 @@ export class Session {
                 deviceId: this._sessionInfo.deviceId,
                 txn
             });
+            if (this._e2eeAccount) {
+                this._setupEncryption();
+            }
         }
         const pendingEventsByRoomId = await this._getPendingEventsByRoom(txn);
         // load rooms
@@ -175,6 +205,7 @@ export class Session {
         }
         if (this._deviceTracker) {
             for (const {room, changes} of roomChanges) {
+                // TODO: move this so the room passes this to it's "encryption" object in its own writeSync method?
                 if (room.isTrackingMembers && changes.memberChanges?.size) {
                     await this._deviceTracker.writeMemberChanges(room, changes.memberChanges, txn);
                 }
@@ -184,6 +215,11 @@ export class Session {
                 await this._deviceTracker.writeDeviceChanges(deviceLists, txn);
             }
         }
+
+        const toDeviceEvents = syncResponse.to_device?.events;
+        if (Array.isArray(toDeviceEvents)) {
+            this._deviceMessageHandler.writeSync(toDeviceEvents, txn);
+        }
         return changes;
     }
 
@@ -199,11 +235,14 @@ export class Session {
 
     async afterSyncCompleted() {
         const needsToUploadOTKs = await this._e2eeAccount.generateOTKsIfNeeded(this._storage);
+        const promises = [this._deviceMessageHandler.decryptPending()];
         if (needsToUploadOTKs) {
             // TODO: we could do this in parallel with sync if it proves to be too slow
             // but I'm not sure how to not swallow errors in that case
-            await this._e2eeAccount.uploadKeys(this._storage);
+            promises.push(this._e2eeAccount.uploadKeys(this._storage));
         }
+        // run key upload and decryption in parallel
+        await Promise.all(promises);
     }
 
     get syncToken() {
diff --git a/src/matrix/SessionContainer.js b/src/matrix/SessionContainer.js
index 1b6e21d8..7917baf4 100644
--- a/src/matrix/SessionContainer.js
+++ b/src/matrix/SessionContainer.js
@@ -151,7 +151,8 @@ export class SessionContainer {
             homeServer: sessionInfo.homeServer,
         };
         const olm = await this._olmPromise;
-        this._session = new Session({storage: this._storage, sessionInfo: filteredSessionInfo, hsApi, olm});
+        this._session = new Session({storage: this._storage,
+            sessionInfo: filteredSessionInfo, hsApi, olm, clock: this._clock});
         await this._session.load();
         await this._session.beforeFirstSync(isNewLogin);
         

From e09fbf566d2260489512dae91e1b8b7f81da6da8 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Wed, 2 Sep 2020 14:30:49 +0200
Subject: [PATCH 057/173] TODO

---
 src/matrix/Sync.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/matrix/Sync.js b/src/matrix/Sync.js
index e3d519bd..041cacf2 100644
--- a/src/matrix/Sync.js
+++ b/src/matrix/Sync.js
@@ -102,6 +102,7 @@ export class Sync {
             }
             if (!this._error) {
                 try {
+                    // TODO: run this in parallel with the next sync request
                     await this._session.afterSyncCompleted();
                 } catch (err) {
                     console.err("error during after sync completed, continuing to sync.",  err.stack);

From 1f66868566112a3d8c4c208f71f2f2f947d330fc Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Wed, 2 Sep 2020 14:52:02 +0200
Subject: [PATCH 058/173] forgot to await

---
 src/matrix/DeviceMessageHandler.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js
index 43c084ef..9c81f11e 100644
--- a/src/matrix/DeviceMessageHandler.js
+++ b/src/matrix/DeviceMessageHandler.js
@@ -34,7 +34,7 @@ export class DeviceMessageHandler {
     async writeSync(toDeviceEvents, txn) {
         const encryptedEvents = toDeviceEvents.filter(e => e.type === "m.room.encrypted");
         // store encryptedEvents
-        let pendingEvents = this._getPendingEvents(txn);
+        let pendingEvents = await this._getPendingEvents(txn);
         pendingEvents = pendingEvents.concat(encryptedEvents);
         txn.session.set(PENDING_ENCRYPTED_EVENTS, pendingEvents);
         // we don't handle anything other for now
@@ -63,7 +63,7 @@ export class DeviceMessageHandler {
             return;
         }
         const readTxn = await this._storage.readTxn([this._storage.storeNames.session]);
-        const pendingEvents = this._getPendingEvents(readTxn);
+        const pendingEvents = await this._getPendingEvents(readTxn);
         // only know olm for now
         const olmEvents = pendingEvents.filter(e => e.content?.algorithm === OLM_ALGORITHM);
         const decryptChanges = await this._olmDecryption.decryptAll(olmEvents);

From 14cba7ec6e296cf49a510dbada01e0433fb7fa42 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Wed, 2 Sep 2020 14:52:19 +0200
Subject: [PATCH 059/173] need to pass in olm

---
 src/matrix/Session.js                | 2 +-
 src/matrix/e2ee/megolm/Decryption.js | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index 18a0af2c..e07d0d03 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -63,7 +63,7 @@ export class Session {
             storage: this._storage,
             olm: this._olm,
         });
-        const megolmDecryption = new MegOlmDecryption({pickleKey: PICKLE_KEY});
+        const megolmDecryption = new MegOlmDecryption({pickleKey: PICKLE_KEY, olm: this._olm});
         this._deviceMessageHandler.enableEncryption({olmDecryption, megolmDecryption});
     }
 
diff --git a/src/matrix/e2ee/megolm/Decryption.js b/src/matrix/e2ee/megolm/Decryption.js
index 1627564d..94b3ed10 100644
--- a/src/matrix/e2ee/megolm/Decryption.js
+++ b/src/matrix/e2ee/megolm/Decryption.js
@@ -16,8 +16,9 @@ limitations under the License.
 
 // senderKey is a curve25519 key
 export class Decryption {
-    constructor({pickleKey}) {
+    constructor({pickleKey, olm}) {
         this._pickleKey = pickleKey;
+        this._olm = olm;
     }
 
     async addRoomKeys(payloads, txn) {

From 95fcbe1598299f021f2f9b96938f5c41e72c6fc6 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Wed, 2 Sep 2020 14:52:33 +0200
Subject: [PATCH 060/173] typo

---
 src/matrix/Sync.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/matrix/Sync.js b/src/matrix/Sync.js
index 041cacf2..3c04f71a 100644
--- a/src/matrix/Sync.js
+++ b/src/matrix/Sync.js
@@ -105,7 +105,7 @@ export class Sync {
                     // TODO: run this in parallel with the next sync request
                     await this._session.afterSyncCompleted();
                 } catch (err) {
-                    console.err("error during after sync completed, continuing to sync.",  err.stack);
+                    console.error("error during after sync completed, continuing to sync.",  err.stack);
                     // swallowing error here apart from logging
                 }
             }

From 1ab356cd9cb824eb600e280eda5f206d47ccfce8 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Wed, 2 Sep 2020 14:53:50 +0200
Subject: [PATCH 061/173] wrong store name

---
 src/matrix/e2ee/megolm/Decryption.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/matrix/e2ee/megolm/Decryption.js b/src/matrix/e2ee/megolm/Decryption.js
index 94b3ed10..bb5103e6 100644
--- a/src/matrix/e2ee/megolm/Decryption.js
+++ b/src/matrix/e2ee/megolm/Decryption.js
@@ -49,7 +49,7 @@ export class Decryption {
                         session: session.pickle(this._pickleKey),
                         claimedKeys: event.keys,
                     };
-                    txn.megOlmInboundSessions.set(sessionEntry);
+                    txn.inboundGroupSessions.set(sessionEntry);
                     newSessions.push(sessionEntry);
                 } finally {
                     session.free();

From 1dbabf6240fb3175546a6ffe6ef468637c80d108 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Wed, 2 Sep 2020 14:59:17 +0200
Subject: [PATCH 062/173] cleanup ctor

---
 src/matrix/Session.js | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index e07d0d03..503ec3a5 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -27,7 +27,8 @@ const PICKLE_KEY = "DEFAULT_KEY";
 
 export class Session {
     // sessionInfo contains deviceId, userId and homeServer
-    constructor({storage, hsApi, sessionInfo, olm, clock}) {
+    constructor({clock, storage, hsApi, sessionInfo, olm}) {
+        this._clock = clock;
         this._storage = storage;
         this._hsApi = hsApi;
         this._syncInfo = null;
@@ -36,13 +37,11 @@ export class Session {
         this._sendScheduler = new SendScheduler({hsApi, backoff: new RateLimitingBackoff()});
         this._roomUpdateCallback = (room, params) => this._rooms.update(room.id, params);
         this._user = new User(sessionInfo.userId);
-        this._clock = clock;
+        this._deviceMessageHandler = new DeviceMessageHandler({storage});
         this._olm = olm;
         this._olmUtil = null;
         this._e2eeAccount = null;
         this._deviceTracker = null;
-        this._olmDecryption = null;
-        this._deviceMessageHandler = new DeviceMessageHandler({storage});
         if (olm) {
             this._olmUtil = new olm.Utility();
             this._deviceTracker = new DeviceTracker({

From bd64aaf029e7b2abfb61acaec03699db6e0352c9 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Wed, 2 Sep 2020 17:37:13 +0200
Subject: [PATCH 063/173] create outbound olm session from account

also better error handling
---
 src/matrix/e2ee/Account.js | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/src/matrix/e2ee/Account.js b/src/matrix/e2ee/Account.js
index 0478112b..e5c03cbd 100644
--- a/src/matrix/e2ee/Account.js
+++ b/src/matrix/e2ee/Account.js
@@ -126,8 +126,24 @@ export class Account {
 
     createInboundOlmSession(senderKey, body) {
         const newSession = new this._olm.Session();
-        newSession.create_inbound_from(this._account, senderKey, body);
-        return newSession;
+        try {
+            newSession.create_inbound_from(this._account, senderKey, body);
+            return newSession;
+        } catch (err) {
+            newSession.free();
+            throw err;
+        }
+    }
+
+    createOutboundOlmSession(theirIdentityKey, theirOneTimeKey) {
+        const newSession = new this._olm.Session();
+        try {
+            newSession.create_outbound_from(this._account, theirIdentityKey, theirOneTimeKey);
+            return newSession;
+        } catch (err) {
+            newSession.free();
+            throw err;
+        }
     }
 
     writeRemoveOneTimeKey(session, txn) {

From 0545c1f0c52c6c3d7025040298d4546ddfe41e72 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Wed, 2 Sep 2020 17:37:48 +0200
Subject: [PATCH 064/173] extract verifying a signed object from the device
 tracker

---
 src/matrix/e2ee/DeviceTracker.js | 27 +++++----------------------
 src/matrix/e2ee/common.js        | 23 +++++++++++++++++++++++
 2 files changed, 28 insertions(+), 22 deletions(-)

diff --git a/src/matrix/e2ee/DeviceTracker.js b/src/matrix/e2ee/DeviceTracker.js
index b085be80..095730ff 100644
--- a/src/matrix/e2ee/DeviceTracker.js
+++ b/src/matrix/e2ee/DeviceTracker.js
@@ -14,13 +14,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import anotherjson from "../../../lib/another-json/index.js";
+import {verifyEd25519Signature, SIGNATURE_ALGORITHM} from "./common.js";
 
 const TRACKING_STATUS_OUTDATED = 0;
 const TRACKING_STATUS_UPTODATE = 1;
 
-const DEVICE_KEYS_SIGNATURE_ALGORITHM = "ed25519";
-
 // map 1 device from /keys/query response to DeviceIdentity
 function deviceKeysAsDeviceIdentity(deviceSection) {
     const deviceId = deviceSection["device_id"];
@@ -200,7 +198,7 @@ export class DeviceTracker {
                 if (deviceIdOnKeys !== deviceId) {
                     return false;
                 }
-                return this._verifyUserDeviceKeys(deviceKeys);
+                return this._hasValidSignature(deviceKeys);
             });
             const verifiedKeys = verifiedEntries.map(([, deviceKeys]) => deviceKeys);
             return {userId, verifiedKeys};
@@ -208,26 +206,11 @@ export class DeviceTracker {
         return verifiedKeys;
     }
 
-    _verifyUserDeviceKeys(deviceSection) {
+    _hasValidSignature(deviceSection) {
         const deviceId = deviceSection["device_id"];
         const userId = deviceSection["user_id"];
-        const clone = Object.assign({}, deviceSection);
-        delete clone.unsigned;
-        delete clone.signatures;
-        const canonicalJson = anotherjson.stringify(clone);
-        const key = deviceSection?.keys?.[`${DEVICE_KEYS_SIGNATURE_ALGORITHM}:${deviceId}`];
-        const signature = deviceSection?.signatures?.[userId]?.[`${DEVICE_KEYS_SIGNATURE_ALGORITHM}:${deviceId}`];
-        try {
-            if (!signature) {
-                throw new Error("no signature");
-            }
-            // throws when signature is invalid
-            this._olmUtil.ed25519_verify(key, canonicalJson, signature);
-            return true;
-        } catch (err) {
-            console.warn("Invalid device signature, ignoring device.", key, canonicalJson, signature, err);
-            return false;
-        }
+        const ed25519Key = deviceSection?.keys?.[`${SIGNATURE_ALGORITHM}:${deviceId}`];
+        return verifyEd25519Signature(this._olmUtil, userId, deviceId, ed25519Key, deviceSection);
     }
 
     /**
diff --git a/src/matrix/e2ee/common.js b/src/matrix/e2ee/common.js
index c5e7399f..9b3b2f26 100644
--- a/src/matrix/e2ee/common.js
+++ b/src/matrix/e2ee/common.js
@@ -14,6 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+import anotherjson from "../../../lib/another-json/index.js";
+
 // use common prefix so it's easy to clear properties that are not e2ee related during session clear
 export const SESSION_KEY_PREFIX = "e2ee:";
 export const OLM_ALGORITHM = "m.olm.v1.curve25519-aes-sha2";
@@ -27,3 +29,24 @@ export class DecryptionError extends Error {
         this.details = detailsObj;
     }
 }
+
+export const SIGNATURE_ALGORITHM = "ed25519";
+
+export function verifyEd25519Signature(olmUtil, userId, deviceOrKeyId, ed25519Key, value) {
+    const clone = Object.assign({}, value);
+    delete clone.unsigned;
+    delete clone.signatures;
+    const canonicalJson = anotherjson.stringify(clone);
+    const signature = value?.signatures?.[userId]?.[`${SIGNATURE_ALGORITHM}:${deviceOrKeyId}`];
+    try {
+        if (!signature) {
+            throw new Error("no signature");
+        }
+        // throws when signature is invalid
+        this._olmUtil.ed25519_verify(ed25519Key, canonicalJson, signature);
+        return true;
+    } catch (err) {
+        console.warn("Invalid signature, ignoring.", ed25519Key, canonicalJson, signature, err);
+        return false;
+    }
+}

From f1b78a577856017a434b571448ea9ecf3a5bf46d Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Wed, 2 Sep 2020 17:38:25 +0200
Subject: [PATCH 065/173] extract groupBy function from olm decryption into
 util

---
 src/matrix/e2ee/olm/Decryption.js | 12 ++---------
 src/utils/groupBy.js              | 35 +++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+), 10 deletions(-)
 create mode 100644 src/utils/groupBy.js

diff --git a/src/matrix/e2ee/olm/Decryption.js b/src/matrix/e2ee/olm/Decryption.js
index f701f4df..ec87af74 100644
--- a/src/matrix/e2ee/olm/Decryption.js
+++ b/src/matrix/e2ee/olm/Decryption.js
@@ -15,6 +15,7 @@ limitations under the License.
 */
 
 import {DecryptionError} from "../common.js";
+import {groupBy} from "../../../utils/groupBy.js";
 
 const SESSION_LIMIT_PER_SENDER_KEY = 4;
 
@@ -49,16 +50,7 @@ export class Decryption {
     // 
     // doing it one by one would be possible, but we would lose the opportunity for parallelization
     async decryptAll(events) {
-        const eventsPerSenderKey = events.reduce((map, event) => {
-            const senderKey = event.content?.["sender_key"];
-            let list = map.get(senderKey);
-            if (!list) {
-                list = [];
-                map.set(senderKey, list);
-            }
-            list.push(event);
-            return map;
-        }, new Map());
+        const eventsPerSenderKey = groupBy(events, event => event.content?.["sender_key"]);
         const timestamp = this._now();
         const readSessionsTxn = await this._storage.readTxn([this._storage.storeNames.olmSessions]);
         // decrypt events for different sender keys in parallel
diff --git a/src/utils/groupBy.js b/src/utils/groupBy.js
new file mode 100644
index 00000000..5df2f36d
--- /dev/null
+++ b/src/utils/groupBy.js
@@ -0,0 +1,35 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export function groupBy(array, groupFn) {
+    return groupByWithCreator(array, groupFn,
+        () => {return [];},
+        (array, value) => array.push(value)
+    );
+}
+
+export function groupByWithCreator(array, groupFn, createCollectionFn, addCollectionFn) {
+    return array.reduce((map, value) => {
+        const key = groupFn(value);
+        let collection = map.get(key);
+        if (!collection) {
+            collection = createCollectionFn();
+            map.set(key, collection);
+        }
+        addCollectionFn(collection, value);
+        return map;
+    }, new Map());
+}

From 3cb46b38ff29baee4a43ff0483337538a231c1d7 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Wed, 2 Sep 2020 17:38:46 +0200
Subject: [PATCH 066/173] extract olm/Session into own file

---
 src/matrix/e2ee/olm/Decryption.js | 39 +--------------------
 src/matrix/e2ee/olm/Session.js    | 58 +++++++++++++++++++++++++++++++
 2 files changed, 59 insertions(+), 38 deletions(-)
 create mode 100644 src/matrix/e2ee/olm/Session.js

diff --git a/src/matrix/e2ee/olm/Decryption.js b/src/matrix/e2ee/olm/Decryption.js
index ec87af74..01362266 100644
--- a/src/matrix/e2ee/olm/Decryption.js
+++ b/src/matrix/e2ee/olm/Decryption.js
@@ -16,6 +16,7 @@ limitations under the License.
 
 import {DecryptionError} from "../common.js";
 import {groupBy} from "../../../utils/groupBy.js";
+import {Session} from "./Session.js";
 
 const SESSION_LIMIT_PER_SENDER_KEY = 4;
 
@@ -169,44 +170,6 @@ export class Decryption {
     }
 }
 
-class Session {
-    constructor(data, pickleKey, olm, isNew = false) {
-        this.data = data;
-        this._olm = olm;
-        this._pickleKey = pickleKey;
-        this.isNew = isNew;
-        this.isModified = isNew;
-    }
-
-    static create(senderKey, olmSession, olm, pickleKey, timestamp) {
-        return new Session({
-            session: olmSession.pickle(pickleKey),
-            sessionId: olmSession.session_id(),
-            senderKey,
-            lastUsed: timestamp,
-        }, pickleKey, olm, true);
-    }
-
-    get id() {
-        return this.data.sessionId;
-    }
-
-    load() {
-        const session = new this._olm.Session();
-        session.unpickle(this._pickleKey, this.data.session);
-        return session;
-    }
-
-    unload(olmSession) {
-        olmSession.free();
-    }
-
-    save(olmSession) {
-        this.data.session = olmSession.pickle(this._pickleKey);
-        this.isModified = true;
-    }
-}
-
 // decryption helper for a single senderKey
 class SenderKeyDecryption {
     constructor(senderKey, sessions, olm, timestamp) {
diff --git a/src/matrix/e2ee/olm/Session.js b/src/matrix/e2ee/olm/Session.js
new file mode 100644
index 00000000..9b5f4db0
--- /dev/null
+++ b/src/matrix/e2ee/olm/Session.js
@@ -0,0 +1,58 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export function createSessionEntry(olmSession, senderKey, timestamp, pickleKey) {
+    return {
+        session: olmSession.pickle(pickleKey),
+        sessionId: olmSession.session_id(),
+        senderKey,
+        lastUsed: timestamp,
+    };
+}
+
+export class Session {
+    constructor(data, pickleKey, olm, isNew = false) {
+        this.data = data;
+        this._olm = olm;
+        this._pickleKey = pickleKey;
+        this.isNew = isNew;
+        this.isModified = isNew;
+    }
+
+    static create(senderKey, olmSession, olm, pickleKey, timestamp) {
+        const data = createSessionEntry(olmSession, senderKey, timestamp, pickleKey);
+        return new Session(data, pickleKey, olm, true);
+    }
+
+    get id() {
+        return this.data.sessionId;
+    }
+
+    load() {
+        const session = new this._olm.Session();
+        session.unpickle(this._pickleKey, this.data.session);
+        return session;
+    }
+
+    unload(olmSession) {
+        olmSession.free();
+    }
+
+    save(olmSession) {
+        this.data.session = olmSession.pickle(this._pickleKey);
+        this.isModified = true;
+    }
+}

From e3daef5ca9428224801f01494752e550116e08b2 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Wed, 2 Sep 2020 17:58:01 +0200
Subject: [PATCH 067/173] first draft of olm encryption

---
 src/matrix/e2ee/olm/Encryption.js | 224 ++++++++++++++++++++++++++++++
 1 file changed, 224 insertions(+)
 create mode 100644 src/matrix/e2ee/olm/Encryption.js

diff --git a/src/matrix/e2ee/olm/Encryption.js b/src/matrix/e2ee/olm/Encryption.js
new file mode 100644
index 00000000..928fc6a9
--- /dev/null
+++ b/src/matrix/e2ee/olm/Encryption.js
@@ -0,0 +1,224 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {groupByWithCreator} from "../../../utils/groupBy.js";
+import {verifyEd25519Signature, OLM_ALGORITHM} from "./common.js";
+import {createSessionEntry} from "./Session.js";
+
+function findFirstSessionId(sessionIds) {
+    return sessionIds.reduce((first, sessionId) => {
+        if (!first || sessionId < first) {
+            return sessionId;
+        } else {
+            return first;
+        }
+    }, null);
+}
+
+const OTK_ALGORITHM = "signed_curve25519";
+
+export class Encryption {
+    constructor({account, olm, olmUtil, userId, storage, now, pickleKey}) {
+        this._account = account;
+        this._olm = olm;
+        this._olmUtil = olmUtil;
+        this._userId = userId;
+        this._storage = storage;
+        this._now = now;
+        this._pickleKey = pickleKey;
+    }
+
+    async encrypt(type, content, devices, hsApi) {
+        const txn = this._storage.readTxn([this._storage.storeNames.olmSessions]);
+        const sessionIdsForDevice = await Promise.all(devices.map(async device => {
+            return await txn.olmSessions.getSessionIds(device.curve25519Key);
+        }));
+        const devicesWithoutSession = devices.filter((_, i) => {
+            const sessionIds = sessionIdsForDevice[i];
+            return !(sessionIds?.length);
+        });
+
+        const timestamp = this._now(); 
+
+        let encryptionTuples = [];
+
+        if (devicesWithoutSession.length) {
+            const newEncryptionTuples = await this._claimOneTimeKeys(hsApi, devicesWithoutSession);
+            try {
+                for (const tuple of newEncryptionTuples) {
+                    const {device, oneTimeKey} = tuple;
+                    tuple.session = this._account.createOutboundOlmSession(device.curve25519Key, oneTimeKey);
+                }
+                this._storeSessions(newEncryptionTuples, timestamp);
+            } catch (err) {
+                for (const tuple of newEncryptionTuples) {
+                    tuple.dispose();
+                }
+            }
+            encryptionTuples = encryptionTuples.concat(newEncryptionTuples);
+        }
+
+        const existingEncryptionTuples = devices.map((device, i) => {
+            const sessionIds = sessionIdsForDevice[i];
+            if (sessionIds?.length > 0) {
+                const sessionId = findFirstSessionId(sessionIds);
+                return EncryptionTuple.fromSessionId(device, sessionId);
+            }
+        }).filter(tuple => !!tuple);
+        
+        // TODO: if we read and write in two different txns,
+        // is there a chance we overwrite a session modified by the decryption during sync?
+        // I think so. We'll have to have a lock while sending ...
+        await this._loadSessions(existingEncryptionTuples);
+        encryptionTuples = encryptionTuples.concat(existingEncryptionTuples);
+        const ciphertext = this._buildCipherText(type, content, encryptionTuples);
+        await this._storeSessions(encryptionTuples, timestamp);
+        return {
+            type: "m.room.encrypted",
+            content: {
+                algorithm: OLM_ALGORITHM,
+                sender_key: this._account.identityKeys.curve25519,
+                ciphertext
+            }
+        };
+    }
+
+    _buildPlainTextMessageForDevice(type, content, device) {
+        return {
+            keys: {
+                "ed25519": this._account.identityKeys.ed25519
+            },
+            recipient_keys: {
+                "ed25519": device.ed25519Key
+            },
+            recipient: device.userId,
+            sender: this._userId,
+            content,
+            type
+        }
+    }
+
+    _buildCipherText(type, content, encryptionTuples) {
+        const ciphertext = {};
+        for (const {device, session} of encryptionTuples) {
+            if (session) {
+                const message = session.encrypt(this._buildPlainTextMessageForDevice(type, content, device));
+                ciphertext[device.curve25519Key] = message;
+            }
+        }
+        return ciphertext;
+    }
+
+    async _claimOneTimeKeys(hsApi, deviceIdentities) {
+        // create a Map>
+        const devicesByUser = groupByWithCreator(deviceIdentities,
+            device => device.userId,
+            () => new Map(),
+            (deviceMap, device) => deviceMap.set(device.deviceId, device)
+        );
+        const oneTimeKeys = Array.from(devicesByUser.entries()).reduce((usersObj, [userId, deviceMap]) => {
+            usersObj[userId] = Array.from(deviceMap.values()).reduce((devicesObj, device) => {
+                devicesObj[device.deviceId] = OTK_ALGORITHM;
+                return devicesObj;
+            }, {});
+            return usersObj;
+        }, {});
+        const claimResponse = await hsApi.claimKeys({
+            timeout: 10000,
+            one_time_keys: oneTimeKeys
+        }).response();
+        // TODO: log claimResponse.failures
+        const userKeyMap = claimResponse?.["one_time_keys"];
+        return this._verifyAndCreateOTKTuples(userKeyMap, devicesByUser);
+    }
+
+    _verifyAndCreateOTKTuples(userKeyMap, devicesByUser) {
+        const verifiedEncryptionTuples = [];
+        for (const [userId, userSection] of Object.entries(userKeyMap)) {
+            for (const [deviceId, deviceSection] of Object.entries(userSection)) {
+                const [firstPropName, keySection] = Object.entries(deviceSection)[0];
+                const [keyAlgorithm] = firstPropName.split(":");
+                if (keyAlgorithm === OTK_ALGORITHM) {
+                    const device = devicesByUser.get(userId)?.get(deviceId);
+                    if (device) {
+                        const isValidSignature = verifyEd25519Signature(
+                            this._olmUtil, userId, deviceId, device.ed25519Key, keySection);
+                        if (isValidSignature) {
+                            verifiedEncryptionTuples.push(EncryptionTuple.fromOTK(device, keySection.key));
+                        }
+                    }
+                }
+            } 
+        }
+        return verifiedEncryptionTuples;
+    }
+
+    async _loadSessions(encryptionTuples) {
+        const txn = this._storage.readTxn([this._storage.storeNames.olmSessions]);
+        await Promise.all(encryptionTuples.map(async encryptionTuple => {
+            const sessionEntry = await txn.olmSessions.get(
+                encryptionTuple.device.curve25519Key, encryptionTuple.sessionId);
+            if (sessionEntry) {
+                const olmSession = new this._olm.Session();
+                encryptionTuple.session = 
+            }
+
+        }));
+    }
+
+    async _storeSessions(encryptionTuples, timestamp) {
+        const txn = this._storage.readWriteTxn([this._storage.storeNames.olmSessions]);
+        try {
+            for (const tuple of encryptionTuples) {
+                const sessionEntry = createSessionEntry(
+                    tuple.session, tuple.device.curve25519Key, timestamp, this._pickleKey);
+                txn.olmSessions.set(sessionEntry);
+            }
+        } catch (err) {
+            txn.abort();
+            throw err;
+        }
+        await txn.complete();
+    }
+}
+
+// just a container needed to encrypt a message for a recipient device
+// it is constructed with either a oneTimeKey
+// (and later converted to a session) in case of a new session
+// or an existing session
+class EncryptionTuple {
+    constructor(device, oneTimeKey, sessionId) {
+        this.device = device;
+        this.oneTimeKey = oneTimeKey;
+        this.sessionId = sessionId;
+        // an olmSession, should probably be called olmSession
+        this.session = null;
+    }
+
+    static fromOTK(device, oneTimeKey) {
+        return new EncryptionTuple(device, oneTimeKey, null);
+    }
+
+    static fromSessionId(device, sessionId) {
+        return new EncryptionTuple(device, null, sessionId);
+    }
+
+    dispose() {
+        if (this.session) {
+            this.session.free();
+        }
+    }
+}

From 1492b6b6f8b403792c09d1bd79b713c9270426e0 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 3 Sep 2020 09:53:16 +0200
Subject: [PATCH 068/173] cleanup of olm encryption

---
 src/matrix/e2ee/olm/Encryption.js | 166 ++++++++++++++++++------------
 1 file changed, 102 insertions(+), 64 deletions(-)

diff --git a/src/matrix/e2ee/olm/Encryption.js b/src/matrix/e2ee/olm/Encryption.js
index 928fc6a9..fee485a4 100644
--- a/src/matrix/e2ee/olm/Encryption.js
+++ b/src/matrix/e2ee/olm/Encryption.js
@@ -42,6 +42,39 @@ export class Encryption {
     }
 
     async encrypt(type, content, devices, hsApi) {
+        const {
+            devicesWithoutSession,
+            existingEncryptionTargets
+        } = await this._findExistingSessions(devices);
+    
+        const timestamp = this._now(); 
+
+        let encryptionTargets = [];
+        try {
+            if (devicesWithoutSession.length) {
+                const newEncryptionTargets = await this._createNewSessions(
+                    devicesWithoutSession, hsApi, timestamp);
+                encryptionTargets = encryptionTargets.concat(newEncryptionTargets);
+            }
+            // TODO: if we read and write in two different txns,
+            // is there a chance we overwrite a session modified by the decryption during sync?
+            // I think so. We'll have to have a lock while sending ...
+            await this._loadSessions(existingEncryptionTargets);
+            encryptionTargets = encryptionTargets.concat(existingEncryptionTargets);
+            const messages = encryptionTargets.map(target => {
+                const content = this._encryptForDevice(type, content, target);
+                return new EncryptedMessage(content, target.device);
+            });
+            await this._storeSessions(encryptionTargets, timestamp);
+            return messages;
+        } finally {
+            for (const target of encryptionTargets) {
+                target.dispose();
+            }
+        }
+    }
+
+    async _findExistingSessions(devices) {
         const txn = this._storage.readTxn([this._storage.storeNames.olmSessions]);
         const sessionIdsForDevice = await Promise.all(devices.map(async device => {
             return await txn.olmSessions.getSessionIds(device.curve25519Key);
@@ -51,49 +84,28 @@ export class Encryption {
             return !(sessionIds?.length);
         });
 
-        const timestamp = this._now(); 
-
-        let encryptionTuples = [];
-
-        if (devicesWithoutSession.length) {
-            const newEncryptionTuples = await this._claimOneTimeKeys(hsApi, devicesWithoutSession);
-            try {
-                for (const tuple of newEncryptionTuples) {
-                    const {device, oneTimeKey} = tuple;
-                    tuple.session = this._account.createOutboundOlmSession(device.curve25519Key, oneTimeKey);
-                }
-                this._storeSessions(newEncryptionTuples, timestamp);
-            } catch (err) {
-                for (const tuple of newEncryptionTuples) {
-                    tuple.dispose();
-                }
-            }
-            encryptionTuples = encryptionTuples.concat(newEncryptionTuples);
-        }
-
-        const existingEncryptionTuples = devices.map((device, i) => {
+        const existingEncryptionTargets = devices.map((device, i) => {
             const sessionIds = sessionIdsForDevice[i];
             if (sessionIds?.length > 0) {
                 const sessionId = findFirstSessionId(sessionIds);
-                return EncryptionTuple.fromSessionId(device, sessionId);
+                return EncryptionTarget.fromSessionId(device, sessionId);
             }
-        }).filter(tuple => !!tuple);
-        
-        // TODO: if we read and write in two different txns,
-        // is there a chance we overwrite a session modified by the decryption during sync?
-        // I think so. We'll have to have a lock while sending ...
-        await this._loadSessions(existingEncryptionTuples);
-        encryptionTuples = encryptionTuples.concat(existingEncryptionTuples);
-        const ciphertext = this._buildCipherText(type, content, encryptionTuples);
-        await this._storeSessions(encryptionTuples, timestamp);
-        return {
-            type: "m.room.encrypted",
-            content: {
-                algorithm: OLM_ALGORITHM,
-                sender_key: this._account.identityKeys.curve25519,
-                ciphertext
+        }).filter(target => !!target);
+
+        return {devicesWithoutSession, existingEncryptionTargets};
+    }
+
+    _encryptForDevice(type, content, target) {
+        const {session, device} = target;
+        const message = session.encrypt(this._buildPlainTextMessageForDevice(type, content, device));
+        const encryptedContent = {
+            algorithm: OLM_ALGORITHM,
+            sender_key: this._account.identityKeys.curve25519,
+            ciphertext: {
+                [device.curve25519Key]: message
             }
         };
+        return encryptedContent;
     }
 
     _buildPlainTextMessageForDevice(type, content, device) {
@@ -111,15 +123,20 @@ export class Encryption {
         }
     }
 
-    _buildCipherText(type, content, encryptionTuples) {
-        const ciphertext = {};
-        for (const {device, session} of encryptionTuples) {
-            if (session) {
-                const message = session.encrypt(this._buildPlainTextMessageForDevice(type, content, device));
-                ciphertext[device.curve25519Key] = message;
+    async _createNewSessions(devicesWithoutSession, hsApi, timestamp) {
+        const newEncryptionTargets = await this._claimOneTimeKeys(hsApi, devicesWithoutSession);
+        try {
+            for (const target of newEncryptionTargets) {
+                const {device, oneTimeKey} = target;
+                target.session = this._account.createOutboundOlmSession(device.curve25519Key, oneTimeKey);
+            }
+            this._storeSessions(newEncryptionTargets, timestamp);
+        } catch (err) {
+            for (const target of newEncryptionTargets) {
+                target.dispose();
             }
         }
-        return ciphertext;
+        return newEncryptionTargets;
     }
 
     async _claimOneTimeKeys(hsApi, deviceIdentities) {
@@ -142,11 +159,11 @@ export class Encryption {
         }).response();
         // TODO: log claimResponse.failures
         const userKeyMap = claimResponse?.["one_time_keys"];
-        return this._verifyAndCreateOTKTuples(userKeyMap, devicesByUser);
+        return this._verifyAndCreateOTKTargets(userKeyMap, devicesByUser);
     }
 
-    _verifyAndCreateOTKTuples(userKeyMap, devicesByUser) {
-        const verifiedEncryptionTuples = [];
+    _verifyAndCreateOTKTargets(userKeyMap, devicesByUser) {
+        const verifiedEncryptionTargets = [];
         for (const [userId, userSection] of Object.entries(userKeyMap)) {
             for (const [deviceId, deviceSection] of Object.entries(userSection)) {
                 const [firstPropName, keySection] = Object.entries(deviceSection)[0];
@@ -157,34 +174,48 @@ export class Encryption {
                         const isValidSignature = verifyEd25519Signature(
                             this._olmUtil, userId, deviceId, device.ed25519Key, keySection);
                         if (isValidSignature) {
-                            verifiedEncryptionTuples.push(EncryptionTuple.fromOTK(device, keySection.key));
+                            const target = EncryptionTarget.fromOTK(device, keySection.key);
+                            verifiedEncryptionTargets.push(target);
                         }
                     }
                 }
             } 
         }
-        return verifiedEncryptionTuples;
+        return verifiedEncryptionTargets;
     }
 
-    async _loadSessions(encryptionTuples) {
+    async _loadSessions(encryptionTargets) {
         const txn = this._storage.readTxn([this._storage.storeNames.olmSessions]);
-        await Promise.all(encryptionTuples.map(async encryptionTuple => {
-            const sessionEntry = await txn.olmSessions.get(
-                encryptionTuple.device.curve25519Key, encryptionTuple.sessionId);
-            if (sessionEntry) {
-                const olmSession = new this._olm.Session();
-                encryptionTuple.session = 
+        // given we run loading in parallel, there might still be some
+        // storage requests that will finish later once one has failed.
+        // those should not allocate a session anymore.
+        let failed = false;
+        try {
+            await Promise.all(encryptionTargets.map(async encryptionTarget => {
+                const sessionEntry = await txn.olmSessions.get(
+                    encryptionTarget.device.curve25519Key, encryptionTarget.sessionId);
+                if (sessionEntry && !failed) {
+                    const olmSession = new this._olm.Session();
+                    olmSession.unpickle(this._pickleKey, sessionEntry.session);
+                    encryptionTarget.session = olmSession;
+                }
+            }));
+        } catch (err) {
+            failed = true;
+            // clean up the sessions that did load
+            for (const target of encryptionTargets) {
+                target.dispose();
             }
-
-        }));
+            throw err;
+        }
     }
 
-    async _storeSessions(encryptionTuples, timestamp) {
+    async _storeSessions(encryptionTargets, timestamp) {
         const txn = this._storage.readWriteTxn([this._storage.storeNames.olmSessions]);
         try {
-            for (const tuple of encryptionTuples) {
+            for (const target of encryptionTargets) {
                 const sessionEntry = createSessionEntry(
-                    tuple.session, tuple.device.curve25519Key, timestamp, this._pickleKey);
+                    target.session, target.device.curve25519Key, timestamp, this._pickleKey);
                 txn.olmSessions.set(sessionEntry);
             }
         } catch (err) {
@@ -199,7 +230,7 @@ export class Encryption {
 // it is constructed with either a oneTimeKey
 // (and later converted to a session) in case of a new session
 // or an existing session
-class EncryptionTuple {
+class EncryptionTarget {
     constructor(device, oneTimeKey, sessionId) {
         this.device = device;
         this.oneTimeKey = oneTimeKey;
@@ -209,11 +240,11 @@ class EncryptionTuple {
     }
 
     static fromOTK(device, oneTimeKey) {
-        return new EncryptionTuple(device, oneTimeKey, null);
+        return new EncryptionTarget(device, oneTimeKey, null);
     }
 
     static fromSessionId(device, sessionId) {
-        return new EncryptionTuple(device, null, sessionId);
+        return new EncryptionTarget(device, null, sessionId);
     }
 
     dispose() {
@@ -222,3 +253,10 @@ class EncryptionTuple {
         }
     }
 }
+
+class EncryptedMessage {
+    constructor(content, device) {
+        this.content = content;
+        this.device = device;
+    }
+}

From 279b55e8e6c560bd804dc4da98df2bdca76dc9fc Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 3 Sep 2020 11:31:00 +0200
Subject: [PATCH 069/173] fix test

---
 src/matrix/Session.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index 503ec3a5..2752fca7 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -301,7 +301,7 @@ export function tests() {
                     }
                 }
             };
-            const newSessionData = session.writeSync("b", 6, {}, syncTxn);
+            const newSessionData = await session.writeSync({next_batch: "b"}, 6, {}, syncTxn);
             assert(syncSet);
             assert.equal(session.syncToken, "a");
             assert.equal(session.syncFilterId, 5);

From 4f4808b94c3f501fab0d4a74723eb15940cfaad1 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 3 Sep 2020 12:12:33 +0200
Subject: [PATCH 070/173] lock on senderKey while enc/decrypting olm sessions

---
 src/matrix/Session.js             |  4 ++
 src/matrix/e2ee/olm/Decryption.js | 74 +++++++++++++++---------
 src/matrix/e2ee/olm/Encryption.js | 62 ++++++++++++---------
 src/utils/Lock.js                 | 86 ++++++++++++++++++++++++++++
 src/utils/LockMap.js              | 93 +++++++++++++++++++++++++++++++
 5 files changed, 266 insertions(+), 53 deletions(-)
 create mode 100644 src/utils/Lock.js
 create mode 100644 src/utils/LockMap.js

diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index 2752fca7..803aadc8 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -23,6 +23,8 @@ import {DeviceMessageHandler} from "./DeviceMessageHandler.js";
 import {Decryption as OlmDecryption} from "./e2ee/olm/Decryption.js";
 import {Decryption as MegOlmDecryption} from "./e2ee/megolm/Decryption.js";
 import {DeviceTracker} from "./e2ee/DeviceTracker.js";
+import {LockMap} from "../utils/LockMap.js";
+
 const PICKLE_KEY = "DEFAULT_KEY";
 
 export class Session {
@@ -54,6 +56,7 @@ export class Session {
 
     // called once this._e2eeAccount is assigned
     _setupEncryption() {
+        const senderKeyLock = new LockMap();
         const olmDecryption = new OlmDecryption({
             account: this._e2eeAccount,
             pickleKey: PICKLE_KEY,
@@ -61,6 +64,7 @@ export class Session {
             ownUserId: this._user.id,
             storage: this._storage,
             olm: this._olm,
+            senderKeyLock
         });
         const megolmDecryption = new MegOlmDecryption({pickleKey: PICKLE_KEY, olm: this._olm});
         this._deviceMessageHandler.enableEncryption({olmDecryption, megolmDecryption});
diff --git a/src/matrix/e2ee/olm/Decryption.js b/src/matrix/e2ee/olm/Decryption.js
index 01362266..c21c4b3d 100644
--- a/src/matrix/e2ee/olm/Decryption.js
+++ b/src/matrix/e2ee/olm/Decryption.js
@@ -31,14 +31,14 @@ function sortSessions(sessions) {
 }
 
 export class Decryption {
-    constructor({account, pickleKey, now, ownUserId, storage, olm}) {
+    constructor({account, pickleKey, now, ownUserId, storage, olm, senderKeyLock}) {
         this._account = account;
         this._pickleKey = pickleKey;
         this._now = now;
         this._ownUserId = ownUserId;
         this._storage = storage;
         this._olm = olm;
-        this._createOutboundSessionPromise = null;
+        this._senderKeyLock = senderKeyLock;
     }
 
     // we need decryptAll because there is some parallelization we can do for decrypting different sender keys at once
@@ -53,15 +53,28 @@ export class Decryption {
     async decryptAll(events) {
         const eventsPerSenderKey = groupBy(events, event => event.content?.["sender_key"]);
         const timestamp = this._now();
-        const readSessionsTxn = await this._storage.readTxn([this._storage.storeNames.olmSessions]);
-        // decrypt events for different sender keys in parallel
-        const results = await Promise.all(Array.from(eventsPerSenderKey.entries()).map(([senderKey, events]) => {
-            return this._decryptAllForSenderKey(senderKey, events, timestamp, readSessionsTxn);
+        // take a lock on all senderKeys so encryption or other calls to decryptAll (should not happen)
+        // don't modify the sessions at the same time
+        const locks = await Promise.all(Array.from(eventsPerSenderKey.keys()).map(senderKey => {
+            return this._senderKeyLock.takeLock(senderKey);
         }));
-        const payloads = results.reduce((all, r) => all.concat(r.payloads), []);
-        const errors = results.reduce((all, r) => all.concat(r.errors), []);
-        const senderKeyDecryptions = results.map(r => r.senderKeyDecryption);
-        return new DecryptionChanges(senderKeyDecryptions, payloads, errors);
+        try {
+            const readSessionsTxn = await this._storage.readTxn([this._storage.storeNames.olmSessions]);
+            // decrypt events for different sender keys in parallel
+            const results = await Promise.all(Array.from(eventsPerSenderKey.entries()).map(([senderKey, events]) => {
+                return this._decryptAllForSenderKey(senderKey, events, timestamp, readSessionsTxn);
+            }));
+            const payloads = results.reduce((all, r) => all.concat(r.payloads), []);
+            const errors = results.reduce((all, r) => all.concat(r.errors), []);
+            const senderKeyDecryptions = results.map(r => r.senderKeyDecryption);
+            return new DecryptionChanges(senderKeyDecryptions, payloads, errors, locks);
+        } catch (err) {
+            // make sure the locks are release if something throws
+            for (const lock of locks) {
+                lock.release();
+            }
+            throw err;
+        }
     }
 
     async _decryptAllForSenderKey(senderKey, events, timestamp, readSessionsTxn) {
@@ -235,11 +248,12 @@ class SenderKeyDecryption {
 }
 
 class DecryptionChanges {
-    constructor(senderKeyDecryptions, payloads, errors, account) {
+    constructor(senderKeyDecryptions, payloads, errors, account, locks) {
         this._senderKeyDecryptions = senderKeyDecryptions;
         this._account = account;    
         this.payloads = payloads;
         this.errors = errors;
+        this._locks = locks;
     }
 
     get hasNewSessions() {
@@ -247,25 +261,31 @@ class DecryptionChanges {
     }
 
     write(txn) {
-        for (const senderKeyDecryption of this._senderKeyDecryptions) {
-            for (const session of senderKeyDecryption.getModifiedSessions()) {
-                txn.olmSessions.set(session.data);
-                if (session.isNew) {
-                    const olmSession = session.load();
-                    try {
-                        this._account.writeRemoveOneTimeKey(olmSession, txn);
-                    } finally {
-                        session.unload(olmSession);
+        try {
+            for (const senderKeyDecryption of this._senderKeyDecryptions) {
+                for (const session of senderKeyDecryption.getModifiedSessions()) {
+                    txn.olmSessions.set(session.data);
+                    if (session.isNew) {
+                        const olmSession = session.load();
+                        try {
+                            this._account.writeRemoveOneTimeKey(olmSession, txn);
+                        } finally {
+                            session.unload(olmSession);
+                        }
+                    }
+                }
+                if (senderKeyDecryption.sessions.length > SESSION_LIMIT_PER_SENDER_KEY) {
+                    const {senderKey, sessions} = senderKeyDecryption;
+                    // >= because index is zero-based
+                    for (let i = sessions.length - 1; i >= SESSION_LIMIT_PER_SENDER_KEY ; i -= 1) {
+                        const session = sessions[i];
+                        txn.olmSessions.remove(senderKey, session.id);
                     }
                 }
             }
-            if (senderKeyDecryption.sessions.length > SESSION_LIMIT_PER_SENDER_KEY) {
-                const {senderKey, sessions} = senderKeyDecryption;
-                // >= because index is zero-based
-                for (let i = sessions.length - 1; i >= SESSION_LIMIT_PER_SENDER_KEY ; i -= 1) {
-                    const session = sessions[i];
-                    txn.olmSessions.remove(senderKey, session.id);
-                }
+        } finally {
+            for (const lock of this._locks) {
+                lock.release();
             }
         }
     }
diff --git a/src/matrix/e2ee/olm/Encryption.js b/src/matrix/e2ee/olm/Encryption.js
index fee485a4..1461b94c 100644
--- a/src/matrix/e2ee/olm/Encryption.js
+++ b/src/matrix/e2ee/olm/Encryption.js
@@ -31,7 +31,7 @@ function findFirstSessionId(sessionIds) {
 const OTK_ALGORITHM = "signed_curve25519";
 
 export class Encryption {
-    constructor({account, olm, olmUtil, userId, storage, now, pickleKey}) {
+    constructor({account, olm, olmUtil, userId, storage, now, pickleKey, senderKeyLock}) {
         this._account = account;
         this._olm = olm;
         this._olmUtil = olmUtil;
@@ -39,37 +39,47 @@ export class Encryption {
         this._storage = storage;
         this._now = now;
         this._pickleKey = pickleKey;
+        this._senderKeyLock = senderKeyLock;
     }
 
     async encrypt(type, content, devices, hsApi) {
-        const {
-            devicesWithoutSession,
-            existingEncryptionTargets
-        } = await this._findExistingSessions(devices);
-    
-        const timestamp = this._now(); 
-
-        let encryptionTargets = [];
+        // TODO: see if we can only hold some of the locks until after the /keys/claim call (if needed) 
+        // take a lock on all senderKeys so decryption and other calls to encrypt (should not happen)
+        // don't modify the sessions at the same time
+        const locks = await Promise.all(devices.map(device => {
+            return this._senderKeyLock.takeLock(device.curve25519Key);
+        }));
         try {
-            if (devicesWithoutSession.length) {
-                const newEncryptionTargets = await this._createNewSessions(
-                    devicesWithoutSession, hsApi, timestamp);
-                encryptionTargets = encryptionTargets.concat(newEncryptionTargets);
+            const {
+                devicesWithoutSession,
+                existingEncryptionTargets,
+            } = await this._findExistingSessions(devices);
+        
+            const timestamp = this._now(); 
+
+            let encryptionTargets = [];
+            try {
+                if (devicesWithoutSession.length) {
+                    const newEncryptionTargets = await this._createNewSessions(
+                        devicesWithoutSession, hsApi, timestamp);
+                    encryptionTargets = encryptionTargets.concat(newEncryptionTargets);
+                }
+                await this._loadSessions(existingEncryptionTargets);
+                encryptionTargets = encryptionTargets.concat(existingEncryptionTargets);
+                const messages = encryptionTargets.map(target => {
+                    const content = this._encryptForDevice(type, content, target);
+                    return new EncryptedMessage(content, target.device);
+                });
+                await this._storeSessions(encryptionTargets, timestamp);
+                return messages;
+            } finally {
+                for (const target of encryptionTargets) {
+                    target.dispose();
+                }
             }
-            // TODO: if we read and write in two different txns,
-            // is there a chance we overwrite a session modified by the decryption during sync?
-            // I think so. We'll have to have a lock while sending ...
-            await this._loadSessions(existingEncryptionTargets);
-            encryptionTargets = encryptionTargets.concat(existingEncryptionTargets);
-            const messages = encryptionTargets.map(target => {
-                const content = this._encryptForDevice(type, content, target);
-                return new EncryptedMessage(content, target.device);
-            });
-            await this._storeSessions(encryptionTargets, timestamp);
-            return messages;
         } finally {
-            for (const target of encryptionTargets) {
-                target.dispose();
+            for (const lock of locks) {
+                lock.release();
             }
         }
     }
diff --git a/src/utils/Lock.js b/src/utils/Lock.js
new file mode 100644
index 00000000..21d5d7a2
--- /dev/null
+++ b/src/utils/Lock.js
@@ -0,0 +1,86 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export class Lock {
+    constructor() {
+        this._promise = null;
+        this._resolve = null;
+    }
+
+    take() {
+        if (!this._promise) {
+            this._promise = new Promise(resolve => {
+                this._resolve = resolve;
+            });
+            return true;
+        }
+        return false;
+    }
+
+    get isTaken() {
+        return !!this._promise;
+    }
+
+    release() {
+        if (this._resolve) {
+            this._promise = null;
+            const resolve = this._resolve;
+            this._resolve = null;
+            resolve();
+        }
+    }
+
+    released() {
+        return this._promise;
+    }
+}
+
+export function tests() {
+    return {
+        "taking a lock twice returns false": assert => {
+            const lock = new Lock();
+            assert.equal(lock.take(), true);
+            assert.equal(lock.isTaken, true);
+            assert.equal(lock.take(), false);
+        },
+        "can take a released lock again": assert => {
+            const lock = new Lock();
+            lock.take();
+            lock.release();
+            assert.equal(lock.isTaken, false);
+            assert.equal(lock.take(), true);
+        },
+        "2 waiting for lock, only first one gets it": async assert => {
+            const lock = new Lock();
+            lock.take();
+
+            let first = false;
+            lock.released().then(() => first = lock.take());
+            let second = false;
+            lock.released().then(() => second = lock.take());
+            const promise = lock.released();
+            lock.release();
+            await promise;
+            assert.equal(first, true);
+            assert.equal(second, false);
+        },
+        "await non-taken lock": async assert => {
+            const lock = new Lock();
+            await lock.released();
+            assert(true);
+        }
+    }
+}
diff --git a/src/utils/LockMap.js b/src/utils/LockMap.js
new file mode 100644
index 00000000..f99776cc
--- /dev/null
+++ b/src/utils/LockMap.js
@@ -0,0 +1,93 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {Lock} from "./Lock.js";
+
+export class LockMap {
+    constructor() {
+        this._map = new Map();
+    }
+
+    async takeLock(key) {
+        let lock = this._map.get(key);
+        if (lock) {
+            while (!lock.take()) {
+                await lock.released();
+            }
+        } else {
+            lock = new Lock();
+            lock.take();
+            this._map.set(key, lock);
+        }
+        // don't leave old locks lying around
+        lock.released().then(() => {
+            // give others a chance to take the lock first
+            Promise.resolve().then(() => {
+                if (!lock.isTaken) {
+                    this._map.delete(key);
+                }
+            });
+        });
+        return lock;
+    }
+}
+
+export function tests() {
+    return {
+        "taking a lock on the same key blocks": async assert => {
+            const lockMap = new LockMap();
+            const lock = await lockMap.takeLock("foo");
+            let second = false;
+            const prom = lockMap.takeLock("foo").then(() => {
+                second = true;
+            });
+            assert.equal(second, false);
+            // do a delay to make sure prom does not resolve on its own
+            await Promise.resolve();
+            lock.release();
+            await prom;
+            assert.equal(second, true);
+        },
+        "lock is not cleaned up with second request": async assert => {
+            const lockMap = new LockMap();
+            const lock = await lockMap.takeLock("foo");
+            let ranSecond = false;
+            const prom = lockMap.takeLock("foo").then(returnedLock => {
+                ranSecond = true;
+                assert.equal(returnedLock.isTaken, true);
+                // peek into internals, naughty
+                assert.equal(lockMap._map.get("foo"), returnedLock);
+            });
+            lock.release();
+            await prom;
+            // double delay to make sure cleanup logic ran
+            await Promise.resolve();
+            await Promise.resolve();
+            assert.equal(ranSecond, true);
+        },
+        "lock is cleaned up without other request": async assert => {
+            const lockMap = new LockMap();
+            const lock = await lockMap.takeLock("foo");
+            await Promise.resolve();
+            lock.release();
+            // double delay to make sure cleanup logic ran
+            await Promise.resolve();
+            await Promise.resolve();
+            assert.equal(lockMap._map.has("foo"), false);
+        },
+        
+    };
+}

From af423b1c7f20c914d52832e378e99633396ce7d2 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 3 Sep 2020 12:17:01 +0200
Subject: [PATCH 071/173] ensure second promise has run in test

---
 src/utils/Lock.js | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/utils/Lock.js b/src/utils/Lock.js
index 21d5d7a2..6a198097 100644
--- a/src/utils/Lock.js
+++ b/src/utils/Lock.js
@@ -67,15 +67,15 @@ export function tests() {
             const lock = new Lock();
             lock.take();
 
-            let first = false;
+            let first;
             lock.released().then(() => first = lock.take());
-            let second = false;
+            let second;
             lock.released().then(() => second = lock.take());
             const promise = lock.released();
             lock.release();
             await promise;
-            assert.equal(first, true);
-            assert.equal(second, false);
+            assert.strictEqual(first, true);
+            assert.strictEqual(second, false);
         },
         "await non-taken lock": async assert => {
             const lock = new Lock();

From 8d64fa54fe2bf5278cfe30ce97584d84010b4bd0 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 3 Sep 2020 15:27:00 +0200
Subject: [PATCH 072/173] using wrong method here

---
 src/matrix/e2ee/Account.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/matrix/e2ee/Account.js b/src/matrix/e2ee/Account.js
index e5c03cbd..9d83465c 100644
--- a/src/matrix/e2ee/Account.js
+++ b/src/matrix/e2ee/Account.js
@@ -138,7 +138,7 @@ export class Account {
     createOutboundOlmSession(theirIdentityKey, theirOneTimeKey) {
         const newSession = new this._olm.Session();
         try {
-            newSession.create_outbound_from(this._account, theirIdentityKey, theirOneTimeKey);
+            newSession.create_outbound(this._account, theirIdentityKey, theirOneTimeKey);
             return newSession;
         } catch (err) {
             newSession.free();

From eda15e11418b28b664845ee492228ece27dc1b37 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 3 Sep 2020 15:27:40 +0200
Subject: [PATCH 073/173] forgot to remove this after extracting function

---
 src/matrix/e2ee/common.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/matrix/e2ee/common.js b/src/matrix/e2ee/common.js
index 9b3b2f26..3312032b 100644
--- a/src/matrix/e2ee/common.js
+++ b/src/matrix/e2ee/common.js
@@ -43,7 +43,7 @@ export function verifyEd25519Signature(olmUtil, userId, deviceOrKeyId, ed25519Ke
             throw new Error("no signature");
         }
         // throws when signature is invalid
-        this._olmUtil.ed25519_verify(ed25519Key, canonicalJson, signature);
+        olmUtil.ed25519_verify(ed25519Key, canonicalJson, signature);
         return true;
     } catch (err) {
         console.warn("Invalid signature, ignoring.", ed25519Key, canonicalJson, signature, err);

From e22131bf57a3fb89f6b538cdc6284c4564d581c0 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 3 Sep 2020 15:28:03 +0200
Subject: [PATCH 074/173] don't store or return our own device

---
 src/matrix/Session.js            |  2 ++
 src/matrix/e2ee/DeviceTracker.js | 14 ++++++++++++--
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index 803aadc8..6929430f 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -50,6 +50,8 @@ export class Session {
                 storage,
                 getSyncToken: () => this.syncToken,
                 olmUtil: this._olmUtil,
+                ownUserId: sessionInfo.userId,
+                ownDeviceId: sessionInfo.deviceId,
             });
         }
     }
diff --git a/src/matrix/e2ee/DeviceTracker.js b/src/matrix/e2ee/DeviceTracker.js
index 095730ff..84da2f37 100644
--- a/src/matrix/e2ee/DeviceTracker.js
+++ b/src/matrix/e2ee/DeviceTracker.js
@@ -34,11 +34,13 @@ function deviceKeysAsDeviceIdentity(deviceSection) {
 }
 
 export class DeviceTracker {
-    constructor({storage, getSyncToken, olmUtil}) {
+    constructor({storage, getSyncToken, olmUtil, ownUserId, ownDeviceId}) {
         this._storage = storage;
         this._getSyncToken = getSyncToken;
         this._identityChangedForRoom = null;
         this._olmUtil = olmUtil;
+        this._ownUserId = ownUserId;
+        this._ownDeviceId = ownDeviceId;
     }
 
     async writeDeviceChanges(deviceLists, txn) {
@@ -198,6 +200,10 @@ export class DeviceTracker {
                 if (deviceIdOnKeys !== deviceId) {
                     return false;
                 }
+                // don't store our own device
+                if (userId === this._ownUserId && deviceId === this._ownDeviceId) {
+                    return false;
+                }
                 return this._hasValidSignature(deviceKeys);
             });
             const verifiedKeys = verifiedEntries.map(([, deviceKeys]) => deviceKeys);
@@ -258,6 +264,10 @@ export class DeviceTracker {
         if (queriedDevices && queriedDevices.length) {
             flattenedDevices = flattenedDevices.concat(queriedDevices);
         }
-        return flattenedDevices;
+        // filter out our own devices if it got in somehow (even though we should not store it)
+        const devices = flattenedDevices.filter(device => {
+            return !(device.userId === this._ownUserId && device.deviceId === this._ownDeviceId);
+        });
+        return devices;
     }
 }

From 1f8005cdfd28c6955c13389421cc3bb9b6a42281 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 3 Sep 2020 15:28:38 +0200
Subject: [PATCH 075/173] forgot to pass account

---
 src/matrix/e2ee/olm/Decryption.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/matrix/e2ee/olm/Decryption.js b/src/matrix/e2ee/olm/Decryption.js
index c21c4b3d..5f4c7de3 100644
--- a/src/matrix/e2ee/olm/Decryption.js
+++ b/src/matrix/e2ee/olm/Decryption.js
@@ -67,7 +67,7 @@ export class Decryption {
             const payloads = results.reduce((all, r) => all.concat(r.payloads), []);
             const errors = results.reduce((all, r) => all.concat(r.errors), []);
             const senderKeyDecryptions = results.map(r => r.senderKeyDecryption);
-            return new DecryptionChanges(senderKeyDecryptions, payloads, errors, locks);
+            return new DecryptionChanges(senderKeyDecryptions, payloads, errors, this._account, locks);
         } catch (err) {
             // make sure the locks are release if something throws
             for (const lock of locks) {

From b2fffee0370ec2eea762a5295fd87bc9af6d93a5 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 3 Sep 2020 15:28:49 +0200
Subject: [PATCH 076/173] give better error when olm plaintext is not json

---
 src/matrix/e2ee/olm/Decryption.js | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/matrix/e2ee/olm/Decryption.js b/src/matrix/e2ee/olm/Decryption.js
index 5f4c7de3..dfde7674 100644
--- a/src/matrix/e2ee/olm/Decryption.js
+++ b/src/matrix/e2ee/olm/Decryption.js
@@ -111,7 +111,12 @@ export class Decryption {
             plaintext = createResult.plaintext;
         }
         if (typeof plaintext === "string") {
-            const payload = JSON.parse(plaintext);
+            let payload;
+            try {
+                payload = JSON.parse(plaintext);
+            } catch (err) {
+                throw new DecryptionError("Could not JSON decode plaintext", event, {plaintext, err});
+            }
             this._validatePayload(payload, event);
             return {event: payload, senderKey};
         } else {

From 8d0d4570dd18a4235ecd1893fe66cebaed80349c Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 3 Sep 2020 15:29:09 +0200
Subject: [PATCH 077/173] fix import path

---
 src/matrix/e2ee/olm/Encryption.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/matrix/e2ee/olm/Encryption.js b/src/matrix/e2ee/olm/Encryption.js
index 1461b94c..55481bcb 100644
--- a/src/matrix/e2ee/olm/Encryption.js
+++ b/src/matrix/e2ee/olm/Encryption.js
@@ -15,7 +15,7 @@ limitations under the License.
 */
 
 import {groupByWithCreator} from "../../../utils/groupBy.js";
-import {verifyEd25519Signature, OLM_ALGORITHM} from "./common.js";
+import {verifyEd25519Signature, OLM_ALGORITHM} from "../common.js";
 import {createSessionEntry} from "./Session.js";
 
 function findFirstSessionId(sessionIds) {

From 71ba2dd714d9a813eec955e832aec680b277a524 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 3 Sep 2020 15:29:35 +0200
Subject: [PATCH 078/173] name userId -> ownUserId as elsewhere

---
 src/matrix/e2ee/olm/Encryption.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/matrix/e2ee/olm/Encryption.js b/src/matrix/e2ee/olm/Encryption.js
index 55481bcb..f4731b25 100644
--- a/src/matrix/e2ee/olm/Encryption.js
+++ b/src/matrix/e2ee/olm/Encryption.js
@@ -31,11 +31,11 @@ function findFirstSessionId(sessionIds) {
 const OTK_ALGORITHM = "signed_curve25519";
 
 export class Encryption {
-    constructor({account, olm, olmUtil, userId, storage, now, pickleKey, senderKeyLock}) {
+    constructor({account, olm, olmUtil, ownUserId, storage, now, pickleKey, senderKeyLock}) {
         this._account = account;
         this._olm = olm;
         this._olmUtil = olmUtil;
-        this._userId = userId;
+        this._ownUserId = ownUserId;
         this._storage = storage;
         this._now = now;
         this._pickleKey = pickleKey;
@@ -127,7 +127,7 @@ export class Encryption {
                 "ed25519": device.ed25519Key
             },
             recipient: device.userId,
-            sender: this._userId,
+            sender: this._ownUserId,
             content,
             type
         }

From a943467e7174a85abfef3772c39176ffcc598072 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 3 Sep 2020 15:30:01 +0200
Subject: [PATCH 079/173] await txns

---
 src/matrix/e2ee/olm/Encryption.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/matrix/e2ee/olm/Encryption.js b/src/matrix/e2ee/olm/Encryption.js
index f4731b25..e529fa13 100644
--- a/src/matrix/e2ee/olm/Encryption.js
+++ b/src/matrix/e2ee/olm/Encryption.js
@@ -85,7 +85,7 @@ export class Encryption {
     }
 
     async _findExistingSessions(devices) {
-        const txn = this._storage.readTxn([this._storage.storeNames.olmSessions]);
+        const txn = await this._storage.readTxn([this._storage.storeNames.olmSessions]);
         const sessionIdsForDevice = await Promise.all(devices.map(async device => {
             return await txn.olmSessions.getSessionIds(device.curve25519Key);
         }));
@@ -195,7 +195,7 @@ export class Encryption {
     }
 
     async _loadSessions(encryptionTargets) {
-        const txn = this._storage.readTxn([this._storage.storeNames.olmSessions]);
+        const txn = await this._storage.readTxn([this._storage.storeNames.olmSessions]);
         // given we run loading in parallel, there might still be some
         // storage requests that will finish later once one has failed.
         // those should not allocate a session anymore.
@@ -221,7 +221,7 @@ export class Encryption {
     }
 
     async _storeSessions(encryptionTargets, timestamp) {
-        const txn = this._storage.readWriteTxn([this._storage.storeNames.olmSessions]);
+        const txn = await this._storage.readWriteTxn([this._storage.storeNames.olmSessions]);
         try {
             for (const target of encryptionTargets) {
                 const sessionEntry = createSessionEntry(

From 8676909a2665dbff7bcd57e3b7a69ba7b54b5148 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 3 Sep 2020 15:30:09 +0200
Subject: [PATCH 080/173] don't swallow errors!

---
 src/matrix/e2ee/olm/Encryption.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/matrix/e2ee/olm/Encryption.js b/src/matrix/e2ee/olm/Encryption.js
index e529fa13..c8961656 100644
--- a/src/matrix/e2ee/olm/Encryption.js
+++ b/src/matrix/e2ee/olm/Encryption.js
@@ -145,6 +145,7 @@ export class Encryption {
             for (const target of newEncryptionTargets) {
                 target.dispose();
             }
+            throw err;
         }
         return newEncryptionTargets;
     }

From 620fc0d2108f4284d3f4234bc47cc64b410fe3b5 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 3 Sep 2020 15:30:18 +0200
Subject: [PATCH 081/173] JSON stringify payload, olm_encrypt does not do
 objects

---
 src/matrix/e2ee/olm/Encryption.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/matrix/e2ee/olm/Encryption.js b/src/matrix/e2ee/olm/Encryption.js
index c8961656..b5f3449b 100644
--- a/src/matrix/e2ee/olm/Encryption.js
+++ b/src/matrix/e2ee/olm/Encryption.js
@@ -107,7 +107,8 @@ export class Encryption {
 
     _encryptForDevice(type, content, target) {
         const {session, device} = target;
-        const message = session.encrypt(this._buildPlainTextMessageForDevice(type, content, device));
+        const plaintext = JSON.stringify(this._buildPlainTextMessageForDevice(type, content, device));
+        const message = session.encrypt(plaintext);
         const encryptedContent = {
             algorithm: OLM_ALGORITHM,
             sender_key: this._account.identityKeys.curve25519,

From 408ff3322dadf3f99f2b5ab3c8711062d5a24d1c Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 3 Sep 2020 15:30:42 +0200
Subject: [PATCH 082/173] content already exists here

---
 src/matrix/e2ee/olm/Encryption.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/matrix/e2ee/olm/Encryption.js b/src/matrix/e2ee/olm/Encryption.js
index b5f3449b..680ce154 100644
--- a/src/matrix/e2ee/olm/Encryption.js
+++ b/src/matrix/e2ee/olm/Encryption.js
@@ -67,8 +67,8 @@ export class Encryption {
                 await this._loadSessions(existingEncryptionTargets);
                 encryptionTargets = encryptionTargets.concat(existingEncryptionTargets);
                 const messages = encryptionTargets.map(target => {
-                    const content = this._encryptForDevice(type, content, target);
-                    return new EncryptedMessage(content, target.device);
+                    const encryptedContent = this._encryptForDevice(type, content, target);
+                    return new EncryptedMessage(encryptedContent, target.device);
                 });
                 await this._storeSessions(encryptionTargets, timestamp);
                 return messages;

From 4401012312260c0758906aa739d4de6cec3229cc Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 3 Sep 2020 15:30:54 +0200
Subject: [PATCH 083/173] no need to call decrypt when there are no events

---
 src/matrix/DeviceMessageHandler.js | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js
index 9c81f11e..537b948d 100644
--- a/src/matrix/DeviceMessageHandler.js
+++ b/src/matrix/DeviceMessageHandler.js
@@ -64,6 +64,9 @@ export class DeviceMessageHandler {
         }
         const readTxn = await this._storage.readTxn([this._storage.storeNames.session]);
         const pendingEvents = await this._getPendingEvents(readTxn);
+        if (pendingEvents.length === 0) {
+           return;
+        }
         // only know olm for now
         const olmEvents = pendingEvents.filter(e => e.content?.algorithm === OLM_ALGORITHM);
         const decryptChanges = await this._olmDecryption.decryptAll(olmEvents);

From 1d4a5cd6d4f52a2547d6abeaf1cbf10a63636729 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 3 Sep 2020 15:32:08 +0200
Subject: [PATCH 084/173] instantiate olm encryption in session

---
 src/matrix/Session.js | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index 6929430f..1e3f8382 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -21,6 +21,7 @@ import {User} from "./User.js";
 import {Account as E2EEAccount} from "./e2ee/Account.js";
 import {DeviceMessageHandler} from "./DeviceMessageHandler.js";
 import {Decryption as OlmDecryption} from "./e2ee/olm/Decryption.js";
+import {Encryption as OlmEncryption} from "./e2ee/olm/Encryption.js";
 import {Decryption as MegOlmDecryption} from "./e2ee/megolm/Decryption.js";
 import {DeviceTracker} from "./e2ee/DeviceTracker.js";
 import {LockMap} from "../utils/LockMap.js";
@@ -44,6 +45,7 @@ export class Session {
         this._olmUtil = null;
         this._e2eeAccount = null;
         this._deviceTracker = null;
+        this._olmEncryption = null;
         if (olm) {
             this._olmUtil = new olm.Utility();
             this._deviceTracker = new DeviceTracker({
@@ -68,6 +70,16 @@ export class Session {
             olm: this._olm,
             senderKeyLock
         });
+        this._olmEncryption = new OlmEncryption({
+            account: this._e2eeAccount,
+            pickleKey: PICKLE_KEY,
+            now: this._clock.now,
+            ownUserId: this._user.id,
+            storage: this._storage,
+            olm: this._olm,
+            olmUtil: this._olmUtil,
+            senderKeyLock
+        });
         const megolmDecryption = new MegOlmDecryption({pickleKey: PICKLE_KEY, olm: this._olm});
         this._deviceMessageHandler.enableEncryption({olmDecryption, megolmDecryption});
     }

From 792f0cf9a0ce2cc993582869feeb9d6f526ce433 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 3 Sep 2020 15:32:33 +0200
Subject: [PATCH 085/173] log our identity keys after load

---
 src/matrix/Session.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index 1e3f8382..652ac18e 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -60,6 +60,7 @@ export class Session {
 
     // called once this._e2eeAccount is assigned
     _setupEncryption() {
+        console.log("loaded e2ee account with keys", this._e2eeAccount.identityKeys);
         const senderKeyLock = new LockMap();
         const olmDecryption = new OlmDecryption({
             account: this._e2eeAccount,

From dde8c6619691d5e9feddcfcb043d550911c5455f Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 3 Sep 2020 15:33:03 +0200
Subject: [PATCH 086/173] implement store changes for olm encryption

---
 .../storage/idb/stores/OlmSessionStore.js     | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/src/matrix/storage/idb/stores/OlmSessionStore.js b/src/matrix/storage/idb/stores/OlmSessionStore.js
index c94b3bfd..4648f09c 100644
--- a/src/matrix/storage/idb/stores/OlmSessionStore.js
+++ b/src/matrix/storage/idb/stores/OlmSessionStore.js
@@ -18,11 +18,31 @@ function encodeKey(senderKey, sessionId) {
     return `${senderKey}|${sessionId}`;
 }
 
+function decodeKey(key) {
+    const [senderKey, sessionId] = key.split("|");
+    return {senderKey, sessionId};
+}
+
 export class OlmSessionStore {
     constructor(store) {
         this._store = store;
     }
 
+    async getSessionIds(senderKey) {
+        const sessionIds = [];
+        const range = IDBKeyRange.lowerBound(encodeKey(senderKey, ""));
+        await this._store.iterateKeys(range, key => {
+            const decodedKey = decodeKey(key);
+            // prevent running into the next room
+            if (decodedKey.senderKey === senderKey) {
+                sessionIds.push(decodedKey.sessionId);
+                return false;   // fetch more
+            }
+            return true; // done
+        });
+        return sessionIds;
+    }
+
     getAll(senderKey) {
         const range = IDBKeyRange.lowerBound(encodeKey(senderKey, ""));
         return this._store.selectWhile(range, session => {

From 2a40c89a24330010eb54901d08942f8f897e7a39 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 3 Sep 2020 15:33:23 +0200
Subject: [PATCH 087/173] implement hsapi /keys/claim endpoint

---
 src/matrix/net/HomeServerApi.js | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/matrix/net/HomeServerApi.js b/src/matrix/net/HomeServerApi.js
index 42d1b0e0..9ea7dc26 100644
--- a/src/matrix/net/HomeServerApi.js
+++ b/src/matrix/net/HomeServerApi.js
@@ -168,6 +168,10 @@ export class HomeServerApi {
         return this._post("/keys/query", null, queryRequest, options);
     }
 
+    claimKeys(payload, options = null) {
+        return this._post("/keys/claim", null, payload, options);
+    }
+
     get mediaRepository() {
         return this._mediaRepository;
     }

From 5cafef96f53806c20503276028c43dda8747bf80 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 3 Sep 2020 15:36:17 +0200
Subject: [PATCH 088/173] add RoomEncryption to room

---
 src/matrix/Session.js                | 29 +++++++++++---
 src/matrix/common.js                 | 22 ++++++++++
 src/matrix/e2ee/RoomEncryption.js    | 60 ++++++++++++++++++++++++++++
 src/matrix/net/HomeServerApi.js      |  4 ++
 src/matrix/room/Room.js              | 15 ++++++-
 src/matrix/room/sending/SendQueue.js |  7 +---
 6 files changed, 124 insertions(+), 13 deletions(-)
 create mode 100644 src/matrix/common.js
 create mode 100644 src/matrix/e2ee/RoomEncryption.js

diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index 652ac18e..c1a26ded 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -23,6 +23,7 @@ import {DeviceMessageHandler} from "./DeviceMessageHandler.js";
 import {Decryption as OlmDecryption} from "./e2ee/olm/Decryption.js";
 import {Encryption as OlmEncryption} from "./e2ee/olm/Encryption.js";
 import {Decryption as MegOlmDecryption} from "./e2ee/megolm/Decryption.js";
+import {RoomEncryption} from "./e2ee/RoomEncryption.js";
 import {DeviceTracker} from "./e2ee/DeviceTracker.js";
 import {LockMap} from "../utils/LockMap.js";
 
@@ -56,6 +57,7 @@ export class Session {
                 ownDeviceId: sessionInfo.deviceId,
             });
         }
+        this._createRoomEncryption = this._createRoomEncryption.bind(this);
     }
 
     // called once this._e2eeAccount is assigned
@@ -85,6 +87,26 @@ export class Session {
         this._deviceMessageHandler.enableEncryption({olmDecryption, megolmDecryption});
     }
 
+    _createRoomEncryption(room, encryptionEventContent) {
+        // TODO: this will actually happen when users start using the e2ee version for the first time
+
+        // this should never happen because either a session was already synced once
+        // and thus an e2ee account was created as well and _setupEncryption is called from load
+        // OR
+        // this is a new session and loading it will load zero rooms, thus not calling this method.
+        // in this case _setupEncryption is called from beforeFirstSync, right after load,
+        // so any incoming synced rooms won't be there yet
+        if (!this._olmEncryption) {
+            throw new Error("creating room encryption before encryption got globally enabled");
+        }
+        return new RoomEncryption({
+            room,
+            deviceTracker: this._deviceTracker,
+            olmEncryption: this._olmEncryption,
+            encryptionEventContent
+        });
+    }
+
     // called after load
     async beforeFirstSync(isNewLogin) {
         if (this._olm) {
@@ -202,6 +224,7 @@ export class Session {
             sendScheduler: this._sendScheduler,
             pendingEvents,
             user: this._user,
+            createRoomEncryption: this._createRoomEncryption
         });
         this._rooms.add(roomId, room);
         return room;
@@ -222,12 +245,6 @@ export class Session {
             changes.syncInfo = syncInfo;
         }
         if (this._deviceTracker) {
-            for (const {room, changes} of roomChanges) {
-                // TODO: move this so the room passes this to it's "encryption" object in its own writeSync method?
-                if (room.isTrackingMembers && changes.memberChanges?.size) {
-                    await this._deviceTracker.writeMemberChanges(room, changes.memberChanges, txn);
-                }
-            } 
             const deviceLists = syncResponse.device_lists;
             if (deviceLists) {
                 await this._deviceTracker.writeDeviceChanges(deviceLists, txn);
diff --git a/src/matrix/common.js b/src/matrix/common.js
new file mode 100644
index 00000000..3c893234
--- /dev/null
+++ b/src/matrix/common.js
@@ -0,0 +1,22 @@
+/*
+Copyright 2020 Bruno Windels 
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export function makeTxnId() {
+    const n = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER);
+    const str = n.toString(16);
+    return "t" + "0".repeat(14 - str.length) + str;
+}
\ No newline at end of file
diff --git a/src/matrix/e2ee/RoomEncryption.js b/src/matrix/e2ee/RoomEncryption.js
new file mode 100644
index 00000000..32bd061c
--- /dev/null
+++ b/src/matrix/e2ee/RoomEncryption.js
@@ -0,0 +1,60 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {groupBy} from "../../utils/groupBy.js";
+import {makeTxnId} from "../common.js";
+
+
+export class RoomEncryption {
+    constructor({room, deviceTracker, olmEncryption, encryptionEventContent}) {
+        this._room = room;
+        this._deviceTracker = deviceTracker;
+        this._olmEncryption = olmEncryption;
+        // content of the m.room.encryption event
+        this._encryptionEventContent = encryptionEventContent;
+    }
+
+    async writeMemberChanges(memberChanges, txn) {
+        return await this._deviceTracker.writeMemberChanges(this._room, memberChanges, txn);
+    }
+
+    async encrypt(type, content, hsApi) {
+        await this._deviceTracker.trackRoom(this._room);
+        const devices = await this._deviceTracker.deviceIdentitiesForTrackedRoom(this._room.id, hsApi);
+        const messages = await this._olmEncryption.encrypt("m.foo", {body: "hello at " + new Date()}, devices, hsApi);
+        await this._sendMessagesToDevices("m.room.encrypted", messages, hsApi);
+        return {type, content};
+        // return {
+        //     type: "m.room.encrypted",
+        //     content: encryptedContent,
+        // }
+    }
+
+    async _sendMessagesToDevices(type, messages, hsApi) {
+        const messagesByUser = groupBy(messages, message => message.device.userId);
+        const payload = {
+            messages: Array.from(messagesByUser.entries()).reduce((userMap, [userId, messages]) => {
+                userMap[userId] = messages.reduce((deviceMap, message) => {
+                    deviceMap[message.device.deviceId] = message.content;
+                    return deviceMap;
+                }, {});
+                return userMap;
+            }, {})
+        };
+        const txnId = makeTxnId();
+        await hsApi.sendToDevice(type, payload, txnId).response();
+    }
+}
diff --git a/src/matrix/net/HomeServerApi.js b/src/matrix/net/HomeServerApi.js
index 9ea7dc26..b1a89634 100644
--- a/src/matrix/net/HomeServerApi.js
+++ b/src/matrix/net/HomeServerApi.js
@@ -172,6 +172,10 @@ export class HomeServerApi {
         return this._post("/keys/claim", null, payload, options);
     }
 
+    sendToDevice(type, payload, txnId, options = null) {
+        return this._put(`/sendToDevice/${encodeURIComponent(type)}/${encodeURIComponent(txnId)}`, null, payload, options);
+    }
+
     get mediaRepository() {
         return this._mediaRepository;
     }
diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index 98cde3a5..dbf1e5e3 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -27,7 +27,7 @@ import {MemberList} from "./members/MemberList.js";
 import {Heroes} from "./members/Heroes.js";
 
 export class Room extends EventEmitter {
-	constructor({roomId, storage, hsApi, emitCollectionChange, sendScheduler, pendingEvents, user}) {
+	constructor({roomId, storage, hsApi, emitCollectionChange, sendScheduler, pendingEvents, user, createRoomEncryption}) {
         super();
         this._roomId = roomId;
         this._storage = storage;
@@ -41,6 +41,8 @@ export class Room extends EventEmitter {
         this._user = user;
         this._changedMembersDuringSync = null;
         this._memberList = null;
+        this._createRoomEncryption = createRoomEncryption;
+        this._roomEncryption = null;
 	}
 
     /** @package */
@@ -62,6 +64,10 @@ export class Room extends EventEmitter {
             }
             heroChanges = await this._heroes.calculateChanges(summaryChanges.heroes, memberChanges, txn);
         }
+        // pass member changes to device tracker
+        if (this._roomEncryption && this.isTrackingMembers && memberChanges?.size) {
+            await this._roomEncryption.writeMemberChanges(memberChanges, txn);
+        }
         let removedPendingEvents;
         if (roomResponse.timeline && roomResponse.timeline.events) {
             removedPendingEvents = this._sendQueue.removeRemoteEchos(roomResponse.timeline.events, txn);
@@ -79,6 +85,10 @@ export class Room extends EventEmitter {
     /** @package */
     afterSync({summaryChanges, newTimelineEntries, newLiveKey, removedPendingEvents, memberChanges, heroChanges}) {
         this._syncWriter.afterSync(newLiveKey);
+        // encryption got enabled
+        if (!this._summary.encryption && summaryChanges.encryption && !this._roomEncryption) {
+            this._roomEncryption = this._createRoomEncryption(this, summaryChanges.encryption);
+        }
         if (memberChanges.size) {
             if (this._changedMembersDuringSync) {
                 for (const [userId, memberChange] of memberChanges.entries()) {
@@ -125,6 +135,9 @@ export class Room extends EventEmitter {
 	async load(summary, txn) {
         try {
             this._summary.load(summary);
+            if (this._summary.encryption) {
+                this._roomEncryption = this._createRoomEncryption(this, this._summary.encryption);
+            }
             // need to load members for name?
             if (this._summary.needsHeroes) {
                 this._heroes = new Heroes(this._roomId);
diff --git a/src/matrix/room/sending/SendQueue.js b/src/matrix/room/sending/SendQueue.js
index ba215e04..9a094798 100644
--- a/src/matrix/room/sending/SendQueue.js
+++ b/src/matrix/room/sending/SendQueue.js
@@ -17,12 +17,7 @@ limitations under the License.
 import {SortedArray} from "../../../observable/list/SortedArray.js";
 import {ConnectionError} from "../../error.js";
 import {PendingEvent} from "./PendingEvent.js";
-
-function makeTxnId() {
-    const n = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER);
-    const str = n.toString(16);
-    return "t" + "0".repeat(14 - str.length) + str;
-}
+import {makeTxnId} from "../../common.js";
 
 export class SendQueue {
     constructor({roomId, storage, sendScheduler, pendingEvents}) {

From b1226d9220ecc0f53d7b27fe28dac993b666a0ab Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 3 Sep 2020 15:36:48 +0200
Subject: [PATCH 089/173] add infrastructure to encrypt while sending

---
 src/matrix/SendScheduler.js             |  2 +-
 src/matrix/room/Room.js                 |  2 ++
 src/matrix/room/sending/PendingEvent.js |  7 +++++++
 src/matrix/room/sending/SendQueue.js    | 15 ++++++++++++++-
 4 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/src/matrix/SendScheduler.js b/src/matrix/SendScheduler.js
index 6e6196d3..e7627c81 100644
--- a/src/matrix/SendScheduler.js
+++ b/src/matrix/SendScheduler.js
@@ -121,7 +121,7 @@ export class SendScheduler {
                     }
                     this._sendRequests = [];
                 }
-                console.error("error for request", request);
+                console.error("error for request", err);
                 request.reject(err);
                 break;
             }
diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index dbf1e5e3..e8315538 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -88,6 +88,7 @@ export class Room extends EventEmitter {
         // encryption got enabled
         if (!this._summary.encryption && summaryChanges.encryption && !this._roomEncryption) {
             this._roomEncryption = this._createRoomEncryption(this, summaryChanges.encryption);
+            this._sendQueue.enableEncryption(this._roomEncryption);
         }
         if (memberChanges.size) {
             if (this._changedMembersDuringSync) {
@@ -137,6 +138,7 @@ export class Room extends EventEmitter {
             this._summary.load(summary);
             if (this._summary.encryption) {
                 this._roomEncryption = this._createRoomEncryption(this, this._summary.encryption);
+                this._sendQueue.enableEncryption(this._roomEncryption);
             }
             // need to load members for name?
             if (this._summary.needsHeroes) {
diff --git a/src/matrix/room/sending/PendingEvent.js b/src/matrix/room/sending/PendingEvent.js
index 2b4f7477..fb2d1a47 100644
--- a/src/matrix/room/sending/PendingEvent.js
+++ b/src/matrix/room/sending/PendingEvent.js
@@ -26,5 +26,12 @@ export class PendingEvent {
     get remoteId() { return this._data.remoteId; }
     set remoteId(value) { this._data.remoteId = value; }
     get content() { return this._data.content; }
+    get needsEncryption() { return this._data.needsEncryption; }
     get data() { return this._data; }
+
+    setEncrypted(type, content) {
+        this._data.eventType = type;
+        this._data.content = content;
+        this._data.needsEncryption = false;
+    }
 }
diff --git a/src/matrix/room/sending/SendQueue.js b/src/matrix/room/sending/SendQueue.js
index 9a094798..fe7afe77 100644
--- a/src/matrix/room/sending/SendQueue.js
+++ b/src/matrix/room/sending/SendQueue.js
@@ -33,6 +33,11 @@ export class SendQueue {
         this._isSending = false;
         this._offline = false;
         this._amountSent = 0;
+        this._roomEncryption = null;
+    }
+
+    enableEncryption(roomEncryption) {
+        this._roomEncryption = roomEncryption;
     }
 
     async _sendLoop() {
@@ -45,6 +50,13 @@ export class SendQueue {
                 if (pendingEvent.remoteId) {
                     continue;
                 }
+                if (pendingEvent.needsEncryption) {
+                    const {type, content} = await this._sendScheduler.request(async hsApi => {
+                        return await this._roomEncryption.encrypt(pendingEvent.eventType, pendingEvent.content, hsApi);
+                    });
+                    pendingEvent.setEncrypted(type, content);
+                    await this._tryUpdateEvent(pendingEvent);
+                }
                 console.log("really sending now");
                 const response = await this._sendScheduler.request(hsApi => {
                     console.log("got sendScheduler slot");
@@ -156,7 +168,8 @@ export class SendQueue {
                 queueIndex,
                 eventType,
                 content,
-                txnId: makeTxnId()
+                txnId: makeTxnId(),
+                needsEncryption: !!this._roomEncryption
             });
             console.log("_createAndStoreEvent: adding to pendingEventsStore");
             pendingEventsStore.add(pendingEvent.data);

From 6bc30bb8247f0184cfb661be7d5eda4409636914 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 3 Sep 2020 17:48:59 +0200
Subject: [PATCH 090/173] implement megolm encryption

---
 src/matrix/e2ee/megolm/Encryption.js | 147 +++++++++++++++++++++++++++
 1 file changed, 147 insertions(+)
 create mode 100644 src/matrix/e2ee/megolm/Encryption.js

diff --git a/src/matrix/e2ee/megolm/Encryption.js b/src/matrix/e2ee/megolm/Encryption.js
new file mode 100644
index 00000000..0b374f48
--- /dev/null
+++ b/src/matrix/e2ee/megolm/Encryption.js
@@ -0,0 +1,147 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {MEGOLM_ALGORITHM} from "../common.js";
+
+export class Encryption {
+    constructor({pickleKey, olm, account, storage, now, ownDeviceId}) {
+        this._pickleKey = pickleKey;
+        this._olm = olm;
+        this._account = account;
+        this._storage = storage;
+        this._now = now;
+        this._ownDeviceId = ownDeviceId;
+    }
+
+    async encrypt(roomId, type, content, encryptionParams) {
+        let session = new this._olm.OutboundGroupSession();
+        try {
+            const txn = await this._storage.readWriteTxn([
+                this._storage.storeNames.inboundGroupSessions,
+                this._storage.storeNames.outboundGroupSessions,
+            ]);
+            let roomKeyMessage;
+            let encryptedContent;
+            try {
+                let sessionEntry = await txn.outboundGroupSessions.get(roomId);
+                if (sessionEntry) {
+                    session.unpickle(this._pickleKey, sessionEntry.session);
+                }
+                if (!sessionEntry || this._needsToRotate(session, sessionEntry.createdAt, encryptionParams)) {
+                    // in the case of rotating, recreate a session as we already unpickled into it
+                    if (session) {
+                        session.free();
+                        session = new this._olm.OutboundGroupSession();
+                    }
+                    session.create();
+                    roomKeyMessage = this._createRoomKeyMessage(session, roomId);
+                    this._storeAsInboundSession(session, roomId, txn);
+                    // TODO: we could tell the Decryption here that we have a new session so it can add it to its cache
+                }
+                encryptedContent = this._encryptContent(roomId, session, type, content);
+                txn.outboundGroupSessions.set({
+                    roomId,
+                    session: session.pickle(this._pickleKey),
+                    createdAt: sessionEntry?.createdAt || this._now(),
+                });
+
+            } catch (err) {
+                txn.abort();
+                throw err;
+            }
+            await txn.complete();
+            return new EncryptionResult(encryptedContent, roomKeyMessage);
+        } finally {
+            if (session) {
+                session.free();
+            }
+        }
+    }
+
+    _needsToRotate(session, createdAt, encryptionParams) {
+        let rotationPeriodMs = 604800000; // default
+        if (Number.isSafeInteger(encryptionParams?.rotation_period_ms)) {
+            rotationPeriodMs = encryptionParams?.rotation_period_ms;
+        }
+        let rotationPeriodMsgs = 100; // default
+        if (Number.isSafeInteger(encryptionParams?.rotation_period_msgs)) {
+            rotationPeriodMsgs = encryptionParams?.rotation_period_msgs;
+        }
+
+        if (this._now() > (createdAt + rotationPeriodMs)) {
+            return true;
+        }
+        if (session.message_index() >= rotationPeriodMsgs) {
+            return true;
+        }  
+    }
+
+    _encryptContent(roomId, session, type, content) {
+        const plaintext = JSON.stringify({
+            room_id: roomId,
+            type,
+            content
+        });
+        const ciphertext = session.encrypt(plaintext);
+
+        const encryptedContent = {
+            algorithm: MEGOLM_ALGORITHM,
+            sender_key: this._account.identityKeys.curve25519,
+            ciphertext,
+            session_id: session.session_id(),
+            device_id: this._ownDeviceId
+        };
+
+        return encryptedContent;
+    }
+
+    _createRoomKeyMessage(session, roomId) {
+        return {
+            room_id: roomId,
+            session_id: session.session_id(),
+            session_key: session.session_key(),
+            algorithm: MEGOLM_ALGORITHM,
+            // chain_index: session.message_index()
+        }
+    }
+
+    _storeAsInboundSession(outboundSession, roomId, txn) {
+        const {identityKeys} = this._account;
+        const claimedKeys = {ed25519: identityKeys.ed25519};
+        const session = new this._olm.InboundGroupSession();
+        try {
+            session.create(outboundSession.session_key());
+            const sessionEntry = {
+                roomId,
+                senderKey: identityKeys.curve25519,
+                sessionId: session.session_id(),
+                session: session.pickle(this._pickleKey),
+                claimedKeys,
+            };
+            txn.inboundGroupSessions.set(sessionEntry);
+            return sessionEntry;
+        } finally {
+            session.free();
+        }
+    }
+}
+
+class EncryptionResult {
+    constructor(content, roomKeyMessage) {
+        this.content = content;
+        this.roomKeyMessage = roomKeyMessage;
+    }
+}

From be4d8871782e46ce7d1df30af3e0fff6dee890f8 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 3 Sep 2020 17:49:20 +0200
Subject: [PATCH 091/173] add outbound group session storage

---
 src/matrix/storage/common.js                  |  1 +
 src/matrix/storage/idb/Transaction.js         |  6 +++-
 src/matrix/storage/idb/schema.js              |  7 +++++
 .../idb/stores/OutboundGroupSessionStore.js   | 29 +++++++++++++++++++
 4 files changed, 42 insertions(+), 1 deletion(-)
 create mode 100644 src/matrix/storage/idb/stores/OutboundGroupSessionStore.js

diff --git a/src/matrix/storage/common.js b/src/matrix/storage/common.js
index 76a60e66..473b8eb6 100644
--- a/src/matrix/storage/common.js
+++ b/src/matrix/storage/common.js
@@ -26,6 +26,7 @@ export const STORE_NAMES = Object.freeze([
     "deviceIdentities",
     "olmSessions",
     "inboundGroupSessions",
+    "outboundGroupSessions",
 ]);
 
 export const STORE_MAP = Object.freeze(STORE_NAMES.reduce((nameMap, name) => {
diff --git a/src/matrix/storage/idb/Transaction.js b/src/matrix/storage/idb/Transaction.js
index fa862c08..8b42b0f7 100644
--- a/src/matrix/storage/idb/Transaction.js
+++ b/src/matrix/storage/idb/Transaction.js
@@ -28,6 +28,7 @@ import {UserIdentityStore} from "./stores/UserIdentityStore.js";
 import {DeviceIdentityStore} from "./stores/DeviceIdentityStore.js";
 import {OlmSessionStore} from "./stores/OlmSessionStore.js";
 import {InboundGroupSessionStore} from "./stores/InboundGroupSessionStore.js";
+import {OutboundGroupSessionStore} from "./stores/OutboundGroupSessionStore.js";
 
 export class Transaction {
     constructor(txn, allowedStoreNames) {
@@ -100,7 +101,10 @@ export class Transaction {
     get inboundGroupSessions() {
         return this._store("inboundGroupSessions", idbStore => new InboundGroupSessionStore(idbStore));
     }
-
+    
+    get outboundGroupSessions() {
+        return this._store("outboundGroupSessions", idbStore => new OutboundGroupSessionStore(idbStore));
+    }
     complete() {
         return txnAsPromise(this._txn);
     }
diff --git a/src/matrix/storage/idb/schema.js b/src/matrix/storage/idb/schema.js
index 81a56991..2060cb39 100644
--- a/src/matrix/storage/idb/schema.js
+++ b/src/matrix/storage/idb/schema.js
@@ -12,6 +12,7 @@ export const schema = [
     createIdentityStores,
     createOlmSessionStore,
     createInboundGroupSessionsStore,
+    createOutboundGroupSessionsStore,
 ];
 // TODO: how to deal with git merge conflicts of this array?
 
@@ -82,3 +83,9 @@ function createOlmSessionStore(db) {
 function createInboundGroupSessionsStore(db) {
     db.createObjectStore("inboundGroupSessions", {keyPath: "key"});
 }
+
+//v7
+function createOutboundGroupSessionsStore(db) {
+    db.createObjectStore("outboundGroupSessions", {keyPath: "roomId"});
+}
+
diff --git a/src/matrix/storage/idb/stores/OutboundGroupSessionStore.js b/src/matrix/storage/idb/stores/OutboundGroupSessionStore.js
new file mode 100644
index 00000000..ef9224be
--- /dev/null
+++ b/src/matrix/storage/idb/stores/OutboundGroupSessionStore.js
@@ -0,0 +1,29 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export class OutboundGroupSessionStore {
+    constructor(store) {
+        this._store = store;
+    }
+
+    get(roomId) {
+        return this._store.get(roomId);
+    }
+
+    set(session) {
+        this._store.put(session);
+    }
+}

From c5c9505ce2474ca9b1cb897438aaf801a69a9d14 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 3 Sep 2020 17:50:28 +0200
Subject: [PATCH 092/173] hookup megolm encryption in session

---
 src/matrix/Session.js             | 14 ++++++++++++--
 src/matrix/e2ee/RoomEncryption.js | 28 +++++++++++++++++-----------
 2 files changed, 29 insertions(+), 13 deletions(-)

diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index c1a26ded..e3f82bda 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -23,6 +23,7 @@ import {DeviceMessageHandler} from "./DeviceMessageHandler.js";
 import {Decryption as OlmDecryption} from "./e2ee/olm/Decryption.js";
 import {Encryption as OlmEncryption} from "./e2ee/olm/Encryption.js";
 import {Decryption as MegOlmDecryption} from "./e2ee/megolm/Decryption.js";
+import {Encryption as MegOlmEncryption} from "./e2ee/megolm/Encryption.js";
 import {RoomEncryption} from "./e2ee/RoomEncryption.js";
 import {DeviceTracker} from "./e2ee/DeviceTracker.js";
 import {LockMap} from "../utils/LockMap.js";
@@ -83,11 +84,19 @@ export class Session {
             olmUtil: this._olmUtil,
             senderKeyLock
         });
+        this._megolmEncryption = new MegOlmEncryption({
+            account: this._e2eeAccount,
+            pickleKey: PICKLE_KEY,
+            olm: this._olm,
+            storage: this._storage,
+            now: this._clock.now,
+            ownDeviceId: this._sessionInfo.deviceId,
+        })
         const megolmDecryption = new MegOlmDecryption({pickleKey: PICKLE_KEY, olm: this._olm});
         this._deviceMessageHandler.enableEncryption({olmDecryption, megolmDecryption});
     }
 
-    _createRoomEncryption(room, encryptionEventContent) {
+    _createRoomEncryption(room, encryptionParams) {
         // TODO: this will actually happen when users start using the e2ee version for the first time
 
         // this should never happen because either a session was already synced once
@@ -103,7 +112,8 @@ export class Session {
             room,
             deviceTracker: this._deviceTracker,
             olmEncryption: this._olmEncryption,
-            encryptionEventContent
+            megolmEncryption: this._megolmEncryption,
+            encryptionParams
         });
     }
 
diff --git a/src/matrix/e2ee/RoomEncryption.js b/src/matrix/e2ee/RoomEncryption.js
index 32bd061c..5f0c4cc1 100644
--- a/src/matrix/e2ee/RoomEncryption.js
+++ b/src/matrix/e2ee/RoomEncryption.js
@@ -17,14 +17,16 @@ limitations under the License.
 import {groupBy} from "../../utils/groupBy.js";
 import {makeTxnId} from "../common.js";
 
+const ENCRYPTED_TYPE = "m.room.encrypted";
 
 export class RoomEncryption {
-    constructor({room, deviceTracker, olmEncryption, encryptionEventContent}) {
+    constructor({room, deviceTracker, olmEncryption, megolmEncryption, encryptionParams}) {
         this._room = room;
         this._deviceTracker = deviceTracker;
         this._olmEncryption = olmEncryption;
+        this._megolmEncryption = megolmEncryption;
         // content of the m.room.encryption event
-        this._encryptionEventContent = encryptionEventContent;
+        this._encryptionParams = encryptionParams;
     }
 
     async writeMemberChanges(memberChanges, txn) {
@@ -32,15 +34,19 @@ export class RoomEncryption {
     }
 
     async encrypt(type, content, hsApi) {
-        await this._deviceTracker.trackRoom(this._room);
-        const devices = await this._deviceTracker.deviceIdentitiesForTrackedRoom(this._room.id, hsApi);
-        const messages = await this._olmEncryption.encrypt("m.foo", {body: "hello at " + new Date()}, devices, hsApi);
-        await this._sendMessagesToDevices("m.room.encrypted", messages, hsApi);
-        return {type, content};
-        // return {
-        //     type: "m.room.encrypted",
-        //     content: encryptedContent,
-        // }
+        const megolmResult = await this._megolmEncryption.encrypt(this._room.id, type, content, this._encryptionParams);
+        // share the new megolm session if needed
+        if (megolmResult.roomKeyMessage) {
+            await this._deviceTracker.trackRoom(this._room);
+            const devices = await this._deviceTracker.deviceIdentitiesForTrackedRoom(this._room.id, hsApi);
+            const messages = await this._olmEncryption.encrypt(
+                "m.room_key", megolmResult.roomKeyMessage, devices, hsApi);
+            await this._sendMessagesToDevices(ENCRYPTED_TYPE, messages, hsApi);
+        }
+        return {
+            type: ENCRYPTED_TYPE,
+            content: megolmResult.content
+        };
     }
 
     async _sendMessagesToDevices(type, messages, hsApi) {

From c5efa582b1b018880b8e4a17d27c77c584ddcbde Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 3 Sep 2020 17:51:00 +0200
Subject: [PATCH 093/173] check algorithm

---
 src/matrix/Session.js   | 5 +++++
 src/matrix/room/Room.js | 8 ++++++--
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index e3f82bda..68afcf5b 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -24,6 +24,7 @@ import {Decryption as OlmDecryption} from "./e2ee/olm/Decryption.js";
 import {Encryption as OlmEncryption} from "./e2ee/olm/Encryption.js";
 import {Decryption as MegOlmDecryption} from "./e2ee/megolm/Decryption.js";
 import {Encryption as MegOlmEncryption} from "./e2ee/megolm/Encryption.js";
+import {MEGOLM_ALGORITHM} from "./e2ee/common.js";
 import {RoomEncryption} from "./e2ee/RoomEncryption.js";
 import {DeviceTracker} from "./e2ee/DeviceTracker.js";
 import {LockMap} from "../utils/LockMap.js";
@@ -108,6 +109,10 @@ export class Session {
         if (!this._olmEncryption) {
             throw new Error("creating room encryption before encryption got globally enabled");
         }
+        // only support megolm
+        if (encryptionParams.algorithm !== MEGOLM_ALGORITHM) {
+            return null;
+        }
         return new RoomEncryption({
             room,
             deviceTracker: this._deviceTracker,
diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index e8315538..d2272b3d 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -88,7 +88,9 @@ export class Room extends EventEmitter {
         // encryption got enabled
         if (!this._summary.encryption && summaryChanges.encryption && !this._roomEncryption) {
             this._roomEncryption = this._createRoomEncryption(this, summaryChanges.encryption);
-            this._sendQueue.enableEncryption(this._roomEncryption);
+            if (this._roomEncryption) {
+                this._sendQueue.enableEncryption(this._roomEncryption);
+            }
         }
         if (memberChanges.size) {
             if (this._changedMembersDuringSync) {
@@ -138,7 +140,9 @@ export class Room extends EventEmitter {
             this._summary.load(summary);
             if (this._summary.encryption) {
                 this._roomEncryption = this._createRoomEncryption(this, this._summary.encryption);
-                this._sendQueue.enableEncryption(this._roomEncryption);
+                if (this._roomEncryption) {
+                    this._sendQueue.enableEncryption(this._roomEncryption);
+                }
             }
             // need to load members for name?
             if (this._summary.needsHeroes) {

From 8ac80314c21c2c0ae8002536abbc340d9dc6120c Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 3 Sep 2020 17:51:11 +0200
Subject: [PATCH 094/173] cleanup

---
 src/matrix/Session.js                | 10 +++++-----
 src/matrix/e2ee/megolm/Decryption.js |  1 -
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index 68afcf5b..1d0ac73e 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -18,8 +18,8 @@ import {Room} from "./room/Room.js";
 import { ObservableMap } from "../observable/index.js";
 import { SendScheduler, RateLimitingBackoff } from "./SendScheduler.js";
 import {User} from "./User.js";
-import {Account as E2EEAccount} from "./e2ee/Account.js";
 import {DeviceMessageHandler} from "./DeviceMessageHandler.js";
+import {Account as E2EEAccount} from "./e2ee/Account.js";
 import {Decryption as OlmDecryption} from "./e2ee/olm/Decryption.js";
 import {Encryption as OlmEncryption} from "./e2ee/olm/Encryption.js";
 import {Decryption as MegOlmDecryption} from "./e2ee/megolm/Decryption.js";
@@ -69,19 +69,19 @@ export class Session {
         const olmDecryption = new OlmDecryption({
             account: this._e2eeAccount,
             pickleKey: PICKLE_KEY,
+            olm: this._olm,
+            storage: this._storage,
             now: this._clock.now,
             ownUserId: this._user.id,
-            storage: this._storage,
-            olm: this._olm,
             senderKeyLock
         });
         this._olmEncryption = new OlmEncryption({
             account: this._e2eeAccount,
             pickleKey: PICKLE_KEY,
+            olm: this._olm,
+            storage: this._storage,
             now: this._clock.now,
             ownUserId: this._user.id,
-            storage: this._storage,
-            olm: this._olm,
             olmUtil: this._olmUtil,
             senderKeyLock
         });
diff --git a/src/matrix/e2ee/megolm/Decryption.js b/src/matrix/e2ee/megolm/Decryption.js
index bb5103e6..68bca6fe 100644
--- a/src/matrix/e2ee/megolm/Decryption.js
+++ b/src/matrix/e2ee/megolm/Decryption.js
@@ -14,7 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// senderKey is a curve25519 key
 export class Decryption {
     constructor({pickleKey, olm}) {
         this._pickleKey = pickleKey;

From 80ede4f4111df1379ba3270f1fdc038b6c1e0c27 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 4 Sep 2020 12:05:58 +0200
Subject: [PATCH 095/173] session will always be true here, we want to check
 sessionEntry

---
 src/matrix/e2ee/megolm/Encryption.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/matrix/e2ee/megolm/Encryption.js b/src/matrix/e2ee/megolm/Encryption.js
index 0b374f48..e39280ea 100644
--- a/src/matrix/e2ee/megolm/Encryption.js
+++ b/src/matrix/e2ee/megolm/Encryption.js
@@ -42,7 +42,7 @@ export class Encryption {
                 }
                 if (!sessionEntry || this._needsToRotate(session, sessionEntry.createdAt, encryptionParams)) {
                     // in the case of rotating, recreate a session as we already unpickled into it
-                    if (session) {
+                    if (sessionEntry) {
                         session.free();
                         session = new this._olm.OutboundGroupSession();
                     }

From fab58e8724353882e66676d813ea86f18c9b75a1 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 4 Sep 2020 12:06:26 +0200
Subject: [PATCH 096/173] first draft of megolm decryption

---
 src/matrix/e2ee/megolm/Decryption.js | 129 ++++++++++++++++++++++++++-
 1 file changed, 125 insertions(+), 4 deletions(-)

diff --git a/src/matrix/e2ee/megolm/Decryption.js b/src/matrix/e2ee/megolm/Decryption.js
index 68bca6fe..6d7941a0 100644
--- a/src/matrix/e2ee/megolm/Decryption.js
+++ b/src/matrix/e2ee/megolm/Decryption.js
@@ -14,12 +14,97 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+import {DecryptionError} from "../common.js";
+
+const CACHE_MAX_SIZE = 10;
+
 export class Decryption {
     constructor({pickleKey, olm}) {
         this._pickleKey = pickleKey;
         this._olm = olm;
     }
 
+    createSessionCache() {
+        return new SessionCache();
+    }
+
+    async decryptNewEvent(roomId, event, sessionCache, txn) {
+        const {payload, messageIndex} = this._decrypt(roomId, event, sessionCache, txn);
+        const sessionId = event.content?.["session_id"];
+        this._handleReplayAttacks(roomId, sessionId, messageIndex, event, txn);
+        return payload;
+    }
+
+    async decryptStoredEvent(roomId, event, sessionCache, txn) {
+        const {payload} = this._decrypt(roomId, event, sessionCache, txn);
+        return payload;
+    }
+
+    async _decrypt(roomId, event, sessionCache, txn) {
+        const senderKey = event.content?.["sender_key"];
+        const sessionId = event.content?.["session_id"];
+        const ciphertext = event.content?.ciphertext;
+
+        if (
+            typeof senderKey !== "string" ||
+            typeof sessionId !== "string" ||
+            typeof ciphertext !== "string"
+        ) {
+            throw new DecryptionError("MEGOLM_INVALID_EVENT", event);
+        }
+
+        let session = sessionCache.get(roomId, senderKey, sessionId);
+        if (!session) {
+            const sessionEntry = await txn.inboundGroupSessions.get(roomId, senderKey, sessionId);
+            if (sessionEntry) {
+                session = new this._olm.InboundGroupSession();
+                try {
+                    session.unpickle(this._pickleKey, sessionEntry.session);
+                } catch (err) {
+                    session.free();
+                    throw err;
+                }
+                sessionCache.add(roomId, senderKey, session);
+            }
+        }
+        if (!session) {
+            return;
+        }
+        const {plaintext, message_index: messageIndex} = session.decrypt(ciphertext);
+        let payload;
+        try {
+            payload = JSON.parse(plaintext);
+        } catch (err) {
+            throw new DecryptionError("NOT_JSON", event, {plaintext, err});
+        }
+        if (payload.room_id !== roomId) {
+            throw new DecryptionError("MEGOLM_WRONG_ROOM", event,
+                {encryptedRoomId: payload.room_id, eventRoomId: roomId});
+        }
+        return {payload, messageIndex};
+    }
+
+    async _handleReplayAttacks(roomId, sessionId, messageIndex, event, txn) {
+        const eventId = event.event_id;
+        const timestamp = event.origin_server_ts;
+        const decryption = await txn.groupSessionDecryptions.get(roomId, sessionId, messageIndex);
+        if (decryption && decryption.eventId !== eventId) {
+            // the one with the newest timestamp should be the attack
+            const decryptedEventIsBad = decryption.timestamp < timestamp;
+            const badEventId = decryptedEventIsBad ? eventId : decryption.eventId;
+            throw new DecryptionError("MEGOLM_REPLAY_ATTACK", event, {badEventId, otherEventId: decryption.eventId});
+        }
+        if (!decryption) {
+            txn.groupSessionDecryptions.set({
+                roomId,
+                sessionId,
+                messageIndex,
+                eventId,
+                timestamp
+            });
+        }
+    }
+
     async addRoomKeys(payloads, txn) {
         const newSessions = [];
         for (const {senderKey, event} of payloads) {
@@ -56,13 +141,49 @@ export class Decryption {
             }
 
         }
+        // this will be passed to the Room in notifyRoomKeys
         return newSessions;
     }
+}
 
-    applyRoomKeyChanges(newSessions) {
-        // retry decryption with the new sessions
-        if (newSessions.length) {
-            console.log(`I have ${newSessions.length} new inbound group sessions`, newSessions)
+class SessionCache {
+    constructor() {
+        this._sessions = [];
+    }
+
+    get(roomId, senderKey, sessionId) {
+        const idx = this._sessions.findIndex(s => {
+            return s.roomId === roomId &&
+                s.senderKey === senderKey &&
+                sessionId === s.session.session_id();
+        });
+        if (idx !== -1) {
+            const entry = this._sessions[idx];
+            // move to top
+            if (idx > 0) {
+                this._sessions.splice(idx, 1);
+                this._sessions.unshift(entry);
+            }
+            return entry.session;
         }
     }
+
+    add(roomId, senderKey, session) {
+        // add new at top
+        this._sessions.unshift({roomId, senderKey, session});
+        if (this._sessions.length > CACHE_MAX_SIZE) {
+            // free sessions we're about to remove
+            for (let i = CACHE_MAX_SIZE; i < this._sessions.length; i += 1) {
+                this._sessions[i].session.free();
+            }
+            this._sessions = this._sessions.slice(0, CACHE_MAX_SIZE);
+        }
+    }
+
+    dispose() {
+        for (const entry of this._sessions) {
+            entry.session.free();
+        }
+
+    }
 }

From 502ba5deea7c7b8937ebf42ea2e48ca2213c7013 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 4 Sep 2020 12:09:19 +0200
Subject: [PATCH 097/173] first draft of decryption in Room and RoomEncryption

---
 src/matrix/DeviceMessageHandler.js            | 17 ++++++------
 src/matrix/Session.js                         |  4 +--
 src/matrix/e2ee/RoomEncryption.js             | 27 +++++++++++++++++++
 src/matrix/room/Room.js                       | 27 ++++++++++++++++++-
 .../room/timeline/entries/EventEntry.js       | 13 +++++++++
 5 files changed, 77 insertions(+), 11 deletions(-)

diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js
index 537b948d..51a2378c 100644
--- a/src/matrix/DeviceMessageHandler.js
+++ b/src/matrix/DeviceMessageHandler.js
@@ -15,6 +15,7 @@ limitations under the License.
 */
 
 import {OLM_ALGORITHM, MEGOLM_ALGORITHM} from "./e2ee/common.js";
+import {groupBy} from "../utils/groupBy.js";
 
 // key to store in session store
 const PENDING_ENCRYPTED_EVENTS = "pendingEncryptedDeviceEvents";
@@ -44,21 +45,21 @@ export class DeviceMessageHandler {
         const megOlmRoomKeysPayloads = payloads.filter(p => {
             return p.event?.type === "m.room_key" && p.event.content?.algorithm === MEGOLM_ALGORITHM;
         });
-        let megolmChanges;
+        let roomKeys;
         if (megOlmRoomKeysPayloads.length) {
-            megolmChanges = await this._megolmDecryption.addRoomKeys(megOlmRoomKeysPayloads, txn);
+            roomKeys = await this._megolmDecryption.addRoomKeys(megOlmRoomKeysPayloads, txn);
         }
-        return {megolmChanges};
+        return {roomKeys};
     }
 
-    _applyDecryptChanges({megolmChanges}) {
-        if (megolmChanges) {
-            this._megolmDecryption.applyRoomKeyChanges(megolmChanges);
+    _applyDecryptChanges(rooms, {roomKeys}) {
+        const roomKeysByRoom = groupBy(roomKeys, s => s.roomId);
+        for (const [roomId, roomKeys] of roomKeysByRoom) {
         }
     }
 
     // not safe to call multiple times without awaiting first call
-    async decryptPending() {
+    async decryptPending(rooms) {
         if (!this._olmDecryption) {
             return;
         }
@@ -89,7 +90,7 @@ export class DeviceMessageHandler {
             throw err;
         }
         await txn.complete();
-        this._applyDecryptChanges(changes);
+        this._applyDecryptChanges(rooms, changes);
     }
 
     async _getPendingEvents(txn) {
diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index 1d0ac73e..c3429075 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -150,7 +150,7 @@ export class Session {
             }
             await this._e2eeAccount.generateOTKsIfNeeded(this._storage);
             await this._e2eeAccount.uploadKeys(this._storage);
-            await this._deviceMessageHandler.decryptPending();
+            await this._deviceMessageHandler.decryptPending(this.rooms);
         }
     }
 
@@ -285,7 +285,7 @@ export class Session {
 
     async afterSyncCompleted() {
         const needsToUploadOTKs = await this._e2eeAccount.generateOTKsIfNeeded(this._storage);
-        const promises = [this._deviceMessageHandler.decryptPending()];
+        const promises = [this._deviceMessageHandler.decryptPending(this.rooms)];
         if (needsToUploadOTKs) {
             // TODO: we could do this in parallel with sync if it proves to be too slow
             // but I'm not sure how to not swallow errors in that case
diff --git a/src/matrix/e2ee/RoomEncryption.js b/src/matrix/e2ee/RoomEncryption.js
index 5f0c4cc1..f5e91913 100644
--- a/src/matrix/e2ee/RoomEncryption.js
+++ b/src/matrix/e2ee/RoomEncryption.js
@@ -27,12 +27,39 @@ export class RoomEncryption {
         this._megolmEncryption = megolmEncryption;
         // content of the m.room.encryption event
         this._encryptionParams = encryptionParams;
+
+        this._megolmBackfillCache = this._megolmDecryption.createSessionCache();
+        this._megolmSyncCache = this._megolmDecryption.createSessionCache();
+    }
+
+    notifyTimelineClosed() {
+        // empty the backfill cache when closing the timeline
+        this._megolmBackfillCache.dispose();
+        this._megolmBackfillCache = this._megolmDecryption.createSessionCache();
     }
 
     async writeMemberChanges(memberChanges, txn) {
         return await this._deviceTracker.writeMemberChanges(this._room, memberChanges, txn);
     }
 
+    async decryptNewSyncEvent(id, event, txn) {
+        const payload = await this._megolmDecryption.decryptNewEvent(
+            this._room.id, event, this._megolmSyncCache, txn);
+        return payload;
+    }
+
+    async decryptNewGapEvent(id, event, txn) {
+        const payload = await this._megolmDecryption.decryptNewEvent(
+            this._room.id, event, this._megolmBackfillCache, txn);
+        return payload;
+    }
+
+    async decryptStoredEvent(id, event, txn) {
+        const payload = await this._megolmDecryption.decryptStoredEvent(
+            this._room.id, event, this._megolmBackfillCache, txn);
+        return payload;
+    }
+
     async encrypt(type, content, hsApi) {
         const megolmResult = await this._megolmEncryption.encrypt(this._room.id, type, content, this._encryptionParams);
         // share the new megolm session if needed
diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index d2272b3d..14579b73 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -45,6 +45,22 @@ export class Room extends EventEmitter {
         this._roomEncryption = null;
 	}
 
+    async _decryptSyncEntries(entries, txn) {
+        await Promise.all(entries.map(async e => {
+            if (e.eventType === "m.room.encrypted") {
+                try {
+                    const decryptedEvent = await this._roomEncryption.decryptNewSyncEvent(e.internalId, e.event, txn);
+                    if (decryptedEvent) {
+                        e.replaceWithDecrypted(decryptedEvent);
+                    }
+                } catch (err) {
+                    e.setDecryptionError(err);
+                }
+            }
+        }));
+        return entries;
+    }
+
     /** @package */
     async writeSync(roomResponse, membership, isInitialSync, txn) {
         const isTimelineOpen = !!this._timeline;
@@ -53,7 +69,13 @@ export class Room extends EventEmitter {
             membership,
             isInitialSync, isTimelineOpen,
             txn);
-		const {entries, newLiveKey, memberChanges} = await this._syncWriter.writeSync(roomResponse, txn);
+		const {entries: encryptedEntries, newLiveKey, memberChanges} =
+            await this._syncWriter.writeSync(roomResponse, txn);
+        // decrypt if applicable
+        let entries = encryptedEntries;
+        if (this._roomEncryption) {
+            entries = await this._decryptSyncEntries(encryptedEntries, txn);
+        }
         // fetch new members while we have txn open,
         // but don't make any in-memory changes yet
         let heroChanges;
@@ -341,6 +363,9 @@ export class Room extends EventEmitter {
             closeCallback: () => {
                 console.log(`closing the timeline for ${this._roomId}`);
                 this._timeline = null;
+                if (this._roomEncryption) {
+                    this._roomEncryption.notifyTimelineClosed();
+                }
             },
             user: this._user,
         });
diff --git a/src/matrix/room/timeline/entries/EventEntry.js b/src/matrix/room/timeline/entries/EventEntry.js
index 4dce9834..2bf5d941 100644
--- a/src/matrix/room/timeline/entries/EventEntry.js
+++ b/src/matrix/room/timeline/entries/EventEntry.js
@@ -21,6 +21,7 @@ export class EventEntry extends BaseEntry {
     constructor(eventEntry, fragmentIdComparer) {
         super(fragmentIdComparer);
         this._eventEntry = eventEntry;
+        this._decryptionError = null;
     }
 
     get fragmentId() {
@@ -31,6 +32,10 @@ export class EventEntry extends BaseEntry {
         return this._eventEntry.eventIndex;
     }
 
+    get internalId() {
+        return `${this.fragmentId}|${this.entryIndex}`;
+    }
+
     get content() {
         return this._eventEntry.event.content;
     }
@@ -66,4 +71,12 @@ export class EventEntry extends BaseEntry {
     get id() {
         return this._eventEntry.event.event_id;
     }
+
+    replaceWithDecrypted(event) {
+        this._eventEntry.event = event;
+    }
+
+    setDecryptionError(err) {
+        this._decryptionError = err;
+    }
 }

From fe9245dd04afa6123df15e7c341bca6b44b0d939 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 4 Sep 2020 12:10:12 +0200
Subject: [PATCH 098/173] first draft of retrying decryption when receiving
 room keys

---
 src/matrix/DeviceMessageHandler.js |  2 ++
 src/matrix/e2ee/RoomEncryption.js  | 37 ++++++++++++++++++++++++++++++
 src/matrix/room/Room.js            |  9 ++++++++
 3 files changed, 48 insertions(+)

diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js
index 51a2378c..d8698127 100644
--- a/src/matrix/DeviceMessageHandler.js
+++ b/src/matrix/DeviceMessageHandler.js
@@ -55,6 +55,8 @@ export class DeviceMessageHandler {
     _applyDecryptChanges(rooms, {roomKeys}) {
         const roomKeysByRoom = groupBy(roomKeys, s => s.roomId);
         for (const [roomId, roomKeys] of roomKeysByRoom) {
+            const room = rooms.get(roomId);
+            room?.notifyRoomKeys(roomKeys);
         }
     }
 
diff --git a/src/matrix/e2ee/RoomEncryption.js b/src/matrix/e2ee/RoomEncryption.js
index f5e91913..caec39ce 100644
--- a/src/matrix/e2ee/RoomEncryption.js
+++ b/src/matrix/e2ee/RoomEncryption.js
@@ -30,6 +30,8 @@ export class RoomEncryption {
 
         this._megolmBackfillCache = this._megolmDecryption.createSessionCache();
         this._megolmSyncCache = this._megolmDecryption.createSessionCache();
+        // not `event_id`, but an internal event id passed in to the decrypt methods
+        this._eventIdsByMissingSession = new Map();
     }
 
     notifyTimelineClosed() {
@@ -45,21 +47,56 @@ export class RoomEncryption {
     async decryptNewSyncEvent(id, event, txn) {
         const payload = await this._megolmDecryption.decryptNewEvent(
             this._room.id, event, this._megolmSyncCache, txn);
+        if (!payload) {
+            this._addMissingSessionEvent(id, event);
+        }
         return payload;
     }
 
     async decryptNewGapEvent(id, event, txn) {
         const payload = await this._megolmDecryption.decryptNewEvent(
             this._room.id, event, this._megolmBackfillCache, txn);
+        if (!payload) {
+            this._addMissingSessionEvent(id, event);
+        }
         return payload;
     }
 
     async decryptStoredEvent(id, event, txn) {
         const payload = await this._megolmDecryption.decryptStoredEvent(
             this._room.id, event, this._megolmBackfillCache, txn);
+        if (!payload) {
+            this._addMissingSessionEvent(id, event);
+        }
         return payload;
     }
 
+    _addMissingSessionEvent(id, event) {
+        const senderKey = event.content?.["sender_key"];
+        const sessionId = event.content?.["session_id"];
+        const key = `${senderKey}|${sessionId}`;
+        let eventIds = this._eventIdsByMissingSession.get(key);
+        if (!eventIds) {
+            eventIds = new Set();
+            this._eventIdsByMissingSession.set(key, eventIds);
+        }
+        eventIds.add(id);
+    }
+
+    applyRoomKeys(roomKeys) {
+        // retry decryption with the new sessions
+        const idsToRetry = [];
+        for (const roomKey of roomKeys) {
+            const key = `${roomKey.senderKey}|${roomKey.sessionId}`;
+            const idsForSession = this._eventIdsByMissingSession.get(key);
+            if (idsForSession) {
+                this._eventIdsByMissingSession.delete(key);
+                idsToRetry.push(...Array.from(idsForSession));
+            }
+        }
+        return idsToRetry;
+    }
+
     async encrypt(type, content, hsApi) {
         const megolmResult = await this._megolmEncryption.encrypt(this._room.id, type, content, this._encryptionParams);
         // share the new megolm session if needed
diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index 14579b73..493febdb 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -45,6 +45,15 @@ export class Room extends EventEmitter {
         this._roomEncryption = null;
 	}
 
+    notifyRoomKeys(roomKeys) {
+        if (this._roomEncryption) {
+            const internalIdsToRetry = this._roomEncryption.applyRoomKeys(roomKeys);
+            if (this._timeline) {
+
+            }
+        }
+    }
+
     async _decryptSyncEntries(entries, txn) {
         await Promise.all(entries.map(async e => {
             if (e.eventType === "m.room.encrypted") {

From 28b46a1e5bcb9b81afe0774f0bc7c70edbb9e722 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 4 Sep 2020 12:10:28 +0200
Subject: [PATCH 099/173] add some comments

---
 src/matrix/e2ee/megolm/Encryption.js | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/matrix/e2ee/megolm/Encryption.js b/src/matrix/e2ee/megolm/Encryption.js
index e39280ea..9849ce55 100644
--- a/src/matrix/e2ee/megolm/Encryption.js
+++ b/src/matrix/e2ee/megolm/Encryption.js
@@ -36,6 +36,7 @@ export class Encryption {
             let roomKeyMessage;
             let encryptedContent;
             try {
+                // TODO: we could consider keeping the session in memory for the current room
                 let sessionEntry = await txn.outboundGroupSessions.get(roomId);
                 if (sessionEntry) {
                     session.unpickle(this._pickleKey, sessionEntry.session);
@@ -114,6 +115,11 @@ export class Encryption {
             session_id: session.session_id(),
             session_key: session.session_key(),
             algorithm: MEGOLM_ALGORITHM,
+            // if we need to do this, do we need to create
+            // the room key message after or before having encrypted
+            // with the new session? I guess before as we do now
+            // because the chain_index is where you should start decrypting?
+            // 
             // chain_index: session.message_index()
         }
     }

From 565fdb0f8c82ae04d3f86d56eceb4a0981ed6d61 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 4 Sep 2020 12:10:36 +0200
Subject: [PATCH 100/173] use proper error codes

---
 src/matrix/e2ee/olm/Decryption.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/matrix/e2ee/olm/Decryption.js b/src/matrix/e2ee/olm/Decryption.js
index dfde7674..d37b394f 100644
--- a/src/matrix/e2ee/olm/Decryption.js
+++ b/src/matrix/e2ee/olm/Decryption.js
@@ -115,13 +115,13 @@ export class Decryption {
             try {
                 payload = JSON.parse(plaintext);
             } catch (err) {
-                throw new DecryptionError("Could not JSON decode plaintext", event, {plaintext, err});
+                throw new DecryptionError("NOT_JSON", event, {plaintext, err});
             }
             this._validatePayload(payload, event);
             return {event: payload, senderKey};
         } else {
-            throw new DecryptionError("Didn't find any session to decrypt with", event,
-                {sessionIds: senderKeyDecryption.sessions.map(s => s.id)});
+            throw new DecryptionError("OLM_NO_MATCHING_SESSION", event,
+                {knownSessionIds: senderKeyDecryption.sessions.map(s => s.id)});
         }
     }
 

From 62bcb277847e33057a2c4513e903b4442277a80c Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 4 Sep 2020 15:28:22 +0200
Subject: [PATCH 101/173] implement decryption retrying and decrypting of
 gap/load entries

turns out we do have to always check for replay attacks because
failing to decrypt doesn't prevent an item from being stored,
so if you reload and then load you might be decrypting it
for the first time
---
 src/matrix/Sync.js                   |   2 +
 src/matrix/e2ee/RoomEncryption.js    |  47 ++++-------
 src/matrix/e2ee/megolm/Decryption.js |  24 ++----
 src/matrix/room/Room.js              | 112 ++++++++++++++++++++-------
 src/matrix/room/timeline/Timeline.js |  26 +++++++
 src/observable/list/SortedArray.js   |  16 ++++
 6 files changed, 151 insertions(+), 76 deletions(-)

diff --git a/src/matrix/Sync.js b/src/matrix/Sync.js
index 3c04f71a..09ed1824 100644
--- a/src/matrix/Sync.js
+++ b/src/matrix/Sync.js
@@ -133,6 +133,8 @@ export class Sync {
             storeNames.timelineFragments,
             storeNames.pendingEvents,
             storeNames.userIdentities,
+            storeNames.inboundGroupSessions,
+            storeNames.groupSessionDecryptions,
         ]);
         const roomChanges = [];
         let sessionChanges;
diff --git a/src/matrix/e2ee/RoomEncryption.js b/src/matrix/e2ee/RoomEncryption.js
index caec39ce..2fd3fc3f 100644
--- a/src/matrix/e2ee/RoomEncryption.js
+++ b/src/matrix/e2ee/RoomEncryption.js
@@ -14,6 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+import {MEGOLM_ALGORITHM} from "./common.js";
 import {groupBy} from "../../utils/groupBy.js";
 import {makeTxnId} from "../common.js";
 
@@ -44,57 +45,43 @@ export class RoomEncryption {
         return await this._deviceTracker.writeMemberChanges(this._room, memberChanges, txn);
     }
 
-    async decryptNewSyncEvent(id, event, txn) {
-        const payload = await this._megolmDecryption.decryptNewEvent(
-            this._room.id, event, this._megolmSyncCache, txn);
+    async decrypt(event, isSync, retryData, txn) {
+        if (event.content?.algorithm !== MEGOLM_ALGORITHM) {
+            throw new Error("Unsupported algorithm: " + event.content?.algorithm);
+        }
+        let sessionCache = isSync ? this._megolmSyncCache : this._megolmBackfillCache;
+        const payload = await this._megolmDecryption.decrypt(
+            this._room.id, event, sessionCache, txn);
         if (!payload) {
-            this._addMissingSessionEvent(id, event);
+            this._addMissingSessionEvent(event, isSync, retryData);
         }
         return payload;
     }
 
-    async decryptNewGapEvent(id, event, txn) {
-        const payload = await this._megolmDecryption.decryptNewEvent(
-            this._room.id, event, this._megolmBackfillCache, txn);
-        if (!payload) {
-            this._addMissingSessionEvent(id, event);
-        }
-        return payload;
-    }
-
-    async decryptStoredEvent(id, event, txn) {
-        const payload = await this._megolmDecryption.decryptStoredEvent(
-            this._room.id, event, this._megolmBackfillCache, txn);
-        if (!payload) {
-            this._addMissingSessionEvent(id, event);
-        }
-        return payload;
-    }
-
-    _addMissingSessionEvent(id, event) {
+    _addMissingSessionEvent(event, isSync, data) {
         const senderKey = event.content?.["sender_key"];
         const sessionId = event.content?.["session_id"];
         const key = `${senderKey}|${sessionId}`;
         let eventIds = this._eventIdsByMissingSession.get(key);
         if (!eventIds) {
-            eventIds = new Set();
+            eventIds = new Map();
             this._eventIdsByMissingSession.set(key, eventIds);
         }
-        eventIds.add(id);
+        eventIds.set(event.event_id, {data, isSync});
     }
 
     applyRoomKeys(roomKeys) {
         // retry decryption with the new sessions
-        const idsToRetry = [];
+        const retryEntries = [];
         for (const roomKey of roomKeys) {
             const key = `${roomKey.senderKey}|${roomKey.sessionId}`;
-            const idsForSession = this._eventIdsByMissingSession.get(key);
-            if (idsForSession) {
+            const entriesForSession = this._eventIdsByMissingSession.get(key);
+            if (entriesForSession) {
                 this._eventIdsByMissingSession.delete(key);
-                idsToRetry.push(...Array.from(idsForSession));
+                retryEntries.push(...entriesForSession.values());
             }
         }
-        return idsToRetry;
+        return retryEntries;
     }
 
     async encrypt(type, content, hsApi) {
diff --git a/src/matrix/e2ee/megolm/Decryption.js b/src/matrix/e2ee/megolm/Decryption.js
index 6d7941a0..395b03a0 100644
--- a/src/matrix/e2ee/megolm/Decryption.js
+++ b/src/matrix/e2ee/megolm/Decryption.js
@@ -28,19 +28,7 @@ export class Decryption {
         return new SessionCache();
     }
 
-    async decryptNewEvent(roomId, event, sessionCache, txn) {
-        const {payload, messageIndex} = this._decrypt(roomId, event, sessionCache, txn);
-        const sessionId = event.content?.["session_id"];
-        this._handleReplayAttacks(roomId, sessionId, messageIndex, event, txn);
-        return payload;
-    }
-
-    async decryptStoredEvent(roomId, event, sessionCache, txn) {
-        const {payload} = this._decrypt(roomId, event, sessionCache, txn);
-        return payload;
-    }
-
-    async _decrypt(roomId, event, sessionCache, txn) {
+    async decrypt(roomId, event, sessionCache, txn) {
         const senderKey = event.content?.["sender_key"];
         const sessionId = event.content?.["session_id"];
         const ciphertext = event.content?.ciphertext;
@@ -75,16 +63,18 @@ export class Decryption {
         try {
             payload = JSON.parse(plaintext);
         } catch (err) {
-            throw new DecryptionError("NOT_JSON", event, {plaintext, err});
+            throw new DecryptionError("PLAINTEXT_NOT_JSON", event, {plaintext, err});
         }
         if (payload.room_id !== roomId) {
             throw new DecryptionError("MEGOLM_WRONG_ROOM", event,
                 {encryptedRoomId: payload.room_id, eventRoomId: roomId});
         }
-        return {payload, messageIndex};
+        await this._handleReplayAttack(roomId, sessionId, messageIndex, event, txn);
+        // TODO: verify event came from said senderKey
+        return payload;
     }
 
-    async _handleReplayAttacks(roomId, sessionId, messageIndex, event, txn) {
+    async _handleReplayAttack(roomId, sessionId, messageIndex, event, txn) {
         const eventId = event.event_id;
         const timestamp = event.origin_server_ts;
         const decryption = await txn.groupSessionDecryptions.get(roomId, sessionId, messageIndex);
@@ -92,7 +82,7 @@ export class Decryption {
             // the one with the newest timestamp should be the attack
             const decryptedEventIsBad = decryption.timestamp < timestamp;
             const badEventId = decryptedEventIsBad ? eventId : decryption.eventId;
-            throw new DecryptionError("MEGOLM_REPLAY_ATTACK", event, {badEventId, otherEventId: decryption.eventId});
+            throw new DecryptionError("MEGOLM_REPLAYED_INDEX", event, {badEventId, otherEventId: decryption.eventId});
         }
         if (!decryption) {
             txn.groupSessionDecryptions.set({
diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index 493febdb..7cfe4307 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -25,6 +25,8 @@ import {WrappedError} from "../error.js"
 import {fetchOrLoadMembers} from "./members/load.js";
 import {MemberList} from "./members/MemberList.js";
 import {Heroes} from "./members/Heroes.js";
+import {EventEntry} from "./timeline/entries/EventEntry.js";
+import {EventKey} from "./timeline/EventKey.js";
 
 export class Room extends EventEmitter {
 	constructor({roomId, storage, hsApi, emitCollectionChange, sendScheduler, pendingEvents, user, createRoomEncryption}) {
@@ -45,29 +47,75 @@ export class Room extends EventEmitter {
         this._roomEncryption = null;
 	}
 
-    notifyRoomKeys(roomKeys) {
+    async notifyRoomKeys(roomKeys) {
         if (this._roomEncryption) {
-            const internalIdsToRetry = this._roomEncryption.applyRoomKeys(roomKeys);
-            if (this._timeline) {
-
+            // array of {data, source}
+            let retryEntries = this._roomEncryption.applyRoomKeys(roomKeys);
+            let decryptedEntries = [];
+            if (retryEntries.length) {
+                // groupSessionDecryptions can be written, the other stores not
+                const txn = await this._storage.readWriteTxn([
+                    this._storage.storeNames.timelineEvents,
+                    this._storage.storeNames.inboundGroupSessions,
+                    this._storage.storeNames.groupSessionDecryptions,
+                ]);
+                try {
+                    for (const retryEntry of retryEntries) {
+                        const {data: eventKey} = retryEntry;
+                        let entry = this._timeline?.findEntry(eventKey);
+                        if (!entry) {
+                            const storageEntry = await txn.timelineEvents.get(this._roomId, eventKey.fragmentId, eventKey.entryIndex);
+                            if (storageEntry) {
+                                entry = new EventEntry(storageEntry, this._fragmentIdComparer);
+                            }
+                        }
+                        if (entry) {
+                            entry = await this._decryptEntry(entry, txn, retryEntry.isSync);
+                            decryptedEntries.push(entry);
+                        }
+                    }
+                } catch (err) {
+                    txn.abort();
+                    throw err;
+                }
+                await txn.complete();
             }
+            if (this._timeline) {
+                // only adds if already present
+                this._timeline.replaceEntries(decryptedEntries);
+            }
+            // pass decryptedEntries to roomSummary
         }
     }
 
-    async _decryptSyncEntries(entries, txn) {
-        await Promise.all(entries.map(async e => {
-            if (e.eventType === "m.room.encrypted") {
-                try {
-                    const decryptedEvent = await this._roomEncryption.decryptNewSyncEvent(e.internalId, e.event, txn);
-                    if (decryptedEvent) {
-                        e.replaceWithDecrypted(decryptedEvent);
-                    }
-                } catch (err) {
-                    e.setDecryptionError(err);
+    _enableEncryption(encryptionParams) {
+        this._roomEncryption = this._createRoomEncryption(this, encryptionParams);
+        if (this._roomEncryption) {
+            this._sendQueue.enableEncryption(this._roomEncryption);
+            this._timeline.enableEncryption(this._decryptEntries.bind(this));
+        }
+    }
+
+    async _decryptEntry(entry, txn, isSync) {
+        if (entry.eventType === "m.room.encrypted") {
+            try {
+                const {fragmentId, entryIndex} = entry;
+                const key = new EventKey(fragmentId, entryIndex);
+                const decryptedEvent = await this._roomEncryption.decrypt(
+                    entry.event, isSync, key, txn);
+                if (decryptedEvent) {
+                    entry.replaceWithDecrypted(decryptedEvent);
                 }
+            } catch (err) {
+                console.warn("event decryption error", err, entry.event);
+                entry.setDecryptionError(err);
             }
-        }));
-        return entries;
+        }
+        return entry;
+    }
+
+    async _decryptEntries(entries, txn, isSync = false) {
+        return await Promise.all(entries.map(async e => this._decryptEntry(e, txn, isSync)));
     }
 
     /** @package */
@@ -83,7 +131,7 @@ export class Room extends EventEmitter {
         // decrypt if applicable
         let entries = encryptedEntries;
         if (this._roomEncryption) {
-            entries = await this._decryptSyncEntries(encryptedEntries, txn);
+            entries = await this._decryptEntries(encryptedEntries, txn, true);
         }
         // fetch new members while we have txn open,
         // but don't make any in-memory changes yet
@@ -116,12 +164,8 @@ export class Room extends EventEmitter {
     /** @package */
     afterSync({summaryChanges, newTimelineEntries, newLiveKey, removedPendingEvents, memberChanges, heroChanges}) {
         this._syncWriter.afterSync(newLiveKey);
-        // encryption got enabled
         if (!this._summary.encryption && summaryChanges.encryption && !this._roomEncryption) {
-            this._roomEncryption = this._createRoomEncryption(this, summaryChanges.encryption);
-            if (this._roomEncryption) {
-                this._sendQueue.enableEncryption(this._roomEncryption);
-            }
+            this._enableEncryption(summaryChanges.encryption);
         }
         if (memberChanges.size) {
             if (this._changedMembersDuringSync) {
@@ -170,10 +214,7 @@ export class Room extends EventEmitter {
         try {
             this._summary.load(summary);
             if (this._summary.encryption) {
-                this._roomEncryption = this._createRoomEncryption(this, this._summary.encryption);
-                if (this._roomEncryption) {
-                    this._sendQueue.enableEncryption(this._roomEncryption);
-                }
+                this._enableEncryption(this._summary.encryption);
             }
             // need to load members for name?
             if (this._summary.needsHeroes) {
@@ -231,11 +272,18 @@ export class Room extends EventEmitter {
             }
         }).response();
 
-        const txn = await this._storage.readWriteTxn([
+        let stores = [
             this._storage.storeNames.pendingEvents,
             this._storage.storeNames.timelineEvents,
             this._storage.storeNames.timelineFragments,
-        ]);
+        ];
+        if (this._roomEncryption) {
+            stores = stores.concat([
+                this._storage.storeNames.inboundGroupSessions,
+                this._storage.storeNames.groupSessionDecryptions,
+            ]);
+        }
+        const txn = await this._storage.readWriteTxn(stores);
         let removedPendingEvents;
         let gapResult;
         try {
@@ -245,9 +293,12 @@ export class Room extends EventEmitter {
             const gapWriter = new GapWriter({
                 roomId: this._roomId,
                 storage: this._storage,
-                fragmentIdComparer: this._fragmentIdComparer
+                fragmentIdComparer: this._fragmentIdComparer,
             });
             gapResult = await gapWriter.writeFragmentFill(fragmentEntry, response, txn);
+            if (this._roomEncryption) {
+                gapResult.entries = await this._decryptEntries(gapResult.entries, false, txn);
+            }
         } catch (err) {
             txn.abort();
             throw err;
@@ -378,6 +429,9 @@ export class Room extends EventEmitter {
             },
             user: this._user,
         });
+        if (this._roomEncryption) {
+            this._timeline.enableEncryption(this._decryptEntries.bind(this));
+        }
         await this._timeline.load();
         return this._timeline;
     }
diff --git a/src/matrix/room/timeline/Timeline.js b/src/matrix/room/timeline/Timeline.js
index a64be169..c2e9d0ce 100644
--- a/src/matrix/room/timeline/Timeline.js
+++ b/src/matrix/room/timeline/Timeline.js
@@ -18,6 +18,7 @@ import {SortedArray, MappedList, ConcatList} from "../../../observable/index.js"
 import {Direction} from "./Direction.js";
 import {TimelineReader} from "./persistence/TimelineReader.js";
 import {PendingEventEntry} from "./entries/PendingEventEntry.js";
+import {EventEntry} from "./entries/EventEntry.js";
 
 export class Timeline {
     constructor({roomId, storage, closeCallback, fragmentIdComparer, pendingEvents, user}) {
@@ -45,6 +46,27 @@ export class Timeline {
         this._remoteEntries.setManySorted(entries);
     }
 
+    findEntry(eventKey) {
+        // a storage event entry has a fragmentId and eventIndex property, used for sorting,
+        // just like an EventKey, so this will work, but perhaps a bit brittle.
+        const entry = new EventEntry(eventKey, this._fragmentIdComparer);
+        try {
+            const idx = this._remoteEntries.indexOf(entry);
+            if (idx !== -1) {
+                return this._remoteEntries.get(idx);
+            }
+        } catch (err) {
+            // fragmentIdComparer threw, ignore
+            return;
+        }
+    }
+
+    replaceEntries(entries) {
+        for (const entry of entries) {
+            this._remoteEntries.replace(entry);
+        }
+    }
+
     // TODO: should we rather have generic methods for
     // - adding new entries
     // - updating existing entries (redaction, relations)
@@ -84,4 +106,8 @@ export class Timeline {
             this._closeCallback = null;
         }
     }
+
+    enableEncryption(decryptEntries) {
+        this._timelineReader.enableEncryption(decryptEntries);
+    }
 }
diff --git a/src/observable/list/SortedArray.js b/src/observable/list/SortedArray.js
index 2245dbd9..3348307b 100644
--- a/src/observable/list/SortedArray.js
+++ b/src/observable/list/SortedArray.js
@@ -41,6 +41,22 @@ export class SortedArray extends BaseObservableList {
         }
     }
 
+    replace(item) {
+        const idx = this.indexOf(item);
+        if (idx !== -1) {
+            this._items[idx] = item;
+        }
+    }
+
+    indexOf(item) {
+        const idx = sortedIndex(this._items, item, this._comparator);
+        if (idx < this._items.length && this._comparator(this._items[idx], item) === 0) {
+            return idx;
+        } else {
+            return -1;
+        }
+    }
+
     set(item, updateParams = null) {
         const idx = sortedIndex(this._items, item, this._comparator);
         if (idx >= this._items.length || this._comparator(this._items[idx], item) !== 0) {

From 5a731903da46fd2cc487d0f9494a466ff7d9ac08 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 4 Sep 2020 15:30:06 +0200
Subject: [PATCH 102/173] implement decrypting when loading timeline

---
 .../timeline/persistence/TimelineReader.js    | 70 +++++++++++++------
 1 file changed, 49 insertions(+), 21 deletions(-)

diff --git a/src/matrix/room/timeline/persistence/TimelineReader.js b/src/matrix/room/timeline/persistence/TimelineReader.js
index 928d6b64..6b3ab23e 100644
--- a/src/matrix/room/timeline/persistence/TimelineReader.js
+++ b/src/matrix/room/timeline/persistence/TimelineReader.js
@@ -24,18 +24,41 @@ export class TimelineReader {
         this._roomId = roomId;
         this._storage = storage;
         this._fragmentIdComparer = fragmentIdComparer;
+        this._decryptEntries = null;
+    }
+
+    enableEncryption(decryptEntries) {
+        this._decryptEntries = decryptEntries;
     }
 
     _openTxn() {
-        return this._storage.readTxn([
-            this._storage.storeNames.timelineEvents,
-            this._storage.storeNames.timelineFragments,
-        ]);
+        if (this._decryptEntries) {
+            return this._storage.readWriteTxn([
+                this._storage.storeNames.timelineEvents,
+                this._storage.storeNames.timelineFragments,
+                this._storage.storeNames.inboundGroupSessions,
+                this._storage.storeNames.groupSessionDecryptions,
+            ]);
+
+        } else {
+            return this._storage.readTxn([
+                this._storage.storeNames.timelineEvents,
+                this._storage.storeNames.timelineFragments,
+            ]);
+        }
     }
 
     async readFrom(eventKey, direction, amount) {
         const txn = await this._openTxn();
-        return this._readFrom(eventKey, direction, amount, txn);
+        let entries;
+        try {
+            entries = await this._readFrom(eventKey, direction, amount, txn);
+        } catch (err) {
+            txn.abort();
+            throw err;
+        }
+        await txn.complete();
+        return entries;
     }
 
     async _readFrom(eventKey, direction, amount, txn) {
@@ -50,7 +73,10 @@ export class TimelineReader {
             } else {
                 eventsWithinFragment = await timelineStore.eventsBefore(this._roomId, eventKey, amount);
             }
-            const eventEntries = eventsWithinFragment.map(e => new EventEntry(e, this._fragmentIdComparer));
+            let eventEntries = eventsWithinFragment.map(e => new EventEntry(e, this._fragmentIdComparer));
+            if (this._decryptEntries) {
+                eventEntries = await this._decryptEntries(eventEntries, txn);
+            }
             entries = directionalConcat(entries, eventEntries, direction);
             // prepend or append eventsWithinFragment to entries, and wrap them in EventEntry
 
@@ -78,22 +104,24 @@ export class TimelineReader {
 
     async readFromEnd(amount) {
         const txn = await this._openTxn();
-        const liveFragment = await txn.timelineFragments.liveFragment(this._roomId);
-        // room hasn't been synced yet
-        if (!liveFragment) {
-            return [];
+        let entries;
+        try {
+            const liveFragment = await txn.timelineFragments.liveFragment(this._roomId);
+            // room hasn't been synced yet
+            if (!liveFragment) {
+                entries = [];
+            } else {
+                this._fragmentIdComparer.add(liveFragment);
+                const liveFragmentEntry = FragmentBoundaryEntry.end(liveFragment, this._fragmentIdComparer);
+                const eventKey = liveFragmentEntry.asEventKey();
+                entries = await this._readFrom(eventKey, Direction.Backward, amount, txn);
+                entries.unshift(liveFragmentEntry);
+            }
+        } catch (err) {
+            txn.abort();
+            throw err;
         }
-        this._fragmentIdComparer.add(liveFragment);
-        const liveFragmentEntry = FragmentBoundaryEntry.end(liveFragment, this._fragmentIdComparer);
-        const eventKey = liveFragmentEntry.asEventKey();
-        const entries = await this._readFrom(eventKey, Direction.Backward, amount, txn);
-        entries.unshift(liveFragmentEntry);
+        await txn.complete();
         return entries;
     }
-
-    // reads distance up and down from eventId
-    // or just expose eventIdToKey?
-    readAtEventId(eventId, distance) {
-        return null;
-    }
 }

From 32a399afec0f050243bad9c3d9a74d324c4d02c6 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 4 Sep 2020 15:31:00 +0200
Subject: [PATCH 103/173] implement storage changes for megolm decryption

---
 src/matrix/storage/common.js                  |  1 +
 src/matrix/storage/idb/Transaction.js         |  6 ++++
 src/matrix/storage/idb/schema.js              |  5 +++
 .../idb/stores/GroupSessionDecryptionStore.js | 34 +++++++++++++++++++
 .../idb/stores/InboundGroupSessionStore.js    |  4 +++
 5 files changed, 50 insertions(+)
 create mode 100644 src/matrix/storage/idb/stores/GroupSessionDecryptionStore.js

diff --git a/src/matrix/storage/common.js b/src/matrix/storage/common.js
index 473b8eb6..4a4060b2 100644
--- a/src/matrix/storage/common.js
+++ b/src/matrix/storage/common.js
@@ -27,6 +27,7 @@ export const STORE_NAMES = Object.freeze([
     "olmSessions",
     "inboundGroupSessions",
     "outboundGroupSessions",
+    "groupSessionDecryptions",
 ]);
 
 export const STORE_MAP = Object.freeze(STORE_NAMES.reduce((nameMap, name) => {
diff --git a/src/matrix/storage/idb/Transaction.js b/src/matrix/storage/idb/Transaction.js
index 8b42b0f7..946b06cc 100644
--- a/src/matrix/storage/idb/Transaction.js
+++ b/src/matrix/storage/idb/Transaction.js
@@ -29,6 +29,7 @@ import {DeviceIdentityStore} from "./stores/DeviceIdentityStore.js";
 import {OlmSessionStore} from "./stores/OlmSessionStore.js";
 import {InboundGroupSessionStore} from "./stores/InboundGroupSessionStore.js";
 import {OutboundGroupSessionStore} from "./stores/OutboundGroupSessionStore.js";
+import {GroupSessionDecryptionStore} from "./stores/GroupSessionDecryptionStore.js";
 
 export class Transaction {
     constructor(txn, allowedStoreNames) {
@@ -105,6 +106,11 @@ export class Transaction {
     get outboundGroupSessions() {
         return this._store("outboundGroupSessions", idbStore => new OutboundGroupSessionStore(idbStore));
     }
+
+    get groupSessionDecryptions() {
+        return this._store("groupSessionDecryptions", idbStore => new OutboundGroupSessionStore(idbStore));
+    }
+
     complete() {
         return txnAsPromise(this._txn);
     }
diff --git a/src/matrix/storage/idb/schema.js b/src/matrix/storage/idb/schema.js
index 2060cb39..63458916 100644
--- a/src/matrix/storage/idb/schema.js
+++ b/src/matrix/storage/idb/schema.js
@@ -13,6 +13,7 @@ export const schema = [
     createOlmSessionStore,
     createInboundGroupSessionsStore,
     createOutboundGroupSessionsStore,
+    createGroupSessionDecryptions,
 ];
 // TODO: how to deal with git merge conflicts of this array?
 
@@ -89,3 +90,7 @@ function createOutboundGroupSessionsStore(db) {
     db.createObjectStore("outboundGroupSessions", {keyPath: "roomId"});
 }
 
+//v8
+function createGroupSessionDecryptions(db) {
+    db.createObjectStore("groupSessionDecryptions", {keyPath: "key"});
+}
diff --git a/src/matrix/storage/idb/stores/GroupSessionDecryptionStore.js b/src/matrix/storage/idb/stores/GroupSessionDecryptionStore.js
new file mode 100644
index 00000000..299b1520
--- /dev/null
+++ b/src/matrix/storage/idb/stores/GroupSessionDecryptionStore.js
@@ -0,0 +1,34 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+function encodeKey(roomId, senderKey, sessionId) {
+    return `${roomId}|${senderKey}|${sessionId}`;
+}
+
+export class GroupSessionDecryptionStore {
+    constructor(store) {
+        this._store = store;
+    }
+
+    get(roomId, sessionId, messageIndex) {
+        return this._store.get(encodeKey(roomId, sessionId, messageIndex));
+    }
+
+    set(decryption) {
+        decryption.key = encodeKey(decryption.roomId, decryption.sessionId, decryption.messageIndex);
+        this._store.put(decryption);
+    }
+}
diff --git a/src/matrix/storage/idb/stores/InboundGroupSessionStore.js b/src/matrix/storage/idb/stores/InboundGroupSessionStore.js
index 3de5a103..d05c67ff 100644
--- a/src/matrix/storage/idb/stores/InboundGroupSessionStore.js
+++ b/src/matrix/storage/idb/stores/InboundGroupSessionStore.js
@@ -29,6 +29,10 @@ export class InboundGroupSessionStore {
         return key === fetchedKey;
     }
 
+    get(roomId, senderKey, sessionId) {
+        return this._store.get(encodeKey(roomId, senderKey, sessionId));
+    }
+
     set(session) {
         session.key = encodeKey(session.roomId, session.senderKey, session.sessionId);
         this._store.put(session);

From baad4bd37f7b2a49d230b6ca09d3b92c233ba8c1 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 4 Sep 2020 15:31:27 +0200
Subject: [PATCH 104/173] hookup megolm decryption in session

---
 src/matrix/Session.js | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index c3429075..d48e0d00 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -49,6 +49,9 @@ export class Session {
         this._e2eeAccount = null;
         this._deviceTracker = null;
         this._olmEncryption = null;
+        this._megolmEncryption = null;
+        this._megolmDecryption = null;
+
         if (olm) {
             this._olmUtil = new olm.Utility();
             this._deviceTracker = new DeviceTracker({
@@ -92,9 +95,12 @@ export class Session {
             storage: this._storage,
             now: this._clock.now,
             ownDeviceId: this._sessionInfo.deviceId,
-        })
-        const megolmDecryption = new MegOlmDecryption({pickleKey: PICKLE_KEY, olm: this._olm});
-        this._deviceMessageHandler.enableEncryption({olmDecryption, megolmDecryption});
+        });
+        this._megolmDecryption = new MegOlmDecryption({
+            pickleKey: PICKLE_KEY,
+            olm: this._olm,
+        });
+        this._deviceMessageHandler.enableEncryption({olmDecryption, megolmDecryption: this._megolmDecryption});
     }
 
     _createRoomEncryption(room, encryptionParams) {
@@ -118,6 +124,7 @@ export class Session {
             deviceTracker: this._deviceTracker,
             olmEncryption: this._olmEncryption,
             megolmEncryption: this._megolmEncryption,
+            megolmDecryption: this._megolmDecryption,
             encryptionParams
         });
     }

From dc0576f2db2fa8638e45f1a0121d39363e5f0fdc Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 4 Sep 2020 15:31:45 +0200
Subject: [PATCH 105/173] cleanup

---
 src/matrix/e2ee/olm/Decryption.js              | 2 +-
 src/matrix/room/timeline/entries/EventEntry.js | 6 ++----
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/matrix/e2ee/olm/Decryption.js b/src/matrix/e2ee/olm/Decryption.js
index d37b394f..dde9522c 100644
--- a/src/matrix/e2ee/olm/Decryption.js
+++ b/src/matrix/e2ee/olm/Decryption.js
@@ -115,7 +115,7 @@ export class Decryption {
             try {
                 payload = JSON.parse(plaintext);
             } catch (err) {
-                throw new DecryptionError("NOT_JSON", event, {plaintext, err});
+                throw new DecryptionError("PLAINTEXT_NOT_JSON", event, {plaintext, err});
             }
             this._validatePayload(payload, event);
             return {event: payload, senderKey};
diff --git a/src/matrix/room/timeline/entries/EventEntry.js b/src/matrix/room/timeline/entries/EventEntry.js
index 2bf5d941..23296367 100644
--- a/src/matrix/room/timeline/entries/EventEntry.js
+++ b/src/matrix/room/timeline/entries/EventEntry.js
@@ -22,6 +22,7 @@ export class EventEntry extends BaseEntry {
         super(fragmentIdComparer);
         this._eventEntry = eventEntry;
         this._decryptionError = null;
+        this._isEncrypted = false;
     }
 
     get fragmentId() {
@@ -32,10 +33,6 @@ export class EventEntry extends BaseEntry {
         return this._eventEntry.eventIndex;
     }
 
-    get internalId() {
-        return `${this.fragmentId}|${this.entryIndex}`;
-    }
-
     get content() {
         return this._eventEntry.event.content;
     }
@@ -74,6 +71,7 @@ export class EventEntry extends BaseEntry {
 
     replaceWithDecrypted(event) {
         this._eventEntry.event = event;
+        this._isEncrypted = true;
     }
 
     setDecryptionError(err) {

From 9b771120e48f3e632c582aed50f3153adafda24a Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 4 Sep 2020 16:27:14 +0200
Subject: [PATCH 106/173] actually accept megolm decryption dep

---
 src/matrix/e2ee/RoomEncryption.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/matrix/e2ee/RoomEncryption.js b/src/matrix/e2ee/RoomEncryption.js
index 2fd3fc3f..729e1bee 100644
--- a/src/matrix/e2ee/RoomEncryption.js
+++ b/src/matrix/e2ee/RoomEncryption.js
@@ -21,11 +21,12 @@ import {makeTxnId} from "../common.js";
 const ENCRYPTED_TYPE = "m.room.encrypted";
 
 export class RoomEncryption {
-    constructor({room, deviceTracker, olmEncryption, megolmEncryption, encryptionParams}) {
+    constructor({room, deviceTracker, olmEncryption, megolmEncryption, megolmDecryption, encryptionParams}) {
         this._room = room;
         this._deviceTracker = deviceTracker;
         this._olmEncryption = olmEncryption;
         this._megolmEncryption = megolmEncryption;
+        this._megolmDecryption = megolmDecryption;
         // content of the m.room.encryption event
         this._encryptionParams = encryptionParams;
 

From 1af118a44323fb1c6b1f77392fed324ee24d4db9 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 4 Sep 2020 16:27:39 +0200
Subject: [PATCH 107/173] don't assume we have a timeline

---
 src/matrix/room/Room.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index 7cfe4307..6339b62c 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -92,7 +92,9 @@ export class Room extends EventEmitter {
         this._roomEncryption = this._createRoomEncryption(this, encryptionParams);
         if (this._roomEncryption) {
             this._sendQueue.enableEncryption(this._roomEncryption);
-            this._timeline.enableEncryption(this._decryptEntries.bind(this));
+            if (this._timeline) {
+                this._timeline.enableEncryption(this._decryptEntries.bind(this));
+            }
         }
     }
 

From e06cb1eb5f2292694cfadb896a29becf437df7f9 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 4 Sep 2020 16:29:20 +0200
Subject: [PATCH 108/173] fix param order

---
 src/matrix/room/Room.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index 6339b62c..4ec6e43c 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -299,7 +299,7 @@ export class Room extends EventEmitter {
             });
             gapResult = await gapWriter.writeFragmentFill(fragmentEntry, response, txn);
             if (this._roomEncryption) {
-                gapResult.entries = await this._decryptEntries(gapResult.entries, false, txn);
+                gapResult.entries = await this._decryptEntries(gapResult.entries, txn, false);
             }
         } catch (err) {
             txn.abort();

From 8e5d5db32b37aea54e4f6696aa7150056d073e2a Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 4 Sep 2020 16:40:15 +0200
Subject: [PATCH 109/173] add event prop on entry

---
 src/matrix/room/timeline/entries/EventEntry.js | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/matrix/room/timeline/entries/EventEntry.js b/src/matrix/room/timeline/entries/EventEntry.js
index 23296367..f305cb09 100644
--- a/src/matrix/room/timeline/entries/EventEntry.js
+++ b/src/matrix/room/timeline/entries/EventEntry.js
@@ -25,6 +25,10 @@ export class EventEntry extends BaseEntry {
         this._isEncrypted = false;
     }
 
+    get event() {
+        return this._eventEntry.event;
+    }
+
     get fragmentId() {
         return this._eventEntry.fragmentId;
     }

From f31efe3e8759c29ac5faac1173e83a0d561f7886 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 4 Sep 2020 16:40:39 +0200
Subject: [PATCH 110/173] encode key with proper names

---
 src/matrix/storage/idb/stores/GroupSessionDecryptionStore.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/matrix/storage/idb/stores/GroupSessionDecryptionStore.js b/src/matrix/storage/idb/stores/GroupSessionDecryptionStore.js
index 299b1520..99ededb9 100644
--- a/src/matrix/storage/idb/stores/GroupSessionDecryptionStore.js
+++ b/src/matrix/storage/idb/stores/GroupSessionDecryptionStore.js
@@ -14,8 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-function encodeKey(roomId, senderKey, sessionId) {
-    return `${roomId}|${senderKey}|${sessionId}`;
+function encodeKey(roomId, sessionId, messageIndex) {
+    return `${roomId}|${sessionId}|${messageIndex}`;
 }
 
 export class GroupSessionDecryptionStore {

From 7bfcfc9eede1d8e8fc8fe03ae9b021399ccec06b Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 4 Sep 2020 16:40:51 +0200
Subject: [PATCH 111/173] correct store name

---
 src/matrix/storage/idb/Transaction.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/matrix/storage/idb/Transaction.js b/src/matrix/storage/idb/Transaction.js
index 946b06cc..e0982c54 100644
--- a/src/matrix/storage/idb/Transaction.js
+++ b/src/matrix/storage/idb/Transaction.js
@@ -108,7 +108,7 @@ export class Transaction {
     }
 
     get groupSessionDecryptions() {
-        return this._store("groupSessionDecryptions", idbStore => new OutboundGroupSessionStore(idbStore));
+        return this._store("groupSessionDecryptions", idbStore => new GroupSessionDecryptionStore(idbStore));
     }
 
     complete() {

From a817a9aaf9b9b240982a411b60a2c20536c64516 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 4 Sep 2020 16:41:03 +0200
Subject: [PATCH 112/173] return decrypted type and content

---
 src/matrix/room/timeline/entries/EventEntry.js | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/matrix/room/timeline/entries/EventEntry.js b/src/matrix/room/timeline/entries/EventEntry.js
index f305cb09..d6d2f335 100644
--- a/src/matrix/room/timeline/entries/EventEntry.js
+++ b/src/matrix/room/timeline/entries/EventEntry.js
@@ -22,7 +22,7 @@ export class EventEntry extends BaseEntry {
         super(fragmentIdComparer);
         this._eventEntry = eventEntry;
         this._decryptionError = null;
-        this._isEncrypted = false;
+        this._decryptedEvent = null;
     }
 
     get event() {
@@ -38,7 +38,7 @@ export class EventEntry extends BaseEntry {
     }
 
     get content() {
-        return this._eventEntry.event.content;
+        return this._decryptedEvent?.content || this._eventEntry.event.content;
     }
 
     get prevContent() {
@@ -46,7 +46,7 @@ export class EventEntry extends BaseEntry {
     }
 
     get eventType() {
-        return this._eventEntry.event.type;
+        return this._decryptedEvent?.type || this._eventEntry.event.type;
     }
 
     get stateKey() {
@@ -74,8 +74,7 @@ export class EventEntry extends BaseEntry {
     }
 
     replaceWithDecrypted(event) {
-        this._eventEntry.event = event;
-        this._isEncrypted = true;
+        this._decryptedEvent = event;
     }
 
     setDecryptionError(err) {

From fbb534fa1671b58f929fe6496a4f16bfed17b874 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 4 Sep 2020 16:46:13 +0200
Subject: [PATCH 113/173] add todo

---
 src/matrix/e2ee/megolm/Decryption.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/matrix/e2ee/megolm/Decryption.js b/src/matrix/e2ee/megolm/Decryption.js
index 395b03a0..79a58ffa 100644
--- a/src/matrix/e2ee/megolm/Decryption.js
+++ b/src/matrix/e2ee/megolm/Decryption.js
@@ -111,6 +111,7 @@ export class Decryption {
                 return;
             }
 
+            // TODO: compare first_known_index to see which session to keep
             const hasSession = await txn.inboundGroupSessions.has(roomId, senderKey, sessionId);
             if (!hasSession) {
                 const session = new this._olm.InboundGroupSession();

From 9137d5dcbbd93fb2c4ca63b8a108e634a6dc7a01 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 10:48:11 +0200
Subject: [PATCH 114/173] make decryption algorithms return DecryptionResult

which contains curve25519 key and claimed ed25519 key as well as payload
---
 src/matrix/DeviceMessageHandler.js            | 18 +++--
 src/matrix/e2ee/DecryptionResult.js           | 70 +++++++++++++++++++
 src/matrix/e2ee/RoomEncryption.js             |  8 +--
 src/matrix/e2ee/megolm/Decryption.js          | 41 ++++++++---
 src/matrix/e2ee/olm/Decryption.js             | 43 ++++++++----
 src/matrix/room/Room.js                       | 11 ++-
 .../room/timeline/entries/EventEntry.js       | 13 ++--
 7 files changed, 159 insertions(+), 45 deletions(-)
 create mode 100644 src/matrix/e2ee/DecryptionResult.js

diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js
index d8698127..690cbc49 100644
--- a/src/matrix/DeviceMessageHandler.js
+++ b/src/matrix/DeviceMessageHandler.js
@@ -41,13 +41,19 @@ export class DeviceMessageHandler {
         // we don't handle anything other for now
     }
 
-    async _writeDecryptedEvents(payloads, txn) {
-        const megOlmRoomKeysPayloads = payloads.filter(p => {
-            return p.event?.type === "m.room_key" && p.event.content?.algorithm === MEGOLM_ALGORITHM;
+    /**
+     * [_writeDecryptedEvents description]
+     * @param  {Array} olmResults
+     * @param  {[type]} txn        [description]
+     * @return {[type]}            [description]
+     */
+    async _writeDecryptedEvents(olmResults, txn) {
+        const megOlmRoomKeysResults = olmResults.filter(r => {
+            return r.event?.type === "m.room_key" && r.event.content?.algorithm === MEGOLM_ALGORITHM;
         });
         let roomKeys;
-        if (megOlmRoomKeysPayloads.length) {
-            roomKeys = await this._megolmDecryption.addRoomKeys(megOlmRoomKeysPayloads, txn);
+        if (megOlmRoomKeysResults.length) {
+            roomKeys = await this._megolmDecryption.addRoomKeys(megOlmRoomKeysResults, txn);
         }
         return {roomKeys};
     }
@@ -84,7 +90,7 @@ export class DeviceMessageHandler {
         ]);
         let changes;
         try {
-            changes = await this._writeDecryptedEvents(decryptChanges.payloads, txn);
+            changes = await this._writeDecryptedEvents(decryptChanges.results, txn);
             decryptChanges.write(txn);
             txn.session.remove(PENDING_ENCRYPTED_EVENTS);
         } catch (err) {
diff --git a/src/matrix/e2ee/DecryptionResult.js b/src/matrix/e2ee/DecryptionResult.js
new file mode 100644
index 00000000..c109e689
--- /dev/null
+++ b/src/matrix/e2ee/DecryptionResult.js
@@ -0,0 +1,70 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+
+/**
+ * @property {object} event the plaintext event (type and content property)
+ * @property {string} senderCurve25519Key the curve25519 sender key of the olm event
+ * @property {string} claimedEd25519Key The ed25519 fingerprint key retrieved from the decryption payload.
+ *                                      The sender of the olm event claims this is the ed25519 fingerprint key
+ *                                      that matches the curve25519 sender key.
+ *                                      The caller needs to check if this key does indeed match the senderKey
+ *                                      for a device with a valid signature returned from /keys/query,
+ *                                      see DeviceTracker
+ */
+
+
+
+export class DecryptionResult {
+    constructor(event, senderCurve25519Key, claimedKeys) {
+        this.event = event;
+        this.senderCurve25519Key = senderCurve25519Key;
+        this.claimedEd25519Key = claimedKeys.ed25519;
+        this._device = null;
+        this._roomTracked = true;
+    }
+
+    setDevice(device) {
+        this._device = device;
+    }
+
+    setRoomNotTrackedYet() {
+        this._roomTracked = false;
+    }
+
+    get isVerified() {
+        if (this._device) {
+            const comesFromDevice = this._device.ed25519Key === this.claimedEd25519Key;
+            return comesFromDevice;
+        }
+        return false;
+    }
+
+    get isUnverified() {
+        if (this._device) {
+            return !this.isVerified;
+        } else if (this.isVerificationUnknown) {
+            return false;
+        } else {
+            return true;
+        }
+    }
+
+    get isVerificationUnknown() {
+        // verification is unknown if we haven't yet fetched the devices for the room
+        return !this._device && !this._roomTracked;
+    }
+}
diff --git a/src/matrix/e2ee/RoomEncryption.js b/src/matrix/e2ee/RoomEncryption.js
index 729e1bee..d81908df 100644
--- a/src/matrix/e2ee/RoomEncryption.js
+++ b/src/matrix/e2ee/RoomEncryption.js
@@ -46,17 +46,17 @@ export class RoomEncryption {
         return await this._deviceTracker.writeMemberChanges(this._room, memberChanges, txn);
     }
 
-    async decrypt(event, isSync, retryData, txn) {
+    async decrypt(event, isSync, isTimelineOpen, retryData, txn) {
         if (event.content?.algorithm !== MEGOLM_ALGORITHM) {
             throw new Error("Unsupported algorithm: " + event.content?.algorithm);
         }
         let sessionCache = isSync ? this._megolmSyncCache : this._megolmBackfillCache;
-        const payload = await this._megolmDecryption.decrypt(
+        const result = await this._megolmDecryption.decrypt(
             this._room.id, event, sessionCache, txn);
-        if (!payload) {
+        if (!result) {
             this._addMissingSessionEvent(event, isSync, retryData);
         }
-        return payload;
+        return result;
     }
 
     _addMissingSessionEvent(event, isSync, data) {
diff --git a/src/matrix/e2ee/megolm/Decryption.js b/src/matrix/e2ee/megolm/Decryption.js
index 79a58ffa..f1561a79 100644
--- a/src/matrix/e2ee/megolm/Decryption.js
+++ b/src/matrix/e2ee/megolm/Decryption.js
@@ -15,6 +15,7 @@ limitations under the License.
 */
 
 import {DecryptionError} from "../common.js";
+import {DecryptionResult} from "../DecryptionResult.js";
 
 const CACHE_MAX_SIZE = 10;
 
@@ -28,6 +29,14 @@ export class Decryption {
         return new SessionCache();
     }
 
+    /**
+     * [decrypt description]
+     * @param  {[type]} roomId       [description]
+     * @param  {[type]} event        [description]
+     * @param  {[type]} sessionCache [description]
+     * @param  {[type]} txn          [description]
+     * @return {DecryptionResult?} the decrypted event result, or undefined if the session id is not known.
+     */
     async decrypt(roomId, event, sessionCache, txn) {
         const senderKey = event.content?.["sender_key"];
         const sessionId = event.content?.["session_id"];
@@ -41,8 +50,13 @@ export class Decryption {
             throw new DecryptionError("MEGOLM_INVALID_EVENT", event);
         }
 
-        let session = sessionCache.get(roomId, senderKey, sessionId);
-        if (!session) {
+        let session;
+        let claimedKeys;
+        const cacheEntry = sessionCache.get(roomId, senderKey, sessionId);
+        if (cacheEntry) {
+            session = cacheEntry.session;
+            claimedKeys = cacheEntry.claimedKeys;
+        } else {
             const sessionEntry = await txn.inboundGroupSessions.get(roomId, senderKey, sessionId);
             if (sessionEntry) {
                 session = new this._olm.InboundGroupSession();
@@ -52,7 +66,8 @@ export class Decryption {
                     session.free();
                     throw err;
                 }
-                sessionCache.add(roomId, senderKey, session);
+                claimedKeys = sessionEntry.claimedKeys;
+                sessionCache.add(roomId, senderKey, session, claimedKeys);
             }
         }
         if (!session) {
@@ -70,8 +85,7 @@ export class Decryption {
                 {encryptedRoomId: payload.room_id, eventRoomId: roomId});
         }
         await this._handleReplayAttack(roomId, sessionId, messageIndex, event, txn);
-        // TODO: verify event came from said senderKey
-        return payload;
+        return new DecryptionResult(payload, senderKey, claimedKeys);
     }
 
     async _handleReplayAttack(roomId, sessionId, messageIndex, event, txn) {
@@ -142,6 +156,17 @@ class SessionCache {
         this._sessions = [];
     }
 
+    /**
+     * @type {CacheEntry}
+     * @property {InboundGroupSession} session the unpickled session
+     * @property {Object} claimedKeys an object with the claimed ed25519 key
+     *
+     * 
+     * @param  {string} roomId
+     * @param  {string} senderKey
+     * @param  {string} sessionId
+     * @return {CacheEntry?}
+     */
     get(roomId, senderKey, sessionId) {
         const idx = this._sessions.findIndex(s => {
             return s.roomId === roomId &&
@@ -155,13 +180,13 @@ class SessionCache {
                 this._sessions.splice(idx, 1);
                 this._sessions.unshift(entry);
             }
-            return entry.session;
+            return entry;
         }
     }
 
-    add(roomId, senderKey, session) {
+    add(roomId, senderKey, session, claimedKeys) {
         // add new at top
-        this._sessions.unshift({roomId, senderKey, session});
+        this._sessions.unshift({roomId, senderKey, session, claimedKeys});
         if (this._sessions.length > CACHE_MAX_SIZE) {
             // free sessions we're about to remove
             for (let i = CACHE_MAX_SIZE; i < this._sessions.length; i += 1) {
diff --git a/src/matrix/e2ee/olm/Decryption.js b/src/matrix/e2ee/olm/Decryption.js
index dde9522c..b3f47cd6 100644
--- a/src/matrix/e2ee/olm/Decryption.js
+++ b/src/matrix/e2ee/olm/Decryption.js
@@ -17,6 +17,7 @@ limitations under the License.
 import {DecryptionError} from "../common.js";
 import {groupBy} from "../../../utils/groupBy.js";
 import {Session} from "./Session.js";
+import {DecryptionResult} from "../DecryptionResult.js";
 
 const SESSION_LIMIT_PER_SENDER_KEY = 4;
 
@@ -50,6 +51,13 @@ export class Decryption {
     // and also can avoid side-effects before all can be stored this way
     // 
     // doing it one by one would be possible, but we would lose the opportunity for parallelization
+    // 
+    
+    /**
+     * [decryptAll description]
+     * @param  {[type]} events
+     * @return {Promise}        [description]
+     */
     async decryptAll(events) {
         const eventsPerSenderKey = groupBy(events, event => event.content?.["sender_key"]);
         const timestamp = this._now();
@@ -61,15 +69,16 @@ export class Decryption {
         try {
             const readSessionsTxn = await this._storage.readTxn([this._storage.storeNames.olmSessions]);
             // decrypt events for different sender keys in parallel
-            const results = await Promise.all(Array.from(eventsPerSenderKey.entries()).map(([senderKey, events]) => {
+            const senderKeyOperations = await Promise.all(Array.from(eventsPerSenderKey.entries()).map(([senderKey, events]) => {
                 return this._decryptAllForSenderKey(senderKey, events, timestamp, readSessionsTxn);
             }));
-            const payloads = results.reduce((all, r) => all.concat(r.payloads), []);
-            const errors = results.reduce((all, r) => all.concat(r.errors), []);
-            const senderKeyDecryptions = results.map(r => r.senderKeyDecryption);
-            return new DecryptionChanges(senderKeyDecryptions, payloads, errors, this._account, locks);
+            const results = senderKeyOperations.reduce((all, r) => all.concat(r.results), []);
+            const errors = senderKeyOperations.reduce((all, r) => all.concat(r.errors), []);
+            const senderKeyDecryptions = senderKeyOperations.map(r => r.senderKeyDecryption);
+            return new DecryptionChanges(senderKeyDecryptions, results, errors, this._account, locks);
         } catch (err) {
             // make sure the locks are release if something throws
+            // otherwise they will be released in DecryptionChanges after having written
             for (const lock of locks) {
                 lock.release();
             }
@@ -80,18 +89,18 @@ export class Decryption {
     async _decryptAllForSenderKey(senderKey, events, timestamp, readSessionsTxn) {
         const sessions = await this._getSessions(senderKey, readSessionsTxn);
         const senderKeyDecryption = new SenderKeyDecryption(senderKey, sessions, this._olm, timestamp);
-        const payloads = [];
+        const results = [];
         const errors = [];
         // events for a single senderKey need to be decrypted one by one
         for (const event of events) {
             try {
-                const payload = this._decryptForSenderKey(senderKeyDecryption, event, timestamp);
-                payloads.push(payload);
+                const result = this._decryptForSenderKey(senderKeyDecryption, event, timestamp);
+                results.push(result);
             } catch (err) {
                 errors.push(err);
             }
         }
-        return {payloads, errors, senderKeyDecryption};
+        return {results, errors, senderKeyDecryption};
     }
 
     _decryptForSenderKey(senderKeyDecryption, event, timestamp) {
@@ -118,7 +127,7 @@ export class Decryption {
                 throw new DecryptionError("PLAINTEXT_NOT_JSON", event, {plaintext, err});
             }
             this._validatePayload(payload, event);
-            return {event: payload, senderKey};
+            return new DecryptionResult(payload, senderKey, payload.keys);
         } else {
             throw new DecryptionError("OLM_NO_MATCHING_SESSION", event,
                 {knownSessionIds: senderKeyDecryption.sessions.map(s => s.id)});
@@ -182,9 +191,9 @@ export class Decryption {
         if (!payload.content) {
             throw new DecryptionError("missing content on payload", event, {payload});
         }
-        // TODO: how important is it to verify the message?
-        // we should look at payload.keys.ed25519 for that... and compare it to the key we have fetched
-        // from /keys/query, which we might not have done yet at this point.
+        if (typeof payload.keys?.ed25519 !== "string") {
+            throw new DecryptionError("Missing or invalid claimed ed25519 key on payload", event, {payload});
+        }
     }
 }
 
@@ -252,11 +261,15 @@ class SenderKeyDecryption {
     }
 }
 
+/**
+ * @property {Array} results
+ * @property {Array} errors  see DecryptionError.event to retrieve the event that failed to decrypt.
+ */
 class DecryptionChanges {
-    constructor(senderKeyDecryptions, payloads, errors, account, locks) {
+    constructor(senderKeyDecryptions, results, errors, account, locks) {
         this._senderKeyDecryptions = senderKeyDecryptions;
         this._account = account;    
-        this.payloads = payloads;
+        this.results = results;
         this.errors = errors;
         this._locks = locks;
     }
diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index 4ec6e43c..5f7b0941 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -26,7 +26,6 @@ import {fetchOrLoadMembers} from "./members/load.js";
 import {MemberList} from "./members/MemberList.js";
 import {Heroes} from "./members/Heroes.js";
 import {EventEntry} from "./timeline/entries/EventEntry.js";
-import {EventKey} from "./timeline/EventKey.js";
 
 export class Room extends EventEmitter {
 	constructor({roomId, storage, hsApi, emitCollectionChange, sendScheduler, pendingEvents, user, createRoomEncryption}) {
@@ -101,12 +100,10 @@ export class Room extends EventEmitter {
     async _decryptEntry(entry, txn, isSync) {
         if (entry.eventType === "m.room.encrypted") {
             try {
-                const {fragmentId, entryIndex} = entry;
-                const key = new EventKey(fragmentId, entryIndex);
-                const decryptedEvent = await this._roomEncryption.decrypt(
-                    entry.event, isSync, key, txn);
-                if (decryptedEvent) {
-                    entry.replaceWithDecrypted(decryptedEvent);
+                const decryptionResult = await this._roomEncryption.decrypt(
+                    entry.event, isSync, !!this._timeline, entry.asEventKey(), txn);
+                if (decryptionResult) {
+                    entry.setDecryptionResult(decryptionResult);
                 }
             } catch (err) {
                 console.warn("event decryption error", err, entry.event);
diff --git a/src/matrix/room/timeline/entries/EventEntry.js b/src/matrix/room/timeline/entries/EventEntry.js
index d6d2f335..def94b79 100644
--- a/src/matrix/room/timeline/entries/EventEntry.js
+++ b/src/matrix/room/timeline/entries/EventEntry.js
@@ -22,7 +22,7 @@ export class EventEntry extends BaseEntry {
         super(fragmentIdComparer);
         this._eventEntry = eventEntry;
         this._decryptionError = null;
-        this._decryptedEvent = null;
+        this._decryptionResult = null;
     }
 
     get event() {
@@ -38,15 +38,16 @@ export class EventEntry extends BaseEntry {
     }
 
     get content() {
-        return this._decryptedEvent?.content || this._eventEntry.event.content;
+        return this._decryptionResult?.event?.content || this._eventEntry.event.content;
     }
 
     get prevContent() {
+        // doesn't look at _decryptionResult because state events are not encrypted
         return getPrevContentFromStateEvent(this._eventEntry.event);
     }
 
     get eventType() {
-        return this._decryptedEvent?.type || this._eventEntry.event.type;
+        return this._decryptionResult?.event?.type || this._eventEntry.event.type;
     }
 
     get stateKey() {
@@ -73,8 +74,10 @@ export class EventEntry extends BaseEntry {
         return this._eventEntry.event.event_id;
     }
 
-    replaceWithDecrypted(event) {
-        this._decryptedEvent = event;
+    setDecryptionResult(result) {
+        this._decryptionResult = result;
+    }
+
     }
 
     setDecryptionError(err) {

From c32ac2c764f839d449db5bf6defc771bc67d8f3c Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 10:50:39 +0200
Subject: [PATCH 115/173] use decryption result to show message verification
 status in timeline

---
 .../session/room/timeline/tiles/MessageTile.js |  4 ++++
 src/matrix/Sync.js                             |  1 +
 src/matrix/e2ee/DeviceTracker.js               |  4 ++++
 src/matrix/e2ee/RoomEncryption.js              | 18 ++++++++++++++++++
 src/matrix/room/Room.js                        |  2 ++
 src/matrix/room/timeline/entries/EventEntry.js | 10 ++++++++++
 .../timeline/persistence/TimelineReader.js     |  1 +
 src/matrix/storage/idb/schema.js               |  7 +++++++
 .../storage/idb/stores/DeviceIdentityStore.js  |  4 ++++
 src/ui/web/css/themes/element/theme.css        |  4 ++++
 src/ui/web/session/room/timeline/common.js     |  1 +
 11 files changed, 56 insertions(+)

diff --git a/src/domain/session/room/timeline/tiles/MessageTile.js b/src/domain/session/room/timeline/tiles/MessageTile.js
index 5b3bb603..99513fbe 100644
--- a/src/domain/session/room/timeline/tiles/MessageTile.js
+++ b/src/domain/session/room/timeline/tiles/MessageTile.js
@@ -71,6 +71,10 @@ export class MessageTile extends SimpleTile {
         return this._isContinuation;
     }
 
+    get isUnverified() {
+        return this._entry.isUnverified;
+    }
+
     _getContent() {
         return this._entry.content;
     }
diff --git a/src/matrix/Sync.js b/src/matrix/Sync.js
index 09ed1824..4198618a 100644
--- a/src/matrix/Sync.js
+++ b/src/matrix/Sync.js
@@ -135,6 +135,7 @@ export class Sync {
             storeNames.userIdentities,
             storeNames.inboundGroupSessions,
             storeNames.groupSessionDecryptions,
+            storeNames.deviceIdentities,
         ]);
         const roomChanges = [];
         let sessionChanges;
diff --git a/src/matrix/e2ee/DeviceTracker.js b/src/matrix/e2ee/DeviceTracker.js
index 84da2f37..0b4c73a2 100644
--- a/src/matrix/e2ee/DeviceTracker.js
+++ b/src/matrix/e2ee/DeviceTracker.js
@@ -270,4 +270,8 @@ export class DeviceTracker {
         });
         return devices;
     }
+
+    async getDeviceByCurve25519Key(curve25519Key, txn) {
+        return await txn.deviceIdentities.getByCurve25519Key(curve25519Key);
+    }
 }
diff --git a/src/matrix/e2ee/RoomEncryption.js b/src/matrix/e2ee/RoomEncryption.js
index d81908df..7d540784 100644
--- a/src/matrix/e2ee/RoomEncryption.js
+++ b/src/matrix/e2ee/RoomEncryption.js
@@ -34,12 +34,14 @@ export class RoomEncryption {
         this._megolmSyncCache = this._megolmDecryption.createSessionCache();
         // not `event_id`, but an internal event id passed in to the decrypt methods
         this._eventIdsByMissingSession = new Map();
+        this._senderDeviceCache = new Map();
     }
 
     notifyTimelineClosed() {
         // empty the backfill cache when closing the timeline
         this._megolmBackfillCache.dispose();
         this._megolmBackfillCache = this._megolmDecryption.createSessionCache();
+        this._senderDeviceCache = new Map();    // purge the sender device cache
     }
 
     async writeMemberChanges(memberChanges, txn) {
@@ -56,9 +58,25 @@ export class RoomEncryption {
         if (!result) {
             this._addMissingSessionEvent(event, isSync, retryData);
         }
+        if (result && isTimelineOpen) {
+            await this._verifyDecryptionResult(result, txn);
+        }
         return result;
     }
 
+    async _verifyDecryptionResult(result, txn) {
+        let device = this._senderDeviceCache.get(result.senderCurve25519Key);
+        if (!device) {
+            device = await this._deviceTracker.getDeviceByCurve25519Key(result.senderCurve25519Key, txn);
+            this._senderDeviceCache.set(result.senderCurve25519Key, device);
+        }
+        if (device) {
+            result.setDevice(device);
+        } else if (!this._room.isTrackingMembers) {
+            result.setRoomNotTrackedYet();
+        }
+    }
+
     _addMissingSessionEvent(event, isSync, data) {
         const senderKey = event.content?.["sender_key"];
         const sessionId = event.content?.["session_id"];
diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index 5f7b0941..a26be995 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -57,6 +57,7 @@ export class Room extends EventEmitter {
                     this._storage.storeNames.timelineEvents,
                     this._storage.storeNames.inboundGroupSessions,
                     this._storage.storeNames.groupSessionDecryptions,
+                    this._storage.storeNames.deviceIdentities,
                 ]);
                 try {
                     for (const retryEntry of retryEntries) {
@@ -280,6 +281,7 @@ export class Room extends EventEmitter {
             stores = stores.concat([
                 this._storage.storeNames.inboundGroupSessions,
                 this._storage.storeNames.groupSessionDecryptions,
+                this._storage.storeNames.deviceIdentities,
             ]);
         }
         const txn = await this._storage.readWriteTxn(stores);
diff --git a/src/matrix/room/timeline/entries/EventEntry.js b/src/matrix/room/timeline/entries/EventEntry.js
index def94b79..8c7029d4 100644
--- a/src/matrix/room/timeline/entries/EventEntry.js
+++ b/src/matrix/room/timeline/entries/EventEntry.js
@@ -78,6 +78,16 @@ export class EventEntry extends BaseEntry {
         this._decryptionResult = result;
     }
 
+    get isEncrypted() {
+        return this._eventEntry.event.type === "m.room.encrypted";
+    }
+
+    get isVerified() {
+        return this.isEncrypted && this._decryptionResult?.isVerified;
+    }
+
+    get isUnverified() {
+        return this.isEncrypted && this._decryptionResult?.isUnverified;
     }
 
     setDecryptionError(err) {
diff --git a/src/matrix/room/timeline/persistence/TimelineReader.js b/src/matrix/room/timeline/persistence/TimelineReader.js
index 6b3ab23e..4446eaf1 100644
--- a/src/matrix/room/timeline/persistence/TimelineReader.js
+++ b/src/matrix/room/timeline/persistence/TimelineReader.js
@@ -38,6 +38,7 @@ export class TimelineReader {
                 this._storage.storeNames.timelineFragments,
                 this._storage.storeNames.inboundGroupSessions,
                 this._storage.storeNames.groupSessionDecryptions,
+                this._storage.storeNames.deviceIdentities,
             ]);
 
         } else {
diff --git a/src/matrix/storage/idb/schema.js b/src/matrix/storage/idb/schema.js
index 63458916..f1817c24 100644
--- a/src/matrix/storage/idb/schema.js
+++ b/src/matrix/storage/idb/schema.js
@@ -14,6 +14,7 @@ export const schema = [
     createInboundGroupSessionsStore,
     createOutboundGroupSessionsStore,
     createGroupSessionDecryptions,
+    addSenderKeyIndexToDeviceStore
 ];
 // TODO: how to deal with git merge conflicts of this array?
 
@@ -94,3 +95,9 @@ function createOutboundGroupSessionsStore(db) {
 function createGroupSessionDecryptions(db) {
     db.createObjectStore("groupSessionDecryptions", {keyPath: "key"});
 }
+
+//v9
+function addSenderKeyIndexToDeviceStore(db, txn) {
+    const deviceIdentities = txn.objectStore("deviceIdentities");
+    deviceIdentities.createIndex("byCurve25519Key", "curve25519Key", {unique: true});
+}
diff --git a/src/matrix/storage/idb/stores/DeviceIdentityStore.js b/src/matrix/storage/idb/stores/DeviceIdentityStore.js
index aec337fc..d1bf7eaa 100644
--- a/src/matrix/storage/idb/stores/DeviceIdentityStore.js
+++ b/src/matrix/storage/idb/stores/DeviceIdentityStore.js
@@ -38,4 +38,8 @@ export class DeviceIdentityStore {
         deviceIdentity.key = encodeKey(deviceIdentity.userId, deviceIdentity.deviceId);
         return this._store.put(deviceIdentity);
     }
+
+    getByCurve25519Key(curve25519Key) {
+        return this._store.index("byCurve25519Key").get(curve25519Key);
+    }
 }
diff --git a/src/ui/web/css/themes/element/theme.css b/src/ui/web/css/themes/element/theme.css
index bf81bcd0..bdcd599f 100644
--- a/src/ui/web/css/themes/element/theme.css
+++ b/src/ui/web/css/themes/element/theme.css
@@ -373,6 +373,10 @@ ul.Timeline > li.continuation time {
     color: #ccc;
 }
 
+.TextMessageView.unverified .message-container {
+    color: #ff4b55;
+}
+
 .message-container p {
     margin: 3px 0;
     line-height: 2.2rem;
diff --git a/src/ui/web/session/room/timeline/common.js b/src/ui/web/session/room/timeline/common.js
index 36ccb624..00751f4c 100644
--- a/src/ui/web/session/room/timeline/common.js
+++ b/src/ui/web/session/room/timeline/common.js
@@ -22,6 +22,7 @@ export function renderMessage(t, vm, children) {
         "TextMessageView": true,
         own: vm.isOwn,
         pending: vm.isPending,
+        unverified: vm.isUnverified,
         continuation: vm => vm.isContinuation,
     };
 

From 3e100ff5ec552494762b7c3a6274dabfcd78fa53 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 10:51:01 +0200
Subject: [PATCH 116/173] ensure /keys/query devices have the keys we need

---
 src/matrix/e2ee/DeviceTracker.js | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/matrix/e2ee/DeviceTracker.js b/src/matrix/e2ee/DeviceTracker.js
index 0b4c73a2..6b6f3894 100644
--- a/src/matrix/e2ee/DeviceTracker.js
+++ b/src/matrix/e2ee/DeviceTracker.js
@@ -26,8 +26,8 @@ function deviceKeysAsDeviceIdentity(deviceSection) {
     return {
         userId,
         deviceId,
-        ed25519Key: deviceSection.keys?.[`ed25519:${deviceId}`],
-        curve25519Key: deviceSection.keys?.[`curve25519:${deviceId}`],
+        ed25519Key: deviceSection.keys[`ed25519:${deviceId}`],
+        curve25519Key: deviceSection.keys[`curve25519:${deviceId}`],
         algorithms: deviceSection.algorithms,
         displayName: deviceSection.unsigned?.device_display_name,
     };
@@ -200,6 +200,11 @@ export class DeviceTracker {
                 if (deviceIdOnKeys !== deviceId) {
                     return false;
                 }
+                const ed25519Key = deviceKeys.keys?.[`ed25519:${deviceId}`];
+                const curve25519Key = deviceKeys.keys?.[`curve25519:${deviceId}`];
+                if (typeof ed25519Key !== "string" || typeof curve25519Key !== "string") {
+                    return false;
+                }
                 // don't store our own device
                 if (userId === this._ownUserId && deviceId === this._ownDeviceId) {
                     return false;

From 4cf3b3569d09da189ffca43bd87a3dca4a4e4332 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 10:51:28 +0200
Subject: [PATCH 117/173] storage method takes EventKey actually

---
 src/matrix/room/Room.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index a26be995..3942f1c5 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -64,7 +64,7 @@ export class Room extends EventEmitter {
                         const {data: eventKey} = retryEntry;
                         let entry = this._timeline?.findEntry(eventKey);
                         if (!entry) {
-                            const storageEntry = await txn.timelineEvents.get(this._roomId, eventKey.fragmentId, eventKey.entryIndex);
+                            const storageEntry = await txn.timelineEvents.get(this._roomId, eventKey);
                             if (storageEntry) {
                                 entry = new EventEntry(storageEntry, this._fragmentIdComparer);
                             }

From 40ed66dc5e88fb915d512b0e5ad4bf3e78df14c0 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 10:51:45 +0200
Subject: [PATCH 118/173] document return type

---
 src/matrix/e2ee/megolm/Encryption.js | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/src/matrix/e2ee/megolm/Encryption.js b/src/matrix/e2ee/megolm/Encryption.js
index 9849ce55..d3c613f6 100644
--- a/src/matrix/e2ee/megolm/Encryption.js
+++ b/src/matrix/e2ee/megolm/Encryption.js
@@ -26,6 +26,14 @@ export class Encryption {
         this._ownDeviceId = ownDeviceId;
     }
 
+    /**
+     * Encrypts a message with megolm
+     * @param  {string} roomId           
+     * @param  {string} type             event type to encrypt
+     * @param  {string} content          content to encrypt
+     * @param  {object} encryptionParams the content of the m.room.encryption event
+     * @return {Promise}
+     */
     async encrypt(roomId, type, content, encryptionParams) {
         let session = new this._olm.OutboundGroupSession();
         try {
@@ -145,6 +153,14 @@ export class Encryption {
     }
 }
 
+/**
+ * @property {object?} roomKeyMessage  if encrypting this message
+ *                                     created a new outbound session,
+ *                                     this contains the content of the m.room_key message
+ *                                     that should be sent out over olm.
+ * @property {object} content  the encrypted message as the content of
+ *                             the m.room.encrypted event that should be sent out   
+ */
 class EncryptionResult {
     constructor(content, roomKeyMessage) {
         this.content = content;

From 2b59c8bb7c09301507ecf0289230b4a9e29238f4 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 10:52:02 +0200
Subject: [PATCH 119/173] store ed25519 key from olm event rather than one in
 m.room_key payload

that's the docs/js-sdk do it, even though it probably
doesn't matter much as we verify the key anyway
---
 src/matrix/e2ee/megolm/Decryption.js | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/matrix/e2ee/megolm/Decryption.js b/src/matrix/e2ee/megolm/Decryption.js
index f1561a79..2a43c00b 100644
--- a/src/matrix/e2ee/megolm/Decryption.js
+++ b/src/matrix/e2ee/megolm/Decryption.js
@@ -109,9 +109,19 @@ export class Decryption {
         }
     }
 
-    async addRoomKeys(payloads, txn) {
+    /**
+     * @type {MegolmInboundSessionDescription}
+     * @property {string} senderKey the sender key of the session
+     * @property {string} sessionId the session identifier
+     * 
+     * Adds room keys as inbound group sessions
+     * @param {Array} decryptionResults an array of m.room_key decryption results.
+     * @param {[type]} txn      a storage transaction with read/write on inboundGroupSessions
+     * @return {Promise>} an array with the newly added sessions
+     */
+    async addRoomKeys(decryptionResults, txn) {
         const newSessions = [];
-        for (const {senderKey, event} of payloads) {
+        for (const {senderCurve25519Key: senderKey, event, claimedEd25519Key} of decryptionResults) {
             const roomId = event.content?.["room_id"];
             const sessionId = event.content?.["session_id"];
             const sessionKey = event.content?.["session_key"];
@@ -136,7 +146,7 @@ export class Decryption {
                         senderKey,
                         sessionId,
                         session: session.pickle(this._pickleKey),
-                        claimedKeys: event.keys,
+                        claimedKeys: {ed25519: claimedEd25519Key},
                     };
                     txn.inboundGroupSessions.set(sessionEntry);
                     newSessions.push(sessionEntry);

From 2c5c3ac8e2c62addbf49d3126892e9d1e8f06132 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 10:53:02 +0200
Subject: [PATCH 120/173] formatting

---
 src/matrix/e2ee/megolm/Decryption.js | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/matrix/e2ee/megolm/Decryption.js b/src/matrix/e2ee/megolm/Decryption.js
index 2a43c00b..bd3665b3 100644
--- a/src/matrix/e2ee/megolm/Decryption.js
+++ b/src/matrix/e2ee/megolm/Decryption.js
@@ -96,7 +96,11 @@ export class Decryption {
             // the one with the newest timestamp should be the attack
             const decryptedEventIsBad = decryption.timestamp < timestamp;
             const badEventId = decryptedEventIsBad ? eventId : decryption.eventId;
-            throw new DecryptionError("MEGOLM_REPLAYED_INDEX", event, {badEventId, otherEventId: decryption.eventId});
+            throw new DecryptionError("MEGOLM_REPLAYED_INDEX", event, {
+                messageIndex,
+                badEventId,
+                otherEventId: decryption.eventId
+            });
         }
         if (!decryption) {
             txn.groupSessionDecryptions.set({

From dea9fd90b490be50a2444c707a976b1c3138445e Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 10:53:15 +0200
Subject: [PATCH 121/173] name devices at login "Hydrogen"

so you can somewhat identify them in a device list
---
 src/matrix/SessionContainer.js  | 2 +-
 src/matrix/net/HomeServerApi.js | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/matrix/SessionContainer.js b/src/matrix/SessionContainer.js
index 7917baf4..9b64115f 100644
--- a/src/matrix/SessionContainer.js
+++ b/src/matrix/SessionContainer.js
@@ -89,7 +89,7 @@ export class SessionContainer {
         let sessionInfo;
         try {
             const hsApi = new HomeServerApi({homeServer, request: this._request, createTimeout: this._clock.createTimeout});
-            const loginData = await hsApi.passwordLogin(username, password).response();
+            const loginData = await hsApi.passwordLogin(username, password, "Hydrogen").response();
             const sessionId = this.createNewSessionId();
             sessionInfo = {
                 id: sessionId,
diff --git a/src/matrix/net/HomeServerApi.js b/src/matrix/net/HomeServerApi.js
index b1a89634..649a7462 100644
--- a/src/matrix/net/HomeServerApi.js
+++ b/src/matrix/net/HomeServerApi.js
@@ -141,14 +141,15 @@ export class HomeServerApi {
             {}, {}, options);
     }
 
-    passwordLogin(username, password, options = null) {
+    passwordLogin(username, password, initialDeviceDisplayName, options = null) {
         return this._post("/login", null, {
           "type": "m.login.password",
           "identifier": {
             "type": "m.id.user",
             "user": username
           },
-          "password": password
+          "password": password,
+          "initial_device_display_name": initialDeviceDisplayName
         }, options);
     }
 

From cd172f6df20150a94895db8c8356319c09e8f6e9 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 10:55:38 +0200
Subject: [PATCH 122/173] log new room keys for debugging

---
 src/matrix/DeviceMessageHandler.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js
index 690cbc49..e0398256 100644
--- a/src/matrix/DeviceMessageHandler.js
+++ b/src/matrix/DeviceMessageHandler.js
@@ -53,6 +53,7 @@ export class DeviceMessageHandler {
         });
         let roomKeys;
         if (megOlmRoomKeysResults.length) {
+            console.log("new room keys", megOlmRoomKeysResults);
             roomKeys = await this._megolmDecryption.addRoomKeys(megOlmRoomKeysResults, txn);
         }
         return {roomKeys};

From d184be2d22b07bfc7c4e1e6d63ef62d254b21430 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 11:09:09 +0200
Subject: [PATCH 123/173] rotate outbound megolm session when somebody leaves
 the room

---
 src/matrix/e2ee/RoomEncryption.js                          | 3 +++
 src/matrix/e2ee/megolm/Encryption.js                       | 4 ++++
 src/matrix/room/members/RoomMember.js                      | 4 ++++
 src/matrix/storage/idb/stores/OutboundGroupSessionStore.js | 4 ++++
 4 files changed, 15 insertions(+)

diff --git a/src/matrix/e2ee/RoomEncryption.js b/src/matrix/e2ee/RoomEncryption.js
index 7d540784..190852d3 100644
--- a/src/matrix/e2ee/RoomEncryption.js
+++ b/src/matrix/e2ee/RoomEncryption.js
@@ -45,6 +45,9 @@ export class RoomEncryption {
     }
 
     async writeMemberChanges(memberChanges, txn) {
+        if (memberChanges.some(m => m.hasLeft)) {
+            this._megolmEncryption.discardOutboundSession(this._room.id, txn);
+        }
         return await this._deviceTracker.writeMemberChanges(this._room, memberChanges, txn);
     }
 
diff --git a/src/matrix/e2ee/megolm/Encryption.js b/src/matrix/e2ee/megolm/Encryption.js
index d3c613f6..9b7df4eb 100644
--- a/src/matrix/e2ee/megolm/Encryption.js
+++ b/src/matrix/e2ee/megolm/Encryption.js
@@ -26,6 +26,10 @@ export class Encryption {
         this._ownDeviceId = ownDeviceId;
     }
 
+    discardOutboundSession(roomId, txn) {
+        txn.outboundGroupSessions.remove(roomId);
+    }
+
     /**
      * Encrypts a message with megolm
      * @param  {string} roomId           
diff --git a/src/matrix/room/members/RoomMember.js b/src/matrix/room/members/RoomMember.js
index 954de0a4..6b13c721 100644
--- a/src/matrix/room/members/RoomMember.js
+++ b/src/matrix/room/members/RoomMember.js
@@ -134,4 +134,8 @@ export class MemberChange {
     get membership() {
         return this._memberEvent.content?.membership;
     }
+
+    get hasLeft() {
+        return this.previousMembership === "join" && this.membership !== "join";
+    }
 }
diff --git a/src/matrix/storage/idb/stores/OutboundGroupSessionStore.js b/src/matrix/storage/idb/stores/OutboundGroupSessionStore.js
index ef9224be..9710765f 100644
--- a/src/matrix/storage/idb/stores/OutboundGroupSessionStore.js
+++ b/src/matrix/storage/idb/stores/OutboundGroupSessionStore.js
@@ -19,6 +19,10 @@ export class OutboundGroupSessionStore {
         this._store = store;
     }
 
+    remove(roomId) {
+        this._store.delete(roomId);
+    }
+
     get(roomId) {
         return this._store.get(roomId);
     }

From bbaf3a56053fce09d9696b4bf4171bd62a42c79a Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 14:22:11 +0200
Subject: [PATCH 124/173] write needsRoomKey flag when new members joins to
 tracked e2ee room

---
 src/matrix/room/Room.js                       |  2 +-
 src/matrix/room/members/RoomMember.js         | 12 +++
 .../room/timeline/persistence/SyncWriter.js   | 73 ++++++++++++-------
 3 files changed, 60 insertions(+), 27 deletions(-)

diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index 3942f1c5..b4e19a80 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -127,7 +127,7 @@ export class Room extends EventEmitter {
             isInitialSync, isTimelineOpen,
             txn);
 		const {entries: encryptedEntries, newLiveKey, memberChanges} =
-            await this._syncWriter.writeSync(roomResponse, txn);
+            await this._syncWriter.writeSync(roomResponse, this.isTrackingMembers, txn);
         // decrypt if applicable
         let entries = encryptedEntries;
         if (this._roomEncryption) {
diff --git a/src/matrix/room/members/RoomMember.js b/src/matrix/room/members/RoomMember.js
index 6b13c721..084e6168 100644
--- a/src/matrix/room/members/RoomMember.js
+++ b/src/matrix/room/members/RoomMember.js
@@ -67,6 +67,14 @@ export class RoomMember {
         });
     }
 
+    get needsRoomKey() {
+        return this._data.needsRoomKey;
+    }
+
+    set needsRoomKey(value) {
+        this._data.needsRoomKey = !!value;
+    }
+
     get membership() {
         return this._data.membership;
     }
@@ -138,4 +146,8 @@ export class MemberChange {
     get hasLeft() {
         return this.previousMembership === "join" && this.membership !== "join";
     }
+
+    get hasJoined() {
+        return this.previousMembership !== "join" && this.membership === "join";
+    }
 }
diff --git a/src/matrix/room/timeline/persistence/SyncWriter.js b/src/matrix/room/timeline/persistence/SyncWriter.js
index fdc4035b..0d9bea9d 100644
--- a/src/matrix/room/timeline/persistence/SyncWriter.js
+++ b/src/matrix/room/timeline/persistence/SyncWriter.js
@@ -98,39 +98,47 @@ export class SyncWriter {
         return {oldFragment, newFragment};
     }
 
-    _writeStateEvent(event, txn) {
-        if (event.type === MEMBER_EVENT_TYPE) {
-            const userId = event.state_key;
-            if (userId) {
-                const memberChange = new MemberChange(this._roomId, event);
-                if (memberChange.member) {
-                    // as this is sync, we can just replace the member
-                    // if it is there already
-                    txn.roomMembers.set(memberChange.member.serialize());
-                    return memberChange;
+    async _writeMember(event, trackNewlyJoined, txn) {
+        const userId = event.state_key;
+        if (userId) {
+            const memberChange = new MemberChange(this._roomId, event);
+            const {member} = memberChange;
+            if (member) {
+                if (trackNewlyJoined) {
+                    const existingMemberData = await txn.roomMembers.get(this._roomId, userId);
+                    // mark new members so we know who needs our the room key for our outbound megolm session
+                    member.needsRoomKey = existingMemberData.needsRoomKey || memberChange.hasJoined;
                 }
+                txn.roomMembers.set(member.serialize());
+                return memberChange;
             }
+        }
+    }
+
+    async _writeStateEvent(event, trackNewlyJoined, txn) {
+        if (event.type === MEMBER_EVENT_TYPE) {
+            return await this._writeMember(event, trackNewlyJoined, txn);
         } else {
             txn.roomState.set(this._roomId, event);
         }
     }
 
-    _writeStateEvents(roomResponse, txn) {
+    async _writeStateEvents(roomResponse, trackNewlyJoined, txn) {
         const memberChanges = new Map();
         // persist state
         const {state} = roomResponse;
         if (Array.isArray(state?.events)) {
-            for (const event of state.events) {
-                const memberChange = this._writeStateEvent(event, txn);
+            await Promise.all(state.events.map(async event => {
+                const memberChange = await this._writeStateEvent(event, trackNewlyJoined, txn);
                 if (memberChange) {
                     memberChanges.set(memberChange.userId, memberChange);
                 }
-            }
+            }));
         }
         return memberChanges;
     }
 
-    async _writeTimeline(entries, timeline, currentKey, txn) {
+    async _writeTimeline(entries, timeline, currentKey, trackNewlyJoined, txn) {
         const memberChanges = new Map();
         if (timeline.events) {
             const events = deduplicateEvents(timeline.events);
@@ -145,15 +153,17 @@ export class SyncWriter {
                 }
                 txn.timelineEvents.insert(entry);
                 entries.push(new EventEntry(entry, this._fragmentIdComparer));
-                
-                // process live state events first, so new member info is available
-                if (typeof event.state_key === "string") {
-                    const memberChange = this._writeStateEvent(event, txn);
-                    if (memberChange) {
-                        memberChanges.set(memberChange.userId, memberChange);
-                    }
-                }
             }
+            // process live state events first, so new member info is available
+            // also run async state event writing in parallel
+            await Promise.all(events.filter(event => {
+                return typeof event.state_key === "string";
+            }).map(async stateEvent => {
+                const memberChange = await this._writeStateEvent(stateEvent, trackNewlyJoined, txn);
+                if (memberChange) {
+                    memberChanges.set(memberChange.userId, memberChange);
+                }
+            }));
         }
         return {currentKey, memberChanges};
     }
@@ -176,7 +186,18 @@ export class SyncWriter {
         }
     }
 
-    async writeSync(roomResponse, txn) {
+    /**
+     * @type {SyncWriterResult}
+     * @property {Array} entries new timeline entries written
+     * @property {EventKey} newLiveKey the advanced key to write events at
+     * @property {Map} memberChanges member changes in the processed sync ny user id
+     * 
+     * @param  {Object}  roomResponse [description]
+     * @param  {Boolean} trackNewlyJoined  needed to know if we need to keep track whether a user needs keys when they join an encrypted room
+     * @param  {Transaction}  txn     
+     * @return {SyncWriterResult}
+     */
+    async writeSync(roomResponse, trackNewlyJoined, txn) {
         const entries = [];
         const {timeline} = roomResponse;
         let currentKey = this._lastLiveKey;
@@ -198,8 +219,8 @@ export class SyncWriter {
         }
         // important this happens before _writeTimeline so
         // members are available in the transaction
-        const memberChanges = this._writeStateEvents(roomResponse, txn);
-        const timelineResult = await this._writeTimeline(entries, timeline, currentKey, txn);
+        const memberChanges = this._writeStateEvents(roomResponse, trackNewlyJoined, txn);
+        const timelineResult = await this._writeTimeline(entries, timeline, currentKey, trackNewlyJoined, txn);
         currentKey = timelineResult.currentKey;
         // merge member changes from state and timeline, giving precedence to the latter
         for (const [userId, memberChange] of timelineResult.memberChanges.entries()) {

From 7b35a3c46cf847df93884532187f39782f74651e Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 14:23:38 +0200
Subject: [PATCH 125/173] memberChanges is a map, not array

---
 src/matrix/e2ee/RoomEncryption.js | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/matrix/e2ee/RoomEncryption.js b/src/matrix/e2ee/RoomEncryption.js
index 190852d3..d179c23f 100644
--- a/src/matrix/e2ee/RoomEncryption.js
+++ b/src/matrix/e2ee/RoomEncryption.js
@@ -45,8 +45,11 @@ export class RoomEncryption {
     }
 
     async writeMemberChanges(memberChanges, txn) {
-        if (memberChanges.some(m => m.hasLeft)) {
-            this._megolmEncryption.discardOutboundSession(this._room.id, txn);
+        for (const m of memberChanges.values()) {
+            if (m.hasLeft) {
+                this._megolmEncryption.discardOutboundSession(this._room.id, txn);
+                break;
+            }
         }
         return await this._deviceTracker.writeMemberChanges(this._room, memberChanges, txn);
     }

From 52c3c7c03d5524e1b7110d0cb3f9519523d38efd Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 14:24:48 +0200
Subject: [PATCH 126/173] support sending out room key in room encryption for
 newly joined members

---
 src/matrix/e2ee/DeviceTracker.js     | 28 +++++++++---
 src/matrix/e2ee/RoomEncryption.js    | 65 +++++++++++++++++++++++++---
 src/matrix/e2ee/megolm/Encryption.js | 22 +++++++---
 3 files changed, 99 insertions(+), 16 deletions(-)

diff --git a/src/matrix/e2ee/DeviceTracker.js b/src/matrix/e2ee/DeviceTracker.js
index 6b6f3894..0045522f 100644
--- a/src/matrix/e2ee/DeviceTracker.js
+++ b/src/matrix/e2ee/DeviceTracker.js
@@ -65,7 +65,7 @@ export class DeviceTracker {
     }
 
     async trackRoom(room) {
-        if (room.isTrackingMembers) {
+        if (room.isTrackingMembers || !room.isEncrypted) {
             return;
         }
         const memberList = await room.loadMemberList();
@@ -230,8 +230,7 @@ export class DeviceTracker {
      * @param  {String} roomId [description]
      * @return {[type]}        [description]
      */
-    async deviceIdentitiesForTrackedRoom(roomId, hsApi) {
-        let identities;
+    async devicesForTrackedRoom(roomId, hsApi) {
         const txn = await this._storage.readTxn([
             this._storage.storeNames.roomMembers,
             this._storage.storeNames.userIdentities,
@@ -243,8 +242,27 @@ export class DeviceTracker {
         
         // So, this will also contain non-joined memberships
         const userIds = await txn.roomMembers.getAllUserIds(roomId);
-        const allMemberIdentities = await Promise.all(userIds.map(userId => txn.userIdentities.get(userId)));
-        identities = allMemberIdentities.filter(identity => {
+
+        return await this._devicesForUserIds(roomId, userIds, txn, hsApi);
+    }
+
+    async devicesForRoomMembers(roomId, userIds, hsApi) {
+        const txn = await this._storage.readTxn([
+            this._storage.storeNames.userIdentities,
+        ]);
+        return await this._devicesForUserIds(roomId, userIds, txn, hsApi);
+    }
+
+    /**
+     * @param  {string} roomId  [description]
+     * @param  {Array} userIds a set of user ids to try and find the identity for. Will be check to belong to roomId.
+     * @param  {Transaction} userIdentityTxn to read the user identities
+     * @param  {HomeServerApi} hsApi
+     * @return {Array}
+     */
+    async _devicesForUserIds(roomId, userIds, userIdentityTxn, hsApi) {
+        const allMemberIdentities = await Promise.all(userIds.map(userId => userIdentityTxn.userIdentities.get(userId)));
+        const identities = allMemberIdentities.filter(identity => {
             // identity will be missing for any userIds that don't have 
             // membership join in any of your encrypted rooms
             return identity && identity.roomIds.includes(roomId);
diff --git a/src/matrix/e2ee/RoomEncryption.js b/src/matrix/e2ee/RoomEncryption.js
index d179c23f..cd918292 100644
--- a/src/matrix/e2ee/RoomEncryption.js
+++ b/src/matrix/e2ee/RoomEncryption.js
@@ -21,7 +21,7 @@ import {makeTxnId} from "../common.js";
 const ENCRYPTED_TYPE = "m.room.encrypted";
 
 export class RoomEncryption {
-    constructor({room, deviceTracker, olmEncryption, megolmEncryption, megolmDecryption, encryptionParams}) {
+    constructor({room, deviceTracker, olmEncryption, megolmEncryption, megolmDecryption, encryptionParams, storage}) {
         this._room = room;
         this._deviceTracker = deviceTracker;
         this._olmEncryption = olmEncryption;
@@ -35,6 +35,7 @@ export class RoomEncryption {
         // not `event_id`, but an internal event id passed in to the decrypt methods
         this._eventIdsByMissingSession = new Map();
         this._senderDeviceCache = new Map();
+        this._storage = storage;
     }
 
     notifyTimelineClosed() {
@@ -114,10 +115,12 @@ export class RoomEncryption {
         // share the new megolm session if needed
         if (megolmResult.roomKeyMessage) {
             await this._deviceTracker.trackRoom(this._room);
-            const devices = await this._deviceTracker.deviceIdentitiesForTrackedRoom(this._room.id, hsApi);
-            const messages = await this._olmEncryption.encrypt(
-                "m.room_key", megolmResult.roomKeyMessage, devices, hsApi);
-            await this._sendMessagesToDevices(ENCRYPTED_TYPE, messages, hsApi);
+            const devices = await this._deviceTracker.devicesForTrackedRoom(this._room.id, hsApi);
+            await this._sendRoomKey(megolmResult.roomKeyMessage, devices, hsApi);
+            // if we happen to rotate the session before we have sent newly joined members the room key
+            // then mark those members as not needing the key anymore
+            const userIds = Array.from(devices.reduce((set, device) => set.add(device.userId), new Set()));
+            await this._clearNeedsRoomKeyFlag(userIds);
         }
         return {
             type: ENCRYPTED_TYPE,
@@ -125,6 +128,58 @@ export class RoomEncryption {
         };
     }
 
+    async shareRoomKeyForMemberChanges(memberChanges, hsApi) {
+        const pendingUserIds = [];
+        for (const m of memberChanges.values()) {
+            if (m.member.needsRoomKey) {
+                pendingUserIds.push(m.userId);
+            }
+        }
+        return await this._shareRoomKey(pendingUserIds, hsApi);
+    }
+
+    async _shareRoomKey(userIds, hsApi) {
+        if (userIds.length === 0) {
+            return;
+        }
+        const readRoomKeyTxn = await this._storage.readTxn([this._storage.storeNames.outboundGroupSessions]);
+        const roomKeyMessage = await this._megolmEncryption.createRoomKeyMessage(this._room.id, readRoomKeyTxn);
+        // no room key if we haven't created a session yet
+        // (or we removed it and will create a new one on the next send)
+        if (roomKeyMessage) {
+            const devices = await this._deviceTracker.devicesForRoomMembers(this._room.id, userIds, hsApi);
+            await this._sendRoomKey(roomKeyMessage, devices, hsApi);
+            const actuallySentUserIds = Array.from(devices.reduce((set, device) => set.add(device.userId), new Set()));
+            await this._clearNeedsRoomKeyFlag(actuallySentUserIds);
+        } else {
+            // we don't have a session yet, clear them all
+            await this._clearNeedsRoomKeyFlag(userIds);
+        }
+    }
+
+    async _clearNeedsRoomKeyFlag(userIds) {
+        const txn = await this._storage.readWriteTxn([this._storage.storeNames.roomMembers]);
+        try {
+            await Promise.all(userIds.map(async userId => {
+                const memberData = await txn.roomMembers.get(this._room.id, userId);
+                if (memberData.needsRoomKey) {
+                    memberData.needsRoomKey = false;
+                    txn.roomMembers.set(memberData);
+                }
+            }));
+        } catch (err) {
+            txn.abort();
+            throw err;
+        }
+        await txn.complete();
+    }
+
+    async _sendRoomKey(roomKeyMessage, devices, hsApi) {
+        const messages = await this._olmEncryption.encrypt(
+            "m.room_key", roomKeyMessage, devices, hsApi);
+        await this._sendMessagesToDevices(ENCRYPTED_TYPE, messages, hsApi);
+    }
+
     async _sendMessagesToDevices(type, messages, hsApi) {
         const messagesByUser = groupBy(messages, message => message.device.userId);
         const payload = {
diff --git a/src/matrix/e2ee/megolm/Encryption.js b/src/matrix/e2ee/megolm/Encryption.js
index 9b7df4eb..cb0dddf8 100644
--- a/src/matrix/e2ee/megolm/Encryption.js
+++ b/src/matrix/e2ee/megolm/Encryption.js
@@ -30,6 +30,19 @@ export class Encryption {
         txn.outboundGroupSessions.remove(roomId);
     }
 
+    async createRoomKeyMessage(roomId, txn) {
+        let sessionEntry = await txn.outboundGroupSessions.get(roomId);
+        if (sessionEntry) {
+            const session = new this._olm.OutboundGroupSession();
+            try {
+                session.unpickle(this._pickleKey, sessionEntry.session);
+                return this._createRoomKeyMessage(session, roomId);
+            } finally {
+                session.free();
+            }
+        }
+    }
+
     /**
      * Encrypts a message with megolm
      * @param  {string} roomId           
@@ -127,12 +140,9 @@ export class Encryption {
             session_id: session.session_id(),
             session_key: session.session_key(),
             algorithm: MEGOLM_ALGORITHM,
-            // if we need to do this, do we need to create
-            // the room key message after or before having encrypted
-            // with the new session? I guess before as we do now
-            // because the chain_index is where you should start decrypting?
-            // 
-            // chain_index: session.message_index()
+            // chain_index is ignored by element-web if not all clients
+            // but let's send it anyway, as element-web does so
+            chain_index: session.message_index()
         }
     }
 

From c158e3da771410fab72dca25c883aee736321fbb Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 14:37:24 +0200
Subject: [PATCH 127/173] support running afterSyncCompleted step on rooms as
 well

and make it in parallel with next sync request
---
 src/matrix/Sync.js | 119 ++++++++++++++++++++++++++++++---------------
 1 file changed, 79 insertions(+), 40 deletions(-)

diff --git a/src/matrix/Sync.js b/src/matrix/Sync.js
index 4198618a..1295e7db 100644
--- a/src/matrix/Sync.js
+++ b/src/matrix/Sync.js
@@ -87,12 +87,16 @@ export class Sync {
     }
 
     async _syncLoop(syncToken) {
+        let afterSyncCompletedPromise = Promise.resolve();
         // if syncToken is falsy, it will first do an initial sync ... 
         while(this._status.get() !== SyncStatus.Stopped) {
+            let roomChanges;
             try {
                 console.log(`starting sync request with since ${syncToken} ...`);
                 const timeout = syncToken ? INCREMENTAL_TIMEOUT : undefined; 
-                syncToken = await this._syncRequest(syncToken, timeout);
+                const syncResult = await this._syncRequest(syncToken, timeout, afterSyncCompletedPromise);
+                syncToken = syncResult.syncToken;
+                roomChanges = syncResult.roomChanges;
                 this._status.set(SyncStatus.Syncing);
             } catch (err) {
                 if (!(err instanceof AbortError)) {
@@ -101,18 +105,39 @@ export class Sync {
                 }
             }
             if (!this._error) {
-                try {
-                    // TODO: run this in parallel with the next sync request
-                    await this._session.afterSyncCompleted();
-                } catch (err) {
-                    console.error("error during after sync completed, continuing to sync.",  err.stack);
-                    // swallowing error here apart from logging
-                }
+                afterSyncCompletedPromise = this._runAfterSyncCompleted(roomChanges);
             }
         }
     }
 
-    async _syncRequest(syncToken, timeout) {
+    async _runAfterSyncCompleted(roomChanges) {
+        const sessionPromise = (async () => {
+            try {
+                await this._session.afterSyncCompleted();
+            } catch (err) {
+                console.error("error during session afterSyncCompleted, continuing",  err.stack);
+            }
+        })();
+        let allPromises = [sessionPromise];
+
+        const roomsNeedingAfterSyncCompleted = roomChanges.filter(rc => {
+            return rc.changes.needsAfterSyncCompleted;
+        });
+        if (roomsNeedingAfterSyncCompleted.length) {
+            allPromises = allPromises.concat(roomsNeedingAfterSyncCompleted.map(async ({room, changes}) => {
+                try {
+                    await room.afterSyncCompleted(changes);
+                } catch (err) {
+                    console.error(`error during room ${room.id} afterSyncCompleted, continuing`,  err.stack);
+                }
+            }));
+        }
+        // run everything in parallel,
+        // we don't want to delay the next sync too much
+        await Promise.all(allPromises);
+    }
+
+    async _syncRequest(syncToken, timeout, prevAfterSyncCompletedPromise) {
         let {syncFilterId} = this._session;
         if (typeof syncFilterId !== "string") {
             this._currentRequest = this._hsApi.createFilter(this._session.user.id, {room: {state: {lazy_load_members: true}}});
@@ -121,43 +146,20 @@ export class Sync {
         const totalRequestTimeout = timeout + (80 * 1000);  // same as riot-web, don't get stuck on wedged long requests
         this._currentRequest = this._hsApi.sync(syncToken, syncFilterId, timeout, {timeout: totalRequestTimeout});
         const response = await this._currentRequest.response();
+        // wait here for the afterSyncCompleted step of the previous sync to complete
+        // before we continue processing this sync response
+        await prevAfterSyncCompletedPromise;
+
         const isInitialSync = !syncToken;
         syncToken = response.next_batch;
-        const storeNames = this._storage.storeNames;
-        const syncTxn = await this._storage.readWriteTxn([
-            storeNames.session,
-            storeNames.roomSummary,
-            storeNames.roomState,
-            storeNames.roomMembers,
-            storeNames.timelineEvents,
-            storeNames.timelineFragments,
-            storeNames.pendingEvents,
-            storeNames.userIdentities,
-            storeNames.inboundGroupSessions,
-            storeNames.groupSessionDecryptions,
-            storeNames.deviceIdentities,
-        ]);
-        const roomChanges = [];
+        const syncTxn = await this._openSyncTxn();
+        let roomChanges = [];
         let sessionChanges;
         try {
             // to_device
             // presence
             if (response.rooms) {
-                const promises = parseRooms(response.rooms, async (roomId, roomResponse, membership) => {
-                    // ignore rooms with empty timelines during initial sync,
-                    // see https://github.com/vector-im/hydrogen-web/issues/15
-                    if (isInitialSync && timelineIsEmpty(roomResponse)) {
-                        return;
-                    }
-                    let room = this._session.rooms.get(roomId);
-                    if (!room) {
-                        room = this._session.createRoom(roomId);
-                    }
-                    console.log(` * applying sync response to room ${roomId} ...`);
-                    const changes = await room.writeSync(roomResponse, membership, isInitialSync, syncTxn);
-                    roomChanges.push({room, changes});
-                });
-                await Promise.all(promises);
+                roomChanges = await this._writeRoomResponses(response.rooms, isInitialSync, syncTxn);
             }
             sessionChanges = await this._session.writeSync(response, syncFilterId, roomChanges, syncTxn);
         } catch(err) {
@@ -182,7 +184,44 @@ export class Sync {
             room.afterSync(changes);
         }
 
-        return syncToken;
+        return {syncToken, roomChanges};
+    }
+
+    async _writeRoomResponses(roomResponses, isInitialSync, syncTxn) {
+        const roomChanges = [];
+        const promises = parseRooms(roomResponses, async (roomId, roomResponse, membership) => {
+            // ignore rooms with empty timelines during initial sync,
+            // see https://github.com/vector-im/hydrogen-web/issues/15
+            if (isInitialSync && timelineIsEmpty(roomResponse)) {
+                return;
+            }
+            let room = this._session.rooms.get(roomId);
+            if (!room) {
+                room = this._session.createRoom(roomId);
+            }
+            console.log(` * applying sync response to room ${roomId} ...`);
+            const changes = await room.writeSync(roomResponse, membership, isInitialSync, syncTxn);
+            roomChanges.push({room, changes});
+        });
+        await Promise.all(promises);
+        return roomChanges;
+    }
+
+    async _openSyncTxn() {
+        const storeNames = this._storage.storeNames;
+        return await this._storage.readWriteTxn([
+            storeNames.session,
+            storeNames.roomSummary,
+            storeNames.roomState,
+            storeNames.roomMembers,
+            storeNames.timelineEvents,
+            storeNames.timelineFragments,
+            storeNames.pendingEvents,
+            storeNames.userIdentities,
+            storeNames.inboundGroupSessions,
+            storeNames.groupSessionDecryptions,
+            storeNames.deviceIdentities,
+        ]);
     }
 
     stop() {

From 31d4b6f75de01e88d32329f70d2ff5e505c0d225 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 14:38:27 +0200
Subject: [PATCH 128/173] send room keys to newly joined members in
 afterSyncCompleted stage

---
 src/matrix/Session.js                            |  1 +
 src/matrix/e2ee/RoomEncryption.js                | 16 ++++++++++++++++
 src/matrix/room/Room.js                          | 14 +++++++++++++-
 src/matrix/storage/idb/QueryTarget.js            |  8 ++++++++
 src/matrix/storage/idb/stores/RoomMemberStore.js | 15 +++++++++++++++
 5 files changed, 53 insertions(+), 1 deletion(-)

diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index d48e0d00..63b954af 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -125,6 +125,7 @@ export class Session {
             olmEncryption: this._olmEncryption,
             megolmEncryption: this._megolmEncryption,
             megolmDecryption: this._megolmDecryption,
+            storage: this._storage,
             encryptionParams
         });
     }
diff --git a/src/matrix/e2ee/RoomEncryption.js b/src/matrix/e2ee/RoomEncryption.js
index cd918292..3538381f 100644
--- a/src/matrix/e2ee/RoomEncryption.js
+++ b/src/matrix/e2ee/RoomEncryption.js
@@ -128,6 +128,22 @@ export class RoomEncryption {
         };
     }
 
+    needsToShareKeys(memberChanges) {
+        for (const m of memberChanges.values()) {
+            if (m.member.needsRoomKey) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    async shareRoomKeyToPendingMembers(hsApi) {
+        // sucks to call this for all encrypted rooms on startup?
+        const txn = await this._storage.readTxn([this._storage.storeNames.roomMembers]);
+        const pendingUserIds = await txn.roomMembers.getUserIdsNeedingRoomKey(this._room.id);
+        return await this._shareRoomKey(pendingUserIds, hsApi);
+    }
+
     async shareRoomKeyForMemberChanges(memberChanges, hsApi) {
         const pendingUserIds = [];
         for (const m of memberChanges.values()) {
diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index b4e19a80..55bb1ccb 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -157,7 +157,8 @@ export class Room extends EventEmitter {
             newLiveKey,
             removedPendingEvents,
             memberChanges,
-            heroChanges
+            heroChanges,
+            needsAfterSyncCompleted: this._roomEncryption?.needsToShareKeys(memberChanges)
         };
     }
 
@@ -204,6 +205,17 @@ export class Room extends EventEmitter {
         }
 	}
 
+    /**
+     * Only called if the result of writeSync had `needsAfterSyncCompleted` set.
+     * Can be used to do longer running operations that resulted from the last sync,
+     * like network operations.
+     */
+    async afterSyncCompleted({memberChanges}) {
+        if (this._roomEncryption) {
+            await this._roomEncryption.shareRoomKeyForMemberChanges(memberChanges, this._hsApi);
+        }
+    }
+
     /** @package */
     resumeSending() {
         this._sendQueue.resumeSending();
diff --git a/src/matrix/storage/idb/QueryTarget.js b/src/matrix/storage/idb/QueryTarget.js
index fd3050bd..6d11932f 100644
--- a/src/matrix/storage/idb/QueryTarget.js
+++ b/src/matrix/storage/idb/QueryTarget.js
@@ -187,6 +187,14 @@ export class QueryTarget {
         return results;
     }
 
+    async iterateWhile(range, predicate) {
+        const cursor = this._openCursor(range, "next");
+        await iterateCursor(cursor, (value) => {
+            const passesPredicate = predicate(value);
+            return {done: !passesPredicate};
+        });
+    }
+
     async _find(range, predicate, direction) {
         const cursor = this._openCursor(range, direction);
         let result;
diff --git a/src/matrix/storage/idb/stores/RoomMemberStore.js b/src/matrix/storage/idb/stores/RoomMemberStore.js
index be2b16ec..6d354dd9 100644
--- a/src/matrix/storage/idb/stores/RoomMemberStore.js
+++ b/src/matrix/storage/idb/stores/RoomMemberStore.js
@@ -60,4 +60,19 @@ export class RoomMemberStore {
         });
         return userIds;
     }
+
+    async getUserIdsNeedingRoomKey(roomId) {
+        const userIds = [];
+        const range = IDBKeyRange.lowerBound(encodeKey(roomId, ""));
+        await this._roomMembersStore.iterateWhile(range, member => {
+            if (member.roomId !== roomId) {
+                return false;
+            }
+            if (member.needsRoomKey) {
+                userIds.push(member.userId);
+            }
+            return true;
+        });
+        return userIds;
+    }
 }

From 1aa044667c812da212324761a3d2824a2ee5f135 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 14:39:07 +0200
Subject: [PATCH 129/173] try sending out pending room keys after first sync

---
 src/matrix/Session.js   |  2 +-
 src/matrix/room/Room.js | 11 ++++++++++-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index 63b954af..b4cbf3b2 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -217,7 +217,7 @@ export class Session {
 
         this._sendScheduler.start();
         for (const [, room] of this._rooms) {
-            room.resumeSending();
+            room.start();
         }
     }
 
diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index 55bb1ccb..3c417082 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -217,7 +217,16 @@ export class Room extends EventEmitter {
     }
 
     /** @package */
-    resumeSending() {
+    async start() {
+        if (this._roomEncryption) {
+            try {
+                // if we got interrupted last time sending keys to newly joined members
+                await this._roomEncryption.shareRoomKeyToPendingMembers(this._hsApi);
+            } catch (err) {
+                // we should not throw here
+                console.error(`could not send out pending room keys for room ${this.id}`, err.stack);
+            }
+        }
         this._sendQueue.resumeSending();
     }
 

From 5e65eb10ef3e0df34d5b4dc60357426dd4e75783 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 14:39:33 +0200
Subject: [PATCH 130/173] docs

---
 src/matrix/room/Room.js | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index 3c417082..8727edd9 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -162,6 +162,11 @@ export class Room extends EventEmitter {
         };
     }
 
+    /**
+     * @package
+     * Called with the changes returned from `writeSync` to apply them and emit changes.
+     * No storage or network operations should be done here.
+     */
     /** @package */
     afterSync({summaryChanges, newTimelineEntries, newLiveKey, removedPendingEvents, memberChanges, heroChanges}) {
         this._syncWriter.afterSync(newLiveKey);

From 7bba83aa9eda05cc151efe29120c2cd650b87cea Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 15:00:00 +0200
Subject: [PATCH 131/173] add outbound session store to sync txn

---
 src/matrix/Sync.js | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/matrix/Sync.js b/src/matrix/Sync.js
index 1295e7db..598b9169 100644
--- a/src/matrix/Sync.js
+++ b/src/matrix/Sync.js
@@ -221,6 +221,8 @@ export class Sync {
             storeNames.inboundGroupSessions,
             storeNames.groupSessionDecryptions,
             storeNames.deviceIdentities,
+            // to discard outbound session when somebody leaves a room
+            storeNames.outboundGroupSessions
         ]);
     }
 

From 5a8aac57ac0487fb915801fc6e53d5cce838dc1f Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 15:00:20 +0200
Subject: [PATCH 132/173] there might not be a member yet

---
 src/matrix/room/timeline/persistence/SyncWriter.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/matrix/room/timeline/persistence/SyncWriter.js b/src/matrix/room/timeline/persistence/SyncWriter.js
index 0d9bea9d..d265782f 100644
--- a/src/matrix/room/timeline/persistence/SyncWriter.js
+++ b/src/matrix/room/timeline/persistence/SyncWriter.js
@@ -107,7 +107,7 @@ export class SyncWriter {
                 if (trackNewlyJoined) {
                     const existingMemberData = await txn.roomMembers.get(this._roomId, userId);
                     // mark new members so we know who needs our the room key for our outbound megolm session
-                    member.needsRoomKey = existingMemberData.needsRoomKey || memberChange.hasJoined;
+                    member.needsRoomKey = existingMemberData?.needsRoomKey || memberChange.hasJoined;
                 }
                 txn.roomMembers.set(member.serialize());
                 return memberChange;

From 650df6fea8ef323064dbc305724e6ea10d5742be Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 15:00:29 +0200
Subject: [PATCH 133/173] forgot await

---
 src/matrix/room/timeline/persistence/SyncWriter.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/matrix/room/timeline/persistence/SyncWriter.js b/src/matrix/room/timeline/persistence/SyncWriter.js
index d265782f..130b22d1 100644
--- a/src/matrix/room/timeline/persistence/SyncWriter.js
+++ b/src/matrix/room/timeline/persistence/SyncWriter.js
@@ -219,7 +219,7 @@ export class SyncWriter {
         }
         // important this happens before _writeTimeline so
         // members are available in the transaction
-        const memberChanges = this._writeStateEvents(roomResponse, trackNewlyJoined, txn);
+        const memberChanges = await this._writeStateEvents(roomResponse, trackNewlyJoined, txn);
         const timelineResult = await this._writeTimeline(entries, timeline, currentKey, trackNewlyJoined, txn);
         currentKey = timelineResult.currentKey;
         // merge member changes from state and timeline, giving precedence to the latter

From 65660a1e3bf5c455795a395890b5142cbbac07a9 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 15:06:44 +0200
Subject: [PATCH 134/173] remove double jsdoc

---
 src/matrix/room/Room.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index 8727edd9..daec1dd6 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -167,7 +167,6 @@ export class Room extends EventEmitter {
      * Called with the changes returned from `writeSync` to apply them and emit changes.
      * No storage or network operations should be done here.
      */
-    /** @package */
     afterSync({summaryChanges, newTimelineEntries, newLiveKey, removedPendingEvents, memberChanges, heroChanges}) {
         this._syncWriter.afterSync(newLiveKey);
         if (!this._summary.encryption && summaryChanges.encryption && !this._roomEncryption) {

From b653022a5a75ee435fc662b6abb763ff1a25db27 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 15:24:36 +0200
Subject: [PATCH 135/173] do store our own device, otherwise need special case
 verifying own msgs

---
 src/matrix/e2ee/DeviceTracker.js | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/src/matrix/e2ee/DeviceTracker.js b/src/matrix/e2ee/DeviceTracker.js
index 0045522f..6fa1333c 100644
--- a/src/matrix/e2ee/DeviceTracker.js
+++ b/src/matrix/e2ee/DeviceTracker.js
@@ -205,10 +205,6 @@ export class DeviceTracker {
                 if (typeof ed25519Key !== "string" || typeof curve25519Key !== "string") {
                     return false;
                 }
-                // don't store our own device
-                if (userId === this._ownUserId && deviceId === this._ownDeviceId) {
-                    return false;
-                }
                 return this._hasValidSignature(deviceKeys);
             });
             const verifiedKeys = verifiedEntries.map(([, deviceKeys]) => deviceKeys);
@@ -287,7 +283,7 @@ export class DeviceTracker {
         if (queriedDevices && queriedDevices.length) {
             flattenedDevices = flattenedDevices.concat(queriedDevices);
         }
-        // filter out our own devices if it got in somehow (even though we should not store it)
+        // filter out our own device
         const devices = flattenedDevices.filter(device => {
             return !(device.userId === this._ownUserId && device.deviceId === this._ownDeviceId);
         });

From 773cb3420f93d219a31177ea1e7c155ddf6841b7 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 17:14:23 +0200
Subject: [PATCH 136/173] ignore duplicate curve25519 keys in /keys/query
 response

---
 src/matrix/e2ee/DeviceTracker.js | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/matrix/e2ee/DeviceTracker.js b/src/matrix/e2ee/DeviceTracker.js
index 6fa1333c..1ee704c8 100644
--- a/src/matrix/e2ee/DeviceTracker.js
+++ b/src/matrix/e2ee/DeviceTracker.js
@@ -190,6 +190,7 @@ export class DeviceTracker {
     }
 
     _filterVerifiedDeviceKeys(keyQueryDeviceKeysResponse) {
+        const curve25519Keys = new Set();
         const verifiedKeys = Object.entries(keyQueryDeviceKeysResponse).map(([userId, keysByDevice]) => {
             const verifiedEntries = Object.entries(keysByDevice).filter(([deviceId, deviceKeys]) => {
                 const deviceIdOnKeys = deviceKeys["device_id"];
@@ -205,6 +206,11 @@ export class DeviceTracker {
                 if (typeof ed25519Key !== "string" || typeof curve25519Key !== "string") {
                     return false;
                 }
+                if (curve25519Keys.has(curve25519Key)) {
+                    console.warn("ignoring device with duplicate curve25519 key in /keys/query response", deviceKeys);
+                    return false;
+                }
+                curve25519Keys.add(curve25519Key);
                 return this._hasValidSignature(deviceKeys);
             });
             const verifiedKeys = verifiedEntries.map(([, deviceKeys]) => deviceKeys);

From 36a8ec0110008a3b5fe112496aa21a12c409d894 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 17:16:01 +0200
Subject: [PATCH 137/173] dont attempt to decrypt redacted events

this will show them as undecryptable for now though
---
 src/matrix/e2ee/RoomEncryption.js | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/matrix/e2ee/RoomEncryption.js b/src/matrix/e2ee/RoomEncryption.js
index 3538381f..c8f993e5 100644
--- a/src/matrix/e2ee/RoomEncryption.js
+++ b/src/matrix/e2ee/RoomEncryption.js
@@ -56,6 +56,9 @@ export class RoomEncryption {
     }
 
     async decrypt(event, isSync, isTimelineOpen, retryData, txn) {
+        if (event.redacted_because || event.unsigned?.redacted_because) {
+            return;
+        }
         if (event.content?.algorithm !== MEGOLM_ALGORITHM) {
             throw new Error("Unsupported algorithm: " + event.content?.algorithm);
         }

From 4c1aaaf41614a361199befadbce8003808ae5b52 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 17:16:34 +0200
Subject: [PATCH 138/173] show "setting up encryption keys..." step during
 login

---
 src/domain/SessionLoadViewModel.js | 2 ++
 src/matrix/SessionContainer.js     | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/src/domain/SessionLoadViewModel.js b/src/domain/SessionLoadViewModel.js
index e747ce00..fbb96986 100644
--- a/src/domain/SessionLoadViewModel.js
+++ b/src/domain/SessionLoadViewModel.js
@@ -127,6 +127,8 @@ export class SessionLoadViewModel extends ViewModel {
                             return `Something went wrong while checking your login and password.`;
                     }
                     break;
+                case LoadStatus.SessionSetup:
+                    return `Setting up your encryption keys…`;
                 case LoadStatus.Loading:
                     return `Loading your conversations…`;
                 case LoadStatus.FirstSync:
diff --git a/src/matrix/SessionContainer.js b/src/matrix/SessionContainer.js
index 9b64115f..c07190eb 100644
--- a/src/matrix/SessionContainer.js
+++ b/src/matrix/SessionContainer.js
@@ -28,6 +28,7 @@ export const LoadStatus = createEnum(
     "Login",
     "LoginFailed",
     "Loading",
+    "SessionSetup", // upload e2ee keys, ...
     "Migrating",    //not used atm, but would fit here
     "FirstSync",
     "Error",
@@ -154,6 +155,7 @@ export class SessionContainer {
         this._session = new Session({storage: this._storage,
             sessionInfo: filteredSessionInfo, hsApi, olm, clock: this._clock});
         await this._session.load();
+        this._status.set(LoadStatus.SessionSetup);
         await this._session.beforeFirstSync(isNewLogin);
         
         this._sync = new Sync({hsApi, storage: this._storage, session: this._session});

From 4a2faed1984b6d5a517d96fa962307e52f0badcb Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 18:27:12 +0200
Subject: [PATCH 139/173] don't assume roomKeys is an array

---
 src/matrix/DeviceMessageHandler.js | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js
index e0398256..4c7d0e75 100644
--- a/src/matrix/DeviceMessageHandler.js
+++ b/src/matrix/DeviceMessageHandler.js
@@ -60,10 +60,12 @@ export class DeviceMessageHandler {
     }
 
     _applyDecryptChanges(rooms, {roomKeys}) {
-        const roomKeysByRoom = groupBy(roomKeys, s => s.roomId);
-        for (const [roomId, roomKeys] of roomKeysByRoom) {
-            const room = rooms.get(roomId);
-            room?.notifyRoomKeys(roomKeys);
+        if (roomKeys && roomKeys.length) {
+            const roomKeysByRoom = groupBy(roomKeys, s => s.roomId);
+            for (const [roomId, roomKeys] of roomKeysByRoom) {
+                const room = rooms.get(roomId);
+                room?.notifyRoomKeys(roomKeys);
+            }
         }
     }
 

From 9a7abb18997a4783a99125749eb9f875352a5e4d Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 18:27:35 +0200
Subject: [PATCH 140/173] make logic more explicit

---
 src/matrix/e2ee/DeviceTracker.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/matrix/e2ee/DeviceTracker.js b/src/matrix/e2ee/DeviceTracker.js
index 1ee704c8..8b90caf0 100644
--- a/src/matrix/e2ee/DeviceTracker.js
+++ b/src/matrix/e2ee/DeviceTracker.js
@@ -291,7 +291,8 @@ export class DeviceTracker {
         }
         // filter out our own device
         const devices = flattenedDevices.filter(device => {
-            return !(device.userId === this._ownUserId && device.deviceId === this._ownDeviceId);
+            const isOwnDevice = device.userId === this._ownUserId && device.deviceId === this._ownDeviceId;
+            return !isOwnDevice;
         });
         return devices;
     }

From 10b5614fd9a28851cbdfb57b2daef627a629d462 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 18:27:49 +0200
Subject: [PATCH 141/173] m.dummy events don't have content

---
 src/matrix/e2ee/olm/Decryption.js | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/matrix/e2ee/olm/Decryption.js b/src/matrix/e2ee/olm/Decryption.js
index b3f47cd6..fc16852a 100644
--- a/src/matrix/e2ee/olm/Decryption.js
+++ b/src/matrix/e2ee/olm/Decryption.js
@@ -188,9 +188,6 @@ export class Decryption {
         if (!payload.type) {
             throw new DecryptionError("missing type on payload", event, {payload});
         }
-        if (!payload.content) {
-            throw new DecryptionError("missing content on payload", event, {payload});
-        }
         if (typeof payload.keys?.ed25519 !== "string") {
             throw new DecryptionError("Missing or invalid claimed ed25519 key on payload", event, {payload});
         }

From 4ca5ff9b9f2f4b9095e3a70e19a94ed718841251 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 18:30:06 +0200
Subject: [PATCH 142/173] only load 50 olm sessions at once

---
 src/matrix/e2ee/olm/Encryption.js | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/src/matrix/e2ee/olm/Encryption.js b/src/matrix/e2ee/olm/Encryption.js
index 680ce154..eb291c40 100644
--- a/src/matrix/e2ee/olm/Encryption.js
+++ b/src/matrix/e2ee/olm/Encryption.js
@@ -29,6 +29,9 @@ function findFirstSessionId(sessionIds) {
 }
 
 const OTK_ALGORITHM = "signed_curve25519";
+// only encrypt this amount of olm messages at once otherwise we run out of wasm memory
+// with all the sessions loaded at the same time
+const MAX_BATCH_SIZE = 50;
 
 export class Encryption {
     constructor({account, olm, olmUtil, ownUserId, storage, now, pickleKey, senderKeyLock}) {
@@ -43,6 +46,16 @@ export class Encryption {
     }
 
     async encrypt(type, content, devices, hsApi) {
+        let messages = [];
+        for (let i = 0; i < devices.length ; i += MAX_BATCH_SIZE) {
+            const batchDevices = devices.slice(i, i + MAX_BATCH_SIZE);
+            const batchMessages = await this._encryptForMaxDevices(type, content, batchDevices, hsApi);
+            messages = messages.concat(batchMessages);
+        }
+        return messages;
+    }
+
+    async _encryptForMaxDevices(type, content, devices, hsApi) {
         // TODO: see if we can only hold some of the locks until after the /keys/claim call (if needed) 
         // take a lock on all senderKeys so decryption and other calls to encrypt (should not happen)
         // don't modify the sessions at the same time

From da4b710e49e78208d2de7e9e7dbff30f72f7f0a3 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 18:30:45 +0200
Subject: [PATCH 143/173] don't return promise here, not used

---
 src/matrix/storage/idb/stores/DeviceIdentityStore.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/matrix/storage/idb/stores/DeviceIdentityStore.js b/src/matrix/storage/idb/stores/DeviceIdentityStore.js
index d1bf7eaa..d3aba963 100644
--- a/src/matrix/storage/idb/stores/DeviceIdentityStore.js
+++ b/src/matrix/storage/idb/stores/DeviceIdentityStore.js
@@ -36,7 +36,7 @@ export class DeviceIdentityStore {
 
     set(deviceIdentity) {
         deviceIdentity.key = encodeKey(deviceIdentity.userId, deviceIdentity.deviceId);
-        return this._store.put(deviceIdentity);
+        this._store.put(deviceIdentity);
     }
 
     getByCurve25519Key(curve25519Key) {

From 0ed2d1488744d752499aabdf39ccc874f79be45a Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 18:32:12 +0200
Subject: [PATCH 144/173] log OTK claim failures

---
 src/matrix/e2ee/olm/Encryption.js | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/matrix/e2ee/olm/Encryption.js b/src/matrix/e2ee/olm/Encryption.js
index eb291c40..919acc45 100644
--- a/src/matrix/e2ee/olm/Encryption.js
+++ b/src/matrix/e2ee/olm/Encryption.js
@@ -182,6 +182,9 @@ export class Encryption {
             timeout: 10000,
             one_time_keys: oneTimeKeys
         }).response();
+        if (Object.keys(claimResponse.failures).length) {
+            console.warn("failures for claiming one time keys", oneTimeKeys, claimResponse.failures);
+        }
         // TODO: log claimResponse.failures
         const userKeyMap = claimResponse?.["one_time_keys"];
         return this._verifyAndCreateOTKTargets(userKeyMap, devicesByUser);

From 1f9d6191c2e34b9cb427c5c272ea96383a6a15f5 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Tue, 8 Sep 2020 18:32:51 +0200
Subject: [PATCH 145/173] this happens often when room is not tracked yet, so
 don't log

---
 src/matrix/e2ee/DeviceTracker.js | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/matrix/e2ee/DeviceTracker.js b/src/matrix/e2ee/DeviceTracker.js
index 8b90caf0..aef62e10 100644
--- a/src/matrix/e2ee/DeviceTracker.js
+++ b/src/matrix/e2ee/DeviceTracker.js
@@ -51,8 +51,6 @@ export class DeviceTracker {
                 if (user) {
                     user.deviceTrackingStatus = TRACKING_STATUS_OUTDATED;
                     userIdentities.set(user);
-                } else {
-                    console.warn("changed device userid not found", userId);
                 }
             }));
         }

From 212efe823c6ab01df14da89e72b04e1bfca79866 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Wed, 9 Sep 2020 09:50:03 +0200
Subject: [PATCH 146/173] fix memberlist not containing all members

we were using the prev_batch of the last sync to pass to
/members, but this points at the timeline *before* the last
sync, so wouldn't contain all members. Use the sync token instead.
---
 src/matrix/Session.js           |  4 +++-
 src/matrix/room/Room.js         |  4 +++-
 src/matrix/room/RoomSummary.js  | 15 +++------------
 src/matrix/room/members/load.js |  4 ++--
 4 files changed, 11 insertions(+), 16 deletions(-)

diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index b4cbf3b2..19725b58 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -51,12 +51,13 @@ export class Session {
         this._olmEncryption = null;
         this._megolmEncryption = null;
         this._megolmDecryption = null;
+        this._getSyncToken = () => this.syncToken;
 
         if (olm) {
             this._olmUtil = new olm.Utility();
             this._deviceTracker = new DeviceTracker({
                 storage,
-                getSyncToken: () => this.syncToken,
+                getSyncToken: this._getSyncToken,
                 olmUtil: this._olmUtil,
                 ownUserId: sessionInfo.userId,
                 ownDeviceId: sessionInfo.deviceId,
@@ -241,6 +242,7 @@ export class Session {
     createRoom(roomId, pendingEvents) {
         const room = new Room({
             roomId,
+            getSyncToken: this._getSyncToken,
             storage: this._storage,
             emitCollectionChange: this._roomUpdateCallback,
             hsApi: this._hsApi,
diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index daec1dd6..b8454df7 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -28,7 +28,7 @@ import {Heroes} from "./members/Heroes.js";
 import {EventEntry} from "./timeline/entries/EventEntry.js";
 
 export class Room extends EventEmitter {
-	constructor({roomId, storage, hsApi, emitCollectionChange, sendScheduler, pendingEvents, user, createRoomEncryption}) {
+	constructor({roomId, storage, hsApi, emitCollectionChange, sendScheduler, pendingEvents, user, createRoomEncryption, getSyncToken}) {
         super();
         this._roomId = roomId;
         this._storage = storage;
@@ -44,6 +44,7 @@ export class Room extends EventEmitter {
         this._memberList = null;
         this._createRoomEncryption = createRoomEncryption;
         this._roomEncryption = null;
+        this._getSyncToken = getSyncToken;
 	}
 
     async notifyRoomKeys(roomKeys) {
@@ -270,6 +271,7 @@ export class Room extends EventEmitter {
                 roomId: this._roomId,
                 hsApi: this._hsApi,
                 storage: this._storage,
+                syncToken: this._getSyncToken(),
                 // to handle race between /members and /sync
                 setChangedMembersMap: map => this._changedMembersDuringSync = map,
             });
diff --git a/src/matrix/room/RoomSummary.js b/src/matrix/room/RoomSummary.js
index 3b550527..270fa690 100644
--- a/src/matrix/room/RoomSummary.js
+++ b/src/matrix/room/RoomSummary.js
@@ -31,12 +31,8 @@ function applySyncResponse(data, roomResponse, membership, isInitialSync, isTime
     if (roomResponse.state) {
         data = roomResponse.state.events.reduce(processStateEvent, data);
     }
-    if (roomResponse.timeline) {
-        const {timeline} = roomResponse;
-        if (timeline.prev_batch) {
-            data = data.cloneIfNeeded();
-            data.lastPaginationToken = timeline.prev_batch;
-        }
+    const {timeline} = roomResponse;
+    if (timeline && Array.isArray(timeline.events)) {
         data = timeline.events.reduce((data, event) => {
             if (typeof event.state_key === "string") {
                 return processStateEvent(data, event);
@@ -150,7 +146,6 @@ class SummaryData {
         this.canonicalAlias = copy ? copy.canonicalAlias : null;
         this.hasFetchedMembers = copy ? copy.hasFetchedMembers : false;
         this.isTrackingMembers = copy ? copy.isTrackingMembers : false;
-        this.lastPaginationToken = copy ? copy.lastPaginationToken : null;
         this.avatarUrl = copy ? copy.avatarUrl : null;
         this.notificationCount = copy ? copy.notificationCount : 0;
         this.highlightCount = copy ? copy.highlightCount : 0;
@@ -244,11 +239,7 @@ export class RoomSummary {
     get isTrackingMembers() {
         return this._data.isTrackingMembers;
     }
-
-    get lastPaginationToken() {
-        return this._data.lastPaginationToken;
-    }
-
+    
     get tags() {
         return this._data.tags;
     }
diff --git a/src/matrix/room/members/load.js b/src/matrix/room/members/load.js
index 667bec96..18fc4eb4 100644
--- a/src/matrix/room/members/load.js
+++ b/src/matrix/room/members/load.js
@@ -25,13 +25,13 @@ async function loadMembers({roomId, storage}) {
     return memberDatas.map(d => new RoomMember(d));
 }
 
-async function fetchMembers({summary, roomId, hsApi, storage, setChangedMembersMap}) {
+async function fetchMembers({summary, syncToken, roomId, hsApi, storage, setChangedMembersMap}) {
     // if any members are changed by sync while we're fetching members,
     // they will end up here, so we check not to override them
     const changedMembersDuringSync = new Map();
     setChangedMembersMap(changedMembersDuringSync);
     
-    const memberResponse = await hsApi.members(roomId, {at: summary.lastPaginationToken}).response();
+    const memberResponse = await hsApi.members(roomId, {at: syncToken}).response();
 
     const txn = await storage.readWriteTxn([
         storage.storeNames.roomSummary,

From a18d2c0e78413b23c5cd987dbfe5be2ba45143f3 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Wed, 9 Sep 2020 09:51:48 +0200
Subject: [PATCH 147/173] update comment

---
 src/matrix/room/Room.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index b8454df7..a2b84717 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -49,7 +49,7 @@ export class Room extends EventEmitter {
 
     async notifyRoomKeys(roomKeys) {
         if (this._roomEncryption) {
-            // array of {data, source}
+            // array of {data, isSync}
             let retryEntries = this._roomEncryption.applyRoomKeys(roomKeys);
             let decryptedEntries = [];
             if (retryEntries.length) {

From f8e3a754711c44567e272298ea74f5b115fd7929 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Wed, 9 Sep 2020 10:22:29 +0200
Subject: [PATCH 148/173] fix typo

---
 src/domain/SessionLoadViewModel.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/domain/SessionLoadViewModel.js b/src/domain/SessionLoadViewModel.js
index fbb96986..0511d8a7 100644
--- a/src/domain/SessionLoadViewModel.js
+++ b/src/domain/SessionLoadViewModel.js
@@ -82,7 +82,7 @@ export class SessionLoadViewModel extends ViewModel {
             if (this._sessionContainer) {
                 this._sessionContainer.stop();
                 if (this._deleteSessionOnCancel) {
-                    await this._sessionContainer.deletSession();
+                    await this._sessionContainer.deleteSession();
                 }
                 this._sessionContainer = null;
             }

From 18a8f291dc4f7f5868ad375f72927a1aea156a92 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Wed, 9 Sep 2020 10:32:05 +0100
Subject: [PATCH 149/173] make build script work on windows

---
 scripts/build.mjs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/build.mjs b/scripts/build.mjs
index 7599d108..15d090be 100644
--- a/scripts/build.mjs
+++ b/scripts/build.mjs
@@ -358,7 +358,7 @@ async function copyFolder(srcRoot, dstRoot, filter) {
 
 function resource(relPath, content) {
     let fullPath = relPath;
-    if (!relPath.startsWith("/")) {
+    if (!path.isAbsolute(relPath)) {
         fullPath = path.join(targetDir, relPath);
     }
     const hash = contentHash(Buffer.from(content));

From a4c8e56ab0de4517107926f6b4ac3e7716f73239 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Wed, 9 Sep 2020 11:42:26 +0100
Subject: [PATCH 150/173] fix getKey not working on IE11

---
 src/matrix/storage/idb/QueryTarget.js              | 10 +++++++++-
 src/matrix/storage/idb/Store.js                    |  8 ++++++++
 src/matrix/storage/idb/stores/PendingEventStore.js |  8 +-------
 3 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/src/matrix/storage/idb/QueryTarget.js b/src/matrix/storage/idb/QueryTarget.js
index 6d11932f..c0c8ed2c 100644
--- a/src/matrix/storage/idb/QueryTarget.js
+++ b/src/matrix/storage/idb/QueryTarget.js
@@ -42,7 +42,15 @@ export class QueryTarget {
     }
 
     getKey(key) {
-        return reqAsPromise(this._target.getKey(key));
+        if (this._target.supports("getKey")) {
+            return reqAsPromise(this._target.getKey(key));
+        } else {
+            return reqAsPromise(this._target.get(key)).then(value => {
+                if (value) {
+                    return value[this._target.keyPath];
+                }
+            });
+        }
     }
 
     reduce(range, reducer, initialValue) {
diff --git a/src/matrix/storage/idb/Store.js b/src/matrix/storage/idb/Store.js
index 8cb8fd4c..a0ca0b96 100644
--- a/src/matrix/storage/idb/Store.js
+++ b/src/matrix/storage/idb/Store.js
@@ -23,6 +23,14 @@ class QueryTargetWrapper {
         this._qt = qt;
     }
 
+    get keyPath() {
+        if (this._qt.objectStore) {
+            return this._qt.objectStore.keyPath;
+        } else {
+            return this._qt.keyPath;
+        }
+    }
+
     supports(methodName) {
         return !!this._qt[methodName];
     }
diff --git a/src/matrix/storage/idb/stores/PendingEventStore.js b/src/matrix/storage/idb/stores/PendingEventStore.js
index 6492a272..00898311 100644
--- a/src/matrix/storage/idb/stores/PendingEventStore.js
+++ b/src/matrix/storage/idb/stores/PendingEventStore.js
@@ -52,13 +52,7 @@ export class PendingEventStore {
 
     async exists(roomId, queueIndex) {
         const keyRange = IDBKeyRange.only(encodeKey(roomId, queueIndex));
-        let key;
-        if (this._eventStore.supports("getKey")) {
-            key = await this._eventStore.getKey(keyRange);
-        } else {
-            const value = await this._eventStore.get(keyRange);
-            key = value && value.key;
-        }
+        const key = await this._eventStore.getKey(keyRange);
         return !!key;
     }
     

From 7c1f9dbed02c34a64a03cdedb76449925e5ab9b6 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Wed, 9 Sep 2020 16:28:43 +0200
Subject: [PATCH 151/173] split up megolm decryption so it can happen in
 multiple steps,see README

---
 src/matrix/e2ee/megolm/Decryption.js          | 187 +++++++-----------
 .../megolm/decryption/DecryptionChanges.js    |  78 ++++++++
 .../decryption/DecryptionPreparation.js       |  52 +++++
 src/matrix/e2ee/megolm/decryption/README.md   |   6 +
 .../megolm/decryption/ReplayDetectionEntry.js |  24 +++
 .../e2ee/megolm/decryption/SessionCache.js    |  68 +++++++
 .../megolm/decryption/SessionDecryption.js    |  75 +++++++
 .../e2ee/megolm/decryption/SessionInfo.js     |  44 +++++
 src/utils/mergeMap.js                         |  41 ++++
 9 files changed, 455 insertions(+), 120 deletions(-)
 create mode 100644 src/matrix/e2ee/megolm/decryption/DecryptionChanges.js
 create mode 100644 src/matrix/e2ee/megolm/decryption/DecryptionPreparation.js
 create mode 100644 src/matrix/e2ee/megolm/decryption/README.md
 create mode 100644 src/matrix/e2ee/megolm/decryption/ReplayDetectionEntry.js
 create mode 100644 src/matrix/e2ee/megolm/decryption/SessionCache.js
 create mode 100644 src/matrix/e2ee/megolm/decryption/SessionDecryption.js
 create mode 100644 src/matrix/e2ee/megolm/decryption/SessionInfo.js
 create mode 100644 src/utils/mergeMap.js

diff --git a/src/matrix/e2ee/megolm/Decryption.js b/src/matrix/e2ee/megolm/Decryption.js
index bd3665b3..d4352926 100644
--- a/src/matrix/e2ee/megolm/Decryption.js
+++ b/src/matrix/e2ee/megolm/Decryption.js
@@ -15,102 +15,101 @@ limitations under the License.
 */
 
 import {DecryptionError} from "../common.js";
-import {DecryptionResult} from "../DecryptionResult.js";
+import {groupBy} from "../../../utils/groupBy.js";
 
-const CACHE_MAX_SIZE = 10;
+import {SessionInfo} from "./decryption/SessionInfo.js";
+import {DecryptionPreparation} from "./decryption/DecryptionPreparation.js";
+import {SessionDecryption} from "./decryption/SessionDecryption.js";
+import {SessionCache} from "./decryption/SessionCache.js";
+
+function getSenderKey(event) {
+    return event.content?.["sender_key"];
+}
+
+function getSessionId(event) {
+    return event.content?.["session_id"];
+}
+
+function getCiphertext(event) {
+    return event.content?.ciphertext;
+}
 
 export class Decryption {
     constructor({pickleKey, olm}) {
         this._pickleKey = pickleKey;
         this._olm = olm;
+        // this._worker = new MessageHandler(new Worker("worker-2580578233.js"));
     }
 
-    createSessionCache() {
-        return new SessionCache();
+    createSessionCache(fallback) {
+        return new SessionCache(fallback);
     }
 
     /**
-     * [decrypt description]
+     * Reads all the state from storage to be able to decrypt the given events.
+     * Decryption can then happen outside of a storage transaction.
      * @param  {[type]} roomId       [description]
-     * @param  {[type]} event        [description]
+     * @param  {[type]} events        [description]
      * @param  {[type]} sessionCache [description]
      * @param  {[type]} txn          [description]
-     * @return {DecryptionResult?} the decrypted event result, or undefined if the session id is not known.
+     * @return {DecryptionPreparation}
      */
-    async decrypt(roomId, event, sessionCache, txn) {
-        const senderKey = event.content?.["sender_key"];
-        const sessionId = event.content?.["session_id"];
-        const ciphertext = event.content?.ciphertext;
+    async prepareDecryptAll(roomId, events, sessionCache, txn) {
+        const errors = new Map();
+        const validEvents = [];
 
-        if (
-            typeof senderKey !== "string" ||
-            typeof sessionId !== "string" ||
-            typeof ciphertext !== "string"
-        ) {
-            throw new DecryptionError("MEGOLM_INVALID_EVENT", event);
+        for (const event of events) {
+            const isValid = typeof getSenderKey(event) === "string" &&
+                            typeof getSessionId(event) === "string" &&
+                            typeof getCiphertext(event) === "string";
+            if (isValid) {
+                validEvents.push(event);
+            } else {
+                errors.set(event.event_id, new DecryptionError("MEGOLM_INVALID_EVENT", event))
+            }
         }
 
-        let session;
-        let claimedKeys;
-        const cacheEntry = sessionCache.get(roomId, senderKey, sessionId);
-        if (cacheEntry) {
-            session = cacheEntry.session;
-            claimedKeys = cacheEntry.claimedKeys;
-        } else {
+        const eventsBySession = groupBy(validEvents, event => {
+            return `${getSenderKey(event)}|${getSessionId(event)}`;
+        });
+
+        const sessionDecryptions = [];
+
+        await Promise.all(Array.from(eventsBySession.values()).map(async eventsForSession => {
+            const first = eventsForSession[0];
+            const senderKey = getSenderKey(first);
+            const sessionId = getSessionId(first);
+            const sessionInfo = await this._getSessionInfo(roomId, senderKey, sessionId, sessionCache, txn);
+            if (!sessionInfo) {
+                for (const event of eventsForSession) {
+                    errors.set(event.event_id, new DecryptionError("MEGOLM_NO_SESSION", event));
+                }
+            } else {
+                sessionDecryptions.push(new SessionDecryption(sessionInfo, eventsForSession));
+            }
+        }));
+
+        return new DecryptionPreparation(roomId, sessionDecryptions, errors);
+    }
+
+    async _getSessionInfo(roomId, senderKey, sessionId, sessionCache, txn) {
+        let sessionInfo;
+        sessionInfo = sessionCache.get(roomId, senderKey, sessionId);
+        if (!sessionInfo) {
             const sessionEntry = await txn.inboundGroupSessions.get(roomId, senderKey, sessionId);
             if (sessionEntry) {
-                session = new this._olm.InboundGroupSession();
+                let session = new this._olm.InboundGroupSession();
                 try {
                     session.unpickle(this._pickleKey, sessionEntry.session);
+                    sessionInfo = new SessionInfo(roomId, senderKey, session, sessionEntry.claimedKeys);
                 } catch (err) {
                     session.free();
                     throw err;
                 }
-                claimedKeys = sessionEntry.claimedKeys;
-                sessionCache.add(roomId, senderKey, session, claimedKeys);
+                sessionCache.add(sessionInfo);
             }
         }
-        if (!session) {
-            return;
-        }
-        const {plaintext, message_index: messageIndex} = session.decrypt(ciphertext);
-        let payload;
-        try {
-            payload = JSON.parse(plaintext);
-        } catch (err) {
-            throw new DecryptionError("PLAINTEXT_NOT_JSON", event, {plaintext, err});
-        }
-        if (payload.room_id !== roomId) {
-            throw new DecryptionError("MEGOLM_WRONG_ROOM", event,
-                {encryptedRoomId: payload.room_id, eventRoomId: roomId});
-        }
-        await this._handleReplayAttack(roomId, sessionId, messageIndex, event, txn);
-        return new DecryptionResult(payload, senderKey, claimedKeys);
-    }
-
-    async _handleReplayAttack(roomId, sessionId, messageIndex, event, txn) {
-        const eventId = event.event_id;
-        const timestamp = event.origin_server_ts;
-        const decryption = await txn.groupSessionDecryptions.get(roomId, sessionId, messageIndex);
-        if (decryption && decryption.eventId !== eventId) {
-            // the one with the newest timestamp should be the attack
-            const decryptedEventIsBad = decryption.timestamp < timestamp;
-            const badEventId = decryptedEventIsBad ? eventId : decryption.eventId;
-            throw new DecryptionError("MEGOLM_REPLAYED_INDEX", event, {
-                messageIndex,
-                badEventId,
-                otherEventId: decryption.eventId
-            });
-        }
-        if (!decryption) {
-            txn.groupSessionDecryptions.set({
-                roomId,
-                sessionId,
-                messageIndex,
-                eventId,
-                timestamp
-            });
-        }
+        return sessionInfo;
     }
 
     /**
@@ -165,55 +164,3 @@ export class Decryption {
     }
 }
 
-class SessionCache {
-    constructor() {
-        this._sessions = [];
-    }
-
-    /**
-     * @type {CacheEntry}
-     * @property {InboundGroupSession} session the unpickled session
-     * @property {Object} claimedKeys an object with the claimed ed25519 key
-     *
-     * 
-     * @param  {string} roomId
-     * @param  {string} senderKey
-     * @param  {string} sessionId
-     * @return {CacheEntry?}
-     */
-    get(roomId, senderKey, sessionId) {
-        const idx = this._sessions.findIndex(s => {
-            return s.roomId === roomId &&
-                s.senderKey === senderKey &&
-                sessionId === s.session.session_id();
-        });
-        if (idx !== -1) {
-            const entry = this._sessions[idx];
-            // move to top
-            if (idx > 0) {
-                this._sessions.splice(idx, 1);
-                this._sessions.unshift(entry);
-            }
-            return entry;
-        }
-    }
-
-    add(roomId, senderKey, session, claimedKeys) {
-        // add new at top
-        this._sessions.unshift({roomId, senderKey, session, claimedKeys});
-        if (this._sessions.length > CACHE_MAX_SIZE) {
-            // free sessions we're about to remove
-            for (let i = CACHE_MAX_SIZE; i < this._sessions.length; i += 1) {
-                this._sessions[i].session.free();
-            }
-            this._sessions = this._sessions.slice(0, CACHE_MAX_SIZE);
-        }
-    }
-
-    dispose() {
-        for (const entry of this._sessions) {
-            entry.session.free();
-        }
-
-    }
-}
diff --git a/src/matrix/e2ee/megolm/decryption/DecryptionChanges.js b/src/matrix/e2ee/megolm/decryption/DecryptionChanges.js
new file mode 100644
index 00000000..5597aaf7
--- /dev/null
+++ b/src/matrix/e2ee/megolm/decryption/DecryptionChanges.js
@@ -0,0 +1,78 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {DecryptionError} from "../../common.js";
+
+export class DecryptionChanges {
+    constructor(roomId, results, errors, replayEntries) {
+        this._roomId = roomId;
+        this._results = results;
+        this._errors = errors;
+        this._replayEntries = replayEntries;
+    }
+
+    /**
+     * @type MegolmBatchDecryptionResult
+     * @property {Map} results a map of event id to decryption result
+     * @property {Map} errors event id -> errors
+     * 
+     * Handle replay attack detection, and return result
+     * @param  {[type]} txn [description]
+     * @return {MegolmBatchDecryptionResult}
+     */
+    async write(txn) {
+        await Promise.all(this._replayEntries.map(async replayEntry => {
+            try {
+                this._handleReplayAttack(this._roomId, replayEntry, txn);
+            } catch (err) {
+                this._errors.set(replayEntry.eventId, err);
+            }
+        }));
+        return {
+            results: this._results,
+            errors: this._errors
+        };
+    }
+
+    async _handleReplayAttack(roomId, replayEntry, txn) {
+        const {messageIndex, sessionId, eventId, timestamp} = replayEntry;
+        const decryption = await txn.groupSessionDecryptions.get(roomId, sessionId, messageIndex);
+
+        if (decryption && decryption.eventId !== eventId) {
+            // the one with the newest timestamp should be the attack
+            const decryptedEventIsBad = decryption.timestamp < timestamp;
+            const badEventId = decryptedEventIsBad ? eventId : decryption.eventId;
+            // discard result
+            this._results.delete(eventId);
+
+            throw new DecryptionError("MEGOLM_REPLAYED_INDEX", event, {
+                messageIndex,
+                badEventId,
+                otherEventId: decryption.eventId
+            });
+        }
+
+        if (!decryption) {
+            txn.groupSessionDecryptions.set({
+                roomId,
+                sessionId,
+                messageIndex,
+                eventId,
+                timestamp
+            });
+        }
+    }
+}
diff --git a/src/matrix/e2ee/megolm/decryption/DecryptionPreparation.js b/src/matrix/e2ee/megolm/decryption/DecryptionPreparation.js
new file mode 100644
index 00000000..02ee32df
--- /dev/null
+++ b/src/matrix/e2ee/megolm/decryption/DecryptionPreparation.js
@@ -0,0 +1,52 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {DecryptionChanges} from "./DecryptionChanges.js";
+import {mergeMap} from "../../../../utils/mergeMap.js";
+
+/**
+ * Class that contains all the state loaded from storage to decrypt the given events
+ */
+export class DecryptionPreparation {
+    constructor(roomId, sessionDecryptions, errors) {
+        this._roomId = roomId;
+        this._sessionDecryptions = sessionDecryptions;
+        this._initialErrors = errors;
+    }
+
+    async decrypt() {
+        try {
+            const errors = this._initialErrors;
+            const results = new Map();
+            const replayEntries = [];
+            await Promise.all(this._sessionDecryptions.map(async sessionDecryption => {
+                const sessionResult = await sessionDecryption.decryptAll();
+                mergeMap(sessionResult.errors, errors);
+                mergeMap(sessionResult.results, results);
+                replayEntries.push(...sessionResult.replayEntries);
+            }));
+            return new DecryptionChanges(this._roomId, results, errors, replayEntries);
+        } finally {
+            this.dispose();
+        }
+    }
+
+    dispose() {
+        for (const sd of this._sessionDecryptions) {
+            sd.dispose();
+        }
+    }
+}
diff --git a/src/matrix/e2ee/megolm/decryption/README.md b/src/matrix/e2ee/megolm/decryption/README.md
new file mode 100644
index 00000000..b9bb3568
--- /dev/null
+++ b/src/matrix/e2ee/megolm/decryption/README.md
@@ -0,0 +1,6 @@
+Lots of classes here. The complexity comes from needing to offload decryption to a webworker, mainly for IE11. We can't keep a idb transaction open while waiting for the response from the worker, so need to batch decryption of multiple events and do decryption in multiple steps:
+
+ 1. Read all used inbound sessions for the batch of events, requires a read txn. This happens in `Decryption`. Sessions are loaded into `SessionInfo` objects, which are also kept in a `SessionCache` to prevent having to read and unpickle them all the time.
+ 2. Actually decrypt. No txn can stay open during this step, as it can be offloaded to a worker and is thus async. This happens in `DecryptionPreparation`, which delegates to `SessionDecryption` per session.
+ 3. Read and write for the replay detection, requires a read/write txn. This happens in `DecryptionChanges`
+ 4. Return the decrypted entries, and errors if any
diff --git a/src/matrix/e2ee/megolm/decryption/ReplayDetectionEntry.js b/src/matrix/e2ee/megolm/decryption/ReplayDetectionEntry.js
new file mode 100644
index 00000000..e5ce2845
--- /dev/null
+++ b/src/matrix/e2ee/megolm/decryption/ReplayDetectionEntry.js
@@ -0,0 +1,24 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export class ReplayDetectionEntry {
+    constructor(sessionId, messageIndex, event) {
+        this.sessionId = sessionId;
+        this.messageIndex = messageIndex;
+        this.eventId = event.event_id;
+        this.timestamp = event.origin_server_ts;
+    }
+}
diff --git a/src/matrix/e2ee/megolm/decryption/SessionCache.js b/src/matrix/e2ee/megolm/decryption/SessionCache.js
new file mode 100644
index 00000000..efb7ef54
--- /dev/null
+++ b/src/matrix/e2ee/megolm/decryption/SessionCache.js
@@ -0,0 +1,68 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+const CACHE_MAX_SIZE = 10;
+
+/**
+ * Cache of unpickled inbound megolm session.
+ */
+export class SessionCache {
+    constructor() {
+        this._sessions = [];
+    }
+
+    /**
+     * @param  {string} roomId
+     * @param  {string} senderKey
+     * @param  {string} sessionId
+     * @return {SessionInfo?}
+     */
+    get(roomId, senderKey, sessionId) {
+        const idx = this._sessions.findIndex(s => {
+            return s.roomId === roomId &&
+                s.senderKey === senderKey &&
+                sessionId === s.session.session_id();
+        });
+        if (idx !== -1) {
+            const sessionInfo = this._sessions[idx];
+            // move to top
+            if (idx > 0) {
+                this._sessions.splice(idx, 1);
+                this._sessions.unshift(sessionInfo);
+            }
+            return sessionInfo;
+        }
+    }
+
+    add(sessionInfo) {
+        sessionInfo.retain();
+        // add new at top
+        this._sessions.unshift(sessionInfo);
+        if (this._sessions.length > CACHE_MAX_SIZE) {
+            // free sessions we're about to remove
+            for (let i = CACHE_MAX_SIZE; i < this._sessions.length; i += 1) {
+                this._sessions[i].release();
+            }
+            this._sessions = this._sessions.slice(0, CACHE_MAX_SIZE);
+        }
+    }
+
+    dispose() {
+        for (const sessionInfo of this._sessions) {
+            sessionInfo.release();
+        }
+    }
+}
diff --git a/src/matrix/e2ee/megolm/decryption/SessionDecryption.js b/src/matrix/e2ee/megolm/decryption/SessionDecryption.js
new file mode 100644
index 00000000..6abda029
--- /dev/null
+++ b/src/matrix/e2ee/megolm/decryption/SessionDecryption.js
@@ -0,0 +1,75 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {DecryptionResult} from "../../DecryptionResult.js";
+import {DecryptionError} from "../../common.js";
+import {ReplayDetectionEntry} from "./ReplayDetectionEntry.js";
+
+/**
+ * Does the actual decryption of all events for a given megolm session in a batch
+ */
+export class SessionDecryption {
+    constructor(sessionInfo, events) {
+        sessionInfo.retain();
+        this._sessionInfo = sessionInfo;
+        this._events = events;
+    }
+
+    async decryptAll() {
+        const replayEntries = [];
+        const results = new Map();
+        let errors;
+        const roomId = this._sessionInfo.roomId;
+
+        await Promise.all(this._events.map(async event => {
+            try {
+                const {session} = this._sessionInfo;
+                const ciphertext = event.content.ciphertext;
+                const {plaintext, message_index: messageIndex} = await this._decrypt(session, ciphertext);
+                let payload;
+                try {
+                    payload = JSON.parse(plaintext);
+                } catch (err) {
+                    throw new DecryptionError("PLAINTEXT_NOT_JSON", event, {plaintext, err});
+                }
+                if (payload.room_id !== roomId) {
+                    throw new DecryptionError("MEGOLM_WRONG_ROOM", event,
+                        {encryptedRoomId: payload.room_id, eventRoomId: roomId});
+                }
+                replayEntries.push(new ReplayDetectionEntry(session.session_id(), messageIndex, event));
+                const result = new DecryptionResult(payload, this._sessionInfo.senderKey, this._sessionInfo.claimedKeys);
+                results.set(event.event_id, result);
+            } catch (err) {
+                if (!errors) {
+                    errors = new Map();
+                }
+                errors.set(event.event_id, err);
+            }
+        }));
+
+        return {results, errors, replayEntries};
+    }
+
+    async _decrypt(session, ciphertext) {
+        // const sessionKey = session.export_session(session.first_known_index());
+        // return this._worker.decrypt(sessionKey, ciphertext);
+        return session.decrypt(ciphertext);
+    }
+
+    dispose() {
+        this._sessionInfo.release();
+    }
+}
diff --git a/src/matrix/e2ee/megolm/decryption/SessionInfo.js b/src/matrix/e2ee/megolm/decryption/SessionInfo.js
new file mode 100644
index 00000000..dedc3222
--- /dev/null
+++ b/src/matrix/e2ee/megolm/decryption/SessionInfo.js
@@ -0,0 +1,44 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * session loaded in memory with everything needed to create DecryptionResults
+ * and to store/retrieve it in the SessionCache
+ */
+export class SessionInfo {
+    constructor(roomId, senderKey, session, claimedKeys) {
+        this.roomId = roomId;
+        this.senderKey = senderKey;
+        this.session = session;
+        this.claimedKeys = claimedKeys;
+        this._refCounter = 0;
+    }
+
+    retain() {
+        this._refCounter += 1;
+    }
+
+    release() {
+        this._refCounter -= 1;
+        if (this._refCounter <= 0) {
+            this.dispose();
+        }
+    }
+
+    dispose() {
+        this.session.free();
+    }
+}
diff --git a/src/utils/mergeMap.js b/src/utils/mergeMap.js
new file mode 100644
index 00000000..a0aed207
--- /dev/null
+++ b/src/utils/mergeMap.js
@@ -0,0 +1,41 @@
+/*
+Copyright 2020 Bruno Windels 
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export function mergeMap(src, dst) {
+    if (src) {
+        for (const [key, value] of src.entries()) {
+            dst.set(key, value);
+        }
+    }
+}
+
+export function tests() {
+    return {
+        "mergeMap with src": assert => {
+            const src = new Map();
+            src.set(1, "a");
+            const dst = new Map();
+            dst.set(2, "b");
+            mergeMap(src, dst);
+            assert.equal(dst.get(1), "a");
+            assert.equal(dst.get(2), "b");
+            assert.equal(src.get(2), null);
+        },
+        "mergeMap without src doesn't fail": () => {
+            mergeMap(undefined, new Map());
+        }
+    }
+}

From 1c77c3b8763a389bff220b4b344135848449d28c Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 10 Sep 2020 12:09:17 +0200
Subject: [PATCH 152/173] expose multi-step decryption from RoomEncryption,
 adjust room timeline

sync code hasn't been adjusted yet
---
 src/matrix/e2ee/RoomEncryption.js             | 136 +++++++++++++++---
 src/matrix/e2ee/common.js                     |   3 +
 src/matrix/room/Room.js                       | 116 +++++++--------
 src/matrix/room/timeline/Timeline.js          |  15 --
 .../timeline/persistence/TimelineReader.js    |  62 +++-----
 5 files changed, 199 insertions(+), 133 deletions(-)

diff --git a/src/matrix/e2ee/RoomEncryption.js b/src/matrix/e2ee/RoomEncryption.js
index c8f993e5..1341389b 100644
--- a/src/matrix/e2ee/RoomEncryption.js
+++ b/src/matrix/e2ee/RoomEncryption.js
@@ -14,8 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import {MEGOLM_ALGORITHM} from "./common.js";
+import {MEGOLM_ALGORITHM, DecryptionSource} from "./common.js";
 import {groupBy} from "../../utils/groupBy.js";
+import {mergeMap} from "../../utils/mergeMap.js";
 import {makeTxnId} from "../common.js";
 
 const ENCRYPTED_TYPE = "m.room.encrypted";
@@ -55,23 +56,54 @@ export class RoomEncryption {
         return await this._deviceTracker.writeMemberChanges(this._room, memberChanges, txn);
     }
 
-    async decrypt(event, isSync, isTimelineOpen, retryData, txn) {
-        if (event.redacted_because || event.unsigned?.redacted_because) {
-            return;
+    // this happens before entries exists, as they are created by the syncwriter
+    // but we want to be able to map it back to something in the timeline easily
+    // when retrying decryption.
+    async prepareDecryptAll(events, source, isTimelineOpen, txn) {
+        const errors = [];
+        const validEvents = [];
+        for (const event of events) {
+            if (event.redacted_because || event.unsigned?.redacted_because) {
+                continue;
+            }
+            if (event.content?.algorithm !== MEGOLM_ALGORITHM) {
+                errors.set(event.event_id, new Error("Unsupported algorithm: " + event.content?.algorithm));
+            }
+            validEvents.push(event);
         }
-        if (event.content?.algorithm !== MEGOLM_ALGORITHM) {
-            throw new Error("Unsupported algorithm: " + event.content?.algorithm);
+        let customCache;
+        let sessionCache;
+        if (source === DecryptionSource.Sync) {
+            sessionCache = this._megolmSyncCache;
+        } else if (source === DecryptionSource.Timeline) {
+            sessionCache = this._megolmBackfillCache;
+        } else if (source === DecryptionSource.Retry) {
+            // when retrying, we could have mixed events from at the bottom of the timeline (sync)
+            // and somewhere else, so create a custom cache we use just for this operation.
+            customCache = this._megolmEncryption.createSessionCache();
+            sessionCache = customCache;
+        } else {
+            throw new Error("Unknown source: " + source);
         }
-        let sessionCache = isSync ? this._megolmSyncCache : this._megolmBackfillCache;
-        const result = await this._megolmDecryption.decrypt(
-            this._room.id, event, sessionCache, txn);
-        if (!result) {
-            this._addMissingSessionEvent(event, isSync, retryData);
+        const preparation = await this._megolmDecryption.prepareDecryptAll(
+            this._room.id, validEvents, sessionCache, txn);
+        if (customCache) {
+            customCache.dispose();
         }
-        if (result && isTimelineOpen) {
-            await this._verifyDecryptionResult(result, txn);
+        return new DecryptionPreparation(preparation, errors, {isTimelineOpen}, this);
+    }
+
+    async _processDecryptionResults(results, errors, flags, txn) {
+        for (const error of errors.values()) {
+            if (error.code === "MEGOLM_NO_SESSION") {
+                this._addMissingSessionEvent(error.event);
+            }
+        }
+        if (flags.isTimelineOpen) {
+            for (const result of results.values()) {
+                await this._verifyDecryptionResult(result, txn);
+            }
         }
-        return result;
     }
 
     async _verifyDecryptionResult(result, txn) {
@@ -87,30 +119,30 @@ export class RoomEncryption {
         }
     }
 
-    _addMissingSessionEvent(event, isSync, data) {
+    _addMissingSessionEvent(event) {
         const senderKey = event.content?.["sender_key"];
         const sessionId = event.content?.["session_id"];
         const key = `${senderKey}|${sessionId}`;
         let eventIds = this._eventIdsByMissingSession.get(key);
         if (!eventIds) {
-            eventIds = new Map();
+            eventIds = new Set();
             this._eventIdsByMissingSession.set(key, eventIds);
         }
-        eventIds.set(event.event_id, {data, isSync});
+        eventIds.add(event.event_id);
     }
 
     applyRoomKeys(roomKeys) {
         // retry decryption with the new sessions
-        const retryEntries = [];
+        const retryEventIds = [];
         for (const roomKey of roomKeys) {
             const key = `${roomKey.senderKey}|${roomKey.sessionId}`;
             const entriesForSession = this._eventIdsByMissingSession.get(key);
             if (entriesForSession) {
                 this._eventIdsByMissingSession.delete(key);
-                retryEntries.push(...entriesForSession.values());
+                retryEventIds.push(...entriesForSession);
             }
         }
-        return retryEntries;
+        return retryEventIds;
     }
 
     async encrypt(type, content, hsApi) {
@@ -214,3 +246,67 @@ export class RoomEncryption {
         await hsApi.sendToDevice(type, payload, txnId).response();
     }
 }
+
+/**
+ * wrappers around megolm decryption classes to be able to post-process
+ * the decryption results before turning them
+ */
+class DecryptionPreparation {
+    constructor(megolmDecryptionPreparation, extraErrors, flags, roomEncryption) {
+        this._megolmDecryptionPreparation = megolmDecryptionPreparation;
+        this._extraErrors = extraErrors;
+        this._flags = flags;
+        this._roomEncryption = roomEncryption;
+    }
+
+    async decrypt() {
+        return new DecryptionChanges(
+            await this._megolmDecryptionPreparation.decrypt(),
+            this._extraErrors,
+            this._flags,
+            this._roomEncryption);
+    }
+
+    dispose() {
+        this._megolmDecryptionChanges.dispose();
+    }
+}
+
+class DecryptionChanges {
+    constructor(megolmDecryptionChanges, extraErrors, flags, roomEncryption) {
+        this._megolmDecryptionChanges = megolmDecryptionChanges;
+        this._extraErrors = extraErrors;
+        this._flags = flags;
+        this._roomEncryption = roomEncryption;
+    }
+
+    async write(txn) {
+        const {results, errors} = await this._megolmDecryptionChanges.write(txn);
+        mergeMap(this._extraErrors, errors);
+        await this._roomEncryption._processDecryptionResults(results, errors, this._flags, txn);
+        return new BatchDecryptionResult(results, errors);
+    }
+}
+
+class BatchDecryptionResult {
+    constructor(results, errors) {
+        this.results = results;
+        this.errors = errors;
+        console.log("BatchDecryptionResult", this);
+    }
+
+    applyToEntries(entries) {
+        console.log("BatchDecryptionResult.applyToEntries", this);
+        for (const entry of entries) {
+            const result = this.results.get(entry.id);
+            if (result) {
+                entry.setDecryptionResult(result);
+            } else {
+                const error = this.errors.get(entry.id);
+                if (error) {
+                    entry.setDecryptionError(error);
+                }
+            }
+        }
+    }
+}
diff --git a/src/matrix/e2ee/common.js b/src/matrix/e2ee/common.js
index 3312032b..190f2fa2 100644
--- a/src/matrix/e2ee/common.js
+++ b/src/matrix/e2ee/common.js
@@ -15,6 +15,9 @@ limitations under the License.
 */
 
 import anotherjson from "../../../lib/another-json/index.js";
+import {createEnum} from "../../utils/enum.js";
+
+export const DecryptionSource = createEnum(["Sync", "Timeline", "Retry"]);
 
 // use common prefix so it's easy to clear properties that are not e2ee related during session clear
 export const SESSION_KEY_PREFIX = "e2ee:";
diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index a2b84717..6a18f34f 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -26,6 +26,9 @@ import {fetchOrLoadMembers} from "./members/load.js";
 import {MemberList} from "./members/MemberList.js";
 import {Heroes} from "./members/Heroes.js";
 import {EventEntry} from "./timeline/entries/EventEntry.js";
+import {DecryptionSource} from "../e2ee/common.js";
+
+const EVENT_ENCRYPTED_TYPE = "m.room.encrypted";
 
 export class Room extends EventEmitter {
 	constructor({roomId, storage, hsApi, emitCollectionChange, sendScheduler, pendingEvents, user, createRoomEncryption, getSyncToken}) {
@@ -49,43 +52,26 @@ export class Room extends EventEmitter {
 
     async notifyRoomKeys(roomKeys) {
         if (this._roomEncryption) {
-            // array of {data, isSync}
-            let retryEntries = this._roomEncryption.applyRoomKeys(roomKeys);
-            let decryptedEntries = [];
-            if (retryEntries.length) {
-                // groupSessionDecryptions can be written, the other stores not
-                const txn = await this._storage.readWriteTxn([
+            let retryEventIds = this._roomEncryption.applyRoomKeys(roomKeys);
+            if (retryEventIds.length) {
+                const retryEntries = [];
+                const txn = await this._storage.readTxn([
                     this._storage.storeNames.timelineEvents,
                     this._storage.storeNames.inboundGroupSessions,
-                    this._storage.storeNames.groupSessionDecryptions,
-                    this._storage.storeNames.deviceIdentities,
                 ]);
-                try {
-                    for (const retryEntry of retryEntries) {
-                        const {data: eventKey} = retryEntry;
-                        let entry = this._timeline?.findEntry(eventKey);
-                        if (!entry) {
-                            const storageEntry = await txn.timelineEvents.get(this._roomId, eventKey);
-                            if (storageEntry) {
-                                entry = new EventEntry(storageEntry, this._fragmentIdComparer);
-                            }
-                        }
-                        if (entry) {
-                            entry = await this._decryptEntry(entry, txn, retryEntry.isSync);
-                            decryptedEntries.push(entry);
-                        }
+                for (const eventId of retryEventIds) {
+                    const storageEntry = await txn.timelineEvents.getByEventId(this._roomId, eventId);
+                    if (storageEntry) {
+                        retryEntries.push(new EventEntry(storageEntry, this._fragmentIdComparer));
                     }
-                } catch (err) {
-                    txn.abort();
-                    throw err;
                 }
-                await txn.complete();
+                await this._decryptEntries(DecryptionSource.Retry, retryEntries, txn);
+                if (this._timeline) {
+                    // only adds if already present
+                    this._timeline.replaceEntries(retryEntries);
+                }
+                // pass decryptedEntries to roomSummary
             }
-            if (this._timeline) {
-                // only adds if already present
-                this._timeline.replaceEntries(decryptedEntries);
-            }
-            // pass decryptedEntries to roomSummary
         }
     }
 
@@ -94,22 +80,42 @@ export class Room extends EventEmitter {
         if (this._roomEncryption) {
             this._sendQueue.enableEncryption(this._roomEncryption);
             if (this._timeline) {
-                this._timeline.enableEncryption(this._decryptEntries.bind(this));
+                this._timeline.enableEncryption(this._decryptEntries.bind(this, DecryptionSource.Timeline));
             }
         }
     }
 
-    async _decryptEntry(entry, txn, isSync) {
-        if (entry.eventType === "m.room.encrypted") {
-            try {
-                const decryptionResult = await this._roomEncryption.decrypt(
-                    entry.event, isSync, !!this._timeline, entry.asEventKey(), txn);
-                if (decryptionResult) {
-                    entry.setDecryptionResult(decryptionResult);
-                }
-            } catch (err) {
-                console.warn("event decryption error", err, entry.event);
-                entry.setDecryptionError(err);
+    /**
+     * Used for decrypting when loading/filling the timeline, and retrying decryption,
+     * not during sync, where it is split up during the multiple phases.
+     */
+    async _decryptEntries(source, entries, inboundSessionTxn = null) {
+        if (!inboundSessionTxn) {
+            inboundSessionTxn = await this._storage.readTxn([this._storage.storeNames.inboundGroupSessions]);
+        }
+        const events = entries.filter(entry => {
+            return entry.eventType === EVENT_ENCRYPTED_TYPE;
+        }).map(entry => entry.event);
+        const isTimelineOpen = this._isTimelineOpen;
+        const preparation = await this._roomEncryption.prepareDecryptAll(events, source, isTimelineOpen, inboundSessionTxn);
+        const changes = await preparation.decrypt();
+        const stores = [this._storage.storeNames.groupSessionDecryptions];
+        if (isTimelineOpen) {
+            // read to fetch devices if timeline is open
+            stores.push(this._storage.storeNames.deviceIdentities);
+        }
+        const writeTxn = await this._storage.readWriteTxn(stores);
+        let decryption;
+        try {
+            decryption = await changes.write(writeTxn);
+        } catch (err) {
+            writeTxn.abort();
+            throw err;
+        }
+        await writeTxn.complete();
+        decryption.applyToEntries(entries);
+    }
+
             }
         }
         return entry;
@@ -299,19 +305,11 @@ export class Room extends EventEmitter {
             }
         }).response();
 
-        let stores = [
+        const txn = await this._storage.readWriteTxn([
             this._storage.storeNames.pendingEvents,
             this._storage.storeNames.timelineEvents,
             this._storage.storeNames.timelineFragments,
-        ];
-        if (this._roomEncryption) {
-            stores = stores.concat([
-                this._storage.storeNames.inboundGroupSessions,
-                this._storage.storeNames.groupSessionDecryptions,
-                this._storage.storeNames.deviceIdentities,
-            ]);
-        }
-        const txn = await this._storage.readWriteTxn(stores);
+        ]);
         let removedPendingEvents;
         let gapResult;
         try {
@@ -324,14 +322,14 @@ export class Room extends EventEmitter {
                 fragmentIdComparer: this._fragmentIdComparer,
             });
             gapResult = await gapWriter.writeFragmentFill(fragmentEntry, response, txn);
-            if (this._roomEncryption) {
-                gapResult.entries = await this._decryptEntries(gapResult.entries, txn, false);
-            }
         } catch (err) {
             txn.abort();
             throw err;
         }
         await txn.complete();
+        if (this._roomEncryption) {
+            await this._decryptEntries(DecryptionSource.Timeline, gapResult.entries);
+        }
         // once txn is committed, update in-memory state & emit events
         for (const fragment of gapResult.fragments) {
             this._fragmentIdComparer.add(fragment);
@@ -406,6 +404,10 @@ export class Room extends EventEmitter {
         }
     }
 
+    get _isTimelineOpen() {
+        return !!this._timeline;
+    }
+
     async clearUnread() {
         if (this.isUnread || this.notificationCount) {
             const txn = await this._storage.readWriteTxn([
@@ -458,7 +460,7 @@ export class Room extends EventEmitter {
             user: this._user,
         });
         if (this._roomEncryption) {
-            this._timeline.enableEncryption(this._decryptEntries.bind(this));
+            this._timeline.enableEncryption(this._decryptEntries.bind(this, DecryptionSource.Timeline));
         }
         await this._timeline.load();
         return this._timeline;
diff --git a/src/matrix/room/timeline/Timeline.js b/src/matrix/room/timeline/Timeline.js
index c2e9d0ce..7245568d 100644
--- a/src/matrix/room/timeline/Timeline.js
+++ b/src/matrix/room/timeline/Timeline.js
@@ -46,21 +46,6 @@ export class Timeline {
         this._remoteEntries.setManySorted(entries);
     }
 
-    findEntry(eventKey) {
-        // a storage event entry has a fragmentId and eventIndex property, used for sorting,
-        // just like an EventKey, so this will work, but perhaps a bit brittle.
-        const entry = new EventEntry(eventKey, this._fragmentIdComparer);
-        try {
-            const idx = this._remoteEntries.indexOf(entry);
-            if (idx !== -1) {
-                return this._remoteEntries.get(idx);
-            }
-        } catch (err) {
-            // fragmentIdComparer threw, ignore
-            return;
-        }
-    }
-
     replaceEntries(entries) {
         for (const entry of entries) {
             this._remoteEntries.replace(entry);
diff --git a/src/matrix/room/timeline/persistence/TimelineReader.js b/src/matrix/room/timeline/persistence/TimelineReader.js
index 4446eaf1..d451d76e 100644
--- a/src/matrix/room/timeline/persistence/TimelineReader.js
+++ b/src/matrix/room/timeline/persistence/TimelineReader.js
@@ -32,34 +32,19 @@ export class TimelineReader {
     }
 
     _openTxn() {
+        const stores = [
+            this._storage.storeNames.timelineEvents,
+            this._storage.storeNames.timelineFragments,
+        ];
         if (this._decryptEntries) {
-            return this._storage.readWriteTxn([
-                this._storage.storeNames.timelineEvents,
-                this._storage.storeNames.timelineFragments,
-                this._storage.storeNames.inboundGroupSessions,
-                this._storage.storeNames.groupSessionDecryptions,
-                this._storage.storeNames.deviceIdentities,
-            ]);
-
-        } else {
-            return this._storage.readTxn([
-                this._storage.storeNames.timelineEvents,
-                this._storage.storeNames.timelineFragments,
-            ]);
+            stores.push(this._storage.storeNames.inboundGroupSessions);
         }
+        return this._storage.readTxn(stores);
     }
 
     async readFrom(eventKey, direction, amount) {
         const txn = await this._openTxn();
-        let entries;
-        try {
-            entries = await this._readFrom(eventKey, direction, amount, txn);
-        } catch (err) {
-            txn.abort();
-            throw err;
-        }
-        await txn.complete();
-        return entries;
+        return await this._readFrom(eventKey, direction, amount, txn);
     }
 
     async _readFrom(eventKey, direction, amount, txn) {
@@ -75,9 +60,6 @@ export class TimelineReader {
                 eventsWithinFragment = await timelineStore.eventsBefore(this._roomId, eventKey, amount);
             }
             let eventEntries = eventsWithinFragment.map(e => new EventEntry(e, this._fragmentIdComparer));
-            if (this._decryptEntries) {
-                eventEntries = await this._decryptEntries(eventEntries, txn);
-            }
             entries = directionalConcat(entries, eventEntries, direction);
             // prepend or append eventsWithinFragment to entries, and wrap them in EventEntry
 
@@ -100,29 +82,27 @@ export class TimelineReader {
             }
         }
 
+        if (this._decryptEntries) {
+            await this._decryptEntries(entries, txn);
+        }
+
         return entries;
     }
 
     async readFromEnd(amount) {
         const txn = await this._openTxn();
+        const liveFragment = await txn.timelineFragments.liveFragment(this._roomId);
         let entries;
-        try {
-            const liveFragment = await txn.timelineFragments.liveFragment(this._roomId);
-            // room hasn't been synced yet
-            if (!liveFragment) {
-                entries = [];
-            } else {
-                this._fragmentIdComparer.add(liveFragment);
-                const liveFragmentEntry = FragmentBoundaryEntry.end(liveFragment, this._fragmentIdComparer);
-                const eventKey = liveFragmentEntry.asEventKey();
-                entries = await this._readFrom(eventKey, Direction.Backward, amount, txn);
-                entries.unshift(liveFragmentEntry);
-            }
-        } catch (err) {
-            txn.abort();
-            throw err;
+        // room hasn't been synced yet
+        if (!liveFragment) {
+            entries = [];
+        } else {
+            this._fragmentIdComparer.add(liveFragment);
+            const liveFragmentEntry = FragmentBoundaryEntry.end(liveFragment, this._fragmentIdComparer);
+            const eventKey = liveFragmentEntry.asEventKey();
+            entries = await this._readFrom(eventKey, Direction.Backward, amount, txn);
+            entries.unshift(liveFragmentEntry);
         }
-        await txn.complete();
         return entries;
     }
 }

From 94b0cfbd72c46416e42a0c54007b635b28091eab Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 10 Sep 2020 12:11:43 +0200
Subject: [PATCH 153/173] add prepareSync and afterPrepareSync steps to sync,
 run decryption in it

---
 src/matrix/Session.js     |   4 +-
 src/matrix/Sync.js        | 165 ++++++++++++++++++++++++--------------
 src/matrix/e2ee/README.md |  44 ++++++++++
 src/matrix/room/Room.js   |  61 +++++++++-----
 4 files changed, 190 insertions(+), 84 deletions(-)
 create mode 100644 src/matrix/e2ee/README.md

diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index 19725b58..c5c0c94f 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -255,7 +255,7 @@ export class Session {
         return room;
     }
 
-    async writeSync(syncResponse, syncFilterId, roomChanges, txn) {
+    async writeSync(syncResponse, syncFilterId, txn) {
         const changes = {};
         const syncToken = syncResponse.next_batch;
         const deviceOneTimeKeysCount = syncResponse.device_one_time_keys_count;
@@ -362,7 +362,7 @@ export function tests() {
                     }
                 }
             };
-            const newSessionData = await session.writeSync({next_batch: "b"}, 6, {}, syncTxn);
+            const newSessionData = await session.writeSync({next_batch: "b"}, 6, syncTxn);
             assert(syncSet);
             assert.equal(session.syncToken, "a");
             assert.equal(session.syncFilterId, 5);
diff --git a/src/matrix/Sync.js b/src/matrix/Sync.js
index 598b9169..c81acee0 100644
--- a/src/matrix/Sync.js
+++ b/src/matrix/Sync.js
@@ -29,21 +29,6 @@ export const SyncStatus = createEnum(
     "Stopped"
 );
 
-function parseRooms(roomsSection, roomCallback) {
-    if (roomsSection) {
-        const allMemberships = ["join", "invite", "leave"];
-        for(const membership of allMemberships) {
-            const membershipSection = roomsSection[membership];
-            if (membershipSection) {
-                return Object.entries(membershipSection).map(([roomId, roomResponse]) => {
-                    return roomCallback(roomId, roomResponse, membership);
-                });
-            }
-        }
-    }
-    return [];
-}
-
 function timelineIsEmpty(roomResponse) {
     try {
         const events = roomResponse?.timeline?.events;
@@ -53,6 +38,26 @@ function timelineIsEmpty(roomResponse) {
     }
 }
 
+/**
+ * Sync steps in js-pseudocode:
+ * ```js
+ * let preparation;
+ * if (room.needsPrepareSync) {
+ *     // can only read some stores
+ *     preparation = await room.prepareSync(roomResponse, prepareTxn);
+ *     // can do async work that is not related to storage (such as decryption)
+ *     preparation = await room.afterPrepareSync(preparation);
+ * }
+ * // writes and calculates changes
+ * const changes = await room.writeSync(roomResponse, membership, isInitialSync, preparation, syncTxn);
+ * // applies and emits changes once syncTxn is committed
+ * room.afterSync(changes);
+ * if (room.needsAfterSyncCompleted(changes)) {
+ *     // can do network requests
+ *     await room.afterSyncCompleted(changes);
+ * }
+ * ```
+ */
 export class Sync {
     constructor({hsApi, session, storage}) {
         this._hsApi = hsApi;
@@ -90,13 +95,13 @@ export class Sync {
         let afterSyncCompletedPromise = Promise.resolve();
         // if syncToken is falsy, it will first do an initial sync ... 
         while(this._status.get() !== SyncStatus.Stopped) {
-            let roomChanges;
+            let roomStates;
             try {
                 console.log(`starting sync request with since ${syncToken} ...`);
                 const timeout = syncToken ? INCREMENTAL_TIMEOUT : undefined; 
                 const syncResult = await this._syncRequest(syncToken, timeout, afterSyncCompletedPromise);
                 syncToken = syncResult.syncToken;
-                roomChanges = syncResult.roomChanges;
+                roomStates = syncResult.roomStates;
                 this._status.set(SyncStatus.Syncing);
             } catch (err) {
                 if (!(err instanceof AbortError)) {
@@ -105,12 +110,12 @@ export class Sync {
                 }
             }
             if (!this._error) {
-                afterSyncCompletedPromise = this._runAfterSyncCompleted(roomChanges);
+                afterSyncCompletedPromise = this._runAfterSyncCompleted(roomStates);
             }
         }
     }
 
-    async _runAfterSyncCompleted(roomChanges) {
+    async _runAfterSyncCompleted(roomStates) {
         const sessionPromise = (async () => {
             try {
                 await this._session.afterSyncCompleted();
@@ -118,23 +123,22 @@ export class Sync {
                 console.error("error during session afterSyncCompleted, continuing",  err.stack);
             }
         })();
-        let allPromises = [sessionPromise];
 
-        const roomsNeedingAfterSyncCompleted = roomChanges.filter(rc => {
-            return rc.changes.needsAfterSyncCompleted;
+        const roomsNeedingAfterSyncCompleted = roomStates.filter(rs => {
+            return rs.room.needsAfterSyncCompleted(rs.changes);
+        });
+        const roomsPromises = roomsNeedingAfterSyncCompleted.map(async rs => {
+            try {
+                await rs.room.afterSyncCompleted(rs.changes);
+            } catch (err) {
+                console.error(`error during room ${rs.room.id} afterSyncCompleted, continuing`,  err.stack);
+            }
         });
-        if (roomsNeedingAfterSyncCompleted.length) {
-            allPromises = allPromises.concat(roomsNeedingAfterSyncCompleted.map(async ({room, changes}) => {
-                try {
-                    await room.afterSyncCompleted(changes);
-                } catch (err) {
-                    console.error(`error during room ${room.id} afterSyncCompleted, continuing`,  err.stack);
-                }
-            }));
-        }
         // run everything in parallel,
         // we don't want to delay the next sync too much
-        await Promise.all(allPromises);
+        // Also, since all promises won't reject (as they have a try/catch)
+        // it's fine to use Promise.all
+        await Promise.all(roomsPromises.concat(sessionPromise));
     }
 
     async _syncRequest(syncToken, timeout, prevAfterSyncCompletedPromise) {
@@ -152,16 +156,17 @@ export class Sync {
 
         const isInitialSync = !syncToken;
         syncToken = response.next_batch;
-        const syncTxn = await this._openSyncTxn();
-        let roomChanges = [];
+        const roomStates = this._parseRoomsResponse(response.rooms, isInitialSync);
+        await this._prepareRooms(roomStates);
         let sessionChanges;
+        const syncTxn = await this._openSyncTxn();
         try {
-            // to_device
-            // presence
-            if (response.rooms) {
-                roomChanges = await this._writeRoomResponses(response.rooms, isInitialSync, syncTxn);
-            }
-            sessionChanges = await this._session.writeSync(response, syncFilterId, roomChanges, syncTxn);
+            await Promise.all(roomStates.map(async rs => {
+                console.log(` * applying sync response to room ${rs.room.id} ...`);
+                rs.changes = await rs.room.writeSync(
+                    rs.roomResponse, rs.membership, isInitialSync, rs.preparation, syncTxn);
+            }));
+            sessionChanges = await this._session.writeSync(response, syncFilterId, syncTxn);
         } catch(err) {
             console.warn("aborting syncTxn because of error");
             console.error(err);
@@ -180,31 +185,31 @@ export class Sync {
         }
         this._session.afterSync(sessionChanges);
         // emit room related events after txn has been closed
-        for(let {room, changes} of roomChanges) {
-            room.afterSync(changes);
+        for(let rs of roomStates) {
+            rs.room.afterSync(rs.changes);
         }
 
-        return {syncToken, roomChanges};
+        return {syncToken, roomStates};
     }
 
-    async _writeRoomResponses(roomResponses, isInitialSync, syncTxn) {
-        const roomChanges = [];
-        const promises = parseRooms(roomResponses, async (roomId, roomResponse, membership) => {
-            // ignore rooms with empty timelines during initial sync,
-            // see https://github.com/vector-im/hydrogen-web/issues/15
-            if (isInitialSync && timelineIsEmpty(roomResponse)) {
-                return;
-            }
-            let room = this._session.rooms.get(roomId);
-            if (!room) {
-                room = this._session.createRoom(roomId);
-            }
-            console.log(` * applying sync response to room ${roomId} ...`);
-            const changes = await room.writeSync(roomResponse, membership, isInitialSync, syncTxn);
-            roomChanges.push({room, changes});
-        });
-        await Promise.all(promises);
-        return roomChanges;
+    async _openPrepareSyncTxn() {
+        const storeNames = this._storage.storeNames;
+        return await this._storage.readTxn([
+            storeNames.inboundGroupSessions,
+        ]);
+    }
+
+    async _prepareRooms(roomStates) {
+        const prepareRoomStates = roomStates.filter(rs => rs.room.needsPrepareSync);
+        if (prepareRoomStates.length) {
+            const prepareTxn = await this._openPrepareSyncTxn();
+            await Promise.all(prepareRoomStates.map(async rs => {
+                rs.preparation = await rs.room.prepareSync(rs.roomResponse, prepareTxn);
+            }));
+            await Promise.all(prepareRoomStates.map(async rs => {
+                rs.preparation = await rs.room.afterPrepareSync(rs.preparation);
+            }));
+        }
     }
 
     async _openSyncTxn() {
@@ -218,13 +223,39 @@ export class Sync {
             storeNames.timelineFragments,
             storeNames.pendingEvents,
             storeNames.userIdentities,
-            storeNames.inboundGroupSessions,
             storeNames.groupSessionDecryptions,
             storeNames.deviceIdentities,
             // to discard outbound session when somebody leaves a room
             storeNames.outboundGroupSessions
         ]);
     }
+    
+    _parseRoomsResponse(roomsSection, isInitialSync) {
+        const roomStates = [];
+        if (roomsSection) {
+            // don't do "invite", "leave" for now
+            const allMemberships = ["join"];
+            for(const membership of allMemberships) {
+                const membershipSection = roomsSection[membership];
+                if (membershipSection) {
+                    for (const [roomId, roomResponse] of Object.entries(membershipSection)) {
+                        // ignore rooms with empty timelines during initial sync,
+                        // see https://github.com/vector-im/hydrogen-web/issues/15
+                        if (isInitialSync && timelineIsEmpty(roomResponse)) {
+                            return;
+                        }
+                        let room = this._session.rooms.get(roomId);
+                        if (!room) {
+                            room = this._session.createRoom(roomId);
+                        }
+                        roomStates.push(new RoomSyncProcessState(room, roomResponse, membership));
+                    }
+                }
+            }
+        }
+        return roomStates;
+    }
+
 
     stop() {
         if (this._status.get() === SyncStatus.Stopped) {
@@ -237,3 +268,13 @@ export class Sync {
         }
     }
 }
+
+class RoomSyncProcessState {
+    constructor(room, roomResponse, membership) {
+        this.room = room;
+        this.roomResponse = roomResponse;
+        this.membership = membership;
+        this.preparation = null;
+        this.changes = null;
+    }
+}
diff --git a/src/matrix/e2ee/README.md b/src/matrix/e2ee/README.md
new file mode 100644
index 00000000..46f4e95f
--- /dev/null
+++ b/src/matrix/e2ee/README.md
@@ -0,0 +1,44 @@
+## Integratation within the sync lifetime cycle
+
+### prepareSync
+    
+    The session can start its own read/write transactions here, rooms only read from a shared transaction
+
+    - session
+        - device handler
+            - txn
+                - write pending encrypted
+            - txn
+                - olm decryption read
+            - olm async decryption
+                - dispatch to worker
+            - txn
+                - olm decryption write / remove pending encrypted
+    - rooms (with shared read txn)
+        - megolm decryption read
+
+### afterPrepareSync
+
+    - rooms    
+        - megolm async decryption   
+            - dispatch to worker
+
+### writeSync
+
+    - rooms (with shared readwrite txn)
+        - megolm decryption write, yielding decrypted events
+        - use decrypted events to write room summary
+
+### afterSync
+
+    - rooms
+        - emit changes
+
+### afterSyncCompleted
+
+    - session
+        - e2ee account
+            - generate more otks if needed
+            - upload new otks if needed or device keys if not uploaded before
+    - rooms
+        - share new room keys if needed
diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index 6a18f34f..f281f166 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -116,30 +116,52 @@ export class Room extends EventEmitter {
         decryption.applyToEntries(entries);
     }
 
-            }
-        }
-        return entry;
+    get needsPrepareSync() {
+        // only encrypted rooms need the prepare sync steps
+        return !!this._roomEncryption;
     }
 
-    async _decryptEntries(entries, txn, isSync = false) {
-        return await Promise.all(entries.map(async e => this._decryptEntry(e, txn, isSync)));
+    async prepareSync(roomResponse, txn) {
+        if (this._roomEncryption) {
+            const events = roomResponse?.timeline?.events;
+            if (Array.isArray(events)) {
+                const eventsToDecrypt = events.filter(event => {
+                    return event?.type === EVENT_ENCRYPTED_TYPE;
+                });
+                const preparation = await this._roomEncryption.prepareDecryptAll(
+                    eventsToDecrypt, DecryptionSource.Sync, this._isTimelineOpen, txn);
+                return preparation;
+            }
+        }
+    }
+
+    async afterPrepareSync(preparation) {
+        if (preparation) {
+            const decryptChanges = await preparation.decrypt();
+            return decryptChanges;
+        }
     }
 
     /** @package */
-    async writeSync(roomResponse, membership, isInitialSync, txn) {
-        const isTimelineOpen = !!this._timeline;
+    async writeSync(roomResponse, membership, isInitialSync, decryptChanges, txn) {
+        let decryption;
+        if (this._roomEncryption && decryptChanges) {
+            decryption = await decryptChanges.write(txn);
+        }
+		const {entries, newLiveKey, memberChanges} =
+            await this._syncWriter.writeSync(roomResponse, this.isTrackingMembers, txn);
+        if (decryption) {
+            decryption.applyToEntries(entries);
+        }
+        // pass member changes to device tracker
+        if (this._roomEncryption && this.isTrackingMembers && memberChanges?.size) {
+            await this._roomEncryption.writeMemberChanges(memberChanges, txn);
+        }
 		const summaryChanges = this._summary.writeSync(
             roomResponse,
             membership,
-            isInitialSync, isTimelineOpen,
+            isInitialSync, this._isTimelineOpen,
             txn);
-		const {entries: encryptedEntries, newLiveKey, memberChanges} =
-            await this._syncWriter.writeSync(roomResponse, this.isTrackingMembers, txn);
-        // decrypt if applicable
-        let entries = encryptedEntries;
-        if (this._roomEncryption) {
-            entries = await this._decryptEntries(encryptedEntries, txn, true);
-        }
         // fetch new members while we have txn open,
         // but don't make any in-memory changes yet
         let heroChanges;
@@ -150,10 +172,6 @@ export class Room extends EventEmitter {
             }
             heroChanges = await this._heroes.calculateChanges(summaryChanges.heroes, memberChanges, txn);
         }
-        // pass member changes to device tracker
-        if (this._roomEncryption && this.isTrackingMembers && memberChanges?.size) {
-            await this._roomEncryption.writeMemberChanges(memberChanges, txn);
-        }
         let removedPendingEvents;
         if (roomResponse.timeline && roomResponse.timeline.events) {
             removedPendingEvents = this._sendQueue.removeRemoteEchos(roomResponse.timeline.events, txn);
@@ -165,7 +183,6 @@ export class Room extends EventEmitter {
             removedPendingEvents,
             memberChanges,
             heroChanges,
-            needsAfterSyncCompleted: this._roomEncryption?.needsToShareKeys(memberChanges)
         };
     }
 
@@ -216,6 +233,10 @@ export class Room extends EventEmitter {
         }
 	}
 
+    needsAfterSyncCompleted({memberChanges}) {
+        return this._roomEncryption?.needsToShareKeys(memberChanges);
+    }
+
     /**
      * Only called if the result of writeSync had `needsAfterSyncCompleted` set.
      * Can be used to do longer running operations that resulted from the last sync,

From 17412bbb2f18e520b88e023006acbab1e5ea9e45 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 10 Sep 2020 12:12:39 +0200
Subject: [PATCH 154/173] more validation

---
 src/matrix/room/timeline/persistence/SyncWriter.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/matrix/room/timeline/persistence/SyncWriter.js b/src/matrix/room/timeline/persistence/SyncWriter.js
index 130b22d1..9f42163d 100644
--- a/src/matrix/room/timeline/persistence/SyncWriter.js
+++ b/src/matrix/room/timeline/persistence/SyncWriter.js
@@ -140,7 +140,7 @@ export class SyncWriter {
 
     async _writeTimeline(entries, timeline, currentKey, trackNewlyJoined, txn) {
         const memberChanges = new Map();
-        if (timeline.events) {
+        if (Array.isArray(timeline.events)) {
             const events = deduplicateEvents(timeline.events);
             for(const event of events) {
                 // store event in timeline
@@ -220,6 +220,7 @@ export class SyncWriter {
         // important this happens before _writeTimeline so
         // members are available in the transaction
         const memberChanges = await this._writeStateEvents(roomResponse, trackNewlyJoined, txn);
+        // TODO: remove trackNewlyJoined and pass in memberChanges
         const timelineResult = await this._writeTimeline(entries, timeline, currentKey, trackNewlyJoined, txn);
         currentKey = timelineResult.currentKey;
         // merge member changes from state and timeline, giving precedence to the latter

From fdbc5f3c1da40fad6cc1ca4df9e9324353ee02ad Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 10 Sep 2020 13:00:11 +0200
Subject: [PATCH 155/173] WIP worker work

---
 scripts/build.mjs                             |  55 +++++++--
 .../room/timeline/TimelineViewModel.js        |   2 +-
 src/legacy-polyfill.js                        |   2 +-
 src/matrix/e2ee/megolm/Decryption.js          |   8 +-
 .../megolm/decryption/DecryptionWorker.js     |  64 +++++++++++
 .../megolm/decryption/SessionDecryption.js    |  11 +-
 src/matrix/room/timeline/Timeline.js          |   2 +-
 src/worker-polyfill.js                        |  19 +++
 src/worker.js                                 | 108 ++++++++++++++++++
 9 files changed, 248 insertions(+), 23 deletions(-)
 create mode 100644 src/matrix/e2ee/megolm/decryption/DecryptionWorker.js
 create mode 100644 src/worker-polyfill.js
 create mode 100644 src/worker.js

diff --git a/scripts/build.mjs b/scripts/build.mjs
index 15d090be..d4393d50 100644
--- a/scripts/build.mjs
+++ b/scripts/build.mjs
@@ -84,10 +84,11 @@ async function build() {
     // also creates the directories where the theme css bundles are placed in,
     // so do it first
     const themeAssets = await copyThemeAssets(themes, legacy);
-    const jsBundlePath = await buildJs();
-    const jsLegacyBundlePath = await buildJsLegacy();
+    const jsBundlePath = await buildJs("src/main.js", `${PROJECT_ID}.js`);
+    const jsLegacyBundlePath = await buildJsLegacy("src/main.js", `${PROJECT_ID}-legacy.js`);
+    const jsWorkerPath = await buildWorkerJsLegacy("src/worker.js", `worker.js`);
     const cssBundlePaths = await buildCssBundles(legacy ? buildCssLegacy : buildCss, themes, themeAssets);
-    const assetPaths = createAssetPaths(jsBundlePath, jsLegacyBundlePath, cssBundlePaths, themeAssets);
+    const assetPaths = createAssetPaths(jsBundlePath, jsLegacyBundlePath, jsWorkerPath, cssBundlePaths, themeAssets);
 
     let manifestPath;
     if (offline) {
@@ -98,7 +99,7 @@ async function build() {
     console.log(`built ${PROJECT_ID} ${version} successfully`);
 }
 
-function createAssetPaths(jsBundlePath, jsLegacyBundlePath, cssBundlePaths, themeAssets) {
+function createAssetPaths(jsBundlePath, jsLegacyBundlePath, jsWorkerPath, cssBundlePaths, themeAssets) {
     function trim(path) {
         if (!path.startsWith(targetDir)) {
             throw new Error("invalid target path: " + targetDir);
@@ -108,6 +109,7 @@ function createAssetPaths(jsBundlePath, jsLegacyBundlePath, cssBundlePaths, them
     return {
         jsBundle: () => trim(jsBundlePath),
         jsLegacyBundle: () => trim(jsLegacyBundlePath),
+        jsWorker: () => trim(jsWorkerPath),
         cssMainBundle: () => trim(cssBundlePaths.main),
         cssThemeBundle: themeName => trim(cssBundlePaths.themes[themeName]),
         cssThemeBundles: () => Object.values(cssBundlePaths.themes).map(a => trim(a)),
@@ -180,23 +182,24 @@ async function buildHtml(doc, version, assetPaths, manifestPath) {
     await fs.writeFile(path.join(targetDir, "index.html"), doc.html(), "utf8");
 }
 
-async function buildJs() {
+async function buildJs(inputFile, outputName) {
     // create js bundle
     const bundle = await rollup({
-        input: 'src/main.js',
+        input: inputFile,
         plugins: [removeJsComments({comments: "none"})]
     });
     const {output} = await bundle.generate({
         format: 'es',
+        // TODO: can remove this?
         name: `${PROJECT_ID}Bundle`
     });
     const code = output[0].code;
-    const bundlePath = resource(`${PROJECT_ID}.js`, code);
+    const bundlePath = resource(outputName, code);
     await fs.writeFile(bundlePath, code, "utf8");
     return bundlePath;
 }
 
-async function buildJsLegacy() {
+async function buildJsLegacy(inputFile, outputName) {
     // compile down to whatever IE 11 needs
     const babelPlugin = babel.babel({
         babelHelpers: 'bundled',
@@ -214,7 +217,7 @@ async function buildJsLegacy() {
     });
     // create js bundle
     const rollupConfig = {
-        input: ['src/legacy-polyfill.js', 'src/main.js'],
+        input: ['src/legacy-polyfill.js', inputFile],
         plugins: [multi(), commonjs(), nodeResolve(), babelPlugin, removeJsComments({comments: "none"})]
     };
     const bundle = await rollup(rollupConfig);
@@ -223,7 +226,39 @@ async function buildJsLegacy() {
         name: `${PROJECT_ID}Bundle`
     });
     const code = output[0].code;
-    const bundlePath = resource(`${PROJECT_ID}-legacy.js`, code);
+    const bundlePath = resource(outputName, code);
+    await fs.writeFile(bundlePath, code, "utf8");
+    return bundlePath;
+}
+
+async function buildWorkerJsLegacy(inputFile, outputName) {
+    // compile down to whatever IE 11 needs
+    const babelPlugin = babel.babel({
+        babelHelpers: 'bundled',
+        exclude: 'node_modules/**',
+        presets: [
+            [
+                "@babel/preset-env",
+                {
+                    useBuiltIns: "entry",
+                    corejs: "3",
+                    targets: "IE 11"
+                }
+            ]
+        ]
+    });
+    // create js bundle
+    const rollupConfig = {
+        input: ['src/worker-polyfill.js', inputFile],
+        plugins: [multi(), commonjs(), nodeResolve(), babelPlugin, removeJsComments({comments: "none"})]
+    };
+    const bundle = await rollup(rollupConfig);
+    const {output} = await bundle.generate({
+        format: 'iife',
+        name: `${PROJECT_ID}Bundle`
+    });
+    const code = output[0].code;
+    const bundlePath = resource(outputName, code);
     await fs.writeFile(bundlePath, code, "utf8");
     return bundlePath;
 }
diff --git a/src/domain/session/room/timeline/TimelineViewModel.js b/src/domain/session/room/timeline/TimelineViewModel.js
index 1527d3ae..16d529cb 100644
--- a/src/domain/session/room/timeline/TimelineViewModel.js
+++ b/src/domain/session/room/timeline/TimelineViewModel.js
@@ -54,7 +54,7 @@ export class TimelineViewModel extends ViewModel {
         if (firstTile.shape === "gap") {
             return firstTile.fill();
         } else {
-            await this._timeline.loadAtTop(50);
+            await this._timeline.loadAtTop(10);
             return false;
         }
     }
diff --git a/src/legacy-polyfill.js b/src/legacy-polyfill.js
index 5665158c..a48416c7 100644
--- a/src/legacy-polyfill.js
+++ b/src/legacy-polyfill.js
@@ -23,4 +23,4 @@ if (!Element.prototype.remove) {
     Element.prototype.remove = function remove() {
         this.parentNode.removeChild(this);
     };
-}
\ No newline at end of file
+}
diff --git a/src/matrix/e2ee/megolm/Decryption.js b/src/matrix/e2ee/megolm/Decryption.js
index d4352926..077e9df4 100644
--- a/src/matrix/e2ee/megolm/Decryption.js
+++ b/src/matrix/e2ee/megolm/Decryption.js
@@ -21,6 +21,7 @@ import {SessionInfo} from "./decryption/SessionInfo.js";
 import {DecryptionPreparation} from "./decryption/DecryptionPreparation.js";
 import {SessionDecryption} from "./decryption/SessionDecryption.js";
 import {SessionCache} from "./decryption/SessionCache.js";
+import {DecryptionWorker} from "./decryption/DecryptionWorker.js";
 
 function getSenderKey(event) {
     return event.content?.["sender_key"];
@@ -38,7 +39,9 @@ export class Decryption {
     constructor({pickleKey, olm}) {
         this._pickleKey = pickleKey;
         this._olm = olm;
-        // this._worker = new MessageHandler(new Worker("worker-2580578233.js"));
+        // this._decryptor = new DecryptionWorker(new Worker("./src/worker.js"));
+        this._decryptor = new DecryptionWorker(new Worker("worker-3074010154.js"));
+        this._initPromise = this._decryptor.init();
     }
 
     createSessionCache(fallback) {
@@ -55,6 +58,7 @@ export class Decryption {
      * @return {DecryptionPreparation}
      */
     async prepareDecryptAll(roomId, events, sessionCache, txn) {
+        await this._initPromise;
         const errors = new Map();
         const validEvents = [];
 
@@ -85,7 +89,7 @@ export class Decryption {
                     errors.set(event.event_id, new DecryptionError("MEGOLM_NO_SESSION", event));
                 }
             } else {
-                sessionDecryptions.push(new SessionDecryption(sessionInfo, eventsForSession));
+                sessionDecryptions.push(new SessionDecryption(sessionInfo, eventsForSession, this._decryptor));
             }
         }));
 
diff --git a/src/matrix/e2ee/megolm/decryption/DecryptionWorker.js b/src/matrix/e2ee/megolm/decryption/DecryptionWorker.js
new file mode 100644
index 00000000..df7bb748
--- /dev/null
+++ b/src/matrix/e2ee/megolm/decryption/DecryptionWorker.js
@@ -0,0 +1,64 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export class DecryptionWorker {
+    constructor(worker) {
+        this._worker = worker;
+        this._requests = new Map();
+        this._counter = 0;
+        this._worker.addEventListener("message", this);
+    }
+
+    handleEvent(e) {
+        if (e.type === "message") {
+            const message = e.data;
+            console.log("worker reply", message);
+            const request = this._requests.get(message.replyToId);
+            if (request) {
+                if (message.type === "success") {
+                    request.resolve(message.payload);
+                } else if (message.type === "error") {
+                    request.reject(new Error(message.stack));
+                }
+                this._requests.delete(message.ref_id);
+            }
+        }
+    }
+
+    _send(message) {
+        this._counter += 1;
+        message.id = this._counter;
+        let resolve;
+        let reject;
+        const promise = new Promise((_resolve, _reject) => {
+            resolve = _resolve;
+            reject = _reject;
+        });
+        this._requests.set(message.id, {reject, resolve});
+        this._worker.postMessage(message);
+        return promise;
+    }
+
+    decrypt(session, ciphertext) {
+        const sessionKey = session.export_session(session.first_known_index());
+        return this._send({type: "megolm_decrypt", ciphertext, sessionKey});
+    }
+
+    init() {
+        return this._send({type: "load_olm", path: "olm_legacy-3232457086.js"});
+        // return this._send({type: "load_olm", path: "../lib/olm/olm_legacy.js"});
+    }
+}
diff --git a/src/matrix/e2ee/megolm/decryption/SessionDecryption.js b/src/matrix/e2ee/megolm/decryption/SessionDecryption.js
index 6abda029..5fc32f58 100644
--- a/src/matrix/e2ee/megolm/decryption/SessionDecryption.js
+++ b/src/matrix/e2ee/megolm/decryption/SessionDecryption.js
@@ -22,10 +22,11 @@ import {ReplayDetectionEntry} from "./ReplayDetectionEntry.js";
  * Does the actual decryption of all events for a given megolm session in a batch
  */
 export class SessionDecryption {
-    constructor(sessionInfo, events) {
+    constructor(sessionInfo, events, decryptor) {
         sessionInfo.retain();
         this._sessionInfo = sessionInfo;
         this._events = events;
+        this._decryptor = decryptor;
     }
 
     async decryptAll() {
@@ -38,7 +39,7 @@ export class SessionDecryption {
             try {
                 const {session} = this._sessionInfo;
                 const ciphertext = event.content.ciphertext;
-                const {plaintext, message_index: messageIndex} = await this._decrypt(session, ciphertext);
+                const {plaintext, message_index: messageIndex} = await this._decryptor.decrypt(session, ciphertext);
                 let payload;
                 try {
                     payload = JSON.parse(plaintext);
@@ -63,12 +64,6 @@ export class SessionDecryption {
         return {results, errors, replayEntries};
     }
 
-    async _decrypt(session, ciphertext) {
-        // const sessionKey = session.export_session(session.first_known_index());
-        // return this._worker.decrypt(sessionKey, ciphertext);
-        return session.decrypt(ciphertext);
-    }
-
     dispose() {
         this._sessionInfo.release();
     }
diff --git a/src/matrix/room/timeline/Timeline.js b/src/matrix/room/timeline/Timeline.js
index 7245568d..53c26d82 100644
--- a/src/matrix/room/timeline/Timeline.js
+++ b/src/matrix/room/timeline/Timeline.js
@@ -42,7 +42,7 @@ export class Timeline {
 
     /** @package */
     async load() {
-        const entries = await this._timelineReader.readFromEnd(50);
+        const entries = await this._timelineReader.readFromEnd(25);
         this._remoteEntries.setManySorted(entries);
     }
 
diff --git a/src/worker-polyfill.js b/src/worker-polyfill.js
new file mode 100644
index 00000000..08bbf652
--- /dev/null
+++ b/src/worker-polyfill.js
@@ -0,0 +1,19 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// polyfills needed for IE11
+import "regenerator-runtime/runtime";
+import "core-js/modules/es.promise";
diff --git a/src/worker.js b/src/worker.js
new file mode 100644
index 00000000..e470eaa7
--- /dev/null
+++ b/src/worker.js
@@ -0,0 +1,108 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+function asErrorMessage(err) {
+    return {
+        type: "error",
+        message: err.message,
+        stack: err.stack
+    };
+}
+
+function asSuccessMessage(payload) {
+    return {
+        type: "success",
+        payload
+    };
+}
+
+class MessageHandler {
+    constructor() {
+        this._olm = null;
+    }
+
+    handleEvent(e) {
+        if (e.type === "message") {
+            this._handleMessage(e.data);
+        }
+    }
+
+    _sendReply(refMessage, reply) {
+        reply.replyToId = refMessage.id;
+        self.postMessage(reply);
+    }
+
+    _toMessage(fn) {
+        try {
+            let payload = fn();
+            if (payload instanceof Promise) {
+                return payload.then(
+                    payload => asSuccessMessage(payload),
+                    err => asErrorMessage(err)
+                );
+            } else {
+                return asSuccessMessage(payload);
+            }
+        } catch (err) {
+            return asErrorMessage(err);
+        }
+    }
+
+    _loadOlm(path) {
+        return this._toMessage(async () => {
+            // might have some problems here with window vs self as global object?
+            if (self.msCrypto && !self.crypto) {
+                self.crypto = self.msCrypto;
+            }
+            self.importScripts(path);
+            const olm = self.olm_exports;
+            // mangle the globals enough to make olm load believe it is running in a browser
+            self.window = self;
+            self.document = {};
+            await olm.init();
+            delete self.document;
+            delete self.window;
+            this._olm = olm;
+        });
+    }
+
+    _megolmDecrypt(sessionKey, ciphertext) {
+        return this._toMessage(() => {
+            let session;
+            try {
+                session = new this._olm.InboundGroupSession();
+                session.import_session(sessionKey);
+                // returns object with plaintext and message_index
+                return session.decrypt(ciphertext);
+            } finally {
+                session?.free();
+            }
+        });
+    }
+
+    async _handleMessage(message) {
+        switch (message.type) {
+            case "load_olm":
+                this._sendReply(message, await this._loadOlm(message.path));
+                break;
+            case "megolm_decrypt":
+                this._sendReply(message, this._megolmDecrypt(message.sessionKey, message.ciphertext));
+                break;
+        }
+    }
+}
+
+self.addEventListener("message", new MessageHandler());

From 0bf1723d9963df586860b7b87de10478e5f6587b Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 10 Sep 2020 15:40:30 +0100
Subject: [PATCH 156/173] Worker WIP

---
 src/domain/ViewModel.js                       |   3 +-
 src/domain/session/room/RoomViewModel.js      |   3 +-
 src/matrix/e2ee/RoomEncryption.js             |   2 +-
 src/matrix/e2ee/megolm/Decryption.js          |   4 +-
 .../megolm/decryption/DecryptionWorker.js     | 191 ++++++++++++++++--
 .../megolm/decryption/SessionDecryption.js    |  22 +-
 src/matrix/room/Room.js                       |  89 +++++---
 src/matrix/room/timeline/Timeline.js          |  26 ++-
 .../timeline/persistence/TimelineReader.js    |  72 ++++---
 src/utils/Disposables.js                      |   1 +
 src/worker-polyfill.js                        |   4 +
 src/worker.js                                 |  14 +-
 12 files changed, 339 insertions(+), 92 deletions(-)

diff --git a/src/domain/ViewModel.js b/src/domain/ViewModel.js
index bc35fabd..15812f8c 100644
--- a/src/domain/ViewModel.js
+++ b/src/domain/ViewModel.js
@@ -36,8 +36,7 @@ export class ViewModel extends EventEmitter {
         if (!this.disposables) {
             this.disposables = new Disposables();
         }
-        this.disposables.track(disposable);
-        return disposable;
+        return this.disposables.track(disposable);
     }
 
     dispose() {
diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js
index 32e09fbe..59bcd52f 100644
--- a/src/domain/session/room/RoomViewModel.js
+++ b/src/domain/session/room/RoomViewModel.js
@@ -38,7 +38,8 @@ export class RoomViewModel extends ViewModel {
     async load() {
         this._room.on("change", this._onRoomChange);
         try {
-            this._timeline = await this._room.openTimeline();
+            this._timeline = this._room.openTimeline();
+            await this._timeline.load();
             this._timelineVM = new TimelineViewModel(this.childOptions({
                 room: this._room,
                 timeline: this._timeline,
diff --git a/src/matrix/e2ee/RoomEncryption.js b/src/matrix/e2ee/RoomEncryption.js
index 1341389b..544081b3 100644
--- a/src/matrix/e2ee/RoomEncryption.js
+++ b/src/matrix/e2ee/RoomEncryption.js
@@ -268,7 +268,7 @@ class DecryptionPreparation {
     }
 
     dispose() {
-        this._megolmDecryptionChanges.dispose();
+        this._megolmDecryptionPreparation.dispose();
     }
 }
 
diff --git a/src/matrix/e2ee/megolm/Decryption.js b/src/matrix/e2ee/megolm/Decryption.js
index 077e9df4..6121192d 100644
--- a/src/matrix/e2ee/megolm/Decryption.js
+++ b/src/matrix/e2ee/megolm/Decryption.js
@@ -21,7 +21,7 @@ import {SessionInfo} from "./decryption/SessionInfo.js";
 import {DecryptionPreparation} from "./decryption/DecryptionPreparation.js";
 import {SessionDecryption} from "./decryption/SessionDecryption.js";
 import {SessionCache} from "./decryption/SessionCache.js";
-import {DecryptionWorker} from "./decryption/DecryptionWorker.js";
+import {DecryptionWorker, WorkerPool} from "./decryption/DecryptionWorker.js";
 
 function getSenderKey(event) {
     return event.content?.["sender_key"];
@@ -40,7 +40,7 @@ export class Decryption {
         this._pickleKey = pickleKey;
         this._olm = olm;
         // this._decryptor = new DecryptionWorker(new Worker("./src/worker.js"));
-        this._decryptor = new DecryptionWorker(new Worker("worker-3074010154.js"));
+        this._decryptor = new DecryptionWorker(new WorkerPool("worker-1039452087.js", 4));
         this._initPromise = this._decryptor.init();
     }
 
diff --git a/src/matrix/e2ee/megolm/decryption/DecryptionWorker.js b/src/matrix/e2ee/megolm/decryption/DecryptionWorker.js
index df7bb748..9cc64d74 100644
--- a/src/matrix/e2ee/megolm/decryption/DecryptionWorker.js
+++ b/src/matrix/e2ee/megolm/decryption/DecryptionWorker.js
@@ -14,51 +14,200 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-export class DecryptionWorker {
+import {AbortError} from "../../../../utils/error.js";
+
+class WorkerState {
     constructor(worker) {
-        this._worker = worker;
+        this.worker = worker;
+        this.busy = false;
+    }
+
+    attach(pool) {
+        this.worker.addEventListener("message", pool);
+        this.worker.addEventListener("error", pool);
+    }
+
+    detach(pool) {
+        this.worker.removeEventListener("message", pool);
+        this.worker.removeEventListener("error", pool);
+    }
+}
+
+class Request {
+    constructor(message, pool) {
+        this._promise = new Promise((_resolve, _reject) => {
+            this._resolve = _resolve;
+            this._reject = _reject;
+        });
+        this._message = message;
+        this._pool = pool;
+        this._worker = null;
+    }
+
+    abort() {
+        if (this._isNotDisposed) {
+            this._pool._abortRequest(this);
+            this._dispose();
+        }
+    }
+
+    response() {
+        return this._promise;
+    }
+
+    _dispose() {
+        this._reject = null;
+        this._resolve = null;
+    }
+
+    get _isNotDisposed() {
+        return this._resolve && this._reject;
+    }
+}
+
+export class WorkerPool {
+    constructor(path, amount) {
+        this._workers = [];
+        for (let i = 0; i < amount ; ++i) {
+            const worker = new WorkerState(new Worker(path));
+            worker.attach(this);
+            this._workers[i] = worker;
+        }
         this._requests = new Map();
         this._counter = 0;
-        this._worker.addEventListener("message", this);
+        this._pendingFlag = false;
     }
 
     handleEvent(e) {
         if (e.type === "message") {
             const message = e.data;
-            console.log("worker reply", message);
             const request = this._requests.get(message.replyToId);
             if (request) {
-                if (message.type === "success") {
-                    request.resolve(message.payload);
-                } else if (message.type === "error") {
-                    request.reject(new Error(message.stack));
+                request._worker.busy = false;
+                if (request._isNotDisposed) {
+                    if (message.type === "success") {
+                        request._resolve(message.payload);
+                    } else if (message.type === "error") {
+                        request._reject(new Error(message.stack));
+                    }
+                    request._dispose();
                 }
-                this._requests.delete(message.ref_id);
+                this._requests.delete(message.replyToId);
+            }
+            console.log("got worker reply", message, this._requests.size);
+            this._sendPending();
+        } else if (e.type === "error") {
+            console.error("worker error", e);
+        }
+    }
+
+    _getPendingRequest() {
+        for (const r of this._requests.values()) {
+            if (!r._worker) {
+                return r;
             }
         }
     }
 
-    _send(message) {
+    _getFreeWorker() {
+        for (const w of this._workers) {
+            if (!w.busy) {
+                return w;
+            }
+        }
+    }
+
+    _sendPending() {
+        this._pendingFlag = false;
+        console.log("seeing if there is anything to send", this._requests.size);
+        let success;
+        do {
+            success = false;
+            const request = this._getPendingRequest();
+            if (request) {
+                console.log("sending pending request", request);
+                const worker = this._getFreeWorker();
+                if (worker) {
+                    this._sendWith(request, worker);
+                    success = true;
+                }
+            }
+        } while (success);
+    }
+
+    _sendWith(request, worker) {
+        request._worker = worker;
+        worker.busy = true;
+        console.log("sending message to worker", request._message);
+        worker.worker.postMessage(request._message);
+    }
+
+    _enqueueRequest(message) {
         this._counter += 1;
         message.id = this._counter;
-        let resolve;
-        let reject;
-        const promise = new Promise((_resolve, _reject) => {
-            resolve = _resolve;
-            reject = _reject;
+        const request = new Request(message, this);
+        this._requests.set(message.id, request);
+        return request;
+    }
+
+    send(message) {
+        const request = this._enqueueRequest(message);
+        const worker = this._getFreeWorker();
+        if (worker) {
+            this._sendWith(request, worker);
+        }
+        return request;
+    }
+
+    // assumes all workers are free atm
+    sendAll(message) {
+        const promises = this._workers.map(worker => {
+            const request = this._enqueueRequest(message);
+            this._sendWith(request, worker);
+            return request.response();
         });
-        this._requests.set(message.id, {reject, resolve});
-        this._worker.postMessage(message);
-        return promise;
+        return Promise.all(promises);
+    }
+
+    dispose() {
+        for (const w of this._workers) {
+            w.worker.terminate();
+            w.detach(this);
+        }
+    }
+
+    _trySendPendingInNextTick() {
+        if (!this._pendingFlag) {
+            this._pendingFlag = true;
+            Promise.resolve().then(() => {
+                this._sendPending();
+            });
+        }
+    }
+
+    _abortRequest(request) {
+        request._reject(new AbortError());
+        if (request._worker) {
+            request._worker.busy = false;
+        }
+        this._requests.delete(request._message.id);
+        // allow more requests to be aborted before trying to send other pending
+        this._trySendPendingInNextTick();
+    }
+}
+
+export class DecryptionWorker {
+    constructor(workerPool) {
+        this._workerPool = workerPool;
     }
 
     decrypt(session, ciphertext) {
         const sessionKey = session.export_session(session.first_known_index());
-        return this._send({type: "megolm_decrypt", ciphertext, sessionKey});
+        return this._workerPool.send({type: "megolm_decrypt", ciphertext, sessionKey});
     }
 
-    init() {
-        return this._send({type: "load_olm", path: "olm_legacy-3232457086.js"});
+    async init() {
+        await this._workerPool.sendAll({type: "load_olm", path: "olm_legacy-3232457086.js"});
         // return this._send({type: "load_olm", path: "../lib/olm/olm_legacy.js"});
     }
 }
diff --git a/src/matrix/e2ee/megolm/decryption/SessionDecryption.js b/src/matrix/e2ee/megolm/decryption/SessionDecryption.js
index 5fc32f58..30ca432e 100644
--- a/src/matrix/e2ee/megolm/decryption/SessionDecryption.js
+++ b/src/matrix/e2ee/megolm/decryption/SessionDecryption.js
@@ -27,6 +27,7 @@ export class SessionDecryption {
         this._sessionInfo = sessionInfo;
         this._events = events;
         this._decryptor = decryptor;
+        this._decryptionRequests = decryptor ? [] : null;
     }
 
     async decryptAll() {
@@ -39,7 +40,16 @@ export class SessionDecryption {
             try {
                 const {session} = this._sessionInfo;
                 const ciphertext = event.content.ciphertext;
-                const {plaintext, message_index: messageIndex} = await this._decryptor.decrypt(session, ciphertext);
+                let decryptionResult;
+                if (this._decryptor) {
+                    const request = this._decryptor.decrypt(session, ciphertext);
+                    this._decryptionRequests.push(request);
+                    decryptionResult = await request.response();
+                } else {
+                    decryptionResult = session.decrypt(ciphertext);
+                }
+                const plaintext = decryptionResult.plaintext;
+                const messageIndex = decryptionResult.message_index;
                 let payload;
                 try {
                     payload = JSON.parse(plaintext);
@@ -54,6 +64,10 @@ export class SessionDecryption {
                 const result = new DecryptionResult(payload, this._sessionInfo.senderKey, this._sessionInfo.claimedKeys);
                 results.set(event.event_id, result);
             } catch (err) {
+                // ignore AbortError from cancelling decryption requests in dispose method
+                if (err.name === "AbortError") {
+                    return;
+                }
                 if (!errors) {
                     errors = new Map();
                 }
@@ -65,6 +79,12 @@ export class SessionDecryption {
     }
 
     dispose() {
+        if (this._decryptionRequests) {
+            for (const r of this._decryptionRequests) {
+                r.abort();
+            }
+        }
+        // TODO: cancel decryptions here
         this._sessionInfo.release();
     }
 }
diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index f281f166..233990a3 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -65,7 +65,8 @@ export class Room extends EventEmitter {
                         retryEntries.push(new EventEntry(storageEntry, this._fragmentIdComparer));
                     }
                 }
-                await this._decryptEntries(DecryptionSource.Retry, retryEntries, txn);
+                const decryptRequest = this._decryptEntries(DecryptionSource.Retry, retryEntries, txn);
+                await decryptRequest.complete();
                 if (this._timeline) {
                     // only adds if already present
                     this._timeline.replaceEntries(retryEntries);
@@ -89,31 +90,39 @@ export class Room extends EventEmitter {
      * Used for decrypting when loading/filling the timeline, and retrying decryption,
      * not during sync, where it is split up during the multiple phases.
      */
-    async _decryptEntries(source, entries, inboundSessionTxn = null) {
-        if (!inboundSessionTxn) {
-            inboundSessionTxn = await this._storage.readTxn([this._storage.storeNames.inboundGroupSessions]);
-        }
-        const events = entries.filter(entry => {
-            return entry.eventType === EVENT_ENCRYPTED_TYPE;
-        }).map(entry => entry.event);
-        const isTimelineOpen = this._isTimelineOpen;
-        const preparation = await this._roomEncryption.prepareDecryptAll(events, source, isTimelineOpen, inboundSessionTxn);
-        const changes = await preparation.decrypt();
-        const stores = [this._storage.storeNames.groupSessionDecryptions];
-        if (isTimelineOpen) {
-            // read to fetch devices if timeline is open
-            stores.push(this._storage.storeNames.deviceIdentities);
-        }
-        const writeTxn = await this._storage.readWriteTxn(stores);
-        let decryption;
-        try {
-            decryption = await changes.write(writeTxn);
-        } catch (err) {
-            writeTxn.abort();
-            throw err;
-        }
-        await writeTxn.complete();
-        decryption.applyToEntries(entries);
+    _decryptEntries(source, entries, inboundSessionTxn = null) {
+        const request = new DecryptionRequest(async r => {
+            if (!inboundSessionTxn) {
+                inboundSessionTxn = await this._storage.readTxn([this._storage.storeNames.inboundGroupSessions]);
+            }
+            if (r.cancelled) return;
+            const events = entries.filter(entry => {
+                return entry.eventType === EVENT_ENCRYPTED_TYPE;
+            }).map(entry => entry.event);
+            const isTimelineOpen = this._isTimelineOpen;
+            r.preparation = await this._roomEncryption.prepareDecryptAll(events, source, isTimelineOpen, inboundSessionTxn);
+            if (r.cancelled) return;
+            // TODO: should this throw an AbortError?
+            const changes = await r.preparation.decrypt();
+            r.preparation = null;
+            if (r.cancelled) return;
+            const stores = [this._storage.storeNames.groupSessionDecryptions];
+            if (isTimelineOpen) {
+                // read to fetch devices if timeline is open
+                stores.push(this._storage.storeNames.deviceIdentities);
+            }
+            const writeTxn = await this._storage.readWriteTxn(stores);
+            let decryption;
+            try {
+                decryption = await changes.write(writeTxn);
+            } catch (err) {
+                writeTxn.abort();
+                throw err;
+            }
+            await writeTxn.complete();
+            decryption.applyToEntries(entries);
+        });
+        return request;
     }
 
     get needsPrepareSync() {
@@ -349,7 +358,8 @@ export class Room extends EventEmitter {
         }
         await txn.complete();
         if (this._roomEncryption) {
-            await this._decryptEntries(DecryptionSource.Timeline, gapResult.entries);
+            const decryptRequest = this._decryptEntries(DecryptionSource.Timeline, gapResult.entries);
+            await decryptRequest.complete();
         }
         // once txn is committed, update in-memory state & emit events
         for (const fragment of gapResult.fragments) {
@@ -461,7 +471,7 @@ export class Room extends EventEmitter {
     }
 
     /** @public */
-    async openTimeline() {
+    openTimeline() {
         if (this._timeline) {
             throw new Error("not dealing with load race here for now");
         }
@@ -483,7 +493,6 @@ export class Room extends EventEmitter {
         if (this._roomEncryption) {
             this._timeline.enableEncryption(this._decryptEntries.bind(this, DecryptionSource.Timeline));
         }
-        await this._timeline.load();
         return this._timeline;
     }
 
@@ -502,3 +511,25 @@ export class Room extends EventEmitter {
     }
 }
 
+class DecryptionRequest {
+    constructor(decryptFn) {
+        this._cancelled = false;
+        this.preparation = null;
+        this._promise = decryptFn(this);
+    }
+
+    complete() {
+        return this._promise;
+    }
+
+    get cancelled() {
+        return this._cancelled;
+    }
+
+    dispose() {
+        this._cancelled = true;
+        if (this.preparation) {
+            this.preparation.dispose();
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/matrix/room/timeline/Timeline.js b/src/matrix/room/timeline/Timeline.js
index 53c26d82..38dc1c6a 100644
--- a/src/matrix/room/timeline/Timeline.js
+++ b/src/matrix/room/timeline/Timeline.js
@@ -15,6 +15,7 @@ limitations under the License.
 */
 
 import {SortedArray, MappedList, ConcatList} from "../../../observable/index.js";
+import {Disposables} from "../../../utils/Disposables.js";
 import {Direction} from "./Direction.js";
 import {TimelineReader} from "./persistence/TimelineReader.js";
 import {PendingEventEntry} from "./entries/PendingEventEntry.js";
@@ -26,12 +27,14 @@ export class Timeline {
         this._storage = storage;
         this._closeCallback = closeCallback;
         this._fragmentIdComparer = fragmentIdComparer;
+        this._disposables = new Disposables();
         this._remoteEntries = new SortedArray((a, b) => a.compare(b));
         this._timelineReader = new TimelineReader({
             roomId: this._roomId,
             storage: this._storage,
             fragmentIdComparer: this._fragmentIdComparer
         });
+        this._readerRequest = null;
         const localEntries = new MappedList(pendingEvents, pe => {
             return new PendingEventEntry({pendingEvent: pe, user});
         }, (pee, params) => {
@@ -42,8 +45,14 @@ export class Timeline {
 
     /** @package */
     async load() {
-        const entries = await this._timelineReader.readFromEnd(25);
-        this._remoteEntries.setManySorted(entries);
+        // 30 seems to be a good amount to fill the entire screen
+        const readerRequest = this._disposables.track(this._timelineReader.readFromEnd(30));
+        try {
+            const entries = await readerRequest.complete();
+            this._remoteEntries.setManySorted(entries);
+        } finally {
+            this._disposables.disposeTracked(readerRequest);
+        }
     }
 
     replaceEntries(entries) {
@@ -71,12 +80,17 @@ export class Timeline {
         if (!firstEventEntry) {
             return;
         }
-        const entries = await this._timelineReader.readFrom(
+        const readerRequest = this._disposables.track(this._timelineReader.readFrom(
             firstEventEntry.asEventKey(),
             Direction.Backward,
             amount
-        );
-        this._remoteEntries.setManySorted(entries);
+        ));
+        try {
+            const entries = await readerRequest.complete();
+            this._remoteEntries.setManySorted(entries);
+        } finally {
+            this._disposables.disposeTracked(readerRequest);
+        }
     }
 
     /** @public */
@@ -87,6 +101,8 @@ export class Timeline {
     /** @public */
     close() {
         if (this._closeCallback) {
+            this._readerRequest?.dispose();
+            this._readerRequest = null;
             this._closeCallback();
             this._closeCallback = null;
         }
diff --git a/src/matrix/room/timeline/persistence/TimelineReader.js b/src/matrix/room/timeline/persistence/TimelineReader.js
index d451d76e..f5983a19 100644
--- a/src/matrix/room/timeline/persistence/TimelineReader.js
+++ b/src/matrix/room/timeline/persistence/TimelineReader.js
@@ -19,6 +19,24 @@ import {Direction} from "../Direction.js";
 import {EventEntry} from "../entries/EventEntry.js";
 import {FragmentBoundaryEntry} from "../entries/FragmentBoundaryEntry.js";
 
+class ReaderRequest {
+    constructor(fn) {
+        this.decryptRequest = null;
+        this._promise = fn(this);
+    }
+
+    complete() {
+        return this._promise;
+    }
+
+    dispose() {
+        if (this.decryptRequest) {
+            this.decryptRequest.dispose();
+            this.decryptRequest = null;
+        }
+    }
+}
+
 export class TimelineReader {
     constructor({roomId, storage, fragmentIdComparer}) {
         this._roomId = roomId;
@@ -42,12 +60,33 @@ export class TimelineReader {
         return this._storage.readTxn(stores);
     }
 
-    async readFrom(eventKey, direction, amount) {
-        const txn = await this._openTxn();
-        return await this._readFrom(eventKey, direction, amount, txn);
+    readFrom(eventKey, direction, amount) {
+        return new ReaderRequest(async r => {
+            const txn = await this._openTxn();
+            return await this._readFrom(eventKey, direction, amount, r, txn);
+        });
     }
 
-    async _readFrom(eventKey, direction, amount, txn) {
+    readFromEnd(amount) {
+        return new ReaderRequest(async r => {
+            const txn = await this._openTxn();
+            const liveFragment = await txn.timelineFragments.liveFragment(this._roomId);
+            let entries;
+            // room hasn't been synced yet
+            if (!liveFragment) {
+                entries = [];
+            } else {
+                this._fragmentIdComparer.add(liveFragment);
+                const liveFragmentEntry = FragmentBoundaryEntry.end(liveFragment, this._fragmentIdComparer);
+                const eventKey = liveFragmentEntry.asEventKey();
+                entries = await this._readFrom(eventKey, Direction.Backward, amount, r, txn);
+                entries.unshift(liveFragmentEntry);
+            }
+            return entries;
+        });
+    }
+
+    async _readFrom(eventKey, direction, amount, r, txn) {
         let entries = [];
         const timelineStore = txn.timelineEvents;
         const fragmentStore = txn.timelineFragments;
@@ -83,25 +122,12 @@ export class TimelineReader {
         }
 
         if (this._decryptEntries) {
-            await this._decryptEntries(entries, txn);
-        }
-
-        return entries;
-    }
-
-    async readFromEnd(amount) {
-        const txn = await this._openTxn();
-        const liveFragment = await txn.timelineFragments.liveFragment(this._roomId);
-        let entries;
-        // room hasn't been synced yet
-        if (!liveFragment) {
-            entries = [];
-        } else {
-            this._fragmentIdComparer.add(liveFragment);
-            const liveFragmentEntry = FragmentBoundaryEntry.end(liveFragment, this._fragmentIdComparer);
-            const eventKey = liveFragmentEntry.asEventKey();
-            entries = await this._readFrom(eventKey, Direction.Backward, amount, txn);
-            entries.unshift(liveFragmentEntry);
+            r.decryptRequest = this._decryptEntries(entries, txn);
+            try {
+                await r.decryptRequest.complete();
+            } finally {
+                r.decryptRequest = null;
+            }
         }
         return entries;
     }
diff --git a/src/utils/Disposables.js b/src/utils/Disposables.js
index e5690319..e020ef83 100644
--- a/src/utils/Disposables.js
+++ b/src/utils/Disposables.js
@@ -29,6 +29,7 @@ export class Disposables {
 
     track(disposable) {
         this._disposables.push(disposable);
+        return disposable;
     }
 
     dispose() {
diff --git a/src/worker-polyfill.js b/src/worker-polyfill.js
index 08bbf652..15b955d5 100644
--- a/src/worker-polyfill.js
+++ b/src/worker-polyfill.js
@@ -14,6 +14,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+
 // polyfills needed for IE11
+// just enough to run olm, have promises and async/await
 import "regenerator-runtime/runtime";
 import "core-js/modules/es.promise";
+import "core-js/modules/es.math.imul";
+import "core-js/modules/es.math.clz32";
diff --git a/src/worker.js b/src/worker.js
index e470eaa7..4b2a1e43 100644
--- a/src/worker.js
+++ b/src/worker.js
@@ -94,13 +94,13 @@ class MessageHandler {
     }
 
     async _handleMessage(message) {
-        switch (message.type) {
-            case "load_olm":
-                this._sendReply(message, await this._loadOlm(message.path));
-                break;
-            case "megolm_decrypt":
-                this._sendReply(message, this._megolmDecrypt(message.sessionKey, message.ciphertext));
-                break;
+        const {type} = message;
+        if (type === "ping") {
+            this._sendReply(message, {type: "pong"});
+        } else if (type === "load_olm") {
+            this._sendReply(message, await this._loadOlm(message.path));
+        } else if (type === "megolm_decrypt") {
+            this._sendReply(message, this._megolmDecrypt(message.sessionKey, message.ciphertext));
         }
     }
 }

From de1cc0d7395f4831032be3b061ba6c0925cd63d7 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 10 Sep 2020 17:43:01 +0200
Subject: [PATCH 157/173] abort decrypt requests when changing room

---
 src/domain/session/room/RoomViewModel.js               | 10 ++++------
 src/matrix/e2ee/RoomEncryption.js                      |  2 --
 src/matrix/e2ee/megolm/Decryption.js                   |  2 +-
 .../e2ee/megolm/decryption/DecryptionPreparation.js    |  3 +++
 src/matrix/e2ee/megolm/decryption/DecryptionWorker.js  |  8 ++------
 src/matrix/room/Room.js                                |  2 +-
 src/matrix/room/timeline/Timeline.js                   |  6 ++----
 src/utils/Disposables.js                               |  9 ++++++++-
 8 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js
index 59bcd52f..a2ea5f66 100644
--- a/src/domain/session/room/RoomViewModel.js
+++ b/src/domain/session/room/RoomViewModel.js
@@ -38,7 +38,7 @@ export class RoomViewModel extends ViewModel {
     async load() {
         this._room.on("change", this._onRoomChange);
         try {
-            this._timeline = this._room.openTimeline();
+            this._timeline = this.track(this._room.openTimeline());
             await this._timeline.load();
             this._timelineVM = new TimelineViewModel(this.childOptions({
                 room: this._room,
@@ -63,17 +63,15 @@ export class RoomViewModel extends ViewModel {
     }
 
     dispose() {
-        // this races with enable, on the await openTimeline()
-        if (this._timeline) {
-            // will stop the timeline from delivering updates on entries
-            this._timeline.close();
-        }
+        super.dispose();
         if (this._clearUnreadTimout) {
             this._clearUnreadTimout.abort();
             this._clearUnreadTimout = null;
         }
     }
 
+    // called from view to close room
+    // parent vm will dispose this vm
     close() {
         this._closeCallback();
     }
diff --git a/src/matrix/e2ee/RoomEncryption.js b/src/matrix/e2ee/RoomEncryption.js
index 544081b3..44229b97 100644
--- a/src/matrix/e2ee/RoomEncryption.js
+++ b/src/matrix/e2ee/RoomEncryption.js
@@ -292,11 +292,9 @@ class BatchDecryptionResult {
     constructor(results, errors) {
         this.results = results;
         this.errors = errors;
-        console.log("BatchDecryptionResult", this);
     }
 
     applyToEntries(entries) {
-        console.log("BatchDecryptionResult.applyToEntries", this);
         for (const entry of entries) {
             const result = this.results.get(entry.id);
             if (result) {
diff --git a/src/matrix/e2ee/megolm/Decryption.js b/src/matrix/e2ee/megolm/Decryption.js
index 6121192d..9726e6d8 100644
--- a/src/matrix/e2ee/megolm/Decryption.js
+++ b/src/matrix/e2ee/megolm/Decryption.js
@@ -39,8 +39,8 @@ export class Decryption {
     constructor({pickleKey, olm}) {
         this._pickleKey = pickleKey;
         this._olm = olm;
-        // this._decryptor = new DecryptionWorker(new Worker("./src/worker.js"));
         this._decryptor = new DecryptionWorker(new WorkerPool("worker-1039452087.js", 4));
+        //this._decryptor = new DecryptionWorker(new WorkerPool("./src/worker.js", 4));
         this._initPromise = this._decryptor.init();
     }
 
diff --git a/src/matrix/e2ee/megolm/decryption/DecryptionPreparation.js b/src/matrix/e2ee/megolm/decryption/DecryptionPreparation.js
index 02ee32df..e24d70af 100644
--- a/src/matrix/e2ee/megolm/decryption/DecryptionPreparation.js
+++ b/src/matrix/e2ee/megolm/decryption/DecryptionPreparation.js
@@ -28,6 +28,9 @@ export class DecryptionPreparation {
     }
 
     async decrypt() {
+        // console.log("start sleeping");
+        // await new Promise(resolve => setTimeout(resolve, 5000));
+        // console.log("done sleeping");
         try {
             const errors = this._initialErrors;
             const results = new Map();
diff --git a/src/matrix/e2ee/megolm/decryption/DecryptionWorker.js b/src/matrix/e2ee/megolm/decryption/DecryptionWorker.js
index 9cc64d74..38a474ed 100644
--- a/src/matrix/e2ee/megolm/decryption/DecryptionWorker.js
+++ b/src/matrix/e2ee/megolm/decryption/DecryptionWorker.js
@@ -94,7 +94,6 @@ export class WorkerPool {
                 }
                 this._requests.delete(message.replyToId);
             }
-            console.log("got worker reply", message, this._requests.size);
             this._sendPending();
         } else if (e.type === "error") {
             console.error("worker error", e);
@@ -119,13 +118,11 @@ export class WorkerPool {
 
     _sendPending() {
         this._pendingFlag = false;
-        console.log("seeing if there is anything to send", this._requests.size);
         let success;
         do {
             success = false;
             const request = this._getPendingRequest();
             if (request) {
-                console.log("sending pending request", request);
                 const worker = this._getFreeWorker();
                 if (worker) {
                     this._sendWith(request, worker);
@@ -138,7 +135,6 @@ export class WorkerPool {
     _sendWith(request, worker) {
         request._worker = worker;
         worker.busy = true;
-        console.log("sending message to worker", request._message);
         worker.worker.postMessage(request._message);
     }
 
@@ -162,7 +158,7 @@ export class WorkerPool {
     // assumes all workers are free atm
     sendAll(message) {
         const promises = this._workers.map(worker => {
-            const request = this._enqueueRequest(message);
+            const request = this._enqueueRequest(Object.assign({}, message));
             this._sendWith(request, worker);
             return request.response();
         });
@@ -208,6 +204,6 @@ export class DecryptionWorker {
 
     async init() {
         await this._workerPool.sendAll({type: "load_olm", path: "olm_legacy-3232457086.js"});
-        // return this._send({type: "load_olm", path: "../lib/olm/olm_legacy.js"});
+        //await this._workerPool.sendAll({type: "load_olm", path: "../lib/olm/olm_legacy.js"});
     }
 }
diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index 233990a3..98f114a5 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -532,4 +532,4 @@ class DecryptionRequest {
             this.preparation.dispose();
         }
     }
-}
\ No newline at end of file
+}
diff --git a/src/matrix/room/timeline/Timeline.js b/src/matrix/room/timeline/Timeline.js
index 38dc1c6a..74362a13 100644
--- a/src/matrix/room/timeline/Timeline.js
+++ b/src/matrix/room/timeline/Timeline.js
@@ -19,7 +19,6 @@ import {Disposables} from "../../../utils/Disposables.js";
 import {Direction} from "./Direction.js";
 import {TimelineReader} from "./persistence/TimelineReader.js";
 import {PendingEventEntry} from "./entries/PendingEventEntry.js";
-import {EventEntry} from "./entries/EventEntry.js";
 
 export class Timeline {
     constructor({roomId, storage, closeCallback, fragmentIdComparer, pendingEvents, user}) {
@@ -99,10 +98,9 @@ export class Timeline {
     }
 
     /** @public */
-    close() {
+    dispose() {
         if (this._closeCallback) {
-            this._readerRequest?.dispose();
-            this._readerRequest = null;
+            this._disposables.dispose();
             this._closeCallback();
             this._closeCallback = null;
         }
diff --git a/src/utils/Disposables.js b/src/utils/Disposables.js
index e020ef83..efc49897 100644
--- a/src/utils/Disposables.js
+++ b/src/utils/Disposables.js
@@ -28,6 +28,9 @@ export class Disposables {
     }
 
     track(disposable) {
+        if (this.isDisposed) {
+            throw new Error("Already disposed, check isDisposed after await if needed");
+        }
         this._disposables.push(disposable);
         return disposable;
     }
@@ -41,8 +44,12 @@ export class Disposables {
         }
     }
 
+    get isDisposed() {
+        return this._disposables === null;
+    }
+
     disposeTracked(value) {
-        if (value === undefined || value === null) {
+        if (value === undefined || value === null || this.isDisposed) {
             return null;
         }
         const idx = this._disposables.indexOf(value);

From af36c71a5974de8241f7fafa78e48b998a21f3ee Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 10 Sep 2020 18:41:23 +0200
Subject: [PATCH 158/173] load worker in main and pass paths so it works both
 on compiled and non-compiled

---
 index.html                                    |   9 +-
 scripts/build.mjs                             |   8 +-
 src/main.js                                   |  30 ++-
 src/matrix/Session.js                         |   5 +-
 src/matrix/SessionContainer.js                |  10 +-
 src/matrix/e2ee/megolm/Decryption.js          |   9 +-
 .../megolm/decryption/DecryptionWorker.js     | 183 ---------------
 src/utils/WorkerPool.js                       | 212 ++++++++++++++++++
 src/worker.js                                 |   2 +-
 9 files changed, 268 insertions(+), 200 deletions(-)
 create mode 100644 src/utils/WorkerPool.js

diff --git a/index.html b/index.html
index b09286a0..74e44c99 100644
--- a/index.html
+++ b/index.html
@@ -19,9 +19,12 @@
 		
         ` +
+        `` +
         `` +
-        ``);
+        ``);
     removeOrEnableScript(doc("script#service-worker"), offline);
 
     const versionScript = doc("script#version");
diff --git a/src/main.js b/src/main.js
index 79f5698d..6f279910 100644
--- a/src/main.js
+++ b/src/main.js
@@ -25,6 +25,7 @@ import {BrawlViewModel} from "./domain/BrawlViewModel.js";
 import {BrawlView} from "./ui/web/BrawlView.js";
 import {Clock} from "./ui/web/dom/Clock.js";
 import {OnlineStatus} from "./ui/web/dom/OnlineStatus.js";
+import {WorkerPool} from "./utils/WorkerPool.js";
 
 function addScript(src) {
     return new Promise(function (resolve, reject) {
@@ -55,10 +56,27 @@ async function loadOlm(olmPaths) {
     return null;
 }
 
+// make path relative to basePath,
+// assuming it and basePath are relative to document
+function relPath(path, basePath) {
+    const idx = basePath.lastIndexOf("/");
+    const dir = idx === -1 ? "" : basePath.slice(0, idx);
+    const dirCount = dir.length ? dir.split("/").length : 0;
+    return "../".repeat(dirCount) + path;
+}
+
+async function loadWorker(paths) {
+    const workerPool = new WorkerPool(paths.worker, 4);
+    await workerPool.init();
+    const path = relPath(paths.olm.legacyBundle, paths.worker);
+    await workerPool.sendAll({type: "load_olm", path});
+    return workerPool;
+}
+
 // Don't use a default export here, as we use multiple entries during legacy build,
 // which does not support default exports,
 // see https://github.com/rollup/plugins/tree/master/packages/multi-entry
-export async function main(container, olmPaths) {
+export async function main(container, paths) {
     try {
         // to replay:
         // const fetchLog = await (await fetch("/fetchlogs/constrainterror.json")).json();
@@ -79,6 +97,13 @@ export async function main(container, olmPaths) {
         const sessionInfoStorage = new SessionInfoStorage("brawl_sessions_v1");
         const storageFactory = new StorageFactory();
 
+        // if wasm is not supported, we'll want
+        // to run some olm operations in a worker (mainly for IE11)
+        let workerPromise;
+        if (!window.WebAssembly) {
+            workerPromise = loadWorker(paths);
+        }
+
         const vm = new BrawlViewModel({
             createSessionContainer: () => {
                 return new SessionContainer({
@@ -88,7 +113,8 @@ export async function main(container, olmPaths) {
                     sessionInfoStorage,
                     request,
                     clock,
-                    olmPromise: loadOlm(olmPaths),
+                    olmPromise: loadOlm(paths.olm),
+                    workerPromise,
                 });
             },
             sessionInfoStorage,
diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index c5c0c94f..be3f6a06 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -33,7 +33,7 @@ const PICKLE_KEY = "DEFAULT_KEY";
 
 export class Session {
     // sessionInfo contains deviceId, userId and homeServer
-    constructor({clock, storage, hsApi, sessionInfo, olm}) {
+    constructor({clock, storage, hsApi, sessionInfo, olm, workerPool}) {
         this._clock = clock;
         this._storage = storage;
         this._hsApi = hsApi;
@@ -52,6 +52,7 @@ export class Session {
         this._megolmEncryption = null;
         this._megolmDecryption = null;
         this._getSyncToken = () => this.syncToken;
+        this._workerPool = workerPool;
 
         if (olm) {
             this._olmUtil = new olm.Utility();
@@ -100,6 +101,7 @@ export class Session {
         this._megolmDecryption = new MegOlmDecryption({
             pickleKey: PICKLE_KEY,
             olm: this._olm,
+            workerPool: this._workerPool,
         });
         this._deviceMessageHandler.enableEncryption({olmDecryption, megolmDecryption: this._megolmDecryption});
     }
@@ -202,6 +204,7 @@ export class Session {
     }
 
     stop() {
+        this._workerPool?.dispose();
         this._sendScheduler.stop();
     }
 
diff --git a/src/matrix/SessionContainer.js b/src/matrix/SessionContainer.js
index c07190eb..1e868eba 100644
--- a/src/matrix/SessionContainer.js
+++ b/src/matrix/SessionContainer.js
@@ -42,7 +42,7 @@ export const LoginFailure = createEnum(
 );
 
 export class SessionContainer {
-    constructor({clock, random, onlineStatus, request, storageFactory, sessionInfoStorage, olmPromise}) {
+    constructor({clock, random, onlineStatus, request, storageFactory, sessionInfoStorage, olmPromise, workerPromise}) {
         this._random = random;
         this._clock = clock;
         this._onlineStatus = onlineStatus;
@@ -59,6 +59,7 @@ export class SessionContainer {
         this._sessionId = null;
         this._storage = null;
         this._olmPromise = olmPromise;
+        this._workerPromise = workerPromise;
     }
 
     createNewSessionId() {
@@ -152,8 +153,13 @@ export class SessionContainer {
             homeServer: sessionInfo.homeServer,
         };
         const olm = await this._olmPromise;
+        let workerPool = null;
+        if (this._workerPromise) {
+            workerPool = await this._workerPromise;
+        }
         this._session = new Session({storage: this._storage,
-            sessionInfo: filteredSessionInfo, hsApi, olm, clock: this._clock});
+            sessionInfo: filteredSessionInfo, hsApi, olm,
+            clock: this._clock, workerPool});
         await this._session.load();
         this._status.set(LoadStatus.SessionSetup);
         await this._session.beforeFirstSync(isNewLogin);
diff --git a/src/matrix/e2ee/megolm/Decryption.js b/src/matrix/e2ee/megolm/Decryption.js
index 9726e6d8..544fa0a3 100644
--- a/src/matrix/e2ee/megolm/Decryption.js
+++ b/src/matrix/e2ee/megolm/Decryption.js
@@ -21,7 +21,7 @@ import {SessionInfo} from "./decryption/SessionInfo.js";
 import {DecryptionPreparation} from "./decryption/DecryptionPreparation.js";
 import {SessionDecryption} from "./decryption/SessionDecryption.js";
 import {SessionCache} from "./decryption/SessionCache.js";
-import {DecryptionWorker, WorkerPool} from "./decryption/DecryptionWorker.js";
+import {DecryptionWorker} from "./decryption/DecryptionWorker.js";
 
 function getSenderKey(event) {
     return event.content?.["sender_key"];
@@ -36,12 +36,10 @@ function getCiphertext(event) {
 }
 
 export class Decryption {
-    constructor({pickleKey, olm}) {
+    constructor({pickleKey, olm, workerPool}) {
         this._pickleKey = pickleKey;
         this._olm = olm;
-        this._decryptor = new DecryptionWorker(new WorkerPool("worker-1039452087.js", 4));
-        //this._decryptor = new DecryptionWorker(new WorkerPool("./src/worker.js", 4));
-        this._initPromise = this._decryptor.init();
+        this._decryptor = workerPool ? new DecryptionWorker(workerPool) : null;
     }
 
     createSessionCache(fallback) {
@@ -58,7 +56,6 @@ export class Decryption {
      * @return {DecryptionPreparation}
      */
     async prepareDecryptAll(roomId, events, sessionCache, txn) {
-        await this._initPromise;
         const errors = new Map();
         const validEvents = [];
 
diff --git a/src/matrix/e2ee/megolm/decryption/DecryptionWorker.js b/src/matrix/e2ee/megolm/decryption/DecryptionWorker.js
index 38a474ed..b44694a0 100644
--- a/src/matrix/e2ee/megolm/decryption/DecryptionWorker.js
+++ b/src/matrix/e2ee/megolm/decryption/DecryptionWorker.js
@@ -14,184 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import {AbortError} from "../../../../utils/error.js";
-
-class WorkerState {
-    constructor(worker) {
-        this.worker = worker;
-        this.busy = false;
-    }
-
-    attach(pool) {
-        this.worker.addEventListener("message", pool);
-        this.worker.addEventListener("error", pool);
-    }
-
-    detach(pool) {
-        this.worker.removeEventListener("message", pool);
-        this.worker.removeEventListener("error", pool);
-    }
-}
-
-class Request {
-    constructor(message, pool) {
-        this._promise = new Promise((_resolve, _reject) => {
-            this._resolve = _resolve;
-            this._reject = _reject;
-        });
-        this._message = message;
-        this._pool = pool;
-        this._worker = null;
-    }
-
-    abort() {
-        if (this._isNotDisposed) {
-            this._pool._abortRequest(this);
-            this._dispose();
-        }
-    }
-
-    response() {
-        return this._promise;
-    }
-
-    _dispose() {
-        this._reject = null;
-        this._resolve = null;
-    }
-
-    get _isNotDisposed() {
-        return this._resolve && this._reject;
-    }
-}
-
-export class WorkerPool {
-    constructor(path, amount) {
-        this._workers = [];
-        for (let i = 0; i < amount ; ++i) {
-            const worker = new WorkerState(new Worker(path));
-            worker.attach(this);
-            this._workers[i] = worker;
-        }
-        this._requests = new Map();
-        this._counter = 0;
-        this._pendingFlag = false;
-    }
-
-    handleEvent(e) {
-        if (e.type === "message") {
-            const message = e.data;
-            const request = this._requests.get(message.replyToId);
-            if (request) {
-                request._worker.busy = false;
-                if (request._isNotDisposed) {
-                    if (message.type === "success") {
-                        request._resolve(message.payload);
-                    } else if (message.type === "error") {
-                        request._reject(new Error(message.stack));
-                    }
-                    request._dispose();
-                }
-                this._requests.delete(message.replyToId);
-            }
-            this._sendPending();
-        } else if (e.type === "error") {
-            console.error("worker error", e);
-        }
-    }
-
-    _getPendingRequest() {
-        for (const r of this._requests.values()) {
-            if (!r._worker) {
-                return r;
-            }
-        }
-    }
-
-    _getFreeWorker() {
-        for (const w of this._workers) {
-            if (!w.busy) {
-                return w;
-            }
-        }
-    }
-
-    _sendPending() {
-        this._pendingFlag = false;
-        let success;
-        do {
-            success = false;
-            const request = this._getPendingRequest();
-            if (request) {
-                const worker = this._getFreeWorker();
-                if (worker) {
-                    this._sendWith(request, worker);
-                    success = true;
-                }
-            }
-        } while (success);
-    }
-
-    _sendWith(request, worker) {
-        request._worker = worker;
-        worker.busy = true;
-        worker.worker.postMessage(request._message);
-    }
-
-    _enqueueRequest(message) {
-        this._counter += 1;
-        message.id = this._counter;
-        const request = new Request(message, this);
-        this._requests.set(message.id, request);
-        return request;
-    }
-
-    send(message) {
-        const request = this._enqueueRequest(message);
-        const worker = this._getFreeWorker();
-        if (worker) {
-            this._sendWith(request, worker);
-        }
-        return request;
-    }
-
-    // assumes all workers are free atm
-    sendAll(message) {
-        const promises = this._workers.map(worker => {
-            const request = this._enqueueRequest(Object.assign({}, message));
-            this._sendWith(request, worker);
-            return request.response();
-        });
-        return Promise.all(promises);
-    }
-
-    dispose() {
-        for (const w of this._workers) {
-            w.worker.terminate();
-            w.detach(this);
-        }
-    }
-
-    _trySendPendingInNextTick() {
-        if (!this._pendingFlag) {
-            this._pendingFlag = true;
-            Promise.resolve().then(() => {
-                this._sendPending();
-            });
-        }
-    }
-
-    _abortRequest(request) {
-        request._reject(new AbortError());
-        if (request._worker) {
-            request._worker.busy = false;
-        }
-        this._requests.delete(request._message.id);
-        // allow more requests to be aborted before trying to send other pending
-        this._trySendPendingInNextTick();
-    }
-}
-
 export class DecryptionWorker {
     constructor(workerPool) {
         this._workerPool = workerPool;
@@ -201,9 +23,4 @@ export class DecryptionWorker {
         const sessionKey = session.export_session(session.first_known_index());
         return this._workerPool.send({type: "megolm_decrypt", ciphertext, sessionKey});
     }
-
-    async init() {
-        await this._workerPool.sendAll({type: "load_olm", path: "olm_legacy-3232457086.js"});
-        //await this._workerPool.sendAll({type: "load_olm", path: "../lib/olm/olm_legacy.js"});
-    }
 }
diff --git a/src/utils/WorkerPool.js b/src/utils/WorkerPool.js
new file mode 100644
index 00000000..554067fe
--- /dev/null
+++ b/src/utils/WorkerPool.js
@@ -0,0 +1,212 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {AbortError} from "./error.js";
+
+class WorkerState {
+    constructor(worker) {
+        this.worker = worker;
+        this.busy = false;
+    }
+
+    attach(pool) {
+        this.worker.addEventListener("message", pool);
+        this.worker.addEventListener("error", pool);
+    }
+
+    detach(pool) {
+        this.worker.removeEventListener("message", pool);
+        this.worker.removeEventListener("error", pool);
+    }
+}
+
+class Request {
+    constructor(message, pool) {
+        this._promise = new Promise((_resolve, _reject) => {
+            this._resolve = _resolve;
+            this._reject = _reject;
+        });
+        this._message = message;
+        this._pool = pool;
+        this._worker = null;
+    }
+
+    abort() {
+        if (this._isNotDisposed) {
+            this._pool._abortRequest(this);
+            this._dispose();
+        }
+    }
+
+    response() {
+        return this._promise;
+    }
+
+    _dispose() {
+        this._reject = null;
+        this._resolve = null;
+    }
+
+    get _isNotDisposed() {
+        return this._resolve && this._reject;
+    }
+}
+
+export class WorkerPool {
+    // TODO: extract DOM specific bits and write unit tests
+    constructor(path, amount) {
+        this._workers = [];
+        for (let i = 0; i < amount ; ++i) {
+            const worker = new WorkerState(new Worker(path));
+            worker.attach(this);
+            this._workers[i] = worker;
+        }
+        this._requests = new Map();
+        this._counter = 0;
+        this._pendingFlag = false;
+        this._init = null;
+
+    }
+
+    init() {
+        const promise = new Promise((resolve, reject) => {
+            this._init = {resolve, reject};
+        });
+        this.sendAll({type: "ping"})
+            .then(this._init.resolve, this._init.reject)
+            .finally(() => {
+                this._init = null;
+            });
+        return promise;
+    }
+
+    handleEvent(e) {
+        console.log("WorkerPool event", e);
+        if (e.type === "message") {
+            const message = e.data;
+            const request = this._requests.get(message.replyToId);
+            if (request) {
+                request._worker.busy = false;
+                if (request._isNotDisposed) {
+                    if (message.type === "success") {
+                        request._resolve(message.payload);
+                    } else if (message.type === "error") {
+                        request._reject(new Error(message.stack));
+                    }
+                    request._dispose();
+                }
+                this._requests.delete(message.replyToId);
+            }
+            this._sendPending();
+        } else if (e.type === "error") {
+            if (this._init) {
+                this._init.reject(new Error("worker error during init"));
+            }
+            console.error("worker error", e);
+        }
+    }
+
+    _getPendingRequest() {
+        for (const r of this._requests.values()) {
+            if (!r._worker) {
+                return r;
+            }
+        }
+    }
+
+    _getFreeWorker() {
+        for (const w of this._workers) {
+            if (!w.busy) {
+                return w;
+            }
+        }
+    }
+
+    _sendPending() {
+        this._pendingFlag = false;
+        let success;
+        do {
+            success = false;
+            const request = this._getPendingRequest();
+            if (request) {
+                const worker = this._getFreeWorker();
+                if (worker) {
+                    this._sendWith(request, worker);
+                    success = true;
+                }
+            }
+        } while (success);
+    }
+
+    _sendWith(request, worker) {
+        request._worker = worker;
+        worker.busy = true;
+        worker.worker.postMessage(request._message);
+    }
+
+    _enqueueRequest(message) {
+        this._counter += 1;
+        message.id = this._counter;
+        const request = new Request(message, this);
+        this._requests.set(message.id, request);
+        return request;
+    }
+
+    send(message) {
+        const request = this._enqueueRequest(message);
+        const worker = this._getFreeWorker();
+        if (worker) {
+            this._sendWith(request, worker);
+        }
+        return request;
+    }
+
+    // assumes all workers are free atm
+    sendAll(message) {
+        const promises = this._workers.map(worker => {
+            const request = this._enqueueRequest(Object.assign({}, message));
+            this._sendWith(request, worker);
+            return request.response();
+        });
+        return Promise.all(promises);
+    }
+
+    dispose() {
+        for (const w of this._workers) {
+            w.detach(this);
+            w.worker.terminate();
+        }
+    }
+
+    _trySendPendingInNextTick() {
+        if (!this._pendingFlag) {
+            this._pendingFlag = true;
+            Promise.resolve().then(() => {
+                this._sendPending();
+            });
+        }
+    }
+
+    _abortRequest(request) {
+        request._reject(new AbortError());
+        if (request._worker) {
+            request._worker.busy = false;
+        }
+        this._requests.delete(request._message.id);
+        // allow more requests to be aborted before trying to send other pending
+        this._trySendPendingInNextTick();
+    }
+}
diff --git a/src/worker.js b/src/worker.js
index 4b2a1e43..7c6642fb 100644
--- a/src/worker.js
+++ b/src/worker.js
@@ -96,7 +96,7 @@ class MessageHandler {
     async _handleMessage(message) {
         const {type} = message;
         if (type === "ping") {
-            this._sendReply(message, {type: "pong"});
+            this._sendReply(message, {type: "success"});
         } else if (type === "load_olm") {
             this._sendReply(message, await this._loadOlm(message.path));
         } else if (type === "megolm_decrypt") {

From 78fecd003a847f25da1f9b7d19c27d3d763d4dea Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Thu, 10 Sep 2020 18:57:29 +0200
Subject: [PATCH 159/173] cleanup

---
 src/matrix/e2ee/megolm/decryption/DecryptionPreparation.js | 3 ---
 src/matrix/room/Room.js                                    | 1 -
 src/utils/WorkerPool.js                                    | 1 -
 3 files changed, 5 deletions(-)

diff --git a/src/matrix/e2ee/megolm/decryption/DecryptionPreparation.js b/src/matrix/e2ee/megolm/decryption/DecryptionPreparation.js
index e24d70af..02ee32df 100644
--- a/src/matrix/e2ee/megolm/decryption/DecryptionPreparation.js
+++ b/src/matrix/e2ee/megolm/decryption/DecryptionPreparation.js
@@ -28,9 +28,6 @@ export class DecryptionPreparation {
     }
 
     async decrypt() {
-        // console.log("start sleeping");
-        // await new Promise(resolve => setTimeout(resolve, 5000));
-        // console.log("done sleeping");
         try {
             const errors = this._initialErrors;
             const results = new Map();
diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index 98f114a5..3704223e 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -102,7 +102,6 @@ export class Room extends EventEmitter {
             const isTimelineOpen = this._isTimelineOpen;
             r.preparation = await this._roomEncryption.prepareDecryptAll(events, source, isTimelineOpen, inboundSessionTxn);
             if (r.cancelled) return;
-            // TODO: should this throw an AbortError?
             const changes = await r.preparation.decrypt();
             r.preparation = null;
             if (r.cancelled) return;
diff --git a/src/utils/WorkerPool.js b/src/utils/WorkerPool.js
index 554067fe..56feaf8c 100644
--- a/src/utils/WorkerPool.js
+++ b/src/utils/WorkerPool.js
@@ -94,7 +94,6 @@ export class WorkerPool {
     }
 
     handleEvent(e) {
-        console.log("WorkerPool event", e);
         if (e.type === "message") {
             const message = e.data;
             const request = this._requests.get(message.replyToId);

From 0b26e6f53ac1765df54db638075759274fd1cc94 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 11 Sep 2020 08:40:43 +0200
Subject: [PATCH 160/173] compress new e2ee stores into one new idb version

---
 src/matrix/storage/idb/schema.js | 34 ++++----------------------------
 1 file changed, 4 insertions(+), 30 deletions(-)

diff --git a/src/matrix/storage/idb/schema.js b/src/matrix/storage/idb/schema.js
index f1817c24..1ed9cadb 100644
--- a/src/matrix/storage/idb/schema.js
+++ b/src/matrix/storage/idb/schema.js
@@ -9,12 +9,7 @@ export const schema = [
     createInitialStores,
     createMemberStore,
     migrateSession,
-    createIdentityStores,
-    createOlmSessionStore,
-    createInboundGroupSessionsStore,
-    createOutboundGroupSessionsStore,
-    createGroupSessionDecryptions,
-    addSenderKeyIndexToDeviceStore
+    createE2EEStores
 ];
 // TODO: how to deal with git merge conflicts of this array?
 
@@ -71,33 +66,12 @@ async function migrateSession(db, txn) {
     }
 }
 //v4
-function createIdentityStores(db) {
+function createE2EEStores(db) {
     db.createObjectStore("userIdentities", {keyPath: "userId"});
-    db.createObjectStore("deviceIdentities", {keyPath: "key"});
-}
-
-//v5
-function createOlmSessionStore(db) {
+    const deviceIdentities = db.createObjectStore("deviceIdentities", {keyPath: "key"});
+    deviceIdentities.createIndex("byCurve25519Key", "curve25519Key", {unique: true});
     db.createObjectStore("olmSessions", {keyPath: "key"});
-}
-
-//v6
-function createInboundGroupSessionsStore(db) {
     db.createObjectStore("inboundGroupSessions", {keyPath: "key"});
-}
-
-//v7
-function createOutboundGroupSessionsStore(db) {
     db.createObjectStore("outboundGroupSessions", {keyPath: "roomId"});
-}
-
-//v8
-function createGroupSessionDecryptions(db) {
     db.createObjectStore("groupSessionDecryptions", {keyPath: "key"});
 }
-
-//v9
-function addSenderKeyIndexToDeviceStore(db, txn) {
-    const deviceIdentities = txn.objectStore("deviceIdentities");
-    deviceIdentities.createIndex("byCurve25519Key", "curve25519Key", {unique: true});
-}

From e0d9d703b70f43fd2e700d8aafc8a3e69603998b Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 11 Sep 2020 10:43:17 +0200
Subject: [PATCH 161/173] offload olm account creation in worker

---
 src/main.js                                   | 12 ++--
 src/matrix/Session.js                         | 35 ++++------
 src/matrix/SessionContainer.js                |  6 +-
 src/matrix/e2ee/Account.js                    | 38 +++++++----
 .../DecryptionWorker.js => OlmWorker.js}      | 21 +++++-
 src/matrix/e2ee/megolm/Decryption.js          |  7 +-
 .../megolm/decryption/SessionDecryption.js    | 10 +--
 src/worker.js                                 | 66 ++++++++++++++++---
 8 files changed, 134 insertions(+), 61 deletions(-)
 rename src/matrix/e2ee/{megolm/decryption/DecryptionWorker.js => OlmWorker.js} (53%)

diff --git a/src/main.js b/src/main.js
index 6f279910..ca0afcaf 100644
--- a/src/main.js
+++ b/src/main.js
@@ -26,6 +26,7 @@ import {BrawlView} from "./ui/web/BrawlView.js";
 import {Clock} from "./ui/web/dom/Clock.js";
 import {OnlineStatus} from "./ui/web/dom/OnlineStatus.js";
 import {WorkerPool} from "./utils/WorkerPool.js";
+import {OlmWorker} from "./matrix/e2ee/OlmWorker.js";
 
 function addScript(src) {
     return new Promise(function (resolve, reject) {
@@ -65,12 +66,13 @@ function relPath(path, basePath) {
     return "../".repeat(dirCount) + path;
 }
 
-async function loadWorker(paths) {
+async function loadOlmWorker(paths) {
     const workerPool = new WorkerPool(paths.worker, 4);
     await workerPool.init();
     const path = relPath(paths.olm.legacyBundle, paths.worker);
     await workerPool.sendAll({type: "load_olm", path});
-    return workerPool;
+    const olmWorker = new OlmWorker(workerPool);
+    return olmWorker;
 }
 
 // Don't use a default export here, as we use multiple entries during legacy build,
@@ -100,9 +102,9 @@ export async function main(container, paths) {
         // if wasm is not supported, we'll want
         // to run some olm operations in a worker (mainly for IE11)
         let workerPromise;
-        if (!window.WebAssembly) {
-            workerPromise = loadWorker(paths);
-        }
+        // if (!window.WebAssembly) {
+            workerPromise = loadOlmWorker(paths);
+        // }
 
         const vm = new BrawlViewModel({
             createSessionContainer: () => {
diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index be3f6a06..53c3898b 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -33,7 +33,7 @@ const PICKLE_KEY = "DEFAULT_KEY";
 
 export class Session {
     // sessionInfo contains deviceId, userId and homeServer
-    constructor({clock, storage, hsApi, sessionInfo, olm, workerPool}) {
+    constructor({clock, storage, hsApi, sessionInfo, olm, olmWorker}) {
         this._clock = clock;
         this._storage = storage;
         this._hsApi = hsApi;
@@ -52,7 +52,7 @@ export class Session {
         this._megolmEncryption = null;
         this._megolmDecryption = null;
         this._getSyncToken = () => this.syncToken;
-        this._workerPool = workerPool;
+        this._olmWorker = olmWorker;
 
         if (olm) {
             this._olmUtil = new olm.Utility();
@@ -101,7 +101,7 @@ export class Session {
         this._megolmDecryption = new MegOlmDecryption({
             pickleKey: PICKLE_KEY,
             olm: this._olm,
-            workerPool: this._workerPool,
+            olmWorker: this._olmWorker,
         });
         this._deviceMessageHandler.enableEncryption({olmDecryption, megolmDecryption: this._megolmDecryption});
     }
@@ -140,23 +140,15 @@ export class Session {
                 throw new Error("there should not be an e2ee account already on a fresh login");
             }
             if (!this._e2eeAccount) {
-                const txn = await this._storage.readWriteTxn([
-                    this._storage.storeNames.session
-                ]);
-                try {
-                    this._e2eeAccount = await E2EEAccount.create({
-                        hsApi: this._hsApi,
-                        olm: this._olm,
-                        pickleKey: PICKLE_KEY,
-                        userId: this._sessionInfo.userId,
-                        deviceId: this._sessionInfo.deviceId,
-                        txn
-                    });
-                } catch (err) {
-                    txn.abort();
-                    throw err;
-                }
-                await txn.complete();
+                this._e2eeAccount = await E2EEAccount.create({
+                    hsApi: this._hsApi,
+                    olm: this._olm,
+                    pickleKey: PICKLE_KEY,
+                    userId: this._sessionInfo.userId,
+                    deviceId: this._sessionInfo.deviceId,
+                    olmWorker: this._olmWorker,
+                    storage: this._storage,
+                });
                 this._setupEncryption();
             }
             await this._e2eeAccount.generateOTKsIfNeeded(this._storage);
@@ -184,6 +176,7 @@ export class Session {
                 pickleKey: PICKLE_KEY,
                 userId: this._sessionInfo.userId,
                 deviceId: this._sessionInfo.deviceId,
+                olmWorker: this._olmWorker,
                 txn
             });
             if (this._e2eeAccount) {
@@ -204,7 +197,7 @@ export class Session {
     }
 
     stop() {
-        this._workerPool?.dispose();
+        this._olmWorker?.dispose();
         this._sendScheduler.stop();
     }
 
diff --git a/src/matrix/SessionContainer.js b/src/matrix/SessionContainer.js
index 1e868eba..98ffd2b1 100644
--- a/src/matrix/SessionContainer.js
+++ b/src/matrix/SessionContainer.js
@@ -153,13 +153,13 @@ export class SessionContainer {
             homeServer: sessionInfo.homeServer,
         };
         const olm = await this._olmPromise;
-        let workerPool = null;
+        let olmWorker = null;
         if (this._workerPromise) {
-            workerPool = await this._workerPromise;
+            olmWorker = await this._workerPromise;
         }
         this._session = new Session({storage: this._storage,
             sessionInfo: filteredSessionInfo, hsApi, olm,
-            clock: this._clock, workerPool});
+            clock: this._clock, olmWorker});
         await this._session.load();
         this._status.set(LoadStatus.SessionSetup);
         await this._session.beforeFirstSync(isNewLogin);
diff --git a/src/matrix/e2ee/Account.js b/src/matrix/e2ee/Account.js
index 9d83465c..37fab7d2 100644
--- a/src/matrix/e2ee/Account.js
+++ b/src/matrix/e2ee/Account.js
@@ -23,7 +23,7 @@ const DEVICE_KEY_FLAG_SESSION_KEY = SESSION_KEY_PREFIX + "areDeviceKeysUploaded"
 const SERVER_OTK_COUNT_SESSION_KEY = SESSION_KEY_PREFIX + "serverOTKCount";
 
 export class Account {
-    static async load({olm, pickleKey, hsApi, userId, deviceId, txn}) {
+    static async load({olm, pickleKey, hsApi, userId, deviceId, olmWorker, txn}) {
         const pickledAccount = await txn.session.get(ACCOUNT_SESSION_KEY);
         if (pickledAccount) {
             const account = new olm.Account();
@@ -31,26 +31,39 @@ export class Account {
             account.unpickle(pickleKey, pickledAccount);
             const serverOTKCount = await txn.session.get(SERVER_OTK_COUNT_SESSION_KEY);
             return new Account({pickleKey, hsApi, account, userId,
-                deviceId, areDeviceKeysUploaded, serverOTKCount, olm});
+                deviceId, areDeviceKeysUploaded, serverOTKCount, olm, olmWorker});
         }
     }
 
-    static async create({olm, pickleKey, hsApi, userId, deviceId, txn}) {
+    static async create({olm, pickleKey, hsApi, userId, deviceId, olmWorker, storage}) {
         const account = new olm.Account();
-        account.create();
-        account.generate_one_time_keys(account.max_number_of_one_time_keys());
+        if (olmWorker) {
+            await olmWorker.createAccountAndOTKs(account, account.max_number_of_one_time_keys());
+        } else {
+            account.create();
+            account.generate_one_time_keys(account.max_number_of_one_time_keys());
+        }
         const pickledAccount = account.pickle(pickleKey);
-        // add will throw if the key already exists
-        // we would not want to overwrite olmAccount here
         const areDeviceKeysUploaded = false;
-        await txn.session.add(ACCOUNT_SESSION_KEY, pickledAccount);
-        await txn.session.add(DEVICE_KEY_FLAG_SESSION_KEY, areDeviceKeysUploaded);
-        await txn.session.add(SERVER_OTK_COUNT_SESSION_KEY, 0);
+        const txn = await storage.readWriteTxn([
+            storage.storeNames.session
+        ]);
+        try {
+            // add will throw if the key already exists
+            // we would not want to overwrite olmAccount here
+            txn.session.add(ACCOUNT_SESSION_KEY, pickledAccount);
+            txn.session.add(DEVICE_KEY_FLAG_SESSION_KEY, areDeviceKeysUploaded);
+            txn.session.add(SERVER_OTK_COUNT_SESSION_KEY, 0);
+        } catch (err) {
+            txn.abort();
+            throw err;
+        }
+        await txn.complete();
         return new Account({pickleKey, hsApi, account, userId,
-            deviceId, areDeviceKeysUploaded, serverOTKCount: 0, olm});
+            deviceId, areDeviceKeysUploaded, serverOTKCount: 0, olm, olmWorker});
     }
 
-    constructor({pickleKey, hsApi, account, userId, deviceId, areDeviceKeysUploaded, serverOTKCount, olm}) {
+    constructor({pickleKey, hsApi, account, userId, deviceId, areDeviceKeysUploaded, serverOTKCount, olm, olmWorker}) {
         this._olm = olm;
         this._pickleKey = pickleKey;
         this._hsApi = hsApi;
@@ -59,6 +72,7 @@ export class Account {
         this._deviceId = deviceId;
         this._areDeviceKeysUploaded = areDeviceKeysUploaded;
         this._serverOTKCount = serverOTKCount;
+        this._olmWorker = olmWorker;
         this._identityKeys = JSON.parse(this._account.identity_keys());
     }
 
diff --git a/src/matrix/e2ee/megolm/decryption/DecryptionWorker.js b/src/matrix/e2ee/OlmWorker.js
similarity index 53%
rename from src/matrix/e2ee/megolm/decryption/DecryptionWorker.js
rename to src/matrix/e2ee/OlmWorker.js
index b44694a0..a6edd3cc 100644
--- a/src/matrix/e2ee/megolm/decryption/DecryptionWorker.js
+++ b/src/matrix/e2ee/OlmWorker.js
@@ -14,13 +14,30 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-export class DecryptionWorker {
+export class OlmWorker {
     constructor(workerPool) {
         this._workerPool = workerPool;
     }
 
-    decrypt(session, ciphertext) {
+    megolmDecrypt(session, ciphertext) {
         const sessionKey = session.export_session(session.first_known_index());
         return this._workerPool.send({type: "megolm_decrypt", ciphertext, sessionKey});
     }
+
+    async createAccountAndOTKs(account, otkAmount) {
+        // IE11 does not support getRandomValues in a worker, so we have to generate the values upfront.
+        let randomValues;
+        if (window.msCrypto) {
+            randomValues = [
+                window.msCrypto.getRandomValues(new Uint8Array(64)),
+                window.msCrypto.getRandomValues(new Uint8Array(otkAmount * 32)),
+            ];
+        }
+        const pickle = await this._workerPool.send({type: "olm_create_account_otks", randomValues, otkAmount}).response();
+        account.unpickle("", pickle);
+    }
+
+    dispose() {
+        this._workerPool.dispose();
+    }
 }
diff --git a/src/matrix/e2ee/megolm/Decryption.js b/src/matrix/e2ee/megolm/Decryption.js
index 544fa0a3..b3f1ea71 100644
--- a/src/matrix/e2ee/megolm/Decryption.js
+++ b/src/matrix/e2ee/megolm/Decryption.js
@@ -21,7 +21,6 @@ import {SessionInfo} from "./decryption/SessionInfo.js";
 import {DecryptionPreparation} from "./decryption/DecryptionPreparation.js";
 import {SessionDecryption} from "./decryption/SessionDecryption.js";
 import {SessionCache} from "./decryption/SessionCache.js";
-import {DecryptionWorker} from "./decryption/DecryptionWorker.js";
 
 function getSenderKey(event) {
     return event.content?.["sender_key"];
@@ -36,10 +35,10 @@ function getCiphertext(event) {
 }
 
 export class Decryption {
-    constructor({pickleKey, olm, workerPool}) {
+    constructor({pickleKey, olm, olmWorker}) {
         this._pickleKey = pickleKey;
         this._olm = olm;
-        this._decryptor = workerPool ? new DecryptionWorker(workerPool) : null;
+        this._olmWorker = olmWorker;
     }
 
     createSessionCache(fallback) {
@@ -86,7 +85,7 @@ export class Decryption {
                     errors.set(event.event_id, new DecryptionError("MEGOLM_NO_SESSION", event));
                 }
             } else {
-                sessionDecryptions.push(new SessionDecryption(sessionInfo, eventsForSession, this._decryptor));
+                sessionDecryptions.push(new SessionDecryption(sessionInfo, eventsForSession, this._olmWorker));
             }
         }));
 
diff --git a/src/matrix/e2ee/megolm/decryption/SessionDecryption.js b/src/matrix/e2ee/megolm/decryption/SessionDecryption.js
index 30ca432e..137ae9f8 100644
--- a/src/matrix/e2ee/megolm/decryption/SessionDecryption.js
+++ b/src/matrix/e2ee/megolm/decryption/SessionDecryption.js
@@ -22,12 +22,12 @@ import {ReplayDetectionEntry} from "./ReplayDetectionEntry.js";
  * Does the actual decryption of all events for a given megolm session in a batch
  */
 export class SessionDecryption {
-    constructor(sessionInfo, events, decryptor) {
+    constructor(sessionInfo, events, olmWorker) {
         sessionInfo.retain();
         this._sessionInfo = sessionInfo;
         this._events = events;
-        this._decryptor = decryptor;
-        this._decryptionRequests = decryptor ? [] : null;
+        this._olmWorker = olmWorker;
+        this._decryptionRequests = olmWorker ? [] : null;
     }
 
     async decryptAll() {
@@ -41,8 +41,8 @@ export class SessionDecryption {
                 const {session} = this._sessionInfo;
                 const ciphertext = event.content.ciphertext;
                 let decryptionResult;
-                if (this._decryptor) {
-                    const request = this._decryptor.decrypt(session, ciphertext);
+                if (this._olmWorker) {
+                    const request = this._olmWorker.megolmDecrypt(session, ciphertext);
                     this._decryptionRequests.push(request);
                     decryptionResult = await request.response();
                 } else {
diff --git a/src/worker.js b/src/worker.js
index 7c6642fb..a0016f67 100644
--- a/src/worker.js
+++ b/src/worker.js
@@ -32,6 +32,44 @@ function asSuccessMessage(payload) {
 class MessageHandler {
     constructor() {
         this._olm = null;
+        this._randomValues = self.crypto ? null : [];
+    }
+
+    _feedRandomValues(randomValues) {
+        if (this._randomValues) {
+            this._randomValues.push(...randomValues);
+        }
+    }
+
+    _checkRandomValuesUsed() {
+        if (this._randomValues && this._randomValues.length !== 0) {
+            throw new Error(`${this._randomValues.length} random values left`);
+        }
+    }
+
+    _getRandomValues(typedArray) {
+        if (!(typedArray instanceof Uint8Array)) {
+            throw new Error("only Uint8Array is supported: " + JSON.stringify({
+                Int8Array: typedArray instanceof Int8Array,
+                Uint8Array: typedArray instanceof Uint8Array,
+                Int16Array: typedArray instanceof Int16Array,
+                Uint16Array: typedArray instanceof Uint16Array,
+                Int32Array: typedArray instanceof Int32Array,
+                Uint32Array: typedArray instanceof Uint32Array,
+            }));
+        }
+        if (this._randomValues.length === 0) {
+            throw new Error("no more random values, needed one of length " + typedArray.length);
+        }
+        const precalculated = this._randomValues.shift();
+        if (precalculated.length !== typedArray.length) {
+            throw new Error(`typedArray length (${typedArray.length}) does not match precalculated length (${precalculated.length})`);
+        }
+        // copy values
+        for (let i = 0; i < typedArray.length; ++i) {
+            typedArray[i] = precalculated[i];
+        }
+        return typedArray;
     }
 
     handleEvent(e) {
@@ -47,7 +85,7 @@ class MessageHandler {
 
     _toMessage(fn) {
         try {
-            let payload = fn();
+            const payload = fn();
             if (payload instanceof Promise) {
                 return payload.then(
                     payload => asSuccessMessage(payload),
@@ -63,18 +101,15 @@ class MessageHandler {
 
     _loadOlm(path) {
         return this._toMessage(async () => {
-            // might have some problems here with window vs self as global object?
-            if (self.msCrypto && !self.crypto) {
-                self.crypto = self.msCrypto;
+            if (!self.crypto) {
+                self.crypto = {getRandomValues: this._getRandomValues.bind(this)};
             }
-            self.importScripts(path);
-            const olm = self.olm_exports;
-            // mangle the globals enough to make olm load believe it is running in a browser
+            // mangle the globals enough to make olm believe it is running in a browser
             self.window = self;
             self.document = {};
+            self.importScripts(path);
+            const olm = self.olm_exports;
             await olm.init();
-            delete self.document;
-            delete self.window;
             this._olm = olm;
         });
     }
@@ -93,6 +128,17 @@ class MessageHandler {
         });
     }
 
+    _olmCreateAccountAndOTKs(randomValues, otkAmount) {
+        return this._toMessage(() => {
+            this._feedRandomValues(randomValues);
+            const account = new this._olm.Account();
+            account.create();
+            account.generate_one_time_keys(otkAmount);
+            this._checkRandomValuesUsed();
+            return account.pickle("");
+        });
+    }
+
     async _handleMessage(message) {
         const {type} = message;
         if (type === "ping") {
@@ -101,6 +147,8 @@ class MessageHandler {
             this._sendReply(message, await this._loadOlm(message.path));
         } else if (type === "megolm_decrypt") {
             this._sendReply(message, this._megolmDecrypt(message.sessionKey, message.ciphertext));
+        } else if (type === "olm_create_account_otks") {
+            this._sendReply(message, this._olmCreateAccountAndOTKs(message.randomValues, message.otkAmount));
         }
     }
 }

From b8ce97e7393cfcc1500141a59d1321315d63c7ca Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 11 Sep 2020 10:44:08 +0200
Subject: [PATCH 162/173] remove duplicate code in build script

---
 scripts/build.mjs | 40 ++++++++--------------------------------
 1 file changed, 8 insertions(+), 32 deletions(-)

diff --git a/scripts/build.mjs b/scripts/build.mjs
index bda86d33..c6c37b96 100644
--- a/scripts/build.mjs
+++ b/scripts/build.mjs
@@ -203,7 +203,7 @@ async function buildJs(inputFile, outputName) {
     return bundlePath;
 }
 
-async function buildJsLegacy(inputFile, outputName) {
+async function buildJsLegacy(inputFile, outputName, polyfillFile = null) {
     // compile down to whatever IE 11 needs
     const babelPlugin = babel.babel({
         babelHelpers: 'bundled',
@@ -219,9 +219,12 @@ async function buildJsLegacy(inputFile, outputName) {
             ]
         ]
     });
+    if (!polyfillFile) {
+        polyfillFile = 'src/legacy-polyfill.js';
+    }
     // create js bundle
     const rollupConfig = {
-        input: ['src/legacy-polyfill.js', inputFile],
+        input: [polyfillFile, inputFile],
         plugins: [multi(), commonjs(), nodeResolve(), babelPlugin, removeJsComments({comments: "none"})]
     };
     const bundle = await rollup(rollupConfig);
@@ -235,36 +238,9 @@ async function buildJsLegacy(inputFile, outputName) {
     return bundlePath;
 }
 
-async function buildWorkerJsLegacy(inputFile, outputName) {
-    // compile down to whatever IE 11 needs
-    const babelPlugin = babel.babel({
-        babelHelpers: 'bundled',
-        exclude: 'node_modules/**',
-        presets: [
-            [
-                "@babel/preset-env",
-                {
-                    useBuiltIns: "entry",
-                    corejs: "3",
-                    targets: "IE 11"
-                }
-            ]
-        ]
-    });
-    // create js bundle
-    const rollupConfig = {
-        input: ['src/worker-polyfill.js', inputFile],
-        plugins: [multi(), commonjs(), nodeResolve(), babelPlugin, removeJsComments({comments: "none"})]
-    };
-    const bundle = await rollup(rollupConfig);
-    const {output} = await bundle.generate({
-        format: 'iife',
-        name: `${PROJECT_ID}Bundle`
-    });
-    const code = output[0].code;
-    const bundlePath = resource(outputName, code);
-    await fs.writeFile(bundlePath, code, "utf8");
-    return bundlePath;
+function buildWorkerJsLegacy(inputFile, outputName) {
+    const polyfillFile = 'src/worker-polyfill.js';
+    return buildJsLegacy(inputFile, outputName, polyfillFile);
 }
 
 async function buildOffline(version, assetPaths) {

From 95c6fd5a5bdc36c33cd548e93cb885a69899ecb3 Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 11 Sep 2020 10:53:15 +0200
Subject: [PATCH 163/173] reenable only using worker when wasm is not supported

---
 src/main.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main.js b/src/main.js
index ca0afcaf..2f5165aa 100644
--- a/src/main.js
+++ b/src/main.js
@@ -102,9 +102,9 @@ export async function main(container, paths) {
         // if wasm is not supported, we'll want
         // to run some olm operations in a worker (mainly for IE11)
         let workerPromise;
-        // if (!window.WebAssembly) {
+        if (!window.WebAssembly) {
             workerPromise = loadOlmWorker(paths);
-        // }
+        }
 
         const vm = new BrawlViewModel({
             createSessionContainer: () => {

From 0e3084cce3ca2bb6506500f0d02fc94cffa8298b Mon Sep 17 00:00:00 2001
From: Bruno Windels 
Date: Fri, 11 Sep 2020 11:28:59 +0200
Subject: [PATCH 164/173] provide alternative spinner for ie11

---
 src/main.js                  |  6 +++++
 src/ui/web/common.js         | 15 +++++++++---
 src/ui/web/css/spinner.css   | 45 +++++++++++++++++++++++++++++++-----
 src/ui/web/css/timeline.css  |  2 +-
 src/ui/web/view-gallery.html |  2 +-
 5 files changed, 59 insertions(+), 11 deletions(-)

diff --git a/src/main.js b/src/main.js
index 2f5165aa..8a7cbf0d 100644
--- a/src/main.js
+++ b/src/main.js
@@ -80,6 +80,12 @@ async function loadOlmWorker(paths) {
 // see https://github.com/rollup/plugins/tree/master/packages/multi-entry
 export async function main(container, paths) {
     try {
+        const isIE11 = !!window.MSInputMethodContext && !!document.documentMode;
+        if (isIE11) {
+            document.body.className += " ie11";
+        } else {
+            document.body.className += " not-ie11";
+        }
         // to replay:
         // const fetchLog = await (await fetch("/fetchlogs/constrainterror.json")).json();
         // const replay = new ReplayRequester(fetchLog, {delay: false});
diff --git a/src/ui/web/common.js b/src/ui/web/common.js
index 2883652e..d6c4dddc 100644
--- a/src/ui/web/common.js
+++ b/src/ui/web/common.js
@@ -15,9 +15,18 @@ limitations under the License.
 */
 
 export function spinner(t, extraClasses = undefined) {
-    return t.svg({className: Object.assign({"spinner": true}, extraClasses), viewBox:"0 0 100 100"}, 
-        t.circle({cx:"50%", cy:"50%", r:"45%", pathLength:"100"})
-    );
+    if (document.body.classList.contains("ie11")) {
+        return t.div({className: "spinner"}, [
+            t.div(),
+            t.div(),
+            t.div(),
+            t.div(),
+        ]);
+    } else {
+        return t.svg({className: Object.assign({"spinner": true}, extraClasses), viewBox:"0 0 100 100"}, 
+            t.circle({cx:"50%", cy:"50%", r:"45%", pathLength:"100"})
+        );
+    }
 }
 
 /**
diff --git a/src/ui/web/css/spinner.css b/src/ui/web/css/spinner.css
index 62974da6..1548b9b6 100644
--- a/src/ui/web/css/spinner.css
+++ b/src/ui/web/css/spinner.css
@@ -32,24 +32,57 @@ limitations under the License.
     }
 }
 
-.spinner circle {
+.not-ie11 .spinner circle {
     transform-origin: 50% 50%;
     animation-name: spinner;
     animation-duration: 2s;
     animation-iteration-count: infinite;
     animation-timing-function: linear;
-    /**
-     * TODO
-     * see if with IE11 we can just set a static stroke state and make it rotate?
-     */
     stroke-dasharray: 0 0 85 85;
-
     fill: none;
     stroke: currentcolor;
     stroke-width: 12;
     stroke-linecap: butt;
 }
 
+.ie11 .spinner {
+  display: inline-block;
+  position: relative;
+}
+
+.ie11 .spinner div {
+  box-sizing: border-box;
+  display: block;
+  position: absolute;
+  padding: 2px;
+  border: 2px solid currentcolor;
+  border-radius: 50%;
+  animation: ie-spinner 1.2s cubic-bezier(0.5, 0, 0.5, 1) infinite;
+  border-color: currentcolor transparent transparent transparent;
+  width: var(--size);
+  height: var(--size);
+}
+
+.ie11 .spinner div:nth-child(1) {
+  animation-delay: -0.45s;
+}
+.ie11 .spinner div:nth-child(2) {
+  animation-delay: -0.3s;
+}
+.ie11 .spinner div:nth-child(3) {
+  animation-delay: -0.15s;
+}
+
+@keyframes ie-spinner {
+  0% {
+    transform: rotate(0deg);
+  }
+  100% {
+    transform: rotate(360deg);
+  }
+}
+
+
 .spinner {
     --size: 20px;
     width: var(--size);
diff --git a/src/ui/web/css/timeline.css b/src/ui/web/css/timeline.css
index 469e117f..f96adf6e 100644
--- a/src/ui/web/css/timeline.css
+++ b/src/ui/web/css/timeline.css
@@ -77,6 +77,6 @@ limitations under the License.
 }
 
 .GapView > div {
-    flex: 1;
+    flex: 1 1 0;
     margin-left: 10px;
 }
diff --git a/src/ui/web/view-gallery.html b/src/ui/web/view-gallery.html
index 43827afb..ecf328db 100644
--- a/src/ui/web/view-gallery.html
+++ b/src/ui/web/view-gallery.html
@@ -5,7 +5,7 @@
         
         
 	
-	
+