Merge pull request #231 from vector-im/bwindels/logs

Structured logging

src/domain/ViewModel.js

@@ -108,6 +108,10 @@ export class ViewModel extends EventEmitter {
         return this._options.platform.clock;
     }
 
+    get logger() {
+        return this.platform.logger;
+    }
+
     /**
      * The url router, only meant to be used to create urls with from view models.
      * @return {URLRouter}

src/domain/session/settings/SettingsViewModel.js

@@ -110,5 +110,10 @@ export class SettingsViewModel extends ViewModel {
             return this.i18n`unknown`;
         }
     }
+
+    async exportLogs() {
+        const logExport = await this.logger.export();
+        this.platform.saveFileAs(logExport.asBlob(), `hydrogen-logs-${this.platform.clock.now()}.json`);
+    }
 }
 

src/logging/BaseLogger.js

@@ -0,0 +1,103 @@
+/*
+Copyright 2020 Bruno Windels <bruno@windels.cloud>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {LogItem} from "./LogItem.js";
+import {LogLevel, LogFilter} from "./LogFilter.js";
+
+export class BaseLogger {
+    constructor({platform}) {
+        this._openItems = new Set();
+        this._platform = platform;
+    }
+
+    run(labelOrValues, callback, logLevel = LogLevel.Info, filterCreator = null) {
+        const item = new LogItem(labelOrValues, logLevel, null, this._platform.clock);
+        this._openItems.add(item);
+
+        const finishItem = () => {
+            let filter = new LogFilter();
+            if (filterCreator) {
+                try {
+                    filter = filterCreator(filter, this);
+                } catch (err) {
+                    console.error("Error while creating log filter", err);
+                }
+            } else {
+                // if not filter is specified, filter out anything lower than the initial log level
+                filter = filter.minLevel(logLevel);
+            }
+            try {
+                const serialized = item.serialize(filter, 0);
+                if (serialized) {
+                    this._persistItem(serialized);
+                }
+            } catch (err) {
+                console.error("Could not serialize log item", err);
+            }
+            this._openItems.delete(item);
+        };
+
+        try {
+            const result = item.run(callback);
+            if (result instanceof Promise) {
+                return result.then(promiseResult => {
+                    finishItem();
+                    return promiseResult;
+                }, err => {
+                    finishItem();
+                    throw err;
+                });
+            } else {
+                finishItem();
+                return result;
+            }
+        } catch (err) {
+            finishItem();
+            throw err;
+        }
+    }
+
+    _finishOpenItems() {
+        for (const openItem of this._openItems) {
+            openItem.finish();
+            try {
+                // for now, serialize with an all-permitting filter
+                // as the createFilter function would get a distorted image anyway
+                // about the duration of the item, etc ...
+                const serialized = openItem.serialize(new LogFilter(), 0);
+                if (serialized) {
+                    this._persistItem(serialized);
+                }
+            } catch (err) {
+                console.error("Could not serialize log item", err);
+            }
+        }
+        this._openItems.clear();
+    }
+
+    _persistItem() {
+        throw new Error("not implemented");
+    }
+
+    async export() {
+        throw new Error("not implemented");
+    }
+
+    // expose log level without needing 
+    get level() {
+        return LogLevel;
+    }
+}

src/logging/IDBLogger.js

@@ -0,0 +1,190 @@
+/*
+Copyright 2020 Bruno Windels <bruno@windels.cloud>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {
+    openDatabase,
+    txnAsPromise,
+    reqAsPromise,
+    iterateCursor,
+    fetchResults,
+    encodeUint64
+} from "../matrix/storage/idb/utils.js";
+import {BaseLogger} from "./BaseLogger.js";
+
+export class IDBLogger extends BaseLogger {
+    constructor(options) {
+        super(options);
+        const {name, flushInterval = 60 * 1000, limit = 3000} = options;
+        this._name = name;
+        this._limit = limit;
+        // does not get loaded from idb on startup as we only use it to
+        // differentiate between two items with the same start time
+        this._itemCounter = 0;
+        this._queuedItems = this._loadQueuedItems();
+        // TODO: also listen for unload just in case sync keeps on running after pagehide is fired?
+        window.addEventListener("pagehide", this, false);
+        this._flushInterval = this._platform.clock.createInterval(() => this._tryFlush(), flushInterval);
+    }
+
+    dispose() {
+        window.removeEventListener("pagehide", this, false);
+        this._flushInterval.dispose();
+    }
+
+    handleEvent(evt) {
+        if (evt.type === "pagehide") {
+            this._finishAllAndFlush();
+        }
+    }
+
+    async _tryFlush() {
+        const db = await this._openDB();
+        try {
+            const txn = db.transaction(["logs"], "readwrite");
+            const logs = txn.objectStore("logs");
+            const amount = this._queuedItems.length;
+            for(const i of this._queuedItems) {
+                logs.add(i);
+            }
+            const itemCount = await reqAsPromise(logs.count());
+            if (itemCount > this._limit) {
+                // delete an extra 10% so we don't need to delete every time we flush
+                let deleteAmount = (itemCount - this._limit) + Math.round(0.1 * this._limit);
+                await iterateCursor(logs.openCursor(), (_, __, cursor) => {
+                    cursor.delete();
+                    deleteAmount -= 1;
+                    return {done: deleteAmount === 0};
+                });
+            }
+            await txnAsPromise(txn);
+            this._queuedItems.splice(0, amount);
+        } catch (err) {
+            console.error("Could not flush logs", err);
+        } finally {
+            try {
+                db.close();
+            } catch (e) {}
+        }
+    }
+
+    _finishAllAndFlush() {
+        this._finishOpenItems();
+        this._persistQueuedItems(this._queuedItems);
+    }
+
+    _loadQueuedItems() {
+        const key = `${this._name}_queuedItems`;
+        try {
+            const json = window.localStorage.getItem(key);
+            if (json) {
+                window.localStorage.removeItem(key);
+                return JSON.parse(json);
+            }
+        } catch (err) {
+            console.error("Could not load queued log items", err);
+        }
+        return [];
+    }
+
+    _openDB() {
+        return openDatabase(this._name, db => db.createObjectStore("logs", {keyPath: "id"}), 1);
+    }
+    
+    _persistItem(serializedItem) {
+        this._itemCounter += 1;
+        this._queuedItems.push({
+            id: `${encodeUint64(serializedItem.s)}:${this._itemCounter}`,
+            json: JSON.stringify(serializedItem)
+        });
+    }
+
+    _persistQueuedItems(items) {
+        try {
+            window.localStorage.setItem(`${this._name}_queuedItems`, JSON.stringify(items));
+        } catch (e) {
+            console.error("Could not persist queued log items in localStorage, they will likely be lost", e);
+        }
+    }
+
+    async export() {
+        const db = await this._openDB();
+        try {
+            const txn = db.transaction(["logs"], "readonly");
+            const logs = txn.objectStore("logs");
+            const storedItems = await fetchResults(logs.openCursor(), () => false);
+            const allItems = storedItems.concat(this._queuedItems);
+            const sortedItems = allItems.sort((a, b) => {
+                return a.id > b.id;
+            });
+            return new IDBLogExport(sortedItems, this, this._platform);
+        } finally {
+            try {
+                db.close();
+            } catch (e) {}
+        }
+    }
+
+    async _removeItems(items) {
+        const db = await this._openDB();
+        try {
+            const txn = db.transaction(["logs"], "readwrite");
+            const logs = txn.objectStore("logs");
+            for (const item of items) {
+                const queuedIdx = this._queuedItems.findIndex(i => i.id === item.id);
+                if (queuedIdx === -1) {
+                    logs.delete(item.id);
+                } else {
+                    this._queuedItems.splice(queuedIdx, 1);
+                }
+            }
+            await txnAsPromise(txn);
+        } finally {
+            try {
+                db.close();
+            } catch (e) {}
+        }
+    }
+}
+
+class IDBLogExport {
+    constructor(items, logger, platform) {
+        this._items = items;
+        this._logger = logger;
+        this._platform = platform;
+    }
+    
+    get count() {
+        return this._items.length;
+    }
+
+    /**
+     * @return {Promise}
+     */
+    removeFromStore() {
+        return this._logger._removeItems(this._items);
+    }
+
+    asBlob() {
+        const log = {
+            version: 1,
+            items: this._items.map(i => JSON.parse(i.json))
+        };
+        const json = JSON.stringify(log);
+        const buffer = this._platform.encoding.utf8.encode(json);
+        const blob = this._platform.createBlob(buffer, "application/json");
+        return blob;
+    }
+}

src/logging/LogFilter.js

@@ -0,0 +1,60 @@
+/*
+Copyright 2020 Bruno Windels <bruno@windels.cloud>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export const LogLevel = {
+    All: 1,
+    Debug: 2,
+    Info: 3,
+    Warn: 4,
+    Error: 5,
+    Fatal: 6,
+    Off: 7,
+}
+
+export class LogFilter {
+    constructor(parentFilter) {
+        this._parentFilter = parentFilter;
+        this._min = null;
+        this._maxDepth = null;
+    }
+
+    filter(item, children, depth) {
+        if (this._parentFilter) {
+            if (!this._parentFilter.filter(item, children, depth)) {
+                return false;
+            }
+        }
+        // neither our children or us have a loglevel high enough, filter out.
+        if (this._min !== null && children === null && item.logLevel < this._min) {
+            return false;
+        } if (this._maxDepth !== null && depth > this._maxDepth) {
+            return false;
+        } else {
+            return true;
+        }
+    }
+
+    /* methods to build the filter */
+    minLevel(logLevel) {
+        this._min = logLevel;
+        return this;
+    }
+
+    maxDepth(depth) {
+        this._maxDepth = depth;
+        return this;
+    }
+}

src/logging/LogItem.js

@@ -0,0 +1,190 @@
+/*
+Copyright 2020 Bruno Windels <bruno@windels.cloud>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {LogLevel, LogFilter} from "./LogFilter.js";
+
+export class LogItem {
+    constructor(labelOrValues, logLevel, filterCreator, clock) {
+        this._clock = clock;
+        this._start = clock.now();
+        this._end = null;
+        // (l)abel
+        this._values = typeof labelOrValues === "string" ? {l: labelOrValues} : labelOrValues;
+        this.error = null;
+        this.logLevel = logLevel;
+        this._children = null;
+        this._filterCreator = filterCreator;
+    }
+
+    /**
+     * Creates a new child item and runs it in `callback`.
+     */
+    wrap(labelOrValues, callback, logLevel = LogLevel.Info, filterCreator = null) {
+        const item = this.child(labelOrValues, logLevel, filterCreator);
+        return item.run(callback);
+    }
+
+    get duration() {
+        if (this._end) {
+            return this._end - this._start;
+        } else {
+            return null;
+        }
+    }
+    
+    /**
+     * Creates a new child item that finishes immediately
+     * and can hence not be modified anymore.
+     * 
+     * Hence, the child item is not returned.
+     */
+    log(labelOrValues, logLevel = LogLevel.Info) {
+        const item = this.child(labelOrValues, logLevel, null);
+        item.end = item.start;
+    }
+
+    set(key, value) {
+        if(typeof key === "object") {
+            const values = key;
+            Object.assign(this._values, values);
+        } else {
+            this._values[key] = value;
+        }
+    }
+
+    serialize(filter, depth) {
+        if (this._filterCreator) {
+            try {
+                filter = this._filterCreator(new LogFilter(filter), this);
+            } catch (err) {
+                console.error("Error creating log item", err);
+            }
+        }
+        let children;
+        if (this._children !== null) {
+            children = this._children.reduce((array, c) => {
+                const s = c.serialize(filter, depth + 1);
+                if (s) {
+                    if (array === null) {
+                        array = [];
+                    }
+                    array.push(s);
+                }
+                return array;
+            }, null);
+        }
+        if (!filter.filter(this, children, depth)) {
+            return null;
+        }
+        const item = {
+            // (s)tart
+            s: this._start,
+            // (d)uration
+            d: this.duration,
+            // (v)alues
+            v: this._values,
+            // (l)evel
+            l: this.logLevel
+        };
+        if (this.error) {
+            // (e)rror
+            item.e = {
+                stack: this.error.stack,
+                name: this.error.name
+            };
+        }
+        if (children) {
+            // (c)hildren
+            item.c = children;
+        }
+        return item;
+    }
+
+    /**
+     * You probably want to use `wrap` instead of this.
+     * 
+     * Runs a callback passing this log item,
+     * recording the timing and any error.
+     *
+     * callback can return a Promise.
+     *
+     * Should only be called once.
+     * 
+     * @param  {Function} callback [description]
+     * @return {[type]}            [description]
+     */
+    run(callback) {
+        if (this._end !== null) {
+            console.trace("log item is finished, additional logs will likely not be recorded");
+        }
+        let result;
+        try {
+            result = callback(this);
+            if (result instanceof Promise) {
+                return result.then(promiseResult => {
+                    this.finish();
+                    return promiseResult;
+                }, err => {
+                    throw this.catch(err);
+                });
+            } else {
+                this.finish();
+                return result;
+            }
+        } catch (err) {
+            throw this.catch(err);
+        }
+    }
+
+    /**
+     * finished the item, recording the end time. After finishing, an item can't be modified anymore as it will be persisted.
+     * @internal shouldn't typically be called by hand. allows to force finish if a promise is still running when closing the app
+     */
+    finish() {
+        if (this._end === null) {
+            if (this._children !== null) {
+                for(const c of this._children) {
+                    c.finish();
+                }
+            }
+            this._end = this._clock.now();
+        }
+    }
+
+    // expose log level without needing import everywhere
+    get level() {
+        return LogLevel;
+    }
+
+    catch(err) {
+        this.error = err;
+        this.logLevel = LogLevel.Error;
+        this.finish();
+        return err;
+    }
+
+    child(labelOrValues, logLevel, filterCreator) {
+        if (this._end !== null) {
+            console.trace("log item is finished, additional logs will likely not be recorded");
+        }
+        const item = new LogItem(labelOrValues, logLevel, filterCreator, this._clock);
+        if (this._children === null) {
+            this._children = [];
+        }
+        this._children.push(item);
+        return item;
+    }
+}

src/logging/NullLogger.js

@@ -0,0 +1,57 @@
+/*
+Copyright 2020 Bruno Windels <bruno@windels.cloud>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+import {LogLevel} from "./LogFilter.js";
+
+export class NullLogger {
+    constructor() {
+        this._item = new NullLogItem();
+    }
+
+    run(_, callback) {
+        return callback(this._item);    
+    }
+
+    async export() {
+        return null;
+    }
+
+    get level() {
+        return LogLevel;
+    }
+}
+
+class NullLogItem {
+    wrap(_, callback) {
+        return callback(this);
+    }
+    log() {}
+    set() {}
+    anonymize() {}
+
+    get level() {
+        return LogLevel;
+    }
+
+    catch(err) {
+        return err;
+    }
+
+    child() {
+        return this;
+    }
+
+    finish() {}
+}

src/matrix/Session.js

@@ -207,7 +207,12 @@ export class Session {
 
     async _createSessionBackup(ssssKey, txn) {
         const secretStorage = new SecretStorage({key: ssssKey, platform: this._platform});
-        this._sessionBackup = await SessionBackup.fromSecretStorage({olm: this._olm, secretStorage, hsApi: this._hsApi, txn});
+        this._sessionBackup = await SessionBackup.fromSecretStorage({
+            platform: this._platform,
+            olm: this._olm, secretStorage,
+            hsApi: this._hsApi,
+            txn
+        });
         if (this._sessionBackup) {
             for (const room of this._rooms.values()) {
                 if (room.isEncrypted) {

src/matrix/SessionContainer.js

@@ -184,7 +184,7 @@ export class SessionContainer {
             await this._session.createIdentity();
         }
         
-        this._sync = new Sync({hsApi: this._requestScheduler.hsApi, storage: this._storage, session: this._session});
+        this._sync = new Sync({hsApi: this._requestScheduler.hsApi, storage: this._storage, session: this._session, logger: this._platform.logger});
         // notify sync and session when back online
         this._reconnectSubscription = this._reconnector.connectionStatus.subscribe(state => {
             if (state === ConnectionStatus.Online) {

src/matrix/Sync.js

@@ -56,8 +56,9 @@ function timelineIsEmpty(roomResponse) {
  * ```
  */
 export class Sync {
-    constructor({hsApi, session, storage}) {
+    constructor({hsApi, session, storage, logger}) {
         this._hsApi = hsApi;
+        this._logger = logger;
         this._session = session;
         this._storage = storage;
         this._currentRequest = null;
@@ -108,7 +109,18 @@ export class Sync {
                 //   for us. We do that by calling it with a zero timeout until it
                 //   doesn't give us any more to_device messages.
                 const timeout = this._status.get() === SyncStatus.Syncing ? INCREMENTAL_TIMEOUT : 0; 
-                const syncResult = await this._syncRequest(syncToken, timeout);
+                const syncResult = await this._logger.run("sync",
+                    log => this._syncRequest(syncToken, timeout, log),
+                    this._logger.level.Info,
+                    (filter, log) => {
+                        if (log.duration >= 2000 || this._status.get() === SyncStatus.CatchupSync) {
+                            return filter.minLevel(log.level.Info);
+                        } else if (log.error) {
+                            return filter.minLevel(log.level.Error);
+                        } else {
+                            return filter.maxDepth(0);
+                        }
+                });
                 syncToken = syncResult.syncToken;
                 roomStates = syncResult.roomStates;
                 sessionChanges = syncResult.sessionChanges;
@@ -169,28 +181,30 @@ export class Sync {
         await Promise.all(roomsPromises.concat(sessionPromise));
     }
 
-    async _syncRequest(syncToken, timeout) {
+    async _syncRequest(syncToken, timeout, log) {
         let {syncFilterId} = this._session;
         if (typeof syncFilterId !== "string") {
             this._currentRequest = this._hsApi.createFilter(this._session.user.id, {room: {state: {lazy_load_members: true}}});
             syncFilterId = (await this._currentRequest.response()).filter_id;
         }
         const totalRequestTimeout = timeout + (80 * 1000);  // same as riot-web, don't get stuck on wedged long requests
-        this._currentRequest = this._hsApi.sync(syncToken, syncFilterId, timeout, {timeout: totalRequestTimeout});
+        this._currentRequest = this._hsApi.sync(syncToken, syncFilterId, timeout, {timeout: totalRequestTimeout, log});
         const response = await this._currentRequest.response();
 
         const isInitialSync = !syncToken;
         syncToken = response.next_batch;
+        log.set("syncToken", syncToken);
+        log.set("status", this._status.get());
+
         const roomStates = this._parseRoomsResponse(response.rooms, isInitialSync);
-        await this._prepareRooms(roomStates);
+        await log.wrap("prepare rooms", log => this._prepareRooms(roomStates, log));
         let sessionChanges;
         const syncTxn = this._openSyncTxn();
         try {
-            sessionChanges = await this._session.writeSync(response, syncFilterId, syncTxn);
+            sessionChanges = await log.wrap("session.writeSync", log => this._session.writeSync(response, syncFilterId, syncTxn, log));
             await Promise.all(roomStates.map(async rs => {
-                console.log(` * applying sync response to room ${rs.room.id} ...`);
+                rs.changes = await log.wrap("room.writeSync", log => rs.room.writeSync(
-                rs.changes = await rs.room.writeSync(
+                    rs.roomResponse, isInitialSync, rs.preparation, syncTxn, log));
-                    rs.roomResponse, isInitialSync, rs.preparation, syncTxn);
             }));
         } catch(err) {
             // avoid corrupting state by only
@@ -232,10 +246,10 @@ export class Sync {
         ]);
     }
 
-    async _prepareRooms(roomStates) {
+    async _prepareRooms(roomStates, log) {
         const prepareTxn = this._openPrepareSyncTxn();
         await Promise.all(roomStates.map(async rs => {
-            rs.preparation = await rs.room.prepareSync(rs.roomResponse, rs.membership, prepareTxn);
+            rs.preparation = await log.wrap("room.prepareSync", log => rs.room.prepareSync(rs.roomResponse, rs.membership, prepareTxn, log));
         }));
         // This is needed for safari to not throw TransactionInactiveErrors on the syncTxn. See docs/INDEXEDDB.md
         await prepareTxn.complete();

src/matrix/e2ee/attachment.js

@@ -14,8 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import base64 from "../../../lib/base64-arraybuffer/index.js";
-
 /**
  * Decrypt an attachment.
  * @param {ArrayBuffer} ciphertextBuffer The encrypted attachment data buffer.
@@ -25,12 +23,14 @@ import base64 from "../../../lib/base64-arraybuffer/index.js";
  * @param {string} info.hashes.sha256 Base64 encoded SHA-256 hash of the ciphertext.
  * @return {Promise} A promise that resolves with an ArrayBuffer when the attachment is decrypted.
  */
-export async function decryptAttachment(crypto, ciphertextBuffer, info) {
+export async function decryptAttachment(platform, ciphertextBuffer, info) {
     if (info === undefined || info.key === undefined || info.iv === undefined
         || info.hashes === undefined || info.hashes.sha256 === undefined) {
        throw new Error("Invalid info. Missing info.key, info.iv or info.hashes.sha256 key");
     }
 
+    const {crypto} = platform;
+    const {base64} = platform.encoding;
     var ivArray = base64.decode(info.iv);
     // re-encode to not deal with padded vs unpadded
     var expectedSha256base64 = base64.encode(base64.decode(info.hashes.sha256));
@@ -59,6 +59,7 @@ export async function decryptAttachment(crypto, ciphertextBuffer, info) {
 
 export async function encryptAttachment(platform, blob) {
     const {crypto} = platform;
+    const {base64} = platform.encoding;
     const iv = await crypto.aes.generateIV();
     const key = await crypto.aes.generateKey("jwk", 256);
     const buffer = await blob.readAsBuffer();
@@ -69,20 +70,10 @@ export async function encryptAttachment(platform, blob) {
         info: {
             v: "v2",
             key,
-            iv: encodeUnpaddedBase64(iv),
+            iv: base64.encodeUnpadded(iv),
             hashes: {
-                sha256: encodeUnpaddedBase64(digest)
+                sha256: base64.encodeUnpadded(digest)
             }
         }
     };
 }
-
-function encodeUnpaddedBase64(buffer) {
-    const str = base64.encode(buffer);
-    const paddingIdx = str.indexOf("=");
-    if (paddingIdx !== -1) {
-        return str.substr(0, paddingIdx);
-    } else {
-        return str;
-    }
-}

src/matrix/e2ee/megolm/SessionBackup.js

@@ -14,8 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import base64 from "../../../../lib/base64-arraybuffer/index.js";
-
 export class SessionBackup {
     constructor({backupInfo, decryption, hsApi}) {
         this._backupInfo = backupInfo;
@@ -41,10 +39,10 @@ export class SessionBackup {
         this._decryption.free();
     }
 
-    static async fromSecretStorage({olm, secretStorage, hsApi, txn}) {
+    static async fromSecretStorage({platform, olm, secretStorage, hsApi, txn}) {
         const base64PrivateKey = await secretStorage.readSecret("m.megolm_backup.v1", txn);
         if (base64PrivateKey) {
-            const privateKey = new Uint8Array(base64.decode(base64PrivateKey));
+            const privateKey = new Uint8Array(platform.encoding.base64.decode(base64PrivateKey));
             const backupInfo = await hsApi.roomKeysVersion().response();
             const expectedPubKey = backupInfo.auth_data.public_key;
             const decryption = new olm.PkDecryption();

src/matrix/net/HomeServerApi.js

@@ -19,24 +19,34 @@ import {HomeServerError, ConnectionError} from "../error.js";
 import {encodeQueryParams} from "./common.js";
 
 class RequestWrapper {
-    constructor(method, url, requestResult) {
+    constructor(method, url, requestResult, log) {
+        this._log = log;
         this._requestResult = requestResult;
         this._promise = requestResult.response().then(response => {
+            log?.set("status", response.status);
             // ok?
             if (response.status >= 200 && response.status < 300) {
+                log?.finish();
                 return response.body;
             } else {
                 if (response.status >= 400 && !response.body?.errcode) {
-                    throw new ConnectionError(`HTTP error status ${response.status} without errcode in body, assume this is a load balancer complaining the server is offline.`);
+                    const err = new ConnectionError(`HTTP error status ${response.status} without errcode in body, assume this is a load balancer complaining the server is offline.`);
+                    log?.catch(err);
+                    throw err;
                 } else {
-                    throw new HomeServerError(method, url, response.body, response.status);
+                    const err = new HomeServerError(method, url, response.body, response.status);
+                    log?.catch(err);
+                    throw err;
                 }
             }
         }, err => {
             // if this._requestResult is still set, the abort error came not from calling abort here
             if (err.name === "AbortError" && this._requestResult) {
-                throw new Error(`Request ${method} ${url} was unexpectedly aborted, see #187.`);
+                const err = new Error(`Unexpectedly aborted, see #187.`);
+                log?.catch(err);
+                throw err;
             } else {
+                log?.catch(err);
                 throw err;
             }
         });
@@ -44,6 +54,7 @@ class RequestWrapper {
 
     abort() {
         if (this._requestResult) {
+            this._log?.set("aborted", true);
             this._requestResult.abort();
             // to mark that it was on purpose in above rejection handler
             this._requestResult = null;
@@ -93,6 +104,15 @@ export class HomeServerApi {
     _baseRequest(method, url, queryParams, body, options, accessToken) {
         const queryString = encodeQueryParams(queryParams);
         url = `${url}?${queryString}`;
+        let log;
+        if (options?.log) {
+            const parent = options?.log;
+            log = parent.child({
+                kind: "request",
+                url,
+                method,
+            }, parent.level.Info);
+        }
         let encodedBody;
         const headers = new Map();
         if (accessToken) {
@@ -105,6 +125,7 @@ export class HomeServerApi {
             headers.set("Content-Length", encoded.length);
             encodedBody = encoded.body;
         }
+
         const requestResult = this._requestFn(url, {
             method,
             headers,
@@ -114,7 +135,7 @@ export class HomeServerApi {
             format: "json"  // response format
         });
 
-        const wrapper = new RequestWrapper(method, url, requestResult);
+        const wrapper = new RequestWrapper(method, url, requestResult, log);
         
         if (this._reconnector) {
             wrapper.response().catch(err => {

src/matrix/net/MediaRepository.js

@@ -55,7 +55,7 @@ export class MediaRepository {
     async downloadEncryptedFile(fileEntry, cache = false) {
         const url = this.mxcUrl(fileEntry.url);
         const {body: encryptedBuffer} = await this._platform.request(url, {method: "GET", format: "buffer", cache}).response();
-        const decryptedBuffer = await decryptAttachment(this._platform.crypto, encryptedBuffer, fileEntry);
+        const decryptedBuffer = await decryptAttachment(this._platform, encryptedBuffer, fileEntry);
         return this._platform.createBlob(decryptedBuffer, fileEntry.mimetype);
     }
 

src/matrix/room/Room.js

@@ -175,7 +175,8 @@ export class Room extends EventEmitter {
         return request;
     }
 
-    async prepareSync(roomResponse, membership, txn) {
+    async prepareSync(roomResponse, membership, txn, log) {
+        log.set("roomId", this.id);
         const summaryChanges = this._summary.data.applySyncResponse(roomResponse, membership)
         let roomEncryption = this._roomEncryption;
         // encryption is enabled in this sync
@@ -211,7 +212,8 @@ export class Room extends EventEmitter {
     }
 
     /** @package */
-    async writeSync(roomResponse, isInitialSync, {summaryChanges, decryptChanges, roomEncryption}, txn) {
+    async writeSync(roomResponse, isInitialSync, {summaryChanges, decryptChanges, roomEncryption}, txn, log) {
+        log.set("roomId", this.id);
         const {entries, newLiveKey, memberChanges} =
             await this._syncWriter.writeSync(roomResponse, txn);
         if (decryptChanges) {

src/matrix/ssss/SecretStorage.js

@@ -14,8 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import base64 from "../../../lib/base64-arraybuffer/index.js";
-
 export class SecretStorage {
     constructor({key, platform}) {
         this._key = key;
@@ -40,17 +38,17 @@ export class SecretStorage {
     }
 
     async _decryptAESSecret(type, encryptedData) {
+        const {base64, utf8} = this._platform.encoding;
         // now derive the aes and mac key from the 4s key
         const hkdfKey = await this._platform.crypto.derive.hkdf(
             this._key.binaryKey,
             new Uint8Array(8).buffer,   //zero salt
-            this._platform.utf8.encode(type), // info
+            utf8.encode(type), // info
             "SHA-256",
             512 // 512 bits or 64 bytes
         );
         const aesKey = hkdfKey.slice(0, 32);
         const hmacKey = hkdfKey.slice(32);
-
         const ciphertextBytes = base64.decode(encryptedData.ciphertext);
 
         const isVerified = await this._platform.crypto.hmac.verify(
@@ -67,6 +65,6 @@ export class SecretStorage {
             data: ciphertextBytes
         });
 
-        return this._platform.utf8.decode(plaintextBytes);
+        return utf8.decode(plaintextBytes);
     }
 }

src/matrix/ssss/index.js

@@ -56,7 +56,7 @@ export async function keyFromCredential(type, credential, storage, platform, olm
     if (type === "phrase") {
         key = await keyFromPassphrase(keyDescription, credential, platform);
     } else if (type === "key") {
-        key = keyFromRecoveryKey(olm, keyDescription, credential);
+        key = keyFromRecoveryKey(keyDescription, credential, olm, platform);
     } else {
         throw new Error(`Invalid type: ${type}`);
     }

src/matrix/ssss/passphrase.js

@@ -33,11 +33,12 @@ export async function keyFromPassphrase(keyDescription, passphrase, platform) {
     if (passphraseParams.algorithm !== "m.pbkdf2") {
         throw new Error(`Unsupported passphrase algorithm: ${passphraseParams.algorithm}`);
     }
+    const {utf8} = platform.encoding;
     const keyBits = await platform.crypto.derive.pbkdf2(
-        platform.utf8.encode(passphrase),
+        utf8.encode(passphrase),
         passphraseParams.iterations || DEFAULT_ITERATIONS,
         // salt is just a random string, not encoded in any way
-        platform.utf8.encode(passphraseParams.salt),
+        utf8.encode(passphraseParams.salt),
         "SHA-512",
         passphraseParams.bits || DEFAULT_BITSIZE);
     return new Key(keyDescription, keyBits);

src/matrix/ssss/recoveryKey.js

@@ -13,7 +13,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-import bs58 from "../../../lib/bs58/index.js";
 import {Key} from "./common.js";
 
 const OLM_RECOVERY_KEY_PREFIX = [0x8B, 0x01];
@@ -24,8 +23,8 @@ const OLM_RECOVERY_KEY_PREFIX = [0x8B, 0x01];
  * @param  {string} recoveryKey
  * @return {Key}
  */
-export function keyFromRecoveryKey(olm, keyDescription, recoveryKey) {
+export function keyFromRecoveryKey(keyDescription, recoveryKey, olm, platform) {
-    const result = bs58.decode(recoveryKey.replace(/ /g, ''));
+    const result = platform.encoding.base58.decode(recoveryKey.replace(/ /g, ''));
 
     let parity = 0;
     for (const b of result) {

src/matrix/storage/idb/utils.js

@@ -54,6 +54,12 @@ export function encodeUint32(n) {
     return "0".repeat(8 - hex.length) + hex;
 }
 
+// used for logs where timestamp is part of key, which is larger than 32 bit
+export function encodeUint64(n) {
+    const hex = n.toString(16);
+    return "0".repeat(16 - hex.length) + hex;
+}
+
 export function decodeUint32(str) {
     return parseInt(str, 16);
 }
@@ -110,7 +116,7 @@ export function iterateCursor(cursorRequest, processValue) {
                 needsSyncPromise && Promise._flush && Promise._flush();
                 return; // end of results
             }
-            const result = processValue(cursor.value, cursor.key);
+            const result = processValue(cursor.value, cursor.key, cursor);
             // TODO: don't use object for result and assume it's jumpTo when not === true/false or undefined
             const done = result?.done;
             const jumpTo = result?.jumpTo;

src/platform/web/Platform.js

@@ -19,8 +19,9 @@ import {xhrRequest} from "./dom/request/xhr.js";
 import {StorageFactory} from "../../matrix/storage/idb/StorageFactory.js";
 import {SessionInfoStorage} from "../../matrix/sessioninfo/localstorage/SessionInfoStorage.js";
 import {SettingsStorage} from "./dom/SettingsStorage.js";
-import {UTF8} from "./dom/UTF8.js";
+import {Encoding} from "./utils/Encoding.js";
 import {OlmWorker} from "../../matrix/e2ee/OlmWorker.js";
+import {IDBLogger} from "../../logging/IDBLogger.js";
 import {RootView} from "./ui/RootView.js";
 import {Clock} from "./dom/Clock.js";
 import {ServiceWorkerHandler} from "./dom/ServiceWorkerHandler.js";
@@ -84,8 +85,11 @@ export class Platform {
     constructor(container, paths, cryptoExtras = null) {
         this._paths = paths;
         this._container = container;
-        this.utf8 = new UTF8();
+        this.settingsStorage = new SettingsStorage("hydrogen_setting_v1_");
         this.clock = new Clock();
+        this.encoding = new Encoding();
+        this.random = Math.random;
+        this.logger = new IDBLogger({name: "hydrogen_logs", platform: this});
         this.history = new History();
         this.onlineStatus = new OnlineStatus();
         this._serviceWorkerHandler = null;
@@ -96,9 +100,7 @@ export class Platform {
         this.crypto = new Crypto(cryptoExtras);
         this.storageFactory = new StorageFactory(this._serviceWorkerHandler);
         this.sessionInfoStorage = new SessionInfoStorage("hydrogen_sessions_v1");
-        this.settingsStorage = new SettingsStorage("hydrogen_setting_v1_");
         this.estimateStorageUsage = estimateStorageUsage;
-        this.random = Math.random;
         if (typeof fetch === "function") {
             this.request = createFetchRequest(this.clock.createTimeout);
         } else {

src/platform/web/dom/Crypto.js

@@ -14,6 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+import base64 from "../../../../lib/base64-arraybuffer/index.js";
+
 // turn IE11 result into promise
 function subtleCryptoResult(promiseOrOp, method) {
     if (promiseOrOp instanceof Promise) {
@@ -302,7 +304,6 @@ function rawKeyToJwk(key) {
     };
 }
 
-import base64 from "../../../../lib/base64-arraybuffer/index.js";
 
 class AESLegacyCrypto {
     constructor(aesjs, crypto) {

src/platform/web/dom/SettingsStorage.js

@@ -20,18 +20,34 @@ export class SettingsStorage {
     }
 
     async setInt(key, value) {
-        window.localStorage.setItem(`${this._prefix}${key}`, value);
+        this._set(key, value);
     }
 
-    async getInt(key) {
+    async getInt(key, defaultValue = 0) {
         const value = window.localStorage.getItem(`${this._prefix}${key}`);
         if (typeof value === "string") {
             return parseInt(value, 10);
         }
-        return;
+        return defaultValue;
+    }
+
+    async setBool(key, value) {
+        this._set(key, value);
+    }
+
+    async getBool(key, defaultValue = false) {
+        const value = window.localStorage.getItem(`${this._prefix}${key}`);
+        if (typeof value === "string") {
+            return value === "true";
+        }
+        return defaultValue;
     }
 
     async remove(key) {
         window.localStorage.removeItem(`${this._prefix}${key}`);
     }
+
+    async _set(key, value) {
+        window.localStorage.setItem(`${this._prefix}${key}`, value);
+    }
 }

src/platform/web/ui/session/settings/SettingsView.js

@@ -51,6 +51,9 @@ export class SettingsView extends TemplateView {
                 t.h3("Application"),
                 row(vm.i18n`Version`, version),
                 row(vm.i18n`Storage usage`, vm => `${vm.storageUsage} / ${vm.storageQuota}`),
+                row(vm.i18n`Debug logs`, t.button({onClick: () => vm.exportLogs()}, "Export")),
+                t.p(["Debug logs contain application usage data including your username, the IDs or aliases of the rooms or groups you have visited and the usernames of other users. They do not contain messages. For more information, review our ",
+                    t.a({href: "https://element.io/privacy", target: "_blank", rel: "noopener"}, "privacy policy"), "."]),
             ])
         ]);
     }

src/platform/web/utils/Base58.js

@@ -0,0 +1,27 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import bs58 from "../../../../lib/bs58/index.js";
+
+export class Base58 {
+    encode(buffer) {
+        return bs58.encode(buffer);
+    }
+
+    decode(str) {
+        return bs58.decode(str);
+    }
+}

src/platform/web/utils/Base64.js

@@ -0,0 +1,37 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import base64 from "../../../../lib/base64-arraybuffer/index.js";
+
+export class Base64 {
+    encodeUnpadded(buffer) {
+        const str = base64.encode(buffer);
+        const paddingIdx = str.indexOf("=");
+        if (paddingIdx !== -1) {
+            return str.substr(0, paddingIdx);
+        } else {
+            return str;
+        }
+    }
+
+    encode(buffer) {
+        return base64.encode(buffer);
+    }
+
+    decode(str) {
+        return base64.decode(str);
+    }
+}

src/platform/web/utils/Encoding.js

@@ -0,0 +1,27 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {UTF8} from "../dom/UTF8.js";
+import {Base64} from "./Base64.js";
+import {Base58} from "./Base58.js";
+
+export class Encoding {
+    constructor() {
+        this.utf8 = new UTF8();
+        this.base64 = new Base64();
+        this.base58 = new Base58();
+    }
+}