From 89599e9f870ee046f24a1a77264e31a3d004d244 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Thu, 11 Feb 2021 21:07:18 +0100 Subject: [PATCH] WIP --- src/domain/ViewModel.js | 19 ++-- src/logging/IDBLogger.js | 188 ++++++++++++++++++++++++++++++++ src/logging/LogItem.js | 94 ++++++++++++++++ src/logging/Logger.js | 51 +++++++++ src/matrix/storage/idb/utils.js | 2 +- 5 files changed, 343 insertions(+), 11 deletions(-) create mode 100644 src/logging/IDBLogger.js create mode 100644 src/logging/LogItem.js create mode 100644 src/logging/Logger.js diff --git a/src/domain/ViewModel.js b/src/domain/ViewModel.js index dd2d9819..f0588fc7 100644 --- a/src/domain/ViewModel.js +++ b/src/domain/ViewModel.js @@ -30,8 +30,8 @@ export class ViewModel extends EventEmitter { } childOptions(explicitOptions) { - const {navigation, urlCreator, platform} = this._options; - return Object.assign({navigation, urlCreator, platform}, explicitOptions); + const {navigation, urlCreator, platform, logger} = this._options; + return Object.assign({navigation, urlCreator, platform, logger}, explicitOptions); } // makes it easier to pass through dependencies of a sub-view model @@ -78,14 +78,9 @@ export class ViewModel extends EventEmitter { // we probably are, if we're using routing with a url, we could just refresh. i18n(parts, ...expr) { // just concat for now - let result = ""; - for (let i = 0; i < parts.length; ++i) { - result = result + parts[i]; - if (i < expr.length) { - result = result + expr[i]; - } - } - return result; + return parts.reduce((all, p, i) => { + return all + p + expr[i]; + }); } updateOptions(options) { @@ -108,6 +103,10 @@ export class ViewModel extends EventEmitter { return this._options.platform.clock; } + get logger() { + return this._options.logger; + } + /** * The url router, only meant to be used to create urls with from view models. * @return {URLRouter} diff --git a/src/logging/IDBLogger.js b/src/logging/IDBLogger.js new file mode 100644 index 00000000..1192f0b5 --- /dev/null +++ b/src/logging/IDBLogger.js @@ -0,0 +1,188 @@ +/* +Copyright 2020 Bruno Windels + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {openDatabase, txnAsPromise, reqAsPromise, iterateCursor, fetchResults} from "../matrix/storage/idb/utils.js"; +import {LogItem} from "./LogItem.js"; + +class Logger { + constructor(clock) { + this._openItems = new Set(); + this._clock = clock; + } + + decend(label, callback, logLevel) { + const item = new LogItem(label, this, logLevel, this._clock); + + const failItem = (err) => { + item.catch(err); + finishItem(); + throw err; + }; + + const finishItem = () => { + item.finish(); + this._persistItem(item); + this._openItems.remove(item); + }; + + let result; + try { + result = callback(item); + if (result instanceof Promise) { + result = result.then(promiseResult => { + finishItem(); + return promiseResult; + }, failItem); + } + } catch (err) { + failItem(err); + } + } + + _persistItem(item) { + throw new Error("not implemented"); + } + + async extractItems() { + throw new Error("not implemented"); + } +} + +export function encodeUint64(n) { + const hex = n.toString(16); + return "0".repeat(16 - hex.length) + hex; +} + +export default class IDBLogger extends Logger { + constructor({name, clock, utf8, flushInterval = 2 * 60 * 1000, limit = 1000}) { + super(clock); + this._utf8 = utf8; + this._name = name; + this._limit = limit; + // does not get loaded from idb on startup as we only use it to + // differentiate between two items with the same start time + this._itemCounter = 0; + this._queuedItems = this._loadQueuedItems(); + window.addEventListener("pagehide", this, false); + this._flushInterval = this._clock.createInterval(() => this._tryFlush(), flushInterval); + } + + dispose() { + window.removeEventListener("pagehide", this, false); + this._flushInterval.dispose(); + } + + handleEvent(evt) { + if (evt.type === "pagehide") { + this._finishAllAndFlush(); + } + } + + async _tryFlush() { + const db = await this._openDB(); + try { + const txn = this.db.transaction(["logs"], "readwrite"); + const logs = txn.objectStore("logs"); + const amount = this._queuedItems.length; + for(const i of this._queuedItems) { + logs.add(i); + } + // trim logs if needed + const itemCount = await reqAsPromise(logs.count()); + if (itemCount > this._limit) { + let currentCount = itemCount; + await iterateCursor(logs.openCursor(), (_, __, cursor) => { + cursor.delete(); + currentCount -= 1; + return {done: currentCount <= this._limit}; + }); + } + await txnAsPromise(txn); + this._queuedItems.splice(0, amount); + } finally { + try { + db.close(); + } catch (e) {} + } + } + + _finishAllAndFlush() { + for (const openItem of this._openItems) { + openItem.finish(); + this._persistItem(openItem); + } + this._openItems.clear(); + this._persistQueuedItems(this._queuedItems); + } + + _loadQueuedItems() { + const key = `${this._name}_queuedItems`; + const json = window.localStorage.getItem(key); + if (json) { + window.localStorage.removeItem(key); + return JSON.parse(json); + } + return []; + } + + _openDB() { + return openDatabase(this._name, db => db.createObjectStore("logs", {keyPath: "id"}), 1); + } + + _persistItem(item) { + this._itemCounter += 1; + this._queuedItems.push({ + id: `${encodeUint64(item.start)}:${this._itemCounter}`, + // store as buffer so parsing overhead is lower + content: this._utf8.encode(JSON.stringify(item.serialize())) + }); + } + + _persistQueuedItems(items) { + window.localStorage.setItem(`${this._name}_queuedItems`, JSON.stringify(items)); + } + + // should we actually delete items and just not rely on trimming for it not to grow too large? + // just worried that people will create a file, not do anything with it, hit export logs again and then + // send a mostly empty file. we could just not delete but in _tryFlush (and perhaps make trimming delete a few more than needed to be below limit) + // how does eleweb handle this? + // + // both deletes and reads items from store + async extractItems() { + const db = this._openDB(); + try { + const queuedItems = this._queuedItems.slice(); + const txn = this.db.transaction(["logs"], "readwrite"); + const logs = txn.objectStore("logs"); + const items = await fetchResults(logs.openCursor(), () => false); + // we know we have read all the items as we're doing this in a txn + logs.clear(); + await txnAsPromise(txn); + // once the transaction is complete, remove the queued items + this._queuedItems.splice(0, queuedItems.length); + const sortedItems = items.concat(queuedItems).sort((a, b) => { + return a.id > b.id; + }).map(i => { + + }); + return sortedItems; + } finally { + try { + db.close(); + } catch (e) {} + } + } +} diff --git a/src/logging/LogItem.js b/src/logging/LogItem.js new file mode 100644 index 00000000..7136cf56 --- /dev/null +++ b/src/logging/LogItem.js @@ -0,0 +1,94 @@ +/* +Copyright 2020 Bruno Windels + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +export class LogItem { + constructor(label, parent, logLevel, clock) { + this._clock = clock; + this._start = clock.now(); + this._end = null; + this._values = {label}; + this._parent = parent; + this._error = null; + this._children = []; + this._logLevel = logLevel; + } + + // should this handle errors in the same way as logger.descend/start? + descend(label, logLevel = this._logLevel) { + if (this._end !== null) { + throw new Error("can't descend on finished item"); + } + const item = new LogItem(label, logLevel); + this._children.push(item); + return item; + } + + set(key, value) { + if(typeof key === "object") { + const values = key; + Object.assign(this._values, values); + } else { + this._values[key] = value; + } + } + + // XXX: where will this be called? from wrapAsync? + finish() { + if (this._end === null) { + for(const c of this._children) { + c.finish(); + } + this._end = this._clock.now(); + if (this._parent) { + this._parent._closeChild(this); + } + } + } + + catch(err) { + this._error = err; + console.error(`log item ${this.values.label} failed: ${err.message}:\n${err.stack}`); + } + + serialize() { + let error; + if (this._error) { + error = { + message: this._error.message, + stack: this._error.stack, + }; + } + return { + start: this._start, + end: this._end, + values: this._values, + error, + children: this._children.map(c => c.serialize()), + logLevel: this._logLevel + }; + } + + async wrapAsync(fn) { + try { + const ret = await fn(this); + this.finish(); + return ret; + } catch (err) { + this.fail(err); + throw err; + } + } +} diff --git a/src/logging/Logger.js b/src/logging/Logger.js new file mode 100644 index 00000000..98654aad --- /dev/null +++ b/src/logging/Logger.js @@ -0,0 +1,51 @@ +/* +Copyright 2020 Bruno Windels + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {LogItem} from "./LogItem.js"; + +export class Logger { + constructor(persister) { + this._openItems = []; + this._persister = persister; + this._closing = false; + } + + restore() { + const items = this._persister.loadTempItems(); + return this._persister.persistItems(items); + } + + start(label, logLevel) { + const item = new LogItem(label, logLevel, this); + this._openItems.push(item); + return item; + } + + _closeChild(item) { + if (!this._closing) { + this._persister.persistItems([item.serialize()]); + } + } + + close() { + this._closing = true; + for(const i of this._openItems) { + i.finish(); + } + this._closing = false; + this._persister.persistTempItems(this._openItems.map(i => i.serialize())); + } +} diff --git a/src/matrix/storage/idb/utils.js b/src/matrix/storage/idb/utils.js index 9d3a42d9..e06fa3a0 100644 --- a/src/matrix/storage/idb/utils.js +++ b/src/matrix/storage/idb/utils.js @@ -110,7 +110,7 @@ export function iterateCursor(cursorRequest, processValue) { needsSyncPromise && Promise._flush && Promise._flush(); return; // end of results } - const result = processValue(cursor.value, cursor.key); + const result = processValue(cursor.value, cursor.key, cursor); // TODO: don't use object for result and assume it's jumpTo when not === true/false or undefined const done = result?.done; const jumpTo = result?.jumpTo;