split up RoomPersister to SyncPersister

also rename stores to timelineEvents and timelineFragments

doc/FRAGMENTS.md

@@ -17,7 +17,7 @@
     how will fragments be exposed in timeline store?
         - all read operations are passed a fragment id
  - adapt persister
-    - persist fragments in /sync
+    - DONE: persist fragments in /sync
     - load n items before and after key
     - fill gaps / fragment filling
  - add live fragment id optimization if we haven't done so already

src/matrix/room/room.js

@@ -1,7 +1,8 @@
 import EventEmitter from "../../EventEmitter.js";
 import RoomSummary from "./summary.js";
-import RoomPersister from "./persister.js";
 import Timeline from "./timeline.js";
+import SyncPersister from "./timeline/persistence/SyncPersister.js";
+import FragmentIdComparer from "./timeline/FragmentIdComparer.js";
 
 export default class Room extends EventEmitter {
 	constructor({roomId, storage, hsApi, emitCollectionChange}) {
@@ -10,14 +11,15 @@ export default class Room extends EventEmitter {
         this._storage = storage;
         this._hsApi = hsApi;
 		this._summary = new RoomSummary(roomId);
-		this._persister = new RoomPersister({roomId, storage});
+        this._fragmentIdComparer = new FragmentIdComparer([]);
+		this._syncPersister = new SyncPersister({roomId, storage, fragmentIdComparer: this._fragmentIdComparer});
         this._emitCollectionChange = emitCollectionChange;
         this._timeline = null;
 	}
 
     persistSync(roomResponse, membership, txn) {
 		const summaryChanged = this._summary.applySync(roomResponse, membership, txn);
-		const newTimelineEntries = this._persister.persistSync(roomResponse, txn);
+		const newTimelineEntries = this._syncPersister.persistSync(roomResponse, txn);
         return {summaryChanged, newTimelineEntries};
     }
 
@@ -33,7 +35,7 @@ export default class Room extends EventEmitter {
 
 	load(summary, txn) {
 		this._summary.load(summary);
-		return this._persister.load(txn);
+		return this._syncPersister.load(txn);
 	}
 
     get name() {
@@ -51,7 +53,6 @@ export default class Room extends EventEmitter {
         this._timeline = new Timeline({
             roomId: this.id,
             storage: this._storage,
-            persister: this._persister,
             hsApi: this._hsApi,
             closeCallback: () => this._timeline = null,
         });

src/matrix/room/timeline/FragmentIdComparer.js

@@ -106,7 +106,7 @@ class Island {
 /*
 index for fast lookup of how two fragments can be sorted
 */
-export default class FragmentIdIndex {
+export default class FragmentIdComparer {
     constructor(fragments) {
         this.rebuild(fragments);
     }

src/matrix/room/timeline/persistence/SyncPersister.js

@@ -1,40 +1,13 @@
-import EventKey from "./timeline/EventKey.js";
+import EventKey from "../EventKey.js";
-import FragmentIdIndex from "./timeline/FragmentIdIndex.js";
+import EventEntry from "../entries/EventEntry.js";
-import EventEntry from "./timeline/entries/EventEntry.js";
+import FragmentBoundaryEntry from "../entries/FragmentBoundaryEntry.js";
-import FragmentBoundaryEntry from "./timeline/entries/FragmentBoundaryEntry.js";
+import {createEventEntry} from "./common.js";
 
-function gapEntriesAreEqual(a, b) {
+export default class SyncPersister {
-    if (!a || !b || !a.gap || !b.gap) {
+    constructor({roomId, storage, fragmentIdComparer}) {
-        return false;
-    }
-    const gapA = a.gap, gapB = b.gap;
-    return gapA.prev_batch === gapB.prev_batch && gapA.next_batch === gapB.next_batch;
-}
-
-function replaceGapEntries(roomTimeline, newEntries, gapKey, neighbourEventKey, backwards) {
-    let replacedRange;
-    if (neighbourEventKey) {
-        replacedRange = backwards ?
-            roomTimeline.boundRange(neighbourEventKey, gapKey, false, true) :
-            roomTimeline.boundRange(gapKey, neighbourEventKey, true, false);
-    } else {
-        replacedRange = roomTimeline.onlyRange(gapKey);
-    }
-
-    const removedEntries = roomTimeline.getAndRemoveRange(this._roomId, replacedRange);
-    for (let entry of newEntries) {
-        roomTimeline.add(entry);
-    }
-
-    return removedEntries;
-}
-
-export default class RoomPersister {
-    constructor({roomId, storage}) {
         this._roomId = roomId;
         this._storage = storage;
-        this._lastLiveKey = null;
+        this._fragmentIdComparer = fragmentIdComparer;
-        this._fragmentIdIndex = new FragmentIdIndex([]);   //only used when timeline is loaded ... e.g. "certain" methods on this class... split up?
     }
 
     async load(txn) {
@@ -52,77 +25,6 @@ export default class RoomPersister {
         console.log("room persister load", this._roomId, this._lastLiveKey && this._lastLiveKey.toString());
     }
 
-    async persistGapFill(gapEntry, response) {
-        const backwards = !!gapEntry.prev_batch;
-        const {chunk, start, end} = response;
-        if (!Array.isArray(chunk)) {
-            throw new Error("Invalid chunk in response");
-        }
-        if (typeof end !== "string") {
-            throw new Error("Invalid end token in response");
-        }
-        if ((backwards && start !== gapEntry.prev_batch) || (!backwards && start !== gapEntry.next_batch)) {
-            throw new Error("start is not equal to prev_batch or next_batch");
-        }
-        
-        const gapKey = gapEntry.sortKey;
-        const txn = await this._storage.readWriteTxn([this._storage.storeNames.roomTimeline]);
-        let result;
-        try {
-            const roomTimeline = txn.roomTimeline;
-            // make sure what we've been given is actually persisted
-            // in the timeline, otherwise we're replacing something
-            // that doesn't exist (maybe it has been replaced already, or ...)
-            const persistedEntry = await roomTimeline.get(this._roomId, gapKey);
-            if (!gapEntriesAreEqual(gapEntry, persistedEntry)) {
-                throw new Error("Gap is not present in the timeline");
-            }
-            // find the previous event before the gap we could merge with
-            const neighbourEventEntry = await (backwards ?
-                roomTimeline.previousEvent(this._roomId, gapKey) :
-                roomTimeline.nextEvent(this._roomId, gapKey));
-            const neighbourEventId = neighbourEventEntry ? neighbourEventEntry.event.event_id : undefined;
-            const {newEntries, eventFound} = this._createNewGapEntries(chunk, end, gapKey, neighbourEventId, backwards);
-            const neighbourEventKey = eventFound ? neighbourEventEntry.sortKey : undefined;
-            const replacedEntries = replaceGapEntries(roomTimeline, newEntries, gapKey, neighbourEventKey, backwards);
-            result = {newEntries, replacedEntries};
-        } catch (err) {
-            txn.abort();
-            throw err;
-        }
-
-        await txn.complete();
-
-        return result;
-    }
-
-    _createNewGapEntries(chunk, nextPaginationToken, gapKey, neighbourEventId, backwards) {
-        if (backwards) {
-            // if backwards, the last events are the ones closest to the gap,
-            // and need to be assigned a key derived from the gap first,
-            // so swap order to only need one loop for both directions
-            chunk.reverse();
-        }
-        let sortKey = gapKey;
-        const {newEntries, eventFound} = chunk.reduce((acc, event) => {
-            acc.eventFound = acc.eventFound || event.event_id === neighbourEventId;
-            if (!acc.eventFound) {
-                acc.newEntries.push(this._createEventEntry(sortKey, event));
-                sortKey = backwards ? sortKey.previousKey() : sortKey.nextKey();
-            }
-        }, {newEntries: [], eventFound: false});
-
-        if (!eventFound) {
-            // as we're replacing an existing gap, no need to increment the gap index
-            newEntries.push(this._createGapEntry(sortKey, nextPaginationToken, backwards));
-        }
-        if (backwards) {
-            // swap resulting array order again if needed
-            newEntries.reverse();
-        }
-        return {newEntries, eventFound};
-    }
-
     async _createLiveFragment(txn, previousToken) {
         const liveFragment = await txn.roomFragments.liveFragment(this._roomId);
         if (!liveFragment) {
@@ -160,7 +62,7 @@ export default class RoomPersister {
             nextToken: null
         };
         txn.roomFragments.add(newFragment);
-        return newFragment;
+        return {oldFragment, newFragment};
     }
 
     async persistSync(roomResponse, txn) {
@@ -172,23 +74,23 @@ export default class RoomPersister {
             // (but don't fail if it isn't, we won't be able to back-paginate though)
             let liveFragment = await this._createLiveFragment(txn, timeline.prev_batch);
             this._lastLiveKey = new EventKey(liveFragment.id, EventKey.defaultLiveKey.eventIndex);
-            entries.push(FragmentBoundaryEntry.start(liveFragment, this._fragmentIdIndex));
+            entries.push(FragmentBoundaryEntry.start(liveFragment, this._fragmentIdComparer));
         } else if (timeline.limited) {
             // replace live fragment for limited sync, *only* if we had a live fragment already
             const oldFragmentId = this._lastLiveKey.fragmentId;
             this._lastLiveKey = this._lastLiveKey.nextFragmentKey();
-            const [oldFragment, newFragment] = this._replaceLiveFragment(oldFragmentId, this._lastLiveKey.fragmentId, timeline.prev_batch, txn);
+            const {oldFragment, newFragment} = this._replaceLiveFragment(oldFragmentId, this._lastLiveKey.fragmentId, timeline.prev_batch, txn);
-            entries.push(FragmentBoundaryEntry.end(oldFragment, this._fragmentIdIndex));
+            entries.push(FragmentBoundaryEntry.end(oldFragment, this._fragmentIdComparer));
-            entries.push(FragmentBoundaryEntry.start(newFragment, this._fragmentIdIndex));
+            entries.push(FragmentBoundaryEntry.start(newFragment, this._fragmentIdComparer));
         }
         let currentKey = this._lastLiveKey;
         const timeline = roomResponse.timeline;
         if (timeline.events) {
             for(const event of timeline.events) {
                 currentKey = currentKey.nextKey();
-                const entry = this._createEventEntry(currentKey, event);
+                const entry = createEventEntry(currentKey, event);
                 txn.roomTimeline.insert(entry);
-                entries.push(new EventEntry(entry, this._fragmentIdIndex));
+                entries.push(new EventEntry(entry, this._fragmentIdComparer));
             }
         }
         // right thing to do? if the txn fails, not sure we'll continue anyways ...
@@ -217,14 +119,6 @@ export default class RoomPersister {
 
         return entries;
     }
-
-    _createEventEntry(key, event) {
-        return {
-            fragmentId: key.fragmentId,
-            eventIndex: key.eventIndex,
-            event: event,
-        };
-    }
 }
 
 //#ifdef TESTS

src/matrix/room/timeline/persistence/common.js

@@ -0,0 +1,7 @@
+export function createEventEntry(key, event) {
+    return {
+        fragmentId: key.fragmentId,
+        eventIndex: key.eventIndex,
+        event: event,
+    };
+}

src/matrix/storage/idb/storage.js

@@ -1,6 +1,5 @@
 import Transaction from "./transaction.js";
-
+import { STORE_NAMES } from "../common.js";
-export const STORE_NAMES = ["session", "roomState", "roomSummary", "roomTimeline"];
 
 export default class Storage {
 	constructor(idbDatabase) {

src/matrix/storage/idb/stores/TimelineEventStore.js

@@ -1,4 +1,4 @@
-import SortKey from "../../../room/timeline/SortKey.js";
+import EventKey from "../../../room/timeline/EventKey.js";
 
 class Range {
     constructor(only, lower, upper, lowerOpen, upperOpen) {
@@ -12,14 +12,14 @@ class Range {
     asIDBKeyRange(roomId) {
         // only
         if (this._only) {
-            return IDBKeyRange.only([roomId, this._only.buffer]);
+            return IDBKeyRange.only([roomId, this._only.fragmentId, this._only.eventIndex]);
         }
         // lowerBound
         // also bound as we don't want to move into another roomId
         if (this._lower && !this._upper) {
             return IDBKeyRange.bound(
-                [roomId, this._lower.buffer],
+                [roomId, this._lower.fragmentId, this._lower.eventIndex],
-                [roomId, SortKey.maxKey.buffer],
+                [roomId, EventKey.maxKey.fragmentId, EventKey.maxKey.eventIndex],
                 this._lowerOpen,
                 false
             );
@@ -28,8 +28,8 @@ class Range {
         // also bound as we don't want to move into another roomId
         if (!this._lower && this._upper) {
             return IDBKeyRange.bound(
-                [roomId, SortKey.minKey.buffer],
+                [roomId, EventKey.minKey.fragmentId, EventKey.minKey.eventIndex],
-                [roomId, this._upper.buffer],
+                [roomId, this._upper.fragmentId, this._upper.eventIndex],
                 false,
                 this._upperOpen
             );
@@ -37,8 +37,8 @@ class Range {
         // bound
         if (this._lower && this._upper) {
             return IDBKeyRange.bound(
-                [roomId, this._lower.buffer],
+                [roomId, this._lower.fragmentId, this._lower.eventIndex],
-                [roomId, this._upper.buffer],
+                [roomId, this._upper.fragmentId, this._upper.eventIndex],
                 this._lowerOpen,
                 this._upperOpen
             );
@@ -57,44 +57,44 @@ class Range {
  *
  * @typedef   {Object} Entry
  * @property  {string} roomId
- * @property  {SortKey} sortKey
+ * @property  {EventKey} eventKey
  * @property  {?Event} event if an event entry, the event
  * @property  {?Gap} gap if a gap entry, the gap
 */
-export default class RoomTimelineStore {
+export default class TimelineEventStore {
 	constructor(timelineStore) {
 		this._timelineStore = timelineStore;
 	}
 
     /** Creates a range that only includes the given key
-     *  @param {SortKey} sortKey the key
+     *  @param {EventKey} eventKey the key
      *  @return {Range} the created range
      */
-    onlyRange(sortKey) {
+    onlyRange(eventKey) {
-        return new Range(sortKey);
+        return new Range(eventKey);
     }
 
-    /** Creates a range that includes all keys before sortKey, and optionally also the key itself.
+    /** Creates a range that includes all keys before eventKey, and optionally also the key itself.
-     *  @param {SortKey} sortKey the key
+     *  @param {EventKey} eventKey the key
      *  @param {boolean} [open=false] whether the key is included (false) or excluded (true) from the range at the upper end.
      *  @return {Range} the created range
      */
-    upperBoundRange(sortKey, open=false) {
+    upperBoundRange(eventKey, open=false) {
-        return new Range(undefined, undefined, sortKey, undefined, open);
+        return new Range(undefined, undefined, eventKey, undefined, open);
     }
 
-    /** Creates a range that includes all keys after sortKey, and optionally also the key itself.
+    /** Creates a range that includes all keys after eventKey, and optionally also the key itself.
-     *  @param {SortKey} sortKey the key
+     *  @param {EventKey} eventKey the key
      *  @param {boolean} [open=false] whether the key is included (false) or excluded (true) from the range at the lower end.
      *  @return {Range} the created range
      */
-    lowerBoundRange(sortKey, open=false) {
+    lowerBoundRange(eventKey, open=false) {
-        return new Range(undefined, sortKey, undefined, open);
+        return new Range(undefined, eventKey, undefined, open);
     }
 
     /** Creates a range that includes all keys between `lower` and `upper`, and optionally the given keys as well.
-     *  @param {SortKey} lower the lower key
+     *  @param {EventKey} lower the lower key
-     *  @param {SortKey} upper the upper key
+     *  @param {EventKey} upper the upper key
      *  @param {boolean} [lowerOpen=false] whether the lower key is included (false) or excluded (true) from the range.
      *  @param {boolean} [upperOpen=false] whether the upper key is included (false) or excluded (true) from the range.
      *  @return {Range} the created range
@@ -110,9 +110,9 @@ export default class RoomTimelineStore {
      *  @return {Promise<Entry[]>} a promise resolving to an array with 0 or more entries, in ascending order.
      */
 	async lastEvents(roomId, fragmentId, amount) {
-        const sortKey = SortKey.maxKey;
+        const eventKey = EventKey.maxKey;
-        sortKey.fragmentId = fragmentId;
+        eventKey.fragmentId = fragmentId;
-		return this.eventsBefore(roomId, sortKey, amount);
+		return this.eventsBefore(roomId, eventKey, amount);
 	}
 
     /** Looks up the first `amount` entries in the timeline for `roomId`.
@@ -122,32 +122,32 @@ export default class RoomTimelineStore {
      *  @return {Promise<Entry[]>} a promise resolving to an array with 0 or more entries, in ascending order.
      */
 	async firstEvents(roomId, fragmentId, amount) {
-        const sortKey = SortKey.minKey;
+        const eventKey = EventKey.minKey;
-        sortKey.fragmentId = fragmentId;
+        eventKey.fragmentId = fragmentId;
-		return this.eventsAfter(roomId, sortKey, amount);
+		return this.eventsAfter(roomId, eventKey, amount);
 	}
 
-    /** Looks up `amount` entries after `sortKey` in the timeline for `roomId` within the same fragment.
+    /** Looks up `amount` entries after `eventKey` in the timeline for `roomId` within the same fragment.
-     *  The entry for `sortKey` is not included.
+     *  The entry for `eventKey` is not included.
      *  @param  {string} roomId
-     *  @param  {SortKey} sortKey
+     *  @param  {EventKey} eventKey
      *  @param  {number} amount
      *  @return {Promise<Entry[]>} a promise resolving to an array with 0 or more entries, in ascending order.
      */
-	eventsAfter(roomId, sortKey, amount) {
+	eventsAfter(roomId, eventKey, amount) {
-        const idbRange = this.lowerBoundRange(sortKey, true).asIDBKeyRange(roomId);
+        const idbRange = this.lowerBoundRange(eventKey, true).asIDBKeyRange(roomId);
 		return this._timelineStore.selectLimit(idbRange, amount);
 	}
 
-    /** Looks up `amount` entries before `sortKey` in the timeline for `roomId` within the same fragment.
+    /** Looks up `amount` entries before `eventKey` in the timeline for `roomId` within the same fragment.
-     *  The entry for `sortKey` is not included.
+     *  The entry for `eventKey` is not included.
      *  @param  {string} roomId
-     *  @param  {SortKey} sortKey
+     *  @param  {EventKey} eventKey
      *  @param  {number} amount
      *  @return {Promise<Entry[]>} a promise resolving to an array with 0 or more entries, in ascending order.
      */
-    async eventsBefore(roomId, sortKey, amount) {
+    async eventsBefore(roomId, eventKey, amount) {
-        const range = this.upperBoundRange(sortKey, true).asIDBKeyRange(roomId);
+        const range = this.upperBoundRange(eventKey, true).asIDBKeyRange(roomId);
         const events = await this._timelineStore.selectLimitReverse(range, amount);
         events.reverse(); // because we fetched them backwards
         return events;
@@ -196,7 +196,7 @@ export default class RoomTimelineStore {
         return firstFoundEventId;
     }
 
-    /** Inserts a new entry into the store. The combination of roomId and sortKey should not exist yet, or an error is thrown.
+    /** Inserts a new entry into the store. The combination of roomId and eventKey should not exist yet, or an error is thrown.
      *  @param  {Entry} entry the entry to insert
      *  @return {Promise<>} a promise resolving to undefined if the operation was successful, or a StorageError if not.
      *  @throws {StorageError} ...
@@ -206,7 +206,7 @@ export default class RoomTimelineStore {
         return this._timelineStore.add(entry);
     }
 
-    /** Updates the entry into the store with the given [roomId, sortKey] combination.
+    /** Updates the entry into the store with the given [roomId, eventKey] combination.
      *  If not yet present, will insert. Might be slower than add.
      *  @param  {Entry} entry the entry to update.
      *  @return {Promise<>} a promise resolving to undefined if the operation was successful, or a StorageError if not.
@@ -215,12 +215,16 @@ export default class RoomTimelineStore {
         return this._timelineStore.put(entry);
     }
 
-    get(roomId, sortKey) {
+    get(roomId, eventKey) {
-        return this._timelineStore.get([roomId, sortKey]);
+        return this._timelineStore.get([roomId, eventKey.fragmentId, eventKey.eventIndex]);
     }
     // returns the entries as well!! (or not always needed? I guess not always needed, so extra method)
     removeRange(roomId, range) {
         // TODO: read the entries!
         return this._timelineStore.delete(range.asIDBKeyRange(roomId));
     }
+
+    getByEventId(roomId, eventId) {
+        return this._timelineStore.index("byEventId").get([roomId, eventId]);
+    }
 }

src/matrix/storage/idb/transaction.js

@@ -2,8 +2,9 @@ import {txnAsPromise} from "./utils.js";
 import Store from "./store.js";
 import SessionStore from "./stores/SessionStore.js";
 import RoomSummaryStore from "./stores/RoomSummaryStore.js";
-import RoomTimelineStore from "./stores/RoomTimelineStore.js";
+import TimelineEventStore from "./stores/TimelineEventStore.js";
 import RoomStateStore from "./stores/RoomStateStore.js";
+import TimelineFragmentStore from "./stores/TimelineFragmentStore.js";
 
 export default class Transaction {
 	constructor(txn, allowedStoreNames) {
@@ -41,8 +42,12 @@ export default class Transaction {
 		return this._store("roomSummary", idbStore => new RoomSummaryStore(idbStore));
 	}
 
-	get roomTimeline() {
+    get timelineFragments() {
-		return this._store("roomTimeline", idbStore => new RoomTimelineStore(idbStore));
+        return this._store("timelineFragments", idbStore => new TimelineFragmentStore(idbStore));
+    }
+
+	get timelineEvents() {
+		return this._store("timelineEvents", idbStore => new TimelineEventStore(idbStore));
 	}
 
 	get roomState() {