adjust persister to fragments (untested)

doc/FRAGMENTS.md

@@ -43,3 +43,19 @@ so we'll need to remove previous/nextEvent on the timeline store and come up wit
 thoughts:
     - ranges in timeline store with fragmentId might not make sense anymore as doing queries over multiple fragment ids doesn't make sense anymore ... still makes sense to have them part of SortKey though ...
     - we need a test for querytarget::lookup, or make sure it works well ...
+
+
+# Reading the timeline with fragments
+
+- what format does the persister return newEntries after persisting sync or a gap fill?
+    - a new fragment can be created during a limited sync
+    - when doing a /context or /messages call, we could have joined with another fragment
+    - don't think we need to describe a result spanning multiple fragments here
+    so:
+
+    in case of limited sync, we just say there was a limited sync, this is the fragment that was created for it so we can show a gap in the timeline
+
+    in case of a gap fill, we need to return what was changed to the fragment (was it joined with another fragment, what's the new token), and which events were actually added.
+
+- where do we translate from fragments to gap entries? and back? in the timeline object?
+    that would make sense, that seems to be the only place we need that translation

src/matrix/room/persister.js

@@ -1,5 +1,5 @@
 import SortKey from "./timeline/SortKey.js";
-import FragmentIndex from "./timeline/FragmentIndex.js";
+import FragmentIdIndex from "./timeline/FragmentIdIndex.js";
 
 function gapEntriesAreEqual(a, b) {
     if (!a || !b || !a.gap || !b.gap) {
@@ -31,9 +31,7 @@ export default class RoomPersister {
 	constructor({roomId, storage}) {
 		this._roomId = roomId;
         this._storage = storage;
-        // TODO: load fragmentIndex?
+		this._lastLiveKey = null;
-		this._lastSortKey = new SortKey();
-		this._lastSortKey = null;
         this._fragmentIdIndex = new FragmentIdIndex([]);   //only used when timeline is loaded ... e.g. "certain" methods on this class... split up?
 	}
 
@@ -41,13 +39,18 @@ export default class RoomPersister {
         const liveFragment = await txn.roomFragments.liveFragment(this._roomId);
         if (liveFragment) {
             const [lastEvent] = await txn.roomTimeline.lastEvents(this._roomId, liveFragment.id, 1);
-            // last event needs to come from the fragment (e.g. passing the last fragment id)
+            // sorting and identifying (e.g. sort key and pk to insert) are a bit intertwined here
-            const lastSortKey = new SortKey(this._fragmentIdIndex);
+            // we could split it up into a SortKey (only with compare) and
-            lastSortKey.fragmentId = liveFragment.id;
+            // a EventKey (no compare or fragment index) with nextkey methods and getters/setters for eventIndex/fragmentId
-            lastSortKey.eventIndex = lastEvent.eventIndex;
+            // we probably need to convert from one to the other though, so bother?
-            this._lastSortKey = lastSortKey;
+            const lastLiveKey = new SortKey(this._fragmentIdIndex);
+            lastLiveKey.fragmentId = liveFragment.id;
+            lastLiveKey.eventIndex = lastEvent.eventIndex;
+            this._lastLiveKey = lastLiveKey;
         }
-        console.log("room persister load", this._roomId, this._lastSortKey && this._lastSortKey.toString());
+        // if there is no live fragment, we don't create it here because load gets a readonly txn.
+        // this is on purpose, load shouldn't modify the store
+        console.log("room persister load", this._roomId, this._lastLiveKey && this._lastLiveKey.toString());
 	}
 
     async persistGapFill(gapEntry, response) {
@@ -121,16 +124,63 @@ export default class RoomPersister {
         return {newEntries, eventFound};
     }
 
-	persistSync(roomResponse, txn) {
+    async _getLiveFragment(txn, previousToken) {
-		let nextKey = this._lastSortKey;
+        const liveFragment = await txn.roomFragments.liveFragment(this._roomId);
+        if (!liveFragment) {
+            if (!previousToken) {
+                previousToken = null;
+            }
+            let defaultId = SortKey.firstLiveFragmentId;
+            txn.roomFragments.add({
+                roomId: this._roomId,
+                id: defaultId,
+                previousId: null,
+                nextId: null,
+                previousToken: previousToken,
+                nextToken: null
+            });
+            return defaultId;
+        } else {
+            return liveFragment.id;
+        }
+    }
+
+    async _replaceLiveFragment(oldFragmentId, newFragmentId, previousToken, txn) {
+        const oldFragment = await txn.roomFragments.get(oldFragmentId);
+        if (!oldFragment) {
+            throw new Error(`old live fragment doesn't exist: ${oldFragmentId}`);
+        }
+        oldFragment.nextId = newFragmentId;
+        txn.roomFragments.update(oldFragment);
+        txn.roomFragments.add({
+            roomId: this._roomId,
+            id: newFragmentId,
+            previousId: oldFragmentId,
+            nextId: null,
+            previousToken: previousToken,
+            nextToken: null
+        });
+    }
+
+	async persistSync(roomResponse, txn) {
+        // means we haven't synced this room yet (just joined or did initial sync)
+        if (!this._lastLiveKey) {
+            // as this is probably a limited sync, prev_batch should be there
+            // (but don't fail if it isn't, we won't be able to back-paginate though)
+            const fragmentId = await this._getLiveFragment(txn, timeline.prev_batch);
+            this._lastLiveKey = new SortKey(this._fragmentIdIndex);
+            this._lastLiveKey.fragmentId = fragmentId;
+            this._lastLiveKey.eventIndex = SortKey.firstLiveEventIndex;
+        }
+        // replace live fragment for limited sync, *only* if we had a live fragment already
+        else if (timeline.limited) {
+            const oldFragmentId = this._lastLiveKey.fragmentId;
+			this._lastLiveKey = this._lastLiveKey.nextFragmentKey();
+            this._replaceLiveFragment(oldFragmentId, this._lastLiveKey.fragmentId, timeline.prev_batch, txn);
+        }
+		let nextKey = this._lastLiveKey;
 		const timeline = roomResponse.timeline;
         const entries = [];
-		// is limited true for initial sync???? or do we need to handle that as a special case?
-		// I suppose it will, yes
-		if (timeline.limited) {
-			nextKey = nextKey.nextKeyWithGap();
-            entries.push(this._createBackwardGapEntry(nextKey, timeline.prev_batch));
-        }
         if (timeline.events) {
             for(const event of timeline.events) {
                 nextKey = nextKey.nextKey();
@@ -146,7 +196,7 @@ export default class RoomPersister {
 		// succeeded 
 		txn.complete().then(() => {
 			console.log("txn complete, setting key");
-			this._lastSortKey = nextKey;
+			this._lastLiveKey = nextKey;
 		});
 
 		// persist state
@@ -156,7 +206,7 @@ export default class RoomPersister {
 				txn.roomState.setStateEvent(this._roomId, event)
 			}
 		}
-
+        // persist live state events in timeline
 		if (timeline.events) {
 			for (const event of timeline.events) {
 				if (typeof event.state_key === "string") {
@@ -164,33 +214,15 @@ export default class RoomPersister {
 				}
 			}
 		}
+
         return entries;
 	}
 
-    _createBackwardGapEntry(sortKey, prevBatch) {
+    _createEventEntry(key, event) {
         return {
-            roomId: this._roomId,
+            fragmentId: key.fragmentId,
-            sortKey: sortKey.buffer,
+            eventIndex: key.eventIndex,
-            event: null,
-            gap: {prev_batch: prevBatch}
-        };
-    }
-
-    _createForwardGapEntry(sortKey, nextBatch) {
-        return {
-            roomId: this._roomId,
-            sortKey: sortKey.buffer,
-            event: null,
-            gap: {next_batch: nextBatch}
-        };
-    }
-
-    _createEventEntry(sortKey, event) {
-        return {
-            roomId: this._roomId,
-            sortKey: sortKey.buffer,
             event: event,
-            gap: null
         };
     }
 }

src/matrix/room/timeline/SortKey.js

@@ -86,6 +86,14 @@ export default class SortKey {
         return minKey;
     }
 
+    static get firstLiveFragmentId() {
+        return MID;
+    }
+
+    static get firstLiveEventIndex() {
+        return MID;
+    }
+
     compare(otherKey) {
         const fragmentDiff = this.fragmentId - otherKey.fragmentId;
         if (fragmentDiff === 0) {

src/matrix/storage/idb/create.js

@@ -2,33 +2,34 @@ import Storage from "./storage.js";
 import { openDatabase } from "./utils.js";
 
 export default async function createIdbStorage(databaseName) {
-	const db = await openDatabase(databaseName, createStores, 1);
+    const db = await openDatabase(databaseName, createStores, 1);
-	return new Storage(db);
+    return new Storage(db);
 }
 
 function createStores(db) {
-	db.createObjectStore("session", {keyPath: "key"});
+    db.createObjectStore("session", {keyPath: "key"});
-	// any way to make keys unique here? (just use put?)
+    // any way to make keys unique here? (just use put?)
-	db.createObjectStore("roomSummary", {keyPath: "roomId"});
+    db.createObjectStore("roomSummary", {keyPath: "roomId"});
 
+    // need index to find live fragment? prooobably ok without for now
     db.createObjectStore("timelineFragments", {keyPath: ["roomId", "id"]});
-	// needs roomId separate because it might hold a gap and no event
+    // needs roomId separate because it might hold a gap and no event
-	const timelineEvents = db.createObjectStore("timelineEvents", {keyPath: ["event.room_id", "fragmentId", "sortKey"]});
+    const timelineEvents = db.createObjectStore("timelineEvents", {keyPath: ["event.room_id", "fragmentId", "eventIndex"]});
-	timelineEvents.createIndex("byEventId", [
+    timelineEvents.createIndex("byEventId", [
-		"event.room_id",
+        "event.room_id",
-		"event.event_id"
+        "event.event_id"
-	], {unique: true});
+    ], {unique: true});
 
-	db.createObjectStore("roomState", {keyPath: [
+    db.createObjectStore("roomState", {keyPath: [
-		"roomId",
+        "roomId",
-		"event.type",
+        "event.type",
-		"event.state_key"
+        "event.state_key"
-	]});
+    ]});
-	
+    
-	// const roomMembers = db.createObjectStore("roomMembers", {keyPath: [
+    // const roomMembers = db.createObjectStore("roomMembers", {keyPath: [
-	// 	"event.room_id",
+    //  "event.room_id",
-	// 	"event.content.membership",
+    //  "event.content.membership",
-	// 	"event.state_key"
+    //  "event.state_key"
-	// ]});
+    // ]});
-	// roomMembers.createIndex("byName", ["room_id", "content.name"]);
+    // roomMembers.createIndex("byName", ["room_id", "content.name"]);
 }