WIP for fragment support

This commit is contained in:
Bruno Windels 2019-05-19 20:49:46 +02:00
parent 3324fd3afd
commit 784588440c
11 changed files with 159 additions and 25 deletions

View File

@ -5,21 +5,31 @@
- SortKey
- FragmentId
- EventIndex
- write fragmentStore
- DONE: write fragmentStore
- load all fragments
- add a fragment (live on limited sync, or /context)
- connect two fragments
- update token on fragment (when filling gap or connecting two fragments)
fragments can need connecting when filling a gap or creating a new /context fragment
- adapt timelineStore
- DONE: adapt timelineStore
how will fragments be exposed in timeline store?
- all read operations are passed a fragment id
- adapt persister
- DONE: persist fragments in /sync
- load n items before and after key
- fill gaps / fragment filling
- DONE: fill gaps / fragment filling
- load n items before and after key,
- need to add fragments as we come across boundaries
- also cache fragments? not for now ...
- not doing any of the above, just reloading and rebuilding for now
- adapt Timeline
- turn ObservableArray into ObservableSortedArray
- upsert already sorted sections
- upsert single entry
- adapt TilesCollection & Tile to entry changes
- add live fragment id optimization if we haven't done so already
- lets try to not have to have the fragmentindex in memory if the timeline isn't loaded
- could do this by only loading all fragments into index when filling gaps, backpaginating, ... and on persister load only load the last fragment. This wouldn't even need a FragmentIndex?

View File

@ -22,6 +22,7 @@ export default class TilesCollection extends BaseObservableList {
for (let entry of this._entries) {
if (!currentTile || !currentTile.tryIncludeEntry(entry)) {
currentTile = this._tileCreator(entry);
// if (currentTile) here?
this._tiles.push(currentTile);
}
}

View File

@ -163,7 +163,7 @@ export default class FragmentIdComparer {
export function tests() {
return {
test_1_island_3_fragments(assert) {
const index = new FragmentIdIndex([
const index = new FragmentIdComparer([
{id: 3, previousId: 2},
{id: 1, nextId: 2},
{id: 2, nextId: 3, previousId: 1},
@ -180,7 +180,7 @@ export function tests() {
assert.equal(index.compare(1, 1), 0);
},
test_2_island_dont_compare(assert) {
const index = new FragmentIdIndex([
const index = new FragmentIdComparer([
{id: 1},
{id: 2},
]);
@ -188,7 +188,7 @@ export function tests() {
assert.throws(() => index.compare(2, 1));
},
test_2_island_compare_internally(assert) {
const index = new FragmentIdIndex([
const index = new FragmentIdComparer([
{id: 1, nextId: 2},
{id: 2, previousId: 1},
{id: 11, nextId: 12},
@ -203,12 +203,12 @@ export function tests() {
assert.throws(() => index.compare(12, 2));
},
test_unknown_id(assert) {
const index = new FragmentIdIndex([{id: 1}]);
const index = new FragmentIdComparer([{id: 1}]);
assert.throws(() => index.compare(1, 2));
assert.throws(() => index.compare(2, 1));
},
test_rebuild_flushes_old_state(assert) {
const index = new FragmentIdIndex([
const index = new FragmentIdComparer([
{id: 1, nextId: 2},
{id: 2, previousId: 1},
]);

View File

@ -1,6 +1,7 @@
import { ObservableArray } from "../../../observable/index.js";
import sortedIndex from "../../../utils/sortedIndex.js";
import GapPersister from "./persistence/GapPersister.js";
import TimelineReader from "./persistence/TimelineReader.js";
export default class Timeline {
constructor({roomId, storage, closeCallback, fragmentIdComparer}) {
@ -9,6 +10,11 @@ export default class Timeline {
this._closeCallback = closeCallback;
this._entriesList = new ObservableArray();
this._fragmentIdComparer = fragmentIdComparer;
this._timelineReader = new TimelineReader({
roomId: this._roomId,
storage: this._storage,
fragmentIdComparer: this._fragmentIdComparer
});
}
/** @package */
@ -53,6 +59,7 @@ export default class Timeline {
}
async loadAtTop(amount) {
// TODO: use TimelineReader::readFrom here, and insert returned array at location for first and last entry.
const firstEntry = this._entriesList.at(0);
if (firstEntry) {
const txn = await this._storage.readTxn([this._storage.storeNames.timelineEvents]);

View File

@ -1,14 +1,6 @@
import EventKey from "../EventKey.js";
import EventEntry from "../entries/EventEntry.js";
import {createEventEntry} from "./common.js";
function directionalAppend(array, value, direction) {
if (direction.isForward) {
array.push(value);
} else {
array.splice(0, 0, value);
}
}
import {createEventEntry, directionalAppend} from "./common.js";
export default class GapPersister {
constructor({roomId, storage, fragmentIdComparer}) {

View File

@ -11,7 +11,7 @@ export default class SyncPersister {
}
async load(txn) {
const liveFragment = await txn.roomFragments.liveFragment(this._roomId);
const liveFragment = await txn.timelineFragments.liveFragment(this._roomId);
if (liveFragment) {
const [lastEvent] = await txn.roomTimeline.lastEvents(this._roomId, liveFragment.id, 1);
// sorting and identifying (e.g. sort key and pk to insert) are a bit intertwined here
@ -26,7 +26,7 @@ export default class SyncPersister {
}
async _createLiveFragment(txn, previousToken) {
const liveFragment = await txn.roomFragments.liveFragment(this._roomId);
const liveFragment = await txn.timelineFragments.liveFragment(this._roomId);
if (!liveFragment) {
if (!previousToken) {
previousToken = null;
@ -39,7 +39,7 @@ export default class SyncPersister {
previousToken: previousToken,
nextToken: null
};
txn.roomFragments.add(fragment);
txn.timelineFragments.add(fragment);
return fragment;
} else {
return liveFragment;
@ -47,12 +47,12 @@ export default class SyncPersister {
}
async _replaceLiveFragment(oldFragmentId, newFragmentId, previousToken, txn) {
const oldFragment = await txn.roomFragments.get(oldFragmentId);
const oldFragment = await txn.timelineFragments.get(oldFragmentId);
if (!oldFragment) {
throw new Error(`old live fragment doesn't exist: ${oldFragmentId}`);
}
oldFragment.nextId = newFragmentId;
txn.roomFragments.update(oldFragment);
txn.timelineFragments.update(oldFragment);
const newFragment = {
roomId: this._roomId,
id: newFragmentId,
@ -61,7 +61,7 @@ export default class SyncPersister {
previousToken: previousToken,
nextToken: null
};
txn.roomFragments.add(newFragment);
txn.timelineFragments.add(newFragment);
return {oldFragment, newFragment};
}
@ -117,6 +117,11 @@ export default class SyncPersister {
}
}
if (timeline.limited) {
const fragments = await txn.timelineFragments.all(this._roomId);
this._fragmentIdComparer.rebuild(fragments);
}
return entries;
}
}

View File

@ -0,0 +1,63 @@
import {directionalConcat, directionalAppend} from "./common.js";
import EventKey from "../EventKey.js";
import EventEntry from "../entries/EventEntry.js";
import FragmentBoundaryEntry from "../entries/FragmentBoundaryEntry.js";
export default class TimelineReader {
constructor({roomId, storage, fragmentIdComparer}) {
this._roomId = roomId;
this._storage = storage;
this._fragmentIdComparer = fragmentIdComparer;
}
async readFrom(eventKey, direction, amount) {
const txn = this._storage.readTxn([
this._storage.storeNames.timelineEvents,
this._storage.storeNames.timelineFragments,
]);
let entries = [];
let loadedFragment = false;
const timelineStore = txn.timelineEvents;
const fragmentStore = txn.timelineFragments;
while (entries.length < amount && eventKey) {
let eventsWithinFragment;
if (direction.isForward) {
eventsWithinFragment = timelineStore.eventsAfter(eventKey, amount);
} else {
eventsWithinFragment = timelineStore.eventsBefore(eventKey, amount);
}
const eventEntries = eventsWithinFragment.map(e => new EventEntry(e, this._fragmentIdComparer));
entries = directionalConcat(entries, eventEntries, direction);
// prepend or append eventsWithinFragment to entries, and wrap them in EventEntry
if (entries.length < amount) {
const fragment = await fragmentStore.get(this._roomId, eventKey.fragmentId);
// this._fragmentIdComparer.addFragment(fragment);
let fragmentEntry = new FragmentBoundaryEntry(fragment, direction.isBackward, this._fragmentIdComparer);
// append or prepend fragmentEntry, reuse func from GapPersister?
directionalAppend(entries, fragmentEntry, direction);
// don't count it in amount perhaps? or do?
if (fragmentEntry.linkedFragmentId) {
const nextFragment = await fragmentStore.get(this._roomId, fragmentEntry.linkedFragmentId);
// this._fragmentIdComparer.addFragment(nextFragment);
const nextFragmentEntry = new FragmentBoundaryEntry(nextFragment, direction.isForward, this._fragmentIdComparer);
directionalAppend(entries, nextFragmentEntry, direction);
eventKey = new EventKey(nextFragmentEntry.fragmentId, nextFragmentEntry.eventIndex);
loadedFragment = true;
} else {
eventKey = null;
}
}
}
// reload fragments
if (loadedFragment) {
const fragments = await fragmentStore.all(this._roomId);
this._fragmentIdComparer.rebuild(fragments);
}
return entries;
}
}

View File

@ -5,3 +5,19 @@ export function createEventEntry(key, event) {
event: event,
};
}
export function directionalAppend(array, value, direction) {
if (direction.isForward) {
array.push(value);
} else {
array.unshift(value);
}
}
export function directionalConcat(array, otherArray, direction) {
if (direction.isForward) {
return array.concat(otherArray);
} else {
return otherArray.concat(array);
}
}

View File

@ -18,6 +18,7 @@ export default class Session {
this._storage.storeNames.roomSummary,
this._storage.storeNames.roomState,
this._storage.storeNames.timelineEvents,
this._storage.storeNames.timelineFragments,
]);
// restore session object
this._session = await txn.session.get();

View File

@ -74,6 +74,7 @@ export default class Sync extends EventEmitter {
storeNames.session,
storeNames.roomSummary,
storeNames.timelineEvents,
storeNames.timelineFragments,
storeNames.roomState,
]);
const roomChanges = [];

View File

@ -0,0 +1,38 @@
import BaseObservableList from "./BaseObservableList.js";
import sortedIndex from "../../utils/sortedIndex";
export default class SortedArray extends BaseObservableList {
constructor(comparator) {
super();
this._comparator = comparator;
this._items = [];
}
setSortedMany(items) {
}
set(item) {
const idx = sortedIndex(this._items, item, this._comparator);
if (idx < this._items.length || this._comparator(this._items[idx], item) !== 0) {
this._items.splice(idx, 0, item);
//emitAdd
} else {
this._items[idx] = item;
//emitRemove
//emitAdd
}
}
get array() {
return this._items;
}
get length() {
return this._items.length;
}
[Symbol.iterator]() {
return this._items.values();
}
}