diff --git a/src/matrix/room/timeline/persistence/TimelineReader.js b/src/matrix/room/timeline/persistence/TimelineReader.js
index ea40f8a4..928d6b64 100644
--- a/src/matrix/room/timeline/persistence/TimelineReader.js
+++ b/src/matrix/room/timeline/persistence/TimelineReader.js
@@ -60,8 +60,8 @@ export class TimelineReader {
                 let fragmentEntry = new FragmentBoundaryEntry(fragment, direction.isBackward, this._fragmentIdComparer);
                 // append or prepend fragmentEntry, reuse func from GapWriter?
                 directionalAppend(entries, fragmentEntry, direction);
-                // don't count it in amount perhaps? or do?
-                if (fragmentEntry.hasLinkedFragment) {
+                // only continue loading if the fragment boundary can't be backfilled
+                if (!fragmentEntry.token && fragmentEntry.hasLinkedFragment) {
                     const nextFragment = await fragmentStore.get(this._roomId, fragmentEntry.linkedFragmentId);
                     this._fragmentIdComparer.add(nextFragment);
                     const nextFragmentEntry = new FragmentBoundaryEntry(nextFragment, direction.isForward, this._fragmentIdComparer);