write redactions during sync

src/matrix/room/BaseRoom.js

@@ -288,6 +288,8 @@ export class BaseRoom extends EventEmitter {
                 this._applyGapFill(extraGapFillChanges);
             }
             if (this._timeline) {
+                // these should not be added if not already there
+                this._timeline.replaceEntries(gapResult.updatedEntries);
                 this._timeline.addOrReplaceEntries(gapResult.entries);
             }
         });

src/matrix/room/Room.js

@@ -106,9 +106,8 @@ export class Room extends BaseRoom {
             txn.roomState.removeAllForRoom(this.id);
             txn.roomMembers.removeAllForRoom(this.id);
         }
-        const {entries: newEntries, newLiveKey, memberChanges} =
+        const {entries: newEntries, updatedEntries, newLiveKey, memberChanges} =
             await log.wrap("syncWriter", log => this._syncWriter.writeSync(roomResponse, isRejoin, txn, log), log.level.Detail);
-        let allEntries = newEntries;
         if (decryptChanges) {
             const decryption = await log.wrap("decryptChanges", log => decryptChanges.write(txn, log));
             log.set("decryptionResults", decryption.results.size);
@@ -119,16 +118,18 @@ export class Room extends BaseRoom {
             decryption.applyToEntries(newEntries);
             if (retryEntries?.length) {
                 decryption.applyToEntries(retryEntries);
-                allEntries = retryEntries.concat(allEntries);
+                updatedEntries.push(...retryEntries);
             }
         }
-        log.set("allEntries", allEntries.length);
+        log.set("newEntries", newEntries.length);
+        log.set("updatedEntries", updatedEntries.length);
         let shouldFlushKeyShares = false;
         // pass member changes to device tracker
         if (roomEncryption && this.isTrackingMembers && memberChanges?.size) {
             shouldFlushKeyShares = await roomEncryption.writeMemberChanges(memberChanges, txn, log);
             log.set("shouldFlushKeyShares", shouldFlushKeyShares);
         }
+        const allEntries = newEntries.concat(updatedEntries);
         // also apply (decrypted) timeline entries to the summary changes
         summaryChanges = summaryChanges.applyTimelineEntries(
             allEntries, isInitialSync, !this._isTimelineOpen, this._user.id);
@@ -164,7 +165,7 @@ export class Room extends BaseRoom {
             summaryChanges,
             roomEncryption,
             newEntries,
-            updatedEntries: retryEntries || [],
+            updatedEntries,
             newLiveKey,
             removedPendingEvents,
             memberChanges,

src/matrix/room/common.js

@@ -19,3 +19,5 @@ export function getPrevContentFromStateEvent(event) {
     // see https://matrix.to/#/!NasysSDfxKxZBzJJoE:matrix.org/$DvrAbZJiILkOmOIuRsNoHmh2v7UO5CWp_rYhlGk34fQ?via=matrix.org&via=pixie.town&via=amorgan.xyz
     return event.unsigned?.prev_content || event.prev_content;
 }
+
+export const REDACTION_TYPE = "m.room.redaction";

src/matrix/room/sending/PendingEvent.js

@@ -15,8 +15,8 @@ limitations under the License.
 */
 import {createEnum} from "../../../utils/enum.js";
 import {AbortError} from "../../../utils/error.js";
+import {REDACTION_TYPE} from "../common.js";
 import {isTxnId} from "../../common.js";
-import {REDACTION_TYPE} from "./SendQueue.js";
 
 export const SendStatus = createEnum(
     "Waiting",

src/matrix/room/sending/SendQueue.js

@@ -18,8 +18,7 @@ import {SortedArray} from "../../../observable/list/SortedArray.js";
 import {ConnectionError} from "../../error.js";
 import {PendingEvent} from "./PendingEvent.js";
 import {makeTxnId, isTxnId} from "../../common.js";
-
+import {REDACTION_TYPE} from "../common.js";
-export const REDACTION_TYPE = "m.room.redaction";
 
 export class SendQueue {
     constructor({roomId, storage, hsApi, pendingEvents}) {

src/matrix/room/timeline/entries/EventEntry.js

@@ -108,4 +108,8 @@ export class EventEntry extends BaseEntry {
     get decryptionError() {
         return this._decryptionError;
     }
+
+    get relatedEventId() {
+        return this._eventEntry.event.redacts;
+    }
 }

src/matrix/room/timeline/persistence/GapWriter.js

@@ -14,6 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+import {RelationWriter} from "./RelationWriter.js";
 import {EventKey} from "../EventKey.js";
 import {EventEntry} from "../entries/EventEntry.js";
 import {createEventEntry, directionalAppend} from "./common.js";
@@ -24,6 +25,7 @@ export class GapWriter {
         this._roomId = roomId;
         this._storage = storage;
         this._fragmentIdComparer = fragmentIdComparer;
+        this._relationWriter = new RelationWriter(roomId, fragmentIdComparer);
     }
     // events is in reverse-chronological order (last event comes at index 0) if backwards
     async _findOverlappingEvents(fragmentEntry, events, txn, log) {
@@ -105,6 +107,7 @@ export class GapWriter {
 
     _storeEvents(events, startKey, direction, state, txn) {
         const entries = [];
+        const updatedEntries = [];
         // events is in reverse chronological order for backwards pagination,
         // e.g. order is moving away from the `from` point.
         let key = startKey;
@@ -120,6 +123,10 @@ export class GapWriter {
             txn.timelineEvents.insert(eventStorageEntry);
             const eventEntry = new EventEntry(eventStorageEntry, this._fragmentIdComparer);
             directionalAppend(entries, eventEntry, direction);
+            const updatedRelationTargetEntry = this._relationWriter.writeRelation(eventEntry);
+            if (updatedRelationTargetEntry) {
+                updatedEntries.push(updatedRelationTargetEntry);
+            }
         }
         return entries;
     }
@@ -201,7 +208,6 @@ export class GapWriter {
         // chunk is in reverse-chronological order when backwards
         const {chunk, start, state} = response;
         let {end} = response;
-        let entries;
 
         if (!Array.isArray(chunk)) {
             throw new Error("Invalid chunk in response");
@@ -240,9 +246,9 @@ export class GapWriter {
             end = null;
         }
         // create entries for all events in chunk, add them to entries
-        entries = this._storeEvents(nonOverlappingEvents, lastKey, direction, state, txn);
+        const {entries, updatedEntries} = this._storeEvents(nonOverlappingEvents, lastKey, direction, state, txn);
         const fragments = await this._updateFragments(fragmentEntry, neighbourFragmentEntry, end, entries, txn);
     
-        return {entries, fragments};
+        return {entries, updatedEntries, fragments};
     }
 }

src/matrix/room/timeline/persistence/RelationWriter.js

@@ -0,0 +1,99 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {EventEntry} from "../entries/EventEntry.js";
+import {REDACTION_TYPE} from "../../common.js";
+
+export class RelationWriter {
+    constructor(roomId, fragmentIdComparer) {
+        this._roomId = roomId;
+        this._fragmentIdComparer = fragmentIdComparer;
+    }
+
+    // this needs to happen again after decryption too for edits
+    async writeRelation(sourceEntry, txn) {
+        if (sourceEntry.relatedEventId) {
+            const target = await txn.timelineEvents.getByEventId(this._roomId, sourceEntry.relatedEventId);
+            if (target) {
+                if (this._applyRelation(sourceEntry, target)) {
+                    txn.timelineEvents.update(target);
+                    return new EventEntry(target, this._fragmentIdComparer);
+                }
+            }
+        }
+        return;
+    }
+
+    _applyRelation(sourceEntry, target) {
+        if (sourceEntry.eventType === REDACTION_TYPE) {
+            return this._applyRedaction(sourceEntry.event, target.event);
+        } else {
+            return false;
+        }
+    }
+
+    _applyRedaction(redactionEvent, targetEvent) {
+        // TODO: should we make efforts to preserve the decrypted event type?
+        // probably ok not to, as we'll show whatever is deleted as "deleted message"
+        // reactions are the only thing that comes to mind, but we don't encrypt those (for now)
+        for (const key of Object.keys(targetEvent)) {
+            if (!_REDACT_KEEP_KEY_MAP[key]) {
+                delete targetEvent[key];
+            }
+        }
+        const {content} = targetEvent;
+        const keepMap = _REDACT_KEEP_CONTENT_MAP[targetEvent.type];
+        for (const key of Object.keys(content)) {
+            if (!keepMap?.[key]) {
+                delete content[key];
+            }
+        }
+        targetEvent.unsigned = targetEvent.unsigned || {};
+        targetEvent.unsigned.redacted_because = redactionEvent;
+
+        return true;
+    }
+}
+
+// copied over from matrix-js-sdk, copyright 2016 OpenMarket Ltd
+/* _REDACT_KEEP_KEY_MAP gives the keys we keep when an event is redacted
+ *
+ * This is specified here:
+ *  http://matrix.org/speculator/spec/HEAD/client_server/latest.html#redactions
+ *
+ * Also:
+ *  - We keep 'unsigned' since that is created by the local server
+ *  - We keep user_id for backwards-compat with v1
+ */
+const _REDACT_KEEP_KEY_MAP = [
+    'event_id', 'type', 'room_id', 'user_id', 'sender', 'state_key', 'prev_state',
+    'content', 'unsigned', 'origin_server_ts',
+].reduce(function(ret, val) {
+    ret[val] = 1; return ret;
+}, {});
+
+// a map from event type to the .content keys we keep when an event is redacted
+const _REDACT_KEEP_CONTENT_MAP = {
+    'm.room.member': {'membership': 1},
+    'm.room.create': {'creator': 1},
+    'm.room.join_rules': {'join_rule': 1},
+    'm.room.power_levels': {'ban': 1, 'events': 1, 'events_default': 1,
+                            'kick': 1, 'redact': 1, 'state_default': 1,
+                            'users': 1, 'users_default': 1,
+                           },
+    'm.room.aliases': {'aliases': 1},
+};
+// end of matrix-js-sdk code

src/matrix/room/timeline/persistence/SyncWriter.js

@@ -21,6 +21,7 @@ import {FragmentBoundaryEntry} from "../entries/FragmentBoundaryEntry.js";
 import {createEventEntry} from "./common.js";
 import {EVENT_TYPE as MEMBER_EVENT_TYPE} from "../../members/RoomMember.js";
 import {MemberWriter} from "./MemberWriter.js";
+import {RelationWriter} from "./RelationWriter.js";
 
 // Synapse bug? where the m.room.create event appears twice in sync response
 // when first syncing the room
@@ -40,6 +41,7 @@ export class SyncWriter {
     constructor({roomId, fragmentIdComparer}) {
         this._roomId = roomId;
         this._memberWriter = new MemberWriter(roomId);
+        this._relationWriter = new RelationWriter(roomId, fragmentIdComparer);
         this._fragmentIdComparer = fragmentIdComparer;
         this._lastLiveKey = null;
     }
@@ -151,7 +153,9 @@ export class SyncWriter {
         }
     }
 
-    async _writeTimeline(entries, timeline, currentKey, memberChanges, txn, log) {
+    async _writeTimeline(timeline, currentKey, memberChanges, txn, log) {
+        const entries = [];
+        const updatedEntries = [];
         if (Array.isArray(timeline?.events) && timeline.events.length) {
             // only create a fragment when we will really write an event
             currentKey = await this._ensureLiveFragment(currentKey, entries, timeline, txn, log);
@@ -161,15 +165,19 @@ export class SyncWriter {
             for(const event of events) {
                 // store event in timeline
                 currentKey = currentKey.nextKey();
-                const entry = createEventEntry(currentKey, this._roomId, event);
+                const storageEntry = createEventEntry(currentKey, this._roomId, event);
                 let member = await this._memberWriter.lookupMember(event.sender, event, events, txn);
                 if (member) {
-                    entry.displayName = member.displayName;
+                    storageEntry.displayName = member.displayName;
-                    entry.avatarUrl = member.avatarUrl;
+                    storageEntry.avatarUrl = member.avatarUrl;
+                }
+                txn.timelineEvents.insert(storageEntry);
+                const entry = new EventEntry(storageEntry, this._fragmentIdComparer);
+                entries.push(entry);
+                const updatedRelationTargetEntry = await this._relationWriter.writeRelation(entry);
+                if (updatedRelationTargetEntry) {
+                    updatedEntries.push(updatedRelationTargetEntry);
                 }
-                txn.timelineEvents.insert(entry);
-                entries.push(new EventEntry(entry, this._fragmentIdComparer));
-
                 // update state events after writing event, so for a member event,
                 // we only update the member info after having written the member event
                 // to the timeline, as we want that event to have the old profile info
@@ -187,7 +195,7 @@ export class SyncWriter {
             }
             log.set("timelineStateEventCount", timelineStateEventCount);
         }
-        return currentKey;
+        return {currentKey, entries, updatedEntries};
     }
 
     async _handleRejoinOverlap(timeline, txn, log) {
@@ -226,7 +234,6 @@ export class SyncWriter {
      * @return {SyncWriterResult}
      */
     async writeSync(roomResponse, isRejoin, txn, log) {
-        const entries = [];
         let {timeline} = roomResponse;
         // we have rejoined the room after having synced it before,
         // check for overlap with the last synced event
@@ -238,9 +245,10 @@ export class SyncWriter {
         // important this happens before _writeTimeline so
         // members are available in the transaction
         await this._writeStateEvents(roomResponse, memberChanges, timeline?.limited, txn, log);
-        const currentKey = await this._writeTimeline(entries, timeline, this._lastLiveKey, memberChanges, txn, log);
+        const {currentKey, entries, updatedEntries} =
+            await this._writeTimeline(entries, updatedEntries, timeline, this._lastLiveKey, memberChanges, txn, log);
         log.set("memberChanges", memberChanges.size);
-        return {entries, newLiveKey: currentKey, memberChanges};
+        return {entries, updatedEntries, newLiveKey: currentKey, memberChanges};
     }
 
     afterSync(newLiveKey) {