Merge pull request #1177 from element-hq/midhun/support-authenticated-media

Add support for authenticated media

src/matrix/Client.js

@@ -252,7 +252,7 @@ export class Client {
         this._reconnector = new Reconnector({
             onlineStatus: this._platform.onlineStatus,
             retryDelay: new ExponentialRetryDelay(clock.createTimeout),
-            createMeasure: clock.createMeasure
+            createMeasure: clock.createMeasure,
         });
         const hsApi = new HomeServerApi({
             homeserver: sessionInfo.homeServer,
@@ -261,7 +261,10 @@ export class Client {
             reconnector: this._reconnector,
         });
         this._sessionId = sessionInfo.id;
-        this._storage = await this._platform.storageFactory.create(sessionInfo.id, log);
+        this._storage = await this._platform.storageFactory.create(
+            sessionInfo.id,
+            log
+        );
         // no need to pass access token to session
         const filteredSessionInfo = {
             id: sessionInfo.id,
@@ -275,11 +278,16 @@ export class Client {
         if (this._workerPromise) {
             olmWorker = await this._workerPromise;
         }
-        this._requestScheduler = new RequestScheduler({hsApi, clock});
+        this._requestScheduler = new RequestScheduler({ hsApi, clock });
         this._requestScheduler.start();
+
+        const lastVersionsResponse = await hsApi
+            .versions({ timeout: 10000, log })
+            .response();
         const mediaRepository = new MediaRepository({
             homeserver: sessionInfo.homeServer,
             platform: this._platform,
+            serverVersions: lastVersionsResponse.versions,
         });
         this._session = new Session({
             storage: this._storage,
@@ -289,32 +297,54 @@ export class Client {
             olmWorker,
             mediaRepository,
             platform: this._platform,
-            features: this._features
+            features: this._features,
         });
         await this._session.load(log);
         if (dehydratedDevice) {
-            await log.wrap("dehydrateIdentity", log => this._session.dehydrateIdentity(dehydratedDevice, log));
+            await log.wrap("dehydrateIdentity", (log) =>
-            await this._session.setupDehydratedDevice(dehydratedDevice.key, log);
+                this._session.dehydrateIdentity(dehydratedDevice, log)
+            );
+            await this._session.setupDehydratedDevice(
+                dehydratedDevice.key,
+                log
+            );
         } else if (!this._session.hasIdentity) {
             this._status.set(LoadStatus.SessionSetup);
-            await log.wrap("createIdentity", log => this._session.createIdentity(log));
+            await log.wrap("createIdentity", (log) =>
+                this._session.createIdentity(log)
+            );
         }
 
-        this._sync = new Sync({hsApi: this._requestScheduler.hsApi, storage: this._storage, session: this._session, logger: this._platform.logger});
+        this._sync = new Sync({
-        // notify sync and session when back online
+            hsApi: this._requestScheduler.hsApi,
-        this._reconnectSubscription = this._reconnector.connectionStatus.subscribe(state => {
+            storage: this._storage,
-            if (state === ConnectionStatus.Online) {
+            session: this._session,
-                this._platform.logger.runDetached("reconnect", async log => {
+            logger: this._platform.logger,
-                    // needs to happen before sync and session or it would abort all requests
-                    this._requestScheduler.start();
-                    this._sync.start();
-                    this._sessionStartedByReconnector = true;
-                    const d = dehydratedDevice;
-                    dehydratedDevice = undefined;
-                    await log.wrap("session start", log => this._session.start(this._reconnector.lastVersionsResponse, d, log));
-                });
-            }
         });
+        // notify sync and session when back online
+        this._reconnectSubscription =
+            this._reconnector.connectionStatus.subscribe((state) => {
+                if (state === ConnectionStatus.Online) {
+                    this._platform.logger.runDetached(
+                        "reconnect",
+                        async (log) => {
+                            // needs to happen before sync and session or it would abort all requests
+                            this._requestScheduler.start();
+                            this._sync.start();
+                            this._sessionStartedByReconnector = true;
+                            const d = dehydratedDevice;
+                            dehydratedDevice = undefined;
+                            await log.wrap("session start", (log) =>
+                                this._session.start(
+                                    this._reconnector.lastVersionsResponse,
+                                    d,
+                                    log
+                                )
+                            );
+                        }
+                    );
+                }
+            });
         await log.wrap("wait first sync", () => this._waitForFirstSync());
         if (this._isDisposed) {
             return;
@@ -326,14 +356,15 @@ export class Client {
         // started to session, so check first
         // to prevent an extra /versions request
         if (!this._sessionStartedByReconnector) {
-            const lastVersionsResponse = await hsApi.versions({timeout: 10000, log}).response();
             if (this._isDisposed) {
                 return;
             }
             const d = dehydratedDevice;
             dehydratedDevice = undefined;
             // log as ref as we don't want to await it
-            await log.wrap("session start", log => this._session.start(lastVersionsResponse, d, log));
+            await log.wrap("session start", (log) =>
+                this._session.start(lastVersionsResponse, d, log)
+            );
         }
     }
 

src/matrix/net/MediaRepository.ts

@@ -14,67 +14,183 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import {encodeQueryParams} from "./common";
+import { encodeQueryParams } from "./common";
-import {decryptAttachment} from "../e2ee/attachment.js";
+import { decryptAttachment } from "../e2ee/attachment.js";
-import {Platform} from "../../platform/web/Platform.js";
+import { Platform } from "../../platform/web/Platform.js";
-import {BlobHandle} from "../../platform/web/dom/BlobHandle.js";
+import { BlobHandle } from "../../platform/web/dom/BlobHandle.js";
-import type {Attachment, EncryptedFile} from "./types/response";
+import type {
+    Attachment,
+    EncryptedFile,
+    VersionResponse,
+} from "./types/response";
+
+type ServerVersions = VersionResponse["versions"];
+
+type Params = {
+    homeserver: string;
+    platform: Platform;
+    serverVersions: ServerVersions;
+};
 
 export class MediaRepository {
-    private readonly _homeserver: string;
+    private readonly homeserver: string;
-    private readonly _platform: Platform;
+    private readonly platform: Platform;
+    // Depends on whether the server supports authenticated media
+    private mediaUrlPart: string;
 
-    constructor({homeserver, platform}: {homeserver:string, platform: Platform}) {
+    constructor(params: Params) {
-        this._homeserver = homeserver;
+        this.homeserver = params.homeserver;
-        this._platform = platform;
+        this.platform = params.platform;
+        this.generateMediaUrl(params.serverVersions);
     }
 
-    mxcUrlThumbnail(url: string, width: number, height: number, method: "crop" | "scale"): string | undefined {
+    /**
-        const parts = this._parseMxcUrl(url);
+     * Calculate and store the correct media endpoint depending
+     * on whether the homeserver supports authenticated media (MSC3916)
+     * @see https://github.com/matrix-org/matrix-spec-proposals/pull/3916
+     * @param serverVersions List of supported spec versions
+     */
+    private generateMediaUrl(serverVersions: ServerVersions) {
+        const VERSION_WITH_AUTHENTICATION = "v1.11";
+        if (serverVersions.includes(VERSION_WITH_AUTHENTICATION)) {
+            this.mediaUrlPart = "_matrix/client/v1/media";
+        } else {
+            this.mediaUrlPart = "_matrix/media/v3";
+        }
+    }
+
+    mxcUrlThumbnail(
+        url: string,
+        width: number,
+        height: number,
+        method: "crop" | "scale"
+    ): string | undefined {
+        const parts = this.parseMxcUrl(url);
         if (parts) {
             const [serverName, mediaId] = parts;
-            const httpUrl = `${this._homeserver}/_matrix/media/r0/thumbnail/${encodeURIComponent(serverName)}/${encodeURIComponent(mediaId)}`;
+            const httpUrl = `${this.homeserver}/${
-            return httpUrl + "?" + encodeQueryParams({width: Math.round(width), height: Math.round(height), method});
+                this.mediaUrlPart
+            }/thumbnail/${encodeURIComponent(serverName)}/${encodeURIComponent(
+                mediaId
+            )}`;
+            return (
+                httpUrl +
+                "?" +
+                encodeQueryParams({
+                    width: Math.round(width),
+                    height: Math.round(height),
+                    method,
+                })
+            );
         }
         return undefined;
     }
 
     mxcUrl(url: string): string | undefined {
-        const parts = this._parseMxcUrl(url);
+        const parts = this.parseMxcUrl(url);
         if (parts) {
             const [serverName, mediaId] = parts;
-            return `${this._homeserver}/_matrix/media/r0/download/${encodeURIComponent(serverName)}/${encodeURIComponent(mediaId)}`;
+            return `${this.homeserver}/${
+                this.mediaUrlPart
+            }/download/${encodeURIComponent(serverName)}/${encodeURIComponent(
+                mediaId
+            )}`;
         }
         return undefined;
     }
 
-    private _parseMxcUrl(url: string): string[] | undefined {
+    private parseMxcUrl(url: string): string[] | undefined {
         const prefix = "mxc://";
         if (url.startsWith(prefix)) {
-            return url.substr(prefix.length).split("/", 2);
+            return url.slice(prefix.length).split("/", 2);
         } else {
             return undefined;
         }
     }
 
-    async downloadEncryptedFile(fileEntry: EncryptedFile, cache: boolean = false): Promise<BlobHandle> {
+    async downloadEncryptedFile(
+        fileEntry: EncryptedFile,
+        cache: boolean = false
+    ): Promise<BlobHandle> {
         const url = this.mxcUrl(fileEntry.url);
-        const {body: encryptedBuffer} = await this._platform.request(url, {method: "GET", format: "buffer", cache}).response();
+        const { body: encryptedBuffer } = await this.platform
-        const decryptedBuffer = await decryptAttachment(this._platform, encryptedBuffer, fileEntry);
+            .request(url, { method: "GET", format: "buffer", cache })
-        return this._platform.createBlob(decryptedBuffer, fileEntry.mimetype);
+            .response();
+        const decryptedBuffer = await decryptAttachment(
+            this.platform,
+            encryptedBuffer,
+            fileEntry
+        );
+        return this.platform.createBlob(decryptedBuffer, fileEntry.mimetype);
     }
 
-    async downloadPlaintextFile(mxcUrl: string, mimetype: string, cache: boolean = false): Promise<BlobHandle> {
+    async downloadPlaintextFile(
+        mxcUrl: string,
+        mimetype: string,
+        cache: boolean = false
+    ): Promise<BlobHandle> {
         const url = this.mxcUrl(mxcUrl);
-        const {body: buffer} = await this._platform.request(url, {method: "GET", format: "buffer", cache}).response();
+        const { body: buffer } = await this.platform
-        return this._platform.createBlob(buffer, mimetype);
+            .request(url, { method: "GET", format: "buffer", cache })
+            .response();
+        return this.platform.createBlob(buffer, mimetype);
     }
 
-    async downloadAttachment(content: Attachment, cache: boolean = false): Promise<BlobHandle> {
+    async downloadAttachment(
+        content: Attachment,
+        cache: boolean = false
+    ): Promise<BlobHandle> {
         if (content.file) {
             return this.downloadEncryptedFile(content.file, cache);
         } else {
-            return this.downloadPlaintextFile(content.url!, content.info?.mimetype, cache);
+            return this.downloadPlaintextFile(
+                content.url!,
+                content.info?.mimetype,
+                cache
+            );
         }
     }
 }
+
+export function tests() {
+    return {
+        "Uses correct endpoint when server supports authenticated media": (
+            assert
+        ) => {
+            const homeserver = "matrix.org";
+            const platform = {};
+            // Is it enough to check if v1.11 is present?
+            // or do we check if maxVersion > v1.11
+            const serverVersions = ["v1.1", "v1.11", "v1.10"];
+            const mediaRepository = new MediaRepository({
+                homeserver,
+                platform,
+                serverVersions,
+            });
+
+            const mxcUrl = "mxc://matrix.org/foobartest";
+            assert.match(
+                mediaRepository.mxcUrl(mxcUrl),
+                /_matrix\/client\/v1\/media/
+            );
+        },
+
+        "Uses correct endpoint when server does not supports authenticated media":
+            (assert) => {
+                const homeserver = "matrix.org";
+                const platform = {};
+                const serverVersions = ["v1.1", "v1.11", "v1.10"];
+                const mediaRepository = new MediaRepository({
+                    homeserver,
+                    platform,
+                    serverVersions,
+                });
+
+                const mxcUrl = "mxc://matrix.org/foobartest";
+                assert.match(
+                    mediaRepository.mxcUrl(mxcUrl),
+                    /_matrix\/client\/v1\/media/
+                );
+            },
+    };
+}

src/platform/web/Platform.js

@@ -146,8 +146,13 @@ export class Platform {
         this.onlineStatus = new OnlineStatus();
         this.timeFormatter = new TimeFormatter();
         this._serviceWorkerHandler = null;
+        this.sessionInfoStorage = new SessionInfoStorage(
+            "hydrogen_sessions_v1"
+        );
         if (assetPaths.serviceWorker && "serviceWorker" in navigator) {
-            this._serviceWorkerHandler = new ServiceWorkerHandler();
+            this._serviceWorkerHandler = new ServiceWorkerHandler(
+                this.sessionInfoStorage
+            );
             this._serviceWorkerHandler.registerAndStart(assetPaths.serviceWorker);
         }
         this.notificationService = undefined;
@@ -156,7 +161,6 @@ export class Platform {
             this.crypto = new Crypto(cryptoExtras);
         }
         this.storageFactory = new StorageFactory(this._serviceWorkerHandler);
-        this.sessionInfoStorage = new SessionInfoStorage("hydrogen_sessions_v1");
         this.estimateStorageUsage = estimateStorageUsage;
         if (typeof fetch === "function") {
             this.request = createFetchRequest(this.clock.createTimeout, this._serviceWorkerHandler);

src/platform/web/dom/ServiceWorkerHandler.js

@@ -19,13 +19,14 @@ limitations under the License.
 // - UpdateService (see checkForUpdate method, and should also emit events rather than showing confirm dialog here)
 // - ConcurrentAccessBlocker (see preventConcurrentSessionAccess method)
 export class ServiceWorkerHandler {
-    constructor() {
+    constructor(sessionInfoStorage) {
         this._waitingForReply = new Map();
         this._messageIdCounter = 0;
         this._navigation = null;
         this._registration = null;
         this._registrationPromise = null;
         this._currentController = null;
+        this._sessionInfoStorage = sessionInfoStorage;
         this.haltRequests = false;
     }
 
@@ -50,8 +51,8 @@ export class ServiceWorkerHandler {
         })();
     }
 
-    _onMessage(event) {
+    async _onMessage(event) {
-        const {data} = event;
+        const { data } = event;
         const replyTo = data.replyTo;
         if (replyTo) {
             const resolve = this._waitingForReply.get(replyTo);
@@ -61,37 +62,63 @@ export class ServiceWorkerHandler {
             }
         }
         if (data.type === "hasSessionOpen") {
-            const hasOpen = this._navigation.observe("session").get() === data.payload.sessionId;
+            const hasOpen =
-            event.source.postMessage({replyTo: data.id, payload: hasOpen});
+                this._navigation.observe("session").get() ===
+                data.payload.sessionId;
+            event.source.postMessage({ replyTo: data.id, payload: hasOpen });
         } else if (data.type === "hasRoomOpen") {
-            const hasSessionOpen = this._navigation.observe("session").get() === data.payload.sessionId;
+            const hasSessionOpen =
-            const hasRoomOpen = this._navigation.observe("room").get() === data.payload.roomId;
+                this._navigation.observe("session").get() ===
-            event.source.postMessage({replyTo: data.id, payload: hasSessionOpen && hasRoomOpen});
+                data.payload.sessionId;
+            const hasRoomOpen =
+                this._navigation.observe("room").get() === data.payload.roomId;
+            event.source.postMessage({
+                replyTo: data.id,
+                payload: hasSessionOpen && hasRoomOpen,
+            });
         } else if (data.type === "closeSession") {
-            const {sessionId} = data.payload;
+            const { sessionId } = data.payload;
             this._closeSessionIfNeeded(sessionId).finally(() => {
-                event.source.postMessage({replyTo: data.id});
+                event.source.postMessage({ replyTo: data.id });
             });
         } else if (data.type === "haltRequests") {
             // this flag is read in fetch.js
             this.haltRequests = true;
-            event.source.postMessage({replyTo: data.id});
+            event.source.postMessage({ replyTo: data.id });
         } else if (data.type === "openRoom") {
             this._navigation.push("room", data.payload.roomId);
+        } else if (data.type === "getAccessToken") {
+            const token = await this._getLatestAccessToken();
+            event.source.postMessage({ replyTo: data.id, payload: token });
         }
     }
 
+    /**
+     * Fetch access-token from the storage
+     * @returns access token as string
+     */
+    async _getLatestAccessToken() {
+        const currentSessionId = this._navigation?.path.get("session")?.value;
+        if (!currentSessionId) return null;
+        const { accessToken } = await this._sessionInfoStorage.get(
+            currentSessionId
+        );
+        return accessToken;
+    }
+
     _closeSessionIfNeeded(sessionId) {
         const currentSession = this._navigation?.path.get("session");
         if (sessionId && currentSession?.value === sessionId) {
-            return new Promise(resolve => {
+            return new Promise((resolve) => {
-                const unsubscribe = this._navigation.pathObservable.subscribe(path => {
+                const unsubscribe = this._navigation.pathObservable.subscribe(
-                    const session = path.get("session");
+                    (path) => {
-                    if (!session || session.value !== sessionId) {
+                        const session = path.get("session");
-                        unsubscribe();
+                        if (!session || session.value !== sessionId) {
-                        resolve();
+                            unsubscribe();
+                            resolve();
+                        }
                     }
-                });
+                );
                 this._navigation.push("session");
             });
         } else {
@@ -135,7 +162,10 @@ export class ServiceWorkerHandler {
                 this._onMessage(event);
                 break;
             case "updatefound":
-                this._registration.installing.addEventListener("statechange", this);
+                this._registration.installing.addEventListener(
+                    "statechange",
+                    this
+                );
                 break;
             case "statechange": {
                 if (event.target.state === "installed") {
@@ -149,10 +179,11 @@ export class ServiceWorkerHandler {
                     // Clients.claim() in the SW can trigger a controllerchange event
                     // if we had no SW before. This is fine,
                     // and now our requests will be served from the SW.
-                    this._currentController = navigator.serviceWorker.controller;
+                    this._currentController =
+                        navigator.serviceWorker.controller;
                 } else {
                     // active service worker changed,
-                    // refresh, so we can get all assets 
+                    // refresh, so we can get all assets
                     // (and not only some if we would not refresh)
                     // up to date from it
                     document.location.reload();
@@ -168,7 +199,7 @@ export class ServiceWorkerHandler {
         if (!worker) {
             worker = this._registration.active;
         }
-        worker.postMessage({type, payload});
+        worker.postMessage({ type, payload });
     }
 
     async _sendAndWaitForReply(type, payload, worker = undefined) {
@@ -180,10 +211,10 @@ export class ServiceWorkerHandler {
         }
         this._messageIdCounter += 1;
         const id = this._messageIdCounter;
-        const promise = new Promise(resolve => {
+        const promise = new Promise((resolve) => {
             this._waitingForReply.set(id, resolve);
         });
-        worker.postMessage({type, id, payload});
+        worker.postMessage({ type, id, payload });
         return await promise;
     }
 
@@ -203,7 +234,7 @@ export class ServiceWorkerHandler {
     }
 
     async preventConcurrentSessionAccess(sessionId) {
-        return this._sendAndWaitForReply("closeSession", {sessionId});
+        return this._sendAndWaitForReply("closeSession", { sessionId });
     }
 
     async getRegistration() {

src/platform/web/sw.js

@@ -83,12 +83,15 @@ self.addEventListener("fetch", (event) => {
     This has to do with xhr not being supported in service workers.
     */
     if (event.request.method === "GET") {
-        event.respondWith(handleRequest(event.request));
+        event.respondWith(handleRequest(event));
     }
 });
 
 function isCacheableThumbnail(url) {
-    if (url.pathname.startsWith("/_matrix/media/r0/thumbnail/")) {
+    if (
+        url.pathname.startsWith("/_matrix/media/r0/thumbnail/") ||
+        url.pathname.startsWith("/_matrix/client/v1/media/thumbnail/")
+    ) {
         const width = parseInt(url.searchParams.get("width"), 10);
         const height = parseInt(url.searchParams.get("height"), 10);
         if (width <= 50 && height <= 50) {
@@ -101,22 +104,42 @@ function isCacheableThumbnail(url) {
 const baseURL = new URL(self.registration.scope);
 let pendingFetchAbortController = new AbortController();
 
-async function handleRequest(request) {
+async function handleRequest({ request, clientId }) {
     try {
+        // Special caching strategy for config.json and theme json files
         if (
             request.url.includes("config.json") ||
             /theme-.+\.json/.test(request.url)
         ) {
             return handleStaleWhileRevalidateRequest(request);
         }
-        const url = new URL(request.url);
+
         // rewrite / to /index.html so it hits the cache
+        const url = new URL(request.url);
         if (
             url.origin === baseURL.origin &&
             url.pathname === baseURL.pathname
         ) {
             request = new Request(new URL("index.html", baseURL.href));
         }
+
+        // Add access token for authenticated media endpoints
+        if (request.url.includes("_matrix/client/v1/media")) {
+            const headers = new Headers(request.headers);
+            const client = await self.clients.get(clientId);
+            const accessToken = await sendAndWaitForReply(
+                client,
+                "getAccessToken",
+                {}
+            );
+            headers.set("authorization", `Bearer ${accessToken}`);
+            request = new Request(request, {
+                mode: "cors",
+                credentials: "omit",
+                headers,
+            });
+        }
+
         let response = await readCache(request);
         if (!response) {
             // use cors so the resource in the cache isn't opaque and uses up to 7mb