From 044360afaa4ddd58b6dedd101267819e08ec9e1e Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Thu, 13 Aug 2020 18:59:31 +0200 Subject: [PATCH 1/3] add content hashes to build assets --- package.json | 4 +- scripts/build.mjs | 184 ++++++++++++++++++++++++++++------------------ 2 files changed, 114 insertions(+), 74 deletions(-) diff --git a/package.json b/package.json index f44d51a6..c43220c7 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "@rollup/plugin-multi-entry": "^3.0.1", "@rollup/plugin-node-resolve": "^8.4.0", "cheerio": "^1.0.0-rc.3", + "commander": "^6.0.0", "core-js": "^3.6.5", "finalhandler": "^1.1.1", "impunity": "^0.0.11", @@ -39,6 +40,7 @@ "postcss-import": "^12.0.1", "regenerator-runtime": "^0.13.7", "rollup": "^1.15.6", - "serve-static": "^1.13.2" + "serve-static": "^1.13.2", + "xxhash": "^0.3.0" } } diff --git a/scripts/build.mjs b/scripts/build.mjs index 611738cb..30189714 100644 --- a/scripts/build.mjs +++ b/scripts/build.mjs @@ -19,11 +19,13 @@ import cheerio from "cheerio"; import fsRoot from "fs"; const fs = fsRoot.promises; import path from "path"; +import XXHash from 'xxhash'; import rollup from 'rollup'; import postcss from "postcss"; import postcssImport from "postcss-import"; import { fileURLToPath } from 'url'; import { dirname } from 'path'; +import commander from "commander"; // needed for legacy bundle import babel from '@rollup/plugin-babel'; // needed to find the polyfill modules in the main-legacy.js bundle @@ -43,32 +45,20 @@ const PROJECT_NAME = "Hydrogen Chat"; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); const projectDir = path.join(__dirname, "../"); -const cssDir = path.join(projectDir, "src/ui/web/css/"); -const targetDir = path.join(projectDir, "target"); +const cssSrcDir = path.join(projectDir, "src/ui/web/css/"); +const targetDir = path.join(projectDir, "target/"); -const {debug, noOffline, legacy} = process.argv.reduce((params, param) => { - if (param.startsWith("--")) { - params[param.substr(2)] = true; - } - return params; -}, { - debug: false, - noOffline: false, - legacy: false -}); +const program = new commander.Command(); +program + .option("--legacy", "make a build for IE11") + .option("--no-offline", "make a build without a service worker or appcache manifest") +program.parse(process.argv); +const {debug, noOffline, legacy} = program; const offline = !noOffline; async function build() { // get version number const version = JSON.parse(await fs.readFile(path.join(projectDir, "package.json"), "utf8")).version; - // clear target dir - await removeDirIfExists(targetDir); - await fs.mkdir(targetDir); - let bundleName = `${PROJECT_ID}.js`; - if (legacy) { - bundleName = `${PROJECT_ID}-legacy.js`; - } - const devHtml = await fs.readFile(path.join(projectDir, "index.html"), "utf8"); const doc = cheerio.load(devHtml); @@ -76,25 +66,40 @@ async function build() { findThemes(doc, themeName => { themes.push(themeName); }); - + // clear target dir + await removeDirIfExists(targetDir); + await createDirs(targetDir, themes); // also creates the directories where the theme css bundles are placed in, // so do it first const themeAssets = await copyThemeAssets(themes, legacy); + const jsBundlePath = await (legacy ? buildJsLegacy() : buildJs()); + const cssBundlePaths = await buildCssBundles(legacy ? buildCssLegacy : buildCss, themes); + const assetPaths = createAssetPaths(jsBundlePath, cssBundlePaths, themeAssets); - await buildHtml(doc, version, bundleName); - if (legacy) { - await buildJsLegacy(bundleName); - } else { - await buildJs(bundleName); - } - await buildCssBundles(legacy ? buildCssLegacy : buildCss, themes); if (offline) { - await buildOffline(version, bundleName, themeAssets); + await buildOffline(version, assetPaths); } + await buildHtml(doc, version, assetPaths); console.log(`built ${PROJECT_ID}${legacy ? " legacy" : ""} ${version} successfully`); } +function createAssetPaths(jsBundlePath, cssBundlePaths, themeAssets) { + function trim(path) { + if (!path.startsWith(targetDir)) { + throw new Error("invalid target path: " + targetDir); + } + return path.substr(targetDir.length); + } + return { + jsBundle: () => trim(jsBundlePath), + cssMainBundle: () => trim(cssBundlePaths.main), + cssThemeBundle: themeName => trim(cssBundlePaths.themes[themeName]), + cssThemeBundles: () => Object.values(cssBundlePaths.themes).map(a => trim(a)), + otherAssets: () => themeAssets.map(a => trim(a)) + }; +} + async function findThemes(doc, callback) { doc("link[rel~=stylesheet][title]").each((i, el) => { const theme = doc(el); @@ -110,37 +115,39 @@ async function findThemes(doc, callback) { }); } +async function createDirs(targetDir, themes) { + await fs.mkdir(targetDir); + const themeDir = path.join(targetDir, "themes"); + await fs.mkdir(themeDir); + for (const theme of themes) { + await fs.mkdir(path.join(themeDir, theme)); + } +} + async function copyThemeAssets(themes, legacy) { const assets = []; - // create theme directories and copy assets - await fs.mkdir(path.join(targetDir, "themes")); for (const theme of themes) { - assets.push(`themes/${theme}/bundle.css`); const themeDstFolder = path.join(targetDir, `themes/${theme}`); - await fs.mkdir(themeDstFolder); - const themeSrcFolder = path.join(cssDir, `themes/${theme}`); - await copyFolder(themeSrcFolder, themeDstFolder, file => { + const themeSrcFolder = path.join(cssSrcDir, `themes/${theme}`); + const themeAssets = await copyFolder(themeSrcFolder, themeDstFolder, file => { const isUnneededFont = legacy ? file.endsWith(".woff2") : file.endsWith(".woff"); - if (!file.endsWith(".css") && !isUnneededFont) { - assets.push(file.substr(cssDir.length)); - return true; - } - return false; + return !file.endsWith(".css") && !isUnneededFont; }); + assets.push(...themeAssets); } return assets; } -async function buildHtml(doc, version, bundleName) { +async function buildHtml(doc, version, assetPaths) { // transform html file // change path to main.css to css bundle - doc("link[rel=stylesheet]:not([title])").attr("href", `${PROJECT_ID}.css`); + doc("link[rel=stylesheet]:not([title])").attr("href", assetPaths.cssMainBundle()); // change paths to all theme stylesheets findThemes(doc, (themeName, theme) => { - theme.attr("href", `themes/${themeName}/bundle.css`); + theme.attr("href", assetPaths.cssThemeBundle(themeName)); }); doc("script#main").replaceWith( - `` + + `` + ``); removeOrEnableScript(doc("script#service-worker"), offline); @@ -157,17 +164,20 @@ async function buildHtml(doc, version, bundleName) { await fs.writeFile(path.join(targetDir, "index.html"), doc.html(), "utf8"); } -async function buildJs(bundleName) { +async function buildJs() { // create js bundle const bundle = await rollup.rollup({input: 'src/main.js'}); - await bundle.write({ - file: path.join(targetDir, bundleName), + const {output} = await bundle.generate({ format: 'iife', name: `${PROJECT_ID}Bundle` }); + const code = output[0].code; + const bundlePath = resource(`${PROJECT_ID}.js`, code); + await fs.writeFile(bundlePath, code, "utf8"); + return bundlePath; } -async function buildJsLegacy(bundleName) { +async function buildJsLegacy() { // compile down to whatever IE 11 needs const babelPlugin = babel.babel({ babelHelpers: 'bundled', @@ -189,24 +199,24 @@ async function buildJsLegacy(bundleName) { plugins: [multi(), commonjs(), nodeResolve(), babelPlugin] }; const bundle = await rollup.rollup(rollupConfig); - await bundle.write({ - file: path.join(targetDir, bundleName), + const {output} = await bundle.generate({ format: 'iife', name: `${PROJECT_ID}Bundle` }); + const code = output[0].code; + const bundlePath = resource(`${PROJECT_ID}-legacy.js`, code); + await fs.writeFile(bundlePath, code, "utf8"); + return bundlePath; } -async function buildOffline(version, bundleName, themeAssets) { - const {offlineAssets, cacheAssets} = themeAssets.reduce((result, asset) => { - if (asset.endsWith(".css")) { - result.offlineAssets.push(asset); - } else { - result.cacheAssets.push(asset); - } - return result; - }, {offlineAssets: [], cacheAssets: []}); +async function buildOffline(version, assetPaths) { // write offline availability - const offlineFiles = [bundleName, `${PROJECT_ID}.css`, "index.html", "icon-192.png"].concat(offlineAssets); + const offlineFiles = [ + assetPaths.jsBundle(), + assetPaths.cssMainBundle(), + "index.html", + "icon-192.png", + ].concat(assetPaths.cssThemeBundles()); // write appcache manifest const manifestLines = [ @@ -223,7 +233,7 @@ async function buildOffline(version, bundleName, themeAssets) { let swSource = await fs.readFile(path.join(projectDir, "src/service-worker.template.js"), "utf8"); swSource = swSource.replace(`"%%VERSION%%"`, `"${version}"`); swSource = swSource.replace(`"%%OFFLINE_FILES%%"`, JSON.stringify(offlineFiles)); - swSource = swSource.replace(`"%%CACHE_FILES%%"`, JSON.stringify(cacheAssets)); + swSource = swSource.replace(`"%%CACHE_FILES%%"`, JSON.stringify(assetPaths.otherAssets())); await fs.writeFile(path.join(targetDir, "sw.js"), swSource, "utf8"); // write web manifest const webManifest = { @@ -235,33 +245,37 @@ async function buildOffline(version, bundleName, themeAssets) { }; await fs.writeFile(path.join(targetDir, "manifest.json"), JSON.stringify(webManifest), "utf8"); // copy icon + // should this icon have a content hash as well? let icon = await fs.readFile(path.join(projectDir, "icon.png")); await fs.writeFile(path.join(targetDir, "icon-192.png"), icon); } async function buildCssBundles(buildFn, themes) { - const cssMainFile = path.join(cssDir, "main.css"); - await buildFn(cssMainFile, path.join(targetDir, `${PROJECT_ID}.css`)); + const bundleCss = await buildFn(path.join(cssSrcDir, "main.css")); + const mainDstPath = resource(`${PROJECT_ID}.css`, bundleCss); + await fs.writeFile(mainDstPath, bundleCss, "utf8"); + const bundlePaths = {main: mainDstPath, themes: {}}; for (const theme of themes) { - await buildFn( - path.join(cssDir, `themes/${theme}/theme.css`), - path.join(targetDir, `themes/${theme}/bundle.css`) - ); + const themeCss = await buildFn(path.join(cssSrcDir, `themes/${theme}/theme.css`)); + const themeDstPath = resource(`themes/${theme}/bundle.css`, themeCss); + await fs.writeFile(themeDstPath, themeCss, "utf8"); + bundlePaths.themes[theme] = themeDstPath; } + return bundlePaths; } -async function buildCss(entryPath, bundlePath) { +async function buildCss(entryPath) { const preCss = await fs.readFile(entryPath, "utf8"); const cssBundler = postcss([postcssImport]); const result = await cssBundler.process(preCss, {from: entryPath}); - await fs.writeFile(bundlePath, result.css, "utf8"); + return result.css; } -async function buildCssLegacy(entryPath, bundlePath) { +async function buildCssLegacy(entryPath) { const preCss = await fs.readFile(entryPath, "utf8"); const cssBundler = postcss([postcssImport, cssvariables(), flexbugsFixes()]); const result = await cssBundler.process(preCss, {from: entryPath}); - await fs.writeFile(bundlePath, result.css, "utf8"); + return result.css; } function removeOrEnableScript(scriptNode, enable) { @@ -283,17 +297,41 @@ async function removeDirIfExists(targetDir) { } async function copyFolder(srcRoot, dstRoot, filter) { + const assetPaths = []; const dirEnts = await fs.readdir(srcRoot, {withFileTypes: true}); for (const dirEnt of dirEnts) { const dstPath = path.join(dstRoot, dirEnt.name); const srcPath = path.join(srcRoot, dirEnt.name); if (dirEnt.isDirectory()) { await fs.mkdir(dstPath); - await copyFolder(srcPath, dstPath, filter); + assetPaths.push(... await copyFolder(srcPath, dstPath, filter)); } else if (dirEnt.isFile() && filter(srcPath)) { - await fs.copyFile(srcPath, dstPath); + const content = await fs.readFile(srcPath); + const hashedDstPath = resource(dstPath, content); + await fs.writeFile(hashedDstPath, content); + assetPaths.push(hashedDstPath); } } + return assetPaths; } +function resource(relPath, content) { + let fullPath = relPath; + if (!relPath.startsWith("/")) { + fullPath = path.join(targetDir, relPath); + } + const hash = contentHash(Buffer.from(content)); + const dir = path.dirname(fullPath); + const extname = path.extname(fullPath); + const basename = path.basename(fullPath, extname); + return path.join(dir, `${basename}-${hash}${extname}`); +} + +function contentHash(str) { + var hasher = new XXHash(0); + hasher.update(str); + return hasher.digest(); +} + + build().catch(err => console.error(err)); From 0104e14e0b6b905ff8fe5757154f05a6d43e762b Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Fri, 14 Aug 2020 10:45:14 +0200 Subject: [PATCH 2/3] map urls in theme css bundles to their content-hashed counterparts --- package.json | 1 + scripts/build.mjs | 47 ++++++++++++++++++++++++++++++++++------------- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/package.json b/package.json index c43220c7..a2531924 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "postcss-css-variables": "^0.17.0", "postcss-flexbugs-fixes": "^4.2.1", "postcss-import": "^12.0.1", + "postcss-url": "^8.0.0", "regenerator-runtime": "^0.13.7", "rollup": "^1.15.6", "serve-static": "^1.13.2", diff --git a/scripts/build.mjs b/scripts/build.mjs index 30189714..96fee0c8 100644 --- a/scripts/build.mjs +++ b/scripts/build.mjs @@ -34,6 +34,8 @@ import { nodeResolve } from '@rollup/plugin-node-resolve'; import commonjs from '@rollup/plugin-commonjs'; // multi-entry plugin so we can add polyfill file to main import multi from '@rollup/plugin-multi-entry'; +// replace urls of asset names with content hashed version +import postcssUrl from "postcss-url"; import cssvariables from "postcss-css-variables"; import flexbugsFixes from "postcss-flexbugs-fixes"; @@ -73,7 +75,7 @@ async function build() { // so do it first const themeAssets = await copyThemeAssets(themes, legacy); const jsBundlePath = await (legacy ? buildJsLegacy() : buildJs()); - const cssBundlePaths = await buildCssBundles(legacy ? buildCssLegacy : buildCss, themes); + const cssBundlePaths = await buildCssBundles(legacy ? buildCssLegacy : buildCss, themes, themeAssets); const assetPaths = createAssetPaths(jsBundlePath, cssBundlePaths, themeAssets); if (offline) { @@ -96,7 +98,7 @@ function createAssetPaths(jsBundlePath, cssBundlePaths, themeAssets) { cssMainBundle: () => trim(cssBundlePaths.main), cssThemeBundle: themeName => trim(cssBundlePaths.themes[themeName]), cssThemeBundles: () => Object.values(cssBundlePaths.themes).map(a => trim(a)), - otherAssets: () => themeAssets.map(a => trim(a)) + otherAssets: () => Object.values(themeAssets).map(a => trim(a)) }; } @@ -125,7 +127,7 @@ async function createDirs(targetDir, themes) { } async function copyThemeAssets(themes, legacy) { - const assets = []; + const assets = {}; for (const theme of themes) { const themeDstFolder = path.join(targetDir, `themes/${theme}`); const themeSrcFolder = path.join(cssSrcDir, `themes/${theme}`); @@ -133,7 +135,7 @@ async function copyThemeAssets(themes, legacy) { const isUnneededFont = legacy ? file.endsWith(".woff2") : file.endsWith(".woff"); return !file.endsWith(".css") && !isUnneededFont; }); - assets.push(...themeAssets); + Object.assign(assets, themeAssets); } return assets; } @@ -250,13 +252,20 @@ async function buildOffline(version, assetPaths) { await fs.writeFile(path.join(targetDir, "icon-192.png"), icon); } -async function buildCssBundles(buildFn, themes) { +async function buildCssBundles(buildFn, themes, themeAssets) { const bundleCss = await buildFn(path.join(cssSrcDir, "main.css")); const mainDstPath = resource(`${PROJECT_ID}.css`, bundleCss); await fs.writeFile(mainDstPath, bundleCss, "utf8"); const bundlePaths = {main: mainDstPath, themes: {}}; for (const theme of themes) { - const themeCss = await buildFn(path.join(cssSrcDir, `themes/${theme}/theme.css`)); + const urlBase = path.join(targetDir, `themes/${theme}/`); + const assetUrlMapper = ({absolutePath}) => { + const hashedDstPath = themeAssets[absolutePath]; + if (hashedDstPath && hashedDstPath.startsWith(urlBase)) { + return hashedDstPath.substr(urlBase.length); + } + }; + const themeCss = await buildFn(path.join(cssSrcDir, `themes/${theme}/theme.css`), assetUrlMapper); const themeDstPath = resource(`themes/${theme}/bundle.css`, themeCss); await fs.writeFile(themeDstPath, themeCss, "utf8"); bundlePaths.themes[theme] = themeDstPath; @@ -264,16 +273,28 @@ async function buildCssBundles(buildFn, themes) { return bundlePaths; } -async function buildCss(entryPath) { +async function buildCss(entryPath, urlMapper = null) { const preCss = await fs.readFile(entryPath, "utf8"); - const cssBundler = postcss([postcssImport]); + const options = [postcssImport]; + if (urlMapper) { + options.push(postcssUrl({url: urlMapper})); + } + const cssBundler = postcss(options); const result = await cssBundler.process(preCss, {from: entryPath}); return result.css; } -async function buildCssLegacy(entryPath) { +async function buildCssLegacy(entryPath, urlMapper = null) { const preCss = await fs.readFile(entryPath, "utf8"); - const cssBundler = postcss([postcssImport, cssvariables(), flexbugsFixes()]); + const options = [ + postcssImport, + cssvariables(), + flexbugsFixes() + ]; + if (urlMapper) { + options.push(postcssUrl({url: urlMapper})); + } + const cssBundler = postcss(options); const result = await cssBundler.process(preCss, {from: entryPath}); return result.css; } @@ -297,19 +318,19 @@ async function removeDirIfExists(targetDir) { } async function copyFolder(srcRoot, dstRoot, filter) { - const assetPaths = []; + const assetPaths = {}; const dirEnts = await fs.readdir(srcRoot, {withFileTypes: true}); for (const dirEnt of dirEnts) { const dstPath = path.join(dstRoot, dirEnt.name); const srcPath = path.join(srcRoot, dirEnt.name); if (dirEnt.isDirectory()) { await fs.mkdir(dstPath); - assetPaths.push(... await copyFolder(srcPath, dstPath, filter)); + Object.assign(assetPaths, await copyFolder(srcPath, dstPath, filter)); } else if (dirEnt.isFile() && filter(srcPath)) { const content = await fs.readFile(srcPath); const hashedDstPath = resource(dstPath, content); await fs.writeFile(hashedDstPath, content); - assetPaths.push(hashedDstPath); + assetPaths[srcPath] = hashedDstPath; } } return assetPaths; From 3c4805b2678b0958aa03bfd8297a61eb9f87bf63 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Fri, 14 Aug 2020 11:06:39 +0200 Subject: [PATCH 3/3] also content-hash the webapp manifest --- scripts/build.mjs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/build.mjs b/scripts/build.mjs index 96fee0c8..5ddcbafa 100644 --- a/scripts/build.mjs +++ b/scripts/build.mjs @@ -78,10 +78,11 @@ async function build() { const cssBundlePaths = await buildCssBundles(legacy ? buildCssLegacy : buildCss, themes, themeAssets); const assetPaths = createAssetPaths(jsBundlePath, cssBundlePaths, themeAssets); + let manifestPath; if (offline) { - await buildOffline(version, assetPaths); + manifestPath = await buildOffline(version, assetPaths); } - await buildHtml(doc, version, assetPaths); + await buildHtml(doc, version, assetPaths, manifestPath); console.log(`built ${PROJECT_ID}${legacy ? " legacy" : ""} ${version} successfully`); } @@ -140,7 +141,7 @@ async function copyThemeAssets(themes, legacy) { return assets; } -async function buildHtml(doc, version, assetPaths) { +async function buildHtml(doc, version, assetPaths, manifestPath) { // transform html file // change path to main.css to css bundle doc("link[rel=stylesheet]:not([title])").attr("href", assetPaths.cssMainBundle()); @@ -161,7 +162,7 @@ async function buildHtml(doc, version, assetPaths) { if (offline) { doc("html").attr("manifest", "manifest.appcache"); - doc("head").append(``); + doc("head").append(``); } await fs.writeFile(path.join(targetDir, "index.html"), doc.html(), "utf8"); } @@ -245,11 +246,14 @@ async function buildOffline(version, assetPaths) { start_url: "index.html", icons: [{"src": "icon-192.png", "sizes": "192x192", "type": "image/png"}], }; - await fs.writeFile(path.join(targetDir, "manifest.json"), JSON.stringify(webManifest), "utf8"); + const manifestJson = JSON.stringify(webManifest); + const manifestPath = resource("manifest.json", manifestJson); + await fs.writeFile(manifestPath, manifestJson, "utf8"); // copy icon // should this icon have a content hash as well? let icon = await fs.readFile(path.join(projectDir, "icon.png")); await fs.writeFile(path.join(targetDir, "icon-192.png"), icon); + return manifestPath; } async function buildCssBundles(buildFn, themes, themeAssets) {