Update build script (#1722)

* Add support for making non-buildable variants

* Add support for running a command to assign a value

* Update chrome-dev inheritance

* Add base variant

* Update manifest to auto-fill the version based on most recent git tag

* Add support for changing the default manifest

* Change the default manifest

* Move some manifest utility functions into manifest-util.js

* Move more manifest functionality into ManifestUtil

* Revert "Update manifest to auto-fill the version based on most recent git tag"

This reverts commit 2a66e40ff24f9dc545783503bcf3404f21148356.

dev/build.js

@@ -21,13 +21,10 @@ const assert = require('assert');
 const readline = require('readline');
 const childProcess = require('child_process');
 const util = require('./util');
-const {getAllFiles, getDefaultManifestAndVariants, createManifestString, getArgs, testMain} = util;
+const {getAllFiles, getArgs, testMain} = util;
+const {ManifestUtil} = require('./manifest-util');
 
 
-function clone(value) {
-    return JSON.parse(JSON.stringify(value));
-}
-
 async function createZip(directory, excludeFiles, outputFileName, sevenZipExes, onUpdate, dryRun) {
     try {
         fs.unlinkSync(outputFileName);
@@ -110,179 +107,7 @@ function getIndexOfFilePath(array, item) {
     return -1;
 }
 
-function applyModifications(manifest, modifications) {
+async function build(buildDir, extDir, manifestUtil, variantNames, manifestPath, dryRun, dryRunBuildZip) {
-    if (Array.isArray(modifications)) {
-        for (const modification of modifications) {
-            const {action, path: path2} = modification;
-            switch (action) {
-                case 'set':
-                    {
-                        const {value, before, after} = modification;
-                        const object = getObjectProperties(manifest, path2, path2.length - 1);
-                        const key = path2[path2.length - 1];
-
-                        let {index} = modification;
-                        if (typeof index !== 'number') {
-                            index = -1;
-                        }
-                        if (typeof before === 'string') {
-                            index = getObjectKeyIndex(object, before);
-                        }
-                        if (typeof after === 'string') {
-                            index = getObjectKeyIndex(object, after);
-                            if (index >= 0) { ++index; }
-                        }
-
-                        setObjectKeyAtIndex(object, key, value, index);
-                    }
-                    break;
-                case 'replace':
-                    {
-                        const {pattern, patternFlags, replacement} = modification;
-                        const value = getObjectProperties(manifest, path2, path2.length - 1);
-                        const regex = new RegExp(pattern, patternFlags);
-                        const last = path2[path2.length - 1];
-                        let value2 = value[last];
-                        value2 = `${value2}`.replace(regex, replacement);
-                        value[last] = value2;
-                    }
-                    break;
-                case 'delete':
-                    {
-                        const value = getObjectProperties(manifest, path2, path2.length - 1);
-                        const last = path2[path2.length - 1];
-                        delete value[last];
-                    }
-                    break;
-                case 'remove':
-                    {
-                        const {item} = modification;
-                        const value = getObjectProperties(manifest, path2, path2.length);
-                        const index = value.indexOf(item);
-                        if (index >= 0) { value.splice(index, 1); }
-                    }
-                    break;
-                case 'splice':
-                    {
-                        const {start, deleteCount, items} = modification;
-                        const value = getObjectProperties(manifest, path2, path2.length);
-                        const itemsNew = items.map((v) => clone(v));
-                        value.splice(start, deleteCount, ...itemsNew);
-                    }
-                    break;
-                case 'copy':
-                case 'move':
-                    {
-                        const {newPath, before, after} = modification;
-                        const oldKey = path2[path2.length - 1];
-                        const newKey = newPath[newPath.length - 1];
-                        const oldObject = getObjectProperties(manifest, path2, path2.length - 1);
-                        const newObject = getObjectProperties(manifest, newPath, newPath.length - 1);
-                        const oldObjectIsNewObject = arraysAreSame(path2, newPath, -1);
-                        const value = oldObject[oldKey];
-
-                        let {index} = modification;
-                        if (typeof index !== 'number' || index < 0) {
-                            index = (oldObjectIsNewObject && action !== 'copy') ? getObjectKeyIndex(oldObject, oldKey) : -1;
-                        }
-                        if (typeof before === 'string') {
-                            index = getObjectKeyIndex(newObject, before);
-                        }
-                        if (typeof after === 'string') {
-                            index = getObjectKeyIndex(newObject, after);
-                            if (index >= 0) { ++index; }
-                        }
-
-                        setObjectKeyAtIndex(newObject, newKey, value, index);
-                        if (action !== 'copy' && (!oldObjectIsNewObject || oldKey !== newKey)) {
-                            delete oldObject[oldKey];
-                        }
-                    }
-                    break;
-                case 'add':
-                    {
-                        const {items} = modification;
-                        const value = getObjectProperties(manifest, path2, path2.length);
-                        const itemsNew = items.map((v) => clone(v));
-                        value.push(...itemsNew);
-                    }
-                    break;
-            }
-        }
-    }
-
-    return manifest;
-}
-
-function arraysAreSame(array1, array2, lengthOffset) {
-    let ii = array1.length;
-    if (ii !== array2.length) { return false; }
-    ii += lengthOffset;
-    for (let i = 0; i < ii; ++i) {
-        if (array1[i] !== array2[i]) { return false; }
-    }
-    return true;
-}
-
-function getObjectKeyIndex(object, key) {
-    return Object.keys(object).indexOf(key);
-}
-
-function setObjectKeyAtIndex(object, key, value, index) {
-    if (index < 0 || typeof key === 'number' || Object.prototype.hasOwnProperty.call(object, key)) {
-        object[key] = value;
-        return;
-    }
-
-    const entries = Object.entries(object);
-    index = Math.min(index, entries.length);
-    for (let i = index, ii = entries.length; i < ii; ++i) {
-        const [key2] = entries[i];
-        delete object[key2];
-    }
-    entries.splice(index, 0, [key, value]);
-    for (let i = index, ii = entries.length; i < ii; ++i) {
-        const [key2, value2] = entries[i];
-        object[key2] = value2;
-    }
-}
-
-function getObjectProperties(object, path2, count) {
-    for (let i = 0; i < count; ++i) {
-        object = object[path2[i]];
-    }
-    return object;
-}
-
-function getInheritanceChain(variant, variantMap) {
-    const visited = new Set();
-    const inheritance = [];
-    while (true) {
-        const {name, inherit} = variant;
-        if (visited.has(name)) { break; }
-
-        visited.add(name);
-        inheritance.unshift(variant);
-
-        if (typeof inherit !== 'string') { break; }
-
-        const nextVariant = variantMap.get(inherit);
-        if (typeof nextVariant === 'undefined') { break; }
-
-        variant = nextVariant;
-    }
-    return inheritance;
-}
-
-function createVariantManifest(manifest, variant, variantMap) {
-    let modifiedManifest = clone(manifest);
-    for (const {modifications} of getInheritanceChain(variant, variantMap)) {
-        modifiedManifest = applyModifications(modifiedManifest, modifications);
-    }
-    return modifiedManifest;
-}
-
-async function build(manifest, buildDir, extDir, manifestPath, variantMap, variantNames, dryRun, dryRunBuildZip) {
     const sevenZipExes = ['7za', '7z'];
 
     // Create build directory
@@ -305,8 +130,8 @@ async function build(manifest, buildDir, extDir, manifestPath, variantMap, varia
     };
 
     for (const variantName of variantNames) {
-        const variant = variantMap.get(variantName);
+        const variant = manifestUtil.getVariant(variantName);
-        if (typeof variant === 'undefined') { continue; }
+        if (typeof variant === 'undefined' || variant.buildable === false) { continue; }
 
         const {name, fileName, fileCopies} = variant;
         let {excludeFiles} = variant;
@@ -314,13 +139,13 @@ async function build(manifest, buildDir, extDir, manifestPath, variantMap, varia
 
         process.stdout.write(`Building ${name}...\n`);
 
-        const modifiedManifest = createVariantManifest(manifest, variant, variantMap);
+        const modifiedManifest = manifestUtil.getManifest(variant.name);
 
         const fileNameSafe = path.basename(fileName);
         const fullFileName = path.join(buildDir, fileNameSafe);
         ensureFilesExist(extDir, excludeFiles);
         if (!dryRun) {
-            fs.writeFileSync(manifestPath, createManifestString(modifiedManifest));
+            fs.writeFileSync(manifestPath, ManifestUtil.createManifestString(modifiedManifest));
         }
 
         if (!dryRun || dryRunBuildZip) {
@@ -360,33 +185,27 @@ async function main(argv) {
     const dryRun = args.get('dry-run');
     const dryRunBuildZip = args.get('dry-run-build-zip');
 
-    const {manifest, variants} = getDefaultManifestAndVariants();
+    const manifestUtil = new ManifestUtil();
 
     const rootDir = path.join(__dirname, '..');
     const extDir = path.join(rootDir, 'ext');
     const buildDir = path.join(rootDir, 'builds');
     const manifestPath = path.join(extDir, 'manifest.json');
 
-    const variantMap = new Map();
-    for (const variant of variants) {
-        variantMap.set(variant.name, variant);
-    }
-
     try {
-        const variantNames = (argv.length === 0 || args.get('all') ? variants.map(({name}) => name) : args.get(null));
+        const variantNames = (
-        await build(manifest, buildDir, extDir, manifestPath, variantMap, variantNames, dryRun, dryRunBuildZip);
+            argv.length === 0 || args.get('all') ?
+            manifestUtil.getVariants().filter(({buildable}) => buildable !== false).map(({name}) => name) :
+            args.get(null)
+        );
+        await build(buildDir, extDir, manifestUtil, variantNames, manifestPath, dryRun, dryRunBuildZip);
     } finally {
         // Restore manifest
-        let restoreManifest = manifest;
+        const manifestName = (!args.get('default') && args.get('manifest') !== null) ? args.get('manifest') : null;
-        if (!args.get('default') && args.get('manifest') !== null) {
+        const restoreManifest = manifestUtil.getManifest(manifestName);
-            const variant = variantMap.get(args.get('manifest'));
-            if (typeof variant !== 'undefined') {
-                restoreManifest = createVariantManifest(manifest, variant, variantMap);
-            }
-        }
         process.stdout.write('Restoring manifest...\n');
         if (!dryRun) {
-            fs.writeFileSync(manifestPath, createManifestString(restoreManifest));
+            fs.writeFileSync(manifestPath, ManifestUtil.createManifestString(restoreManifest));
         }
     }
 }

dev/data/manifest-variants.json

@@ -116,9 +116,15 @@
         ],
         "content_security_policy": "default-src 'self'; img-src blob: 'self'; style-src 'self' 'unsafe-inline'; media-src *; connect-src *"
     },
+    "defaultVariant": "base",
     "variants": [
+        {
+            "name": "base",
+            "buildable": false
+        },
         {
             "name": "chrome",
+            "inherit": "base",
             "fileName": "yomichan-chrome.zip",
             "excludeFiles": [
                 "sw.js",
@@ -128,6 +134,7 @@
         },
         {
             "name": "chrome-dev",
+            "inherit": "chrome",
             "fileName": "yomichan-chrome-dev.zip",
             "modifications": [
                 {
@@ -144,15 +151,11 @@
                     "patternFlags": "",
                     "replacement": "$1. This is a development build; get the stable version here: https://tinyurl.com/yaatdjmp"
                 }
-            ],
-            "excludeFiles": [
-                "sw.js",
-                "js/dom/simple-dom-parser.js",
-                "lib/parse5.js"
             ]
         },
         {
             "name": "chrome-mv3",
+            "inherit": "base",
             "fileName": "yomichan-chrome-mv3.zip",
             "modifications": [
                 {"action": "set",    "path": ["manifest_version"], "value": 3},
@@ -180,6 +183,7 @@
         },
         {
             "name": "firefox",
+            "inherit": "base",
             "fileName": "yomichan-firefox.xpi",
             "modifications": [
                 {

dev/manifest-util.js

@@ -0,0 +1,267 @@
+/*
+ * Copyright (C) 2021  Yomichan Authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const childProcess = require('child_process');
+
+
+function clone(value) {
+    return JSON.parse(JSON.stringify(value));
+}
+
+
+class ManifestUtil {
+    constructor() {
+        const fileName = path.join(__dirname, 'data', 'manifest-variants.json');
+        const {manifest, variants, defaultVariant} = JSON.parse(fs.readFileSync(fileName));
+        this._manifest = manifest;
+        this._variants = variants;
+        this._defaultVariant = defaultVariant;
+        this._variantMap = new Map();
+        for (const variant of variants) {
+            this._variantMap.set(variant.name, variant);
+        }
+    }
+
+    getManifest(variantName) {
+        if (typeof variantName === 'string') {
+            const variant = this._variantMap.get(variantName);
+            if (typeof variant !== 'undefined') {
+                return this._createVariantManifest(this._manifest, variant);
+            }
+        }
+
+        if (typeof this._defaultVariant === 'string') {
+            const variant = this._variantMap.get(this._defaultVariant);
+            if (typeof variant !== 'undefined') {
+                return this._createVariantManifest(this._manifest, variant);
+            }
+        }
+
+        return clone(this._manifest);
+    }
+
+    getVariants() {
+        return [...this._variants];
+    }
+
+    getVariant(name) {
+        return this._variantMap.get(name);
+    }
+
+    static createManifestString(manifest) {
+        return JSON.stringify(manifest, null, 4) + '\n';
+    }
+
+    // Private
+
+    _evaluateModificationCommand(data) {
+        const {command, args, trim} = data;
+        const {stdout, status} = childProcess.spawnSync(command, args, {
+            cwd: __dirname,
+            stdio: 'pipe',
+            shell: false
+        });
+        if (status !== 0) {
+            throw new Error(`Failed to execute ${command} ${args.join(' ')}`);
+        }
+        let result = stdout.toString('utf8');
+        if (trim) { result = result.trim(); }
+        return result;
+    }
+
+    _applyModifications(manifest, modifications) {
+        if (Array.isArray(modifications)) {
+            for (const modification of modifications) {
+                const {action, path: path2} = modification;
+                switch (action) {
+                    case 'set':
+                        {
+                            let {value, before, after, command} = modification;
+                            const object = this._getObjectProperties(manifest, path2, path2.length - 1);
+                            const key = path2[path2.length - 1];
+
+                            let {index} = modification;
+                            if (typeof index !== 'number') {
+                                index = -1;
+                            }
+                            if (typeof before === 'string') {
+                                index = this._getObjectKeyIndex(object, before);
+                            }
+                            if (typeof after === 'string') {
+                                index = this._getObjectKeyIndex(object, after);
+                                if (index >= 0) { ++index; }
+                            }
+                            if (typeof command === 'object' && command !== null) {
+                                value = this._evaluateModificationCommand(command);
+                            }
+
+                            this._setObjectKeyAtIndex(object, key, value, index);
+                        }
+                        break;
+                    case 'replace':
+                        {
+                            const {pattern, patternFlags, replacement} = modification;
+                            const value = this._getObjectProperties(manifest, path2, path2.length - 1);
+                            const regex = new RegExp(pattern, patternFlags);
+                            const last = path2[path2.length - 1];
+                            let value2 = value[last];
+                            value2 = `${value2}`.replace(regex, replacement);
+                            value[last] = value2;
+                        }
+                        break;
+                    case 'delete':
+                        {
+                            const value = this._getObjectProperties(manifest, path2, path2.length - 1);
+                            const last = path2[path2.length - 1];
+                            delete value[last];
+                        }
+                        break;
+                    case 'remove':
+                        {
+                            const {item} = modification;
+                            const value = this._getObjectProperties(manifest, path2, path2.length);
+                            const index = value.indexOf(item);
+                            if (index >= 0) { value.splice(index, 1); }
+                        }
+                        break;
+                    case 'splice':
+                        {
+                            const {start, deleteCount, items} = modification;
+                            const value = this._getObjectProperties(manifest, path2, path2.length);
+                            const itemsNew = items.map((v) => clone(v));
+                            value.splice(start, deleteCount, ...itemsNew);
+                        }
+                        break;
+                    case 'copy':
+                    case 'move':
+                        {
+                            const {newPath, before, after} = modification;
+                            const oldKey = path2[path2.length - 1];
+                            const newKey = newPath[newPath.length - 1];
+                            const oldObject = this._getObjectProperties(manifest, path2, path2.length - 1);
+                            const newObject = this._getObjectProperties(manifest, newPath, newPath.length - 1);
+                            const oldObjectIsNewObject = this._arraysAreSame(path2, newPath, -1);
+                            const value = oldObject[oldKey];
+
+                            let {index} = modification;
+                            if (typeof index !== 'number' || index < 0) {
+                                index = (oldObjectIsNewObject && action !== 'copy') ? this._getObjectKeyIndex(oldObject, oldKey) : -1;
+                            }
+                            if (typeof before === 'string') {
+                                index = this._getObjectKeyIndex(newObject, before);
+                            }
+                            if (typeof after === 'string') {
+                                index = this._getObjectKeyIndex(newObject, after);
+                                if (index >= 0) { ++index; }
+                            }
+
+                            this._setObjectKeyAtIndex(newObject, newKey, value, index);
+                            if (action !== 'copy' && (!oldObjectIsNewObject || oldKey !== newKey)) {
+                                delete oldObject[oldKey];
+                            }
+                        }
+                        break;
+                    case 'add':
+                        {
+                            const {items} = modification;
+                            const value = this._getObjectProperties(manifest, path2, path2.length);
+                            const itemsNew = items.map((v) => clone(v));
+                            value.push(...itemsNew);
+                        }
+                        break;
+                }
+            }
+        }
+
+        return manifest;
+    }
+
+    _arraysAreSame(array1, array2, lengthOffset) {
+        let ii = array1.length;
+        if (ii !== array2.length) { return false; }
+        ii += lengthOffset;
+        for (let i = 0; i < ii; ++i) {
+            if (array1[i] !== array2[i]) { return false; }
+        }
+        return true;
+    }
+
+    _getObjectKeyIndex(object, key) {
+        return Object.keys(object).indexOf(key);
+    }
+
+    _setObjectKeyAtIndex(object, key, value, index) {
+        if (index < 0 || typeof key === 'number' || Object.prototype.hasOwnProperty.call(object, key)) {
+            object[key] = value;
+            return;
+        }
+
+        const entries = Object.entries(object);
+        index = Math.min(index, entries.length);
+        for (let i = index, ii = entries.length; i < ii; ++i) {
+            const [key2] = entries[i];
+            delete object[key2];
+        }
+        entries.splice(index, 0, [key, value]);
+        for (let i = index, ii = entries.length; i < ii; ++i) {
+            const [key2, value2] = entries[i];
+            object[key2] = value2;
+        }
+    }
+
+    _getObjectProperties(object, path2, count) {
+        for (let i = 0; i < count; ++i) {
+            object = object[path2[i]];
+        }
+        return object;
+    }
+
+    _getInheritanceChain(variant) {
+        const visited = new Set();
+        const inheritance = [];
+        while (true) {
+            const {name, inherit} = variant;
+            if (visited.has(name)) { break; }
+
+            visited.add(name);
+            inheritance.unshift(variant);
+
+            if (typeof inherit !== 'string') { break; }
+
+            const nextVariant = this._variantMap.get(inherit);
+            if (typeof nextVariant === 'undefined') { break; }
+
+            variant = nextVariant;
+        }
+        return inheritance;
+    }
+
+    _createVariantManifest(manifest, variant) {
+        let modifiedManifest = clone(manifest);
+        for (const {modifications} of this._getInheritanceChain(variant)) {
+            modifiedManifest = this._applyModifications(modifiedManifest, modifications);
+        }
+        return modifiedManifest;
+    }
+}
+
+
+module.exports = {
+    ManifestUtil
+};

dev/util.js

@@ -98,21 +98,6 @@ function getAllFiles(baseDirectory, predicate=null) {
     return results;
 }
 
-function getDefaultManifest() {
-    const {manifest} = getDefaultManifestAndVariants();
-    return manifest;
-}
-
-function getDefaultManifestAndVariants() {
-    const fileName = path.join(__dirname, 'data', 'manifest-variants.json');
-    const {manifest, variants} = JSON.parse(fs.readFileSync(fileName));
-    return {manifest, variants};
-}
-
-function createManifestString(manifest) {
-    return JSON.stringify(manifest, null, 4) + '\n';
-}
-
 function createDictionaryArchive(dictionaryDirectory, dictionaryName) {
     const fileNames = fs.readdirSync(dictionaryDirectory);
 
@@ -151,9 +136,6 @@ module.exports = {
     get JSZip() { return getJSZip(); },
     getArgs,
     getAllFiles,
-    getDefaultManifest,
-    getDefaultManifestAndVariants,
-    createManifestString,
     createDictionaryArchive,
     testMain
 };

test/test-manifest.js

@@ -18,7 +18,8 @@
 const fs = require('fs');
 const path = require('path');
 const assert = require('assert');
-const {getDefaultManifest, createManifestString, testMain} = require('../dev/util');
+const {testMain} = require('../dev/util');
+const {ManifestUtil} = require('../dev/manifest-util');
 
 
 function loadManifestString() {
@@ -27,8 +28,9 @@ function loadManifestString() {
 }
 
 function validateManifest() {
+    const manifestUtil = new ManifestUtil();
     const manifest1 = loadManifestString();
-    const manifest2 = createManifestString(getDefaultManifest());
+    const manifest2 = ManifestUtil.createManifestString(manifestUtil.getManifest());
     assert.strictEqual(manifest1, manifest2, 'Manifest data does not match.');
 }