Update build script (#1722)

* Add support for making non-buildable variants

* Add support for running a command to assign a value

* Update chrome-dev inheritance

* Add base variant

* Update manifest to auto-fill the version based on most recent git tag

* Add support for changing the default manifest

* Change the default manifest

* Move some manifest utility functions into manifest-util.js

* Move more manifest functionality into ManifestUtil

* Revert "Update manifest to auto-fill the version based on most recent git tag"

This reverts commit 2a66e40ff24f9dc545783503bcf3404f21148356.
This commit is contained in:
toasted-nutbread 2021-05-31 13:24:40 -04:00 committed by GitHub
parent 6da81d59c3
commit 003cf791b1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 297 additions and 223 deletions

View File

@ -21,13 +21,10 @@ const assert = require('assert');
const readline = require('readline');
const childProcess = require('child_process');
const util = require('./util');
const {getAllFiles, getDefaultManifestAndVariants, createManifestString, getArgs, testMain} = util;
const {getAllFiles, getArgs, testMain} = util;
const {ManifestUtil} = require('./manifest-util');
function clone(value) {
return JSON.parse(JSON.stringify(value));
}
async function createZip(directory, excludeFiles, outputFileName, sevenZipExes, onUpdate, dryRun) {
try {
fs.unlinkSync(outputFileName);
@ -110,179 +107,7 @@ function getIndexOfFilePath(array, item) {
return -1;
}
function applyModifications(manifest, modifications) {
if (Array.isArray(modifications)) {
for (const modification of modifications) {
const {action, path: path2} = modification;
switch (action) {
case 'set':
{
const {value, before, after} = modification;
const object = getObjectProperties(manifest, path2, path2.length - 1);
const key = path2[path2.length - 1];
let {index} = modification;
if (typeof index !== 'number') {
index = -1;
}
if (typeof before === 'string') {
index = getObjectKeyIndex(object, before);
}
if (typeof after === 'string') {
index = getObjectKeyIndex(object, after);
if (index >= 0) { ++index; }
}
setObjectKeyAtIndex(object, key, value, index);
}
break;
case 'replace':
{
const {pattern, patternFlags, replacement} = modification;
const value = getObjectProperties(manifest, path2, path2.length - 1);
const regex = new RegExp(pattern, patternFlags);
const last = path2[path2.length - 1];
let value2 = value[last];
value2 = `${value2}`.replace(regex, replacement);
value[last] = value2;
}
break;
case 'delete':
{
const value = getObjectProperties(manifest, path2, path2.length - 1);
const last = path2[path2.length - 1];
delete value[last];
}
break;
case 'remove':
{
const {item} = modification;
const value = getObjectProperties(manifest, path2, path2.length);
const index = value.indexOf(item);
if (index >= 0) { value.splice(index, 1); }
}
break;
case 'splice':
{
const {start, deleteCount, items} = modification;
const value = getObjectProperties(manifest, path2, path2.length);
const itemsNew = items.map((v) => clone(v));
value.splice(start, deleteCount, ...itemsNew);
}
break;
case 'copy':
case 'move':
{
const {newPath, before, after} = modification;
const oldKey = path2[path2.length - 1];
const newKey = newPath[newPath.length - 1];
const oldObject = getObjectProperties(manifest, path2, path2.length - 1);
const newObject = getObjectProperties(manifest, newPath, newPath.length - 1);
const oldObjectIsNewObject = arraysAreSame(path2, newPath, -1);
const value = oldObject[oldKey];
let {index} = modification;
if (typeof index !== 'number' || index < 0) {
index = (oldObjectIsNewObject && action !== 'copy') ? getObjectKeyIndex(oldObject, oldKey) : -1;
}
if (typeof before === 'string') {
index = getObjectKeyIndex(newObject, before);
}
if (typeof after === 'string') {
index = getObjectKeyIndex(newObject, after);
if (index >= 0) { ++index; }
}
setObjectKeyAtIndex(newObject, newKey, value, index);
if (action !== 'copy' && (!oldObjectIsNewObject || oldKey !== newKey)) {
delete oldObject[oldKey];
}
}
break;
case 'add':
{
const {items} = modification;
const value = getObjectProperties(manifest, path2, path2.length);
const itemsNew = items.map((v) => clone(v));
value.push(...itemsNew);
}
break;
}
}
}
return manifest;
}
function arraysAreSame(array1, array2, lengthOffset) {
let ii = array1.length;
if (ii !== array2.length) { return false; }
ii += lengthOffset;
for (let i = 0; i < ii; ++i) {
if (array1[i] !== array2[i]) { return false; }
}
return true;
}
function getObjectKeyIndex(object, key) {
return Object.keys(object).indexOf(key);
}
function setObjectKeyAtIndex(object, key, value, index) {
if (index < 0 || typeof key === 'number' || Object.prototype.hasOwnProperty.call(object, key)) {
object[key] = value;
return;
}
const entries = Object.entries(object);
index = Math.min(index, entries.length);
for (let i = index, ii = entries.length; i < ii; ++i) {
const [key2] = entries[i];
delete object[key2];
}
entries.splice(index, 0, [key, value]);
for (let i = index, ii = entries.length; i < ii; ++i) {
const [key2, value2] = entries[i];
object[key2] = value2;
}
}
function getObjectProperties(object, path2, count) {
for (let i = 0; i < count; ++i) {
object = object[path2[i]];
}
return object;
}
function getInheritanceChain(variant, variantMap) {
const visited = new Set();
const inheritance = [];
while (true) {
const {name, inherit} = variant;
if (visited.has(name)) { break; }
visited.add(name);
inheritance.unshift(variant);
if (typeof inherit !== 'string') { break; }
const nextVariant = variantMap.get(inherit);
if (typeof nextVariant === 'undefined') { break; }
variant = nextVariant;
}
return inheritance;
}
function createVariantManifest(manifest, variant, variantMap) {
let modifiedManifest = clone(manifest);
for (const {modifications} of getInheritanceChain(variant, variantMap)) {
modifiedManifest = applyModifications(modifiedManifest, modifications);
}
return modifiedManifest;
}
async function build(manifest, buildDir, extDir, manifestPath, variantMap, variantNames, dryRun, dryRunBuildZip) {
async function build(buildDir, extDir, manifestUtil, variantNames, manifestPath, dryRun, dryRunBuildZip) {
const sevenZipExes = ['7za', '7z'];
// Create build directory
@ -305,8 +130,8 @@ async function build(manifest, buildDir, extDir, manifestPath, variantMap, varia
};
for (const variantName of variantNames) {
const variant = variantMap.get(variantName);
if (typeof variant === 'undefined') { continue; }
const variant = manifestUtil.getVariant(variantName);
if (typeof variant === 'undefined' || variant.buildable === false) { continue; }
const {name, fileName, fileCopies} = variant;
let {excludeFiles} = variant;
@ -314,13 +139,13 @@ async function build(manifest, buildDir, extDir, manifestPath, variantMap, varia
process.stdout.write(`Building ${name}...\n`);
const modifiedManifest = createVariantManifest(manifest, variant, variantMap);
const modifiedManifest = manifestUtil.getManifest(variant.name);
const fileNameSafe = path.basename(fileName);
const fullFileName = path.join(buildDir, fileNameSafe);
ensureFilesExist(extDir, excludeFiles);
if (!dryRun) {
fs.writeFileSync(manifestPath, createManifestString(modifiedManifest));
fs.writeFileSync(manifestPath, ManifestUtil.createManifestString(modifiedManifest));
}
if (!dryRun || dryRunBuildZip) {
@ -360,33 +185,27 @@ async function main(argv) {
const dryRun = args.get('dry-run');
const dryRunBuildZip = args.get('dry-run-build-zip');
const {manifest, variants} = getDefaultManifestAndVariants();
const manifestUtil = new ManifestUtil();
const rootDir = path.join(__dirname, '..');
const extDir = path.join(rootDir, 'ext');
const buildDir = path.join(rootDir, 'builds');
const manifestPath = path.join(extDir, 'manifest.json');
const variantMap = new Map();
for (const variant of variants) {
variantMap.set(variant.name, variant);
}
try {
const variantNames = (argv.length === 0 || args.get('all') ? variants.map(({name}) => name) : args.get(null));
await build(manifest, buildDir, extDir, manifestPath, variantMap, variantNames, dryRun, dryRunBuildZip);
const variantNames = (
argv.length === 0 || args.get('all') ?
manifestUtil.getVariants().filter(({buildable}) => buildable !== false).map(({name}) => name) :
args.get(null)
);
await build(buildDir, extDir, manifestUtil, variantNames, manifestPath, dryRun, dryRunBuildZip);
} finally {
// Restore manifest
let restoreManifest = manifest;
if (!args.get('default') && args.get('manifest') !== null) {
const variant = variantMap.get(args.get('manifest'));
if (typeof variant !== 'undefined') {
restoreManifest = createVariantManifest(manifest, variant, variantMap);
}
}
const manifestName = (!args.get('default') && args.get('manifest') !== null) ? args.get('manifest') : null;
const restoreManifest = manifestUtil.getManifest(manifestName);
process.stdout.write('Restoring manifest...\n');
if (!dryRun) {
fs.writeFileSync(manifestPath, createManifestString(restoreManifest));
fs.writeFileSync(manifestPath, ManifestUtil.createManifestString(restoreManifest));
}
}
}

View File

@ -116,9 +116,15 @@
],
"content_security_policy": "default-src 'self'; img-src blob: 'self'; style-src 'self' 'unsafe-inline'; media-src *; connect-src *"
},
"defaultVariant": "base",
"variants": [
{
"name": "base",
"buildable": false
},
{
"name": "chrome",
"inherit": "base",
"fileName": "yomichan-chrome.zip",
"excludeFiles": [
"sw.js",
@ -128,6 +134,7 @@
},
{
"name": "chrome-dev",
"inherit": "chrome",
"fileName": "yomichan-chrome-dev.zip",
"modifications": [
{
@ -144,15 +151,11 @@
"patternFlags": "",
"replacement": "$1. This is a development build; get the stable version here: https://tinyurl.com/yaatdjmp"
}
],
"excludeFiles": [
"sw.js",
"js/dom/simple-dom-parser.js",
"lib/parse5.js"
]
},
{
"name": "chrome-mv3",
"inherit": "base",
"fileName": "yomichan-chrome-mv3.zip",
"modifications": [
{"action": "set", "path": ["manifest_version"], "value": 3},
@ -180,6 +183,7 @@
},
{
"name": "firefox",
"inherit": "base",
"fileName": "yomichan-firefox.xpi",
"modifications": [
{

267
dev/manifest-util.js Normal file
View File

@ -0,0 +1,267 @@
/*
* Copyright (C) 2021 Yomichan Authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const fs = require('fs');
const path = require('path');
const childProcess = require('child_process');
function clone(value) {
return JSON.parse(JSON.stringify(value));
}
class ManifestUtil {
constructor() {
const fileName = path.join(__dirname, 'data', 'manifest-variants.json');
const {manifest, variants, defaultVariant} = JSON.parse(fs.readFileSync(fileName));
this._manifest = manifest;
this._variants = variants;
this._defaultVariant = defaultVariant;
this._variantMap = new Map();
for (const variant of variants) {
this._variantMap.set(variant.name, variant);
}
}
getManifest(variantName) {
if (typeof variantName === 'string') {
const variant = this._variantMap.get(variantName);
if (typeof variant !== 'undefined') {
return this._createVariantManifest(this._manifest, variant);
}
}
if (typeof this._defaultVariant === 'string') {
const variant = this._variantMap.get(this._defaultVariant);
if (typeof variant !== 'undefined') {
return this._createVariantManifest(this._manifest, variant);
}
}
return clone(this._manifest);
}
getVariants() {
return [...this._variants];
}
getVariant(name) {
return this._variantMap.get(name);
}
static createManifestString(manifest) {
return JSON.stringify(manifest, null, 4) + '\n';
}
// Private
_evaluateModificationCommand(data) {
const {command, args, trim} = data;
const {stdout, status} = childProcess.spawnSync(command, args, {
cwd: __dirname,
stdio: 'pipe',
shell: false
});
if (status !== 0) {
throw new Error(`Failed to execute ${command} ${args.join(' ')}`);
}
let result = stdout.toString('utf8');
if (trim) { result = result.trim(); }
return result;
}
_applyModifications(manifest, modifications) {
if (Array.isArray(modifications)) {
for (const modification of modifications) {
const {action, path: path2} = modification;
switch (action) {
case 'set':
{
let {value, before, after, command} = modification;
const object = this._getObjectProperties(manifest, path2, path2.length - 1);
const key = path2[path2.length - 1];
let {index} = modification;
if (typeof index !== 'number') {
index = -1;
}
if (typeof before === 'string') {
index = this._getObjectKeyIndex(object, before);
}
if (typeof after === 'string') {
index = this._getObjectKeyIndex(object, after);
if (index >= 0) { ++index; }
}
if (typeof command === 'object' && command !== null) {
value = this._evaluateModificationCommand(command);
}
this._setObjectKeyAtIndex(object, key, value, index);
}
break;
case 'replace':
{
const {pattern, patternFlags, replacement} = modification;
const value = this._getObjectProperties(manifest, path2, path2.length - 1);
const regex = new RegExp(pattern, patternFlags);
const last = path2[path2.length - 1];
let value2 = value[last];
value2 = `${value2}`.replace(regex, replacement);
value[last] = value2;
}
break;
case 'delete':
{
const value = this._getObjectProperties(manifest, path2, path2.length - 1);
const last = path2[path2.length - 1];
delete value[last];
}
break;
case 'remove':
{
const {item} = modification;
const value = this._getObjectProperties(manifest, path2, path2.length);
const index = value.indexOf(item);
if (index >= 0) { value.splice(index, 1); }
}
break;
case 'splice':
{
const {start, deleteCount, items} = modification;
const value = this._getObjectProperties(manifest, path2, path2.length);
const itemsNew = items.map((v) => clone(v));
value.splice(start, deleteCount, ...itemsNew);
}
break;
case 'copy':
case 'move':
{
const {newPath, before, after} = modification;
const oldKey = path2[path2.length - 1];
const newKey = newPath[newPath.length - 1];
const oldObject = this._getObjectProperties(manifest, path2, path2.length - 1);
const newObject = this._getObjectProperties(manifest, newPath, newPath.length - 1);
const oldObjectIsNewObject = this._arraysAreSame(path2, newPath, -1);
const value = oldObject[oldKey];
let {index} = modification;
if (typeof index !== 'number' || index < 0) {
index = (oldObjectIsNewObject && action !== 'copy') ? this._getObjectKeyIndex(oldObject, oldKey) : -1;
}
if (typeof before === 'string') {
index = this._getObjectKeyIndex(newObject, before);
}
if (typeof after === 'string') {
index = this._getObjectKeyIndex(newObject, after);
if (index >= 0) { ++index; }
}
this._setObjectKeyAtIndex(newObject, newKey, value, index);
if (action !== 'copy' && (!oldObjectIsNewObject || oldKey !== newKey)) {
delete oldObject[oldKey];
}
}
break;
case 'add':
{
const {items} = modification;
const value = this._getObjectProperties(manifest, path2, path2.length);
const itemsNew = items.map((v) => clone(v));
value.push(...itemsNew);
}
break;
}
}
}
return manifest;
}
_arraysAreSame(array1, array2, lengthOffset) {
let ii = array1.length;
if (ii !== array2.length) { return false; }
ii += lengthOffset;
for (let i = 0; i < ii; ++i) {
if (array1[i] !== array2[i]) { return false; }
}
return true;
}
_getObjectKeyIndex(object, key) {
return Object.keys(object).indexOf(key);
}
_setObjectKeyAtIndex(object, key, value, index) {
if (index < 0 || typeof key === 'number' || Object.prototype.hasOwnProperty.call(object, key)) {
object[key] = value;
return;
}
const entries = Object.entries(object);
index = Math.min(index, entries.length);
for (let i = index, ii = entries.length; i < ii; ++i) {
const [key2] = entries[i];
delete object[key2];
}
entries.splice(index, 0, [key, value]);
for (let i = index, ii = entries.length; i < ii; ++i) {
const [key2, value2] = entries[i];
object[key2] = value2;
}
}
_getObjectProperties(object, path2, count) {
for (let i = 0; i < count; ++i) {
object = object[path2[i]];
}
return object;
}
_getInheritanceChain(variant) {
const visited = new Set();
const inheritance = [];
while (true) {
const {name, inherit} = variant;
if (visited.has(name)) { break; }
visited.add(name);
inheritance.unshift(variant);
if (typeof inherit !== 'string') { break; }
const nextVariant = this._variantMap.get(inherit);
if (typeof nextVariant === 'undefined') { break; }
variant = nextVariant;
}
return inheritance;
}
_createVariantManifest(manifest, variant) {
let modifiedManifest = clone(manifest);
for (const {modifications} of this._getInheritanceChain(variant)) {
modifiedManifest = this._applyModifications(modifiedManifest, modifications);
}
return modifiedManifest;
}
}
module.exports = {
ManifestUtil
};

View File

@ -98,21 +98,6 @@ function getAllFiles(baseDirectory, predicate=null) {
return results;
}
function getDefaultManifest() {
const {manifest} = getDefaultManifestAndVariants();
return manifest;
}
function getDefaultManifestAndVariants() {
const fileName = path.join(__dirname, 'data', 'manifest-variants.json');
const {manifest, variants} = JSON.parse(fs.readFileSync(fileName));
return {manifest, variants};
}
function createManifestString(manifest) {
return JSON.stringify(manifest, null, 4) + '\n';
}
function createDictionaryArchive(dictionaryDirectory, dictionaryName) {
const fileNames = fs.readdirSync(dictionaryDirectory);
@ -151,9 +136,6 @@ module.exports = {
get JSZip() { return getJSZip(); },
getArgs,
getAllFiles,
getDefaultManifest,
getDefaultManifestAndVariants,
createManifestString,
createDictionaryArchive,
testMain
};

View File

@ -18,7 +18,8 @@
const fs = require('fs');
const path = require('path');
const assert = require('assert');
const {getDefaultManifest, createManifestString, testMain} = require('../dev/util');
const {testMain} = require('../dev/util');
const {ManifestUtil} = require('../dev/manifest-util');
function loadManifestString() {
@ -27,8 +28,9 @@ function loadManifestString() {
}
function validateManifest() {
const manifestUtil = new ManifestUtil();
const manifest1 = loadManifestString();
const manifest2 = createManifestString(getDefaultManifest());
const manifest2 = ManifestUtil.createManifestString(manifestUtil.getManifest());
assert.strictEqual(manifest1, manifest2, 'Manifest data does not match.');
}