From 04d47bf8a92a71ae49c320d8d7eccd50c254348e Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sun, 9 Aug 2020 13:09:06 -0400 Subject: [PATCH] Build system changes (#721) * Refactor manifest.json * Change end-of-line convention for built plain-text files * Ignore builds directory * Mark the "dev" directory as using the node environment * Create build script * Register build scripts * Remove old build script * Fix 64x64 icon * Add test to ensure manifest data is updated properly --- .eslintrc.json | 5 +- .gitattributes | 8 +- .gitignore | 1 + build.bat | 1 + build.sh | 2 + build_zip.sh | 7 -- dev/build.js | 214 ++++++++++++++++++++++++++++++++ dev/data/manifest-variants.json | 144 +++++++++++++++++++++ ext/manifest.json | 66 +++++----- package.json | 6 +- test/test-manifest.js | 41 ++++++ 11 files changed, 454 insertions(+), 41 deletions(-) create mode 100644 build.bat create mode 100644 build.sh delete mode 100755 build_zip.sh create mode 100644 dev/build.js create mode 100644 dev/data/manifest-variants.json create mode 100644 test/test-manifest.js diff --git a/.eslintrc.json b/.eslintrc.json index 7f3cdc4b..126e1a3e 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -134,7 +134,10 @@ } }, { - "files": ["test/**/*.js"], + "files": [ + "test/**/*.js", + "dev/**/*.js" + ], "excludedFiles": ["test/data/html/*.js"], "parserOptions": { "ecmaVersion": 8, diff --git a/.gitattributes b/.gitattributes index cb9d99bb..e1f36548 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,7 @@ -*.handlebars text eol=lf +*.sh text eol=lf +ext/*.handlebars text eol=lf +ext/*.js text eol=lf +ext/*.json text eol=lf +ext/*.css text eol=lf +ext/*.html text eol=lf +ext/*.svg text eol=lf diff --git a/.gitignore b/.gitignore index 6dd55d4e..0d57bb9a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.zip node_modules +builds/ diff --git a/build.bat b/build.bat new file mode 100644 index 00000000..d5ded641 --- /dev/null +++ b/build.bat @@ -0,0 +1 @@ +@npm run-script build diff --git a/build.sh b/build.sh new file mode 100644 index 00000000..2d2c12b7 --- /dev/null +++ b/build.sh @@ -0,0 +1,2 @@ +#!/bin/bash +npm run-script build diff --git a/build_zip.sh b/build_zip.sh deleted file mode 100755 index a8e43246..00000000 --- a/build_zip.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -rm yomichan_source.zip -7za a yomichan_source.zip ./ext ./LICENSE ./README.md ./resources ./tmpl - -rm yomichan_extension.zip -7za a yomichan_extension.zip ./ext/* diff --git a/dev/build.js b/dev/build.js new file mode 100644 index 00000000..6b62083e --- /dev/null +++ b/dev/build.js @@ -0,0 +1,214 @@ +/* + * Copyright (C) 2020 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +const fs = require('fs'); +const path = require('path'); +const readline = require('readline'); +const childProcess = require('child_process'); +const yomichanTest = require('../test/yomichan-test'); + + +function getAllFiles(directory, relativeTo) { + const results = []; + const directories = [directory]; + for (const dir of directories) { + const fileNames = fs.readdirSync(dir); + for (const fileName of fileNames) { + const fullFileName = path.join(dir, fileName); + const relativeFileName = path.relative(relativeTo, fullFileName); + const stats = fs.lstatSync(fullFileName); + if (stats.isFile()) { + results.push(relativeFileName); + } else if (stats.isDirectory()) { + directories.push(fullFileName); + } + } + } + return results; +} + +async function createZip(directory, outputFileName, sevenZipExes=[], onUpdate=null) { + for (const exe of sevenZipExes) { + try { + childProcess.execFileSync( + exe, + [ + 'a', + outputFileName, + '.' + ], + { + cwd: directory + } + ); + return; + } catch (e) { + // NOP + } + } + return await createJSZip(directory, outputFileName, onUpdate); +} + +async function createJSZip(directory, outputFileName, onUpdate) { + const JSZip = yomichanTest.JSZip; + const files = getAllFiles(directory, directory); + const zip = new JSZip(); + for (const fileName of files) { + zip.file( + fileName.replace(/\\/g, '/'), + fs.readFileSync(path.join(directory, fileName), {encoding: null, flag: 'r'}), + {} + ); + } + + if (typeof onUpdate !== 'function') { + onUpdate = () => {}; // NOP + } + + const data = await zip.generateAsync({ + type: 'nodebuffer', + compression: 'DEFLATE', + compressionOptions: {level: 9} + }, onUpdate); + process.stdout.write('\n'); + + fs.writeFileSync(outputFileName, data, {encoding: null, flag: 'w'}); +} + +function createModifiedManifest(manifest, modifications) { + manifest = JSON.parse(JSON.stringify(manifest)); + + if (Array.isArray(modifications)) { + for (const modification of modifications) { + const {action, path: path2} = modification; + switch (action) { + case 'set': + { + const value = getObjectProperties(manifest, path2, path2.length - 1); + const last = path2[path2.length - 1]; + value[last] = modification.value; + } + break; + case 'replace': + { + const value = getObjectProperties(manifest, path2, path2.length - 1); + const regex = new RegExp(modification.pattern, modification.patternFlags); + const last = path2[path2.length - 1]; + let value2 = value[last]; + value2 = `${value2}`.replace(regex, modification.replacement); + value[last] = value2; + } + break; + case 'delete': + { + const value = getObjectProperties(manifest, path2, path2.length - 1); + const last = path2[path2.length - 1]; + delete value[last]; + } + break; + } + } + } + + return manifest; +} + +function getObjectProperties(object, path2, count) { + for (let i = 0; i < count; ++i) { + object = object[path2[i]]; + } + return object; +} + +function loadDefaultManifest() { + const {manifest} = loadDefaultManifestAndVariants(); + return manifest; +} + +function loadDefaultManifestAndVariants() { + const fileName = path.join(__dirname, 'data', 'manifest-variants.json'); + const {manifest, variants} = JSON.parse(fs.readFileSync(fileName)); + return {manifest, variants}; +} + +function createManifestString(manifest) { + return JSON.stringify(manifest, null, 4) + '\n'; +} + + +async function main() { + const {manifest, variants} = loadDefaultManifestAndVariants(); + + const rootDir = path.join(__dirname, '..'); + const extDir = path.join(rootDir, 'ext'); + const buildDir = path.join(rootDir, 'builds'); + const manifestPath = path.join(extDir, 'manifest.json'); + const sevenZipExes = ['7za', '7z']; + + // Create build directory + if (!fs.existsSync(buildDir)) { + fs.mkdirSync(buildDir, {recursive: true}); + } + + + const onUpdate = (metadata) => { + let message = `Progress: ${metadata.percent.toFixed(2)}%`; + if (metadata.currentFile) { + message += ` (${metadata.currentFile})`; + } + + readline.clearLine(process.stdout); + readline.cursorTo(process.stdout, 0); + process.stdout.write(message); + }; + + try { + for (const variant of variants) { + const {name, fileName, fileCopies, modifications} = variant; + process.stdout.write(`Building ${name}...\n`); + + const fileNameSafe = path.basename(fileName); + const modifiedManifest = createModifiedManifest(manifest, modifications); + const fullFileName = path.join(buildDir, fileNameSafe); + fs.writeFileSync(manifestPath, createManifestString(modifiedManifest)); + await createZip(extDir, fullFileName, sevenZipExes, onUpdate); + + if (Array.isArray(fileCopies)) { + for (const fileName2 of fileCopies) { + const fileName2Safe = path.basename(fileName2); + fs.copyFileSync(fullFileName, path.join(buildDir, fileName2Safe)); + } + } + + process.stdout.write('\n'); + } + } finally { + // Restore manifest + process.stdout.write('Restoring manifest...\n'); + fs.writeFileSync(manifestPath, createManifestString(manifest)); + } +} + + +module.exports = { + loadDefaultManifest, + loadDefaultManifestAndVariants, + createManifestString +}; + + +if (require.main === module) { main(); } diff --git a/dev/data/manifest-variants.json b/dev/data/manifest-variants.json new file mode 100644 index 00000000..bac781da --- /dev/null +++ b/dev/data/manifest-variants.json @@ -0,0 +1,144 @@ +{ + "manifest": { + "manifest_version": 2, + "name": "Yomichan", + "version": "20.8.3.0", + "description": "Japanese dictionary with Anki integration", + "author": "Alex Yatskov", + "icons": { + "16": "mixed/img/icon16.png", + "19": "mixed/img/icon19.png", + "32": "mixed/img/icon32.png", + "38": "mixed/img/icon38.png", + "48": "mixed/img/icon48.png", + "64": "mixed/img/icon64.png", + "128": "mixed/img/icon128.png" + }, + "browser_action": { + "default_icon": { + "16": "mixed/img/icon16.png", + "19": "mixed/img/icon19.png", + "32": "mixed/img/icon32.png", + "38": "mixed/img/icon38.png", + "48": "mixed/img/icon48.png", + "64": "mixed/img/icon64.png", + "128": "mixed/img/icon128.png" + }, + "default_title": "Yomichan", + "default_popup": "bg/context.html" + }, + "background": { + "page": "bg/background.html", + "persistent": true + }, + "content_scripts": [ + { + "matches": [ + "http://*/*", + "https://*/*", + "file://*/*" + ], + "js": [ + "mixed/js/core.js", + "mixed/js/yomichan.js", + "mixed/js/comm.js", + "mixed/js/dom.js", + "mixed/js/api.js", + "mixed/js/dynamic-loader.js", + "mixed/js/frame-client.js", + "mixed/js/text-scanner.js", + "fg/js/document.js", + "fg/js/dom-text-scanner.js", + "fg/js/popup.js", + "fg/js/source.js", + "fg/js/popup-factory.js", + "fg/js/frame-offset-forwarder.js", + "fg/js/popup-proxy.js", + "fg/js/frontend.js", + "fg/js/content-script-main.js" + ], + "match_about_blank": true, + "all_frames": true + } + ], + "minimum_chrome_version": "57.0.0.0", + "options_page": "bg/settings.html", + "options_ui": { + "page": "bg/settings.html", + "open_in_tab": true + }, + "permissions": [ + "", + "storage", + "clipboardWrite", + "unlimitedStorage", + "nativeMessaging", + "webRequest", + "webRequestBlocking" + ], + "optional_permissions": [ + "clipboardRead" + ], + "commands": { + "toggle": { + "suggested_key": { + "default": "Alt+Delete" + }, + "description": "Toggle text scanning" + }, + "search": { + "suggested_key": { + "default": "Alt+Insert" + }, + "description": "Open search window" + } + }, + "web_accessible_resources": [ + "fg/float.html" + ], + "content_security_policy": "script-src 'self' 'unsafe-eval'; object-src 'self'", + "applications": { + "gecko": { + "id": "alex@foosoft.net", + "strict_min_version": "55.0" + } + } + }, + "variants": [ + { + "name": "default", + "fileName": "yomichan.zip", + "fileCopies": [ + "yomichan.xpi" + ] + }, + { + "name": "dev", + "fileName": "yomichan-dev.zip", + "fileCopies": [ + "yomichan-dev.xpi" + ], + "modifications": [ + { + "action": "replace", + "path": ["name"], + "pattern": "^.*$", + "patternFlags": "", + "replacement": "$& (development build)" + }, + { + "action": "replace", + "path": ["description"], + "pattern": "^(.*)(?:\\.\\s*)?$", + "patternFlags": "", + "replacement": "$1. This is a development build; get the stable version here: https://tinyurl.com/yaatdjmp" + }, + { + "action": "set", + "path": ["applications", "gecko", "id"], + "value": "alex.testing@foosoft.net" + } + ] + } + ] +} \ No newline at end of file diff --git a/ext/manifest.json b/ext/manifest.json index 304a27c8..91516751 100644 --- a/ext/manifest.json +++ b/ext/manifest.json @@ -2,15 +2,15 @@ "manifest_version": 2, "name": "Yomichan", "version": "20.8.3.0", - "description": "Japanese dictionary with Anki integration", + "author": "Alex Yatskov", "icons": { "16": "mixed/img/icon16.png", "19": "mixed/img/icon19.png", "32": "mixed/img/icon32.png", "38": "mixed/img/icon38.png", "48": "mixed/img/icon48.png", - "64": "mixed/img/icon48.png", + "64": "mixed/img/icon64.png", "128": "mixed/img/icon128.png" }, "browser_action": { @@ -20,42 +20,46 @@ "32": "mixed/img/icon32.png", "38": "mixed/img/icon38.png", "48": "mixed/img/icon48.png", - "64": "mixed/img/icon48.png", + "64": "mixed/img/icon64.png", "128": "mixed/img/icon128.png" }, "default_title": "Yomichan", "default_popup": "bg/context.html" }, - - "author": "Alex Yatskov", "background": { "page": "bg/background.html", "persistent": true }, - "content_scripts": [{ - "matches": ["http://*/*", "https://*/*", "file://*/*"], - "js": [ - "mixed/js/core.js", - "mixed/js/yomichan.js", - "mixed/js/comm.js", - "mixed/js/dom.js", - "mixed/js/api.js", - "mixed/js/dynamic-loader.js", - "mixed/js/frame-client.js", - "mixed/js/text-scanner.js", - "fg/js/document.js", - "fg/js/dom-text-scanner.js", - "fg/js/popup.js", - "fg/js/source.js", - "fg/js/popup-factory.js", - "fg/js/frame-offset-forwarder.js", - "fg/js/popup-proxy.js", - "fg/js/frontend.js", - "fg/js/content-script-main.js" - ], - "match_about_blank": true, - "all_frames": true - }], + "content_scripts": [ + { + "matches": [ + "http://*/*", + "https://*/*", + "file://*/*" + ], + "js": [ + "mixed/js/core.js", + "mixed/js/yomichan.js", + "mixed/js/comm.js", + "mixed/js/dom.js", + "mixed/js/api.js", + "mixed/js/dynamic-loader.js", + "mixed/js/frame-client.js", + "mixed/js/text-scanner.js", + "fg/js/document.js", + "fg/js/dom-text-scanner.js", + "fg/js/popup.js", + "fg/js/source.js", + "fg/js/popup-factory.js", + "fg/js/frame-offset-forwarder.js", + "fg/js/popup-proxy.js", + "fg/js/frontend.js", + "fg/js/content-script-main.js" + ], + "match_about_blank": true, + "all_frames": true + } + ], "minimum_chrome_version": "57.0.0.0", "options_page": "bg/settings.html", "options_ui": { @@ -88,7 +92,9 @@ "description": "Open search window" } }, - "web_accessible_resources": ["fg/float.html"], + "web_accessible_resources": [ + "fg/float.html" + ], "content_security_policy": "script-src 'self' 'unsafe-eval'; object-src 'self'", "applications": { "gecko": { diff --git a/package.json b/package.json index f4f8459e..aaa7087e 100644 --- a/package.json +++ b/package.json @@ -6,9 +6,11 @@ "test": "test" }, "scripts": { - "test": "npm run test-lint && npm run test-code", + "build": "node ./dev/build.js", + "test": "npm run test-lint && npm run test-code && npm run test-manifest", "test-lint": "eslint . && node ./test/lint/global-declarations.js", - "test-code": "node ./test/test-schema.js && node ./test/test-dictionary.js && node ./test/test-database.js && node ./test/test-document.js && node ./test/test-object-property-accessor.js && node ./test/test-japanese.js && node ./test/test-text-source-map.js && node ./test/test-dom-text-scanner.js" + "test-code": "node ./test/test-schema.js && node ./test/test-dictionary.js && node ./test/test-database.js && node ./test/test-document.js && node ./test/test-object-property-accessor.js && node ./test/test-japanese.js && node ./test/test-text-source-map.js && node ./test/test-dom-text-scanner.js", + "test-manifest": "node ./test/test-manifest.js" }, "repository": { "type": "git", diff --git a/test/test-manifest.js b/test/test-manifest.js new file mode 100644 index 00000000..cb86ce7b --- /dev/null +++ b/test/test-manifest.js @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2020 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +const fs = require('fs'); +const path = require('path'); +const assert = require('assert'); +const {loadDefaultManifest, createManifestString} = require('../dev/build'); + + +function loadManifestString() { + const manifestPath = path.join(__dirname, '..', 'ext', 'manifest.json'); + return fs.readFileSync(manifestPath, {encoding: 'utf8'}); +} + +function validateManifest() { + const manifest1 = loadManifestString(); + const manifest2 = createManifestString(loadDefaultManifest()); + assert.strictEqual(manifest1, manifest2, 'Manifest data does not match.'); +} + + +function main() { + validateManifest(); +} + + +if (require.main === module) { main(); }