From 8c4a50f68c9543c14cfe76abd2f6f42135b0e13d Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sat, 31 Jul 2021 14:46:09 -0400 Subject: [PATCH] DictionaryImporterThreaded (#1865) * Create new classes for importing dictionaries from a separate thread * Use threaded importer * Update worker tests --- .eslintrc.json | 19 +++++ .../language/dictionary-importer-threaded.js | 85 +++++++++++++++++++ .../dictionary-importer-worker-main.js | 42 +++++++++ ...dictionary-importer-worker-media-loader.js | 65 ++++++++++++++ ext/js/language/dictionary-importer-worker.js | 83 ++++++++++++++++++ .../settings/dictionary-import-controller.js | 34 +++----- ext/settings.html | 3 +- ext/welcome.html | 4 +- test/test-workers.js | 34 ++++++++ 9 files changed, 340 insertions(+), 29 deletions(-) create mode 100644 ext/js/language/dictionary-importer-threaded.js create mode 100644 ext/js/language/dictionary-importer-worker-main.js create mode 100644 ext/js/language/dictionary-importer-worker-media-loader.js create mode 100644 ext/js/language/dictionary-importer-worker.js diff --git a/.eslintrc.json b/.eslintrc.json index 15bb96ea..4ec8f0f6 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -221,6 +221,25 @@ "crypto": "readonly" } }, + { + "files": [ + "ext/js/core.js", + "ext/js/data/database.js", + "ext/js/data/json-schema.js", + "ext/js/general/cache-map.js", + "ext/js/language/dictionary-database.js", + "ext/js/language/dictionary-importer.js", + "ext/js/language/dictionary-importer-worker.js", + "ext/js/language/dictionary-importer-worker-media-loader.js", + "ext/js/media/media-util.js" + ], + "env": { + "browser": false, + "worker": true, + "es2017": true, + "webextensions": true + } + }, { "files": [ "ext/js/**/*.js" diff --git a/ext/js/language/dictionary-importer-threaded.js b/ext/js/language/dictionary-importer-threaded.js new file mode 100644 index 00000000..a251906b --- /dev/null +++ b/ext/js/language/dictionary-importer-threaded.js @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2021 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* global + * DictionaryImporterMediaLoader + */ + +class DictionaryImporterThreaded { + importDictionary(archiveContent, details, onProgress) { + return new Promise((resolve, reject) => { + const dictionaryImporterMediaLoader = new DictionaryImporterMediaLoader(); + const worker = new Worker('/js/language/dictionary-importer-worker-main.js', {}); + const onMessage = (e) => { + const {action, params} = e.data; + switch (action) { + case 'complete': + worker.removeEventListener('message', onMessage); + worker.terminate(); + this._onComplete(params, resolve, reject); + break; + case 'progress': + this._onProgress(params, onProgress); + break; + case 'getImageResolution': + this._onGetImageResolution(params, worker, dictionaryImporterMediaLoader); + break; + } + }; + worker.addEventListener('message', onMessage); + worker.postMessage({ + action: 'import', + params: {details, archiveContent} + }, [archiveContent]); + }); + } + + // Private + + _onComplete(params, resolve, reject) { + const {error} = params; + if (typeof error !== 'undefined') { + reject(deserializeError(error)); + } else { + resolve(this._formatResult(params.result)); + } + } + + _formatResult(data) { + const {result, errors} = data; + const errors2 = errors.map((error) => deserializeError(error)); + return {result, errors: errors2}; + } + + _onProgress(params, onProgress) { + if (typeof onProgress !== 'function') { return; } + const {args} = params; + onProgress(...args); + } + + async _onGetImageResolution(params, worker, dictionaryImporterMediaLoader) { + const {id, mediaType, content} = params; + let response; + try { + const result = await dictionaryImporterMediaLoader.getImageResolution(mediaType, content); + response = {id, result}; + } catch (e) { + response = {id, error: serializeError(e)}; + } + worker.postMessage({action: 'getImageResolution.response', params: response}); + } +} diff --git a/ext/js/language/dictionary-importer-worker-main.js b/ext/js/language/dictionary-importer-worker-main.js new file mode 100644 index 00000000..100bb4fb --- /dev/null +++ b/ext/js/language/dictionary-importer-worker-main.js @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2021 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* global + * DictionaryImporterWorker + */ + +self.importScripts( + '/lib/jszip.min.js', + '/js/core.js', + '/js/data/database.js', + '/js/data/json-schema.js', + '/js/general/cache-map.js', + '/js/language/dictionary-database.js', + '/js/language/dictionary-importer.js', + '/js/language/dictionary-importer-worker.js', + '/js/language/dictionary-importer-worker-media-loader.js', + '/js/media/media-util.js' +); + +(() => { + try { + const dictionaryImporterWorker = new DictionaryImporterWorker(); + dictionaryImporterWorker.prepare(); + } catch (e) { + log.error(e); + } +})(); diff --git a/ext/js/language/dictionary-importer-worker-media-loader.js b/ext/js/language/dictionary-importer-worker-media-loader.js new file mode 100644 index 00000000..5d5d3593 --- /dev/null +++ b/ext/js/language/dictionary-importer-worker-media-loader.js @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2021 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/** + * Class used for loading and validating media from a worker thread + * during the dictionary import process. + */ +class DictionaryImporterWorkerMediaLoader { + /** + * Creates a new instance of the media loader. + */ + constructor() { + this._requests = new Map(); + } + + /** + * Handles a response message posted to the worker thread. + * @param params Details of the response. + */ + handleMessage(params) { + const {id} = params; + const request = this._requests.get(id); + if (typeof request === 'undefined') { return; } + this._requests.delete(id); + const {error} = params; + if (typeof error !== 'undefined') { + request.reject(deserializeError(error)); + } else { + request.resolve(params.result); + } + } + + /** + * Attempts to load an image using a base64 encoded content and a media type + * and returns its resolution. + * @param mediaType The media type for the image content. + * @param content The binary content for the image, encoded in base64. + * @returns A Promise which resolves with {width, height} on success, + * otherwise an error is thrown. + */ + getImageResolution(mediaType, content) { + return new Promise((resolve, reject) => { + const id = generateId(16); + this._requests.set(id, {resolve, reject}); + self.postMessage({ + action: 'getImageResolution', + params: {id, mediaType, content} + }); + }); + } +} diff --git a/ext/js/language/dictionary-importer-worker.js b/ext/js/language/dictionary-importer-worker.js new file mode 100644 index 00000000..f44a10f9 --- /dev/null +++ b/ext/js/language/dictionary-importer-worker.js @@ -0,0 +1,83 @@ +/* + * Copyright (C) 2021 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* global + * DictionaryDatabase + * DictionaryImporter + * DictionaryImporterWorkerMediaLoader + */ + +class DictionaryImporterWorker { + constructor() { + this._mediaLoader = new DictionaryImporterWorkerMediaLoader(); + } + + prepare() { + self.addEventListener('message', this._onMessage.bind(this), false); + } + + // Private + + _onMessage(e) { + const {action, params} = e.data; + switch (action) { + case 'import': + this._onImport(params); + break; + case 'getImageResolution.response': + this._mediaLoader.handleMessage(params); + break; + } + } + + async _onImport({details, archiveContent}) { + const onProgress = (...args) => { + self.postMessage({ + action: 'progress', + params: {args} + }); + }; + let response; + try { + const result = await this._importDictionary(archiveContent, details, onProgress); + response = {result}; + } catch (e) { + response = {error: serializeError(e)}; + } + self.postMessage({action: 'complete', params: response}); + } + + async _importDictionary(archiveContent, importDetails, onProgress) { + const dictionaryDatabase = await this._getPreparedDictionaryDatabase(); + try { + const dictionaryImporter = new DictionaryImporter(this._mediaLoader); + const {result, errors} = await dictionaryImporter.importDictionary(dictionaryDatabase, archiveContent, importDetails, onProgress); + return { + result, + errors: errors.map((error) => serializeError(error)) + }; + } finally { + dictionaryDatabase.close(); + } + } + + async _getPreparedDictionaryDatabase() { + const dictionaryDatabase = new DictionaryDatabase(); + await dictionaryDatabase.prepare(); + return dictionaryDatabase; + } +} diff --git a/ext/js/pages/settings/dictionary-import-controller.js b/ext/js/pages/settings/dictionary-import-controller.js index 128e18cb..5e51a48a 100644 --- a/ext/js/pages/settings/dictionary-import-controller.js +++ b/ext/js/pages/settings/dictionary-import-controller.js @@ -17,9 +17,7 @@ /* global * DictionaryController - * DictionaryDatabase - * DictionaryImporter - * DictionaryImporterMediaLoader + * DictionaryImporterThreaded */ class DictionaryImportController { @@ -183,22 +181,16 @@ class DictionaryImportController { } async _importDictionary(file, importDetails, onProgress) { - const dictionaryDatabase = await this._getPreparedDictionaryDatabase(); - try { - const dictionaryImporterMediaLoader = new DictionaryImporterMediaLoader(); - const dictionaryImporter = new DictionaryImporter(dictionaryImporterMediaLoader); - const archiveContent = await this._readFile(file); - const {result, errors} = await dictionaryImporter.importDictionary(dictionaryDatabase, archiveContent, importDetails, onProgress); - yomichan.api.triggerDatabaseUpdated('dictionary', 'import'); - const errors2 = await this._addDictionarySettings(result.sequenced, result.title); + const dictionaryImporter = new DictionaryImporterThreaded(); + const archiveContent = await this._readFile(file); + const {result, errors} = await dictionaryImporter.importDictionary(archiveContent, importDetails, onProgress); + yomichan.api.triggerDatabaseUpdated('dictionary', 'import'); + const errors2 = await this._addDictionarySettings(result.sequenced, result.title); - if (errors.length > 0) { - const allErrors = [...errors, ...errors2]; - allErrors.push(new Error(`Dictionary may not have been imported properly: ${allErrors.length} error${allErrors.length === 1 ? '' : 's'} reported.`)); - this._showErrors(allErrors); - } - } finally { - dictionaryDatabase.close(); + if (errors.length > 0) { + const allErrors = [...errors, ...errors2]; + allErrors.push(new Error(`Dictionary may not have been imported properly: ${allErrors.length} error${allErrors.length === 1 ? '' : 's'} reported.`)); + this._showErrors(allErrors); } } @@ -311,12 +303,6 @@ class DictionaryImportController { } } - async _getPreparedDictionaryDatabase() { - const dictionaryDatabase = new DictionaryDatabase(); - await dictionaryDatabase.prepare(); - return dictionaryDatabase; - } - async _modifyGlobalSettings(targets) { const results = await this._settingsController.modifyGlobalSettings(targets); const errors = []; diff --git a/ext/settings.html b/ext/settings.html index 3260810c..cd9231c1 100644 --- a/ext/settings.html +++ b/ext/settings.html @@ -3421,7 +3421,6 @@ - @@ -3451,8 +3450,8 @@ - + diff --git a/ext/welcome.html b/ext/welcome.html index 5c9f4469..da17ee2c 100644 --- a/ext/welcome.html +++ b/ext/welcome.html @@ -384,8 +384,6 @@ - - @@ -408,8 +406,8 @@ - + diff --git a/test/test-workers.js b/test/test-workers.js index 9f9c4d13..6cc77823 100644 --- a/test/test-workers.js +++ b/test/test-workers.js @@ -22,6 +22,13 @@ const {VM} = require('../dev/vm'); const assert = require('assert'); +class StubClass { + prepare() { + // NOP + } +} + + function loadEslint() { return JSON.parse(fs.readFileSync(path.join(__dirname, '..', '.eslintrc.json'), {encoding: 'utf8'})); } @@ -87,9 +94,36 @@ function testServiceWorker() { assert.deepStrictEqual(swRules.files, expectedSwRulesFiles); } +function testWorkers() { + testWorker( + 'js/language/dictionary-importer-worker-main.js', + {DictionaryImporterWorker: StubClass} + ); +} + +function testWorker(scriptPath, fields) { + // Get script paths + const scripts = getImportedScripts(scriptPath, fields); + + // Verify that eslint config lists files correctly + const expectedRulesFiles = filterScriptPaths(scripts); + const expectedRulesFilesSet = new Set(expectedRulesFiles); + const eslintConfig = loadEslint(); + const rules = eslintConfig.overrides.find((item) => ( + typeof item.env === 'object' && + item.env !== null && + item.env.worker === true + )); + assert.ok(typeof rules !== 'undefined'); + assert.ok(Array.isArray(rules.files)); + assert.deepStrictEqual(rules.files.filter((v) => expectedRulesFilesSet.has(v)), expectedRulesFiles); +} + + function main() { try { testServiceWorker(); + testWorkers(); } catch (e) { console.error(e); process.exit(-1);