DictionaryImporterThreaded (#1865)
* Create new classes for importing dictionaries from a separate thread * Use threaded importer * Update worker tests
This commit is contained in:
parent
992c8bcf75
commit
8c4a50f68c
@ -221,6 +221,25 @@
|
||||
"crypto": "readonly"
|
||||
}
|
||||
},
|
||||
{
|
||||
"files": [
|
||||
"ext/js/core.js",
|
||||
"ext/js/data/database.js",
|
||||
"ext/js/data/json-schema.js",
|
||||
"ext/js/general/cache-map.js",
|
||||
"ext/js/language/dictionary-database.js",
|
||||
"ext/js/language/dictionary-importer.js",
|
||||
"ext/js/language/dictionary-importer-worker.js",
|
||||
"ext/js/language/dictionary-importer-worker-media-loader.js",
|
||||
"ext/js/media/media-util.js"
|
||||
],
|
||||
"env": {
|
||||
"browser": false,
|
||||
"worker": true,
|
||||
"es2017": true,
|
||||
"webextensions": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"files": [
|
||||
"ext/js/**/*.js"
|
||||
|
85
ext/js/language/dictionary-importer-threaded.js
Normal file
85
ext/js/language/dictionary-importer-threaded.js
Normal file
@ -0,0 +1,85 @@
|
||||
/*
|
||||
* Copyright (C) 2021 Yomichan Authors
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
/* global
|
||||
* DictionaryImporterMediaLoader
|
||||
*/
|
||||
|
||||
class DictionaryImporterThreaded {
|
||||
importDictionary(archiveContent, details, onProgress) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const dictionaryImporterMediaLoader = new DictionaryImporterMediaLoader();
|
||||
const worker = new Worker('/js/language/dictionary-importer-worker-main.js', {});
|
||||
const onMessage = (e) => {
|
||||
const {action, params} = e.data;
|
||||
switch (action) {
|
||||
case 'complete':
|
||||
worker.removeEventListener('message', onMessage);
|
||||
worker.terminate();
|
||||
this._onComplete(params, resolve, reject);
|
||||
break;
|
||||
case 'progress':
|
||||
this._onProgress(params, onProgress);
|
||||
break;
|
||||
case 'getImageResolution':
|
||||
this._onGetImageResolution(params, worker, dictionaryImporterMediaLoader);
|
||||
break;
|
||||
}
|
||||
};
|
||||
worker.addEventListener('message', onMessage);
|
||||
worker.postMessage({
|
||||
action: 'import',
|
||||
params: {details, archiveContent}
|
||||
}, [archiveContent]);
|
||||
});
|
||||
}
|
||||
|
||||
// Private
|
||||
|
||||
_onComplete(params, resolve, reject) {
|
||||
const {error} = params;
|
||||
if (typeof error !== 'undefined') {
|
||||
reject(deserializeError(error));
|
||||
} else {
|
||||
resolve(this._formatResult(params.result));
|
||||
}
|
||||
}
|
||||
|
||||
_formatResult(data) {
|
||||
const {result, errors} = data;
|
||||
const errors2 = errors.map((error) => deserializeError(error));
|
||||
return {result, errors: errors2};
|
||||
}
|
||||
|
||||
_onProgress(params, onProgress) {
|
||||
if (typeof onProgress !== 'function') { return; }
|
||||
const {args} = params;
|
||||
onProgress(...args);
|
||||
}
|
||||
|
||||
async _onGetImageResolution(params, worker, dictionaryImporterMediaLoader) {
|
||||
const {id, mediaType, content} = params;
|
||||
let response;
|
||||
try {
|
||||
const result = await dictionaryImporterMediaLoader.getImageResolution(mediaType, content);
|
||||
response = {id, result};
|
||||
} catch (e) {
|
||||
response = {id, error: serializeError(e)};
|
||||
}
|
||||
worker.postMessage({action: 'getImageResolution.response', params: response});
|
||||
}
|
||||
}
|
42
ext/js/language/dictionary-importer-worker-main.js
Normal file
42
ext/js/language/dictionary-importer-worker-main.js
Normal file
@ -0,0 +1,42 @@
|
||||
/*
|
||||
* Copyright (C) 2021 Yomichan Authors
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
/* global
|
||||
* DictionaryImporterWorker
|
||||
*/
|
||||
|
||||
self.importScripts(
|
||||
'/lib/jszip.min.js',
|
||||
'/js/core.js',
|
||||
'/js/data/database.js',
|
||||
'/js/data/json-schema.js',
|
||||
'/js/general/cache-map.js',
|
||||
'/js/language/dictionary-database.js',
|
||||
'/js/language/dictionary-importer.js',
|
||||
'/js/language/dictionary-importer-worker.js',
|
||||
'/js/language/dictionary-importer-worker-media-loader.js',
|
||||
'/js/media/media-util.js'
|
||||
);
|
||||
|
||||
(() => {
|
||||
try {
|
||||
const dictionaryImporterWorker = new DictionaryImporterWorker();
|
||||
dictionaryImporterWorker.prepare();
|
||||
} catch (e) {
|
||||
log.error(e);
|
||||
}
|
||||
})();
|
65
ext/js/language/dictionary-importer-worker-media-loader.js
Normal file
65
ext/js/language/dictionary-importer-worker-media-loader.js
Normal file
@ -0,0 +1,65 @@
|
||||
/*
|
||||
* Copyright (C) 2021 Yomichan Authors
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Class used for loading and validating media from a worker thread
|
||||
* during the dictionary import process.
|
||||
*/
|
||||
class DictionaryImporterWorkerMediaLoader {
|
||||
/**
|
||||
* Creates a new instance of the media loader.
|
||||
*/
|
||||
constructor() {
|
||||
this._requests = new Map();
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles a response message posted to the worker thread.
|
||||
* @param params Details of the response.
|
||||
*/
|
||||
handleMessage(params) {
|
||||
const {id} = params;
|
||||
const request = this._requests.get(id);
|
||||
if (typeof request === 'undefined') { return; }
|
||||
this._requests.delete(id);
|
||||
const {error} = params;
|
||||
if (typeof error !== 'undefined') {
|
||||
request.reject(deserializeError(error));
|
||||
} else {
|
||||
request.resolve(params.result);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempts to load an image using a base64 encoded content and a media type
|
||||
* and returns its resolution.
|
||||
* @param mediaType The media type for the image content.
|
||||
* @param content The binary content for the image, encoded in base64.
|
||||
* @returns A Promise which resolves with {width, height} on success,
|
||||
* otherwise an error is thrown.
|
||||
*/
|
||||
getImageResolution(mediaType, content) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const id = generateId(16);
|
||||
this._requests.set(id, {resolve, reject});
|
||||
self.postMessage({
|
||||
action: 'getImageResolution',
|
||||
params: {id, mediaType, content}
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
83
ext/js/language/dictionary-importer-worker.js
Normal file
83
ext/js/language/dictionary-importer-worker.js
Normal file
@ -0,0 +1,83 @@
|
||||
/*
|
||||
* Copyright (C) 2021 Yomichan Authors
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
/* global
|
||||
* DictionaryDatabase
|
||||
* DictionaryImporter
|
||||
* DictionaryImporterWorkerMediaLoader
|
||||
*/
|
||||
|
||||
class DictionaryImporterWorker {
|
||||
constructor() {
|
||||
this._mediaLoader = new DictionaryImporterWorkerMediaLoader();
|
||||
}
|
||||
|
||||
prepare() {
|
||||
self.addEventListener('message', this._onMessage.bind(this), false);
|
||||
}
|
||||
|
||||
// Private
|
||||
|
||||
_onMessage(e) {
|
||||
const {action, params} = e.data;
|
||||
switch (action) {
|
||||
case 'import':
|
||||
this._onImport(params);
|
||||
break;
|
||||
case 'getImageResolution.response':
|
||||
this._mediaLoader.handleMessage(params);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
async _onImport({details, archiveContent}) {
|
||||
const onProgress = (...args) => {
|
||||
self.postMessage({
|
||||
action: 'progress',
|
||||
params: {args}
|
||||
});
|
||||
};
|
||||
let response;
|
||||
try {
|
||||
const result = await this._importDictionary(archiveContent, details, onProgress);
|
||||
response = {result};
|
||||
} catch (e) {
|
||||
response = {error: serializeError(e)};
|
||||
}
|
||||
self.postMessage({action: 'complete', params: response});
|
||||
}
|
||||
|
||||
async _importDictionary(archiveContent, importDetails, onProgress) {
|
||||
const dictionaryDatabase = await this._getPreparedDictionaryDatabase();
|
||||
try {
|
||||
const dictionaryImporter = new DictionaryImporter(this._mediaLoader);
|
||||
const {result, errors} = await dictionaryImporter.importDictionary(dictionaryDatabase, archiveContent, importDetails, onProgress);
|
||||
return {
|
||||
result,
|
||||
errors: errors.map((error) => serializeError(error))
|
||||
};
|
||||
} finally {
|
||||
dictionaryDatabase.close();
|
||||
}
|
||||
}
|
||||
|
||||
async _getPreparedDictionaryDatabase() {
|
||||
const dictionaryDatabase = new DictionaryDatabase();
|
||||
await dictionaryDatabase.prepare();
|
||||
return dictionaryDatabase;
|
||||
}
|
||||
}
|
@ -17,9 +17,7 @@
|
||||
|
||||
/* global
|
||||
* DictionaryController
|
||||
* DictionaryDatabase
|
||||
* DictionaryImporter
|
||||
* DictionaryImporterMediaLoader
|
||||
* DictionaryImporterThreaded
|
||||
*/
|
||||
|
||||
class DictionaryImportController {
|
||||
@ -183,12 +181,9 @@ class DictionaryImportController {
|
||||
}
|
||||
|
||||
async _importDictionary(file, importDetails, onProgress) {
|
||||
const dictionaryDatabase = await this._getPreparedDictionaryDatabase();
|
||||
try {
|
||||
const dictionaryImporterMediaLoader = new DictionaryImporterMediaLoader();
|
||||
const dictionaryImporter = new DictionaryImporter(dictionaryImporterMediaLoader);
|
||||
const dictionaryImporter = new DictionaryImporterThreaded();
|
||||
const archiveContent = await this._readFile(file);
|
||||
const {result, errors} = await dictionaryImporter.importDictionary(dictionaryDatabase, archiveContent, importDetails, onProgress);
|
||||
const {result, errors} = await dictionaryImporter.importDictionary(archiveContent, importDetails, onProgress);
|
||||
yomichan.api.triggerDatabaseUpdated('dictionary', 'import');
|
||||
const errors2 = await this._addDictionarySettings(result.sequenced, result.title);
|
||||
|
||||
@ -197,9 +192,6 @@ class DictionaryImportController {
|
||||
allErrors.push(new Error(`Dictionary may not have been imported properly: ${allErrors.length} error${allErrors.length === 1 ? '' : 's'} reported.`));
|
||||
this._showErrors(allErrors);
|
||||
}
|
||||
} finally {
|
||||
dictionaryDatabase.close();
|
||||
}
|
||||
}
|
||||
|
||||
async _addDictionarySettings(sequenced, title) {
|
||||
@ -311,12 +303,6 @@ class DictionaryImportController {
|
||||
}
|
||||
}
|
||||
|
||||
async _getPreparedDictionaryDatabase() {
|
||||
const dictionaryDatabase = new DictionaryDatabase();
|
||||
await dictionaryDatabase.prepare();
|
||||
return dictionaryDatabase;
|
||||
}
|
||||
|
||||
async _modifyGlobalSettings(targets) {
|
||||
const results = await this._settingsController.modifyGlobalSettings(targets);
|
||||
const errors = [];
|
||||
|
@ -3421,7 +3421,6 @@
|
||||
|
||||
|
||||
<!-- Scripts -->
|
||||
<script src="/lib/jszip.min.js"></script>
|
||||
<script src="/lib/wanakana.min.js"></script>
|
||||
|
||||
<script src="/js/core.js"></script>
|
||||
@ -3451,8 +3450,8 @@
|
||||
<script src="/js/general/task-accumulator.js"></script>
|
||||
<script src="/js/input/hotkey-util.js"></script>
|
||||
<script src="/js/language/dictionary-database.js"></script>
|
||||
<script src="/js/language/dictionary-importer.js"></script>
|
||||
<script src="/js/language/dictionary-importer-media-loader.js"></script>
|
||||
<script src="/js/language/dictionary-importer-threaded.js"></script>
|
||||
<script src="/js/language/sandbox/dictionary-data-util.js"></script>
|
||||
<script src="/js/language/sandbox/japanese-util.js"></script>
|
||||
<script src="/js/media/audio-system.js"></script>
|
||||
|
@ -384,8 +384,6 @@
|
||||
|
||||
|
||||
<!-- Scripts -->
|
||||
<script src="/lib/jszip.min.js"></script>
|
||||
|
||||
<script src="/js/core.js"></script>
|
||||
|
||||
<script src="/js/yomichan.js"></script>
|
||||
@ -408,8 +406,8 @@
|
||||
<script src="/js/general/task-accumulator.js"></script>
|
||||
<script src="/js/input/hotkey-util.js"></script>
|
||||
<script src="/js/language/dictionary-database.js"></script>
|
||||
<script src="/js/language/dictionary-importer.js"></script>
|
||||
<script src="/js/language/dictionary-importer-media-loader.js"></script>
|
||||
<script src="/js/language/dictionary-importer-threaded.js"></script>
|
||||
<script src="/js/media/media-util.js"></script>
|
||||
<script src="/js/pages/settings/dictionary-controller.js"></script>
|
||||
<script src="/js/pages/settings/dictionary-import-controller.js"></script>
|
||||
|
@ -22,6 +22,13 @@ const {VM} = require('../dev/vm');
|
||||
const assert = require('assert');
|
||||
|
||||
|
||||
class StubClass {
|
||||
prepare() {
|
||||
// NOP
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function loadEslint() {
|
||||
return JSON.parse(fs.readFileSync(path.join(__dirname, '..', '.eslintrc.json'), {encoding: 'utf8'}));
|
||||
}
|
||||
@ -87,9 +94,36 @@ function testServiceWorker() {
|
||||
assert.deepStrictEqual(swRules.files, expectedSwRulesFiles);
|
||||
}
|
||||
|
||||
function testWorkers() {
|
||||
testWorker(
|
||||
'js/language/dictionary-importer-worker-main.js',
|
||||
{DictionaryImporterWorker: StubClass}
|
||||
);
|
||||
}
|
||||
|
||||
function testWorker(scriptPath, fields) {
|
||||
// Get script paths
|
||||
const scripts = getImportedScripts(scriptPath, fields);
|
||||
|
||||
// Verify that eslint config lists files correctly
|
||||
const expectedRulesFiles = filterScriptPaths(scripts);
|
||||
const expectedRulesFilesSet = new Set(expectedRulesFiles);
|
||||
const eslintConfig = loadEslint();
|
||||
const rules = eslintConfig.overrides.find((item) => (
|
||||
typeof item.env === 'object' &&
|
||||
item.env !== null &&
|
||||
item.env.worker === true
|
||||
));
|
||||
assert.ok(typeof rules !== 'undefined');
|
||||
assert.ok(Array.isArray(rules.files));
|
||||
assert.deepStrictEqual(rules.files.filter((v) => expectedRulesFilesSet.has(v)), expectedRulesFiles);
|
||||
}
|
||||
|
||||
|
||||
function main() {
|
||||
try {
|
||||
testServiceWorker();
|
||||
testWorkers();
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
process.exit(-1);
|
||||
|
Loading…
Reference in New Issue
Block a user