DictionaryImporterThreaded (#1865)

* Create new classes for importing dictionaries from a separate thread

* Use threaded importer

* Update worker tests

.eslintrc.json

@@ -221,6 +221,25 @@
                 "crypto": "readonly"
             }
         },
+        {
+            "files": [
+                "ext/js/core.js",
+                "ext/js/data/database.js",
+                "ext/js/data/json-schema.js",
+                "ext/js/general/cache-map.js",
+                "ext/js/language/dictionary-database.js",
+                "ext/js/language/dictionary-importer.js",
+                "ext/js/language/dictionary-importer-worker.js",
+                "ext/js/language/dictionary-importer-worker-media-loader.js",
+                "ext/js/media/media-util.js"
+            ],
+            "env": {
+                "browser": false,
+                "worker": true,
+                "es2017": true,
+                "webextensions": true
+            }
+        },
         {
             "files": [
                 "ext/js/**/*.js"

ext/js/language/dictionary-importer-threaded.js

@@ -0,0 +1,85 @@
+/*
+ * Copyright (C) 2021  Yomichan Authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/* global
+ * DictionaryImporterMediaLoader
+ */
+
+class DictionaryImporterThreaded {
+    importDictionary(archiveContent, details, onProgress) {
+        return new Promise((resolve, reject) => {
+            const dictionaryImporterMediaLoader = new DictionaryImporterMediaLoader();
+            const worker = new Worker('/js/language/dictionary-importer-worker-main.js', {});
+            const onMessage = (e) => {
+                const {action, params} = e.data;
+                switch (action) {
+                    case 'complete':
+                        worker.removeEventListener('message', onMessage);
+                        worker.terminate();
+                        this._onComplete(params, resolve, reject);
+                        break;
+                    case 'progress':
+                        this._onProgress(params, onProgress);
+                        break;
+                    case 'getImageResolution':
+                        this._onGetImageResolution(params, worker, dictionaryImporterMediaLoader);
+                        break;
+                }
+            };
+            worker.addEventListener('message', onMessage);
+            worker.postMessage({
+                action: 'import',
+                params: {details, archiveContent}
+            }, [archiveContent]);
+        });
+    }
+
+    // Private
+
+    _onComplete(params, resolve, reject) {
+        const {error} = params;
+        if (typeof error !== 'undefined') {
+            reject(deserializeError(error));
+        } else {
+            resolve(this._formatResult(params.result));
+        }
+    }
+
+    _formatResult(data) {
+        const {result, errors} = data;
+        const errors2 = errors.map((error) => deserializeError(error));
+        return {result, errors: errors2};
+    }
+
+    _onProgress(params, onProgress) {
+        if (typeof onProgress !== 'function') { return; }
+        const {args} = params;
+        onProgress(...args);
+    }
+
+    async _onGetImageResolution(params, worker, dictionaryImporterMediaLoader) {
+        const {id, mediaType, content} = params;
+        let response;
+        try {
+            const result = await dictionaryImporterMediaLoader.getImageResolution(mediaType, content);
+            response = {id, result};
+        } catch (e) {
+            response = {id, error: serializeError(e)};
+        }
+        worker.postMessage({action: 'getImageResolution.response', params: response});
+    }
+}

ext/js/language/dictionary-importer-worker-main.js

@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 2021  Yomichan Authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/* global
+ * DictionaryImporterWorker
+ */
+
+self.importScripts(
+    '/lib/jszip.min.js',
+    '/js/core.js',
+    '/js/data/database.js',
+    '/js/data/json-schema.js',
+    '/js/general/cache-map.js',
+    '/js/language/dictionary-database.js',
+    '/js/language/dictionary-importer.js',
+    '/js/language/dictionary-importer-worker.js',
+    '/js/language/dictionary-importer-worker-media-loader.js',
+    '/js/media/media-util.js'
+);
+
+(() => {
+    try {
+        const dictionaryImporterWorker = new DictionaryImporterWorker();
+        dictionaryImporterWorker.prepare();
+    } catch (e) {
+        log.error(e);
+    }
+})();

ext/js/language/dictionary-importer-worker-media-loader.js

@@ -0,0 +1,65 @@
+/*
+ * Copyright (C) 2021  Yomichan Authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * Class used for loading and validating media from a worker thread
+ * during the dictionary import process.
+ */
+class DictionaryImporterWorkerMediaLoader {
+    /**
+     * Creates a new instance of the media loader.
+     */
+    constructor() {
+        this._requests = new Map();
+    }
+
+    /**
+     * Handles a response message posted to the worker thread.
+     * @param params Details of the response.
+     */
+    handleMessage(params) {
+        const {id} = params;
+        const request = this._requests.get(id);
+        if (typeof request === 'undefined') { return; }
+        this._requests.delete(id);
+        const {error} = params;
+        if (typeof error !== 'undefined') {
+            request.reject(deserializeError(error));
+        } else {
+            request.resolve(params.result);
+        }
+    }
+
+    /**
+     * Attempts to load an image using a base64 encoded content and a media type
+     * and returns its resolution.
+     * @param mediaType The media type for the image content.
+     * @param content The binary content for the image, encoded in base64.
+     * @returns A Promise which resolves with {width, height} on success,
+     *   otherwise an error is thrown.
+     */
+    getImageResolution(mediaType, content) {
+        return new Promise((resolve, reject) => {
+            const id = generateId(16);
+            this._requests.set(id, {resolve, reject});
+            self.postMessage({
+                action: 'getImageResolution',
+                params: {id, mediaType, content}
+            });
+        });
+    }
+}

ext/js/language/dictionary-importer-worker.js

@@ -0,0 +1,83 @@
+/*
+ * Copyright (C) 2021  Yomichan Authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/* global
+ * DictionaryDatabase
+ * DictionaryImporter
+ * DictionaryImporterWorkerMediaLoader
+ */
+
+class DictionaryImporterWorker {
+    constructor() {
+        this._mediaLoader = new DictionaryImporterWorkerMediaLoader();
+    }
+
+    prepare() {
+        self.addEventListener('message', this._onMessage.bind(this), false);
+    }
+
+    // Private
+
+    _onMessage(e) {
+        const {action, params} = e.data;
+        switch (action) {
+            case 'import':
+                this._onImport(params);
+                break;
+            case 'getImageResolution.response':
+                this._mediaLoader.handleMessage(params);
+                break;
+        }
+    }
+
+    async _onImport({details, archiveContent}) {
+        const onProgress = (...args) => {
+            self.postMessage({
+                action: 'progress',
+                params: {args}
+            });
+        };
+        let response;
+        try {
+            const result = await this._importDictionary(archiveContent, details, onProgress);
+            response = {result};
+        } catch (e) {
+            response = {error: serializeError(e)};
+        }
+        self.postMessage({action: 'complete', params: response});
+    }
+
+    async _importDictionary(archiveContent, importDetails, onProgress) {
+        const dictionaryDatabase = await this._getPreparedDictionaryDatabase();
+        try {
+            const dictionaryImporter = new DictionaryImporter(this._mediaLoader);
+            const {result, errors} = await dictionaryImporter.importDictionary(dictionaryDatabase, archiveContent, importDetails, onProgress);
+            return {
+                result,
+                errors: errors.map((error) => serializeError(error))
+            };
+        } finally {
+            dictionaryDatabase.close();
+        }
+    }
+
+    async _getPreparedDictionaryDatabase() {
+        const dictionaryDatabase = new DictionaryDatabase();
+        await dictionaryDatabase.prepare();
+        return dictionaryDatabase;
+    }
+}

ext/js/pages/settings/dictionary-import-controller.js

@@ -17,9 +17,7 @@
 
 /* global
  * DictionaryController
- * DictionaryDatabase
- * DictionaryImporter
- * DictionaryImporterMediaLoader
+ * DictionaryImporterThreaded
  */
 
 class DictionaryImportController {
@@ -183,12 +181,9 @@ class DictionaryImportController {
     }
 
     async _importDictionary(file, importDetails, onProgress) {
-        const dictionaryDatabase = await this._getPreparedDictionaryDatabase();
-        try {
-            const dictionaryImporterMediaLoader = new DictionaryImporterMediaLoader();
-            const dictionaryImporter = new DictionaryImporter(dictionaryImporterMediaLoader);
+        const dictionaryImporter = new DictionaryImporterThreaded();
         const archiveContent = await this._readFile(file);
-            const {result, errors} = await dictionaryImporter.importDictionary(dictionaryDatabase, archiveContent, importDetails, onProgress);
+        const {result, errors} = await dictionaryImporter.importDictionary(archiveContent, importDetails, onProgress);
         yomichan.api.triggerDatabaseUpdated('dictionary', 'import');
         const errors2 = await this._addDictionarySettings(result.sequenced, result.title);
 
@@ -197,9 +192,6 @@ class DictionaryImportController {
             allErrors.push(new Error(`Dictionary may not have been imported properly: ${allErrors.length} error${allErrors.length === 1 ? '' : 's'} reported.`));
             this._showErrors(allErrors);
         }
-        } finally {
-            dictionaryDatabase.close();
-        }
     }
 
     async _addDictionarySettings(sequenced, title) {
@@ -311,12 +303,6 @@ class DictionaryImportController {
         }
     }
 
-    async _getPreparedDictionaryDatabase() {
-        const dictionaryDatabase = new DictionaryDatabase();
-        await dictionaryDatabase.prepare();
-        return dictionaryDatabase;
-    }
-
     async _modifyGlobalSettings(targets) {
         const results = await this._settingsController.modifyGlobalSettings(targets);
         const errors = [];

ext/settings.html

@@ -3421,7 +3421,6 @@
 
 
 <!-- Scripts -->
-<script src="/lib/jszip.min.js"></script>
 <script src="/lib/wanakana.min.js"></script>
 
 <script src="/js/core.js"></script>
@@ -3451,8 +3450,8 @@
 <script src="/js/general/task-accumulator.js"></script>
 <script src="/js/input/hotkey-util.js"></script>
 <script src="/js/language/dictionary-database.js"></script>
-<script src="/js/language/dictionary-importer.js"></script>
 <script src="/js/language/dictionary-importer-media-loader.js"></script>
+<script src="/js/language/dictionary-importer-threaded.js"></script>
 <script src="/js/language/sandbox/dictionary-data-util.js"></script>
 <script src="/js/language/sandbox/japanese-util.js"></script>
 <script src="/js/media/audio-system.js"></script>

ext/welcome.html

@@ -384,8 +384,6 @@
 
 
 <!-- Scripts -->
-<script src="/lib/jszip.min.js"></script>
-
 <script src="/js/core.js"></script>
 
 <script src="/js/yomichan.js"></script>
@@ -408,8 +406,8 @@
 <script src="/js/general/task-accumulator.js"></script>
 <script src="/js/input/hotkey-util.js"></script>
 <script src="/js/language/dictionary-database.js"></script>
-<script src="/js/language/dictionary-importer.js"></script>
 <script src="/js/language/dictionary-importer-media-loader.js"></script>
+<script src="/js/language/dictionary-importer-threaded.js"></script>
 <script src="/js/media/media-util.js"></script>
 <script src="/js/pages/settings/dictionary-controller.js"></script>
 <script src="/js/pages/settings/dictionary-import-controller.js"></script>

test/test-workers.js

@@ -22,6 +22,13 @@ const {VM} = require('../dev/vm');
 const assert = require('assert');
 
 
+class StubClass {
+    prepare() {
+        // NOP
+    }
+}
+
+
 function loadEslint() {
     return JSON.parse(fs.readFileSync(path.join(__dirname, '..', '.eslintrc.json'), {encoding: 'utf8'}));
 }
@@ -87,9 +94,36 @@ function testServiceWorker() {
     assert.deepStrictEqual(swRules.files, expectedSwRulesFiles);
 }
 
+function testWorkers() {
+    testWorker(
+        'js/language/dictionary-importer-worker-main.js',
+        {DictionaryImporterWorker: StubClass}
+    );
+}
+
+function testWorker(scriptPath, fields) {
+    // Get script paths
+    const scripts = getImportedScripts(scriptPath, fields);
+
+    // Verify that eslint config lists files correctly
+    const expectedRulesFiles = filterScriptPaths(scripts);
+    const expectedRulesFilesSet = new Set(expectedRulesFiles);
+    const eslintConfig = loadEslint();
+    const rules = eslintConfig.overrides.find((item) => (
+        typeof item.env === 'object' &&
+        item.env !== null &&
+        item.env.worker === true
+    ));
+    assert.ok(typeof rules !== 'undefined');
+    assert.ok(Array.isArray(rules.files));
+    assert.deepStrictEqual(rules.files.filter((v) => expectedRulesFilesSet.has(v)), expectedRulesFiles);
+}
+
+
 function main() {
     try {
         testServiceWorker();
+        testWorkers();
     } catch (e) {
         console.error(e);
         process.exit(-1);