alex/yomichan
1
Fork 0
Code Releases Activity

Dictionary importer refactoring (#1867)

Browse Source 
* Remove map of schemas

* Don't reuse dictionary importer instances

* Refactor

* Update how progress callback is used
This commit is contained in:
toasted-nutbread 2021-07-31 18:08:51 -04:00 committed by GitHub
parent 1d6e437fb2
commit 01c5c5c04b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 38 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
dev/translator-vm.js
View File

@ -76,15 +76,14 @@ class TranslatorVM extends DatabaseVM {
// Setup database
const dictionaryImporterMediaLoader = new DatabaseVMDictionaryImporterMediaLoader();
const dictionaryImporter = new DictionaryImporter(dictionaryImporterMediaLoader);
const dictionaryImporter = new DictionaryImporter(dictionaryImporterMediaLoader, null);
const dictionaryDatabase = new DictionaryDatabase();
await dictionaryDatabase.prepare();
const {errors} = await dictionaryImporter.importDictionary(
dictionaryDatabase,
testDictionaryContent,
{prefixWildcardsSupported: true},
() => {}
{prefixWildcardsSupported: true}
);
assert.deepStrictEqual(errors.length, 0);

34
ext/js/language/dictionary-importer-threaded.js
View File

@ -20,7 +20,11 @@
*/
class DictionaryImporterThreaded {
importDictionary(archiveContent, details, onProgress) {
constructor(onProgress) {
this._onProgress = onProgress;
}
importDictionary(archiveContent, details) {
return new Promise((resolve, reject) => {
const dictionaryImporterMediaLoader = new DictionaryImporterMediaLoader();
const worker = new Worker('/js/language/dictionary-importer-worker-main.js', {});
@ -30,13 +34,13 @@ class DictionaryImporterThreaded {
case 'complete':
worker.removeEventListener('message', onMessage);
worker.terminate();
this._onComplete(params, resolve, reject);
this._onMessageComplete(params, resolve, reject);
break;
case 'progress':
this._onProgress(params, onProgress);
this._onMessageProgress(params);
break;
case 'getImageResolution':
this._onGetImageResolution(params, worker, dictionaryImporterMediaLoader);
this._onMessageGetImageResolution(params, worker, dictionaryImporterMediaLoader);
break;
}
};
@ -50,7 +54,7 @@ class DictionaryImporterThreaded {
// Private
_onComplete(params, resolve, reject) {
_onMessageComplete(params, resolve, reject) {
const {error} = params;
if (typeof error !== 'undefined') {
reject(deserializeError(error));
@ -59,19 +63,13 @@ class DictionaryImporterThreaded {
}
}
_formatResult(data) {
const {result, errors} = data;
const errors2 = errors.map((error) => deserializeError(error));
return {result, errors: errors2};
}
_onProgress(params, onProgress) {
if (typeof onProgress !== 'function') { return; }
_onMessageProgress(params) {
if (typeof this._onProgress !== 'function') { return; }
const {args} = params;
onProgress(...args);
this._onProgress(...args);
}
async _onGetImageResolution(params, worker, dictionaryImporterMediaLoader) {
async _onMessageGetImageResolution(params, worker, dictionaryImporterMediaLoader) {
const {id, mediaType, content} = params;
let response;
try {
@ -82,4 +80,10 @@ class DictionaryImporterThreaded {
}
worker.postMessage({action: 'getImageResolution.response', params: response});
}
_formatResult(data) {
const {result, errors} = data;
const errors2 = errors.map((error) => deserializeError(error));
return {result, errors: errors2};
}
}

4
ext/js/language/dictionary-importer-worker.js
View File

@ -64,8 +64,8 @@ class DictionaryImporterWorker {
async _importDictionary(archiveContent, importDetails, onProgress) {
const dictionaryDatabase = await this._getPreparedDictionaryDatabase();
try {
const dictionaryImporter = new DictionaryImporter(this._mediaLoader);
const {result, errors} = await dictionaryImporter.importDictionary(dictionaryDatabase, archiveContent, importDetails, onProgress);
const dictionaryImporter = new DictionaryImporter(this._mediaLoader, onProgress);
const {result, errors} = await dictionaryImporter.importDictionary(dictionaryDatabase, archiveContent, importDetails);
return {
result,
errors: errors.map((error) => serializeError(error))

21
ext/js/language/dictionary-importer.js
View File

@ -22,12 +22,12 @@
*/
class DictionaryImporter {
constructor(mediaLoader) {
constructor(mediaLoader, onProgress) {
this._mediaLoader = mediaLoader;
this._schemas = new Map();
this._onProgress = onProgress;
}
async importDictionary(dictionaryDatabase, archiveContent, details, onProgress) {
async importDictionary(dictionaryDatabase, archiveContent, details) {
if (!dictionaryDatabase) {
throw new Error('Invalid database');
}
@ -35,7 +35,7 @@ class DictionaryImporter {
throw new Error('Database is not ready');
}
const hasOnProgress = (typeof onProgress === 'function');
const hasOnProgress = (typeof this._onProgress === 'function');
// Read archive
const archive = await JSZip.loadAsync(archiveContent);
@ -143,7 +143,7 @@ class DictionaryImporter {
loadedCount += count;
if (hasOnProgress) {
onProgress(total, loadedCount);
this._onProgress(total, loadedCount);
}
}
};
@ -178,17 +178,6 @@ class DictionaryImporter {
}
async _getSchema(fileName) {
let schemaPromise = this._schemas.get(fileName);
if (typeof schemaPromise !== 'undefined') {
return schemaPromise;
}
schemaPromise = this._createSchema(fileName);
this._schemas.set(fileName, schemaPromise);
return schemaPromise;
}
async _createSchema(fileName) {
const schema = await this._fetchJsonAsset(fileName);
return new JsonSchema(schema);
}

4
ext/js/pages/settings/dictionary-import-controller.js
View File

@ -181,9 +181,9 @@ class DictionaryImportController {
}
async _importDictionary(file, importDetails, onProgress) {
const dictionaryImporter = new DictionaryImporterThreaded();
const dictionaryImporter = new DictionaryImporterThreaded(onProgress);
const archiveContent = await this._readFile(file);
const {result, errors} = await dictionaryImporter.importDictionary(archiveContent, importDetails, onProgress);
const {result, errors} = await dictionaryImporter.importDictionary(archiveContent, importDetails);
yomichan.api.triggerDatabaseUpdated('dictionary', 'import');
const errors2 = await this._addDictionarySettings(result.sequenced, result.title);

21
test/test-database.js
View File

@ -41,9 +41,9 @@ function createTestDictionaryArchive(dictionary, dictionaryName) {
}
function createDictionaryImporter() {
function createDictionaryImporter(onProgress) {
const dictionaryImporterMediaLoader = new DatabaseVMDictionaryImporterMediaLoader();
return new DictionaryImporter(dictionaryImporterMediaLoader);
return new DictionaryImporter(dictionaryImporterMediaLoader, onProgress);
}
@ -131,7 +131,6 @@ async function testDatabase1() {
];
// Setup database
const dictionaryImporter = createDictionaryImporter();
const dictionaryDatabase = new DictionaryDatabase();
await dictionaryDatabase.prepare();
@ -146,13 +145,11 @@ async function testDatabase1() {
// Import data
let progressEvent = false;
const dictionaryImporter = createDictionaryImporter(() => { progressEvent = true; });
const {result, errors} = await dictionaryImporter.importDictionary(
dictionaryDatabase,
testDictionarySource,
{prefixWildcardsSupported: true},
() => {
progressEvent = true;
}
{prefixWildcardsSupported: true}
);
vm.assert.deepStrictEqual(errors, []);
vm.assert.deepStrictEqual(result, expectedSummary);
@ -781,7 +778,6 @@ async function testDatabase2() {
]);
// Setup database
const dictionaryImporter = createDictionaryImporter();
const dictionaryDatabase = new DictionaryDatabase();
// Error: not prepared
@ -796,17 +792,17 @@ async function testDatabase2() {
await assert.rejects(async () => await dictionaryDatabase.findTagForTitle('tag', title));
await assert.rejects(async () => await dictionaryDatabase.getDictionaryInfo());
await assert.rejects(async () => await dictionaryDatabase.getDictionaryCounts(titles, true));
await assert.rejects(async () => await dictionaryImporter.importDictionary(dictionaryDatabase, testDictionarySource, {}, () => {}));
await assert.rejects(async () => await createDictionaryImporter().importDictionary(dictionaryDatabase, testDictionarySource, {}));
await dictionaryDatabase.prepare();
// Error: already prepared
await assert.rejects(async () => await dictionaryDatabase.prepare());
await dictionaryImporter.importDictionary(dictionaryDatabase, testDictionarySource, {}, () => {});
await createDictionaryImporter().importDictionary(dictionaryDatabase, testDictionarySource, {});
// Error: dictionary already imported
await assert.rejects(async () => await dictionaryImporter.importDictionary(dictionaryDatabase, testDictionarySource, {}, () => {}));
await assert.rejects(async () => await createDictionaryImporter().importDictionary(dictionaryDatabase, testDictionarySource, {}));
await dictionaryDatabase.close();
}
@ -823,7 +819,6 @@ async function testDatabase3() {
];
// Setup database
const dictionaryImporter = createDictionaryImporter();
const dictionaryDatabase = new DictionaryDatabase();
await dictionaryDatabase.prepare();
@ -833,7 +828,7 @@ async function testDatabase3() {
let error = null;
try {
await dictionaryImporter.importDictionary(dictionaryDatabase, testDictionarySource, {}, () => {});
await createDictionaryImporter().importDictionary(dictionaryDatabase, testDictionarySource, {});
} catch (e) {
error = e;
}