alex/yomichan
1
Fork 0
Code Releases Activity

Dictionary importer refactoring (#1867)

Browse Source 
* Remove map of schemas

* Don't reuse dictionary importer instances

* Refactor

* Update how progress callback is used
This commit is contained in:
toasted-nutbread 2021-07-31 18:08:51 -04:00 committed by GitHub
parent 1d6e437fb2
commit 01c5c5c04b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 38 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
dev/translator-vm.js
View File

@ -76,15 +76,14 @@ class TranslatorVM extends DatabaseVM {
// Setup database // Setup database
const dictionaryImporterMediaLoader = new DatabaseVMDictionaryImporterMediaLoader(); const dictionaryImporterMediaLoader = new DatabaseVMDictionaryImporterMediaLoader();
const dictionaryImporter = new DictionaryImporter(dictionaryImporterMediaLoader); const dictionaryImporter = new DictionaryImporter(dictionaryImporterMediaLoader, null);
const dictionaryDatabase = new DictionaryDatabase(); const dictionaryDatabase = new DictionaryDatabase();
await dictionaryDatabase.prepare(); await dictionaryDatabase.prepare();
const {errors} = await dictionaryImporter.importDictionary( const {errors} = await dictionaryImporter.importDictionary(
dictionaryDatabase, dictionaryDatabase,
testDictionaryContent, testDictionaryContent,
{prefixWildcardsSupported: true}, {prefixWildcardsSupported: true}
() => {}
); );
assert.deepStrictEqual(errors.length, 0); assert.deepStrictEqual(errors.length, 0);

34
ext/js/language/dictionary-importer-threaded.js
View File

@ -20,7 +20,11 @@
*/ */
class DictionaryImporterThreaded { class DictionaryImporterThreaded {
importDictionary(archiveContent, details, onProgress) { constructor(onProgress) {
this._onProgress = onProgress;
}
importDictionary(archiveContent, details) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const dictionaryImporterMediaLoader = new DictionaryImporterMediaLoader(); const dictionaryImporterMediaLoader = new DictionaryImporterMediaLoader();
const worker = new Worker('/js/language/dictionary-importer-worker-main.js', {}); const worker = new Worker('/js/language/dictionary-importer-worker-main.js', {});
@ -30,13 +34,13 @@ class DictionaryImporterThreaded {
case 'complete': case 'complete':
worker.removeEventListener('message', onMessage); worker.removeEventListener('message', onMessage);
worker.terminate(); worker.terminate();
this._onComplete(params, resolve, reject); this._onMessageComplete(params, resolve, reject);
break; break;
case 'progress': case 'progress':
this._onProgress(params, onProgress); this._onMessageProgress(params);
break; break;
case 'getImageResolution': case 'getImageResolution':
this._onGetImageResolution(params, worker, dictionaryImporterMediaLoader); this._onMessageGetImageResolution(params, worker, dictionaryImporterMediaLoader);
break; break;
} }
}; };
@ -50,7 +54,7 @@ class DictionaryImporterThreaded {
// Private // Private
_onComplete(params, resolve, reject) { _onMessageComplete(params, resolve, reject) {
const {error} = params; const {error} = params;
if (typeof error !== 'undefined') { if (typeof error !== 'undefined') {
reject(deserializeError(error)); reject(deserializeError(error));
@ -59,19 +63,13 @@ class DictionaryImporterThreaded {
} }
} }
_formatResult(data) { _onMessageProgress(params) {
const {result, errors} = data; if (typeof this._onProgress !== 'function') { return; }
const errors2 = errors.map((error) => deserializeError(error));
return {result, errors: errors2};
}
_onProgress(params, onProgress) {
if (typeof onProgress !== 'function') { return; }
const {args} = params; const {args} = params;
onProgress(...args); this._onProgress(...args);
} }
async _onGetImageResolution(params, worker, dictionaryImporterMediaLoader) { async _onMessageGetImageResolution(params, worker, dictionaryImporterMediaLoader) {
const {id, mediaType, content} = params; const {id, mediaType, content} = params;
let response; let response;
try { try {
@ -82,4 +80,10 @@ class DictionaryImporterThreaded {
} }
worker.postMessage({action: 'getImageResolution.response', params: response}); worker.postMessage({action: 'getImageResolution.response', params: response});
} }
_formatResult(data) {
const {result, errors} = data;
const errors2 = errors.map((error) => deserializeError(error));
return {result, errors: errors2};
}
} }

4
ext/js/language/dictionary-importer-worker.js
View File

@ -64,8 +64,8 @@ class DictionaryImporterWorker {
async _importDictionary(archiveContent, importDetails, onProgress) { async _importDictionary(archiveContent, importDetails, onProgress) {
const dictionaryDatabase = await this._getPreparedDictionaryDatabase(); const dictionaryDatabase = await this._getPreparedDictionaryDatabase();
try { try {
const dictionaryImporter = new DictionaryImporter(this._mediaLoader); const dictionaryImporter = new DictionaryImporter(this._mediaLoader, onProgress);
const {result, errors} = await dictionaryImporter.importDictionary(dictionaryDatabase, archiveContent, importDetails, onProgress); const {result, errors} = await dictionaryImporter.importDictionary(dictionaryDatabase, archiveContent, importDetails);
return { return {
result, result,
errors: errors.map((error) => serializeError(error)) errors: errors.map((error) => serializeError(error))

21
ext/js/language/dictionary-importer.js
View File

@ -22,12 +22,12 @@
*/ */
class DictionaryImporter { class DictionaryImporter {
constructor(mediaLoader) { constructor(mediaLoader, onProgress) {
this._mediaLoader = mediaLoader; this._mediaLoader = mediaLoader;
this._schemas = new Map(); this._onProgress = onProgress;
} }
async importDictionary(dictionaryDatabase, archiveContent, details, onProgress) { async importDictionary(dictionaryDatabase, archiveContent, details) {
if (!dictionaryDatabase) { if (!dictionaryDatabase) {
throw new Error('Invalid database'); throw new Error('Invalid database');
} }
@ -35,7 +35,7 @@ class DictionaryImporter {
throw new Error('Database is not ready'); throw new Error('Database is not ready');
} }
const hasOnProgress = (typeof onProgress === 'function'); const hasOnProgress = (typeof this._onProgress === 'function');
// Read archive // Read archive
const archive = await JSZip.loadAsync(archiveContent); const archive = await JSZip.loadAsync(archiveContent);
@ -143,7 +143,7 @@ class DictionaryImporter {
loadedCount += count; loadedCount += count;
if (hasOnProgress) { if (hasOnProgress) {
onProgress(total, loadedCount); this._onProgress(total, loadedCount);
} }
} }
}; };
@ -178,17 +178,6 @@ class DictionaryImporter {
} }
async _getSchema(fileName) { async _getSchema(fileName) {
let schemaPromise = this._schemas.get(fileName);
if (typeof schemaPromise !== 'undefined') {
return schemaPromise;
}
schemaPromise = this._createSchema(fileName);
this._schemas.set(fileName, schemaPromise);
return schemaPromise;
}
async _createSchema(fileName) {
const schema = await this._fetchJsonAsset(fileName); const schema = await this._fetchJsonAsset(fileName);
return new JsonSchema(schema); return new JsonSchema(schema);
} }

4
ext/js/pages/settings/dictionary-import-controller.js
View File

@ -181,9 +181,9 @@ class DictionaryImportController {
} }
async _importDictionary(file, importDetails, onProgress) { async _importDictionary(file, importDetails, onProgress) {
const dictionaryImporter = new DictionaryImporterThreaded(); const dictionaryImporter = new DictionaryImporterThreaded(onProgress);
const archiveContent = await this._readFile(file); const archiveContent = await this._readFile(file);
const {result, errors} = await dictionaryImporter.importDictionary(archiveContent, importDetails, onProgress); const {result, errors} = await dictionaryImporter.importDictionary(archiveContent, importDetails);
yomichan.api.triggerDatabaseUpdated('dictionary', 'import'); yomichan.api.triggerDatabaseUpdated('dictionary', 'import');
const errors2 = await this._addDictionarySettings(result.sequenced, result.title); const errors2 = await this._addDictionarySettings(result.sequenced, result.title);

21
test/test-database.js
View File

@ -41,9 +41,9 @@ function createTestDictionaryArchive(dictionary, dictionaryName) {
} }
function createDictionaryImporter() { function createDictionaryImporter(onProgress) {
const dictionaryImporterMediaLoader = new DatabaseVMDictionaryImporterMediaLoader(); const dictionaryImporterMediaLoader = new DatabaseVMDictionaryImporterMediaLoader();
return new DictionaryImporter(dictionaryImporterMediaLoader); return new DictionaryImporter(dictionaryImporterMediaLoader, onProgress);
} }
@ -131,7 +131,6 @@ async function testDatabase1() {
]; ];
// Setup database // Setup database
const dictionaryImporter = createDictionaryImporter();
const dictionaryDatabase = new DictionaryDatabase(); const dictionaryDatabase = new DictionaryDatabase();
await dictionaryDatabase.prepare(); await dictionaryDatabase.prepare();
@ -146,13 +145,11 @@ async function testDatabase1() {
// Import data // Import data
let progressEvent = false; let progressEvent = false;
const dictionaryImporter = createDictionaryImporter(() => { progressEvent = true; });
const {result, errors} = await dictionaryImporter.importDictionary( const {result, errors} = await dictionaryImporter.importDictionary(
dictionaryDatabase, dictionaryDatabase,
testDictionarySource, testDictionarySource,
{prefixWildcardsSupported: true}, {prefixWildcardsSupported: true}
() => {
progressEvent = true;
}
); );
vm.assert.deepStrictEqual(errors, []); vm.assert.deepStrictEqual(errors, []);
vm.assert.deepStrictEqual(result, expectedSummary); vm.assert.deepStrictEqual(result, expectedSummary);
@ -781,7 +778,6 @@ async function testDatabase2() {
]); ]);
// Setup database // Setup database
const dictionaryImporter = createDictionaryImporter();
const dictionaryDatabase = new DictionaryDatabase(); const dictionaryDatabase = new DictionaryDatabase();
// Error: not prepared // Error: not prepared
@ -796,17 +792,17 @@ async function testDatabase2() {
await assert.rejects(async () => await dictionaryDatabase.findTagForTitle('tag', title)); await assert.rejects(async () => await dictionaryDatabase.findTagForTitle('tag', title));
await assert.rejects(async () => await dictionaryDatabase.getDictionaryInfo()); await assert.rejects(async () => await dictionaryDatabase.getDictionaryInfo());
await assert.rejects(async () => await dictionaryDatabase.getDictionaryCounts(titles, true)); await assert.rejects(async () => await dictionaryDatabase.getDictionaryCounts(titles, true));
await assert.rejects(async () => await dictionaryImporter.importDictionary(dictionaryDatabase, testDictionarySource, {}, () => {})); await assert.rejects(async () => await createDictionaryImporter().importDictionary(dictionaryDatabase, testDictionarySource, {}));
await dictionaryDatabase.prepare(); await dictionaryDatabase.prepare();
// Error: already prepared // Error: already prepared
await assert.rejects(async () => await dictionaryDatabase.prepare()); await assert.rejects(async () => await dictionaryDatabase.prepare());
await dictionaryImporter.importDictionary(dictionaryDatabase, testDictionarySource, {}, () => {}); await createDictionaryImporter().importDictionary(dictionaryDatabase, testDictionarySource, {});
// Error: dictionary already imported // Error: dictionary already imported
await assert.rejects(async () => await dictionaryImporter.importDictionary(dictionaryDatabase, testDictionarySource, {}, () => {})); await assert.rejects(async () => await createDictionaryImporter().importDictionary(dictionaryDatabase, testDictionarySource, {}));
await dictionaryDatabase.close(); await dictionaryDatabase.close();
} }
@ -823,7 +819,6 @@ async function testDatabase3() {
]; ];
// Setup database // Setup database
const dictionaryImporter = createDictionaryImporter();
const dictionaryDatabase = new DictionaryDatabase(); const dictionaryDatabase = new DictionaryDatabase();
await dictionaryDatabase.prepare(); await dictionaryDatabase.prepare();
@ -833,7 +828,7 @@ async function testDatabase3() {
let error = null; let error = null;
try { try {
await dictionaryImporter.importDictionary(dictionaryDatabase, testDictionarySource, {}, () => {}); await createDictionaryImporter().importDictionary(dictionaryDatabase, testDictionarySource, {});
} catch (e) { } catch (e) {
error = e; error = e;
} }