Dictionary import progress improvements (#1868)

* Update loop vars

* Update loop

* Improve progress reporting during the import process
This commit is contained in:
toasted-nutbread 2021-07-31 19:13:41 -04:00 committed by GitHub
parent 01c5c5c04b
commit cd3f47a359
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 106 additions and 26 deletions

View File

@ -24,7 +24,8 @@
class DictionaryImporter { class DictionaryImporter {
constructor(mediaLoader, onProgress) { constructor(mediaLoader, onProgress) {
this._mediaLoader = mediaLoader; this._mediaLoader = mediaLoader;
this._onProgress = onProgress; this._onProgress = typeof onProgress === 'function' ? onProgress : () => {};
this._progressData = null;
} }
async importDictionary(dictionaryDatabase, archiveContent, details) { async importDictionary(dictionaryDatabase, archiveContent, details) {
@ -35,7 +36,7 @@ class DictionaryImporter {
throw new Error('Database is not ready'); throw new Error('Database is not ready');
} }
const hasOnProgress = (typeof this._onProgress === 'function'); this._progressReset();
// Read archive // Read archive
const archive = await JSZip.loadAsync(archiveContent); const archive = await JSZip.loadAsync(archiveContent);
@ -72,6 +73,7 @@ class DictionaryImporter {
const convertTagBankEntry = this._convertTagBankEntry.bind(this); const convertTagBankEntry = this._convertTagBankEntry.bind(this);
// Load schemas // Load schemas
this._progressNextStep(0);
const dataBankSchemaPaths = this._getDataBankSchemaPaths(version); const dataBankSchemaPaths = this._getDataBankSchemaPaths(version);
const dataBankSchemas = await Promise.all(dataBankSchemaPaths.map((path) => this._getSchema(path))); const dataBankSchemas = await Promise.all(dataBankSchemaPaths.map((path) => this._getSchema(path)));
@ -83,6 +85,7 @@ class DictionaryImporter {
const tagFiles = this._getArchiveFiles(archive, 'tag_bank_?.json'); const tagFiles = this._getArchiveFiles(archive, 'tag_bank_?.json');
// Load data // Load data
this._progressNextStep(termFiles.length + termMetaFiles.length + kanjiFiles.length + kanjiMetaFiles.length + tagFiles.length);
const termList = await this._readFileSequence(termFiles, convertTermBankEntry, dataBankSchemas[0], dictionaryTitle); const termList = await this._readFileSequence(termFiles, convertTermBankEntry, dataBankSchemas[0], dictionaryTitle);
const termMetaList = await this._readFileSequence(termMetaFiles, convertTermMetaBankEntry, dataBankSchemas[1], dictionaryTitle); const termMetaList = await this._readFileSequence(termMetaFiles, convertTermMetaBankEntry, dataBankSchemas[1], dictionaryTitle);
const kanjiList = await this._readFileSequence(kanjiFiles, convertKanjiBankEntry, dataBankSchemas[2], dictionaryTitle); const kanjiList = await this._readFileSequence(kanjiFiles, convertKanjiBankEntry, dataBankSchemas[2], dictionaryTitle);
@ -100,17 +103,26 @@ class DictionaryImporter {
} }
// Extended data support // Extended data support
this._progressNextStep(termList.length);
const formatProgressInterval = 1000;
const requirements = []; const requirements = [];
for (const entry of termList) { for (let i = 0, ii = termList.length; i < ii; ++i) {
const entry = termList[i];
const glossaryList = entry.glossary; const glossaryList = entry.glossary;
for (let i = 0, ii = glossaryList.length; i < ii; ++i) { for (let j = 0, jj = glossaryList.length; j < jj; ++j) {
const glossary = glossaryList[i]; const glossary = glossaryList[j];
if (typeof glossary !== 'object' || glossary === null) { continue; } if (typeof glossary !== 'object' || glossary === null) { continue; }
glossaryList[i] = this._formatDictionaryTermGlossaryObject(glossary, entry, requirements); glossaryList[j] = this._formatDictionaryTermGlossaryObject(glossary, entry, requirements);
}
if ((i % formatProgressInterval) === 0) {
this._progressData.index = i;
this._progress();
} }
} }
this._progress();
// Async requirements // Async requirements
this._progressNextStep(requirements.length);
const {media} = await this._resolveAsyncRequirements(requirements, archive); const {media} = await this._resolveAsyncRequirements(requirements, archive);
// Add dictionary // Add dictionary
@ -119,15 +131,8 @@ class DictionaryImporter {
dictionaryDatabase.bulkAdd('dictionaries', [summary], 0, 1); dictionaryDatabase.bulkAdd('dictionaries', [summary], 0, 1);
// Add data // Add data
this._progressNextStep(termList.length + termMetaList.length + kanjiList.length + kanjiMetaList.length + tagList.length);
const errors = []; const errors = [];
const total = (
termList.length +
termMetaList.length +
kanjiList.length +
kanjiMetaList.length +
tagList.length
);
let loadedCount = 0;
const maxTransactionLength = 1000; const maxTransactionLength = 1000;
const bulkAdd = async (objectStoreName, entries) => { const bulkAdd = async (objectStoreName, entries) => {
@ -141,10 +146,8 @@ class DictionaryImporter {
errors.push(e); errors.push(e);
} }
loadedCount += count; this._progressData.index += count;
if (hasOnProgress) { this._progress();
this._onProgress(total, loadedCount);
}
} }
}; };
@ -155,9 +158,32 @@ class DictionaryImporter {
await bulkAdd('tagMeta', tagList); await bulkAdd('tagMeta', tagList);
await bulkAdd('media', media); await bulkAdd('media', media);
this._progress();
return {result: summary, errors}; return {result: summary, errors};
} }
_progressReset() {
this._progressData = {
stepIndex: 0,
stepCount: 6,
index: 0,
count: 0
};
this._progress();
}
_progressNextStep(count) {
++this._progressData.stepIndex;
this._progressData.index = 0;
this._progressData.count = count;
this._progress();
}
_progress() {
this._onProgress(this._progressData);
}
_createSummary(dictionaryTitle, version, index, details) { _createSummary(dictionaryTitle, version, index, details) {
const summary = { const summary = {
title: dictionaryTitle, title: dictionaryTitle,
@ -328,6 +354,8 @@ class DictionaryImporter {
return; return;
} }
Object.assign(target, result); Object.assign(target, result);
++this._progressData.index;
this._progress();
} }
async _resolveDictionaryTermGlossaryImage(context, data, entry) { async _resolveDictionaryTermGlossaryImage(context, data, entry) {
@ -500,10 +528,31 @@ class DictionaryImporter {
} }
async _readFileSequence(files, convertEntry, schema, dictionaryTitle) { async _readFileSequence(files, convertEntry, schema, dictionaryTitle) {
const progressData = this._progressData;
let count = 0;
let startIndex = 0;
if (typeof this._onProgress === 'function') {
schema.progressInterval = 1000;
schema.progress = (s) => {
const index = s.getValueStackLength() > 1 ? s.getValueStackItem(1).path : 0;
progressData.index = startIndex + (index / count);
this._progress();
};
}
const results = []; const results = [];
for (const file of files) { for (const file of files) {
const entries = JSON.parse(await file.async('string')); const entries = JSON.parse(await file.async('string'));
count = Array.isArray(entries) ? Math.max(entries.length, 1) : 1;
startIndex = progressData.index;
this._progress();
this._validateJsonSchema(entries, schema, file.name); this._validateJsonSchema(entries, schema, file.name);
progressData.index = startIndex + 1;
this._progress();
for (const entry of entries) { for (const entry of entries) {
results.push(convertEntry(entry, dictionaryTitle)); results.push(convertEntry(entry, dictionaryTitle));
} }

View File

@ -140,13 +140,28 @@ class DictionaryImportController {
prefixWildcardsSupported: optionsFull.global.database.prefixWildcardsSupported prefixWildcardsSupported: optionsFull.global.database.prefixWildcardsSupported
}; };
const onProgress = (total, current) => { let statusPrefix = '';
const percent = (current / total * 100.0); let stepIndex = -2;
const onProgress = (data) => {
const {stepIndex: stepIndex2, index, count} = data;
if (stepIndex !== stepIndex2) {
stepIndex = stepIndex2;
const labelText = `${statusPrefix} - Step ${stepIndex2 + 1} of ${data.stepCount}: ${this._getImportLabel(stepIndex2)}...`;
for (const label of infoLabels) { label.textContent = labelText; }
}
const percent = count > 0 ? (index / count * 100.0) : 0.0;
const cssString = `${percent}%`; const cssString = `${percent}%`;
const statusString = `${percent.toFixed(0)}%`; const statusString = `${Math.floor(percent).toFixed(0)}%`;
for (const progressBar of progressBars) { progressBar.style.width = cssString; } for (const progressBar of progressBars) { progressBar.style.width = cssString; }
for (const label of statusLabels) { label.textContent = statusString; } for (const label of statusLabels) { label.textContent = statusString; }
switch (stepIndex2) {
case -2: // Initialize
case 5: // Data import
this._triggerStorageChanged(); this._triggerStorageChanged();
break;
}
}; };
const fileCount = files.length; const fileCount = files.length;
@ -156,10 +171,13 @@ class DictionaryImportController {
importInfo.textContent = `(${i + 1} of ${fileCount})`; importInfo.textContent = `(${i + 1} of ${fileCount})`;
} }
onProgress(1, 0); statusPrefix = `Importing dictionary${fileCount > 1 ? ` (${i + 1} of ${fileCount})` : ''}`;
onProgress({
const labelText = `Importing dictionary${fileCount > 1 ? ` (${i + 1} of ${fileCount})` : ''}...`; stepIndex: -1,
for (const label of infoLabels) { label.textContent = labelText; } stepCount: 6,
index: 0,
count: 0
});
if (statusFooter !== null) { statusFooter.setTaskActive(progressSelector, true); } if (statusFooter !== null) { statusFooter.setTaskActive(progressSelector, true); }
await this._importDictionary(files[i], importDetails, onProgress); await this._importDictionary(files[i], importDetails, onProgress);
@ -180,6 +198,19 @@ class DictionaryImportController {
} }
} }
_getImportLabel(stepIndex) {
switch (stepIndex) {
case -1:
case 0: return 'Loading dictionary';
case 1: return 'Loading schemas';
case 2: return 'Validating data';
case 3: return 'Processing data';
case 4: return 'Post-processing data';
case 5: return 'Importing data';
default: return '';
}
}
async _importDictionary(file, importDetails, onProgress) { async _importDictionary(file, importDetails, onProgress) {
const dictionaryImporter = new DictionaryImporterThreaded(onProgress); const dictionaryImporter = new DictionaryImporterThreaded(onProgress);
const archiveContent = await this._readFile(file); const archiveContent = await this._readFile(file);