Settings dictionary import refactor (#759)

* Fix .purge not re-opening the database after deletion failure

* Create DictionaryImportController

* Remove backend dictionary import
This commit is contained in:
toasted-nutbread 2020-09-04 17:54:34 -04:00 committed by GitHub
parent 8cd5a2f75f
commit d8f488e28c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 366 additions and 233 deletions

View File

@ -16,7 +16,6 @@
<textarea id="clipboard-paste-target"></textarea>
<script src="/mixed/lib/handlebars.min.js"></script>
<script src="/mixed/lib/jszip.min.js"></script>
<script src="/mixed/lib/wanakana.min.js"></script>
<script src="/mixed/js/core.js"></script>
@ -33,10 +32,8 @@
<script src="/bg/js/conditions.js"></script>
<script src="/bg/js/database.js"></script>
<script src="/bg/js/dictionary-database.js"></script>
<script src="/bg/js/dictionary-importer.js"></script>
<script src="/bg/js/deinflector.js"></script>
<script src="/bg/js/json-schema.js"></script>
<script src="/bg/js/media-utility.js"></script>
<script src="/bg/js/options.js"></script>
<script src="/bg/js/profile-conditions.js"></script>
<script src="/bg/js/request-builder.js"></script>

View File

@ -16,7 +16,6 @@
*/
#dict-spinner, #dict-import-progress,
.storage-hidden {
display: none;
}

View File

@ -22,7 +22,6 @@
* AudioUriBuilder
* ClipboardMonitor
* DictionaryDatabase
* DictionaryImporter
* Environment
* JsonSchemaValidator
* Mecab
@ -39,7 +38,6 @@ class Backend {
constructor() {
this._environment = new Environment();
this._dictionaryDatabase = new DictionaryDatabase();
this._dictionaryImporter = new DictionaryImporter();
this._translator = new Translator(this._dictionaryDatabase);
this._anki = new AnkiConnect();
this._mecab = new Mecab();
@ -130,7 +128,6 @@ class Backend {
['getOrCreateSearchPopup', {async: true, contentScript: true, handler: this._onApiGetOrCreateSearchPopup.bind(this)}]
]);
this._messageHandlersWithProgress = new Map([
['importDictionaryArchive', {async: true, contentScript: false, handler: this._onApiImportDictionaryArchive.bind(this)}],
['deleteDictionary', {async: true, contentScript: false, handler: this._onApiDeleteDictionary.bind(this)}]
]);
@ -755,10 +752,6 @@ class Backend {
return details;
}
async _onApiImportDictionaryArchive({archiveContent, details}, sender, onProgress) {
return await this._dictionaryImporter.importDictionary(this._dictionaryDatabase, archiveContent, details, onProgress);
}
async _onApiDeleteDictionary({dictionaryName}, sender, onProgress) {
this._translator.clearDatabaseCaches();
await this._dictionaryDatabase.deleteDictionary(dictionaryName, {rate: 1000}, onProgress);
@ -1045,10 +1038,6 @@ class Backend {
return true;
}
async _importDictionary(archiveSource, onProgress, details) {
return await this._dictionaryImporter.importDictionary(this._dictionaryDatabase, archiveSource, onProgress, details);
}
async _textParseScanning(text, options) {
const results = [];
while (text.length > 0) {

View File

@ -117,8 +117,15 @@ class DictionaryDatabase {
if (this._db.isOpen()) {
this._db.close();
}
await Database.deleteDatabase(this._dbName);
let result = false;
try {
await Database.deleteDatabase(this._dbName);
result = true;
} catch (e) {
yomichan.logError(e);
}
await this.prepare();
return result;
}
async deleteDictionary(dictionaryName, progressSettings, onProgress) {

View File

@ -190,7 +190,7 @@ class DictionaryImporter {
try {
await dictionaryDatabase.bulkAdd(objectStoreName, entries, i, count);
} catch (e) {
errors.push(errorToJson(e));
errors.push(e);
}
loadedCount += count;

View File

@ -383,24 +383,9 @@ class SettingsDictionaryExtraUI {
}
class DictionaryController {
constructor(settingsController, storageController) {
constructor(settingsController) {
this._settingsController = settingsController;
this._storageController = storageController;
this._dictionaryUI = null;
this._dictionaryErrorToStringOverrides = [
[
'A mutation operation was attempted on a database that did not allow mutations.',
'Access to IndexedDB appears to be restricted. Firefox seems to require that the history preference is set to "Remember history" before IndexedDB use of any kind is allowed.'
],
[
'The operation failed for reasons unrelated to the database itself and not covered by any other error code.',
'Unable to access IndexedDB due to a possibly corrupt user profile. Try using the "Refresh Firefox" feature to reset your user profile.'
],
[
'BulkError',
'Unable to finish importing dictionary data into IndexedDB. This may indicate that you do not have sufficient disk space available to complete this operation.'
]
];
}
async prepare() {
@ -414,14 +399,11 @@ class DictionaryController {
this._dictionaryUI.preventPageExit = this._preventPageExit.bind(this);
this._dictionaryUI.on('databaseUpdated', this._onDatabaseUpdated.bind(this));
document.querySelector('#dict-purge-button').addEventListener('click', this._onPurgeButtonClick.bind(this), false);
document.querySelector('#dict-purge-confirm').addEventListener('click', this._onPurgeConfirmButtonClick.bind(this), false);
document.querySelector('#dict-file-button').addEventListener('click', this._onImportButtonClick.bind(this), false);
document.querySelector('#dict-file').addEventListener('change', this._onImportFileChange.bind(this), false);
document.querySelector('#dict-main').addEventListener('change', this._onDictionaryMainChanged.bind(this), false);
document.querySelector('#database-enable-prefix-wildcard-searches').addEventListener('change', this._onDatabaseEnablePrefixWildcardSearchesChanged.bind(this), false);
this._settingsController.on('optionsChanged', this._onOptionsChanged.bind(this));
this._settingsController.on('databaseUpdated', this._onDatabaseUpdated.bind(this));
await this._onOptionsChanged();
await this._onDatabaseUpdated();
@ -493,76 +475,6 @@ class DictionaryController {
select.value = value;
}
_dictionaryErrorToString(error) {
if (error.toString) {
error = error.toString();
} else {
error = `${error}`;
}
for (const [match, subst] of this._dictionaryErrorToStringOverrides) {
if (error.includes(match)) {
error = subst;
break;
}
}
return error;
}
_dictionaryErrorsShow(errors) {
const dialog = document.querySelector('#dict-error');
dialog.textContent = '';
if (errors !== null && errors.length > 0) {
const uniqueErrors = new Map();
for (let e of errors) {
yomichan.logError(e);
e = this._dictionaryErrorToString(e);
let count = uniqueErrors.get(e);
if (typeof count === 'undefined') {
count = 0;
}
uniqueErrors.set(e, count + 1);
}
for (const [e, count] of uniqueErrors.entries()) {
const div = document.createElement('p');
if (count > 1) {
div.textContent = `${e} `;
const em = document.createElement('em');
em.textContent = `(${count})`;
div.appendChild(em);
} else {
div.textContent = `${e}`;
}
dialog.appendChild(div);
}
dialog.hidden = false;
} else {
dialog.hidden = true;
}
}
_dictionarySpinnerShow(show) {
const spinner = $('#dict-spinner');
if (show) {
spinner.show();
} else {
spinner.hide();
}
}
_dictReadFile(file) {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => resolve(reader.result);
reader.onerror = () => reject(reader.error);
reader.readAsBinaryString(file);
});
}
async _onDatabaseUpdated() {
try {
const dictionaries = await api.getDictionaryInfo();
@ -576,7 +488,7 @@ class DictionaryController {
const {counts, total} = await api.getDictionaryCounts(dictionaries.map((v) => v.title), true);
this._dictionaryUI.setCounts(counts, total);
} catch (e) {
this._dictionaryErrorsShow([e]);
yomichan.logError(e);
}
}
@ -594,124 +506,6 @@ class DictionaryController {
await this._settingsController.save();
}
_onImportButtonClick() {
const dictFile = document.querySelector('#dict-file');
dictFile.click();
}
_onPurgeButtonClick(e) {
e.preventDefault();
$('#dict-purge-modal').modal('show');
}
async _onPurgeConfirmButtonClick(e) {
e.preventDefault();
$('#dict-purge-modal').modal('hide');
const dictControls = $('#dict-importer, #dict-groups, #dict-groups-extra, #dict-main-group').hide();
const dictProgress = document.querySelector('#dict-purge');
dictProgress.hidden = false;
const prevention = this._preventPageExit();
try {
this._dictionaryErrorsShow(null);
this._dictionarySpinnerShow(true);
await api.purgeDatabase();
const optionsFull = await this._settingsController.getOptionsFullMutable();
for (const {options} of toIterable(optionsFull.profiles)) {
options.dictionaries = utilBackgroundIsolate({});
options.general.mainDictionary = '';
}
await this._settingsController.save();
this._onDatabaseUpdated();
} catch (err) {
this._dictionaryErrorsShow([err]);
} finally {
prevention.end();
this._dictionarySpinnerShow(false);
dictControls.show();
dictProgress.hidden = true;
this._storageController.updateStats();
}
}
async _onImportFileChange(e) {
const files = [...e.target.files];
e.target.value = null;
const dictFile = $('#dict-file');
const dictControls = $('#dict-importer').hide();
const dictProgress = $('#dict-import-progress').show();
const dictImportInfo = document.querySelector('#dict-import-info');
const prevention = this._preventPageExit();
try {
this._dictionaryErrorsShow(null);
this._dictionarySpinnerShow(true);
const setProgress = (percent) => dictProgress.find('.progress-bar').css('width', `${percent}%`);
const updateProgress = (total, current) => {
setProgress(current / total * 100.0);
this._storageController.updateStats();
};
const optionsFull = await this._settingsController.getOptionsFull();
const importDetails = {
prefixWildcardsSupported: optionsFull.global.database.prefixWildcardsSupported
};
for (let i = 0, ii = files.length; i < ii; ++i) {
setProgress(0.0);
if (ii > 1) {
dictImportInfo.hidden = false;
dictImportInfo.textContent = `(${i + 1} of ${ii})`;
}
const archiveContent = await this._dictReadFile(files[i]);
const {result, errors} = await api.importDictionaryArchive(archiveContent, importDetails, updateProgress);
const optionsFull2 = await this._settingsController.getOptionsFullMutable();
for (const {options} of toIterable(optionsFull2.profiles)) {
const dictionaryOptions = SettingsDictionaryListUI.createDictionaryOptions();
dictionaryOptions.enabled = true;
options.dictionaries[result.title] = dictionaryOptions;
if (result.sequenced && options.general.mainDictionary === '') {
options.general.mainDictionary = result.title;
}
}
await this._settingsController.save();
if (errors.length > 0) {
const errors2 = errors.map((error) => jsonToError(error));
errors2.push(`Dictionary may not have been imported properly: ${errors2.length} error${errors2.length === 1 ? '' : 's'} reported.`);
this._dictionaryErrorsShow(errors2);
}
this._onDatabaseUpdated();
}
} catch (err) {
this._dictionaryErrorsShow([err]);
} finally {
prevention.end();
this._dictionarySpinnerShow(false);
dictImportInfo.hidden = false;
dictImportInfo.textContent = '';
dictFile.val('');
dictControls.show();
dictProgress.hide();
}
}
async _onDatabaseEnablePrefixWildcardSearchesChanged(e) {
const optionsFull = await this._settingsController.getOptionsFullMutable();
const v = !!e.target.checked;

View File

@ -0,0 +1,334 @@
/*
* Copyright (C) 2020 Yomichan Authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
/* global
* DictionaryDatabase
* DictionaryImporter
* ObjectPropertyAccessor
* api
*/
class DictionaryImportController {
constructor(settingsController, storageController) {
this._settingsController = settingsController;
this._storageController = storageController;
this._modifying = false;
this._purgeButton = null;
this._purgeConfirmButton = null;
this._importFileButton = null;
this._importFileInput = null;
this._purgeConfirmModal = null;
this._errorContainer = null;
this._spinner = null;
this._purgeNotification = null;
this._importInfo = null;
this._errorToStringOverrides = [
[
'A mutation operation was attempted on a database that did not allow mutations.',
'Access to IndexedDB appears to be restricted. Firefox seems to require that the history preference is set to "Remember history" before IndexedDB use of any kind is allowed.'
],
[
'The operation failed for reasons unrelated to the database itself and not covered by any other error code.',
'Unable to access IndexedDB due to a possibly corrupt user profile. Try using the "Refresh Firefox" feature to reset your user profile.'
]
];
}
async prepare() {
this._purgeButton = document.querySelector('#dict-purge-button');
this._purgeConfirmButton = document.querySelector('#dict-purge-confirm');
this._importFileButton = document.querySelector('#dict-file-button');
this._importFileInput = document.querySelector('#dict-file');
this._purgeConfirmModal = document.querySelector('#dict-purge-modal');
this._errorContainer = document.querySelector('#dict-error');
this._spinner = document.querySelector('#dict-spinner');
this._progressContainer = document.querySelector('#dict-import-progress');
this._progressBar = this._progressContainer.querySelector('.progress-bar');
this._purgeNotification = document.querySelector('#dict-purge');
this._importInfo = document.querySelector('#dict-import-info');
this._purgeButton.addEventListener('click', this._onPurgeButtonClick.bind(this), false);
this._purgeConfirmButton.addEventListener('click', this._onPurgeConfirmButtonClick.bind(this), false);
this._importFileButton.addEventListener('click', this._onImportButtonClick.bind(this), false);
this._importFileInput.addEventListener('change', this._onImportFileChange.bind(this), false);
}
// Private
_onImportButtonClick() {
this._importFileInput.click();
}
_onPurgeButtonClick(e) {
e.preventDefault();
this._setPurgeModalVisible(true);
}
_onPurgeConfirmButtonClick(e) {
e.preventDefault();
this._setPurgeModalVisible(false);
this._purgeDatabase();
}
_onImportFileChange(e) {
const node = e.currentTarget;
const files = [...node.files];
node.value = null;
this._importDictionaries(files);
}
async _purgeDatabase() {
if (this._modifying) { return; }
const purgeNotification = this._purgeNotification;
const prevention = this._preventPageExit();
try {
this._setModifying(true);
this._hideErrors();
this._setSpinnerVisible(true);
purgeNotification.hidden = false;
await api.purgeDatabase();
const errors = await this._clearDictionarySettings();
if (errors.length > 0) {
this._showErrors(errors);
}
this._triggerDatabaseUpdated('purge');
} catch (error) {
this._showErrors([error]);
} finally {
prevention.end();
purgeNotification.hidden = true;
this._setSpinnerVisible(false);
this._storageController.updateStats();
this._setModifying(false);
}
}
async _importDictionaries(files) {
if (this._modifying) { return; }
const importInfo = this._importInfo;
const progressContainer = this._progressContainer;
const progressBar = this._progressBar;
const storageController = this._storageController;
const prevention = this._preventPageExit();
try {
this._setModifying(true);
this._hideErrors();
this._setSpinnerVisible(true);
progressContainer.hidden = false;
const optionsFull = await this._settingsController.getOptionsFull();
const importDetails = {
prefixWildcardsSupported: optionsFull.global.database.prefixWildcardsSupported
};
const updateProgress = (total, current) => {
const percent = (current / total * 100.0);
progressBar.style.width = `${percent}%`;
storageController.updateStats();
};
const fileCount = files.length;
for (let i = 0; i < fileCount; ++i) {
progressBar.style.width = '0';
if (fileCount > 1) {
importInfo.hidden = false;
importInfo.textContent = `(${i + 1} of ${fileCount})`;
}
await this._importDictionary(files[i], importDetails, updateProgress);
}
} catch (err) {
this._showErrors([err]);
} finally {
prevention.end();
progressContainer.hidden = true;
importInfo.textContent = '';
importInfo.hidden = true;
this._setSpinnerVisible(false);
this._setModifying(false);
}
}
async _importDictionary(file, importDetails, onProgress) {
const dictionaryDatabase = await this._getPreparedDictionaryDatabase();
try {
const dictionaryImporter = new DictionaryImporter();
const archiveContent = await this._readFile(file);
const {result, errors} = await dictionaryImporter.importDictionary(dictionaryDatabase, archiveContent, importDetails, onProgress);
const errors2 = await this._addDictionarySettings(result.sequenced, result.title);
if (errors.length > 0) {
const allErrors = [...errors, ...errors2];
allErrors.push(new Error(`Dictionary may not have been imported properly: ${allErrors.length} error${allErrors.length === 1 ? '' : 's'} reported.`));
this._showErrors(allErrors);
}
this._triggerDatabaseUpdated('import');
} finally {
dictionaryDatabase.close();
}
}
async _addDictionarySettings(sequenced, title) {
const optionsFull = await this._settingsController.getOptionsFull();
const targets = [];
const profileCount = optionsFull.profiles.length;
for (let i = 0; i < profileCount; ++i) {
const {options} = optionsFull.profiles[i];
const value = this._createDictionaryOptions();
const path1 = ObjectPropertyAccessor.getPathString(['profiles', i, 'options', 'dictionaries', title]);
targets.push({action: 'set', path: path1, value});
if (sequenced && options.general.mainDictionary === '') {
const path2 = ObjectPropertyAccessor.getPathString(['profiles', i, 'options', 'general', 'mainDictionary']);
targets.push({action: 'set', path: path2, value: title});
}
}
return await this._modifyGlobalSettings(targets);
}
async _clearDictionarySettings() {
const optionsFull = await this._settingsController.getOptionsFull();
const targets = [];
const profileCount = optionsFull.profiles.length;
for (let i = 0; i < profileCount; ++i) {
const path1 = ObjectPropertyAccessor.getPathString(['profiles', i, 'options', 'dictionaries']);
targets.push({action: 'set', path: path1, value: {}});
const path2 = ObjectPropertyAccessor.getPathString(['profiles', i, 'options', 'general', 'mainDictionary']);
targets.push({action: 'set', path: path2, value: ''});
}
return await this._modifyGlobalSettings(targets);
}
_setPurgeModalVisible(visible) {
const node = $(this._purgeConfirmModal);
node.modal(visible ? 'show' : 'hide');
}
_setSpinnerVisible(visible) {
this._spinner.hidden = !visible;
}
_preventPageExit() {
return this._settingsController.preventPageExit();
}
_showErrors(errors) {
const uniqueErrors = new Map();
for (const error of errors) {
yomichan.logError(error);
const errorString = this._errorToString(error);
let count = uniqueErrors.get(errorString);
if (typeof count === 'undefined') {
count = 0;
}
uniqueErrors.set(errorString, count + 1);
}
const fragment = document.createDocumentFragment();
for (const [e, count] of uniqueErrors.entries()) {
const div = document.createElement('p');
if (count > 1) {
div.textContent = `${e} `;
const em = document.createElement('em');
em.textContent = `(${count})`;
div.appendChild(em);
} else {
div.textContent = `${e}`;
}
fragment.appendChild(div);
}
this._errorContainer.appendChild(fragment);
this._errorContainer.hidden = false;
}
_hideErrors() {
this._errorContainer.textContent = '';
this._errorContainer.hidden = true;
}
_triggerDatabaseUpdated(cause) {
this._settingsController.triggerDatabaseUpdated(cause);
}
_readFile(file) {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => resolve(reader.result);
reader.onerror = () => reject(reader.error);
reader.readAsBinaryString(file);
});
}
_createDictionaryOptions() {
return {
priority: 0,
enabled: true,
allowSecondarySearches: false
};
}
_errorToString(error) {
error = (typeof error.toString === 'function' ? error.toString() : `${error}`);
for (const [match, newErrorString] of this._errorToStringOverrides) {
if (error.includes(match)) {
return newErrorString;
}
}
return error;
}
_setModifying(value) {
this._modifying = value;
this._setButtonsEnabled(!value);
}
_setButtonsEnabled(value) {
value = !value;
this._purgeButton.disabled = value;
this._importFileButton.disabled = value;
}
async _getPreparedDictionaryDatabase() {
const dictionaryDatabase = new DictionaryDatabase();
await dictionaryDatabase.prepare();
return dictionaryDatabase;
}
async _modifyGlobalSettings(targets) {
const results = await this._settingsController.modifyGlobalSettings(targets);
const errors = [];
for (const {error} of results) {
if (typeof error !== 'undefined') {
errors.push(jsonToError(error));
}
}
return errors;
}
}

View File

@ -21,6 +21,7 @@
* AudioController
* ClipboardPopupsController
* DictionaryController
* DictionaryImportController
* GenericSettingController
* PopupPreviewController
* ProfileController
@ -94,9 +95,12 @@ async function setupEnvironmentInfo() {
const profileController = new ProfileController(settingsController);
profileController.prepare();
const dictionaryController = new DictionaryController(settingsController, storageController);
const dictionaryController = new DictionaryController(settingsController);
dictionaryController.prepare();
const dictionaryImportController = new DictionaryImportController(settingsController, storageController);
dictionaryImportController.prepare();
const ankiController = new AnkiController(settingsController);
ankiController.prepare();

View File

@ -121,6 +121,10 @@ class SettingsController extends EventDispatcher {
return obj;
}
triggerDatabaseUpdated(cause) {
this.trigger('databaseUpdated', {cause});
}
// Private
_setProfileIndex(value) {

View File

@ -585,7 +585,7 @@
<div class="ignore-form-changes">
<div>
<img src="/mixed/img/spinner.gif" class="pull-right" id="dict-spinner" alt>
<img src="/mixed/img/spinner.gif" class="pull-right" id="dict-spinner" alt hidden>
<h3>Dictionaries</h3>
</div>
@ -605,7 +605,7 @@
<div id="dict-groups"></div>
<div id="dict-groups-extra"></div>
<div id="dict-import-progress">
<div id="dict-import-progress" hidden>
Dictionary data is being imported, please be patient...
<span id="dict-import-info" hidden></span>
<div class="progress">
@ -1128,6 +1128,7 @@
<script src="/mixed/lib/jquery.min.js"></script>
<script src="/mixed/lib/bootstrap/js/bootstrap.min.js"></script>
<script src="/mixed/lib/handlebars.min.js"></script>
<script src="/mixed/lib/jszip.min.js"></script>
<script src="/mixed/lib/wanakana.min.js"></script>
<script src="/mixed/js/core.js"></script>
@ -1145,12 +1146,20 @@
<script src="/mixed/js/audio-system.js"></script>
<script src="/mixed/js/document-util.js"></script>
<script src="/bg/js/database.js"></script>
<script src="/bg/js/dictionary-database.js"></script>
<script src="/bg/js/dictionary-importer.js"></script>
<script src="/bg/js/json-schema.js"></script>
<script src="/bg/js/media-utility.js"></script>
<script src="/mixed/js/cache-map.js"></script>
<script src="/bg/js/settings/anki.js"></script>
<script src="/bg/js/settings/anki-templates.js"></script>
<script src="/bg/js/settings/audio.js"></script>
<script src="/bg/js/settings/backup.js"></script>
<script src="/bg/js/settings/clipboard-popups-controller.js"></script>
<script src="/bg/js/settings/dictionaries.js"></script>
<script src="/bg/js/settings/dictionary-import-controller.js"></script>
<script src="/bg/js/settings/generic-setting-controller.js"></script>
<script src="/bg/js/settings/popup-preview.js"></script>
<script src="/bg/js/settings/profiles.js"></script>

View File

@ -203,10 +203,6 @@ const api = (() => {
// Invoke functions with progress
importDictionaryArchive(archiveContent, details, onProgress) {
return this._invokeWithProgress('importDictionaryArchive', {archiveContent, details}, onProgress);
}
deleteDictionary(dictionaryName, onProgress) {
return this._invokeWithProgress('deleteDictionary', {dictionaryName}, onProgress);
}