Passively handle database errors

ext/bg/js/database.js

@@ -228,11 +228,47 @@ class Database {
         }
     }
 
-    async importDictionary(archive, callback) {
+    async importDictionary(archive, progressCallback, exceptions) {
         if (!this.db) {
             throw 'Database not initialized';
         }
 
+        const maxTransactionLength = 1000;
+        const bulkAdd = async (table, items, total, current) => {
+            if (items.length < maxTransactionLength) {
+                if (progressCallback) {
+                    progressCallback(total, current);
+                }
+
+                try {
+                    await table.bulkAdd(items);
+                } catch (e) {
+                    if (exceptions) {
+                        exceptions.push(e);
+                    } else {
+                        throw e;
+                    }
+                }
+            } else {
+                for (let i = 0; i < items.length; i += maxTransactionLength) {
+                    if (progressCallback) {
+                        progressCallback(total, current + i / items.length);
+                    }
+
+                    let count = Math.min(maxTransactionLength, items.length - i);
+                    try {
+                        await table.bulkAdd(items.slice(i, i + count));
+                    } catch (e) {
+                        if (exceptions) {
+                            exceptions.push(e);
+                        } else {
+                            throw e;
+                        }
+                    }
+                }
+            }
+        };
+
         const indexDataLoaded = async summary => {
             if (summary.version > 3) {
                 throw 'Unsupported dictionary version';
@@ -247,10 +283,6 @@ class Database {
         };
 
         const termDataLoaded = async (summary, entries, total, current) => {
-            if (callback) {
-                callback(total, current);
-            }
-
             const rows = [];
             if (summary.version === 1) {
                 for (const [expression, reading, definitionTags, rules, score, ...glossary] of entries) {
@@ -280,14 +312,10 @@ class Database {
                 }
             }
 
-            await this.db.terms.bulkAdd(rows);
+            await bulkAdd(this.db.terms, rows, total, current);
         };
 
         const termMetaDataLoaded = async (summary, entries, total, current) => {
-            if (callback) {
-                callback(total, current);
-            }
-
             const rows = [];
             for (const [expression, mode, data] of entries) {
                 rows.push({
@@ -298,14 +326,10 @@ class Database {
                 });
             }
 
-            await this.db.termMeta.bulkAdd(rows);
+            await bulkAdd(this.db.termMeta, rows, total, current);
         };
 
         const kanjiDataLoaded = async (summary, entries, total, current)  => {
-            if (callback) {
-                callback(total, current);
-            }
-
             const rows = [];
             if (summary.version === 1) {
                 for (const [character, onyomi, kunyomi, tags, ...meanings] of entries) {
@@ -332,14 +356,10 @@ class Database {
                 }
             }
 
-            await this.db.kanji.bulkAdd(rows);
+            await bulkAdd(this.db.kanji, rows, total, current);
         };
 
         const kanjiMetaDataLoaded = async (summary, entries, total, current) => {
-            if (callback) {
-                callback(total, current);
-            }
-
             const rows = [];
             for (const [character, mode, data] of entries) {
                 rows.push({
@@ -350,14 +370,10 @@ class Database {
                 });
             }
 
-            await this.db.kanjiMeta.bulkAdd(rows);
+            await bulkAdd(this.db.kanjiMeta, rows, total, current);
         };
 
         const tagDataLoaded = async (summary, entries, total, current) => {
-            if (callback) {
-                callback(total, current);
-            }
-
             const rows = [];
             for (const [name, category, order, notes, score] of entries) {
                 const row = dictTagSanitize({
@@ -372,7 +388,7 @@ class Database {
                 rows.push(row);
             }
 
-            await this.db.tagMeta.bulkAdd(rows);
+            await bulkAdd(this.db.tagMeta, rows, total, current);
         };
 
         return await Database.importDictionaryZip(

ext/bg/js/settings.js

@@ -193,7 +193,7 @@ async function onReady() {
         await dictionaryGroupsPopulate(options);
         await formMainDictionaryOptionsPopulate(options);
     } catch (e) {
-        dictionaryErrorShow(e);
+        dictionaryErrorsShow([e]);
     }
 
     try {
@@ -214,10 +214,23 @@ $(document).ready(utilAsync(onReady));
  * Dictionary
  */
 
-function dictionaryErrorShow(error) {
+function dictionaryErrorToString(error) {
-    const dialog = $('#dict-error');
+    if (error.toString) {
-    if (error) {
+        error = error.toString();
-        const overrides = [
+    } else {
+        error = `${error}`;
+    }
+
+    for (const [match, subst] of dictionaryErrorToString.overrides) {
+        if (error.includes(match)) {
+            error = subst;
+            break;
+        }
+    }
+
+    return error;
+}
+dictionaryErrorToString.overrides = [
     [
         'A mutation operation was attempted on a database that did not allow mutations.',
         'Access to IndexedDB appears to be restricted. Firefox seems to require that the history preference is set to "Remember history" before IndexedDB use of any kind is allowed.'
@@ -232,18 +245,32 @@ function dictionaryErrorShow(error) {
     ]
 ];
 
-        if (error.toString) {
+function dictionaryErrorsShow(errors) {
-            error = error.toString();
+    const dialog = $('#dict-error');
+    dialog.show().text('');
+
+    if (errors !== null && errors.length > 0) {
+        const uniqueErrors = {};
+        for (let e of errors) {
+            e = dictionaryErrorToString(e);
+            uniqueErrors[e] = uniqueErrors.hasOwnProperty(e) ? uniqueErrors[e] + 1 : 1;
         }
 
-        for (const [match, subst] of overrides) {
+        for (const e in uniqueErrors) {
-            if (error.includes(match)) {
+            const count = uniqueErrors[e];
-                error = subst;
+            const div = document.createElement('p');
-                break;
+            if (count > 1) {
+                div.textContent = `${e} `;
+                const em = document.createElement('em');
+                em.textContent = `(${count})`;
+                div.appendChild(em);
+            } else {
+                div.textContent = `${e}`;
             }
+            dialog.append($(div));
         }
 
-        dialog.show().text(error);
+        dialog.show();
     } else {
         dialog.hide();
     }
@@ -319,7 +346,7 @@ async function onDictionaryPurge(e) {
     const dictProgress = $('#dict-purge').show();
 
     try {
-        dictionaryErrorShow();
+        dictionaryErrorsShow(null);
         dictionarySpinnerShow(true);
 
         await utilDatabasePurge();
@@ -331,7 +358,7 @@ async function onDictionaryPurge(e) {
         await dictionaryGroupsPopulate(options);
         await formMainDictionaryOptionsPopulate(options);
     } catch (e) {
-        dictionaryErrorShow(e);
+        dictionaryErrorsShow([e]);
     } finally {
         dictionarySpinnerShow(false);
 
@@ -346,25 +373,32 @@ async function onDictionaryImport(e) {
     const dictProgress = $('#dict-import-progress').show();
 
     try {
-        dictionaryErrorShow();
+        dictionaryErrorsShow(null);
         dictionarySpinnerShow(true);
 
         const setProgress = percent => dictProgress.find('.progress-bar').css('width', `${percent}%`);
         const updateProgress = (total, current) => setProgress(current / total * 100.0);
         setProgress(0.0);
 
+        const exceptions = [];
         const options = await optionsLoad();
-        const summary = await utilDatabaseImport(e.target.files[0], updateProgress);
+        const summary = await utilDatabaseImport(e.target.files[0], updateProgress, exceptions);
         options.dictionaries[summary.title] = {enabled: true, priority: 0, allowSecondarySearches: false};
         if (summary.sequenced && options.general.mainDictionary === '') {
             options.general.mainDictionary = summary.title;
         }
+
+        if (exceptions.length > 0) {
+            exceptions.push(`Dictionary may not have been imported properly: ${exceptions.length} error${exceptions.length === 1 ? '' : 's'} reported.`);
+            dictionaryErrorsShow(exceptions);
+        }
+
         await optionsSave(options);
 
         await dictionaryGroupsPopulate(options);
         await formMainDictionaryOptionsPopulate(options);
     } catch (e) {
-        dictionaryErrorShow(e);
+        dictionaryErrorsShow([e]);
     } finally {
         dictionarySpinnerShow(false);
 

ext/bg/js/util.js

@@ -87,6 +87,6 @@ function utilDatabasePurge() {
     return utilBackend().translator.database.purge();
 }
 
-function utilDatabaseImport(data, progress) {
+function utilDatabaseImport(data, progress, exceptions) {
-    return utilBackend().translator.database.importDictionary(data, progress);
+    return utilBackend().translator.database.importDictionary(data, progress, exceptions);
 }