Generic database (#600)

* Update test

* Rename db to _db

* Create GenericDatabase class

* Catch prepare error

* Allow database to be purged even if it was not open

* Remove unused functions

* Change static functions to non-static

* Delete and count using the media object store

* Update tests

ext/bg/background.html

@@ -30,6 +30,7 @@
         <script src="/bg/js/audio-uri-builder.js"></script>
         <script src="/bg/js/clipboard-monitor.js"></script>
         <script src="/bg/js/conditions.js"></script>
+        <script src="/bg/js/generic-database.js"></script>
         <script src="/bg/js/database.js"></script>
         <script src="/bg/js/dictionary-importer.js"></script>
         <script src="/bg/js/deinflector.js"></script>

ext/bg/js/backend.js

@@ -172,7 +172,11 @@ class Backend {
             yomichan.on('log', this._onLog.bind(this));
 
             await this.environment.prepare();
+            try {
                 await this.database.prepare();
+            } catch (e) {
+                yomichan.logError(e);
+            }
             await this.translator.prepare();
 
             await profileConditionsDescriptorPromise;

ext/bg/js/database.js

@@ -16,27 +16,26 @@
  */
 
 /* global
+ * GenericDatabase
  * dictFieldSplit
  */
 
 class Database {
     constructor() {
-        this.db = null;
+        this._db = new GenericDatabase();
+        this._dbName = 'dict';
         this._schemas = new Map();
     }
 
     // Public
 
     async prepare() {
-        if (this.db !== null) {
+        await this._db.open(
-            throw new Error('Database already initialized');
+            this._dbName,
-        }
+            60,
-
+            [
-        try {
-            this.db = await Database._open('dict', 6, (db, transaction, oldVersion) => {
-                Database._upgrade(db, transaction, oldVersion, [
                 {
-                        version: 2,
+                    version: 20,
                     stores: {
                         terms: {
                             primaryKey: {keyPath: 'id', autoIncrement: true},
@@ -57,7 +56,7 @@ class Database {
                     }
                 },
                 {
-                        version: 3,
+                    version: 30,
                     stores: {
                         termMeta: {
                             primaryKey: {autoIncrement: true},
@@ -74,7 +73,7 @@ class Database {
                     }
                 },
                 {
-                        version: 4,
+                    version: 40,
                     stores: {
                         terms: {
                             primaryKey: {keyPath: 'id', autoIncrement: true},
@@ -83,7 +82,7 @@ class Database {
                     }
                 },
                 {
-                        version: 5,
+                    version: 50,
                     stores: {
                         terms: {
                             primaryKey: {keyPath: 'id', autoIncrement: true},
@@ -92,7 +91,7 @@ class Database {
                     }
                 },
                 {
-                        version: 6,
+                    version: 60,
                     stores: {
                         media: {
                             primaryKey: {keyPath: 'id', autoIncrement: true},
@@ -100,339 +99,339 @@ class Database {
                         }
                     }
                 }
-                ]);
+            ]
-            });
+        );
-            return true;
-        } catch (e) {
-            yomichan.logError(e);
-            return false;
-        }
     }
 
     async close() {
-        this._validate();
+        this._db.close();
-        this.db.close();
-        this.db = null;
     }
 
     isPrepared() {
-        return this.db !== null;
+        return this._db.isOpen();
     }
 
     async purge() {
-        this._validate();
+        if (this._db.isOpening()) {
-
+            throw new Error('Cannot purge database while opening');
-        this.db.close();
+        }
-        await Database._deleteDatabase(this.db.name);
+        if (this._db.isOpen()) {
-        this.db = null;
+            this._db.close();
-
+        }
+        await GenericDatabase.deleteDatabase(this._dbName);
         await this.prepare();
     }
 
     async deleteDictionary(dictionaryName, progressSettings, onProgress) {
-        this._validate();
-
         const targets = [
             ['dictionaries', 'title'],
             ['kanji', 'dictionary'],
             ['kanjiMeta', 'dictionary'],
             ['terms', 'dictionary'],
             ['termMeta', 'dictionary'],
-            ['tagMeta', 'dictionary']
+            ['tagMeta', 'dictionary'],
+            ['media', 'dictionary']
         ];
-        const promises = [];
+
+        const {rate} = progressSettings;
         const progressData = {
             count: 0,
             processed: 0,
             storeCount: targets.length,
             storesProcesed: 0
         };
-        let progressRate = (typeof progressSettings === 'object' && progressSettings !== null ? progressSettings.rate : 0);
-        if (typeof progressRate !== 'number' || progressRate <= 0) {
-            progressRate = 1000;
-        }
 
-        for (const [objectStoreName, index] of targets) {
+        const filterKeys = (keys) => {
-            const dbTransaction = this.db.transaction([objectStoreName], 'readwrite');
+            ++progressData.storesProcesed;
-            const dbObjectStore = dbTransaction.objectStore(objectStoreName);
+            progressData.count += keys.length;
-            const dbIndex = dbObjectStore.index(index);
+            onProgress(progressData);
-            const only = IDBKeyRange.only(dictionaryName);
+            return keys;
-            promises.push(Database._deleteValues(dbObjectStore, dbIndex, only, onProgress, progressData, progressRate));
+        };
-        }
+        const onProgress2 = () => {
-
+            const processed = progressData.processed + 1;
-        await Promise.all(promises);
+            progressData.processed = processed;
-    }
+            if ((processed % rate) === 0 || processed === progressData.count) {
-
+                onProgress(progressData);
-    async findTermsBulk(termList, dictionaries, wildcard) {
-        this._validate();
-
-        const promises = [];
-        const visited = new Set();
-        const results = [];
-        const processRow = (row, index) => {
-            if (dictionaries.has(row.dictionary) && !visited.has(row.id)) {
-                visited.add(row.id);
-                results.push(Database._createTerm(row, index));
             }
         };
 
+        const promises = [];
+        for (const [objectStoreName, indexName] of targets) {
+            const query = IDBKeyRange.only(dictionaryName);
+            const promise = this._db.bulkDelete(objectStoreName, indexName, query, filterKeys, onProgress2);
+            promises.push(promise);
+        }
+        await Promise.all(promises);
+    }
+
+    findTermsBulk(termList, dictionaries, wildcard) {
+        return new Promise((resolve, reject) => {
+            const results = [];
+            const count = termList.length;
+            if (count === 0) {
+                resolve(results);
+                return;
+            }
+
+            const visited = new Set();
             const useWildcard = !!wildcard;
             const prefixWildcard = wildcard === 'prefix';
 
-        const dbTransaction = this.db.transaction(['terms'], 'readonly');
+            const transaction = this._db.transaction(['terms'], 'readonly');
-        const dbTerms = dbTransaction.objectStore('terms');
+            const terms = transaction.objectStore('terms');
-        const dbIndex1 = dbTerms.index(prefixWildcard ? 'expressionReverse' : 'expression');
+            const index1 = terms.index(prefixWildcard ? 'expressionReverse' : 'expression');
-        const dbIndex2 = dbTerms.index(prefixWildcard ? 'readingReverse' : 'reading');
+            const index2 = terms.index(prefixWildcard ? 'readingReverse' : 'reading');
 
-        for (let i = 0; i < termList.length; ++i) {
+            const count2 = count * 2;
+            let completeCount = 0;
+            for (let i = 0; i < count; ++i) {
+                const inputIndex = i;
                 const term = prefixWildcard ? stringReverse(termList[i]) : termList[i];
                 const query = useWildcard ? IDBKeyRange.bound(term, `${term}\uffff`, false, false) : IDBKeyRange.only(term);
-            promises.push(
+
-                Database._getAll(dbIndex1, query, i, processRow),
+                const onGetAll = (rows) => {
-                Database._getAll(dbIndex2, query, i, processRow)
+                    for (const row of rows) {
-            );
+                        if (dictionaries.has(row.dictionary) && !visited.has(row.id)) {
+                            visited.add(row.id);
+                            results.push(this._createTerm(row, inputIndex));
                         }
-
-        await Promise.all(promises);
-
-        return results;
                     }
-
+                    if (++completeCount >= count2) {
-    async findTermsExactBulk(termList, readingList, dictionaries) {
+                        resolve(results);
-        this._validate();
-
-        const promises = [];
-        const results = [];
-        const processRow = (row, index) => {
-            if (row.reading === readingList[index] && dictionaries.has(row.dictionary)) {
-                results.push(Database._createTerm(row, index));
                     }
                 };
 
-        const dbTransaction = this.db.transaction(['terms'], 'readonly');
+                this._db.getAll(index1, query, onGetAll, reject);
-        const dbTerms = dbTransaction.objectStore('terms');
+                this._db.getAll(index2, query, onGetAll, reject);
-        const dbIndex = dbTerms.index('expression');
-
-        for (let i = 0; i < termList.length; ++i) {
-            const only = IDBKeyRange.only(termList[i]);
-            promises.push(Database._getAll(dbIndex, only, i, processRow));
-        }
-
-        await Promise.all(promises);
-
-        return results;
-    }
-
-    async findTermsBySequenceBulk(sequenceList, mainDictionary) {
-        this._validate();
-
-        const promises = [];
-        const results = [];
-        const processRow = (row, index) => {
-            if (row.dictionary === mainDictionary) {
-                results.push(Database._createTerm(row, index));
-            }
-        };
-
-        const dbTransaction = this.db.transaction(['terms'], 'readonly');
-        const dbTerms = dbTransaction.objectStore('terms');
-        const dbIndex = dbTerms.index('sequence');
-
-        for (let i = 0; i < sequenceList.length; ++i) {
-            const only = IDBKeyRange.only(sequenceList[i]);
-            promises.push(Database._getAll(dbIndex, only, i, processRow));
-        }
-
-        await Promise.all(promises);
-
-        return results;
-    }
-
-    async findTermMetaBulk(termList, dictionaries) {
-        return this._findGenericBulk('termMeta', 'expression', termList, dictionaries, Database._createTermMeta);
-    }
-
-    async findKanjiBulk(kanjiList, dictionaries) {
-        return this._findGenericBulk('kanji', 'character', kanjiList, dictionaries, Database._createKanji);
-    }
-
-    async findKanjiMetaBulk(kanjiList, dictionaries) {
-        return this._findGenericBulk('kanjiMeta', 'character', kanjiList, dictionaries, Database._createKanjiMeta);
-    }
-
-    async findTagForTitle(name, title) {
-        this._validate();
-
-        let result = null;
-        const dbTransaction = this.db.transaction(['tagMeta'], 'readonly');
-        const dbTerms = dbTransaction.objectStore('tagMeta');
-        const dbIndex = dbTerms.index('name');
-        const only = IDBKeyRange.only(name);
-        await Database._getAll(dbIndex, only, null, (row) => {
-            if (title === row.dictionary) {
-                result = row;
             }
         });
-
-        return result;
     }
 
-    async getMedia(targets) {
+    findTermsExactBulk(termList, readingList, dictionaries) {
-        this._validate();
+        return new Promise((resolve, reject) => {
+            const results = [];
+            const count = termList.length;
+            if (count === 0) {
+                resolve(results);
+                return;
+            }
 
-        const count = targets.length;
+            const transaction = this._db.transaction(['terms'], 'readonly');
-        const promises = [];
+            const terms = transaction.objectStore('terms');
-        const results = new Array(count).fill(null);
+            const index = terms.index('expression');
-        const createResult = Database._createMedia;
+
-        const processRow = (row, [index, dictionaryName]) => {
+            let completeCount = 0;
-            if (row.dictionary === dictionaryName) {
+            for (let i = 0; i < count; ++i) {
-                results[index] = createResult(row, index);
+                const inputIndex = i;
+                const reading = readingList[i];
+                const query = IDBKeyRange.only(termList[i]);
+
+                const onGetAll = (rows) => {
+                    for (const row of rows) {
+                        if (row.reading === reading && dictionaries.has(row.dictionary)) {
+                            results.push(this._createTerm(row, inputIndex));
+                        }
+                    }
+                    if (++completeCount >= count) {
+                        resolve(results);
                     }
                 };
 
-        const transaction = this.db.transaction(['media'], 'readonly');
+                this._db.getAll(index, query, onGetAll, reject);
+            }
+        });
+    }
+
+    findTermsBySequenceBulk(sequenceList, mainDictionary) {
+        return new Promise((resolve, reject) => {
+            const results = [];
+            const count = sequenceList.length;
+            if (count === 0) {
+                resolve(results);
+                return;
+            }
+
+            const transaction = this._db.transaction(['terms'], 'readonly');
+            const terms = transaction.objectStore('terms');
+            const index = terms.index('sequence');
+
+            let completeCount = 0;
+            for (let i = 0; i < count; ++i) {
+                const inputIndex = i;
+                const query = IDBKeyRange.only(sequenceList[i]);
+
+                const onGetAll = (rows) => {
+                    for (const row of rows) {
+                        if (row.dictionary === mainDictionary) {
+                            results.push(this._createTerm(row, inputIndex));
+                        }
+                    }
+                    if (++completeCount >= count) {
+                        resolve(results);
+                    }
+                };
+
+                this._db.getAll(index, query, onGetAll, reject);
+            }
+        });
+    }
+
+    findTermMetaBulk(termList, dictionaries) {
+        return this._findGenericBulk('termMeta', 'expression', termList, dictionaries, this._createTermMeta.bind(this));
+    }
+
+    findKanjiBulk(kanjiList, dictionaries) {
+        return this._findGenericBulk('kanji', 'character', kanjiList, dictionaries, this._createKanji.bind(this));
+    }
+
+    findKanjiMetaBulk(kanjiList, dictionaries) {
+        return this._findGenericBulk('kanjiMeta', 'character', kanjiList, dictionaries, this._createKanjiMeta.bind(this));
+    }
+
+    findTagForTitle(name, title) {
+        const query = IDBKeyRange.only(name);
+        return this._db.find('tagMeta', 'name', query, (row) => (row.dictionary === title), null);
+    }
+
+    getMedia(targets) {
+        return new Promise((resolve, reject) => {
+            const count = targets.length;
+            const results = new Array(count).fill(null);
+            if (count === 0) {
+                resolve(results);
+                return;
+            }
+
+            let completeCount = 0;
+            const transaction = this._db.transaction(['media'], 'readonly');
             const objectStore = transaction.objectStore('media');
             const index = objectStore.index('path');
 
             for (let i = 0; i < count; ++i) {
+                const inputIndex = i;
                 const {path, dictionaryName} = targets[i];
-            const only = IDBKeyRange.only(path);
+                const query = IDBKeyRange.only(path);
-            promises.push(Database._getAll(index, only, [i, dictionaryName], processRow));
+
+                const onGetAll = (rows) => {
+                    for (const row of rows) {
+                        if (row.dictionary !== dictionaryName) { continue; }
+                        results[inputIndex] = this._createMedia(row, inputIndex);
                     }
-
+                    if (++completeCount >= count) {
-        await Promise.all(promises);
+                        resolve(results);
-
-        return results;
-    }
-
-    async getDictionaryInfo() {
-        this._validate();
-
-        const results = [];
-        const dbTransaction = this.db.transaction(['dictionaries'], 'readonly');
-        const dbDictionaries = dbTransaction.objectStore('dictionaries');
-
-        await Database._getAll(dbDictionaries, null, null, (info) => results.push(info));
-
-        return results;
-    }
-
-    async getDictionaryCounts(dictionaryNames, getTotal) {
-        this._validate();
-
-        const objectStoreNames = [
-            'kanji',
-            'kanjiMeta',
-            'terms',
-            'termMeta',
-            'tagMeta'
-        ];
-        const dbCountTransaction = this.db.transaction(objectStoreNames, 'readonly');
-
-        const targets = [];
-        for (const objectStoreName of objectStoreNames) {
-            targets.push([
-                objectStoreName,
-                dbCountTransaction.objectStore(objectStoreName).index('dictionary')
-            ]);
-        }
-
-        // Query is required for Edge, otherwise index.count throws an exception.
-        const query1 = IDBKeyRange.lowerBound('', false);
-        const totalPromise = getTotal ? Database._getCounts(targets, query1) : null;
-
-        const counts = [];
-        const countPromises = [];
-        for (let i = 0; i < dictionaryNames.length; ++i) {
-            counts.push(null);
-            const index = i;
-            const query2 = IDBKeyRange.only(dictionaryNames[i]);
-            const countPromise = Database._getCounts(targets, query2).then((v) => counts[index] = v);
-            countPromises.push(countPromise);
-        }
-        await Promise.all(countPromises);
-
-        const result = {counts};
-        if (totalPromise !== null) {
-            result.total = await totalPromise;
-        }
-        return result;
-    }
-
-    async dictionaryExists(title) {
-        this._validate();
-        const transaction = this.db.transaction(['dictionaries'], 'readonly');
-        const index = transaction.objectStore('dictionaries').index('title');
-        const query = IDBKeyRange.only(title);
-        const count = await Database._getCount(index, query);
-        return count > 0;
-    }
-
-    bulkAdd(objectStoreName, items, start, count) {
-        return new Promise((resolve, reject) => {
-            const transaction = this.db.transaction([objectStoreName], 'readwrite');
-            const objectStore = transaction.objectStore(objectStoreName);
-
-            if (start + count > items.length) {
-                count = items.length - start;
-            }
-
-            if (count <= 0) {
-                resolve();
-                return;
-            }
-
-            const end = start + count;
-            let completedCount = 0;
-            const onError = (e) => reject(e);
-            const onSuccess = () => {
-                if (++completedCount >= count) {
-                    resolve();
                     }
                 };
 
-            for (let i = start; i < end; ++i) {
+                this._db.getAll(index, query, onGetAll, reject);
-                const request = objectStore.add(items[i]);
-                request.onerror = onError;
-                request.onsuccess = onSuccess;
             }
         });
     }
 
+    getDictionaryInfo() {
+        return new Promise((resolve, reject) => {
+            const transaction = this._db.transaction(['dictionaries'], 'readonly');
+            const objectStore = transaction.objectStore('dictionaries');
+            this._db.getAll(objectStore, null, resolve, reject);
+        });
+    }
+
+    getDictionaryCounts(dictionaryNames, getTotal) {
+        return new Promise((resolve, reject) => {
+            const targets = [
+                ['kanji', 'dictionary'],
+                ['kanjiMeta', 'dictionary'],
+                ['terms', 'dictionary'],
+                ['termMeta', 'dictionary'],
+                ['tagMeta', 'dictionary'],
+                ['media', 'dictionary']
+            ];
+            const objectStoreNames = targets.map(([objectStoreName]) => objectStoreName);
+            const transaction = this._db.transaction(objectStoreNames, 'readonly');
+            const databaseTargets = targets.map(([objectStoreName, indexName]) => {
+                const objectStore = transaction.objectStore(objectStoreName);
+                const index = objectStore.index(indexName);
+                return {objectStore, index};
+            });
+
+            const countTargets = [];
+            if (getTotal) {
+                for (const {objectStore} of databaseTargets) {
+                    countTargets.push([objectStore, null]);
+                }
+            }
+            for (const dictionaryName of dictionaryNames) {
+                const query = IDBKeyRange.only(dictionaryName);
+                for (const {index} of databaseTargets) {
+                    countTargets.push([index, query]);
+                }
+            }
+
+            const onCountComplete = (results) => {
+                const resultCount = results.length;
+                const targetCount = targets.length;
+                const counts = [];
+                for (let i = 0; i < resultCount; i += targetCount) {
+                    const countGroup = {};
+                    for (let j = 0; j < targetCount; ++j) {
+                        countGroup[targets[j][0]] = results[i + j];
+                    }
+                    counts.push(countGroup);
+                }
+                const total = getTotal ? counts.shift() : null;
+                resolve({total, counts});
+            };
+
+            this._db.bulkCount(countTargets, onCountComplete, reject);
+        });
+    }
+
+    async dictionaryExists(title) {
+        const query = IDBKeyRange.only(title);
+        const result = await this._db.find('dictionaries', 'title', query);
+        return typeof result !== 'undefined';
+    }
+
+    bulkAdd(objectStoreName, items, start, count) {
+        return this._db.bulkAdd(objectStoreName, items, start, count);
+    }
+
     // Private
 
-    _validate() {
+    async _findGenericBulk(objectStoreName, indexName, indexValueList, dictionaries, createResult) {
-        if (this.db === null) {
+        return new Promise((resolve, reject) => {
-            throw new Error('Database not initialized');
-        }
-    }
-
-    async _findGenericBulk(tableName, indexName, indexValueList, dictionaries, createResult) {
-        this._validate();
-
-        const promises = [];
             const results = [];
-        const processRow = (row, index) => {
+            const count = indexValueList.length;
+            if (count === 0) {
+                resolve(results);
+                return;
+            }
+
+            const transaction = this._db.transaction([objectStoreName], 'readonly');
+            const terms = transaction.objectStore(objectStoreName);
+            const index = terms.index(indexName);
+
+            let completeCount = 0;
+            for (let i = 0; i < count; ++i) {
+                const inputIndex = i;
+                const query = IDBKeyRange.only(indexValueList[i]);
+
+                const onGetAll = (rows) => {
+                    for (const row of rows) {
                         if (dictionaries.has(row.dictionary)) {
-                results.push(createResult(row, index));
+                            results.push(createResult(row, inputIndex));
+                        }
+                    }
+                    if (++completeCount >= count) {
+                        resolve(results);
                     }
                 };
 
-        const dbTransaction = this.db.transaction([tableName], 'readonly');
+                this._db.getAll(index, query, onGetAll, reject);
-        const dbTerms = dbTransaction.objectStore(tableName);
+            }
-        const dbIndex = dbTerms.index(indexName);
+        });
-
-        for (let i = 0; i < indexValueList.length; ++i) {
-            const only = IDBKeyRange.only(indexValueList[i]);
-            promises.push(Database._getAll(dbIndex, only, i, processRow));
     }
 
-        await Promise.all(promises);
+    _createTerm(row, index) {
-
-        return results;
-    }
-
-    static _createTerm(row, index) {
         return {
             index,
             expression: row.expression,
@@ -448,7 +447,7 @@ class Database {
         };
     }
 
-    static _createKanji(row, index) {
+    _createKanji(row, index) {
         return {
             index,
             character: row.character,
@@ -461,193 +460,15 @@ class Database {
         };
     }
 
-    static _createTermMeta({expression, mode, data, dictionary}, index) {
+    _createTermMeta({expression, mode, data, dictionary}, index) {
         return {expression, mode, data, dictionary, index};
     }
 
-    static _createKanjiMeta({character, mode, data, dictionary}, index) {
+    _createKanjiMeta({character, mode, data, dictionary}, index) {
         return {character, mode, data, dictionary, index};
     }
 
-    static _createMedia(row, index) {
+    _createMedia(row, index) {
         return Object.assign({}, row, {index});
     }
-
-    static _getAll(dbIndex, query, context, processRow) {
-        const fn = typeof dbIndex.getAll === 'function' ? Database._getAllFast : Database._getAllUsingCursor;
-        return fn(dbIndex, query, context, processRow);
-    }
-
-    static _getAllFast(dbIndex, query, context, processRow) {
-        return new Promise((resolve, reject) => {
-            const request = dbIndex.getAll(query);
-            request.onerror = (e) => reject(e);
-            request.onsuccess = (e) => {
-                for (const row of e.target.result) {
-                    processRow(row, context);
-                }
-                resolve();
-            };
-        });
-    }
-
-    static _getAllUsingCursor(dbIndex, query, context, processRow) {
-        return new Promise((resolve, reject) => {
-            const request = dbIndex.openCursor(query, 'next');
-            request.onerror = (e) => reject(e);
-            request.onsuccess = (e) => {
-                const cursor = e.target.result;
-                if (cursor) {
-                    processRow(cursor.value, context);
-                    cursor.continue();
-                } else {
-                    resolve();
-                }
-            };
-        });
-    }
-
-    static _getCounts(targets, query) {
-        const countPromises = [];
-        const counts = {};
-        for (const [objectStoreName, index] of targets) {
-            const n = objectStoreName;
-            const countPromise = Database._getCount(index, query).then((count) => counts[n] = count);
-            countPromises.push(countPromise);
-        }
-        return Promise.all(countPromises).then(() => counts);
-    }
-
-    static _getCount(dbIndex, query) {
-        return new Promise((resolve, reject) => {
-            const request = dbIndex.count(query);
-            request.onerror = (e) => reject(e);
-            request.onsuccess = (e) => resolve(e.target.result);
-        });
-    }
-
-    static _getAllKeys(dbIndex, query) {
-        const fn = typeof dbIndex.getAllKeys === 'function' ? Database._getAllKeysFast : Database._getAllKeysUsingCursor;
-        return fn(dbIndex, query);
-    }
-
-    static _getAllKeysFast(dbIndex, query) {
-        return new Promise((resolve, reject) => {
-            const request = dbIndex.getAllKeys(query);
-            request.onerror = (e) => reject(e);
-            request.onsuccess = (e) => resolve(e.target.result);
-        });
-    }
-
-    static _getAllKeysUsingCursor(dbIndex, query) {
-        return new Promise((resolve, reject) => {
-            const primaryKeys = [];
-            const request = dbIndex.openKeyCursor(query, 'next');
-            request.onerror = (e) => reject(e);
-            request.onsuccess = (e) => {
-                const cursor = e.target.result;
-                if (cursor) {
-                    primaryKeys.push(cursor.primaryKey);
-                    cursor.continue();
-                } else {
-                    resolve(primaryKeys);
-                }
-            };
-        });
-    }
-
-    static async _deleteValues(dbObjectStore, dbIndex, query, onProgress, progressData, progressRate) {
-        const hasProgress = (typeof onProgress === 'function');
-        const count = await Database._getCount(dbIndex, query);
-        ++progressData.storesProcesed;
-        progressData.count += count;
-        if (hasProgress) {
-            onProgress(progressData);
-        }
-
-        const onValueDeleted = (
-            hasProgress ?
-            () => {
-                const p = ++progressData.processed;
-                if ((p % progressRate) === 0 || p === progressData.count) {
-                    onProgress(progressData);
-                }
-            } :
-            () => {}
-        );
-
-        const promises = [];
-        const primaryKeys = await Database._getAllKeys(dbIndex, query);
-        for (const key of primaryKeys) {
-            const promise = Database._deleteValue(dbObjectStore, key).then(onValueDeleted);
-            promises.push(promise);
-        }
-
-        await Promise.all(promises);
-    }
-
-    static _deleteValue(dbObjectStore, key) {
-        return new Promise((resolve, reject) => {
-            const request = dbObjectStore.delete(key);
-            request.onerror = (e) => reject(e);
-            request.onsuccess = () => resolve();
-        });
-    }
-
-    static _open(name, version, onUpgradeNeeded) {
-        return new Promise((resolve, reject) => {
-            const request = indexedDB.open(name, version * 10);
-
-            request.onupgradeneeded = (event) => {
-                try {
-                    request.transaction.onerror = (e) => reject(e);
-                    onUpgradeNeeded(request.result, request.transaction, event.oldVersion / 10, event.newVersion / 10);
-                } catch (e) {
-                    reject(e);
-                }
-            };
-
-            request.onerror = (e) => reject(e);
-            request.onsuccess = () => resolve(request.result);
-        });
-    }
-
-    static _upgrade(db, transaction, oldVersion, upgrades) {
-        for (const {version, stores} of upgrades) {
-            if (oldVersion >= version) { continue; }
-
-            const objectStoreNames = Object.keys(stores);
-            for (const objectStoreName of objectStoreNames) {
-                const {primaryKey, indices} = stores[objectStoreName];
-
-                const objectStoreNames2 = transaction.objectStoreNames || db.objectStoreNames;
-                const objectStore = (
-                    Database._listContains(objectStoreNames2, objectStoreName) ?
-                    transaction.objectStore(objectStoreName) :
-                    db.createObjectStore(objectStoreName, primaryKey)
-                );
-
-                for (const indexName of indices) {
-                    if (Database._listContains(objectStore.indexNames, indexName)) { continue; }
-
-                    objectStore.createIndex(indexName, indexName, {});
-                }
-            }
-        }
-    }
-
-    static _deleteDatabase(dbName) {
-        return new Promise((resolve, reject) => {
-            const request = indexedDB.deleteDatabase(dbName);
-            request.onerror = (e) => reject(e);
-            request.onsuccess = () => resolve();
-        });
-    }
-
-    static _listContains(list, value) {
-        for (let i = 0, ii = list.length; i < ii; ++i) {
-            if (list[i] === value) { return true; }
-        }
-        return false;
-    }
 }

ext/bg/js/generic-database.js

@@ -0,0 +1,327 @@
+/*
+ * Copyright (C) 2020  Yomichan Authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+class GenericDatabase {
+    constructor() {
+        this._db = null;
+        this._isOpening = false;
+    }
+
+    // Public
+
+    async open(databaseName, version, structure) {
+        if (this._db !== null) {
+            throw new Error('Database already open');
+        }
+        if (this._isOpening) {
+            throw new Error('Already opening');
+        }
+
+        try {
+            this._isOpening = true;
+            this._db = await this._open(databaseName, version, (db, transaction, oldVersion) => {
+                this._upgrade(db, transaction, oldVersion, structure);
+            });
+        } finally {
+            this._isOpening = false;
+        }
+    }
+
+    close() {
+        if (this._db === null) {
+            throw new Error('Database is not open');
+        }
+
+        this._db.close();
+        this._db = null;
+    }
+
+    isOpening() {
+        return this._isOpening;
+    }
+
+    isOpen() {
+        return this._db !== null;
+    }
+
+    transaction(storeNames, mode) {
+        if (this._db === null) {
+            throw new Error(this._isOpening ? 'Database not ready' : 'Database not open');
+        }
+        return this._db.transaction(storeNames, mode);
+    }
+
+    bulkAdd(objectStoreName, items, start, count) {
+        return new Promise((resolve, reject) => {
+            if (start + count > items.length) {
+                count = items.length - start;
+            }
+
+            if (count <= 0) {
+                resolve();
+                return;
+            }
+
+            const end = start + count;
+            let completedCount = 0;
+            const onError = (e) => reject(e.target.error);
+            const onSuccess = () => {
+                if (++completedCount >= count) {
+                    resolve();
+                }
+            };
+
+            const transaction = this.transaction([objectStoreName], 'readwrite');
+            const objectStore = transaction.objectStore(objectStoreName);
+            for (let i = start; i < end; ++i) {
+                const request = objectStore.add(items[i]);
+                request.onerror = onError;
+                request.onsuccess = onSuccess;
+            }
+        });
+    }
+
+    getAll(objectStoreOrIndex, query, resolve, reject) {
+        if (typeof objectStoreOrIndex.getAll === 'function') {
+            this._getAllFast(objectStoreOrIndex, query, resolve, reject);
+        } else {
+            this._getAllUsingCursor(objectStoreOrIndex, query, resolve, reject);
+        }
+    }
+
+    getAllKeys(objectStoreOrIndex, query, resolve, reject) {
+        if (typeof objectStoreOrIndex.getAll === 'function') {
+            this._getAllKeysFast(objectStoreOrIndex, query, resolve, reject);
+        } else {
+            this._getAllKeysUsingCursor(objectStoreOrIndex, query, resolve, reject);
+        }
+    }
+
+    find(objectStoreName, indexName, query, predicate=null, defaultValue) {
+        return new Promise((resolve, reject) => {
+            const transaction = this.transaction([objectStoreName], 'readonly');
+            const objectStore = transaction.objectStore(objectStoreName);
+            const objectStoreOrIndex = indexName !== null ? objectStore.index(indexName) : objectStore;
+            const request = objectStoreOrIndex.openCursor(query, 'next');
+            request.onerror = (e) => reject(e.target.error);
+            request.onsuccess = (e) => {
+                const cursor = e.target.result;
+                if (cursor) {
+                    const value = cursor.value;
+                    if (typeof predicate !== 'function' || predicate(value)) {
+                        resolve(value);
+                    } else {
+                        cursor.continue();
+                    }
+                } else {
+                    resolve(defaultValue);
+                }
+            };
+        });
+    }
+
+    bulkCount(targets, resolve, reject) {
+        const targetCount = targets.length;
+        if (targetCount <= 0) {
+            resolve();
+            return;
+        }
+
+        let completedCount = 0;
+        const results = new Array(targetCount).fill(null);
+
+        const onError = (e) => reject(e.target.error);
+        const onSuccess = (e, index) => {
+            const count = e.target.result;
+            results[index] = count;
+            if (++completedCount >= targetCount) {
+                resolve(results);
+            }
+        };
+
+        for (let i = 0; i < targetCount; ++i) {
+            const index = i;
+            const [objectStoreOrIndex, query] = targets[i];
+            const request = objectStoreOrIndex.count(query);
+            request.onerror = onError;
+            request.onsuccess = (e) => onSuccess(e, index);
+        }
+    }
+
+    delete(objectStoreName, key) {
+        return new Promise((resolve, reject) => {
+            const transaction = this.transaction([objectStoreName], 'readwrite');
+            const objectStore = transaction.objectStore(objectStoreName);
+            const request = objectStore.delete(key);
+            request.onerror = (e) => reject(e.target.error);
+            request.onsuccess = () => resolve();
+        });
+    }
+
+    bulkDelete(objectStoreName, indexName, query, filterKeys=null, onProgress=null) {
+        return new Promise((resolve, reject) => {
+            const transaction = this.transaction([objectStoreName], 'readwrite');
+            const objectStore = transaction.objectStore(objectStoreName);
+            const objectStoreOrIndex = indexName !== null ? objectStore.index(indexName) : objectStore;
+
+            const onGetKeys = (keys) => {
+                try {
+                    if (typeof filterKeys === 'function') {
+                        keys = filterKeys(keys);
+                    }
+                    this._bulkDeleteInternal(objectStore, keys, onProgress, resolve, reject);
+                } catch (e) {
+                    reject(e);
+                }
+            };
+
+            this.getAllKeys(objectStoreOrIndex, query, onGetKeys, reject);
+        });
+    }
+
+    static deleteDatabase(databaseName) {
+        return new Promise((resolve, reject) => {
+            const request = indexedDB.deleteDatabase(databaseName);
+            request.onerror = (e) => reject(e.target.error);
+            request.onsuccess = () => resolve();
+            request.onblocked = () => reject(new Error('Database deletion blocked'));
+        });
+    }
+
+    // Private
+
+    _open(name, version, onUpgradeNeeded) {
+        return new Promise((resolve, reject) => {
+            const request = indexedDB.open(name, version);
+
+            request.onupgradeneeded = (event) => {
+                try {
+                    request.transaction.onerror = (e) => reject(e.target.error);
+                    onUpgradeNeeded(request.result, request.transaction, event.oldVersion, event.newVersion);
+                } catch (e) {
+                    reject(e);
+                }
+            };
+
+            request.onerror = (e) => reject(e.target.error);
+            request.onsuccess = () => resolve(request.result);
+        });
+    }
+
+    _upgrade(db, transaction, oldVersion, upgrades) {
+        for (const {version, stores} of upgrades) {
+            if (oldVersion >= version) { continue; }
+
+            for (const [objectStoreName, {primaryKey, indices}] of Object.entries(stores)) {
+                const existingObjectStoreNames = transaction.objectStoreNames || db.objectStoreNames;
+                const objectStore = (
+                    this._listContains(existingObjectStoreNames, objectStoreName) ?
+                    transaction.objectStore(objectStoreName) :
+                    db.createObjectStore(objectStoreName, primaryKey)
+                );
+                const existingIndexNames = objectStore.indexNames;
+
+                for (const indexName of indices) {
+                    if (this._listContains(existingIndexNames, indexName)) { continue; }
+
+                    objectStore.createIndex(indexName, indexName, {});
+                }
+            }
+        }
+    }
+
+    _listContains(list, value) {
+        for (let i = 0, ii = list.length; i < ii; ++i) {
+            if (list[i] === value) { return true; }
+        }
+        return false;
+    }
+
+    _getAllFast(objectStoreOrIndex, query, resolve, reject) {
+        const request = objectStoreOrIndex.getAll(query);
+        request.onerror = (e) => reject(e.target.error);
+        request.onsuccess = (e) => resolve(e.target.result);
+    }
+
+    _getAllUsingCursor(objectStoreOrIndex, query, resolve, reject) {
+        const results = [];
+        const request = objectStoreOrIndex.openCursor(query, 'next');
+        request.onerror = (e) => reject(e.target.error);
+        request.onsuccess = (e) => {
+            const cursor = e.target.result;
+            if (cursor) {
+                results.push(cursor.value);
+                cursor.continue();
+            } else {
+                resolve(results);
+            }
+        };
+    }
+
+    _getAllKeysFast(objectStoreOrIndex, query, resolve, reject) {
+        const request = objectStoreOrIndex.getAllKeys(query);
+        request.onerror = (e) => reject(e.target.error);
+        request.onsuccess = (e) => resolve(e.target.result);
+    }
+
+    _getAllKeysUsingCursor(objectStoreOrIndex, query, resolve, reject) {
+        const results = [];
+        const request = objectStoreOrIndex.openKeyCursor(query, 'next');
+        request.onerror = (e) => reject(e.target.error);
+        request.onsuccess = (e) => {
+            const cursor = e.target.result;
+            if (cursor) {
+                results.push(cursor.primaryKey);
+                cursor.continue();
+            } else {
+                resolve(results);
+            }
+        };
+    }
+
+    _bulkDeleteInternal(objectStore, keys, onProgress, resolve, reject) {
+        const count = keys.length;
+        if (count === 0) {
+            resolve();
+            return;
+        }
+
+        let completedCount = 0;
+        const hasProgress = (typeof onProgress === 'function');
+
+        const onError = (e) => reject(e.target.error);
+        const onSuccess = () => {
+            ++completedCount;
+            if (hasProgress) {
+                try {
+                    onProgress(completedCount, count);
+                } catch (e) {
+                    // NOP
+                }
+            }
+            if (completedCount >= count) {
+                resolve();
+            }
+        };
+
+        for (const key of keys) {
+            const request = objectStore.delete(key);
+            request.onerror = onError;
+            request.onsuccess = onSuccess;
+        }
+    }
+}

test/test-database.js

@@ -115,6 +115,7 @@ vm.execute([
     'bg/js/media-utility.js',
     'bg/js/request.js',
     'bg/js/dictionary-importer.js',
+    'bg/js/generic-database.js',
     'bg/js/database.js'
 ]);
 const DictionaryImporter = vm.get('DictionaryImporter');
@@ -242,8 +243,8 @@ async function testDatabase1() {
             true
         );
         vm.assert.deepStrictEqual(counts, {
-            counts: [{kanji: 2, kanjiMeta: 2, terms: 33, termMeta: 12, tagMeta: 14}],
+            counts: [{kanji: 2, kanjiMeta: 2, terms: 33, termMeta: 12, tagMeta: 14, media: 1}],
-            total: {kanji: 2, kanjiMeta: 2, terms: 33, termMeta: 12, tagMeta: 14}
+            total: {kanji: 2, kanjiMeta: 2, terms: 33, termMeta: 12, tagMeta: 14, media: 1}
         });
 
         // Test find* functions
@@ -269,7 +270,7 @@ async function testDatabaseEmpty1(database) {
     const counts = await database.getDictionaryCounts([], true);
     vm.assert.deepStrictEqual(counts, {
         counts: [],
-        total: {kanji: 0, kanjiMeta: 0, terms: 0, termMeta: 0, tagMeta: 0}
+        total: {kanji: 0, kanjiMeta: 0, terms: 0, termMeta: 0, tagMeta: 0, media: 0}
     });
 }
 
@@ -863,8 +864,7 @@ async function testDatabase2() {
     const database = new Database();
 
     // Error: not prepared
-    await assert.rejects(async () => await database.purge());
+    await assert.rejects(async () => await database.deleteDictionary(title, {rate: 1000}, () => {}));
-    await assert.rejects(async () => await database.deleteDictionary(title, {}, () => {}));
     await assert.rejects(async () => await database.findTermsBulk(['?'], titles, null));
     await assert.rejects(async () => await database.findTermsExactBulk(['?'], ['?'], titles));
     await assert.rejects(async () => await database.findTermsBySequenceBulk([1], title));