yomichan/ext/bg/js/database.js

605 lines
19 KiB
JavaScript
Raw Normal View History

2016-03-20 02:32:35 +00:00
/*
2017-08-15 04:43:09 +00:00
* Copyright (C) 2016-2017 Alex Yatskov <alex@foosoft.net>
2016-03-20 02:32:35 +00:00
* Author: Alex Yatskov <alex@foosoft.net>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
2016-11-07 16:29:21 +00:00
class Database {
2016-03-21 00:15:40 +00:00
constructor() {
2016-09-12 05:47:08 +00:00
this.db = null;
2017-09-14 00:26:02 +00:00
this.tagCache = {};
2016-08-21 20:32:36 +00:00
}
2017-07-10 20:16:24 +00:00
async prepare() {
2017-07-09 23:29:52 +00:00
if (this.db) {
2019-10-08 00:46:02 +00:00
throw new Error('Database already initialized');
2016-09-13 22:59:18 +00:00
}
2017-07-10 20:16:24 +00:00
this.db = new Dexie('dict');
2017-09-11 04:49:37 +00:00
this.db.version(2).stores({
terms: '++id,dictionary,expression,reading',
2017-09-11 04:49:37 +00:00
kanji: '++,dictionary,character',
tagMeta: '++,dictionary',
2017-07-10 20:16:24 +00:00
dictionaries: '++,title,version'
2016-12-30 18:47:27 +00:00
});
2017-09-11 04:49:37 +00:00
this.db.version(3).stores({
2017-09-23 15:46:34 +00:00
termMeta: '++,dictionary,expression',
kanjiMeta: '++,dictionary,character',
tagMeta: '++,dictionary,name'
2017-09-11 04:49:37 +00:00
});
this.db.version(4).stores({
terms: '++id,dictionary,expression,reading,sequence'
});
2017-07-10 20:16:24 +00:00
await this.db.open();
2016-03-21 00:15:40 +00:00
}
2017-07-10 20:16:24 +00:00
async purge() {
2019-10-08 00:46:02 +00:00
this.validate();
2016-11-14 03:10:28 +00:00
this.db.close();
2017-07-10 20:16:24 +00:00
await this.db.delete();
this.db = null;
2017-09-14 00:26:02 +00:00
this.tagCache = {};
2017-07-10 20:16:24 +00:00
await this.prepare();
2016-11-14 03:10:28 +00:00
}
2019-10-19 14:09:18 +00:00
async findTermsBulk(termList, titles) {
2019-10-19 03:04:06 +00:00
this.validate();
2019-08-31 01:06:21 +00:00
const promises = [];
const visited = {};
const results = [];
const processRow = (row, index) => {
if (titles.includes(row.dictionary) && !visited.hasOwnProperty(row.id)) {
visited[row.id] = true;
2019-10-19 03:09:00 +00:00
results.push(Database.createTerm(row, index));
}
};
2019-08-31 01:06:21 +00:00
const db = this.db.backendDB();
const dbTransaction = db.transaction(['terms'], 'readonly');
const dbTerms = dbTransaction.objectStore('terms');
const dbIndex1 = dbTerms.index('expression');
const dbIndex2 = dbTerms.index('reading');
2019-10-19 14:09:18 +00:00
for (let i = 0; i < termList.length; ++i) {
const only = IDBKeyRange.only(termList[i]);
2019-08-31 01:06:21 +00:00
promises.push(
Database.getAll(dbIndex1, only, i, processRow),
Database.getAll(dbIndex2, only, i, processRow)
2019-08-31 01:06:21 +00:00
);
}
await Promise.all(promises);
return results;
}
2017-10-15 02:19:16 +00:00
async findTermsExact(term, reading, titles) {
2019-10-08 00:46:02 +00:00
this.validate();
2017-10-15 02:19:16 +00:00
const results = [];
await this.db.terms.where('expression').equals(term).each(row => {
if (row.reading === reading && titles.includes(row.dictionary)) {
results.push(Database.createTerm(row));
2017-10-15 02:19:16 +00:00
}
});
return results;
}
2019-10-19 14:09:18 +00:00
async findTermsExactBulk(termList, readingList, titles) {
this.validate();
const promises = [];
const results = [];
const processRow = (row, index) => {
if (row.reading === readingList[index] && titles.includes(row.dictionary)) {
results.push(Database.createTerm(row, index));
}
};
const db = this.db.backendDB();
const dbTransaction = db.transaction(['terms'], 'readonly');
const dbTerms = dbTransaction.objectStore('terms');
const dbIndex = dbTerms.index('expression');
for (let i = 0; i < termList.length; ++i) {
const only = IDBKeyRange.only(termList[i]);
promises.push(Database.getAll(dbIndex, only, i, processRow));
}
await Promise.all(promises);
return results;
}
async findTermsBySequence(sequence, mainDictionary) {
2019-10-08 00:46:02 +00:00
this.validate();
const results = [];
await this.db.terms.where('sequence').equals(sequence).each(row => {
if (row.dictionary === mainDictionary) {
results.push(Database.createTerm(row));
}
});
return results;
}
2019-10-19 14:09:18 +00:00
async findTermsBySequenceBulk(sequenceList, mainDictionary) {
2019-10-19 03:04:06 +00:00
this.validate();
2019-08-31 01:06:21 +00:00
const promises = [];
const results = [];
const processRow = (row, index) => {
2019-10-19 14:09:18 +00:00
if (row.dictionary === mainDictionary) {
results.push(Database.createTerm(row, index));
}
};
2019-08-31 01:06:21 +00:00
const db = this.db.backendDB();
2019-10-19 14:09:18 +00:00
const dbTransaction = db.transaction(['terms'], 'readonly');
const dbTerms = dbTransaction.objectStore('terms');
const dbIndex = dbTerms.index('sequence');
2019-08-31 01:06:21 +00:00
2019-10-19 14:09:18 +00:00
for (let i = 0; i < sequenceList.length; ++i) {
const only = IDBKeyRange.only(sequenceList[i]);
promises.push(Database.getAll(dbIndex, only, i, processRow));
2019-08-31 01:06:21 +00:00
}
await Promise.all(promises);
return results;
}
2019-10-19 14:09:18 +00:00
async findTermMetaBulk(termList, titles) {
return this.findGenericBulk('termMeta', 'expression', termList, titles, Database.createMeta);
}
2017-07-10 21:10:58 +00:00
async findKanji(kanji, titles) {
2019-10-08 00:46:02 +00:00
this.validate();
2016-09-12 05:47:08 +00:00
2016-08-22 02:51:12 +00:00
const results = [];
2017-07-10 20:16:24 +00:00
await this.db.kanji.where('character').equals(kanji).each(row => {
2017-07-10 21:10:58 +00:00
if (titles.includes(row.dictionary)) {
results.push(Database.createKanji(row));
2016-11-13 03:29:30 +00:00
}
});
2017-07-10 20:16:24 +00:00
return results;
}
2016-08-24 03:33:04 +00:00
2019-10-19 14:09:18 +00:00
async findKanjiBulk(kanjiList, titles) {
return this.findGenericBulk('kanji', 'character', kanjiList, titles, Database.createKanji);
}
2017-09-23 15:46:34 +00:00
async findKanjiMeta(kanji, titles) {
2019-10-08 00:46:02 +00:00
this.validate();
2017-09-13 03:20:03 +00:00
const results = [];
2017-09-23 15:46:34 +00:00
await this.db.kanjiMeta.where('character').equals(kanji).each(row => {
2017-09-13 03:20:03 +00:00
if (titles.includes(row.dictionary)) {
results.push(Database.createMeta(row));
2017-09-13 03:20:03 +00:00
}
});
return results;
}
2019-10-19 14:09:18 +00:00
async findKanjiMetaBulk(kanjiList, titles) {
return this.findGenericBulk('kanjiMeta', 'character', kanjiList, titles, Database.createMeta);
}
async findGenericBulk(tableName, indexName, indexValueList, titles, createResult) {
this.validate();
const promises = [];
const results = [];
const processRow = (row, index) => {
if (titles.includes(row.dictionary)) {
results.push(createResult(row, index));
}
};
const db = this.db.backendDB();
const dbTransaction = db.transaction([tableName], 'readonly');
const dbTerms = dbTransaction.objectStore(tableName);
const dbIndex = dbTerms.index(indexName);
for (let i = 0; i < indexValueList.length; ++i) {
const only = IDBKeyRange.only(indexValueList[i]);
promises.push(Database.getAll(dbIndex, only, i, processRow));
}
await Promise.all(promises);
return results;
}
2017-09-13 03:20:03 +00:00
findTagForTitleCached(name, title) {
if (this.tagCache.hasOwnProperty(title)) {
const cache = this.tagCache[title];
if (cache.hasOwnProperty(name)) {
return cache[name];
}
}
}
2017-09-14 01:03:55 +00:00
async findTagForTitle(name, title) {
2019-10-08 00:46:02 +00:00
this.validate();
2016-09-12 05:47:08 +00:00
const cache = (this.tagCache.hasOwnProperty(title) ? this.tagCache[title] : (this.tagCache[title] = {}));
2017-09-14 00:26:02 +00:00
let result = null;
await this.db.tagMeta.where('name').equals(name).each(row => {
if (title === row.dictionary) {
result = row;
}
});
2017-09-14 00:26:02 +00:00
cache[name] = result;
2017-09-13 23:42:04 +00:00
return result;
2016-08-24 05:22:09 +00:00
}
2016-08-24 03:33:04 +00:00
2017-10-29 17:42:39 +00:00
async summarize() {
2019-10-08 00:46:02 +00:00
this.validate();
return this.db.dictionaries.toArray();
2016-11-06 01:24:45 +00:00
}
2019-02-27 02:01:32 +00:00
async importDictionary(archive, progressCallback, exceptions) {
2019-10-08 00:46:02 +00:00
this.validate();
2016-08-24 05:22:09 +00:00
2019-02-27 02:01:32 +00:00
const maxTransactionLength = 1000;
const bulkAdd = async (table, items, total, current) => {
if (items.length < maxTransactionLength) {
if (progressCallback) {
progressCallback(total, current);
}
try {
await table.bulkAdd(items);
} catch (e) {
if (exceptions) {
exceptions.push(e);
} else {
throw e;
}
}
} else {
for (let i = 0; i < items.length; i += maxTransactionLength) {
if (progressCallback) {
progressCallback(total, current + i / items.length);
}
let count = Math.min(maxTransactionLength, items.length - i);
try {
await table.bulkAdd(items.slice(i, i + count));
} catch (e) {
if (exceptions) {
exceptions.push(e);
} else {
throw e;
}
}
}
}
};
const indexDataLoaded = async summary => {
2017-10-29 19:20:56 +00:00
if (summary.version > 3) {
2019-10-08 00:46:02 +00:00
throw new Error('Unsupported dictionary version');
2017-09-17 19:56:34 +00:00
}
2017-09-11 04:49:37 +00:00
const count = await this.db.dictionaries.where('title').equals(summary.title).count();
if (count > 0) {
2019-10-08 00:46:02 +00:00
throw new Error('Dictionary is already imported');
2017-07-10 20:16:24 +00:00
}
2017-09-11 04:49:37 +00:00
await this.db.dictionaries.add(summary);
2016-11-05 23:44:29 +00:00
};
2016-08-24 03:33:04 +00:00
2017-09-17 19:56:34 +00:00
const termDataLoaded = async (summary, entries, total, current) => {
2016-11-05 23:44:29 +00:00
const rows = [];
2017-09-17 19:56:34 +00:00
if (summary.version === 1) {
for (const [expression, reading, definitionTags, rules, score, ...glossary] of entries) {
2017-09-17 19:56:34 +00:00
rows.push({
expression,
reading,
definitionTags,
2017-09-17 19:56:34 +00:00
rules,
score,
glossary,
dictionary: summary.title
});
}
} else {
for (const [expression, reading, definitionTags, rules, score, glossary, sequence, termTags] of entries) {
2017-09-17 19:56:34 +00:00
rows.push({
expression,
reading,
definitionTags,
2017-09-17 19:56:34 +00:00
rules,
score,
glossary,
sequence,
termTags,
2017-09-17 19:56:34 +00:00
dictionary: summary.title
});
}
2016-11-05 23:44:29 +00:00
}
2016-08-24 03:33:04 +00:00
2019-02-27 02:01:32 +00:00
await bulkAdd(this.db.terms, rows, total, current);
2016-11-05 23:44:29 +00:00
};
2017-09-23 15:46:34 +00:00
const termMetaDataLoaded = async (summary, entries, total, current) => {
2017-09-13 01:29:16 +00:00
const rows = [];
2017-09-23 15:46:34 +00:00
for (const [expression, mode, data] of entries) {
2017-09-13 01:29:16 +00:00
rows.push({
expression,
2017-09-23 15:46:34 +00:00
mode,
data,
2017-09-17 19:56:34 +00:00
dictionary: summary.title
2017-09-13 01:29:16 +00:00
});
}
2019-02-27 02:01:32 +00:00
await bulkAdd(this.db.termMeta, rows, total, current);
2017-09-13 01:29:16 +00:00
};
2017-09-17 19:56:34 +00:00
const kanjiDataLoaded = async (summary, entries, total, current) => {
2016-11-05 23:44:29 +00:00
const rows = [];
2017-09-17 19:56:34 +00:00
if (summary.version === 1) {
for (const [character, onyomi, kunyomi, tags, ...meanings] of entries) {
rows.push({
character,
onyomi,
kunyomi,
tags,
meanings,
dictionary: summary.title
});
}
} else {
2017-09-18 02:57:39 +00:00
for (const [character, onyomi, kunyomi, tags, meanings, stats] of entries) {
2017-09-17 19:56:34 +00:00
rows.push({
character,
onyomi,
kunyomi,
tags,
meanings,
stats,
dictionary: summary.title
});
}
2016-08-24 05:22:09 +00:00
}
2019-02-27 02:01:32 +00:00
await bulkAdd(this.db.kanji, rows, total, current);
2017-09-13 03:20:03 +00:00
};
2017-09-23 15:46:34 +00:00
const kanjiMetaDataLoaded = async (summary, entries, total, current) => {
2017-09-13 03:20:03 +00:00
const rows = [];
2017-09-23 15:46:34 +00:00
for (const [character, mode, data] of entries) {
2017-09-13 03:20:03 +00:00
rows.push({
character,
2017-09-23 15:46:34 +00:00
mode,
data,
2017-09-17 19:56:34 +00:00
dictionary: summary.title
2017-09-13 03:20:03 +00:00
});
}
2019-02-27 02:01:32 +00:00
await bulkAdd(this.db.kanjiMeta, rows, total, current);
2016-11-05 23:44:29 +00:00
};
2016-08-24 05:22:09 +00:00
2017-09-17 19:56:34 +00:00
const tagDataLoaded = async (summary, entries, total, current) => {
2017-09-12 20:29:13 +00:00
const rows = [];
2017-10-24 09:38:05 +00:00
for (const [name, category, order, notes, score] of entries) {
2017-09-12 20:29:13 +00:00
const row = dictTagSanitize({
name,
category,
order,
notes,
2017-10-24 09:38:05 +00:00
score,
2017-09-17 19:56:34 +00:00
dictionary: summary.title
2017-09-12 20:29:13 +00:00
});
rows.push(row);
}
2019-02-27 02:01:32 +00:00
await bulkAdd(this.db.tagMeta, rows, total, current);
2017-09-12 20:29:13 +00:00
};
2017-09-11 04:49:37 +00:00
return await Database.importDictionaryZip(
archive,
indexDataLoaded,
termDataLoaded,
2017-09-23 15:46:34 +00:00
termMetaDataLoaded,
2017-09-11 04:49:37 +00:00
kanjiDataLoaded,
2017-09-23 15:46:34 +00:00
kanjiMetaDataLoaded,
2017-09-12 20:29:13 +00:00
tagDataLoaded
2017-09-11 04:49:37 +00:00
);
2016-08-24 05:22:09 +00:00
}
2017-07-10 21:53:06 +00:00
2019-10-08 00:46:02 +00:00
validate() {
if (this.db === null) {
throw new Error('Database not initialized');
}
}
2017-09-11 04:49:37 +00:00
static async importDictionaryZip(
archive,
indexDataLoaded,
termDataLoaded,
2017-09-23 15:46:34 +00:00
termMetaDataLoaded,
2017-09-11 04:49:37 +00:00
kanjiDataLoaded,
2017-09-23 15:46:34 +00:00
kanjiMetaDataLoaded,
2017-09-11 04:49:37 +00:00
tagDataLoaded
) {
2017-09-13 22:41:06 +00:00
const zip = await JSZip.loadAsync(archive);
2017-07-10 21:53:06 +00:00
2017-09-13 22:41:06 +00:00
const indexFile = zip.files['index.json'];
2017-07-10 21:53:06 +00:00
if (!indexFile) {
2019-10-08 00:46:02 +00:00
throw new Error('No dictionary index found in archive');
2017-07-10 21:53:06 +00:00
}
const index = JSON.parse(await indexFile.async('string'));
2017-09-17 19:56:34 +00:00
if (!index.title || !index.revision) {
2019-10-08 00:46:02 +00:00
throw new Error('Unrecognized dictionary format');
2017-07-10 21:53:06 +00:00
}
2017-09-17 19:56:34 +00:00
const summary = {
title: index.title,
revision: index.revision,
sequenced: index.sequenced,
2017-09-17 19:56:34 +00:00
version: index.format || index.version
};
await indexDataLoaded(summary);
2017-07-10 21:53:06 +00:00
2017-09-11 04:49:37 +00:00
const buildTermBankName = index => `term_bank_${index + 1}.json`;
2017-09-23 15:46:34 +00:00
const buildTermMetaBankName = index => `term_meta_bank_${index + 1}.json`;
2017-09-11 04:49:37 +00:00
const buildKanjiBankName = index => `kanji_bank_${index + 1}.json`;
2017-09-23 15:46:34 +00:00
const buildKanjiMetaBankName = index => `kanji_meta_bank_${index + 1}.json`;
2017-09-11 04:49:37 +00:00
const buildTagBankName = index => `tag_bank_${index + 1}.json`;
const countBanks = namer => {
let count = 0;
2017-09-13 22:41:06 +00:00
while (zip.files[namer(count)]) {
2017-09-11 04:49:37 +00:00
++count;
2017-07-10 21:53:06 +00:00
}
2017-09-11 04:49:37 +00:00
return count;
};
const termBankCount = countBanks(buildTermBankName);
2017-09-23 15:46:34 +00:00
const termMetaBankCount = countBanks(buildTermMetaBankName);
2017-09-12 20:29:13 +00:00
const kanjiBankCount = countBanks(buildKanjiBankName);
2017-09-23 15:46:34 +00:00
const kanjiMetaBankCount = countBanks(buildKanjiMetaBankName);
2017-09-12 20:29:13 +00:00
const tagBankCount = countBanks(buildTagBankName);
2017-09-11 04:49:37 +00:00
let bankLoadedCount = 0;
2017-09-12 20:29:13 +00:00
let bankTotalCount =
2017-09-11 04:49:37 +00:00
termBankCount +
2017-09-23 15:46:34 +00:00
termMetaBankCount +
2017-09-12 20:29:13 +00:00
kanjiBankCount +
2017-09-23 15:46:34 +00:00
kanjiMetaBankCount +
2017-09-11 04:49:37 +00:00
tagBankCount;
2017-09-12 20:29:13 +00:00
if (tagDataLoaded && index.tagMeta) {
const bank = [];
for (const name in index.tagMeta) {
const tag = index.tagMeta[name];
2017-10-24 09:38:05 +00:00
bank.push([name, tag.category, tag.order, tag.notes, tag.score]);
2017-09-12 20:29:13 +00:00
}
2017-09-17 19:56:34 +00:00
tagDataLoaded(summary, bank, ++bankTotalCount, bankLoadedCount++);
2017-09-12 20:29:13 +00:00
}
2017-09-17 19:56:34 +00:00
const loadBank = async (summary, namer, count, callback) => {
2017-09-11 04:49:37 +00:00
if (callback) {
for (let i = 0; i < count; ++i) {
2017-09-13 22:41:06 +00:00
const bankFile = zip.files[namer(i)];
2017-09-11 04:49:37 +00:00
const bank = JSON.parse(await bankFile.async('string'));
2017-09-17 19:56:34 +00:00
await callback(summary, bank, bankTotalCount, bankLoadedCount++);
2017-09-11 04:49:37 +00:00
}
}
};
2017-09-17 19:56:34 +00:00
await loadBank(summary, buildTermBankName, termBankCount, termDataLoaded);
2017-09-23 15:46:34 +00:00
await loadBank(summary, buildTermMetaBankName, termMetaBankCount, termMetaDataLoaded);
2017-09-17 19:56:34 +00:00
await loadBank(summary, buildKanjiBankName, kanjiBankCount, kanjiDataLoaded);
2017-09-23 15:46:34 +00:00
await loadBank(summary, buildKanjiMetaBankName, kanjiMetaBankCount, kanjiMetaDataLoaded);
2017-09-17 19:56:34 +00:00
await loadBank(summary, buildTagBankName, tagBankCount, tagDataLoaded);
2017-09-11 04:49:37 +00:00
return summary;
2017-07-10 21:53:06 +00:00
}
2019-08-31 01:06:21 +00:00
static createTerm(row, index) {
return {
2019-08-31 01:06:21 +00:00
index,
expression: row.expression,
reading: row.reading,
definitionTags: dictFieldSplit(row.definitionTags || row.tags || ''),
termTags: dictFieldSplit(row.termTags || ''),
rules: dictFieldSplit(row.rules),
glossary: row.glossary,
score: row.score,
dictionary: row.dictionary,
id: row.id,
sequence: typeof row.sequence === 'undefined' ? -1 : row.sequence
};
}
2019-08-31 01:06:21 +00:00
static createKanji(row, index) {
return {
index,
character: row.character,
onyomi: dictFieldSplit(row.onyomi),
kunyomi: dictFieldSplit(row.kunyomi),
tags: dictFieldSplit(row.tags),
glossary: row.meanings,
stats: row.stats,
dictionary: row.dictionary
};
}
static createMeta(row, index) {
2019-08-31 01:06:21 +00:00
return {
index,
mode: row.mode,
data: row.data,
dictionary: row.dictionary
};
}
static getAll(dbIndex, query, context, processRow) {
2019-08-31 01:06:21 +00:00
const fn = typeof dbIndex.getAll === 'function' ? Database.getAllFast : Database.getAllUsingCursor;
return fn(dbIndex, query, context, processRow);
2019-08-31 01:06:21 +00:00
}
static getAllFast(dbIndex, query, context, processRow) {
2019-08-31 01:06:21 +00:00
return new Promise((resolve, reject) => {
const request = dbIndex.getAll(query);
request.onerror = (e) => reject(e);
request.onsuccess = (e) => {
for (const row of e.target.result) {
processRow(row, context);
2019-08-31 01:06:21 +00:00
}
resolve();
};
});
}
static getAllUsingCursor(dbIndex, query, context, processRow) {
2019-08-31 01:06:21 +00:00
return new Promise((resolve, reject) => {
const request = dbIndex.openCursor(query, 'next');
request.onerror = (e) => reject(e);
request.onsuccess = (e) => {
const cursor = e.target.result;
if (cursor) {
processRow(cursor.value, context);
2019-08-31 01:06:21 +00:00
cursor.continue();
} else {
resolve();
}
};
});
}
2016-03-20 02:32:35 +00:00
}