yomichan/ext/bg/js/database.js
2019-05-04 13:02:56 -04:00

493 lines
16 KiB
JavaScript

/*
* Copyright (C) 2016-2017 Alex Yatskov <alex@foosoft.net>
* Author: Alex Yatskov <alex@foosoft.net>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
class Database {
constructor() {
this.db = null;
this.tagCache = {};
}
async prepare() {
if (this.db) {
throw 'Database already initialized';
}
this.db = new Dexie('dict');
this.db.version(2).stores({
terms: '++id,dictionary,expression,reading',
kanji: '++,dictionary,character',
tagMeta: '++,dictionary',
dictionaries: '++,title,version'
});
this.db.version(3).stores({
termMeta: '++,dictionary,expression',
kanjiMeta: '++,dictionary,character',
tagMeta: '++,dictionary,name'
});
this.db.version(4).stores({
terms: '++id,dictionary,expression,reading,sequence'
});
await this.db.open();
}
async purge() {
if (!this.db) {
throw 'Database not initialized';
}
this.db.close();
await this.db.delete();
this.db = null;
this.tagCache = {};
await this.prepare();
}
async findTerms(term, titles) {
if (!this.db) {
throw 'Database not initialized';
}
const results = [];
await this.db.terms.where('expression').equals(term).or('reading').equals(term).each(row => {
if (titles.includes(row.dictionary)) {
results.push({
expression: row.expression,
reading: row.reading,
definitionTags: dictFieldSplit(row.definitionTags || row.tags || ''),
termTags: dictFieldSplit(row.termTags || ''),
rules: dictFieldSplit(row.rules),
glossary: row.glossary,
score: row.score,
dictionary: row.dictionary,
id: row.id,
sequence: typeof row.sequence === 'undefined' ? -1 : row.sequence
});
}
});
return results;
}
async findTermsExact(term, reading, titles) {
if (!this.db) {
throw 'Database not initialized';
}
const results = [];
await this.db.terms.where('expression').equals(term).each(row => {
if (row.reading === reading && titles.includes(row.dictionary)) {
results.push({
expression: row.expression,
reading: row.reading,
definitionTags: dictFieldSplit(row.definitionTags || row.tags || ''),
termTags: dictFieldSplit(row.termTags || ''),
rules: dictFieldSplit(row.rules),
glossary: row.glossary,
score: row.score,
dictionary: row.dictionary,
id: row.id,
sequence: typeof row.sequence === 'undefined' ? -1 : row.sequence
});
}
});
return results;
}
async findTermsBySequence(sequence, mainDictionary) {
if (!this.db) {
throw 'Database not initialized';
}
const results = [];
await this.db.terms.where('sequence').equals(sequence).each(row => {
if (row.dictionary === mainDictionary) {
results.push({
expression: row.expression,
reading: row.reading,
definitionTags: dictFieldSplit(row.definitionTags || row.tags || ''),
termTags: dictFieldSplit(row.termTags || ''),
rules: dictFieldSplit(row.rules),
glossary: row.glossary,
score: row.score,
dictionary: row.dictionary,
id: row.id,
sequence: typeof row.sequence === 'undefined' ? -1 : row.sequence
});
}
});
return results;
}
async findTermMeta(term, titles) {
if (!this.db) {
throw 'Database not initialized';
}
const results = [];
await this.db.termMeta.where('expression').equals(term).each(row => {
if (titles.includes(row.dictionary)) {
results.push({
mode: row.mode,
data: row.data,
dictionary: row.dictionary
});
}
});
return results;
}
async findKanji(kanji, titles) {
if (!this.db) {
throw 'Database not initialized';
}
const results = [];
await this.db.kanji.where('character').equals(kanji).each(row => {
if (titles.includes(row.dictionary)) {
results.push({
character: row.character,
onyomi: dictFieldSplit(row.onyomi),
kunyomi: dictFieldSplit(row.kunyomi),
tags: dictFieldSplit(row.tags),
glossary: row.meanings,
stats: row.stats,
dictionary: row.dictionary
});
}
});
return results;
}
async findKanjiMeta(kanji, titles) {
if (!this.db) {
throw 'Database not initialized';
}
const results = [];
await this.db.kanjiMeta.where('character').equals(kanji).each(row => {
if (titles.includes(row.dictionary)) {
results.push({
mode: row.mode,
data: row.data,
dictionary: row.dictionary
});
}
});
return results;
}
async findTagForTitle(name, title) {
if (!this.db) {
throw 'Database not initialized';
}
this.tagCache[title] = this.tagCache[title] || {};
let result = this.tagCache[title][name];
if (!result) {
await this.db.tagMeta.where('name').equals(name).each(row => {
if (title === row.dictionary) {
result = row;
}
});
this.tagCache[title][name] = result;
}
return result;
}
async summarize() {
if (this.db) {
return this.db.dictionaries.toArray();
} else {
throw 'Database not initialized';
}
}
async importDictionary(archive, progressCallback, exceptions) {
if (!this.db) {
throw 'Database not initialized';
}
const maxTransactionLength = 1000;
const bulkAdd = async (table, items, total, current) => {
if (items.length < maxTransactionLength) {
if (progressCallback) {
progressCallback(total, current);
}
try {
await table.bulkAdd(items);
} catch (e) {
if (exceptions) {
exceptions.push(e);
} else {
throw e;
}
}
} else {
for (let i = 0; i < items.length; i += maxTransactionLength) {
if (progressCallback) {
progressCallback(total, current + i / items.length);
}
let count = Math.min(maxTransactionLength, items.length - i);
try {
await table.bulkAdd(items.slice(i, i + count));
} catch (e) {
if (exceptions) {
exceptions.push(e);
} else {
throw e;
}
}
}
}
};
const indexDataLoaded = async summary => {
if (summary.version > 3) {
throw 'Unsupported dictionary version';
}
const count = await this.db.dictionaries.where('title').equals(summary.title).count();
if (count > 0) {
throw 'Dictionary is already imported';
}
await this.db.dictionaries.add(summary);
};
const termDataLoaded = async (summary, entries, total, current) => {
const rows = [];
if (summary.version === 1) {
for (const [expression, reading, definitionTags, rules, score, ...glossary] of entries) {
rows.push({
expression,
reading,
definitionTags,
rules,
score,
glossary,
dictionary: summary.title
});
}
} else {
for (const [expression, reading, definitionTags, rules, score, glossary, sequence, termTags] of entries) {
rows.push({
expression,
reading,
definitionTags,
rules,
score,
glossary,
sequence,
termTags,
dictionary: summary.title
});
}
}
await bulkAdd(this.db.terms, rows, total, current);
};
const termMetaDataLoaded = async (summary, entries, total, current) => {
const rows = [];
for (const [expression, mode, data] of entries) {
rows.push({
expression,
mode,
data,
dictionary: summary.title
});
}
await bulkAdd(this.db.termMeta, rows, total, current);
};
const kanjiDataLoaded = async (summary, entries, total, current) => {
const rows = [];
if (summary.version === 1) {
for (const [character, onyomi, kunyomi, tags, ...meanings] of entries) {
rows.push({
character,
onyomi,
kunyomi,
tags,
meanings,
dictionary: summary.title
});
}
} else {
for (const [character, onyomi, kunyomi, tags, meanings, stats] of entries) {
rows.push({
character,
onyomi,
kunyomi,
tags,
meanings,
stats,
dictionary: summary.title
});
}
}
await bulkAdd(this.db.kanji, rows, total, current);
};
const kanjiMetaDataLoaded = async (summary, entries, total, current) => {
const rows = [];
for (const [character, mode, data] of entries) {
rows.push({
character,
mode,
data,
dictionary: summary.title
});
}
await bulkAdd(this.db.kanjiMeta, rows, total, current);
};
const tagDataLoaded = async (summary, entries, total, current) => {
const rows = [];
for (const [name, category, order, notes, score] of entries) {
const row = dictTagSanitize({
name,
category,
order,
notes,
score,
dictionary: summary.title
});
rows.push(row);
}
await bulkAdd(this.db.tagMeta, rows, total, current);
};
return await Database.importDictionaryZip(
archive,
indexDataLoaded,
termDataLoaded,
termMetaDataLoaded,
kanjiDataLoaded,
kanjiMetaDataLoaded,
tagDataLoaded
);
}
static async importDictionaryZip(
archive,
indexDataLoaded,
termDataLoaded,
termMetaDataLoaded,
kanjiDataLoaded,
kanjiMetaDataLoaded,
tagDataLoaded
) {
const zip = await JSZip.loadAsync(archive);
const indexFile = zip.files['index.json'];
if (!indexFile) {
throw 'No dictionary index found in archive';
}
const index = JSON.parse(await indexFile.async('string'));
if (!index.title || !index.revision) {
throw 'Unrecognized dictionary format';
}
const summary = {
title: index.title,
revision: index.revision,
sequenced: index.sequenced,
version: index.format || index.version
};
await indexDataLoaded(summary);
const buildTermBankName = index => `term_bank_${index + 1}.json`;
const buildTermMetaBankName = index => `term_meta_bank_${index + 1}.json`;
const buildKanjiBankName = index => `kanji_bank_${index + 1}.json`;
const buildKanjiMetaBankName = index => `kanji_meta_bank_${index + 1}.json`;
const buildTagBankName = index => `tag_bank_${index + 1}.json`;
const countBanks = namer => {
let count = 0;
while (zip.files[namer(count)]) {
++count;
}
return count;
};
const termBankCount = countBanks(buildTermBankName);
const termMetaBankCount = countBanks(buildTermMetaBankName);
const kanjiBankCount = countBanks(buildKanjiBankName);
const kanjiMetaBankCount = countBanks(buildKanjiMetaBankName);
const tagBankCount = countBanks(buildTagBankName);
let bankLoadedCount = 0;
let bankTotalCount =
termBankCount +
termMetaBankCount +
kanjiBankCount +
kanjiMetaBankCount +
tagBankCount;
if (tagDataLoaded && index.tagMeta) {
const bank = [];
for (const name in index.tagMeta) {
const tag = index.tagMeta[name];
bank.push([name, tag.category, tag.order, tag.notes, tag.score]);
}
tagDataLoaded(summary, bank, ++bankTotalCount, bankLoadedCount++);
}
const loadBank = async (summary, namer, count, callback) => {
if (callback) {
for (let i = 0; i < count; ++i) {
const bankFile = zip.files[namer(i)];
const bank = JSON.parse(await bankFile.async('string'));
await callback(summary, bank, bankTotalCount, bankLoadedCount++);
}
}
};
await loadBank(summary, buildTermBankName, termBankCount, termDataLoaded);
await loadBank(summary, buildTermMetaBankName, termMetaBankCount, termMetaDataLoaded);
await loadBank(summary, buildKanjiBankName, kanjiBankCount, kanjiDataLoaded);
await loadBank(summary, buildKanjiMetaBankName, kanjiMetaBankCount, kanjiMetaDataLoaded);
await loadBank(summary, buildTagBankName, tagBankCount, tagDataLoaded);
return summary;
}
}