yomichan/ext/bg/js/database.js

/*
 * Copyright (C) 2016-2017  Alex Yatskov <alex@foosoft.net>
 * Author: Alex Yatskov <alex@foosoft.net>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */


class Database {
    constructor() {
        this.db = null;
        this.tagCache = {};
    }

    async prepare() {
        if (this.db) {
            throw 'Database already initialized';
        }

        this.db = new Dexie('dict');
        this.db.version(2).stores({
            terms:        '++id,dictionary,expression,reading',
            kanji:        '++,dictionary,character',
            tagMeta:      '++,dictionary',
            dictionaries: '++,title,version'
        });
        this.db.version(3).stores({
            termMeta:  '++,dictionary,expression',
            kanjiMeta: '++,dictionary,character',
            tagMeta:   '++,dictionary,name'
        });
        this.db.version(4).stores({
            terms: '++id,dictionary,expression,reading,sequence'
        });

        await this.db.open();
    }

    async purge() {
        if (!this.db) {
            throw 'Database not initialized';
        }

        this.db.close();
        await this.db.delete();
        this.db = null;
        this.tagCache = {};

        await this.prepare();
    }

    async findTerms(term, titles) {
        if (!this.db) {
            throw 'Database not initialized';
        }

        const results = [];
        await this.db.terms.where('expression').equals(term).or('reading').equals(term).each(row => {
            if (titles.includes(row.dictionary)) {
                results.push({
                    expression: row.expression,
                    reading: row.reading,
                    definitionTags: dictFieldSplit(row.definitionTags || row.tags || ''),
                    termTags: dictFieldSplit(row.termTags || ''),
                    rules: dictFieldSplit(row.rules),
                    glossary: row.glossary,
                    score: row.score,
                    dictionary: row.dictionary,
                    id: row.id,
                    sequence: typeof row.sequence === 'undefined' ? -1 : row.sequence
                });
            }
        });

        return results;
    }

    async findTermsExact(term, reading, titles) {
        if (!this.db) {
            throw 'Database not initialized';
        }

        const results = [];
        await this.db.terms.where('expression').equals(term).each(row => {
            if (row.reading === reading && titles.includes(row.dictionary)) {
                results.push({
                    expression: row.expression,
                    reading: row.reading,
                    definitionTags: dictFieldSplit(row.definitionTags || row.tags || ''),
                    termTags: dictFieldSplit(row.termTags || ''),
                    rules: dictFieldSplit(row.rules),
                    glossary: row.glossary,
                    score: row.score,
                    dictionary: row.dictionary,
                    id: row.id,
                    sequence: typeof row.sequence === 'undefined' ? -1 : row.sequence
                });
            }
        });

        return results;
    }

    async findTermsBySequence(sequence, mainDictionary) {
        if (!this.db) {
            throw 'Database not initialized';
        }

        const results = [];
        await this.db.terms.where('sequence').equals(sequence).each(row => {
            if (row.dictionary === mainDictionary) {
                results.push({
                    expression: row.expression,
                    reading: row.reading,
                    definitionTags: dictFieldSplit(row.definitionTags || row.tags || ''),
                    termTags: dictFieldSplit(row.termTags || ''),
                    rules: dictFieldSplit(row.rules),
                    glossary: row.glossary,
                    score: row.score,
                    dictionary: row.dictionary,
                    id: row.id,
                    sequence: typeof row.sequence === 'undefined' ? -1 : row.sequence
                });
            }
        });

        return results;
    }

    async findTermMeta(term, titles) {
        if (!this.db) {
            throw 'Database not initialized';
        }

        const results = [];
        await this.db.termMeta.where('expression').equals(term).each(row => {
            if (titles.includes(row.dictionary)) {
                results.push({
                    mode: row.mode,
                    data: row.data,
                    dictionary: row.dictionary
                });
            }
        });

        return results;
    }

    async findKanji(kanji, titles) {
        if (!this.db) {
            throw 'Database not initialized';
        }

        const results = [];
        await this.db.kanji.where('character').equals(kanji).each(row => {
            if (titles.includes(row.dictionary)) {
                results.push({
                    character: row.character,
                    onyomi: dictFieldSplit(row.onyomi),
                    kunyomi: dictFieldSplit(row.kunyomi),
                    tags: dictFieldSplit(row.tags),
                    glossary: row.meanings,
                    stats: row.stats,
                    dictionary: row.dictionary
                });
            }
        });

        return results;
    }

    async findKanjiMeta(kanji, titles) {
        if (!this.db) {
            throw 'Database not initialized';
        }

        const results = [];
        await this.db.kanjiMeta.where('character').equals(kanji).each(row => {
            if (titles.includes(row.dictionary)) {
                results.push({
                    mode: row.mode,
                    data: row.data,
                    dictionary: row.dictionary
                });
            }
        });

        return results;
    }

    async findTagForTitle(name, title) {
        if (!this.db) {
            throw 'Database not initialized';
        }

        this.tagCache[title] = this.tagCache[title] || {};

        let result = this.tagCache[title][name];
        if (!result) {
            await this.db.tagMeta.where('name').equals(name).each(row => {
                if (title === row.dictionary) {
                    result = row;
                }
            });

            this.tagCache[title][name] = result;
        }

        return result;
    }

    async summarize() {
        if (this.db) {
            return this.db.dictionaries.toArray();
        } else {
            throw 'Database not initialized';
        }
    }

    async importDictionary(archive, progressCallback, exceptions) {
        if (!this.db) {
            throw 'Database not initialized';
        }

        const maxTransactionLength = 1000;
        const bulkAdd = async (table, items, total, current) => {
            if (items.length < maxTransactionLength) {
                if (progressCallback) {
                    progressCallback(total, current);
                }

                try {
                    await table.bulkAdd(items);
                } catch (e) {
                    if (exceptions) {
                        exceptions.push(e);
                    } else {
                        throw e;
                    }
                }
            } else {
                for (let i = 0; i < items.length; i += maxTransactionLength) {
                    if (progressCallback) {
                        progressCallback(total, current + i / items.length);
                    }

                    let count = Math.min(maxTransactionLength, items.length - i);
                    try {
                        await table.bulkAdd(items.slice(i, i + count));
                    } catch (e) {
                        if (exceptions) {
                            exceptions.push(e);
                        } else {
                            throw e;
                        }
                    }
                }
            }
        };

        const indexDataLoaded = async summary => {
            if (summary.version > 3) {
                throw 'Unsupported dictionary version';
            }

            const count = await this.db.dictionaries.where('title').equals(summary.title).count();
            if (count > 0) {
                throw 'Dictionary is already imported';
            }

            await this.db.dictionaries.add(summary);
        };

        const termDataLoaded = async (summary, entries, total, current) => {
            const rows = [];
            if (summary.version === 1) {
                for (const [expression, reading, definitionTags, rules, score, ...glossary] of entries) {
                    rows.push({
                        expression,
                        reading,
                        definitionTags,
                        rules,
                        score,
                        glossary,
                        dictionary: summary.title
                    });
                }
            } else {
                for (const [expression, reading, definitionTags, rules, score, glossary, sequence, termTags] of entries) {
                    rows.push({
                        expression,
                        reading,
                        definitionTags,
                        rules,
                        score,
                        glossary,
                        sequence,
                        termTags,
                        dictionary: summary.title
                    });
                }
            }

            await bulkAdd(this.db.terms, rows, total, current);
        };

        const termMetaDataLoaded = async (summary, entries, total, current) => {
            const rows = [];
            for (const [expression, mode, data] of entries) {
                rows.push({
                    expression,
                    mode,
                    data,
                    dictionary: summary.title
                });
            }

            await bulkAdd(this.db.termMeta, rows, total, current);
        };

        const kanjiDataLoaded = async (summary, entries, total, current)  => {
            const rows = [];
            if (summary.version === 1) {
                for (const [character, onyomi, kunyomi, tags, ...meanings] of entries) {
                    rows.push({
                        character,
                        onyomi,
                        kunyomi,
                        tags,
                        meanings,
                        dictionary: summary.title
                    });
                }
            } else {
                for (const [character, onyomi, kunyomi, tags, meanings, stats] of entries) {
                    rows.push({
                        character,
                        onyomi,
                        kunyomi,
                        tags,
                        meanings,
                        stats,
                        dictionary: summary.title
                    });
                }
            }

            await bulkAdd(this.db.kanji, rows, total, current);
        };

        const kanjiMetaDataLoaded = async (summary, entries, total, current) => {
            const rows = [];
            for (const [character, mode, data] of entries) {
                rows.push({
                    character,
                    mode,
                    data,
                    dictionary: summary.title
                });
            }

            await bulkAdd(this.db.kanjiMeta, rows, total, current);
        };

        const tagDataLoaded = async (summary, entries, total, current) => {
            const rows = [];
            for (const [name, category, order, notes, score] of entries) {
                const row = dictTagSanitize({
                    name,
                    category,
                    order,
                    notes,
                    score,
                    dictionary: summary.title
                });

                rows.push(row);
            }

            await bulkAdd(this.db.tagMeta, rows, total, current);
        };

        return await Database.importDictionaryZip(
            archive,
            indexDataLoaded,
            termDataLoaded,
            termMetaDataLoaded,
            kanjiDataLoaded,
            kanjiMetaDataLoaded,
            tagDataLoaded
        );
    }

    static async importDictionaryZip(
        archive,
        indexDataLoaded,
        termDataLoaded,
        termMetaDataLoaded,
        kanjiDataLoaded,
        kanjiMetaDataLoaded,
        tagDataLoaded
    ) {
        const zip = await JSZip.loadAsync(archive);

        const indexFile = zip.files['index.json'];
        if (!indexFile) {
            throw 'No dictionary index found in archive';
        }

        const index = JSON.parse(await indexFile.async('string'));
        if (!index.title || !index.revision) {
            throw 'Unrecognized dictionary format';
        }

        const summary = {
            title: index.title,
            revision: index.revision,
            sequenced: index.sequenced,
            version: index.format || index.version
        };

        await indexDataLoaded(summary);

        const buildTermBankName      = index => `term_bank_${index + 1}.json`;
        const buildTermMetaBankName  = index => `term_meta_bank_${index + 1}.json`;
        const buildKanjiBankName     = index => `kanji_bank_${index + 1}.json`;
        const buildKanjiMetaBankName = index => `kanji_meta_bank_${index + 1}.json`;
        const buildTagBankName       = index => `tag_bank_${index + 1}.json`;

        const countBanks = namer => {
            let count = 0;
            while (zip.files[namer(count)]) {
                ++count;
            }

            return count;
        };

        const termBankCount      = countBanks(buildTermBankName);
        const termMetaBankCount  = countBanks(buildTermMetaBankName);
        const kanjiBankCount     = countBanks(buildKanjiBankName);
        const kanjiMetaBankCount = countBanks(buildKanjiMetaBankName);
        const tagBankCount       = countBanks(buildTagBankName);

        let bankLoadedCount = 0;
        let bankTotalCount =
            termBankCount +
            termMetaBankCount +
            kanjiBankCount +
            kanjiMetaBankCount +
            tagBankCount;

        if (tagDataLoaded && index.tagMeta) {
            const bank = [];
            for (const name in index.tagMeta) {
                const tag = index.tagMeta[name];
                bank.push([name, tag.category, tag.order, tag.notes, tag.score]);
            }

            tagDataLoaded(summary, bank, ++bankTotalCount, bankLoadedCount++);
        }

        const loadBank = async (summary, namer, count, callback) => {
            if (callback) {
                for (let i = 0; i < count; ++i) {
                    const bankFile = zip.files[namer(i)];
                    const bank = JSON.parse(await bankFile.async('string'));
                    await callback(summary, bank, bankTotalCount, bankLoadedCount++);
                }
            }
        };

        await loadBank(summary, buildTermBankName, termBankCount, termDataLoaded);
        await loadBank(summary, buildTermMetaBankName, termMetaBankCount, termMetaDataLoaded);
        await loadBank(summary, buildKanjiBankName, kanjiBankCount, kanjiDataLoaded);
        await loadBank(summary, buildKanjiMetaBankName, kanjiMetaBankCount, kanjiMetaDataLoaded);
        await loadBank(summary, buildTagBankName, tagBankCount, tagDataLoaded);

        return summary;
    }
}