alex/yomichan
1
Fork 0
Code Releases Activity

update database for new format

Browse Source
This commit is contained in:
Alex Yatskov 2017-09-23 08:46:34 -07:00
parent 088c608d80
commit 8d85321cf9
2 changed files with 65 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
ext/bg/js/database.js
View File

@ -36,8 +36,8 @@ class Database {
dictionaries: '++,title,version' dictionaries: '++,title,version'
}); });
this.db.version(3).stores({ this.db.version(3).stores({
termFreq: '++,dictionary,expression', termMeta: '++,dictionary,expression',
kanjiFreq: '++,dictionary,character', kanjiMeta: '++,dictionary,character',
tagMeta: '++,dictionary,name' tagMeta: '++,dictionary,name'
}); });
@ -81,15 +81,19 @@ class Database {
return results; return results;
} }
async findTermFreq(term, titles) { async findTermMeta(term, titles) {
if (!this.db) { if (!this.db) {
throw 'Database not initialized'; throw 'Database not initialized';
} }
const results = []; const results = [];
await this.db.termFreq.where('expression').equals(term).each(row => { await this.db.termMeta.where('expression').equals(term).each(row => {
if (titles.includes(row.dictionary)) { if (titles.includes(row.dictionary)) {
results.push({frequency: row.frequency, dictionary: row.dictionary}); results.push({
mode: row.mode,
data: row.data,
dictionary: row.dictionary
});
} }
}); });
@ -119,15 +123,19 @@ class Database {
return results; return results;
} }
async findKanjiFreq(kanji, titles) { async findKanjiMeta(kanji, titles) {
if (!this.db) { if (!this.db) {
throw 'Database not initialized'; throw 'Database not initialized';
} }
const results = []; const results = [];
await this.db.kanjiFreq.where('character').equals(kanji).each(row => { await this.db.kanjiMeta.where('character').equals(kanji).each(row => {
if (titles.includes(row.dictionary)) { if (titles.includes(row.dictionary)) {
results.push({frequency: row.frequency, dictionary: row.dictionary}); results.push({
mode: row.mode,
data: row.data,
dictionary: row.dictionary
});
} }
}); });
@ -216,21 +224,22 @@ class Database {
await this.db.terms.bulkAdd(rows); await this.db.terms.bulkAdd(rows);
}; };
const termFreqDataLoaded = async (summary, entries, total, current) => { const termMetaDataLoaded = async (summary, entries, total, current) => {
if (callback) { if (callback) {
callback(total, current); callback(total, current);
} }
const rows = []; const rows = [];
for (const [expression, frequency] of entries) { for (const [expression, mode, data] of entries) {
rows.push({ rows.push({
expression, expression,
frequency, mode,
data,
dictionary: summary.title dictionary: summary.title
}); });
} }
await this.db.termFreq.bulkAdd(rows); await this.db.termMeta.bulkAdd(rows);
}; };
const kanjiDataLoaded = async (summary, entries, total, current) => { const kanjiDataLoaded = async (summary, entries, total, current) => {
@ -267,21 +276,22 @@ class Database {
await this.db.kanji.bulkAdd(rows); await this.db.kanji.bulkAdd(rows);
}; };
const kanjiFreqDataLoaded = async (summary, entries, total, current) => { const kanjiMetaDataLoaded = async (summary, entries, total, current) => {
if (callback) { if (callback) {
callback(total, current); callback(total, current);
} }
const rows = []; const rows = [];
for (const [character, frequency] of entries) { for (const [character, mode, data] of entries) {
rows.push({ rows.push({
character, character,
frequency, mode,
data,
dictionary: summary.title dictionary: summary.title
}); });
} }
await this.db.kanjiFreq.bulkAdd(rows); await this.db.kanjiMeta.bulkAdd(rows);
}; };
const tagDataLoaded = async (summary, entries, total, current) => { const tagDataLoaded = async (summary, entries, total, current) => {
@ -309,9 +319,9 @@ class Database {
archive, archive,
indexDataLoaded, indexDataLoaded,
termDataLoaded, termDataLoaded,
termFreqDataLoaded, termMetaDataLoaded,
kanjiDataLoaded, kanjiDataLoaded,
kanjiFreqDataLoaded, kanjiMetaDataLoaded,
tagDataLoaded tagDataLoaded
); );
} }
@ -320,9 +330,9 @@ class Database {
archive, archive,
indexDataLoaded, indexDataLoaded,
termDataLoaded, termDataLoaded,
termFreqDataLoaded, termMetaDataLoaded,
kanjiDataLoaded, kanjiDataLoaded,
kanjiFreqDataLoaded, kanjiMetaDataLoaded,
tagDataLoaded tagDataLoaded
) { ) {
const zip = await JSZip.loadAsync(archive); const zip = await JSZip.loadAsync(archive);
@ -348,9 +358,9 @@ class Database {
} }
const buildTermBankName = index => `term_bank_${index + 1}.json`; const buildTermBankName = index => `term_bank_${index + 1}.json`;
const buildTermFreqBankName = index => `termfreq_bank_${index + 1}.json`; const buildTermMetaBankName = index => `term_meta_bank_${index + 1}.json`;
const buildKanjiBankName = index => `kanji_bank_${index + 1}.json`; const buildKanjiBankName = index => `kanji_bank_${index + 1}.json`;
const buildKanjiFreqBankName = index => `kanjifreq_bank_${index + 1}.json`; const buildKanjiMetaBankName = index => `kanji_meta_bank_${index + 1}.json`;
const buildTagBankName = index => `tag_bank_${index + 1}.json`; const buildTagBankName = index => `tag_bank_${index + 1}.json`;
const countBanks = namer => { const countBanks = namer => {
@ -363,17 +373,17 @@ class Database {
}; };
const termBankCount = countBanks(buildTermBankName); const termBankCount = countBanks(buildTermBankName);
const termFreqBankCount = countBanks(buildTermFreqBankName); const termMetaBankCount = countBanks(buildTermMetaBankName);
const kanjiBankCount = countBanks(buildKanjiBankName); const kanjiBankCount = countBanks(buildKanjiBankName);
const kanjiFreqBankCount = countBanks(buildKanjiFreqBankName); const kanjiMetaBankCount = countBanks(buildKanjiMetaBankName);
const tagBankCount = countBanks(buildTagBankName); const tagBankCount = countBanks(buildTagBankName);
let bankLoadedCount = 0; let bankLoadedCount = 0;
let bankTotalCount = let bankTotalCount =
termBankCount + termBankCount +
termFreqBankCount + termMetaBankCount +
kanjiBankCount + kanjiBankCount +
kanjiFreqBankCount + kanjiMetaBankCount +
tagBankCount; tagBankCount;
if (tagDataLoaded && index.tagMeta) { if (tagDataLoaded && index.tagMeta) {
@ -397,9 +407,9 @@ class Database {
}; };
await loadBank(summary, buildTermBankName, termBankCount, termDataLoaded); await loadBank(summary, buildTermBankName, termBankCount, termDataLoaded);
await loadBank(summary, buildTermFreqBankName, termFreqBankCount, termFreqDataLoaded); await loadBank(summary, buildTermMetaBankName, termMetaBankCount, termMetaDataLoaded);
await loadBank(summary, buildKanjiBankName, kanjiBankCount, kanjiDataLoaded); await loadBank(summary, buildKanjiBankName, kanjiBankCount, kanjiDataLoaded);
await loadBank(summary, buildKanjiFreqBankName, kanjiFreqBankCount, kanjiFreqDataLoaded); await loadBank(summary, buildKanjiMetaBankName, kanjiMetaBankCount, kanjiMetaDataLoaded);
await loadBank(summary, buildTagBankName, tagBankCount, tagDataLoaded); await loadBank(summary, buildTagBankName, tagBankCount, tagDataLoaded);
return summary; return summary;

30
ext/bg/js/translator.js
View File

@ -141,16 +141,38 @@ class Translator {
definition.tags = dictTagsSort(tags); definition.tags = dictTagsSort(tags);
definition.stats = await this.expandStats(definition.stats, definition.dictionary); definition.stats = await this.expandStats(definition.stats, definition.dictionary);
definition.frequencies = await this.database.findKanjiFreq(definition.character, titles);
definition.frequencies = [];
const metas = await this.database.findKanjiMeta(definition.character, titles);
for (const meta of metas) {
if (meta.mode === 'freq') {
definition.frequencies.push({
character: meta.character,
frequency: meta.data,
dictionary: meta.dictionary
});
}
}
} }
return definitions; return definitions;
} }
async buildTermFrequencies(definition, titles) { async buildTermFrequencies(definition, titles) {
definition.frequencies = await this.database.findTermFreq(definition.expression, titles); let metas = await this.database.findTermMeta(definition.expression, titles);
if (definition.frequencies.length === 0) { if (metas.length === 0) {
definition.frequencies = await this.database.findTermFreq(definition.reading, titles); metas = await this.database.findTermMeta(definition.reading, titles);
}
definition.frequencies = [];
for (const meta of metas) {
if (meta.mode === 'freq') {
definition.frequencies.push({
expression: meta.expression,
frequency: meta.data,
dictionary: meta.dictionary
});
}
} }
} }