From 127a728675820aa7f92d14cff6559aef99bd78e4 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sat, 12 Oct 2019 19:55:06 -0400 Subject: [PATCH 01/14] Remove unused functions --- ext/bg/js/database.js | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/ext/bg/js/database.js b/ext/bg/js/database.js index 771a71c9..3de4c8b5 100644 --- a/ext/bg/js/database.js +++ b/ext/bg/js/database.js @@ -58,19 +58,6 @@ class Database { await this.prepare(); } - async findTerms(term, titles) { - this.validate(); - - const results = []; - await this.db.terms.where('expression').equals(term).or('reading').equals(term).each(row => { - if (titles.includes(row.dictionary)) { - results.push(Database.createTerm(row)); - } - }); - - return results; - } - async findTermsBulk(terms, titles) { const promises = []; const visited = {}; @@ -128,23 +115,6 @@ class Database { return results; } - async findTermMeta(term, titles) { - this.validate(); - - const results = []; - await this.db.termMeta.where('expression').equals(term).each(row => { - if (titles.includes(row.dictionary)) { - results.push({ - mode: row.mode, - data: row.data, - dictionary: row.dictionary - }); - } - }); - - return results; - } - async findTermMetaBulk(terms, titles) { const promises = []; const results = []; From 31ae00d0d3ad93f5d8b91ae5e1e2046067f49e81 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Fri, 18 Oct 2019 23:04:06 -0400 Subject: [PATCH 02/14] Add missing validate calls --- ext/bg/js/database.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ext/bg/js/database.js b/ext/bg/js/database.js index 3de4c8b5..c46b5228 100644 --- a/ext/bg/js/database.js +++ b/ext/bg/js/database.js @@ -59,6 +59,8 @@ class Database { } async findTermsBulk(terms, titles) { + this.validate(); + const promises = []; const visited = {}; const results = []; @@ -116,6 +118,8 @@ class Database { } async findTermMetaBulk(terms, titles) { + this.validate(); + const promises = []; const results = []; const createResult = Database.createTermMeta; From 6743b785c9f75c0bc6e628360155bdaebdacaa62 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Fri, 18 Oct 2019 23:09:00 -0400 Subject: [PATCH 03/14] Remove temp createResult --- ext/bg/js/database.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ext/bg/js/database.js b/ext/bg/js/database.js index c46b5228..3a87a2cd 100644 --- a/ext/bg/js/database.js +++ b/ext/bg/js/database.js @@ -64,11 +64,10 @@ class Database { const promises = []; const visited = {}; const results = []; - const createResult = Database.createTerm; const processRow = (row, index) => { if (titles.includes(row.dictionary) && !visited.hasOwnProperty(row.id)) { visited[row.id] = true; - results.push(createResult(row, index)); + results.push(Database.createTerm(row, index)); } }; @@ -122,10 +121,9 @@ class Database { const promises = []; const results = []; - const createResult = Database.createTermMeta; const processRow = (row, index) => { if (titles.includes(row.dictionary)) { - results.push(createResult(row, index)); + results.push(Database.createTermMeta(row, index)); } }; From 9364a7cb4ebec4f0ec2c1b51e305ab00a8131856 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Fri, 18 Oct 2019 23:16:33 -0400 Subject: [PATCH 04/14] Create function for creating kanji data and meta --- ext/bg/js/database.js | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/ext/bg/js/database.js b/ext/bg/js/database.js index 3a87a2cd..dfa4ea70 100644 --- a/ext/bg/js/database.js +++ b/ext/bg/js/database.js @@ -123,7 +123,7 @@ class Database { const results = []; const processRow = (row, index) => { if (titles.includes(row.dictionary)) { - results.push(Database.createTermMeta(row, index)); + results.push(Database.createMeta(row, index)); } }; @@ -148,15 +148,7 @@ class Database { const results = []; await this.db.kanji.where('character').equals(kanji).each(row => { if (titles.includes(row.dictionary)) { - results.push({ - character: row.character, - onyomi: dictFieldSplit(row.onyomi), - kunyomi: dictFieldSplit(row.kunyomi), - tags: dictFieldSplit(row.tags), - glossary: row.meanings, - stats: row.stats, - dictionary: row.dictionary - }); + results.push(Database.createKanji(row)); } }); @@ -169,11 +161,7 @@ class Database { const results = []; await this.db.kanjiMeta.where('character').equals(kanji).each(row => { if (titles.includes(row.dictionary)) { - results.push({ - mode: row.mode, - data: row.data, - dictionary: row.dictionary - }); + results.push(Database.createMeta(row)); } }); @@ -494,7 +482,20 @@ class Database { }; } - static createTermMeta(row, index) { + static createKanji(row, index) { + return { + index, + character: row.character, + onyomi: dictFieldSplit(row.onyomi), + kunyomi: dictFieldSplit(row.kunyomi), + tags: dictFieldSplit(row.tags), + glossary: row.meanings, + stats: row.stats, + dictionary: row.dictionary + }; + } + + static createMeta(row, index) { return { index, mode: row.mode, From d212d0aba132efb17d5dd2f354dfe596d6f8d018 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sat, 19 Oct 2019 10:09:18 -0400 Subject: [PATCH 05/14] Add additional database bulk methods --- ext/bg/js/database.js | 86 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 75 insertions(+), 11 deletions(-) diff --git a/ext/bg/js/database.js b/ext/bg/js/database.js index dfa4ea70..37f19359 100644 --- a/ext/bg/js/database.js +++ b/ext/bg/js/database.js @@ -58,7 +58,7 @@ class Database { await this.prepare(); } - async findTermsBulk(terms, titles) { + async findTermsBulk(termList, titles) { this.validate(); const promises = []; @@ -77,8 +77,8 @@ class Database { const dbIndex1 = dbTerms.index('expression'); const dbIndex2 = dbTerms.index('reading'); - for (let i = 0; i < terms.length; ++i) { - const only = IDBKeyRange.only(terms[i]); + for (let i = 0; i < termList.length; ++i) { + const only = IDBKeyRange.only(termList[i]); promises.push( Database.getAll(dbIndex1, only, i, processRow), Database.getAll(dbIndex2, only, i, processRow) @@ -103,6 +103,32 @@ class Database { return results; } + async findTermsExactBulk(termList, readingList, titles) { + this.validate(); + + const promises = []; + const results = []; + const processRow = (row, index) => { + if (row.reading === readingList[index] && titles.includes(row.dictionary)) { + results.push(Database.createTerm(row, index)); + } + }; + + const db = this.db.backendDB(); + const dbTransaction = db.transaction(['terms'], 'readonly'); + const dbTerms = dbTransaction.objectStore('terms'); + const dbIndex = dbTerms.index('expression'); + + for (let i = 0; i < termList.length; ++i) { + const only = IDBKeyRange.only(termList[i]); + promises.push(Database.getAll(dbIndex, only, i, processRow)); + } + + await Promise.all(promises); + + return results; + } + async findTermsBySequence(sequence, mainDictionary) { this.validate(); @@ -116,24 +142,24 @@ class Database { return results; } - async findTermMetaBulk(terms, titles) { + async findTermsBySequenceBulk(sequenceList, mainDictionary) { this.validate(); const promises = []; const results = []; const processRow = (row, index) => { - if (titles.includes(row.dictionary)) { - results.push(Database.createMeta(row, index)); + if (row.dictionary === mainDictionary) { + results.push(Database.createTerm(row, index)); } }; const db = this.db.backendDB(); - const dbTransaction = db.transaction(['termMeta'], 'readonly'); - const dbTerms = dbTransaction.objectStore('termMeta'); - const dbIndex = dbTerms.index('expression'); + const dbTransaction = db.transaction(['terms'], 'readonly'); + const dbTerms = dbTransaction.objectStore('terms'); + const dbIndex = dbTerms.index('sequence'); - for (let i = 0; i < terms.length; ++i) { - const only = IDBKeyRange.only(terms[i]); + for (let i = 0; i < sequenceList.length; ++i) { + const only = IDBKeyRange.only(sequenceList[i]); promises.push(Database.getAll(dbIndex, only, i, processRow)); } @@ -142,6 +168,10 @@ class Database { return results; } + async findTermMetaBulk(termList, titles) { + return this.findGenericBulk('termMeta', 'expression', termList, titles, Database.createMeta); + } + async findKanji(kanji, titles) { this.validate(); @@ -155,6 +185,10 @@ class Database { return results; } + async findKanjiBulk(kanjiList, titles) { + return this.findGenericBulk('kanji', 'character', kanjiList, titles, Database.createKanji); + } + async findKanjiMeta(kanji, titles) { this.validate(); @@ -168,6 +202,36 @@ class Database { return results; } + async findKanjiMetaBulk(kanjiList, titles) { + return this.findGenericBulk('kanjiMeta', 'character', kanjiList, titles, Database.createMeta); + } + + async findGenericBulk(tableName, indexName, indexValueList, titles, createResult) { + this.validate(); + + const promises = []; + const results = []; + const processRow = (row, index) => { + if (titles.includes(row.dictionary)) { + results.push(createResult(row, index)); + } + }; + + const db = this.db.backendDB(); + const dbTransaction = db.transaction([tableName], 'readonly'); + const dbTerms = dbTransaction.objectStore(tableName); + const dbIndex = dbTerms.index(indexName); + + for (let i = 0; i < indexValueList.length; ++i) { + const only = IDBKeyRange.only(indexValueList[i]); + promises.push(Database.getAll(dbIndex, only, i, processRow)); + } + + await Promise.all(promises); + + return results; + } + findTagForTitleCached(name, title) { if (this.tagCache.hasOwnProperty(title)) { const cache = this.tagCache[title]; From a25167df7338187a443f5e1281fd068fe394905e Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sat, 19 Oct 2019 10:21:10 -0400 Subject: [PATCH 06/14] Use findKanjiBulk and findKanjiMetaBulk in Translator.findKanji --- ext/bg/js/translator.js | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/ext/bg/js/translator.js b/ext/bg/js/translator.js index 601ee30c..61acaa79 100644 --- a/ext/bg/js/translator.js +++ b/ext/bg/js/translator.js @@ -277,33 +277,44 @@ class Translator { } async findKanji(text, dictionaries) { - let definitions = []; - const processed = {}; const titles = Object.keys(dictionaries); + const kanjiUnique = {}; + const kanjiList = []; for (const c of text) { - if (!processed[c]) { - definitions.push(...await this.database.findKanji(c, titles)); - processed[c] = true; + if (!kanjiUnique.hasOwnProperty(c)) { + kanjiList.push(c); + kanjiUnique[c] = true; } } + const definitions = await this.database.findKanjiBulk(kanjiList, titles); + if (definitions.length === 0) { + return definitions; + } + + if (definitions.length > 1) { + definitions.sort((a, b) => a.index - b.index); + } + + const kanjiList2 = []; for (const definition of definitions) { + kanjiList2.push(definition.character); + const tags = await this.expandTags(definition.tags, definition.dictionary); tags.push(dictTagBuildSource(definition.dictionary)); definition.tags = dictTagsSort(tags); definition.stats = await this.expandStats(definition.stats, definition.dictionary); - definition.frequencies = []; - for (const meta of await this.database.findKanjiMeta(definition.character, titles)) { - if (meta.mode === 'freq') { - definition.frequencies.push({ - character: meta.character, - frequency: meta.data, - dictionary: meta.dictionary - }); - } - } + } + + for (const meta of await this.database.findKanjiMetaBulk(kanjiList2, titles)) { + if (meta.mode !== 'freq') { continue; } + definitions[meta.index].frequencies.push({ + character: meta.character, + frequency: meta.data, + dictionary: meta.dictionary + }); } return definitions; From a08cbebeb33be8c41f0ec764b0f69e8c64b4c9f3 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sat, 19 Oct 2019 11:34:12 -0400 Subject: [PATCH 07/14] Use findTermsBySequenceBulk instead of findTermsBySequence --- ext/bg/js/translator.js | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/ext/bg/js/translator.js b/ext/bg/js/translator.js index 61acaa79..139fe0fb 100644 --- a/ext/bg/js/translator.js +++ b/ext/bg/js/translator.js @@ -52,23 +52,34 @@ class Translator { return {length, definitions: definitionsGrouped}; } + async getSequencedDefinitions(definitions, mainDictionary) { + const definitionsBySequence = dictTermsMergeBySequence(definitions, mainDictionary); + const defaultDefinitions = definitionsBySequence['-1']; + + const sequenceList = Object.keys(definitionsBySequence).map(v => Number(v)).filter(v => v >= 0); + const sequencedDefinitions = sequenceList.map((key) => ({ + definitions: definitionsBySequence[key], + rawDefinitions: [] + })); + + for (const definition of await this.database.findTermsBySequenceBulk(sequenceList, mainDictionary)) { + sequencedDefinitions[definition.index].rawDefinitions.push(definition); + } + + return {sequencedDefinitions, defaultDefinitions}; + } + async findTermsMerged(text, dictionaries, alphanumeric, options) { const secondarySearchTitles = Object.keys(options.dictionaries).filter(dict => options.dictionaries[dict].allowSecondarySearches); const titles = Object.keys(dictionaries); const {length, definitions} = await this.findTerms(text, dictionaries, alphanumeric); - - const definitionsBySequence = dictTermsMergeBySequence(definitions, options.general.mainDictionary); - + const {sequencedDefinitions, defaultDefinitions} = await this.getSequencedDefinitions(definitions, options.general.mainDictionary); const definitionsMerged = []; const mergedByTermIndices = new Set(); - for (const sequence in definitionsBySequence) { - if (sequence < 0) { - continue; - } - const result = definitionsBySequence[sequence]; - - const rawDefinitionsBySequence = await this.database.findTermsBySequence(Number(sequence), options.general.mainDictionary); + for (const sequencedDefinition of sequencedDefinitions) { + const result = sequencedDefinition.definitions; + const rawDefinitionsBySequence = sequencedDefinition.rawDefinitions; for (const definition of rawDefinitionsBySequence) { const definitionTags = await this.expandTags(definition.definitionTags, definition.dictionary); @@ -100,7 +111,7 @@ class Translator { } } - dictTermsMergeByGloss(result, definitionsBySequence['-1'].concat(secondarySearchResults), definitionsByGloss, mergedByTermIndices); + dictTermsMergeByGloss(result, defaultDefinitions.concat(secondarySearchResults), definitionsByGloss, mergedByTermIndices); for (const gloss in definitionsByGloss) { const definition = definitionsByGloss[gloss]; @@ -139,7 +150,7 @@ class Translator { definitionsMerged.push(result); } - const strayDefinitions = definitionsBySequence['-1'].filter((definition, index) => !mergedByTermIndices.has(index)); + const strayDefinitions = defaultDefinitions.filter((definition, index) => !mergedByTermIndices.has(index)); for (const groupedDefinition of dictTermsGroup(strayDefinitions, dictionaries)) { groupedDefinition.expressions = [{expression: groupedDefinition.expression, reading: groupedDefinition.reading}]; definitionsMerged.push(groupedDefinition); From 96f37fd9dcce1d8bfdaebe2f949c7732eb4c746a Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sat, 19 Oct 2019 12:16:38 -0400 Subject: [PATCH 08/14] Create function for body of findTermsMerged's sequencedDefinitions loop --- ext/bg/js/translator.js | 149 +++++++++++++++++++++------------------- 1 file changed, 80 insertions(+), 69 deletions(-) diff --git a/ext/bg/js/translator.js b/ext/bg/js/translator.js index 139fe0fb..1e6a2492 100644 --- a/ext/bg/js/translator.js +++ b/ext/bg/js/translator.js @@ -69,6 +69,78 @@ class Translator { return {sequencedDefinitions, defaultDefinitions}; } + async getMergedDefinition(text, dictionaries, sequencedDefinition, defaultDefinitions, secondarySearchTitles, mergedByTermIndices) { + const result = sequencedDefinition.definitions; + const rawDefinitionsBySequence = sequencedDefinition.rawDefinitions; + + for (const definition of rawDefinitionsBySequence) { + const definitionTags = await this.expandTags(definition.definitionTags, definition.dictionary); + definitionTags.push(dictTagBuildSource(definition.dictionary)); + definition.definitionTags = definitionTags; + const termTags = await this.expandTags(definition.termTags, definition.dictionary); + definition.termTags = termTags; + } + + const definitionsByGloss = dictTermsMergeByGloss(result, rawDefinitionsBySequence); + + const secondarySearchResults = []; + if (secondarySearchTitles.length > 0) { + for (const expression of result.expressions.keys()) { + if (expression === text) { + continue; + } + + for (const reading of result.expressions.get(expression).keys()) { + for (const definition of await this.database.findTermsExact(expression, reading, secondarySearchTitles)) { + const definitionTags = await this.expandTags(definition.definitionTags, definition.dictionary); + definitionTags.push(dictTagBuildSource(definition.dictionary)); + definition.definitionTags = definitionTags; + const termTags = await this.expandTags(definition.termTags, definition.dictionary); + definition.termTags = termTags; + secondarySearchResults.push(definition); + } + } + } + } + + dictTermsMergeByGloss(result, defaultDefinitions.concat(secondarySearchResults), definitionsByGloss, mergedByTermIndices); + + for (const gloss in definitionsByGloss) { + const definition = definitionsByGloss[gloss]; + dictTagsSort(definition.definitionTags); + result.definitions.push(definition); + } + + dictTermsSort(result.definitions, dictionaries); + + const expressions = []; + for (const expression of result.expressions.keys()) { + for (const reading of result.expressions.get(expression).keys()) { + const termTags = result.expressions.get(expression).get(reading); + expressions.push({ + expression: expression, + reading: reading, + termTags: dictTagsSort(termTags), + termFrequency: (score => { + if (score > 0) { + return 'popular'; + } else if (score < 0) { + return 'rare'; + } else { + return 'normal'; + } + })(termTags.map(tag => tag.score).reduce((p, v) => p + v, 0)) + }); + } + } + + result.expressions = expressions; + result.expression = Array.from(result.expression); + result.reading = Array.from(result.reading); + + return result; + } + async findTermsMerged(text, dictionaries, alphanumeric, options) { const secondarySearchTitles = Object.keys(options.dictionaries).filter(dict => options.dictionaries[dict].allowSecondarySearches); const titles = Object.keys(dictionaries); @@ -78,75 +150,14 @@ class Translator { const mergedByTermIndices = new Set(); for (const sequencedDefinition of sequencedDefinitions) { - const result = sequencedDefinition.definitions; - const rawDefinitionsBySequence = sequencedDefinition.rawDefinitions; - - for (const definition of rawDefinitionsBySequence) { - const definitionTags = await this.expandTags(definition.definitionTags, definition.dictionary); - definitionTags.push(dictTagBuildSource(definition.dictionary)); - definition.definitionTags = definitionTags; - const termTags = await this.expandTags(definition.termTags, definition.dictionary); - definition.termTags = termTags; - } - - const definitionsByGloss = dictTermsMergeByGloss(result, rawDefinitionsBySequence); - - const secondarySearchResults = []; - if (secondarySearchTitles.length > 0) { - for (const expression of result.expressions.keys()) { - if (expression === text) { - continue; - } - - for (const reading of result.expressions.get(expression).keys()) { - for (const definition of await this.database.findTermsExact(expression, reading, secondarySearchTitles)) { - const definitionTags = await this.expandTags(definition.definitionTags, definition.dictionary); - definitionTags.push(dictTagBuildSource(definition.dictionary)); - definition.definitionTags = definitionTags; - const termTags = await this.expandTags(definition.termTags, definition.dictionary); - definition.termTags = termTags; - secondarySearchResults.push(definition); - } - } - } - } - - dictTermsMergeByGloss(result, defaultDefinitions.concat(secondarySearchResults), definitionsByGloss, mergedByTermIndices); - - for (const gloss in definitionsByGloss) { - const definition = definitionsByGloss[gloss]; - dictTagsSort(definition.definitionTags); - result.definitions.push(definition); - } - - dictTermsSort(result.definitions, dictionaries); - - const expressions = []; - for (const expression of result.expressions.keys()) { - for (const reading of result.expressions.get(expression).keys()) { - const termTags = result.expressions.get(expression).get(reading); - expressions.push({ - expression: expression, - reading: reading, - termTags: dictTagsSort(termTags), - termFrequency: (score => { - if (score > 0) { - return 'popular'; - } else if (score < 0) { - return 'rare'; - } else { - return 'normal'; - } - })(termTags.map(tag => tag.score).reduce((p, v) => p + v, 0)) - }); - } - } - - result.expressions = expressions; - - result.expression = Array.from(result.expression); - result.reading = Array.from(result.reading); - + const result = await this.getMergedDefinition( + text, + dictionaries, + sequencedDefinition, + defaultDefinitions, + secondarySearchTitles, + mergedByTermIndices + ); definitionsMerged.push(result); } From 4446cbeabb28931a2c0777ae44dde70ec6aa8e9a Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sat, 19 Oct 2019 12:24:38 -0400 Subject: [PATCH 09/14] Create getMergedSecondarySearchResults --- ext/bg/js/translator.js | 47 +++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/ext/bg/js/translator.js b/ext/bg/js/translator.js index 1e6a2492..66653f4b 100644 --- a/ext/bg/js/translator.js +++ b/ext/bg/js/translator.js @@ -69,6 +69,32 @@ class Translator { return {sequencedDefinitions, defaultDefinitions}; } + async getMergedSecondarySearchResults(text, expressionsMap, secondarySearchTitles) { + const secondarySearchResults = []; + if (secondarySearchTitles.length === 0) { + return secondarySearchResults; + } + + for (const expression of expressionsMap.keys()) { + if (expression === text) { + continue; + } + + for (const reading of expressionsMap.get(expression).keys()) { + for (const definition of await this.database.findTermsExact(expression, reading, secondarySearchTitles)) { + const definitionTags = await this.expandTags(definition.definitionTags, definition.dictionary); + definitionTags.push(dictTagBuildSource(definition.dictionary)); + definition.definitionTags = definitionTags; + const termTags = await this.expandTags(definition.termTags, definition.dictionary); + definition.termTags = termTags; + secondarySearchResults.push(definition); + } + } + } + + return secondarySearchResults; + } + async getMergedDefinition(text, dictionaries, sequencedDefinition, defaultDefinitions, secondarySearchTitles, mergedByTermIndices) { const result = sequencedDefinition.definitions; const rawDefinitionsBySequence = sequencedDefinition.rawDefinitions; @@ -82,26 +108,7 @@ class Translator { } const definitionsByGloss = dictTermsMergeByGloss(result, rawDefinitionsBySequence); - - const secondarySearchResults = []; - if (secondarySearchTitles.length > 0) { - for (const expression of result.expressions.keys()) { - if (expression === text) { - continue; - } - - for (const reading of result.expressions.get(expression).keys()) { - for (const definition of await this.database.findTermsExact(expression, reading, secondarySearchTitles)) { - const definitionTags = await this.expandTags(definition.definitionTags, definition.dictionary); - definitionTags.push(dictTagBuildSource(definition.dictionary)); - definition.definitionTags = definitionTags; - const termTags = await this.expandTags(definition.termTags, definition.dictionary); - definition.termTags = termTags; - secondarySearchResults.push(definition); - } - } - } - } + const secondarySearchResults = await this.getMergedSecondarySearchResults(text, result.expressions, secondarySearchTitles); dictTermsMergeByGloss(result, defaultDefinitions.concat(secondarySearchResults), definitionsByGloss, mergedByTermIndices); From 97de7883a388099d5f0ff235c678e1f01c6858e2 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sat, 19 Oct 2019 12:42:26 -0400 Subject: [PATCH 10/14] Update getMergedSecondarySearchResults to use findTermsExactBulk --- ext/bg/js/translator.js | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/ext/bg/js/translator.js b/ext/bg/js/translator.js index 66653f4b..fa445cbf 100644 --- a/ext/bg/js/translator.js +++ b/ext/bg/js/translator.js @@ -70,29 +70,34 @@ class Translator { } async getMergedSecondarySearchResults(text, expressionsMap, secondarySearchTitles) { - const secondarySearchResults = []; if (secondarySearchTitles.length === 0) { - return secondarySearchResults; + return []; } + const expressionList = []; + const readingList = []; for (const expression of expressionsMap.keys()) { - if (expression === text) { - continue; - } - + if (expression === text) { continue; } for (const reading of expressionsMap.get(expression).keys()) { - for (const definition of await this.database.findTermsExact(expression, reading, secondarySearchTitles)) { - const definitionTags = await this.expandTags(definition.definitionTags, definition.dictionary); - definitionTags.push(dictTagBuildSource(definition.dictionary)); - definition.definitionTags = definitionTags; - const termTags = await this.expandTags(definition.termTags, definition.dictionary); - definition.termTags = termTags; - secondarySearchResults.push(definition); - } + expressionList.push(expression); + readingList.push(reading); } } - return secondarySearchResults; + const definitions = await this.database.findTermsExactBulk(expressionList, readingList, secondarySearchTitles); + for (const definition of definitions) { + const definitionTags = await this.expandTags(definition.definitionTags, definition.dictionary); + definitionTags.push(dictTagBuildSource(definition.dictionary)); + definition.definitionTags = definitionTags; + const termTags = await this.expandTags(definition.termTags, definition.dictionary); + definition.termTags = termTags; + } + + if (definitions.length > 1) { + definitions.sort((a, b) => a.index - b.index); + } + + return definitions; } async getMergedDefinition(text, dictionaries, sequencedDefinition, defaultDefinitions, secondarySearchTitles, mergedByTermIndices) { From 6f5fa6771bac24bb56c137fea6d7387d507aaf39 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sat, 19 Oct 2019 12:44:23 -0400 Subject: [PATCH 11/14] Remove unused database functions --- ext/bg/js/database.js | 52 ------------------------------------------- 1 file changed, 52 deletions(-) diff --git a/ext/bg/js/database.js b/ext/bg/js/database.js index 37f19359..c4d332bf 100644 --- a/ext/bg/js/database.js +++ b/ext/bg/js/database.js @@ -90,19 +90,6 @@ class Database { return results; } - async findTermsExact(term, reading, titles) { - this.validate(); - - const results = []; - await this.db.terms.where('expression').equals(term).each(row => { - if (row.reading === reading && titles.includes(row.dictionary)) { - results.push(Database.createTerm(row)); - } - }); - - return results; - } - async findTermsExactBulk(termList, readingList, titles) { this.validate(); @@ -129,19 +116,6 @@ class Database { return results; } - async findTermsBySequence(sequence, mainDictionary) { - this.validate(); - - const results = []; - await this.db.terms.where('sequence').equals(sequence).each(row => { - if (row.dictionary === mainDictionary) { - results.push(Database.createTerm(row)); - } - }); - - return results; - } - async findTermsBySequenceBulk(sequenceList, mainDictionary) { this.validate(); @@ -172,36 +146,10 @@ class Database { return this.findGenericBulk('termMeta', 'expression', termList, titles, Database.createMeta); } - async findKanji(kanji, titles) { - this.validate(); - - const results = []; - await this.db.kanji.where('character').equals(kanji).each(row => { - if (titles.includes(row.dictionary)) { - results.push(Database.createKanji(row)); - } - }); - - return results; - } - async findKanjiBulk(kanjiList, titles) { return this.findGenericBulk('kanji', 'character', kanjiList, titles, Database.createKanji); } - async findKanjiMeta(kanji, titles) { - this.validate(); - - const results = []; - await this.db.kanjiMeta.where('character').equals(kanji).each(row => { - if (titles.includes(row.dictionary)) { - results.push(Database.createMeta(row)); - } - }); - - return results; - } - async findKanjiMetaBulk(kanjiList, titles) { return this.findGenericBulk('kanjiMeta', 'character', kanjiList, titles, Database.createMeta); } From 610c2b9cca953e102aa31449d604d24d2840cce3 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sat, 19 Oct 2019 13:32:05 -0400 Subject: [PATCH 12/14] Move tagCache out of Database and into Translator --- ext/bg/js/database.js | 15 ------- ext/bg/js/translator.js | 86 ++++++++++++++++++++++++----------------- ext/bg/js/util.js | 2 +- 3 files changed, 52 insertions(+), 51 deletions(-) diff --git a/ext/bg/js/database.js b/ext/bg/js/database.js index c4d332bf..95466e73 100644 --- a/ext/bg/js/database.js +++ b/ext/bg/js/database.js @@ -20,7 +20,6 @@ class Database { constructor() { this.db = null; - this.tagCache = {}; } async prepare() { @@ -53,7 +52,6 @@ class Database { this.db.close(); await this.db.delete(); this.db = null; - this.tagCache = {}; await this.prepare(); } @@ -180,20 +178,9 @@ class Database { return results; } - findTagForTitleCached(name, title) { - if (this.tagCache.hasOwnProperty(title)) { - const cache = this.tagCache[title]; - if (cache.hasOwnProperty(name)) { - return cache[name]; - } - } - } - async findTagForTitle(name, title) { this.validate(); - const cache = (this.tagCache.hasOwnProperty(title) ? this.tagCache[title] : (this.tagCache[title] = {})); - let result = null; await this.db.tagMeta.where('name').equals(name).each(row => { if (title === row.dictionary) { @@ -201,8 +188,6 @@ class Database { } }); - cache[name] = result; - return result; } diff --git a/ext/bg/js/translator.js b/ext/bg/js/translator.js index fa445cbf..de7ebc2b 100644 --- a/ext/bg/js/translator.js +++ b/ext/bg/js/translator.js @@ -21,6 +21,7 @@ class Translator { constructor() { this.database = null; this.deinflector = null; + this.tagCache = {}; } async prepare() { @@ -36,6 +37,11 @@ class Translator { } } + async purgeDatabase() { + this.tagCache = {}; + await this.database.purge(); + } + async findTermsGrouped(text, dictionaries, alphanumeric, options) { const titles = Object.keys(dictionaries); const {length, definitions} = await this.findTerms(text, dictionaries, alphanumeric); @@ -404,56 +410,66 @@ class Translator { } async expandTags(names, title) { - const tags = []; - for (const name of names) { - const base = Translator.getNameBase(name); - let meta = this.database.findTagForTitleCached(base, title); - if (typeof meta === 'undefined') { - meta = await this.database.findTagForTitle(base, title); - } - - const tag = Object.assign({}, meta !== null ? meta : {}, {name}); - - tags.push(dictTagSanitize(tag)); - } - - return tags; + const tagMetaList = await this.getTagMetaList(names, title); + return tagMetaList.map((meta, index) => { + const name = names[index]; + const tag = dictTagSanitize(Object.assign({}, meta !== null ? meta : {}, {name})); + return dictTagSanitize(tag); + }); } async expandStats(items, title) { - const stats = {}; - for (const name in items) { - const base = Translator.getNameBase(name); - let meta = this.database.findTagForTitleCached(base, title); - if (typeof meta === 'undefined') { - meta = await this.database.findTagForTitle(base, title); - if (meta === null) { - continue; - } - } + const names = Object.keys(items); + const tagMetaList = await this.getTagMetaList(names, title); - const group = stats[meta.category] = stats[meta.category] || []; + const stats = {}; + for (let i = 0; i < names.length; ++i) { + const name = names[i]; + const meta = tagMetaList[i]; + if (meta === null) { continue; } + + const category = meta.category; + const group = ( + stats.hasOwnProperty(category) ? + stats[category] : + (stats[category] = []) + ); const stat = Object.assign({}, meta, {name, value: items[name]}); - group.push(dictTagSanitize(stat)); } + const sortCompare = (a, b) => a.notes - b.notes; for (const category in stats) { - stats[category].sort((a, b) => { - if (a.notes < b.notes) { - return -1; - } else if (a.notes > b.notes) { - return 1; - } else { - return 0; - } - }); + stats[category].sort(sortCompare); } return stats; } + async getTagMetaList(names, title) { + const tagMetaList = []; + const cache = ( + this.tagCache.hasOwnProperty(title) ? + this.tagCache[title] : + (this.tagCache[title] = {}) + ); + + for (const name of names) { + const base = Translator.getNameBase(name); + + if (cache.hasOwnProperty(base)) { + tagMetaList.push(cache[base]); + } else { + const tagMeta = await this.database.findTagForTitle(base, title); + cache[base] = tagMeta; + tagMetaList.push(tagMeta); + } + } + + return tagMetaList; + } + static getNameBase(name) { const pos = name.indexOf(':'); return (pos >= 0 ? name.substr(0, pos) : name); diff --git a/ext/bg/js/util.js b/ext/bg/js/util.js index 73a8396f..1ca0833b 100644 --- a/ext/bg/js/util.js +++ b/ext/bg/js/util.js @@ -89,7 +89,7 @@ function utilAnkiGetModelFieldNames(modelName) { } function utilDatabasePurge() { - return utilBackend().translator.database.purge(); + return utilBackend().translator.purgeDatabase(); } async function utilDatabaseImport(data, progress, exceptions) { From 96e4bf1735cef3881e5284ff624ed2114709af82 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sat, 19 Oct 2019 13:41:18 -0400 Subject: [PATCH 13/14] Use native IndexedDB APIs for findTagForTitle --- ext/bg/js/database.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ext/bg/js/database.js b/ext/bg/js/database.js index 95466e73..9f477b24 100644 --- a/ext/bg/js/database.js +++ b/ext/bg/js/database.js @@ -182,7 +182,12 @@ class Database { this.validate(); let result = null; - await this.db.tagMeta.where('name').equals(name).each(row => { + const db = this.db.backendDB(); + const dbTransaction = db.transaction(['tagMeta'], 'readonly'); + const dbTerms = dbTransaction.objectStore('tagMeta'); + const dbIndex = dbTerms.index('name'); + const only = IDBKeyRange.only(name); + await Database.getAll(dbIndex, only, null, row => { if (title === row.dictionary) { result = row; } From eea95eb130e8c444b93353f68f417ad2390795d7 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sat, 19 Oct 2019 14:18:57 -0400 Subject: [PATCH 14/14] Simplify score to term frequency conversion --- ext/bg/js/translator.js | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/ext/bg/js/translator.js b/ext/bg/js/translator.js index de7ebc2b..ee012d96 100644 --- a/ext/bg/js/translator.js +++ b/ext/bg/js/translator.js @@ -135,19 +135,12 @@ class Translator { for (const expression of result.expressions.keys()) { for (const reading of result.expressions.get(expression).keys()) { const termTags = result.expressions.get(expression).get(reading); + const score = termTags.map(tag => tag.score).reduce((p, v) => p + v, 0); expressions.push({ expression: expression, reading: reading, termTags: dictTagsSort(termTags), - termFrequency: (score => { - if (score > 0) { - return 'popular'; - } else if (score < 0) { - return 'rare'; - } else { - return 'normal'; - } - })(termTags.map(tag => tag.score).reduce((p, v) => p + v, 0)) + termFrequency: Translator.scoreToTermFrequency(score) }); } } @@ -470,6 +463,16 @@ class Translator { return tagMetaList; } + static scoreToTermFrequency(score) { + if (score > 0) { + return 'popular'; + } else if (score < 0) { + return 'rare'; + } else { + return 'normal'; + } + } + static getNameBase(name) { const pos = name.indexOf(':'); return (pos >= 0 ? name.substr(0, pos) : name);