Dictionary database improvements (#1527)

* Update formatting

* Add _findMultiBulk

* Update implementation of findTermsBySequenceBulk

* Update tests

* Generalize query creation

* Remove _findGenericBulk

* Reduce function creation

* Add more bindings

* Simplify findTermsExactBulk implementation

* Update var names

* Update _findMultiBulk to support multiple index queries

* Update findTermsBulk

* Update getMedia implementation

* Pass data arg to getAll and findFirst to avoid having multiple closures
This commit is contained in:
toasted-nutbread 2021-03-14 22:51:20 -04:00 committed by GitHub
parent 07df1e0117
commit a52d86a39e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 141 additions and 222 deletions

View File

@ -95,11 +95,11 @@ class Database {
});
}
getAll(objectStoreOrIndex, query, resolve, reject) {
getAll(objectStoreOrIndex, query, resolve, reject, data) {
if (typeof objectStoreOrIndex.getAll === 'function') {
this._getAllFast(objectStoreOrIndex, query, resolve, reject);
this._getAllFast(objectStoreOrIndex, query, resolve, reject, data);
} else {
this._getAllUsingCursor(objectStoreOrIndex, query, resolve, reject);
this._getAllUsingCursor(objectStoreOrIndex, query, resolve, reject, data);
}
}
@ -116,25 +116,25 @@ class Database {
const transaction = this.transaction([objectStoreName], 'readonly');
const objectStore = transaction.objectStore(objectStoreName);
const objectStoreOrIndex = indexName !== null ? objectStore.index(indexName) : objectStore;
this.findFirst(objectStoreOrIndex, query, resolve, reject, predicate, predicateArg, defaultValue);
this.findFirst(objectStoreOrIndex, query, resolve, reject, null, predicate, predicateArg, defaultValue);
});
}
findFirst(objectStoreOrIndex, query, resolve, reject, predicate, predicateArg, defaultValue) {
findFirst(objectStoreOrIndex, query, resolve, reject, data, predicate, predicateArg, defaultValue) {
const noPredicate = (typeof predicate !== 'function');
const request = objectStoreOrIndex.openCursor(query, 'next');
request.onerror = (e) => reject(e.target.error);
request.onerror = (e) => reject(e.target.error, data);
request.onsuccess = (e) => {
const cursor = e.target.result;
if (cursor) {
const {value} = cursor;
if (noPredicate || predicate(value, predicateArg)) {
resolve(value);
resolve(value, data);
} else {
cursor.continue();
}
} else {
resolve(defaultValue);
resolve(defaultValue, data);
}
};
}
@ -256,23 +256,23 @@ class Database {
return false;
}
_getAllFast(objectStoreOrIndex, query, resolve, reject) {
_getAllFast(objectStoreOrIndex, query, resolve, reject, data) {
const request = objectStoreOrIndex.getAll(query);
request.onerror = (e) => reject(e.target.error);
request.onsuccess = (e) => resolve(e.target.result);
request.onerror = (e) => reject(e.target.error, data);
request.onsuccess = (e) => resolve(e.target.result, data);
}
_getAllUsingCursor(objectStoreOrIndex, query, resolve, reject) {
_getAllUsingCursor(objectStoreOrIndex, query, resolve, reject, data) {
const results = [];
const request = objectStoreOrIndex.openCursor(query, 'next');
request.onerror = (e) => reject(e.target.error);
request.onerror = (e) => reject(e.target.error, data);
request.onsuccess = (e) => {
const cursor = e.target.result;
if (cursor) {
results.push(cursor.value);
cursor.continue();
} else {
resolve(results);
resolve(results, data);
}
};
}

View File

@ -24,6 +24,17 @@ class DictionaryDatabase {
this._db = new Database();
this._dbName = 'dict';
this._schemas = new Map();
this._createOnlyQuery1 = (item) => IDBKeyRange.only(item);
this._createOnlyQuery2 = (item) => IDBKeyRange.only(item.query);
this._createOnlyQuery3 = (item) => IDBKeyRange.only(item.expression);
this._createOnlyQuery4 = (item) => IDBKeyRange.only(item.path);
this._createBoundQuery1 = (item) => IDBKeyRange.bound(item, `${item}\uffff`, false, false);
this._createBoundQuery2 = (item) => { item = stringReverse(item); return IDBKeyRange.bound(item, `${item}\uffff`, false, false); };
this._createTermBind = this._createTerm.bind(this);
this._createTermMetaBind = this._createTermMeta.bind(this);
this._createKanjiBind = this._createKanji.bind(this);
this._createKanjiMetaBind = this._createKanjiMeta.bind(this);
this._createMediaBind = this._createMedia.bind(this);
}
// Public
@ -171,132 +182,61 @@ class DictionaryDatabase {
}
findTermsBulk(termList, dictionaries, wildcard) {
return new Promise((resolve, reject) => {
const results = [];
const count = termList.length;
if (count === 0) {
resolve(results);
return;
}
const visited = new Set();
const predicate = (row) => {
if (!dictionaries.has(row.dictionary)) { return false; }
const {id} = row;
if (visited.has(id)) { return false; }
visited.add(id);
return true;
};
const visited = new Set();
const useWildcard = !!wildcard;
const prefixWildcard = wildcard === 'prefix';
const indexNames = (wildcard === 'prefix') ? ['expressionReverse', 'readingReverse'] : ['expression', 'reading'];
const transaction = this._db.transaction(['terms'], 'readonly');
const terms = transaction.objectStore('terms');
const index1 = terms.index(prefixWildcard ? 'expressionReverse' : 'expression');
const index2 = terms.index(prefixWildcard ? 'readingReverse' : 'reading');
let createQuery;
switch (wildcard) {
case 'suffix':
createQuery = this._createBoundQuery1;
break;
case 'prefix':
createQuery = this._createBoundQuery2;
break;
default:
createQuery = this._createOnlyQuery1;
break;
}
const count2 = count * 2;
let completeCount = 0;
for (let i = 0; i < count; ++i) {
const inputIndex = i;
const term = prefixWildcard ? stringReverse(termList[i]) : termList[i];
const query = useWildcard ? IDBKeyRange.bound(term, `${term}\uffff`, false, false) : IDBKeyRange.only(term);
const onGetAll = (rows) => {
for (const row of rows) {
if (dictionaries.has(row.dictionary) && !visited.has(row.id)) {
visited.add(row.id);
results.push(this._createTerm(row, inputIndex));
}
}
if (++completeCount >= count2) {
resolve(results);
}
};
this._db.getAll(index1, query, onGetAll, reject);
this._db.getAll(index2, query, onGetAll, reject);
}
});
return this._findMultiBulk('terms', indexNames, termList, createQuery, predicate, this._createTermBind);
}
findTermsExactBulk(termList, readingList, dictionaries) {
return new Promise((resolve, reject) => {
const results = [];
const count = termList.length;
if (count === 0) {
resolve(results);
return;
}
const transaction = this._db.transaction(['terms'], 'readonly');
const terms = transaction.objectStore('terms');
const index = terms.index('expression');
let completeCount = 0;
for (let i = 0; i < count; ++i) {
const inputIndex = i;
const reading = readingList[i];
const query = IDBKeyRange.only(termList[i]);
const onGetAll = (rows) => {
for (const row of rows) {
if (row.reading === reading && dictionaries.has(row.dictionary)) {
results.push(this._createTerm(row, inputIndex));
}
}
if (++completeCount >= count) {
resolve(results);
}
};
this._db.getAll(index, query, onGetAll, reject);
}
});
findTermsExactBulk(termList, dictionaries) {
const predicate = (row, item) => (row.reading === item.reading && dictionaries.has(row.dictionary));
return this._findMultiBulk('terms', ['expression'], termList, this._createOnlyQuery3, predicate, this._createTermBind);
}
findTermsBySequenceBulk(sequenceList, mainDictionary) {
return new Promise((resolve, reject) => {
const results = [];
const count = sequenceList.length;
if (count === 0) {
resolve(results);
return;
}
const transaction = this._db.transaction(['terms'], 'readonly');
const terms = transaction.objectStore('terms');
const index = terms.index('sequence');
let completeCount = 0;
for (let i = 0; i < count; ++i) {
const inputIndex = i;
const query = IDBKeyRange.only(sequenceList[i]);
const onGetAll = (rows) => {
for (const row of rows) {
if (row.dictionary === mainDictionary) {
results.push(this._createTerm(row, inputIndex));
}
}
if (++completeCount >= count) {
resolve(results);
}
};
this._db.getAll(index, query, onGetAll, reject);
}
});
findTermsBySequenceBulk(items) {
const predicate = (row, item) => (row.dictionary === item.dictionary);
return this._findMultiBulk('terms', ['sequence'], items, this._createOnlyQuery2, predicate, this._createTermBind);
}
findTermMetaBulk(termList, dictionaries) {
return this._findGenericBulk('termMeta', 'expression', termList, dictionaries, this._createTermMeta.bind(this));
const predicate = (row) => dictionaries.has(row.dictionary);
return this._findMultiBulk('termMeta', ['expression'], termList, this._createOnlyQuery1, predicate, this._createTermMetaBind);
}
findKanjiBulk(kanjiList, dictionaries) {
return this._findGenericBulk('kanji', 'character', kanjiList, dictionaries, this._createKanji.bind(this));
const predicate = (row) => dictionaries.has(row.dictionary);
return this._findMultiBulk('kanji', ['character'], kanjiList, this._createOnlyQuery1, predicate, this._createKanjiBind);
}
findKanjiMetaBulk(kanjiList, dictionaries) {
return this._findGenericBulk('kanjiMeta', 'character', kanjiList, dictionaries, this._createKanjiMeta.bind(this));
const predicate = (row) => dictionaries.has(row.dictionary);
return this._findMultiBulk('kanjiMeta', ['character'], kanjiList, this._createOnlyQuery1, predicate, this._createKanjiMetaBind);
}
findTagMetaBulk(items) {
const predicate = (row, item) => (row.dictionary === item.dictionary);
return this._findFirstBulk('tagMeta', 'name', items, predicate);
return this._findFirstBulk('tagMeta', 'name', items, this._createOnlyQuery2, predicate);
}
findTagForTitle(name, title) {
@ -304,38 +244,9 @@ class DictionaryDatabase {
return this._db.find('tagMeta', 'name', query, (row) => (row.dictionary === title), null, null);
}
getMedia(targets) {
return new Promise((resolve, reject) => {
const count = targets.length;
const results = new Array(count).fill(null);
if (count === 0) {
resolve(results);
return;
}
let completeCount = 0;
const transaction = this._db.transaction(['media'], 'readonly');
const objectStore = transaction.objectStore('media');
const index = objectStore.index('path');
for (let i = 0; i < count; ++i) {
const inputIndex = i;
const {path, dictionaryName} = targets[i];
const query = IDBKeyRange.only(path);
const onGetAll = (rows) => {
for (const row of rows) {
if (row.dictionary !== dictionaryName) { continue; }
results[inputIndex] = this._createMedia(row, inputIndex);
}
if (++completeCount >= count) {
resolve(results);
}
};
this._db.getAll(index, query, onGetAll, reject);
}
});
getMedia(items) {
const predicate = (row, item) => (row.dictionary === item.dictionary);
return this._findMultiBulk('media', ['path'], items, this._createOnlyQuery4, predicate, this._createMediaBind);
}
getDictionaryInfo() {
@ -408,66 +319,67 @@ class DictionaryDatabase {
// Private
async _findGenericBulk(objectStoreName, indexName, indexValueList, dictionaries, createResult) {
_findMultiBulk(objectStoreName, indexNames, items, createQuery, predicate, createResult) {
return new Promise((resolve, reject) => {
const itemCount = items.length;
const indexCount = indexNames.length;
const results = [];
const count = indexValueList.length;
if (count === 0) {
if (itemCount === 0 || indexCount === 0) {
resolve(results);
return;
}
const transaction = this._db.transaction([objectStoreName], 'readonly');
const terms = transaction.objectStore(objectStoreName);
const index = terms.index(indexName);
const objectStore = transaction.objectStore(objectStoreName);
const indexList = [];
for (const indexName of indexNames) {
indexList.push(objectStore.index(indexName));
}
let completeCount = 0;
for (let i = 0; i < count; ++i) {
const inputIndex = i;
const query = IDBKeyRange.only(indexValueList[i]);
const onGetAll = (rows) => {
for (const row of rows) {
if (dictionaries.has(row.dictionary)) {
results.push(createResult(row, inputIndex));
}
const requiredCompleteCount = itemCount * indexCount;
const onGetAll = (rows, {item, itemIndex}) => {
for (const row of rows) {
if (predicate(row, item)) {
results.push(createResult(row, itemIndex));
}
if (++completeCount >= count) {
resolve(results);
}
};
this._db.getAll(index, query, onGetAll, reject);
}
if (++completeCount >= requiredCompleteCount) {
resolve(results);
}
};
for (let i = 0; i < itemCount; ++i) {
const item = items[i];
const query = createQuery(item);
for (let j = 0; j < indexCount; ++j) {
this._db.getAll(indexList[j], query, onGetAll, reject, {item, itemIndex: i});
}
}
});
}
_findFirstBulk(objectStoreName, indexName, items, predicate) {
_findFirstBulk(objectStoreName, indexName, items, createQuery, predicate) {
return new Promise((resolve, reject) => {
const count = items.length;
const results = new Array(count);
if (count === 0) {
const itemCount = items.length;
const results = new Array(itemCount);
if (itemCount === 0) {
resolve(results);
return;
}
const transaction = this._db.transaction([objectStoreName], 'readonly');
const terms = transaction.objectStore(objectStoreName);
const index = terms.index(indexName);
const objectStore = transaction.objectStore(objectStoreName);
const index = objectStore.index(indexName);
let completeCount = 0;
for (let i = 0; i < count; ++i) {
const itemIndex = i;
const onFind = (row, itemIndex) => {
results[itemIndex] = row;
if (++completeCount >= itemCount) {
resolve(results);
}
};
for (let i = 0; i < itemCount; ++i) {
const item = items[i];
const query = IDBKeyRange.only(item.query);
const onFind = (row) => {
results[itemIndex] = row;
if (++completeCount >= count) {
resolve(results);
}
};
this._db.findFirst(index, query, onFind, reject, predicate, item, void 0);
const query = createQuery(item);
this._db.findFirst(index, query, onFind, reject, i, predicate, item, void 0);
}
});
}

View File

@ -392,7 +392,8 @@ class Translator {
}
async _addRelatedDefinitions(sequencedDefinitions, unsequencedDefinitions, sequenceList, mainDictionary, enabledDictionaryMap) {
const databaseDefinitions = await this._database.findTermsBySequenceBulk(sequenceList, mainDictionary);
const items = sequenceList.map((query) => ({query, dictionary: mainDictionary}));
const databaseDefinitions = await this._database.findTermsBySequenceBulk(items);
for (const databaseDefinition of databaseDefinitions) {
const {relatedDefinitions, definitionIds} = sequencedDefinitions[databaseDefinition.index];
const {id} = databaseDefinition;
@ -410,8 +411,7 @@ class Translator {
if (unsequencedDefinitions.length === 0 && secondarySearchDictionaryMap.size === 0) { return; }
// Prepare grouping info
const expressionList = [];
const readingList = [];
const termList = [];
const targetList = [];
const targetMap = new Map();
@ -431,8 +431,7 @@ class Translator {
target.sequencedDefinitions.push(sequencedDefinition);
if (!definition.isPrimary && !target.searchSecondary) {
target.searchSecondary = true;
expressionList.push(expression);
readingList.push(reading);
termList.push({expression, reading});
targetList.push(target);
}
}
@ -456,14 +455,14 @@ class Translator {
}
// Search database for additional secondary terms
if (expressionList.length === 0 || secondarySearchDictionaryMap.size === 0) { return; }
if (termList.length === 0 || secondarySearchDictionaryMap.size === 0) { return; }
const databaseDefinitions = await this._database.findTermsExactBulk(expressionList, readingList, secondarySearchDictionaryMap);
const databaseDefinitions = await this._database.findTermsExactBulk(termList, secondarySearchDictionaryMap);
this._sortDatabaseDefinitionsByIndex(databaseDefinitions);
for (const databaseDefinition of databaseDefinitions) {
const {index, id} = databaseDefinition;
const source = expressionList[index];
const source = termList[index].expression;
const target = targetList[index];
for (const {definitionIds, secondaryDefinitions} of target.sequencedDefinitions) {
if (definitionIds.has(id)) { continue; }

View File

@ -26,13 +26,13 @@ class MediaLoader {
this._loadMediaData = [];
}
async loadMedia(path, dictionaryName, onLoad, onUnload) {
async loadMedia(path, dictionary, onLoad, onUnload) {
const token = this._token;
const data = {onUnload, loaded: false};
this._loadMediaData.push(data);
const media = await this.getMedia(path, dictionaryName);
const media = await this.getMedia(path, dictionary);
if (token !== this._token) { return; }
onLoad(media.url);
@ -59,14 +59,14 @@ class MediaLoader {
this._token = {};
}
async getMedia(path, dictionaryName) {
async getMedia(path, dictionary) {
let cachedData;
let dictionaryCache = this._mediaCache.get(dictionaryName);
let dictionaryCache = this._mediaCache.get(dictionary);
if (typeof dictionaryCache !== 'undefined') {
cachedData = dictionaryCache.get(path);
} else {
dictionaryCache = new Map();
this._mediaCache.set(dictionaryName, dictionaryCache);
this._mediaCache.set(dictionary, dictionaryCache);
}
if (typeof cachedData === 'undefined') {
@ -76,15 +76,15 @@ class MediaLoader {
url: null
};
dictionaryCache.set(path, cachedData);
cachedData.promise = this._getMediaData(path, dictionaryName, cachedData);
cachedData.promise = this._getMediaData(path, dictionary, cachedData);
}
return cachedData.promise;
}
async _getMediaData(path, dictionaryName, cachedData) {
async _getMediaData(path, dictionary, cachedData) {
const token = this._token;
const data = (await yomichan.api.getMedia([{path, dictionaryName}]))[0];
const data = (await yomichan.api.getMedia([{path, dictionary}]))[0];
if (token === this._token && data !== null) {
const blob = MediaUtil.createBlobFromBase64Content(data.content, data.mediaType);
const url = URL.createObjectURL(blob);

View File

@ -292,8 +292,11 @@ async function testTindTermsExactBulk1(database, titles) {
{
inputs: [
{
termList: ['打', '打つ', '打ち込む'],
readingList: ['だ', 'うつ', 'うちこむ']
termList: [
{expression: '打', reading: 'だ'},
{expression: '打つ', reading: 'うつ'},
{expression: '打ち込む', reading: 'うちこむ'}
]
}
],
expectedResults: {
@ -313,8 +316,11 @@ async function testTindTermsExactBulk1(database, titles) {
{
inputs: [
{
termList: ['打', '打つ', '打ち込む'],
readingList: ['だ?', 'うつ?', 'うちこむ?']
termList: [
{expression: '打', reading: 'だ?'},
{expression: '打つ', reading: 'うつ?'},
{expression: '打ち込む', reading: 'うちこむ?'}
]
}
],
expectedResults: {
@ -326,8 +332,10 @@ async function testTindTermsExactBulk1(database, titles) {
{
inputs: [
{
termList: ['打つ', '打つ'],
readingList: ['うつ', 'ぶつ']
termList: [
{expression: '打つ', reading: 'うつ'},
{expression: '打つ', reading: 'ぶつ'}
]
}
],
expectedResults: {
@ -344,8 +352,9 @@ async function testTindTermsExactBulk1(database, titles) {
{
inputs: [
{
termList: ['打つ'],
readingList: ['うちこむ']
termList: [
{expression: '打つ', reading: 'うちこむ'}
]
}
],
expectedResults: {
@ -357,8 +366,7 @@ async function testTindTermsExactBulk1(database, titles) {
{
inputs: [
{
termList: [],
readingList: []
termList: []
}
],
expectedResults: {
@ -370,8 +378,8 @@ async function testTindTermsExactBulk1(database, titles) {
];
for (const {inputs, expectedResults} of data) {
for (const {termList, readingList} of inputs) {
const results = await database.findTermsExactBulk(termList, readingList, titles);
for (const {termList} of inputs) {
const results = await database.findTermsExactBulk(termList, titles);
assert.strictEqual(results.length, expectedResults.total);
for (const [expression, count] of expectedResults.expressions) {
assert.strictEqual(countTermsWithExpression(results, expression), count);
@ -520,7 +528,7 @@ async function testFindTermsBySequenceBulk1(database, mainDictionary) {
for (const {inputs, expectedResults} of data) {
for (const {sequenceList} of inputs) {
const results = await database.findTermsBySequenceBulk(sequenceList, mainDictionary);
const results = await database.findTermsBySequenceBulk(sequenceList.map((query) => ({query, dictionary: mainDictionary})));
assert.strictEqual(results.length, expectedResults.total);
for (const [expression, count] of expectedResults.expressions) {
assert.strictEqual(countTermsWithExpression(results, expression), count);
@ -773,8 +781,8 @@ async function testDatabase2() {
// Error: not prepared
await assert.rejects(async () => await dictionaryDatabase.deleteDictionary(title, {rate: 1000}, () => {}));
await assert.rejects(async () => await dictionaryDatabase.findTermsBulk(['?'], titles, null));
await assert.rejects(async () => await dictionaryDatabase.findTermsExactBulk(['?'], ['?'], titles));
await assert.rejects(async () => await dictionaryDatabase.findTermsBySequenceBulk([1], title));
await assert.rejects(async () => await dictionaryDatabase.findTermsExactBulk([{expression: '?', reading: '?'}], titles));
await assert.rejects(async () => await dictionaryDatabase.findTermsBySequenceBulk([{query: 1, dictionary: title}]));
await assert.rejects(async () => await dictionaryDatabase.findTermMetaBulk(['?'], titles));
await assert.rejects(async () => await dictionaryDatabase.findTermMetaBulk(['?'], titles));
await assert.rejects(async () => await dictionaryDatabase.findKanjiBulk(['?'], titles));