alex/yomichan
1
Fork 0
Code Releases Activity

deinflector optimizations

Browse Source
This commit is contained in:
Alex Yatskov 2016-12-19 21:23:17 -08:00
parent 0aa603694c
commit 28b8bae6a7
3 changed files with 683 additions and 703 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1276
ext/bg/data/deinflect.json
View File

File diff suppressed because it is too large Load Diff

35
ext/bg/js/deinflector.js
View File

@ -23,24 +23,25 @@ class Deinflection {
this.term = term; this.term = term;
this.rules = rules; this.rules = rules;
this.reason = reason; this.reason = reason;
this.definitions = [];
} }
deinflect(validator, reasons, entry=false) { deinflect(definer, reasons, entry=false) {
const validate = () => { const validate = () => {
if (entry) { return definer(this.term).then(definitions => {
return Promise.resolve(true); if (entry) {
} this.definitions = definitions;
} else {
return validator(this.term).then(sets => {
for (const rules of sets) {
for (const rule of this.rules) { for (const rule of this.rules) {
if (rules.includes(rule)) { for (const definition of definitions) {
return true; if (definition.rules.includes(rule)) {
this.definitions.push(definition);
}
} }
} }
} }
return false; return this.definitions.length > 0;
}); });
}; };
@ -74,7 +75,7 @@ class Deinflection {
const child = new Deinflection(term, variant.rulesOut, reason); const child = new Deinflection(term, variant.rulesOut, reason);
promises.push( promises.push(
child.deinflect(validator, reasons).then(valid => { child.deinflect(definer, reasons).then(valid => {
if (valid) { if (valid) {
this.children.push(child); this.children.push(child);
} }
@ -90,12 +91,18 @@ class Deinflection {
gather() { gather() {
if (this.children.length === 0) { if (this.children.length === 0) {
return [{root: this.term, rules: this.rules, reasons: []}]; return [{
root: this.term,
rules: this.rules,
definitions: this.definitions,
reasons: []
}];
} }
const paths = []; const paths = [];
for (const child of this.children) { for (const child of this.children) {
for (const path of child.gather()) { for (const path of child.gather()) {
path.definitions = path.definitions.concat(this.definitions);
if (this.reason.length > 0) { if (this.reason.length > 0) {
path.reasons.push(this.reason); path.reasons.push(this.reason);
} }
@ -119,8 +126,8 @@ class Deinflector {
this.reasons = reasons; this.reasons = reasons;
} }
deinflect(term, validator) { deinflect(term, definer) {
const node = new Deinflection(term); const node = new Deinflection(term);
return node.deinflect(validator, this.reasons, true).then(success => success ? node.gather() : []); return node.deinflect(definer, this.reasons, true).then(success => success ? node.gather() : []);
} }
} }

75
ext/bg/js/translator.js
View File

@ -88,68 +88,41 @@ class Translator {
} }
findDeinflectionGroups(text, dictionaries) { findDeinflectionGroups(text, dictionaries) {
const deinflectionGroups = {}; const definer = term => this.database.findTerm(term, dictionaries);
const deinflectionPromises = []; const groups = {};
const promises = [];
for (let i = text.length; i > 0; --i) { for (let i = text.length; i > 0; --i) {
deinflectionPromises.push( promises.push(
this.deinflector.deinflect(text.slice(0, i), term => { this.deinflector.deinflect(text.slice(0, i), definer).then(deinflections => {
return this.database.findTerm(term, dictionaries).then(definitions => definitions.map(definition => definition.rules));
}).then(deinflections => {
const processPromises = [];
for (const deinflection of deinflections) { for (const deinflection of deinflections) {
processPromises.push( this.processDeinflection(groups, deinflection);
this.processDeinflection(
deinflectionGroups,
deinflection.source,
deinflection.rules,
deinflection.reasons,
deinflection.root,
dictionaries
)
);
} }
return Promise.all(processPromises);
}) })
); );
} }
return Promise.all(deinflectionPromises).then(() => deinflectionGroups); return Promise.all(promises).then(() => groups);
} }
processDeinflection(groups, source, rules, reasons, root, dictionaries) { processDeinflection(groups, {source, rules, reasons, root, definitions}, dictionaries) {
return this.database.findTerm(root, dictionaries).then(definitions => { for (const definition of definitions) {
for (const definition of definitions) { if (definition.id in groups) {
if (definition.id in groups) { continue;
continue;
}
let matched = rules.length === 0;
for (const rule of rules) {
if (definition.rules.includes(rule)) {
matched = true;
break;
}
}
if (!matched) {
continue;
}
const tags = definition.tags.map(tag => buildTag(tag, definition.tagMeta));
groups[definition.id] = {
source,
reasons,
score: definition.score,
dictionary: definition.dictionary,
expression: definition.expression,
reading: definition.reading,
glossary: definition.glossary,
tags: sortTags(tags)
};
} }
});
const tags = definition.tags.map(tag => buildTag(tag, definition.tagMeta));
groups[definition.id] = {
source,
reasons,
score: definition.score,
dictionary: definition.dictionary,
expression: definition.expression,
reading: definition.reading,
glossary: definition.glossary,
tags: sortTags(tags)
};
}
} }
processKanji(definitions) { processKanji(definitions) {