alex/yomichan
1
Fork 0
Code Releases Activity

Normalize terms and readings during import (#1941)

Browse Source
This commit is contained in:
toasted-nutbread 2021-09-09 23:25:24 -04:00 committed by GitHub
parent 069ca1a97b
commit 7883d39496
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 14 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
ext/js/language/dictionary-importer.js
View File

@ -476,12 +476,16 @@ class DictionaryImporter {
}
_convertTermBankEntryV1(entry, dictionary) {
const [expression, reading, definitionTags, rules, score, ...glossary] = entry;
let [expression, reading, definitionTags, rules, score, ...glossary] = entry;
expression = this._normalizeTermOrReading(expression);
reading = this._normalizeTermOrReading(reading.length > 0 ? reading : expression);
return {expression, reading, definitionTags, rules, score, glossary, dictionary};
}
_convertTermBankEntryV3(entry, dictionary) {
const [expression, reading, definitionTags, rules, score, glossary, sequence, termTags] = entry;
let [expression, reading, definitionTags, rules, score, glossary, sequence, termTags] = entry;
expression = this._normalizeTermOrReading(expression);
reading = this._normalizeTermOrReading(reading.length > 0 ? reading : expression);
return {expression, reading, definitionTags, rules, score, glossary, sequence, termTags, dictionary};
}
@ -580,4 +584,12 @@ class DictionaryImporter {
}
return counts;
}
_normalizeTermOrReading(text) {
try {
return text.normalize('NFC');
} catch (e) {
return text;
}
}
}