alex/yomichan
1
Fork 0
Code Releases Activity

Normalize terms and readings during import (#1941)

Browse Source
This commit is contained in:
toasted-nutbread 2021-09-09 23:25:24 -04:00 committed by GitHub
parent 069ca1a97b
commit 7883d39496
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 14 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
ext/js/language/dictionary-importer.js
View File

@ -476,12 +476,16 @@ class DictionaryImporter {
} }
_convertTermBankEntryV1(entry, dictionary) { _convertTermBankEntryV1(entry, dictionary) {
const [expression, reading, definitionTags, rules, score, ...glossary] = entry; let [expression, reading, definitionTags, rules, score, ...glossary] = entry;
expression = this._normalizeTermOrReading(expression);
reading = this._normalizeTermOrReading(reading.length > 0 ? reading : expression);
return {expression, reading, definitionTags, rules, score, glossary, dictionary}; return {expression, reading, definitionTags, rules, score, glossary, dictionary};
} }
_convertTermBankEntryV3(entry, dictionary) { _convertTermBankEntryV3(entry, dictionary) {
const [expression, reading, definitionTags, rules, score, glossary, sequence, termTags] = entry; let [expression, reading, definitionTags, rules, score, glossary, sequence, termTags] = entry;
expression = this._normalizeTermOrReading(expression);
reading = this._normalizeTermOrReading(reading.length > 0 ? reading : expression);
return {expression, reading, definitionTags, rules, score, glossary, sequence, termTags, dictionary}; return {expression, reading, definitionTags, rules, score, glossary, sequence, termTags, dictionary};
} }
@ -580,4 +584,12 @@ class DictionaryImporter {
} }
return counts; return counts;
} }
_normalizeTermOrReading(text) {
try {
return text.normalize('NFC');
} catch (e) {
return text;
}
}
} }