Normalize terms and readings during import (#1941)
This commit is contained in:
parent
069ca1a97b
commit
7883d39496
@ -476,12 +476,16 @@ class DictionaryImporter {
|
||||
}
|
||||
|
||||
_convertTermBankEntryV1(entry, dictionary) {
|
||||
const [expression, reading, definitionTags, rules, score, ...glossary] = entry;
|
||||
let [expression, reading, definitionTags, rules, score, ...glossary] = entry;
|
||||
expression = this._normalizeTermOrReading(expression);
|
||||
reading = this._normalizeTermOrReading(reading.length > 0 ? reading : expression);
|
||||
return {expression, reading, definitionTags, rules, score, glossary, dictionary};
|
||||
}
|
||||
|
||||
_convertTermBankEntryV3(entry, dictionary) {
|
||||
const [expression, reading, definitionTags, rules, score, glossary, sequence, termTags] = entry;
|
||||
let [expression, reading, definitionTags, rules, score, glossary, sequence, termTags] = entry;
|
||||
expression = this._normalizeTermOrReading(expression);
|
||||
reading = this._normalizeTermOrReading(reading.length > 0 ? reading : expression);
|
||||
return {expression, reading, definitionTags, rules, score, glossary, sequence, termTags, dictionary};
|
||||
}
|
||||
|
||||
@ -580,4 +584,12 @@ class DictionaryImporter {
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
_normalizeTermOrReading(text) {
|
||||
try {
|
||||
return text.normalize('NFC');
|
||||
} catch (e) {
|
||||
return text;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user