diff --git a/ext/bg/js/translator.js b/ext/bg/js/translator.js index bb78d46d..88a5c6e8 100644 --- a/ext/bg/js/translator.js +++ b/ext/bg/js/translator.js @@ -240,6 +240,7 @@ class Translator { definitions.push({ source: deinflection.source, + rawSource: deinflection.rawSource, reasons: deinflection.reasons, score: definition.score, id: definition.id, @@ -260,7 +261,7 @@ class Translator { let length = 0; for (const definition of definitions) { - length = Math.max(length, definition.source.length); + length = Math.max(length, definition.rawSource.length); } return [definitions, length]; @@ -274,6 +275,7 @@ class Translator { return [{ source: text, + rawSource: text, term: text, rules: 0, definitions, @@ -320,27 +322,71 @@ class Translator { return deinflections.filter((e) => e.definitions.length > 0); } - getAllDeinflections(text, _options) { + getAllDeinflections(text, options) { + const translationOptions = options.translation; const textOptionVariantArray = [ - [false, true] // convert katakana to hiragana + Translator.getTextOptionEntryVariants(translationOptions.convertKatakanaToHiragana), + Translator.getTextOptionEntryVariants(translationOptions.convertHalfWidthCharacters), + Translator.getTextOptionEntryVariants(translationOptions.convertNumericCharacters), + Translator.getTextOptionEntryVariants(translationOptions.convertAlphabeticCharacters) ]; const deinflections = []; const used = new Set(); - for (const [hiragana] of Translator.getArrayVariants(textOptionVariantArray)) { + for (const [hiragana, halfWidth, numeric, alphabetic] of Translator.getArrayVariants(textOptionVariantArray)) { let text2 = text; + let sourceMapping = null; + if (halfWidth) { + if (sourceMapping === null) { sourceMapping = Translator.createTextSourceMapping(text2); } + text2 = jpConvertHalfWidthKanaToFullWidth(text2, sourceMapping); + } + if (numeric) { text2 = jpConvertNumericTofullWidth(text2); } + if (alphabetic) { + if (sourceMapping === null) { sourceMapping = Translator.createTextSourceMapping(text2); } + text2 = jpConvertAlphabeticToKana(text2, sourceMapping); + } if (hiragana) { text2 = jpKatakanaToHiragana(text2); } for (let i = text2.length; i > 0; --i) { const text2Substring = text2.substring(0, i); if (used.has(text2Substring)) { break; } used.add(text2Substring); - deinflections.push(...this.deinflector.deinflect(text2Substring)); + for (const deinflection of this.deinflector.deinflect(text2Substring)) { + deinflection.rawSource = Translator.getDeinflectionRawSource(text, i, sourceMapping); + deinflections.push(deinflection); + } } } return deinflections; } + static getTextOptionEntryVariants(value) { + switch (value) { + case 'true': return [true]; + case 'variant': return [false, true]; + default: return [false]; + } + } + + static getDeinflectionRawSource(source, length, sourceMapping) { + if (sourceMapping === null) { + return source.substring(0, length); + } + + let result = ''; + let index = 0; + for (let i = 0; i < length; ++i) { + const c = sourceMapping[i]; + result += source.substring(index, index + c); + index += c; + } + return result; + } + + static createTextSourceMapping(text) { + return new Array(text.length).fill(1); + } + async findKanji(text, options) { const dictionaries = dictEnabledSet(options); const titles = Object.keys(dictionaries); diff --git a/ext/mixed/js/japanese.js b/ext/mixed/js/japanese.js index 23b2bd36..26349094 100644 --- a/ext/mixed/js/japanese.js +++ b/ext/mixed/js/japanese.js @@ -17,6 +17,65 @@ */ +const jpHalfWidthCharacterMapping = new Map([ + ['ｦ', 'ヲヺ-'], + ['ｧ', 'ァ--'], + ['ｨ', 'ィ--'], + ['ｩ', 'ゥ--'], + ['ｪ', 'ェ--'], + ['ｫ', 'ォ--'], + ['ｬ', 'ャ--'], + ['ｭ', 'ュ--'], + ['ｮ', 'ョ--'], + ['ｯ', 'ッ--'], + ['ｰ', 'ー--'], + ['ｱ', 'ア--'], + ['ｲ', 'イ--'], + ['ｳ', 'ウヴ-'], + ['ｴ', 'エ--'], + ['ｵ', 'オ--'], + ['ｶ', 'カガ-'], + ['ｷ', 'キギ-'], + ['ｸ', 'クグ-'], + ['ｹ', 'ケゲ-'], + ['ｺ', 'コゴ-'], + ['ｻ', 'サザ-'], + ['ｼ', 'シジ-'], + ['ｽ', 'スズ-'], + ['ｾ', 'セゼ-'], + ['ｿ', 'ソゾ-'], + ['ﾀ', 'タダ-'], + ['ﾁ', 'チヂ-'], + ['ﾂ', 'ツヅ-'], + ['ﾃ', 'テデ-'], + ['ﾄ', 'トド-'], + ['ﾅ', 'ナ--'], + ['ﾆ', 'ニ--'], + ['ﾇ', 'ヌ--'], + ['ﾈ', 'ネ--'], + ['ﾉ', 'ノ--'], + ['ﾊ', 'ハバパ'], + ['ﾋ', 'ヒビピ'], + ['ﾌ', 'フブプ'], + ['ﾍ', 'ヘベペ'], + ['ﾎ', 'ホボポ'], + ['ﾏ', 'マ--'], + ['ﾐ', 'ミ--'], + ['ﾑ', 'ム--'], + ['ﾒ', 'メ--'], + ['ﾓ', 'モ--'], + ['ﾔ', 'ヤ--'], + ['ﾕ', 'ユ--'], + ['ﾖ', 'ヨ--'], + ['ﾗ', 'ラ--'], + ['ﾘ', 'リ--'], + ['ﾙ', 'ル--'], + ['ﾚ', 'レ--'], + ['ﾛ', 'ロ--'], + ['ﾜ', 'ワ--'], + ['ﾝ', 'ン--'] +]); + function jpIsKanji(c) { const code = c.charCodeAt(0); return code >= 0x4e00 && code < 0x9fb0 || code >= 0x3400 && code < 0x4dc0; @@ -175,3 +234,145 @@ function jpDistributeFuriganaInflected(expression, reading, source) { return output; } + +function jpConvertHalfWidthKanaToFullWidth(text, sourceMapping) { + let result = ''; + const ii = text.length; + const hasSourceMapping = Array.isArray(sourceMapping); + + for (let i = 0; i < ii; ++i) { + const c = text[i]; + const mapping = jpHalfWidthCharacterMapping.get(c); + if (typeof mapping !== 'string') { + result += c; + continue; + } + + let index = 0; + switch (text.charCodeAt(i + 1)) { + case 0xff9e: // dakuten + index = 1; + break; + case 0xff9f: // handakuten + index = 2; + break; + } + + let c2 = mapping[index]; + if (index > 0) { + if (c2 === '-') { // invalid + index = 0; + c2 = mapping[0]; + } else { + ++i; + } + } + + if (hasSourceMapping && index > 0) { + index = result.length; + const v = sourceMapping.splice(index + 1, 1)[0]; + sourceMapping[index] += v; + } + result += c2; + } + + return result; +} + +function jpConvertNumericTofullWidth(text) { + let result = ''; + for (let i = 0, ii = text.length; i < ii; ++i) { + let c = text.charCodeAt(i); + if (c >= 0x30 && c <= 0x39) { // ['0', '9'] + c += 0xff10 - 0x30; // 0xff10 = '0' full width + result += String.fromCharCode(c); + } else { + result += text[i]; + } + } + return result; +} + +function jpConvertAlphabeticToKana(text, sourceMapping) { + let part = ''; + let result = ''; + const ii = text.length; + + if (sourceMapping.length === ii) { + sourceMapping.length = ii; + sourceMapping.fill(1); + } + + for (let i = 0; i < ii; ++i) { + let c = text.charCodeAt(i); + if (c >= 0x41 && c <= 0x5a) { // ['A', 'Z'] + c -= 0x41; + } else if (c >= 0x61 && c <= 0x7a) { // ['a', 'z'] + c -= 0x61; + } else if (c >= 0xff21 && c <= 0xff3a) { // ['A', 'Z'] full width + c -= 0xff21; + } else if (c >= 0xff41 && c <= 0xff5a) { // ['a', 'z'] full width + c -= 0xff41; + } else { + if (part.length > 0) { + result += jpToHiragana(part, sourceMapping, result.length); + part = ''; + } + result += text[i]; + continue; + } + part += String.fromCharCode(c + 0x61); // + 'a' + } + + if (part.length > 0) { + result += jpToHiragana(part, sourceMapping, result.length); + } + return result; +} + +function jpToHiragana(text, sourceMapping, sourceMappingStart) { + const result = wanakana.toHiragana(text); + + // Generate source mapping + if (Array.isArray(sourceMapping)) { + if (typeof sourceMappingStart !== 'number') { sourceMappingStart = 0; } + let i = 0; + let resultPos = 0; + const ii = text.length; + while (i < ii) { + // Find smallest matching substring + let iNext = i + 1; + let resultPosNext = result.length; + while (iNext < ii) { + const t = wanakana.toHiragana(text.substring(0, iNext)); + if (t === result.substring(0, t.length)) { + resultPosNext = t.length; + break; + } + ++iNext; + } + + // Merge characters + const removals = iNext - i - 1; + if (removals > 0) { + let sum = 0; + const vs = sourceMapping.splice(sourceMappingStart + 1, removals); + for (const v of vs) { sum += v; } + sourceMapping[sourceMappingStart] += sum; + } + ++sourceMappingStart; + + // Empty elements + const additions = resultPosNext - resultPos - 1; + for (let j = 0; j < additions; ++j) { + sourceMapping.splice(sourceMappingStart, 0, 0); + ++sourceMappingStart; + } + + i = iNext; + resultPos = resultPosNext; + } + } + + return result; +}