Convert Japanese utilities to a module-like style

This commit is contained in:
toasted-nutbread 2020-03-11 20:33:01 -04:00
parent 7c5b64f9a4
commit 0f15cca2df
6 changed files with 454 additions and 441 deletions

View File

@ -17,7 +17,7 @@
*/ */
/* global /* global
* jpIsStringEntirelyKana * jp
*/ */
class AudioUriBuilder { class AudioUriBuilder {
@ -66,7 +66,7 @@ class AudioUriBuilder {
let kana = definition.reading; let kana = definition.reading;
let kanji = definition.expression; let kanji = definition.expression;
if (!kana && jpIsStringEntirelyKana(kanji)) { if (!kana && jp.isStringEntirelyKana(kanji)) {
kana = kanji; kana = kanji;
kanji = null; kanji = null;
} }

View File

@ -32,9 +32,7 @@
* dictEnabledSet * dictEnabledSet
* dictTermsSort * dictTermsSort
* handlebarsRenderDynamic * handlebarsRenderDynamic
* jpConvertReading * jp
* jpDistributeFuriganaInflected
* jpKatakanaToHiragana
* optionsLoad * optionsLoad
* optionsSave * optionsSave
* profileConditionsDescriptor * profileConditionsDescriptor
@ -402,13 +400,13 @@ class Backend {
dictTermsSort(definitions); dictTermsSort(definitions);
const {expression, reading} = definitions[0]; const {expression, reading} = definitions[0];
const source = text.substring(0, sourceLength); const source = text.substring(0, sourceLength);
for (const {text: text2, furigana} of jpDistributeFuriganaInflected(expression, reading, source)) { for (const {text: text2, furigana} of jp.distributeFuriganaInflected(expression, reading, source)) {
const reading2 = jpConvertReading(text2, furigana, options.parsing.readingMode); const reading2 = jp.convertReading(text2, furigana, options.parsing.readingMode);
term.push({text: text2, reading: reading2}); term.push({text: text2, reading: reading2});
} }
text = text.substring(source.length); text = text.substring(source.length);
} else { } else {
const reading = jpConvertReading(text[0], null, options.parsing.readingMode); const reading = jp.convertReading(text[0], null, options.parsing.readingMode);
term.push({text: text[0], reading}); term.push({text: text[0], reading});
text = text.substring(1); text = text.substring(1);
} }
@ -427,16 +425,16 @@ class Backend {
for (const {expression, reading, source} of parsedLine) { for (const {expression, reading, source} of parsedLine) {
const term = []; const term = [];
if (expression !== null && reading !== null) { if (expression !== null && reading !== null) {
for (const {text: text2, furigana} of jpDistributeFuriganaInflected( for (const {text: text2, furigana} of jp.distributeFuriganaInflected(
expression, expression,
jpKatakanaToHiragana(reading), jp.convertKatakanaToHiragana(reading),
source source
)) { )) {
const reading2 = jpConvertReading(text2, furigana, options.parsing.readingMode); const reading2 = jp.convertReading(text2, furigana, options.parsing.readingMode);
term.push({text: text2, reading: reading2}); term.push({text: text2, reading: reading2});
} }
} else { } else {
const reading2 = jpConvertReading(source, null, options.parsing.readingMode); const reading2 = jp.convertReading(source, null, options.parsing.readingMode);
term.push({text: source, reading: reading2}); term.push({text: source, reading: reading2});
} }
result.push(term); result.push(term);

View File

@ -17,7 +17,7 @@
*/ */
/* global /* global
* jpIsStringPartiallyJapanese * jp
*/ */
class ClipboardMonitor extends EventDispatcher { class ClipboardMonitor extends EventDispatcher {
@ -54,7 +54,7 @@ class ClipboardMonitor extends EventDispatcher {
text !== this._previousText text !== this._previousText
) { ) {
this._previousText = text; this._previousText = text;
if (jpIsStringPartiallyJapanese(text)) { if (jp.isStringPartiallyJapanese(text)) {
this.trigger('change', {text}); this.trigger('change', {text});
} }
} }

View File

@ -18,8 +18,7 @@
/* global /* global
* Handlebars * Handlebars
* jpDistributeFurigana * jp
* jpIsCodePointKanji
*/ */
function handlebarsEscape(text) { function handlebarsEscape(text) {
@ -33,7 +32,7 @@ function handlebarsDumpObject(options) {
function handlebarsFurigana(options) { function handlebarsFurigana(options) {
const definition = options.fn(this); const definition = options.fn(this);
const segs = jpDistributeFurigana(definition.expression, definition.reading); const segs = jp.distributeFurigana(definition.expression, definition.reading);
let result = ''; let result = '';
for (const seg of segs) { for (const seg of segs) {
@ -49,7 +48,7 @@ function handlebarsFurigana(options) {
function handlebarsFuriganaPlain(options) { function handlebarsFuriganaPlain(options) {
const definition = options.fn(this); const definition = options.fn(this);
const segs = jpDistributeFurigana(definition.expression, definition.reading); const segs = jp.distributeFurigana(definition.expression, definition.reading);
let result = ''; let result = '';
for (const seg of segs) { for (const seg of segs) {
@ -66,7 +65,7 @@ function handlebarsFuriganaPlain(options) {
function handlebarsKanjiLinks(options) { function handlebarsKanjiLinks(options) {
let result = ''; let result = '';
for (const c of options.fn(this)) { for (const c of options.fn(this)) {
if (jpIsCodePointKanji(c.codePointAt(0))) { if (jp.isCodePointKanji(c.codePointAt(0))) {
result += `<a href="#" class="kanji-link">${c}</a>`; result += `<a href="#" class="kanji-link">${c}</a>`;
} else { } else {
result += c; result += c;

View File

@ -20,439 +20,461 @@
* wanakana * wanakana
*/ */
const JP_HALFWIDTH_KATAKANA_MAPPING = new Map([ const jp = (() => {
['ヲ', 'ヲヺ-'], const HALFWIDTH_KATAKANA_MAPPING = new Map([
['ァ', 'ァ--'], ['ヲ', 'ヲヺ-'],
['ィ', 'ィ--'], ['ァ', 'ァ--'],
['ゥ', 'ゥ--'], ['ィ', 'ィ--'],
['ェ', 'ェ--'], ['ゥ', 'ゥ--'],
['ォ', 'ォ--'], ['ェ', 'ェ--'],
['ャ', 'ャ--'], ['ォ', 'ォ--'],
['ュ', 'ュ--'], ['ャ', 'ャ--'],
['ョ', 'ョ--'], ['ュ', 'ュ--'],
['ッ', 'ッ--'], ['ョ', 'ョ--'],
['ー', 'ー--'], ['ッ', 'ッ--'],
['ア', 'ア--'], ['ー', 'ー--'],
['イ', 'イ--'], ['ア', 'ア--'],
['ウ', 'ウヴ-'], ['イ', 'イ--'],
['エ', 'エ--'], ['ウ', 'ウヴ-'],
['オ', 'オ--'], ['エ', 'エ--'],
['カ', 'カガ-'], ['オ', 'オ--'],
['キ', 'キギ-'], ['カ', 'カガ-'],
['ク', 'クグ-'], ['キ', 'キギ-'],
['ケ', 'ケゲ-'], ['ク', 'クグ-'],
['コ', 'コゴ-'], ['ケ', 'ケゲ-'],
['サ', 'サザ-'], ['コ', 'コゴ-'],
['シ', 'シジ-'], ['サ', 'サザ-'],
['ス', 'スズ-'], ['シ', 'シジ-'],
['セ', 'セゼ-'], ['ス', 'スズ-'],
['ソ', 'ソゾ-'], ['セ', 'セゼ-'],
['タ', 'タダ-'], ['ソ', 'ソゾ-'],
['チ', 'チヂ-'], ['タ', 'タダ-'],
['ツ', 'ツヅ-'], ['チ', 'チヂ-'],
['テ', 'テデ-'], ['ツ', 'ツヅ-'],
['ト', 'トド-'], ['テ', 'テデ-'],
['ナ', 'ナ--'], ['ト', 'トド-'],
['ニ', 'ニ--'], ['ナ', 'ナ--'],
['ヌ', 'ヌ--'], ['ニ', 'ニ--'],
['ネ', 'ネ--'], ['ヌ', 'ヌ--'],
['ノ', '--'], ['ネ', 'ネ--'],
['ハ', 'ハバパ'], ['ノ', '--'],
['ヒ', 'ヒビピ'], ['ハ', 'ハバパ'],
['フ', 'フブプ'], ['ヒ', 'ヒビピ'],
['ヘ', 'ヘベペ'], ['フ', 'フブプ'],
['ホ', 'ホボポ'], ['ヘ', 'ヘベペ'],
['マ', 'マ--'], ['ホ', 'ホボポ'],
['ミ', 'ミ--'], ['マ', 'マ--'],
['ム', 'ム--'], ['ミ', 'ミ--'],
['メ', 'メ--'], ['ム', 'ム--'],
['モ', 'モ--'], ['メ', 'メ--'],
['ヤ', 'ヤ--'], ['モ', 'モ--'],
['ユ', 'ユ--'], ['ヤ', 'ヤ--'],
['ヨ', 'ヨ--'], ['ユ', 'ユ--'],
['ラ', 'ラ--'], ['ヨ', 'ヨ--'],
['リ', 'リ--'], ['ラ', 'ラ--'],
['ル', 'ル--'], ['リ', 'リ--'],
['レ', 'レ--'], ['ル', 'ル--'],
['ロ', 'ロ--'], ['レ', 'レ--'],
['ワ', 'ワ--'], ['ロ', 'ロ--'],
['ン', 'ン--'] ['ワ', 'ワ--'],
]); ['ン', 'ン--']
]);
const JP_HIRAGANA_RANGE = [0x3040, 0x309f]; const HIRAGANA_RANGE = [0x3040, 0x309f];
const JP_KATAKANA_RANGE = [0x30a0, 0x30ff]; const KATAKANA_RANGE = [0x30a0, 0x30ff];
const JP_KANA_RANGES = [JP_HIRAGANA_RANGE, JP_KATAKANA_RANGE]; const KANA_RANGES = [HIRAGANA_RANGE, KATAKANA_RANGE];
const JP_CJK_COMMON_RANGE = [0x4e00, 0x9fff]; const CJK_COMMON_RANGE = [0x4e00, 0x9fff];
const JP_CJK_RARE_RANGE = [0x3400, 0x4dbf]; const CJK_RARE_RANGE = [0x3400, 0x4dbf];
const JP_CJK_RANGES = [JP_CJK_COMMON_RANGE, JP_CJK_RARE_RANGE]; const CJK_RANGES = [CJK_COMMON_RANGE, CJK_RARE_RANGE];
const JP_ITERATION_MARK_CHAR_CODE = 0x3005; const ITERATION_MARK_CODE_POINT = 0x3005;
// Japanese character ranges, roughly ordered in order of expected frequency // Japanese character ranges, roughly ordered in order of expected frequency
const JP_JAPANESE_RANGES = [ const JAPANESE_RANGES = [
JP_HIRAGANA_RANGE, HIRAGANA_RANGE,
JP_KATAKANA_RANGE, KATAKANA_RANGE,
JP_CJK_COMMON_RANGE, CJK_COMMON_RANGE,
JP_CJK_RARE_RANGE, CJK_RARE_RANGE,
[0xff66, 0xff9f], // Halfwidth katakana [0xff66, 0xff9f], // Halfwidth katakana
[0x30fb, 0x30fc], // Katakana punctuation [0x30fb, 0x30fc], // Katakana punctuation
[0xff61, 0xff65], // Kana punctuation [0xff61, 0xff65], // Kana punctuation
[0x3000, 0x303f], // CJK punctuation [0x3000, 0x303f], // CJK punctuation
[0xff10, 0xff19], // Fullwidth numbers [0xff10, 0xff19], // Fullwidth numbers
[0xff21, 0xff3a], // Fullwidth upper case Latin letters [0xff21, 0xff3a], // Fullwidth upper case Latin letters
[0xff41, 0xff5a], // Fullwidth lower case Latin letters [0xff41, 0xff5a], // Fullwidth lower case Latin letters
[0xff01, 0xff0f], // Fullwidth punctuation 1 [0xff01, 0xff0f], // Fullwidth punctuation 1
[0xff1a, 0xff1f], // Fullwidth punctuation 2 [0xff1a, 0xff1f], // Fullwidth punctuation 2
[0xff3b, 0xff3f], // Fullwidth punctuation 3 [0xff3b, 0xff3f], // Fullwidth punctuation 3
[0xff5b, 0xff60], // Fullwidth punctuation 4 [0xff5b, 0xff60], // Fullwidth punctuation 4
[0xffe0, 0xffee] // Currency markers [0xffe0, 0xffee] // Currency markers
]; ];
// Helper functions // Character code testing functions
function _jpIsCodePointInRanges(codePoint, ranges) { function isCodePointKanji(codePoint) {
for (const [min, max] of ranges) { return isCodePointInRanges(codePoint, CJK_RANGES);
if (codePoint >= min && codePoint <= max) {
return true;
}
}
return false;
}
// Character code testing functions
function jpIsCodePointKanji(codePoint) {
return _jpIsCodePointInRanges(codePoint, JP_CJK_RANGES);
}
function jpIsCodePointKana(codePoint) {
return _jpIsCodePointInRanges(codePoint, JP_KANA_RANGES);
}
function jpIsCodePointJapanese(codePoint) {
return _jpIsCodePointInRanges(codePoint, JP_JAPANESE_RANGES);
}
// String testing functions
function jpIsStringEntirelyKana(str) {
if (str.length === 0) { return false; }
for (const c of str) {
if (!jpIsCodePointKana(c.codePointAt(0))) {
return false;
}
}
return true;
}
function jpIsStringPartiallyJapanese(str) {
if (str.length === 0) { return false; }
for (const c of str) {
if (jpIsCodePointJapanese(c.codePointAt(0))) {
return true;
}
}
return false;
}
// Conversion functions
function jpKatakanaToHiragana(text) {
let result = '';
for (const c of text) {
if (wanakana.isKatakana(c)) {
result += wanakana.toHiragana(c);
} else {
result += c;
}
} }
return result; function isCodePointKana(codePoint) {
} return isCodePointInRanges(codePoint, KANA_RANGES);
function jpHiraganaToKatakana(text) {
let result = '';
for (const c of text) {
if (wanakana.isHiragana(c)) {
result += wanakana.toKatakana(c);
} else {
result += c;
}
} }
return result; function isCodePointJapanese(codePoint) {
} return isCodePointInRanges(codePoint, JAPANESE_RANGES);
}
function jpToRomaji(text) { function isCodePointInRanges(codePoint, ranges) {
return wanakana.toRomaji(text); for (const [min, max] of ranges) {
} if (codePoint >= min && codePoint <= max) {
return true;
}
}
return false;
}
function jpConvertReading(expressionFragment, readingFragment, readingMode) {
switch (readingMode) { // String testing functions
case 'hiragana':
return jpKatakanaToHiragana(readingFragment || ''); function isStringEntirelyKana(str) {
case 'katakana': if (str.length === 0) { return false; }
return jpHiraganaToKatakana(readingFragment || ''); for (const c of str) {
case 'romaji': if (!isCodePointKana(c.codePointAt(0))) {
if (readingFragment) { return false;
return jpToRomaji(readingFragment); }
}
return true;
}
function isStringPartiallyJapanese(str) {
if (str.length === 0) { return false; }
for (const c of str) {
if (isCodePointJapanese(c.codePointAt(0))) {
return true;
}
}
return false;
}
// Conversion functions
function convertKatakanaToHiragana(text) {
let result = '';
for (const c of text) {
if (wanakana.isKatakana(c)) {
result += wanakana.toHiragana(c);
} else { } else {
if (jpIsStringEntirelyKana(expressionFragment)) { result += c;
return jpToRomaji(expressionFragment); }
}
return result;
}
function convertHiraganaToKatakana(text) {
let result = '';
for (const c of text) {
if (wanakana.isHiragana(c)) {
result += wanakana.toKatakana(c);
} else {
result += c;
}
}
return result;
}
function convertToRomaji(text) {
return wanakana.toRomaji(text);
}
function convertReading(expressionFragment, readingFragment, readingMode) {
switch (readingMode) {
case 'hiragana':
return convertKatakanaToHiragana(readingFragment || '');
case 'katakana':
return convertHiraganaToKatakana(readingFragment || '');
case 'romaji':
if (readingFragment) {
return convertToRomaji(readingFragment);
} else {
if (isStringEntirelyKana(expressionFragment)) {
return convertToRomaji(expressionFragment);
}
}
return readingFragment;
case 'none':
return null;
default:
return readingFragment;
}
}
function convertNumericTofullWidth(text) {
let result = '';
for (const char of text) {
let c = char.codePointAt(0);
if (c >= 0x30 && c <= 0x39) { // ['0', '9']
c += 0xff10 - 0x30; // 0xff10 = '0' full width
result += String.fromCodePoint(c);
} else {
result += char;
}
}
return result;
}
function convertHalfWidthKanaToFullWidth(text, sourceMapping) {
let result = '';
const hasSourceMapping = Array.isArray(sourceMapping);
// This function is safe to use charCodeAt instead of codePointAt, since all
// the relevant characters are represented with a single UTF-16 character code.
for (let i = 0, ii = text.length; i < ii; ++i) {
const c = text[i];
const mapping = HALFWIDTH_KATAKANA_MAPPING.get(c);
if (typeof mapping !== 'string') {
result += c;
continue;
}
let index = 0;
switch (text.charCodeAt(i + 1)) {
case 0xff9e: // dakuten
index = 1;
break;
case 0xff9f: // handakuten
index = 2;
break;
}
let c2 = mapping[index];
if (index > 0) {
if (c2 === '-') { // invalid
index = 0;
c2 = mapping[0];
} else {
++i;
} }
} }
return readingFragment;
case 'none':
return null;
default:
return readingFragment;
}
}
function jpDistributeFurigana(expression, reading) { if (hasSourceMapping && index > 0) {
const fallback = [{furigana: reading, text: expression}]; index = result.length;
if (!reading) { const v = sourceMapping.splice(index + 1, 1)[0];
sourceMapping[index] += v;
}
result += c2;
}
return result;
}
function convertAlphabeticToKana(text, sourceMapping) {
let part = '';
let result = '';
const ii = text.length;
if (sourceMapping.length === ii) {
sourceMapping.length = ii;
sourceMapping.fill(1);
}
for (const char of text) {
// Note: 0x61 is the character code for 'a'
let c = char.codePointAt(0);
if (c >= 0x41 && c <= 0x5a) { // ['A', 'Z']
c += (0x61 - 0x41);
} else if (c >= 0x61 && c <= 0x7a) { // ['a', 'z']
// NOP; c += (0x61 - 0x61);
} else if (c >= 0xff21 && c <= 0xff3a) { // ['A', 'Z'] fullwidth
c += (0x61 - 0xff21);
} else if (c >= 0xff41 && c <= 0xff5a) { // ['a', 'z'] fullwidth
c += (0x61 - 0xff41);
} else if (c === 0x2d || c === 0xff0d) { // '-' or fullwidth dash
c = 0x2d; // '-'
} else {
if (part.length > 0) {
result += convertAlphabeticPartToKana(part, sourceMapping, result.length);
part = '';
}
result += char;
continue;
}
part += String.fromCodePoint(c);
}
if (part.length > 0) {
result += convertAlphabeticPartToKana(part, sourceMapping, result.length);
}
return result;
}
function convertAlphabeticPartToKana(text, sourceMapping, sourceMappingStart) {
const result = wanakana.toHiragana(text);
// Generate source mapping
if (Array.isArray(sourceMapping)) {
if (typeof sourceMappingStart !== 'number') { sourceMappingStart = 0; }
let i = 0;
let resultPos = 0;
const ii = text.length;
while (i < ii) {
// Find smallest matching substring
let iNext = i + 1;
let resultPosNext = result.length;
while (iNext < ii) {
const t = wanakana.toHiragana(text.substring(0, iNext));
if (t === result.substring(0, t.length)) {
resultPosNext = t.length;
break;
}
++iNext;
}
// Merge characters
const removals = iNext - i - 1;
if (removals > 0) {
let sum = 0;
const vs = sourceMapping.splice(sourceMappingStart + 1, removals);
for (const v of vs) { sum += v; }
sourceMapping[sourceMappingStart] += sum;
}
++sourceMappingStart;
// Empty elements
const additions = resultPosNext - resultPos - 1;
for (let j = 0; j < additions; ++j) {
sourceMapping.splice(sourceMappingStart, 0, 0);
++sourceMappingStart;
}
i = iNext;
resultPos = resultPosNext;
}
}
return result;
}
// Furigana distribution
function distributeFurigana(expression, reading) {
const fallback = [{furigana: reading, text: expression}];
if (!reading) {
return fallback;
}
let isAmbiguous = false;
const segmentize = (reading2, groups) => {
if (groups.length === 0 || isAmbiguous) {
return [];
}
const group = groups[0];
if (group.mode === 'kana') {
if (convertKatakanaToHiragana(reading2).startsWith(convertKatakanaToHiragana(group.text))) {
const readingLeft = reading2.substring(group.text.length);
const segs = segmentize(readingLeft, groups.splice(1));
if (segs) {
return [{text: group.text}].concat(segs);
}
}
} else {
let foundSegments = null;
for (let i = reading2.length; i >= group.text.length; --i) {
const readingUsed = reading2.substring(0, i);
const readingLeft = reading2.substring(i);
const segs = segmentize(readingLeft, groups.slice(1));
if (segs) {
if (foundSegments !== null) {
// more than one way to segmentize the tail, mark as ambiguous
isAmbiguous = true;
return null;
}
foundSegments = [{text: group.text, furigana: readingUsed}].concat(segs);
}
// there is only one way to segmentize the last non-kana group
if (groups.length === 1) {
break;
}
}
return foundSegments;
}
};
const groups = [];
let modePrev = null;
for (const c of expression) {
const codePoint = c.codePointAt(0);
const modeCurr = isCodePointKanji(codePoint) || codePoint === ITERATION_MARK_CODE_POINT ? 'kanji' : 'kana';
if (modeCurr === modePrev) {
groups[groups.length - 1].text += c;
} else {
groups.push({mode: modeCurr, text: c});
modePrev = modeCurr;
}
}
const segments = segmentize(reading, groups);
if (segments && !isAmbiguous) {
return segments;
}
return fallback; return fallback;
} }
let isAmbiguous = false; function distributeFuriganaInflected(expression, reading, source) {
const segmentize = (reading2, groups) => { const output = [];
if (groups.length === 0 || isAmbiguous) {
return []; let stemLength = 0;
const shortest = Math.min(source.length, expression.length);
const sourceHiragana = convertKatakanaToHiragana(source);
const expressionHiragana = convertKatakanaToHiragana(expression);
while (stemLength < shortest && sourceHiragana[stemLength] === expressionHiragana[stemLength]) {
++stemLength;
}
const offset = source.length - stemLength;
const stemExpression = source.substring(0, source.length - offset);
const stemReading = reading.substring(
0,
offset === 0 ? reading.length : reading.length - expression.length + stemLength
);
for (const segment of distributeFurigana(stemExpression, stemReading)) {
output.push(segment);
} }
const group = groups[0]; if (stemLength !== source.length) {
if (group.mode === 'kana') { output.push({text: source.substring(stemLength)});
if (jpKatakanaToHiragana(reading2).startsWith(jpKatakanaToHiragana(group.text))) {
const readingLeft = reading2.substring(group.text.length);
const segs = segmentize(readingLeft, groups.splice(1));
if (segs) {
return [{text: group.text}].concat(segs);
}
}
} else {
let foundSegments = null;
for (let i = reading2.length; i >= group.text.length; --i) {
const readingUsed = reading2.substring(0, i);
const readingLeft = reading2.substring(i);
const segs = segmentize(readingLeft, groups.slice(1));
if (segs) {
if (foundSegments !== null) {
// more than one way to segmentize the tail, mark as ambiguous
isAmbiguous = true;
return null;
}
foundSegments = [{text: group.text, furigana: readingUsed}].concat(segs);
}
// there is only one way to segmentize the last non-kana group
if (groups.length === 1) {
break;
}
}
return foundSegments;
} }
return output;
}
// Exports
return {
isCodePointKanji,
isCodePointKana,
isCodePointJapanese,
isStringEntirelyKana,
isStringPartiallyJapanese,
convertKatakanaToHiragana,
convertHiraganaToKatakana,
convertToRomaji,
convertReading,
convertNumericTofullWidth,
convertHalfWidthKanaToFullWidth,
convertAlphabeticToKana,
distributeFurigana,
distributeFuriganaInflected
}; };
})();
const groups = [];
let modePrev = null;
for (const c of expression) {
const codePoint = c.codePointAt(0);
const modeCurr = jpIsCodePointKanji(codePoint) || codePoint === JP_ITERATION_MARK_CHAR_CODE ? 'kanji' : 'kana';
if (modeCurr === modePrev) {
groups[groups.length - 1].text += c;
} else {
groups.push({mode: modeCurr, text: c});
modePrev = modeCurr;
}
}
const segments = segmentize(reading, groups);
if (segments && !isAmbiguous) {
return segments;
}
return fallback;
}
function jpDistributeFuriganaInflected(expression, reading, source) {
const output = [];
let stemLength = 0;
const shortest = Math.min(source.length, expression.length);
const sourceHiragana = jpKatakanaToHiragana(source);
const expressionHiragana = jpKatakanaToHiragana(expression);
while (stemLength < shortest && sourceHiragana[stemLength] === expressionHiragana[stemLength]) {
++stemLength;
}
const offset = source.length - stemLength;
const stemExpression = source.substring(0, source.length - offset);
const stemReading = reading.substring(
0,
offset === 0 ? reading.length : reading.length - expression.length + stemLength
);
for (const segment of jpDistributeFurigana(stemExpression, stemReading)) {
output.push(segment);
}
if (stemLength !== source.length) {
output.push({text: source.substring(stemLength)});
}
return output;
}
function jpConvertHalfWidthKanaToFullWidth(text, sourceMapping) {
let result = '';
const hasSourceMapping = Array.isArray(sourceMapping);
// This function is safe to use charCodeAt instead of codePointAt, since all
// the relevant characters are represented with a single UTF-16 character code.
for (let i = 0, ii = text.length; i < ii; ++i) {
const c = text[i];
const mapping = JP_HALFWIDTH_KATAKANA_MAPPING.get(c);
if (typeof mapping !== 'string') {
result += c;
continue;
}
let index = 0;
switch (text.charCodeAt(i + 1)) {
case 0xff9e: // dakuten
index = 1;
break;
case 0xff9f: // handakuten
index = 2;
break;
}
let c2 = mapping[index];
if (index > 0) {
if (c2 === '-') { // invalid
index = 0;
c2 = mapping[0];
} else {
++i;
}
}
if (hasSourceMapping && index > 0) {
index = result.length;
const v = sourceMapping.splice(index + 1, 1)[0];
sourceMapping[index] += v;
}
result += c2;
}
return result;
}
function jpConvertNumericTofullWidth(text) {
let result = '';
for (const char of text) {
let c = char.codePointAt(0);
if (c >= 0x30 && c <= 0x39) { // ['0', '9']
c += 0xff10 - 0x30; // 0xff10 = '0' full width
result += String.fromCodePoint(c);
} else {
result += char;
}
}
return result;
}
function jpConvertAlphabeticToKana(text, sourceMapping) {
let part = '';
let result = '';
const ii = text.length;
if (sourceMapping.length === ii) {
sourceMapping.length = ii;
sourceMapping.fill(1);
}
for (const char of text) {
// Note: 0x61 is the character code for 'a'
let c = char.codePointAt(0);
if (c >= 0x41 && c <= 0x5a) { // ['A', 'Z']
c += (0x61 - 0x41);
} else if (c >= 0x61 && c <= 0x7a) { // ['a', 'z']
// NOP; c += (0x61 - 0x61);
} else if (c >= 0xff21 && c <= 0xff3a) { // ['A', 'Z'] fullwidth
c += (0x61 - 0xff21);
} else if (c >= 0xff41 && c <= 0xff5a) { // ['a', 'z'] fullwidth
c += (0x61 - 0xff41);
} else if (c === 0x2d || c === 0xff0d) { // '-' or fullwidth dash
c = 0x2d; // '-'
} else {
if (part.length > 0) {
result += jpToHiragana(part, sourceMapping, result.length);
part = '';
}
result += char;
continue;
}
part += String.fromCodePoint(c);
}
if (part.length > 0) {
result += jpToHiragana(part, sourceMapping, result.length);
}
return result;
}
function jpToHiragana(text, sourceMapping, sourceMappingStart) {
const result = wanakana.toHiragana(text);
// Generate source mapping
if (Array.isArray(sourceMapping)) {
if (typeof sourceMappingStart !== 'number') { sourceMappingStart = 0; }
let i = 0;
let resultPos = 0;
const ii = text.length;
while (i < ii) {
// Find smallest matching substring
let iNext = i + 1;
let resultPosNext = result.length;
while (iNext < ii) {
const t = wanakana.toHiragana(text.substring(0, iNext));
if (t === result.substring(0, t.length)) {
resultPosNext = t.length;
break;
}
++iNext;
}
// Merge characters
const removals = iNext - i - 1;
if (removals > 0) {
let sum = 0;
const vs = sourceMapping.splice(sourceMappingStart + 1, removals);
for (const v of vs) { sum += v; }
sourceMapping[sourceMappingStart] += sum;
}
++sourceMappingStart;
// Empty elements
const additions = resultPosNext - resultPos - 1;
for (let j = 0; j < additions; ++j) {
sourceMapping.splice(sourceMappingStart, 0, 0);
++sourceMappingStart;
}
i = iNext;
resultPos = resultPosNext;
}
}
return result;
}

View File

@ -29,13 +29,7 @@
* dictTermsMergeBySequence * dictTermsMergeBySequence
* dictTermsSort * dictTermsSort
* dictTermsUndupe * dictTermsUndupe
* jpConvertAlphabeticToKana * jp
* jpConvertHalfWidthKanaToFullWidth
* jpConvertNumericTofullWidth
* jpDistributeFurigana
* jpHiraganaToKatakana
* jpIsCodePointJapanese
* jpKatakanaToHiragana
* requestJson * requestJson
*/ */
@ -275,7 +269,7 @@ class Translator {
const termTags = await this.expandTags(definition.termTags, definition.dictionary); const termTags = await this.expandTags(definition.termTags, definition.dictionary);
const {expression, reading} = definition; const {expression, reading} = definition;
const furiganaSegments = jpDistributeFurigana(expression, reading); const furiganaSegments = jp.distributeFurigana(expression, reading);
definitions.push({ definitions.push({
source: deinflection.source, source: deinflection.source,
@ -376,20 +370,20 @@ class Translator {
let sourceMapping = null; let sourceMapping = null;
if (halfWidth) { if (halfWidth) {
if (sourceMapping === null) { sourceMapping = Translator.createTextSourceMapping(text2); } if (sourceMapping === null) { sourceMapping = Translator.createTextSourceMapping(text2); }
text2 = jpConvertHalfWidthKanaToFullWidth(text2, sourceMapping); text2 = jp.convertHalfWidthKanaToFullWidth(text2, sourceMapping);
} }
if (numeric) { if (numeric) {
text2 = jpConvertNumericTofullWidth(text2); text2 = jp.convertNumericTofullWidth(text2);
} }
if (alphabetic) { if (alphabetic) {
if (sourceMapping === null) { sourceMapping = Translator.createTextSourceMapping(text2); } if (sourceMapping === null) { sourceMapping = Translator.createTextSourceMapping(text2); }
text2 = jpConvertAlphabeticToKana(text2, sourceMapping); text2 = jp.convertAlphabeticToKana(text2, sourceMapping);
} }
if (katakana) { if (katakana) {
text2 = jpHiraganaToKatakana(text2); text2 = jp.convertHiraganaToKatakana(text2);
} }
if (hiragana) { if (hiragana) {
text2 = jpKatakanaToHiragana(text2); text2 = jp.convertKatakanaToHiragana(text2);
} }
for (let i = text2.length; i > 0; --i) { for (let i = text2.length; i > 0; --i) {
@ -590,7 +584,7 @@ class Translator {
} }
static createExpression(expression, reading, termTags=null, termFrequency=null) { static createExpression(expression, reading, termTags=null, termFrequency=null) {
const furiganaSegments = jpDistributeFurigana(expression, reading); const furiganaSegments = jp.distributeFurigana(expression, reading);
return { return {
expression, expression,
reading, reading,
@ -639,7 +633,7 @@ class Translator {
if (!options.scanning.alphanumeric) { if (!options.scanning.alphanumeric) {
let newText = ''; let newText = '';
for (const c of text) { for (const c of text) {
if (!jpIsCodePointJapanese(c.codePointAt(0))) { if (!jp.isCodePointJapanese(c.codePointAt(0))) {
break; break;
} }
newText += c; newText += c;