alex/yomichan
1
Fork 0
Code Releases Activity

Merge pull request #413 from toasted-nutbread/japanese-utility

Browse Source 
Convert Japanese utilities to a module-like style
This commit is contained in:
toasted-nutbread 2020-03-27 22:53:45 -04:00 committed by GitHub
commit a0c4ce779d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 922 additions and 458 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
ext/bg/background.html
View File

@ -20,6 +20,7 @@
<script src="/mixed/js/core.js"></script>
<script src="/mixed/js/dom.js"></script>
<script src="/mixed/js/japanese.js"></script>
<script src="/bg/js/anki.js"></script>
<script src="/bg/js/anki-note-builder.js"></script>

4
ext/bg/js/audio-uri-builder.js
View File

@ -17,7 +17,7 @@
*/
/* global
* jpIsStringEntirelyKana
* jp
*/
class AudioUriBuilder {
@ -66,7 +66,7 @@ class AudioUriBuilder {
let kana = definition.reading;
let kanji = definition.expression;
if (!kana && jpIsStringEntirelyKana(kanji)) {
if (!kana && jp.isStringEntirelyKana(kanji)) {
kana = kanji;
kanji = null;
}

18
ext/bg/js/backend.js
View File

@ -32,9 +32,7 @@
* dictEnabledSet
* dictTermsSort
* handlebarsRenderDynamic
* jpConvertReading
* jpDistributeFuriganaInflected
* jpKatakanaToHiragana
* jp
* optionsLoad
* optionsSave
* profileConditionsDescriptor
@ -402,13 +400,13 @@ class Backend {
dictTermsSort(definitions);
const {expression, reading} = definitions[0];
const source = text.substring(0, sourceLength);
for (const {text: text2, furigana} of jpDistributeFuriganaInflected(expression, reading, source)) {
const reading2 = jpConvertReading(text2, furigana, options.parsing.readingMode);
for (const {text: text2, furigana} of jp.distributeFuriganaInflected(expression, reading, source)) {
const reading2 = jp.convertReading(text2, furigana, options.parsing.readingMode);
term.push({text: text2, reading: reading2});
}
text = text.substring(source.length);
} else {
const reading = jpConvertReading(text[0], null, options.parsing.readingMode);
const reading = jp.convertReading(text[0], null, options.parsing.readingMode);
term.push({text: text[0], reading});
text = text.substring(1);
}
@ -427,16 +425,16 @@ class Backend {
for (const {expression, reading, source} of parsedLine) {
const term = [];
if (expression !== null && reading !== null) {
for (const {text: text2, furigana} of jpDistributeFuriganaInflected(
for (const {text: text2, furigana} of jp.distributeFuriganaInflected(
expression,
jpKatakanaToHiragana(reading),
jp.convertKatakanaToHiragana(reading),
source
)) {
const reading2 = jpConvertReading(text2, furigana, options.parsing.readingMode);
const reading2 = jp.convertReading(text2, furigana, options.parsing.readingMode);
term.push({text: text2, reading: reading2});
}
} else {
const reading2 = jpConvertReading(source, null, options.parsing.readingMode);
const reading2 = jp.convertReading(source, null, options.parsing.readingMode);
term.push({text: source, reading: reading2});
}
result.push(term);

4
ext/bg/js/clipboard-monitor.js
View File

@ -17,7 +17,7 @@
*/
/* global
* jpIsStringPartiallyJapanese
* jp
*/
class ClipboardMonitor extends EventDispatcher {
@ -54,7 +54,7 @@ class ClipboardMonitor extends EventDispatcher {
text !== this._previousText
) {
this._previousText = text;
if (jpIsStringPartiallyJapanese(text)) {
if (jp.isStringPartiallyJapanese(text)) {
this.trigger('change', {text});
}
}

9
ext/bg/js/handlebars.js
View File

@ -18,8 +18,7 @@
/* global
* Handlebars
* jpDistributeFurigana
* jpIsCodePointKanji
* jp
*/
function handlebarsEscape(text) {
@ -33,7 +32,7 @@ function handlebarsDumpObject(options) {
function handlebarsFurigana(options) {
const definition = options.fn(this);
const segs = jpDistributeFurigana(definition.expression, definition.reading);
const segs = jp.distributeFurigana(definition.expression, definition.reading);
let result = '';
for (const seg of segs) {
@ -49,7 +48,7 @@ function handlebarsFurigana(options) {
function handlebarsFuriganaPlain(options) {
const definition = options.fn(this);
const segs = jpDistributeFurigana(definition.expression, definition.reading);
const segs = jp.distributeFurigana(definition.expression, definition.reading);
let result = '';
for (const seg of segs) {
@ -66,7 +65,7 @@ function handlebarsFuriganaPlain(options) {
function handlebarsKanjiLinks(options) {
let result = '';
for (const c of options.fn(this)) {
if (jpIsCodePointKanji(c.codePointAt(0))) {
if (jp.isCodePointKanji(c.codePointAt(0))) {
result += `<a href="#" class="kanji-link">${c}</a>`;
} else {
result += c;

346
ext/bg/js/japanese.js
View File

@ -17,10 +17,12 @@
*/
/* global
* jp
* wanakana
*/
const JP_HALFWIDTH_KATAKANA_MAPPING = new Map([
(() => {
const HALFWIDTH_KATAKANA_MAPPING = new Map([
['ｦ', 'ヲヺ-'],
['ｧ', 'ァ--'],
['ｨ', 'ィ--'],
@ -79,95 +81,18 @@ const JP_HALFWIDTH_KATAKANA_MAPPING = new Map([
['ﾝ', 'ン--']
]);
const JP_HIRAGANA_RANGE = [0x3040, 0x309f];
const JP_KATAKANA_RANGE = [0x30a0, 0x30ff];
const JP_KANA_RANGES = [JP_HIRAGANA_RANGE, JP_KATAKANA_RANGE];
const JP_CJK_COMMON_RANGE = [0x4e00, 0x9fff];
const JP_CJK_RARE_RANGE = [0x3400, 0x4dbf];
const JP_CJK_RANGES = [JP_CJK_COMMON_RANGE, JP_CJK_RARE_RANGE];
const JP_ITERATION_MARK_CHAR_CODE = 0x3005;
// Japanese character ranges, roughly ordered in order of expected frequency
const JP_JAPANESE_RANGES = [
JP_HIRAGANA_RANGE,
JP_KATAKANA_RANGE,
JP_CJK_COMMON_RANGE,
JP_CJK_RARE_RANGE,
[0xff66, 0xff9f], // Halfwidth katakana
[0x30fb, 0x30fc], // Katakana punctuation
[0xff61, 0xff65], // Kana punctuation
[0x3000, 0x303f], // CJK punctuation
[0xff10, 0xff19], // Fullwidth numbers
[0xff21, 0xff3a], // Fullwidth upper case Latin letters
[0xff41, 0xff5a], // Fullwidth lower case Latin letters
[0xff01, 0xff0f], // Fullwidth punctuation 1
[0xff1a, 0xff1f], // Fullwidth punctuation 2
[0xff3b, 0xff3f], // Fullwidth punctuation 3
[0xff5b, 0xff60], // Fullwidth punctuation 4
[0xffe0, 0xffee] // Currency markers
];
const ITERATION_MARK_CODE_POINT = 0x3005;
// Helper functions
// Existing functions
function _jpIsCodePointInRanges(codePoint, ranges) {
for (const [min, max] of ranges) {
if (codePoint >= min && codePoint <= max) {
return true;
}
}
return false;
}
// Character code testing functions
function jpIsCodePointKanji(codePoint) {
return _jpIsCodePointInRanges(codePoint, JP_CJK_RANGES);
}
function jpIsCodePointKana(codePoint) {
return _jpIsCodePointInRanges(codePoint, JP_KANA_RANGES);
}
function jpIsCodePointJapanese(codePoint) {
return _jpIsCodePointInRanges(codePoint, JP_JAPANESE_RANGES);
}
// String testing functions
function jpIsStringEntirelyKana(str) {
if (str.length === 0) { return false; }
for (const c of str) {
if (!jpIsCodePointKana(c.codePointAt(0))) {
return false;
}
}
return true;
}
function jpIsStringPartiallyJapanese(str) {
if (str.length === 0) { return false; }
for (const c of str) {
if (jpIsCodePointJapanese(c.codePointAt(0))) {
return true;
}
}
return false;
}
const isCodePointKanji = jp.isCodePointKanji;
const isStringEntirelyKana = jp.isStringEntirelyKana;
// Conversion functions
function jpKatakanaToHiragana(text) {
function convertKatakanaToHiragana(text) {
let result = '';
for (const c of text) {
if (wanakana.isKatakana(c)) {
@ -180,7 +105,7 @@ function jpKatakanaToHiragana(text) {
return result;
}
function jpHiraganaToKatakana(text) {
function convertHiraganaToKatakana(text) {
let result = '';
for (const c of text) {
if (wanakana.isHiragana(c)) {
@ -193,22 +118,22 @@ function jpHiraganaToKatakana(text) {
return result;
}
function jpToRomaji(text) {
function convertToRomaji(text) {
return wanakana.toRomaji(text);
}
function jpConvertReading(expressionFragment, readingFragment, readingMode) {
function convertReading(expressionFragment, readingFragment, readingMode) {
switch (readingMode) {
case 'hiragana':
return jpKatakanaToHiragana(readingFragment || '');
return convertKatakanaToHiragana(readingFragment || '');
case 'katakana':
return jpHiraganaToKatakana(readingFragment || '');
return convertHiraganaToKatakana(readingFragment || '');
case 'romaji':
if (readingFragment) {
return jpToRomaji(readingFragment);
return convertToRomaji(readingFragment);
} else {
if (jpIsStringEntirelyKana(expressionFragment)) {
return jpToRomaji(expressionFragment);
if (isStringEntirelyKana(expressionFragment)) {
return convertToRomaji(expressionFragment);
}
}
return readingFragment;
@ -219,99 +144,21 @@ function jpConvertReading(expressionFragment, readingFragment, readingMode) {
}
}
function jpDistributeFurigana(expression, reading) {
const fallback = [{furigana: reading, text: expression}];
if (!reading) {
return fallback;
}
let isAmbiguous = false;
const segmentize = (reading2, groups) => {
if (groups.length === 0 || isAmbiguous) {
return [];
}
const group = groups[0];
if (group.mode === 'kana') {
if (jpKatakanaToHiragana(reading2).startsWith(jpKatakanaToHiragana(group.text))) {
const readingLeft = reading2.substring(group.text.length);
const segs = segmentize(readingLeft, groups.splice(1));
if (segs) {
return [{text: group.text}].concat(segs);
}
}
function convertNumericToFullWidth(text) {
let result = '';
for (const char of text) {
let c = char.codePointAt(0);
if (c >= 0x30 && c <= 0x39) { // ['0', '9']
c += 0xff10 - 0x30; // 0xff10 = '0' full width
result += String.fromCodePoint(c);
} else {
let foundSegments = null;
for (let i = reading2.length; i >= group.text.length; --i) {
const readingUsed = reading2.substring(0, i);
const readingLeft = reading2.substring(i);
const segs = segmentize(readingLeft, groups.slice(1));
if (segs) {
if (foundSegments !== null) {
// more than one way to segmentize the tail, mark as ambiguous
isAmbiguous = true;
return null;
}
foundSegments = [{text: group.text, furigana: readingUsed}].concat(segs);
}
// there is only one way to segmentize the last non-kana group
if (groups.length === 1) {
break;
result += char;
}
}
return foundSegments;
}
};
const groups = [];
let modePrev = null;
for (const c of expression) {
const codePoint = c.codePointAt(0);
const modeCurr = jpIsCodePointKanji(codePoint) || codePoint === JP_ITERATION_MARK_CHAR_CODE ? 'kanji' : 'kana';
if (modeCurr === modePrev) {
groups[groups.length - 1].text += c;
} else {
groups.push({mode: modeCurr, text: c});
modePrev = modeCurr;
}
return result;
}
const segments = segmentize(reading, groups);
if (segments && !isAmbiguous) {
return segments;
}
return fallback;
}
function jpDistributeFuriganaInflected(expression, reading, source) {
const output = [];
let stemLength = 0;
const shortest = Math.min(source.length, expression.length);
const sourceHiragana = jpKatakanaToHiragana(source);
const expressionHiragana = jpKatakanaToHiragana(expression);
while (stemLength < shortest && sourceHiragana[stemLength] === expressionHiragana[stemLength]) {
++stemLength;
}
const offset = source.length - stemLength;
const stemExpression = source.substring(0, source.length - offset);
const stemReading = reading.substring(
0,
offset === 0 ? reading.length : reading.length - expression.length + stemLength
);
for (const segment of jpDistributeFurigana(stemExpression, stemReading)) {
output.push(segment);
}
if (stemLength !== source.length) {
output.push({text: source.substring(stemLength)});
}
return output;
}
function jpConvertHalfWidthKanaToFullWidth(text, sourceMapping) {
function convertHalfWidthKanaToFullWidth(text, sourceMapping) {
let result = '';
const hasSourceMapping = Array.isArray(sourceMapping);
@ -319,7 +166,7 @@ function jpConvertHalfWidthKanaToFullWidth(text, sourceMapping) {
// the relevant characters are represented with a single UTF-16 character code.
for (let i = 0, ii = text.length; i < ii; ++i) {
const c = text[i];
const mapping = JP_HALFWIDTH_KATAKANA_MAPPING.get(c);
const mapping = HALFWIDTH_KATAKANA_MAPPING.get(c);
if (typeof mapping !== 'string') {
result += c;
continue;
@ -356,29 +203,9 @@ function jpConvertHalfWidthKanaToFullWidth(text, sourceMapping) {
return result;
}
function jpConvertNumericTofullWidth(text) {
let result = '';
for (const char of text) {
let c = char.codePointAt(0);
if (c >= 0x30 && c <= 0x39) { // ['0', '9']
c += 0xff10 - 0x30; // 0xff10 = '0' full width
result += String.fromCodePoint(c);
} else {
result += char;
}
}
return result;
}
function jpConvertAlphabeticToKana(text, sourceMapping) {
function convertAlphabeticToKana(text, sourceMapping) {
let part = '';
let result = '';
const ii = text.length;
if (sourceMapping.length === ii) {
sourceMapping.length = ii;
sourceMapping.fill(1);
}
for (const char of text) {
// Note: 0x61 is the character code for 'a'
@ -395,7 +222,7 @@ function jpConvertAlphabeticToKana(text, sourceMapping) {
c = 0x2d; // '-'
} else {
if (part.length > 0) {
result += jpToHiragana(part, sourceMapping, result.length);
result += convertAlphabeticPartToKana(part, sourceMapping, result.length);
part = '';
}
result += char;
@ -405,12 +232,12 @@ function jpConvertAlphabeticToKana(text, sourceMapping) {
}
if (part.length > 0) {
result += jpToHiragana(part, sourceMapping, result.length);
result += convertAlphabeticPartToKana(part, sourceMapping, result.length);
}
return result;
}
function jpToHiragana(text, sourceMapping, sourceMappingStart) {
function convertAlphabeticPartToKana(text, sourceMapping, sourceMappingStart) {
const result = wanakana.toHiragana(text);
// Generate source mapping
@ -456,3 +283,114 @@ function jpToHiragana(text, sourceMapping, sourceMappingStart) {
return result;
}
// Furigana distribution
function distributeFurigana(expression, reading) {
const fallback = [{furigana: reading, text: expression}];
if (!reading) {
return fallback;
}
let isAmbiguous = false;
const segmentize = (reading2, groups) => {
if (groups.length === 0 || isAmbiguous) {
return [];
}
const group = groups[0];
if (group.mode === 'kana') {
if (convertKatakanaToHiragana(reading2).startsWith(convertKatakanaToHiragana(group.text))) {
const readingLeft = reading2.substring(group.text.length);
const segs = segmentize(readingLeft, groups.splice(1));
if (segs) {
return [{text: group.text}].concat(segs);
}
}
} else {
let foundSegments = null;
for (let i = reading2.length; i >= group.text.length; --i) {
const readingUsed = reading2.substring(0, i);
const readingLeft = reading2.substring(i);
const segs = segmentize(readingLeft, groups.slice(1));
if (segs) {
if (foundSegments !== null) {
// more than one way to segmentize the tail, mark as ambiguous
isAmbiguous = true;
return null;
}
foundSegments = [{text: group.text, furigana: readingUsed}].concat(segs);
}
// there is only one way to segmentize the last non-kana group
if (groups.length === 1) {
break;
}
}
return foundSegments;
}
};
const groups = [];
let modePrev = null;
for (const c of expression) {
const codePoint = c.codePointAt(0);
const modeCurr = isCodePointKanji(codePoint) || codePoint === ITERATION_MARK_CODE_POINT ? 'kanji' : 'kana';
if (modeCurr === modePrev) {
groups[groups.length - 1].text += c;
} else {
groups.push({mode: modeCurr, text: c});
modePrev = modeCurr;
}
}
const segments = segmentize(reading, groups);
if (segments && !isAmbiguous) {
return segments;
}
return fallback;
}
function distributeFuriganaInflected(expression, reading, source) {
const output = [];
let stemLength = 0;
const shortest = Math.min(source.length, expression.length);
const sourceHiragana = convertKatakanaToHiragana(source);
const expressionHiragana = convertKatakanaToHiragana(expression);
while (stemLength < shortest && sourceHiragana[stemLength] === expressionHiragana[stemLength]) {
++stemLength;
}
const offset = source.length - stemLength;
const stemExpression = source.substring(0, source.length - offset);
const stemReading = reading.substring(
0,
offset === 0 ? reading.length : reading.length - expression.length + stemLength
);
for (const segment of distributeFurigana(stemExpression, stemReading)) {
output.push(segment);
}
if (stemLength !== source.length) {
output.push({text: source.substring(stemLength)});
}
return output;
}
// Exports
Object.assign(jp, {
convertKatakanaToHiragana,
convertHiraganaToKatakana,
convertToRomaji,
convertReading,
convertNumericToFullWidth,
convertHalfWidthKanaToFullWidth,
convertAlphabeticToKana,
distributeFurigana,
distributeFuriganaInflected
});
})();

24
ext/bg/js/translator.js
View File

@ -29,13 +29,7 @@
* dictTermsMergeBySequence
* dictTermsSort
* dictTermsUndupe
* jpConvertAlphabeticToKana
* jpConvertHalfWidthKanaToFullWidth
* jpConvertNumericTofullWidth
* jpDistributeFurigana
* jpHiraganaToKatakana
* jpIsCodePointJapanese
* jpKatakanaToHiragana
* jp
* requestJson
*/
@ -275,7 +269,7 @@ class Translator {
const termTags = await this.expandTags(definition.termTags, definition.dictionary);
const {expression, reading} = definition;
const furiganaSegments = jpDistributeFurigana(expression, reading);
const furiganaSegments = jp.distributeFurigana(expression, reading);
definitions.push({
source: deinflection.source,
@ -376,20 +370,20 @@ class Translator {
let sourceMapping = null;
if (halfWidth) {
if (sourceMapping === null) { sourceMapping = Translator.createTextSourceMapping(text2); }
text2 = jpConvertHalfWidthKanaToFullWidth(text2, sourceMapping);
text2 = jp.convertHalfWidthKanaToFullWidth(text2, sourceMapping);
}
if (numeric) {
text2 = jpConvertNumericTofullWidth(text2);
text2 = jp.convertNumericToFullWidth(text2);
}
if (alphabetic) {
if (sourceMapping === null) { sourceMapping = Translator.createTextSourceMapping(text2); }
text2 = jpConvertAlphabeticToKana(text2, sourceMapping);
text2 = jp.convertAlphabeticToKana(text2, sourceMapping);
}
if (katakana) {
text2 = jpHiraganaToKatakana(text2);
text2 = jp.convertHiraganaToKatakana(text2);
}
if (hiragana) {
text2 = jpKatakanaToHiragana(text2);
text2 = jp.convertKatakanaToHiragana(text2);
}
for (let i = text2.length; i > 0; --i) {
@ -590,7 +584,7 @@ class Translator {
}
static createExpression(expression, reading, termTags=null, termFrequency=null) {
const furiganaSegments = jpDistributeFurigana(expression, reading);
const furiganaSegments = jp.distributeFurigana(expression, reading);
return {
expression,
reading,
@ -639,7 +633,7 @@ class Translator {
if (!options.scanning.alphanumeric) {
let newText = '';
for (const c of text) {
if (!jpIsCodePointJapanese(c.codePointAt(0))) {
if (!jp.isCodePointJapanese(c.codePointAt(0))) {
break;
}
newText += c;

1
ext/bg/search.html
View File

@ -74,6 +74,7 @@
<script src="/mixed/js/core.js"></script>
<script src="/mixed/js/dom.js"></script>
<script src="/mixed/js/api.js"></script>
<script src="/mixed/js/japanese.js"></script>
<script src="/bg/js/dictionary.js"></script>
<script src="/bg/js/handlebars.js"></script>

1
ext/bg/settings.html
View File

@ -1088,6 +1088,7 @@
<script src="/mixed/js/core.js"></script>
<script src="/mixed/js/dom.js"></script>
<script src="/mixed/js/api.js"></script>
<script src="/mixed/js/japanese.js"></script>
<script src="/bg/js/anki.js"></script>
<script src="/bg/js/anki-note-builder.js"></script>

1
ext/fg/float.html
View File

@ -43,6 +43,7 @@
<script src="/mixed/js/core.js"></script>
<script src="/mixed/js/dom.js"></script>
<script src="/mixed/js/api.js"></script>
<script src="/mixed/js/japanese.js"></script>
<script src="/fg/js/document.js"></script>
<script src="/fg/js/source.js"></script>

11
ext/mixed/js/display-generator.js
View File

@ -19,6 +19,7 @@
/* global
* TemplateHandler
* apiGetDisplayTemplatesHtml
* jp
*/
class DisplayGenerator {
@ -283,7 +284,7 @@ class DisplayGenerator {
_appendKanjiLinks(container, text) {
let part = '';
for (const c of text) {
if (DisplayGenerator._isCharacterKanji(c)) {
if (jp.isCodePointKanji(c.codePointAt(0))) {
if (part.length > 0) {
container.appendChild(document.createTextNode(part));
part = '';
@ -300,14 +301,6 @@ class DisplayGenerator {
}
}
static _isCharacterKanji(c) {
const code = c.codePointAt(0);
return (
code >= 0x4e00 && code < 0x9fb0 ||
code >= 0x3400 && code < 0x4dc0
);
}
static _appendMultiple(container, createItem, detailsArray, fallback=[]) {
if (container === null) { return 0; }

124
ext/mixed/js/japanese.js Normal file
View File

@ -0,0 +1,124 @@
/*
* Copyright (C) 2020 Alex Yatskov <alex@foosoft.net>
* Author: Alex Yatskov <alex@foosoft.net>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const jp = (() => {
const HIRAGANA_RANGE = [0x3040, 0x309f];
const KATAKANA_RANGE = [0x30a0, 0x30ff];
const KANA_RANGES = [HIRAGANA_RANGE, KATAKANA_RANGE];
const CJK_UNIFIED_IDEOGRAPHS_RANGE = [0x4e00, 0x9fff];
const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A_RANGE = [0x3400, 0x4dbf];
const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B_RANGE = [0x20000, 0x2a6df];
const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C_RANGE = [0x2a700, 0x2b73f];
const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D_RANGE = [0x2b740, 0x2b81f];
const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_E_RANGE = [0x2b820, 0x2ceaf];
const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_F_RANGE = [0x2ceb0, 0x2ebef];
const CJK_COMPATIBILITY_IDEOGRAPHS_SUPPLEMENT_RANGE = [0x2f800, 0x2fa1f];
const CJK_UNIFIED_IDEOGRAPHS_RANGES = [
CJK_UNIFIED_IDEOGRAPHS_RANGE,
CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A_RANGE,
CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B_RANGE,
CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C_RANGE,
CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D_RANGE,
CJK_UNIFIED_IDEOGRAPHS_EXTENSION_E_RANGE,
CJK_UNIFIED_IDEOGRAPHS_EXTENSION_F_RANGE,
CJK_COMPATIBILITY_IDEOGRAPHS_SUPPLEMENT_RANGE
];
// Japanese character ranges, roughly ordered in order of expected frequency
const JAPANESE_RANGES = [
HIRAGANA_RANGE,
KATAKANA_RANGE,
...CJK_UNIFIED_IDEOGRAPHS_RANGES,
[0xff66, 0xff9f], // Halfwidth katakana
[0x30fb, 0x30fc], // Katakana punctuation
[0xff61, 0xff65], // Kana punctuation
[0x3000, 0x303f], // CJK punctuation
[0xff10, 0xff19], // Fullwidth numbers
[0xff21, 0xff3a], // Fullwidth upper case Latin letters
[0xff41, 0xff5a], // Fullwidth lower case Latin letters
[0xff01, 0xff0f], // Fullwidth punctuation 1
[0xff1a, 0xff1f], // Fullwidth punctuation 2
[0xff3b, 0xff3f], // Fullwidth punctuation 3
[0xff5b, 0xff60], // Fullwidth punctuation 4
[0xffe0, 0xffee] // Currency markers
];
// Character code testing functions
function isCodePointKanji(codePoint) {
return isCodePointInRanges(codePoint, CJK_UNIFIED_IDEOGRAPHS_RANGES);
}
function isCodePointKana(codePoint) {
return isCodePointInRanges(codePoint, KANA_RANGES);
}
function isCodePointJapanese(codePoint) {
return isCodePointInRanges(codePoint, JAPANESE_RANGES);
}
function isCodePointInRanges(codePoint, ranges) {
for (const [min, max] of ranges) {
if (codePoint >= min && codePoint <= max) {
return true;
}
}
return false;
}
// String testing functions
function isStringEntirelyKana(str) {
if (str.length === 0) { return false; }
for (const c of str) {
if (!isCodePointKana(c.codePointAt(0))) {
return false;
}
}
return true;
}
function isStringPartiallyJapanese(str) {
if (str.length === 0) { return false; }
for (const c of str) {
if (isCodePointJapanese(c.codePointAt(0))) {
return true;
}
}
return false;
}
// Exports
return {
isCodePointKanji,
isCodePointKana,
isCodePointJapanese,
isStringEntirelyKana,
isStringPartiallyJapanese
};
})();

2
package.json
View File

@ -8,7 +8,7 @@
"scripts": {
"test": "npm run test-lint && npm run test-code",
"test-lint": "eslint . && node ./test/lint/global-declarations.js",
"test-code": "node ./test/test-schema.js && node ./test/test-dictionary.js && node ./test/test-database.js && node ./test/test-document.js && node ./test/test-object-property-accessor.js"
"test-code": "node ./test/test-schema.js && node ./test/test-dictionary.js && node ./test/test-database.js && node ./test/test-document.js && node ./test/test-object-property-accessor.js && node ./test/test-japanese.js"
},
"repository": {
"type": "git",

414
test/test-japanese.js Normal file
View File

@ -0,0 +1,414 @@
/*
* Copyright (C) 2020 Alex Yatskov <alex@foosoft.net>
* Author: Alex Yatskov <alex@foosoft.net>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const assert = require('assert');
const {VM} = require('./yomichan-vm');
const vm = new VM();
vm.execute([
'mixed/lib/wanakana.min.js',
'mixed/js/japanese.js',
'bg/js/japanese.js'
]);
const jp = vm.get('jp');
function testIsCodePointKanji() {
const data = [
['力方', true],
['\u53f1\u{20b9f}', true],
['かたカタ々kata、。？,.?', false]
];
for (const [characters, expected] of data) {
for (const character of characters) {
const codePoint = character.codePointAt(0);
const actual = jp.isCodePointKanji(codePoint);
assert.strictEqual(actual, expected, `isCodePointKanji failed for ${character} (\\u{${codePoint.toString(16)}})`);
}
}
}
function testIsCodePointKana() {
const data = [
['かたカタ', true],
['力方々kata、。？,.?', false],
['\u53f1\u{20b9f}', false]
];
for (const [characters, expected] of data) {
for (const character of characters) {
const codePoint = character.codePointAt(0);
const actual = jp.isCodePointKana(codePoint);
assert.strictEqual(actual, expected, `isCodePointKana failed for ${character} (\\u{${codePoint.toString(16)}})`);
}
}
}
function testIsCodePointJapanese() {
const data = [
['かたカタ力方々、。？', true],
['\u53f1\u{20b9f}', true],
['kata,.?', false]
];
for (const [characters, expected] of data) {
for (const character of characters) {
const codePoint = character.codePointAt(0);
const actual = jp.isCodePointJapanese(codePoint);
assert.strictEqual(actual, expected, `isCodePointJapanese failed for ${character} (\\u{${codePoint.toString(16)}})`);
}
}
}
function testIsStringEntirelyKana() {
const data = [
['かたかな', true],
['カタカナ', true],
['ひらがな', true],
['ヒラガナ', true],
['カタカナひらがな', true],
['かたカタ力方々、。？', false],
['\u53f1\u{20b9f}', false],
['kata,.?', false],
['かたカタ力方々、。？invalid', false],
['\u53f1\u{20b9f}invalid', false],
['kata,.?かた', false]
];
for (const [string, expected] of data) {
assert.strictEqual(jp.isStringEntirelyKana(string), expected);
}
}
function testIsStringPartiallyJapanese() {
const data = [
['かたかな', true],
['カタカナ', true],
['ひらがな', true],
['ヒラガナ', true],
['カタカナひらがな', true],
['かたカタ力方々、。？', true],
['\u53f1\u{20b9f}', true],
['kata,.?', false],
['かたカタ力方々、。？invalid', true],
['\u53f1\u{20b9f}invalid', true],
['kata,.?かた', true]
];
for (const [string, expected] of data) {
assert.strictEqual(jp.isStringPartiallyJapanese(string), expected);
}
}
function testConvertKatakanaToHiragana() {
const data = [
['かたかな', 'かたかな'],
['ひらがな', 'ひらがな'],
['カタカナ', 'かたかな'],
['ヒラガナ', 'ひらがな'],
['カタカナかたかな', 'かたかなかたかな'],
['ヒラガナひらがな', 'ひらがなひらがな'],
['chikaraちからチカラ力', 'chikaraちからちから力'],
['katakana', 'katakana'],
['hiragana', 'hiragana']
];
for (const [string, expected] of data) {
assert.strictEqual(jp.convertKatakanaToHiragana(string), expected);
}
}
function testConvertHiraganaToKatakana() {
const data = [
['かたかな', 'カタカナ'],
['ひらがな', 'ヒラガナ'],
['カタカナ', 'カタカナ'],
['ヒラガナ', 'ヒラガナ'],
['カタカナかたかな', 'カタカナカタカナ'],
['ヒラガナひらがな', 'ヒラガナヒラガナ'],
['chikaraちからチカラ力', 'chikaraチカラチカラ力'],
['katakana', 'katakana'],
['hiragana', 'hiragana']
];
for (const [string, expected] of data) {
assert.strictEqual(jp.convertHiraganaToKatakana(string), expected);
}
}
function testConvertToRomaji() {
const data = [
['かたかな', 'katakana'],
['ひらがな', 'hiragana'],
['カタカナ', 'katakana'],
['ヒラガナ', 'hiragana'],
['カタカナかたかな', 'katakanakatakana'],
['ヒラガナひらがな', 'hiraganahiragana'],
['chikaraちからチカラ力', 'chikarachikarachikara力'],
['katakana', 'katakana'],
['hiragana', 'hiragana']
];
for (const [string, expected] of data) {
assert.strictEqual(jp.convertToRomaji(string), expected);
}
}
function testConvertReading() {
const data = [
[['アリガトウ', 'アリガトウ', 'hiragana'], 'ありがとう'],
[['アリガトウ', 'アリガトウ', 'katakana'], 'アリガトウ'],
[['アリガトウ', 'アリガトウ', 'romaji'], 'arigatou'],
[['アリガトウ', 'アリガトウ', 'none'], null],
[['アリガトウ', 'アリガトウ', 'default'], 'アリガトウ'],
[['ありがとう', 'ありがとう', 'hiragana'], 'ありがとう'],
[['ありがとう', 'ありがとう', 'katakana'], 'アリガトウ'],
[['ありがとう', 'ありがとう', 'romaji'], 'arigatou'],
[['ありがとう', 'ありがとう', 'none'], null],
[['ありがとう', 'ありがとう', 'default'], 'ありがとう'],
[['有り難う', 'ありがとう', 'hiragana'], 'ありがとう'],
[['有り難う', 'ありがとう', 'katakana'], 'アリガトウ'],
[['有り難う', 'ありがとう', 'romaji'], 'arigatou'],
[['有り難う', 'ありがとう', 'none'], null],
[['有り難う', 'ありがとう', 'default'], 'ありがとう'],
// Cases with falsy readings
[['ありがとう', '', 'hiragana'], ''],
[['ありがとう', '', 'katakana'], ''],
[['ありがとう', '', 'romaji'], 'arigatou'],
[['ありがとう', '', 'none'], null],
[['ありがとう', '', 'default'], ''],
[['ありがとう', null, 'hiragana'], ''],
[['ありがとう', null, 'katakana'], ''],
[['ありがとう', null, 'romaji'], 'arigatou'],
[['ありがとう', null, 'none'], null],
[['ありがとう', null, 'default'], null],
[['ありがとう', void 0, 'hiragana'], ''],
[['ありがとう', void 0, 'katakana'], ''],
[['ありがとう', void 0, 'romaji'], 'arigatou'],
[['ありがとう', void 0, 'none'], null],
[['ありがとう', void 0, 'default'], void 0],
// Cases with falsy readings and kanji expressions
[['有り難う', '', 'hiragana'], ''],
[['有り難う', '', 'katakana'], ''],
[['有り難う', '', 'romaji'], ''],
[['有り難う', '', 'none'], null],
[['有り難う', '', 'default'], ''],
[['有り難う', null, 'hiragana'], ''],
[['有り難う', null, 'katakana'], ''],
[['有り難う', null, 'romaji'], null],
[['有り難う', null, 'none'], null],
[['有り難う', null, 'default'], null],
[['有り難う', void 0, 'hiragana'], ''],
[['有り難う', void 0, 'katakana'], ''],
[['有り難う', void 0, 'romaji'], void 0],
[['有り難う', void 0, 'none'], null],
[['有り難う', void 0, 'default'], void 0]
];
for (const [[expressionFragment, readingFragment, readingMode], expected] of data) {
assert.strictEqual(jp.convertReading(expressionFragment, readingFragment, readingMode), expected);
}
}
function testConvertNumericToFullWidth() {
const data = [
['0123456789', '０１２３４５６７８９'],
['abcdefghij', 'abcdefghij'],
['カタカナ', 'カタカナ'],
['ひらがな', 'ひらがな']
];
for (const [string, expected] of data) {
assert.strictEqual(jp.convertNumericToFullWidth(string), expected);
}
}
function testConvertHalfWidthKanaToFullWidth() {
const data = [
['0123456789', '0123456789'],
['abcdefghij', 'abcdefghij'],
['カタカナ', 'カタカナ'],
['ひらがな', 'ひらがな'],
['ｶｷ', 'カキ', [1, 1]],
['ｶﾞｷ', 'ガキ', [2, 1]],
['ﾆﾎﾝ', 'ニホン', [1, 1, 1]],
['ﾆｯﾎﾟﾝ', 'ニッポン', [1, 1, 2, 1]]
];
for (const [string, expected, expectedSourceMapping] of data) {
const sourceMapping = new Array(string.length).fill(1);
const actual1 = jp.convertHalfWidthKanaToFullWidth(string, null);
const actual2 = jp.convertHalfWidthKanaToFullWidth(string, sourceMapping);
assert.strictEqual(actual1, expected);
assert.strictEqual(actual2, expected);
if (Array.isArray(expectedSourceMapping)) {
vm.assert.deepStrictEqual(sourceMapping, expectedSourceMapping);
}
}
}
function testConvertAlphabeticToKana() {
const data = [
['0123456789', '0123456789'],
['abcdefghij', 'あbcでfgひj', [1, 1, 1, 2, 1, 1, 2, 1]],
['ABCDEFGHIJ', 'あbcでfgひj', [1, 1, 1, 2, 1, 1, 2, 1]], // wanakana.toHiragana converts text to lower case
['カタカナ', 'カタカナ'],
['ひらがな', 'ひらがな'],
['chikara', 'ちから', [3, 2, 2]],
['CHIKARA', 'ちから', [3, 2, 2]]
];
for (const [string, expected, expectedSourceMapping] of data) {
const sourceMapping = new Array(string.length).fill(1);
const actual1 = jp.convertAlphabeticToKana(string, null);
const actual2 = jp.convertAlphabeticToKana(string, sourceMapping);
assert.strictEqual(actual1, expected);
assert.strictEqual(actual2, expected);
if (Array.isArray(expectedSourceMapping)) {
vm.assert.deepStrictEqual(sourceMapping, expectedSourceMapping);
}
}
}
function testDistributeFurigana() {
const data = [
[
['有り難う', 'ありがとう'],
[
{text: '有', furigana: 'あ'},
{text: 'り'},
{text: '難', furigana: 'がと'},
{text: 'う'}
]
],
[
['方々', 'かたがた'],
[
{text: '方々', furigana: 'かたがた'}
]
],
[
['お祝い', 'おいわい'],
[
{text: 'お'},
{text: '祝', furigana: 'いわ'},
{text: 'い'}
]
],
[
['美味しい', 'おいしい'],
[
{text: '美味', furigana: 'おい'},
{text: 'しい'}
]
],
[
['食べ物', 'たべもの'],
[
{text: '食', furigana: 'た'},
{text: 'べ'},
{text: '物', furigana: 'もの'}
]
],
[
['試し切り', 'ためしぎり'],
[
{text: '試', furigana: 'ため'},
{text: 'し'},
{text: '切', furigana: 'ぎ'},
{text: 'り'}
]
],
// Ambiguous
[
['飼い犬', 'かいいぬ'],
[
{text: '飼い犬', furigana: 'かいいぬ'}
]
],
[
['長い間', 'ながいあいだ'],
[
{text: '長い間', furigana: 'ながいあいだ'}
]
]
];
for (const [[expression, reading], expected] of data) {
const actual = jp.distributeFurigana(expression, reading);
vm.assert.deepStrictEqual(actual, expected);
}
}
function testDistributeFuriganaInflected() {
const data = [
[
['美味しい', 'おいしい', '美味しかた'],
[
{text: '美味', furigana: 'おい'},
{text: 'し'},
{text: 'かた'}
]
],
[
['食べる', 'たべる', '食べた'],
[
{text: '食', furigana: 'た'},
{text: 'べ'},
{text: 'た'}
]
]
];
for (const [[expression, reading, source], expected] of data) {
const actual = jp.distributeFuriganaInflected(expression, reading, source);
vm.assert.deepStrictEqual(actual, expected);
}
}
function main() {
testIsCodePointKanji();
testIsCodePointKana();
testIsCodePointJapanese();
testIsStringEntirelyKana();
testIsStringPartiallyJapanese();
testConvertKatakanaToHiragana();
testConvertHiraganaToKatakana();
testConvertToRomaji();
testConvertReading();
testConvertNumericToFullWidth();
testConvertHalfWidthKanaToFullWidth();
testConvertAlphabeticToKana();
testDistributeFurigana();
testDistributeFuriganaInflected();
}
if (require.main === module) { main(); }