Merge pull request #413 from toasted-nutbread/japanese-utility

Convert Japanese utilities to a module-like style

ext/bg/background.html

@@ -20,6 +20,7 @@
 
         <script src="/mixed/js/core.js"></script>
         <script src="/mixed/js/dom.js"></script>
+        <script src="/mixed/js/japanese.js"></script>
 
         <script src="/bg/js/anki.js"></script>
         <script src="/bg/js/anki-note-builder.js"></script>

ext/bg/js/audio-uri-builder.js

@@ -17,7 +17,7 @@
  */
 
 /* global
- * jpIsStringEntirelyKana
+ * jp
  */
 
 class AudioUriBuilder {
@@ -66,7 +66,7 @@ class AudioUriBuilder {
         let kana = definition.reading;
         let kanji = definition.expression;
 
-        if (!kana && jpIsStringEntirelyKana(kanji)) {
+        if (!kana && jp.isStringEntirelyKana(kanji)) {
             kana = kanji;
             kanji = null;
         }

ext/bg/js/backend.js

@@ -32,9 +32,7 @@
  * dictEnabledSet
  * dictTermsSort
  * handlebarsRenderDynamic
- * jpConvertReading
+ * jp
- * jpDistributeFuriganaInflected
- * jpKatakanaToHiragana
  * optionsLoad
  * optionsSave
  * profileConditionsDescriptor
@@ -402,13 +400,13 @@ class Backend {
                 dictTermsSort(definitions);
                 const {expression, reading} = definitions[0];
                 const source = text.substring(0, sourceLength);
-                for (const {text: text2, furigana} of jpDistributeFuriganaInflected(expression, reading, source)) {
+                for (const {text: text2, furigana} of jp.distributeFuriganaInflected(expression, reading, source)) {
-                    const reading2 = jpConvertReading(text2, furigana, options.parsing.readingMode);
+                    const reading2 = jp.convertReading(text2, furigana, options.parsing.readingMode);
                     term.push({text: text2, reading: reading2});
                 }
                 text = text.substring(source.length);
             } else {
-                const reading = jpConvertReading(text[0], null, options.parsing.readingMode);
+                const reading = jp.convertReading(text[0], null, options.parsing.readingMode);
                 term.push({text: text[0], reading});
                 text = text.substring(1);
             }
@@ -427,16 +425,16 @@ class Backend {
                 for (const {expression, reading, source} of parsedLine) {
                     const term = [];
                     if (expression !== null && reading !== null) {
-                        for (const {text: text2, furigana} of jpDistributeFuriganaInflected(
+                        for (const {text: text2, furigana} of jp.distributeFuriganaInflected(
                             expression,
-                            jpKatakanaToHiragana(reading),
+                            jp.convertKatakanaToHiragana(reading),
                             source
                         )) {
-                            const reading2 = jpConvertReading(text2, furigana, options.parsing.readingMode);
+                            const reading2 = jp.convertReading(text2, furigana, options.parsing.readingMode);
                             term.push({text: text2, reading: reading2});
                         }
                     } else {
-                        const reading2 = jpConvertReading(source, null, options.parsing.readingMode);
+                        const reading2 = jp.convertReading(source, null, options.parsing.readingMode);
                         term.push({text: source, reading: reading2});
                     }
                     result.push(term);

ext/bg/js/clipboard-monitor.js

@@ -17,7 +17,7 @@
  */
 
 /* global
- * jpIsStringPartiallyJapanese
+ * jp
  */
 
 class ClipboardMonitor extends EventDispatcher {
@@ -54,7 +54,7 @@ class ClipboardMonitor extends EventDispatcher {
                 text !== this._previousText
             ) {
                 this._previousText = text;
-                if (jpIsStringPartiallyJapanese(text)) {
+                if (jp.isStringPartiallyJapanese(text)) {
                     this.trigger('change', {text});
                 }
             }

ext/bg/js/handlebars.js

@@ -18,8 +18,7 @@
 
 /* global
  * Handlebars
- * jpDistributeFurigana
+ * jp
- * jpIsCodePointKanji
  */
 
 function handlebarsEscape(text) {
@@ -33,7 +32,7 @@ function handlebarsDumpObject(options) {
 
 function handlebarsFurigana(options) {
     const definition = options.fn(this);
-    const segs = jpDistributeFurigana(definition.expression, definition.reading);
+    const segs = jp.distributeFurigana(definition.expression, definition.reading);
 
     let result = '';
     for (const seg of segs) {
@@ -49,7 +48,7 @@ function handlebarsFurigana(options) {
 
 function handlebarsFuriganaPlain(options) {
     const definition = options.fn(this);
-    const segs = jpDistributeFurigana(definition.expression, definition.reading);
+    const segs = jp.distributeFurigana(definition.expression, definition.reading);
 
     let result = '';
     for (const seg of segs) {
@@ -66,7 +65,7 @@ function handlebarsFuriganaPlain(options) {
 function handlebarsKanjiLinks(options) {
     let result = '';
     for (const c of options.fn(this)) {
-        if (jpIsCodePointKanji(c.codePointAt(0))) {
+        if (jp.isCodePointKanji(c.codePointAt(0))) {
             result += `<a href="#" class="kanji-link">${c}</a>`;
         } else {
             result += c;

ext/bg/js/japanese.js

@@ -17,10 +17,12 @@
  */
 
 /* global
+ * jp
  * wanakana
  */
 
-const JP_HALFWIDTH_KATAKANA_MAPPING = new Map([
+(() => {
+    const HALFWIDTH_KATAKANA_MAPPING = new Map([
         ['ｦ', 'ヲヺ-'],
         ['ｧ', 'ァ--'],
         ['ｨ', 'ィ--'],
@@ -77,97 +79,20 @@ const JP_HALFWIDTH_KATAKANA_MAPPING = new Map([
         ['ﾛ', 'ロ--'],
         ['ﾜ', 'ワ--'],
         ['ﾝ', 'ン--']
-]);
+    ]);
 
-const JP_HIRAGANA_RANGE = [0x3040, 0x309f];
+    const ITERATION_MARK_CODE_POINT = 0x3005;
-const JP_KATAKANA_RANGE = [0x30a0, 0x30ff];
-const JP_KANA_RANGES = [JP_HIRAGANA_RANGE, JP_KATAKANA_RANGE];
-
-const JP_CJK_COMMON_RANGE = [0x4e00, 0x9fff];
-const JP_CJK_RARE_RANGE = [0x3400, 0x4dbf];
-const JP_CJK_RANGES = [JP_CJK_COMMON_RANGE, JP_CJK_RARE_RANGE];
-
-const JP_ITERATION_MARK_CHAR_CODE = 0x3005;
-
-// Japanese character ranges, roughly ordered in order of expected frequency
-const JP_JAPANESE_RANGES = [
-    JP_HIRAGANA_RANGE,
-    JP_KATAKANA_RANGE,
-
-    JP_CJK_COMMON_RANGE,
-    JP_CJK_RARE_RANGE,
-
-    [0xff66, 0xff9f], // Halfwidth katakana
-
-    [0x30fb, 0x30fc], // Katakana punctuation
-    [0xff61, 0xff65], // Kana punctuation
-    [0x3000, 0x303f], // CJK punctuation
-
-    [0xff10, 0xff19], // Fullwidth numbers
-    [0xff21, 0xff3a], // Fullwidth upper case Latin letters
-    [0xff41, 0xff5a], // Fullwidth lower case Latin letters
-
-    [0xff01, 0xff0f], // Fullwidth punctuation 1
-    [0xff1a, 0xff1f], // Fullwidth punctuation 2
-    [0xff3b, 0xff3f], // Fullwidth punctuation 3
-    [0xff5b, 0xff60], // Fullwidth punctuation 4
-    [0xffe0, 0xffee]  // Currency markers
-];
 
 
-// Helper functions
+    // Existing functions
 
-function _jpIsCodePointInRanges(codePoint, ranges) {
+    const isCodePointKanji = jp.isCodePointKanji;
-    for (const [min, max] of ranges) {
+    const isStringEntirelyKana = jp.isStringEntirelyKana;
-        if (codePoint >= min && codePoint <= max) {
-            return true;
-        }
-    }
-    return false;
-}
 
 
-// Character code testing functions
+    // Conversion functions
 
-function jpIsCodePointKanji(codePoint) {
+    function convertKatakanaToHiragana(text) {
-    return _jpIsCodePointInRanges(codePoint, JP_CJK_RANGES);
-}
-
-function jpIsCodePointKana(codePoint) {
-    return _jpIsCodePointInRanges(codePoint, JP_KANA_RANGES);
-}
-
-function jpIsCodePointJapanese(codePoint) {
-    return _jpIsCodePointInRanges(codePoint, JP_JAPANESE_RANGES);
-}
-
-
-// String testing functions
-
-function jpIsStringEntirelyKana(str) {
-    if (str.length === 0) { return false; }
-    for (const c of str) {
-        if (!jpIsCodePointKana(c.codePointAt(0))) {
-            return false;
-        }
-    }
-    return true;
-}
-
-function jpIsStringPartiallyJapanese(str) {
-    if (str.length === 0) { return false; }
-    for (const c of str) {
-        if (jpIsCodePointJapanese(c.codePointAt(0))) {
-            return true;
-        }
-    }
-    return false;
-}
-
-
-// Conversion functions
-
-function jpKatakanaToHiragana(text) {
         let result = '';
         for (const c of text) {
             if (wanakana.isKatakana(c)) {
@@ -178,9 +103,9 @@ function jpKatakanaToHiragana(text) {
         }
 
         return result;
-}
+    }
 
-function jpHiraganaToKatakana(text) {
+    function convertHiraganaToKatakana(text) {
         let result = '';
         for (const c of text) {
             if (wanakana.isHiragana(c)) {
@@ -191,24 +116,24 @@ function jpHiraganaToKatakana(text) {
         }
 
         return result;
-}
+    }
 
-function jpToRomaji(text) {
+    function convertToRomaji(text) {
         return wanakana.toRomaji(text);
-}
+    }
 
-function jpConvertReading(expressionFragment, readingFragment, readingMode) {
+    function convertReading(expressionFragment, readingFragment, readingMode) {
         switch (readingMode) {
             case 'hiragana':
-            return jpKatakanaToHiragana(readingFragment || '');
+                return convertKatakanaToHiragana(readingFragment || '');
             case 'katakana':
-            return jpHiraganaToKatakana(readingFragment || '');
+                return convertHiraganaToKatakana(readingFragment || '');
             case 'romaji':
                 if (readingFragment) {
-                return jpToRomaji(readingFragment);
+                    return convertToRomaji(readingFragment);
                 } else {
-                if (jpIsStringEntirelyKana(expressionFragment)) {
+                    if (isStringEntirelyKana(expressionFragment)) {
-                    return jpToRomaji(expressionFragment);
+                        return convertToRomaji(expressionFragment);
                     }
                 }
                 return readingFragment;
@@ -217,101 +142,23 @@ function jpConvertReading(expressionFragment, readingFragment, readingMode) {
             default:
                 return readingFragment;
         }
-}
-
-function jpDistributeFurigana(expression, reading) {
-    const fallback = [{furigana: reading, text: expression}];
-    if (!reading) {
-        return fallback;
     }
 
-    let isAmbiguous = false;
+    function convertNumericToFullWidth(text) {
-    const segmentize = (reading2, groups) => {
+        let result = '';
-        if (groups.length === 0 || isAmbiguous) {
+        for (const char of text) {
-            return [];
+            let c = char.codePointAt(0);
-        }
+            if (c >= 0x30 && c <= 0x39) { // ['0', '9']
-
+                c += 0xff10 - 0x30; // 0xff10 = '0' full width
-        const group = groups[0];
+                result += String.fromCodePoint(c);
-        if (group.mode === 'kana') {
-            if (jpKatakanaToHiragana(reading2).startsWith(jpKatakanaToHiragana(group.text))) {
-                const readingLeft = reading2.substring(group.text.length);
-                const segs = segmentize(readingLeft, groups.splice(1));
-                if (segs) {
-                    return [{text: group.text}].concat(segs);
-                }
-            }
             } else {
-            let foundSegments = null;
+                result += char;
-            for (let i = reading2.length; i >= group.text.length; --i) {
-                const readingUsed = reading2.substring(0, i);
-                const readingLeft = reading2.substring(i);
-                const segs = segmentize(readingLeft, groups.slice(1));
-                if (segs) {
-                    if (foundSegments !== null) {
-                        // more than one way to segmentize the tail, mark as ambiguous
-                        isAmbiguous = true;
-                        return null;
-                    }
-                    foundSegments = [{text: group.text, furigana: readingUsed}].concat(segs);
-                }
-                // there is only one way to segmentize the last non-kana group
-                if (groups.length === 1) {
-                    break;
             }
         }
-            return foundSegments;
+        return result;
-        }
-    };
-
-    const groups = [];
-    let modePrev = null;
-    for (const c of expression) {
-        const codePoint = c.codePointAt(0);
-        const modeCurr = jpIsCodePointKanji(codePoint) || codePoint === JP_ITERATION_MARK_CHAR_CODE ? 'kanji' : 'kana';
-        if (modeCurr === modePrev) {
-            groups[groups.length - 1].text += c;
-        } else {
-            groups.push({mode: modeCurr, text: c});
-            modePrev = modeCurr;
-        }
     }
 
-    const segments = segmentize(reading, groups);
+    function convertHalfWidthKanaToFullWidth(text, sourceMapping) {
-    if (segments && !isAmbiguous) {
-        return segments;
-    }
-    return fallback;
-}
-
-function jpDistributeFuriganaInflected(expression, reading, source) {
-    const output = [];
-
-    let stemLength = 0;
-    const shortest = Math.min(source.length, expression.length);
-    const sourceHiragana = jpKatakanaToHiragana(source);
-    const expressionHiragana = jpKatakanaToHiragana(expression);
-    while (stemLength < shortest && sourceHiragana[stemLength] === expressionHiragana[stemLength]) {
-        ++stemLength;
-    }
-    const offset = source.length - stemLength;
-
-    const stemExpression = source.substring(0, source.length - offset);
-    const stemReading = reading.substring(
-        0,
-        offset === 0 ? reading.length : reading.length - expression.length + stemLength
-    );
-    for (const segment of jpDistributeFurigana(stemExpression, stemReading)) {
-        output.push(segment);
-    }
-
-    if (stemLength !== source.length) {
-        output.push({text: source.substring(stemLength)});
-    }
-
-    return output;
-}
-
-function jpConvertHalfWidthKanaToFullWidth(text, sourceMapping) {
         let result = '';
         const hasSourceMapping = Array.isArray(sourceMapping);
 
@@ -319,7 +166,7 @@ function jpConvertHalfWidthKanaToFullWidth(text, sourceMapping) {
         // the relevant characters are represented with a single UTF-16 character code.
         for (let i = 0, ii = text.length; i < ii; ++i) {
             const c = text[i];
-        const mapping = JP_HALFWIDTH_KATAKANA_MAPPING.get(c);
+            const mapping = HALFWIDTH_KATAKANA_MAPPING.get(c);
             if (typeof mapping !== 'string') {
                 result += c;
                 continue;
@@ -354,31 +201,11 @@ function jpConvertHalfWidthKanaToFullWidth(text, sourceMapping) {
         }
 
         return result;
-}
-
-function jpConvertNumericTofullWidth(text) {
-    let result = '';
-    for (const char of text) {
-        let c = char.codePointAt(0);
-        if (c >= 0x30 && c <= 0x39) { // ['0', '9']
-            c += 0xff10 - 0x30; // 0xff10 = '0' full width
-            result += String.fromCodePoint(c);
-        } else {
-            result += char;
     }
-    }
-    return result;
-}
 
-function jpConvertAlphabeticToKana(text, sourceMapping) {
+    function convertAlphabeticToKana(text, sourceMapping) {
         let part = '';
         let result = '';
-    const ii = text.length;
-
-    if (sourceMapping.length === ii) {
-        sourceMapping.length = ii;
-        sourceMapping.fill(1);
-    }
 
         for (const char of text) {
             // Note: 0x61 is the character code for 'a'
@@ -395,7 +222,7 @@ function jpConvertAlphabeticToKana(text, sourceMapping) {
                 c = 0x2d; // '-'
             } else {
                 if (part.length > 0) {
-                result += jpToHiragana(part, sourceMapping, result.length);
+                    result += convertAlphabeticPartToKana(part, sourceMapping, result.length);
                     part = '';
                 }
                 result += char;
@@ -405,12 +232,12 @@ function jpConvertAlphabeticToKana(text, sourceMapping) {
         }
 
         if (part.length > 0) {
-        result += jpToHiragana(part, sourceMapping, result.length);
+            result += convertAlphabeticPartToKana(part, sourceMapping, result.length);
         }
         return result;
-}
+    }
 
-function jpToHiragana(text, sourceMapping, sourceMappingStart) {
+    function convertAlphabeticPartToKana(text, sourceMapping, sourceMappingStart) {
         const result = wanakana.toHiragana(text);
 
         // Generate source mapping
@@ -455,4 +282,115 @@ function jpToHiragana(text, sourceMapping, sourceMappingStart) {
         }
 
         return result;
-}
+    }
+
+
+    // Furigana distribution
+
+    function distributeFurigana(expression, reading) {
+        const fallback = [{furigana: reading, text: expression}];
+        if (!reading) {
+            return fallback;
+        }
+
+        let isAmbiguous = false;
+        const segmentize = (reading2, groups) => {
+            if (groups.length === 0 || isAmbiguous) {
+                return [];
+            }
+
+            const group = groups[0];
+            if (group.mode === 'kana') {
+                if (convertKatakanaToHiragana(reading2).startsWith(convertKatakanaToHiragana(group.text))) {
+                    const readingLeft = reading2.substring(group.text.length);
+                    const segs = segmentize(readingLeft, groups.splice(1));
+                    if (segs) {
+                        return [{text: group.text}].concat(segs);
+                    }
+                }
+            } else {
+                let foundSegments = null;
+                for (let i = reading2.length; i >= group.text.length; --i) {
+                    const readingUsed = reading2.substring(0, i);
+                    const readingLeft = reading2.substring(i);
+                    const segs = segmentize(readingLeft, groups.slice(1));
+                    if (segs) {
+                        if (foundSegments !== null) {
+                            // more than one way to segmentize the tail, mark as ambiguous
+                            isAmbiguous = true;
+                            return null;
+                        }
+                        foundSegments = [{text: group.text, furigana: readingUsed}].concat(segs);
+                    }
+                    // there is only one way to segmentize the last non-kana group
+                    if (groups.length === 1) {
+                        break;
+                    }
+                }
+                return foundSegments;
+            }
+        };
+
+        const groups = [];
+        let modePrev = null;
+        for (const c of expression) {
+            const codePoint = c.codePointAt(0);
+            const modeCurr = isCodePointKanji(codePoint) || codePoint === ITERATION_MARK_CODE_POINT ? 'kanji' : 'kana';
+            if (modeCurr === modePrev) {
+                groups[groups.length - 1].text += c;
+            } else {
+                groups.push({mode: modeCurr, text: c});
+                modePrev = modeCurr;
+            }
+        }
+
+        const segments = segmentize(reading, groups);
+        if (segments && !isAmbiguous) {
+            return segments;
+        }
+        return fallback;
+    }
+
+    function distributeFuriganaInflected(expression, reading, source) {
+        const output = [];
+
+        let stemLength = 0;
+        const shortest = Math.min(source.length, expression.length);
+        const sourceHiragana = convertKatakanaToHiragana(source);
+        const expressionHiragana = convertKatakanaToHiragana(expression);
+        while (stemLength < shortest && sourceHiragana[stemLength] === expressionHiragana[stemLength]) {
+            ++stemLength;
+        }
+        const offset = source.length - stemLength;
+
+        const stemExpression = source.substring(0, source.length - offset);
+        const stemReading = reading.substring(
+            0,
+            offset === 0 ? reading.length : reading.length - expression.length + stemLength
+        );
+        for (const segment of distributeFurigana(stemExpression, stemReading)) {
+            output.push(segment);
+        }
+
+        if (stemLength !== source.length) {
+            output.push({text: source.substring(stemLength)});
+        }
+
+        return output;
+    }
+
+
+    // Exports
+
+    Object.assign(jp, {
+        convertKatakanaToHiragana,
+        convertHiraganaToKatakana,
+        convertToRomaji,
+        convertReading,
+        convertNumericToFullWidth,
+        convertHalfWidthKanaToFullWidth,
+        convertAlphabeticToKana,
+        distributeFurigana,
+        distributeFuriganaInflected
+    });
+})();

ext/bg/js/translator.js

@@ -29,13 +29,7 @@
  * dictTermsMergeBySequence
  * dictTermsSort
  * dictTermsUndupe
- * jpConvertAlphabeticToKana
+ * jp
- * jpConvertHalfWidthKanaToFullWidth
- * jpConvertNumericTofullWidth
- * jpDistributeFurigana
- * jpHiraganaToKatakana
- * jpIsCodePointJapanese
- * jpKatakanaToHiragana
  * requestJson
  */
 
@@ -275,7 +269,7 @@ class Translator {
                 const termTags = await this.expandTags(definition.termTags, definition.dictionary);
 
                 const {expression, reading} = definition;
-                const furiganaSegments = jpDistributeFurigana(expression, reading);
+                const furiganaSegments = jp.distributeFurigana(expression, reading);
 
                 definitions.push({
                     source: deinflection.source,
@@ -376,20 +370,20 @@ class Translator {
             let sourceMapping = null;
             if (halfWidth) {
                 if (sourceMapping === null) { sourceMapping = Translator.createTextSourceMapping(text2); }
-                text2 = jpConvertHalfWidthKanaToFullWidth(text2, sourceMapping);
+                text2 = jp.convertHalfWidthKanaToFullWidth(text2, sourceMapping);
             }
             if (numeric) {
-                text2 = jpConvertNumericTofullWidth(text2);
+                text2 = jp.convertNumericToFullWidth(text2);
             }
             if (alphabetic) {
                 if (sourceMapping === null) { sourceMapping = Translator.createTextSourceMapping(text2); }
-                text2 = jpConvertAlphabeticToKana(text2, sourceMapping);
+                text2 = jp.convertAlphabeticToKana(text2, sourceMapping);
             }
             if (katakana) {
-                text2 = jpHiraganaToKatakana(text2);
+                text2 = jp.convertHiraganaToKatakana(text2);
             }
             if (hiragana) {
-                text2 = jpKatakanaToHiragana(text2);
+                text2 = jp.convertKatakanaToHiragana(text2);
             }
 
             for (let i = text2.length; i > 0; --i) {
@@ -590,7 +584,7 @@ class Translator {
     }
 
     static createExpression(expression, reading, termTags=null, termFrequency=null) {
-        const furiganaSegments = jpDistributeFurigana(expression, reading);
+        const furiganaSegments = jp.distributeFurigana(expression, reading);
         return {
             expression,
             reading,
@@ -639,7 +633,7 @@ class Translator {
         if (!options.scanning.alphanumeric) {
             let newText = '';
             for (const c of text) {
-                if (!jpIsCodePointJapanese(c.codePointAt(0))) {
+                if (!jp.isCodePointJapanese(c.codePointAt(0))) {
                     break;
                 }
                 newText += c;

ext/bg/search.html

@@ -74,6 +74,7 @@
         <script src="/mixed/js/core.js"></script>
         <script src="/mixed/js/dom.js"></script>
         <script src="/mixed/js/api.js"></script>
+        <script src="/mixed/js/japanese.js"></script>
 
         <script src="/bg/js/dictionary.js"></script>
         <script src="/bg/js/handlebars.js"></script>

ext/bg/settings.html

@@ -1088,6 +1088,7 @@
         <script src="/mixed/js/core.js"></script>
         <script src="/mixed/js/dom.js"></script>
         <script src="/mixed/js/api.js"></script>
+        <script src="/mixed/js/japanese.js"></script>
 
         <script src="/bg/js/anki.js"></script>
         <script src="/bg/js/anki-note-builder.js"></script>

ext/fg/float.html

@@ -43,6 +43,7 @@
         <script src="/mixed/js/core.js"></script>
         <script src="/mixed/js/dom.js"></script>
         <script src="/mixed/js/api.js"></script>
+        <script src="/mixed/js/japanese.js"></script>
 
         <script src="/fg/js/document.js"></script>
         <script src="/fg/js/source.js"></script>

ext/mixed/js/display-generator.js

@@ -19,6 +19,7 @@
 /* global
  * TemplateHandler
  * apiGetDisplayTemplatesHtml
+ * jp
  */
 
 class DisplayGenerator {
@@ -283,7 +284,7 @@ class DisplayGenerator {
     _appendKanjiLinks(container, text) {
         let part = '';
         for (const c of text) {
-            if (DisplayGenerator._isCharacterKanji(c)) {
+            if (jp.isCodePointKanji(c.codePointAt(0))) {
                 if (part.length > 0) {
                     container.appendChild(document.createTextNode(part));
                     part = '';
@@ -300,14 +301,6 @@ class DisplayGenerator {
         }
     }
 
-    static _isCharacterKanji(c) {
-        const code = c.codePointAt(0);
-        return (
-            code >= 0x4e00 && code < 0x9fb0 ||
-            code >= 0x3400 && code < 0x4dc0
-        );
-    }
-
     static _appendMultiple(container, createItem, detailsArray, fallback=[]) {
         if (container === null) { return 0; }
 

ext/mixed/js/japanese.js

@@ -0,0 +1,124 @@
+/*
+ * Copyright (C) 2020  Alex Yatskov <alex@foosoft.net>
+ * Author: Alex Yatskov <alex@foosoft.net>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+const jp = (() => {
+    const HIRAGANA_RANGE = [0x3040, 0x309f];
+    const KATAKANA_RANGE = [0x30a0, 0x30ff];
+    const KANA_RANGES = [HIRAGANA_RANGE, KATAKANA_RANGE];
+
+    const CJK_UNIFIED_IDEOGRAPHS_RANGE = [0x4e00, 0x9fff];
+    const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A_RANGE = [0x3400, 0x4dbf];
+    const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B_RANGE = [0x20000, 0x2a6df];
+    const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C_RANGE = [0x2a700, 0x2b73f];
+    const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D_RANGE = [0x2b740, 0x2b81f];
+    const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_E_RANGE = [0x2b820, 0x2ceaf];
+    const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_F_RANGE = [0x2ceb0, 0x2ebef];
+    const CJK_COMPATIBILITY_IDEOGRAPHS_SUPPLEMENT_RANGE = [0x2f800, 0x2fa1f];
+    const CJK_UNIFIED_IDEOGRAPHS_RANGES = [
+        CJK_UNIFIED_IDEOGRAPHS_RANGE,
+        CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A_RANGE,
+        CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B_RANGE,
+        CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C_RANGE,
+        CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D_RANGE,
+        CJK_UNIFIED_IDEOGRAPHS_EXTENSION_E_RANGE,
+        CJK_UNIFIED_IDEOGRAPHS_EXTENSION_F_RANGE,
+        CJK_COMPATIBILITY_IDEOGRAPHS_SUPPLEMENT_RANGE
+    ];
+
+    // Japanese character ranges, roughly ordered in order of expected frequency
+    const JAPANESE_RANGES = [
+        HIRAGANA_RANGE,
+        KATAKANA_RANGE,
+
+        ...CJK_UNIFIED_IDEOGRAPHS_RANGES,
+
+        [0xff66, 0xff9f], // Halfwidth katakana
+
+        [0x30fb, 0x30fc], // Katakana punctuation
+        [0xff61, 0xff65], // Kana punctuation
+        [0x3000, 0x303f], // CJK punctuation
+
+        [0xff10, 0xff19], // Fullwidth numbers
+        [0xff21, 0xff3a], // Fullwidth upper case Latin letters
+        [0xff41, 0xff5a], // Fullwidth lower case Latin letters
+
+        [0xff01, 0xff0f], // Fullwidth punctuation 1
+        [0xff1a, 0xff1f], // Fullwidth punctuation 2
+        [0xff3b, 0xff3f], // Fullwidth punctuation 3
+        [0xff5b, 0xff60], // Fullwidth punctuation 4
+        [0xffe0, 0xffee]  // Currency markers
+    ];
+
+
+    // Character code testing functions
+
+    function isCodePointKanji(codePoint) {
+        return isCodePointInRanges(codePoint, CJK_UNIFIED_IDEOGRAPHS_RANGES);
+    }
+
+    function isCodePointKana(codePoint) {
+        return isCodePointInRanges(codePoint, KANA_RANGES);
+    }
+
+    function isCodePointJapanese(codePoint) {
+        return isCodePointInRanges(codePoint, JAPANESE_RANGES);
+    }
+
+    function isCodePointInRanges(codePoint, ranges) {
+        for (const [min, max] of ranges) {
+            if (codePoint >= min && codePoint <= max) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+
+    // String testing functions
+
+    function isStringEntirelyKana(str) {
+        if (str.length === 0) { return false; }
+        for (const c of str) {
+            if (!isCodePointKana(c.codePointAt(0))) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    function isStringPartiallyJapanese(str) {
+        if (str.length === 0) { return false; }
+        for (const c of str) {
+            if (isCodePointJapanese(c.codePointAt(0))) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+
+    // Exports
+
+    return {
+        isCodePointKanji,
+        isCodePointKana,
+        isCodePointJapanese,
+        isStringEntirelyKana,
+        isStringPartiallyJapanese
+    };
+})();

package.json

@@ -8,7 +8,7 @@
     "scripts": {
         "test": "npm run test-lint && npm run test-code",
         "test-lint": "eslint . && node ./test/lint/global-declarations.js",
-        "test-code": "node ./test/test-schema.js && node ./test/test-dictionary.js && node ./test/test-database.js && node ./test/test-document.js && node ./test/test-object-property-accessor.js"
+        "test-code": "node ./test/test-schema.js && node ./test/test-dictionary.js && node ./test/test-database.js && node ./test/test-document.js && node ./test/test-object-property-accessor.js && node ./test/test-japanese.js"
     },
     "repository": {
         "type": "git",

test/test-japanese.js

@@ -0,0 +1,414 @@
+/*
+ * Copyright (C) 2020  Alex Yatskov <alex@foosoft.net>
+ * Author: Alex Yatskov <alex@foosoft.net>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+const assert = require('assert');
+const {VM} = require('./yomichan-vm');
+
+const vm = new VM();
+vm.execute([
+    'mixed/lib/wanakana.min.js',
+    'mixed/js/japanese.js',
+    'bg/js/japanese.js'
+]);
+const jp = vm.get('jp');
+
+
+function testIsCodePointKanji() {
+    const data = [
+        ['力方', true],
+        ['\u53f1\u{20b9f}', true],
+        ['かたカタ々kata、。？,.?', false]
+    ];
+
+    for (const [characters, expected] of data) {
+        for (const character of characters) {
+            const codePoint = character.codePointAt(0);
+            const actual = jp.isCodePointKanji(codePoint);
+            assert.strictEqual(actual, expected, `isCodePointKanji failed for ${character} (\\u{${codePoint.toString(16)}})`);
+        }
+    }
+}
+
+function testIsCodePointKana() {
+    const data = [
+        ['かたカタ', true],
+        ['力方々kata、。？,.?', false],
+        ['\u53f1\u{20b9f}', false]
+    ];
+
+    for (const [characters, expected] of data) {
+        for (const character of characters) {
+            const codePoint = character.codePointAt(0);
+            const actual = jp.isCodePointKana(codePoint);
+            assert.strictEqual(actual, expected, `isCodePointKana failed for ${character} (\\u{${codePoint.toString(16)}})`);
+        }
+    }
+}
+
+function testIsCodePointJapanese() {
+    const data = [
+        ['かたカタ力方々、。？', true],
+        ['\u53f1\u{20b9f}', true],
+        ['kata,.?', false]
+    ];
+
+    for (const [characters, expected] of data) {
+        for (const character of characters) {
+            const codePoint = character.codePointAt(0);
+            const actual = jp.isCodePointJapanese(codePoint);
+            assert.strictEqual(actual, expected, `isCodePointJapanese failed for ${character} (\\u{${codePoint.toString(16)}})`);
+        }
+    }
+}
+
+function testIsStringEntirelyKana() {
+    const data = [
+        ['かたかな', true],
+        ['カタカナ', true],
+        ['ひらがな', true],
+        ['ヒラガナ', true],
+        ['カタカナひらがな', true],
+        ['かたカタ力方々、。？', false],
+        ['\u53f1\u{20b9f}', false],
+        ['kata,.?', false],
+        ['かたカタ力方々、。？invalid', false],
+        ['\u53f1\u{20b9f}invalid', false],
+        ['kata,.?かた', false]
+    ];
+
+    for (const [string, expected] of data) {
+        assert.strictEqual(jp.isStringEntirelyKana(string), expected);
+    }
+}
+
+function testIsStringPartiallyJapanese() {
+    const data = [
+        ['かたかな', true],
+        ['カタカナ', true],
+        ['ひらがな', true],
+        ['ヒラガナ', true],
+        ['カタカナひらがな', true],
+        ['かたカタ力方々、。？', true],
+        ['\u53f1\u{20b9f}', true],
+        ['kata,.?', false],
+        ['かたカタ力方々、。？invalid', true],
+        ['\u53f1\u{20b9f}invalid', true],
+        ['kata,.?かた', true]
+    ];
+
+    for (const [string, expected] of data) {
+        assert.strictEqual(jp.isStringPartiallyJapanese(string), expected);
+    }
+}
+
+function testConvertKatakanaToHiragana() {
+    const data = [
+        ['かたかな', 'かたかな'],
+        ['ひらがな', 'ひらがな'],
+        ['カタカナ', 'かたかな'],
+        ['ヒラガナ', 'ひらがな'],
+        ['カタカナかたかな', 'かたかなかたかな'],
+        ['ヒラガナひらがな', 'ひらがなひらがな'],
+        ['chikaraちからチカラ力', 'chikaraちからちから力'],
+        ['katakana', 'katakana'],
+        ['hiragana', 'hiragana']
+    ];
+
+    for (const [string, expected] of data) {
+        assert.strictEqual(jp.convertKatakanaToHiragana(string), expected);
+    }
+}
+
+function testConvertHiraganaToKatakana() {
+    const data = [
+        ['かたかな', 'カタカナ'],
+        ['ひらがな', 'ヒラガナ'],
+        ['カタカナ', 'カタカナ'],
+        ['ヒラガナ', 'ヒラガナ'],
+        ['カタカナかたかな', 'カタカナカタカナ'],
+        ['ヒラガナひらがな', 'ヒラガナヒラガナ'],
+        ['chikaraちからチカラ力', 'chikaraチカラチカラ力'],
+        ['katakana', 'katakana'],
+        ['hiragana', 'hiragana']
+    ];
+
+    for (const [string, expected] of data) {
+        assert.strictEqual(jp.convertHiraganaToKatakana(string), expected);
+    }
+}
+
+function testConvertToRomaji() {
+    const data = [
+        ['かたかな', 'katakana'],
+        ['ひらがな', 'hiragana'],
+        ['カタカナ', 'katakana'],
+        ['ヒラガナ', 'hiragana'],
+        ['カタカナかたかな', 'katakanakatakana'],
+        ['ヒラガナひらがな', 'hiraganahiragana'],
+        ['chikaraちからチカラ力', 'chikarachikarachikara力'],
+        ['katakana', 'katakana'],
+        ['hiragana', 'hiragana']
+    ];
+
+    for (const [string, expected] of data) {
+        assert.strictEqual(jp.convertToRomaji(string), expected);
+    }
+}
+
+function testConvertReading() {
+    const data = [
+        [['アリガトウ', 'アリガトウ', 'hiragana'], 'ありがとう'],
+        [['アリガトウ', 'アリガトウ', 'katakana'], 'アリガトウ'],
+        [['アリガトウ', 'アリガトウ', 'romaji'], 'arigatou'],
+        [['アリガトウ', 'アリガトウ', 'none'], null],
+        [['アリガトウ', 'アリガトウ', 'default'], 'アリガトウ'],
+
+        [['ありがとう', 'ありがとう', 'hiragana'], 'ありがとう'],
+        [['ありがとう', 'ありがとう', 'katakana'], 'アリガトウ'],
+        [['ありがとう', 'ありがとう', 'romaji'], 'arigatou'],
+        [['ありがとう', 'ありがとう', 'none'], null],
+        [['ありがとう', 'ありがとう', 'default'], 'ありがとう'],
+
+        [['有り難う', 'ありがとう', 'hiragana'], 'ありがとう'],
+        [['有り難う', 'ありがとう', 'katakana'], 'アリガトウ'],
+        [['有り難う', 'ありがとう', 'romaji'], 'arigatou'],
+        [['有り難う', 'ありがとう', 'none'], null],
+        [['有り難う', 'ありがとう', 'default'], 'ありがとう'],
+
+        // Cases with falsy readings
+
+        [['ありがとう', '', 'hiragana'], ''],
+        [['ありがとう', '', 'katakana'], ''],
+        [['ありがとう', '', 'romaji'], 'arigatou'],
+        [['ありがとう', '', 'none'], null],
+        [['ありがとう', '', 'default'], ''],
+
+        [['ありがとう', null, 'hiragana'], ''],
+        [['ありがとう', null, 'katakana'], ''],
+        [['ありがとう', null, 'romaji'], 'arigatou'],
+        [['ありがとう', null, 'none'], null],
+        [['ありがとう', null, 'default'], null],
+
+        [['ありがとう', void 0, 'hiragana'], ''],
+        [['ありがとう', void 0, 'katakana'], ''],
+        [['ありがとう', void 0, 'romaji'], 'arigatou'],
+        [['ありがとう', void 0, 'none'], null],
+        [['ありがとう', void 0, 'default'], void 0],
+
+        // Cases with falsy readings and kanji expressions
+
+        [['有り難う', '', 'hiragana'], ''],
+        [['有り難う', '', 'katakana'], ''],
+        [['有り難う', '', 'romaji'], ''],
+        [['有り難う', '', 'none'], null],
+        [['有り難う', '', 'default'], ''],
+
+        [['有り難う', null, 'hiragana'], ''],
+        [['有り難う', null, 'katakana'], ''],
+        [['有り難う', null, 'romaji'], null],
+        [['有り難う', null, 'none'], null],
+        [['有り難う', null, 'default'], null],
+
+        [['有り難う', void 0, 'hiragana'], ''],
+        [['有り難う', void 0, 'katakana'], ''],
+        [['有り難う', void 0, 'romaji'], void 0],
+        [['有り難う', void 0, 'none'], null],
+        [['有り難う', void 0, 'default'], void 0]
+    ];
+
+    for (const [[expressionFragment, readingFragment, readingMode], expected] of data) {
+        assert.strictEqual(jp.convertReading(expressionFragment, readingFragment, readingMode), expected);
+    }
+}
+
+function testConvertNumericToFullWidth() {
+    const data = [
+        ['0123456789', '０１２３４５６７８９'],
+        ['abcdefghij', 'abcdefghij'],
+        ['カタカナ', 'カタカナ'],
+        ['ひらがな', 'ひらがな']
+    ];
+
+    for (const [string, expected] of data) {
+        assert.strictEqual(jp.convertNumericToFullWidth(string), expected);
+    }
+}
+
+function testConvertHalfWidthKanaToFullWidth() {
+    const data = [
+        ['0123456789', '0123456789'],
+        ['abcdefghij', 'abcdefghij'],
+        ['カタカナ', 'カタカナ'],
+        ['ひらがな', 'ひらがな'],
+        ['ｶｷ', 'カキ', [1, 1]],
+        ['ｶﾞｷ', 'ガキ', [2, 1]],
+        ['ﾆﾎﾝ', 'ニホン', [1, 1, 1]],
+        ['ﾆｯﾎﾟﾝ', 'ニッポン', [1, 1, 2, 1]]
+    ];
+
+    for (const [string, expected, expectedSourceMapping] of data) {
+        const sourceMapping = new Array(string.length).fill(1);
+        const actual1 = jp.convertHalfWidthKanaToFullWidth(string, null);
+        const actual2 = jp.convertHalfWidthKanaToFullWidth(string, sourceMapping);
+        assert.strictEqual(actual1, expected);
+        assert.strictEqual(actual2, expected);
+        if (Array.isArray(expectedSourceMapping)) {
+            vm.assert.deepStrictEqual(sourceMapping, expectedSourceMapping);
+        }
+    }
+}
+
+function testConvertAlphabeticToKana() {
+    const data = [
+        ['0123456789', '0123456789'],
+        ['abcdefghij', 'あbcでfgひj', [1, 1, 1, 2, 1, 1, 2, 1]],
+        ['ABCDEFGHIJ', 'あbcでfgひj', [1, 1, 1, 2, 1, 1, 2, 1]], // wanakana.toHiragana converts text to lower case
+        ['カタカナ', 'カタカナ'],
+        ['ひらがな', 'ひらがな'],
+        ['chikara', 'ちから', [3, 2, 2]],
+        ['CHIKARA', 'ちから', [3, 2, 2]]
+    ];
+
+    for (const [string, expected, expectedSourceMapping] of data) {
+        const sourceMapping = new Array(string.length).fill(1);
+        const actual1 = jp.convertAlphabeticToKana(string, null);
+        const actual2 = jp.convertAlphabeticToKana(string, sourceMapping);
+        assert.strictEqual(actual1, expected);
+        assert.strictEqual(actual2, expected);
+        if (Array.isArray(expectedSourceMapping)) {
+            vm.assert.deepStrictEqual(sourceMapping, expectedSourceMapping);
+        }
+    }
+}
+
+function testDistributeFurigana() {
+    const data = [
+        [
+            ['有り難う', 'ありがとう'],
+            [
+                {text: '有', furigana: 'あ'},
+                {text: 'り'},
+                {text: '難', furigana: 'がと'},
+                {text: 'う'}
+            ]
+        ],
+        [
+            ['方々', 'かたがた'],
+            [
+                {text: '方々', furigana: 'かたがた'}
+            ]
+        ],
+        [
+            ['お祝い', 'おいわい'],
+            [
+                {text: 'お'},
+                {text: '祝', furigana: 'いわ'},
+                {text: 'い'}
+            ]
+        ],
+        [
+            ['美味しい', 'おいしい'],
+            [
+                {text: '美味', furigana: 'おい'},
+                {text: 'しい'}
+            ]
+        ],
+        [
+            ['食べ物', 'たべもの'],
+            [
+                {text: '食', furigana: 'た'},
+                {text: 'べ'},
+                {text: '物', furigana: 'もの'}
+            ]
+        ],
+        [
+            ['試し切り', 'ためしぎり'],
+            [
+                {text: '試', furigana: 'ため'},
+                {text: 'し'},
+                {text: '切', furigana: 'ぎ'},
+                {text: 'り'}
+            ]
+        ],
+        // Ambiguous
+        [
+            ['飼い犬', 'かいいぬ'],
+            [
+                {text: '飼い犬', furigana: 'かいいぬ'}
+            ]
+        ],
+        [
+            ['長い間', 'ながいあいだ'],
+            [
+                {text: '長い間', furigana: 'ながいあいだ'}
+            ]
+        ]
+    ];
+
+    for (const [[expression, reading], expected] of data) {
+        const actual = jp.distributeFurigana(expression, reading);
+        vm.assert.deepStrictEqual(actual, expected);
+    }
+}
+
+function testDistributeFuriganaInflected() {
+    const data = [
+        [
+            ['美味しい', 'おいしい', '美味しかた'],
+            [
+                {text: '美味', furigana: 'おい'},
+                {text: 'し'},
+                {text: 'かた'}
+            ]
+        ],
+        [
+            ['食べる', 'たべる', '食べた'],
+            [
+                {text: '食', furigana: 'た'},
+                {text: 'べ'},
+                {text: 'た'}
+            ]
+        ]
+    ];
+
+    for (const [[expression, reading, source], expected] of data) {
+        const actual = jp.distributeFuriganaInflected(expression, reading, source);
+        vm.assert.deepStrictEqual(actual, expected);
+    }
+}
+
+
+function main() {
+    testIsCodePointKanji();
+    testIsCodePointKana();
+    testIsCodePointJapanese();
+    testIsStringEntirelyKana();
+    testIsStringPartiallyJapanese();
+    testConvertKatakanaToHiragana();
+    testConvertHiraganaToKatakana();
+    testConvertToRomaji();
+    testConvertReading();
+    testConvertNumericToFullWidth();
+    testConvertHalfWidthKanaToFullWidth();
+    testConvertAlphabeticToKana();
+    testDistributeFurigana();
+    testDistributeFuriganaInflected();
+}
+
+
+if (require.main === module) { main(); }