Clean up translator (#1505)

* Remove unused: _removeUsedDefinitions

* Remove unused: _scoreToTermFrequency

* Remove unused: _getTermTagsScoreSum

* Add RegexUtil

* Update Translator to use RegexUtil

* Update sw.js

* Update tests

.eslintrc.json

@@ -208,6 +208,7 @@
                 "ext/js/dom/simple-dom-parser.js",
                 "ext/js/general/cache-map.js",
                 "ext/js/general/object-property-accessor.js",
+                "ext/js/general/regex-util.js",
                 "ext/js/general/text-source-map.js",
                 "ext/js/language/deinflector.js",
                 "ext/js/language/dictionary-data-util.js",

ext/background.html

@@ -39,6 +39,7 @@
 <script src="/js/dom/native-simple-dom-parser.js"></script>
 <script src="/js/general/cache-map.js"></script>
 <script src="/js/general/object-property-accessor.js"></script>
+<script src="/js/general/regex-util.js"></script>
 <script src="/js/general/text-source-map.js"></script>
 <script src="/js/language/deinflector.js"></script>
 <script src="/js/language/dictionary-data-util.js"></script>

ext/js/general/regex-util.js

@@ -0,0 +1,93 @@
+/*
+ * Copyright (C) 2021  Yomichan Authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * This class provides some general utility functions for regular expressions.
+ */
+class RegexUtil {
+    /**
+     * Applies string.replace using a regular expression and replacement string as arguments.
+     * A source map of the changes is also maintained.
+     * @param text A string of the text to replace.
+     * @param sourceMap An instance of `TextSourceMap` which corresponds to `text`.
+     * @param pattern A regular expression to use as the replacement.
+     * @param replacement A replacement string that follows the format of the standard
+     *   JavaScript regular expression replacement string.
+     * @return A new string with the pattern replacements applied and the source map updated.
+     */
+    static applyTextReplacement(text, sourceMap, pattern, replacement) {
+        const isGlobal = pattern.global;
+        if (isGlobal) { pattern.lastIndex = 0; }
+        for (let loop = true; loop; loop = isGlobal) {
+            const match = pattern.exec(text);
+            if (match === null) { break; }
+
+            const matchText = match[0];
+            const index = match.index;
+            const actualReplacement = this.applyMatchReplacement(replacement, match);
+            const actualReplacementLength = actualReplacement.length;
+            const delta = actualReplacementLength - (matchText.length > 0 ? matchText.length : -1);
+
+            text = `${text.substring(0, index)}${actualReplacement}${text.substring(index + matchText.length)}`;
+            pattern.lastIndex += delta;
+
+            if (actualReplacementLength > 0) {
+                sourceMap.insert(index, ...(new Array(actualReplacementLength).fill(0)));
+                sourceMap.combine(index - 1 + actualReplacementLength, matchText.length);
+            } else {
+                sourceMap.combine(index, matchText.length);
+            }
+        }
+        return text;
+    }
+
+    /**
+     * Applies the replacement string for a given regular expression match.
+     * @param replacement The replacement string that follows the format of the standard
+     *   JavaScript regular expression replacement string.
+     * @param match A match object returned from RegExp.match.
+     * @return A new string with the pattern replacement applied.
+     */
+    static applyMatchReplacement(replacement, match) {
+        const pattern = this._matchReplacementPattern;
+        pattern.lastIndex = 0;
+        return replacement.replace(pattern, (g0, g1, g2) => {
+            if (typeof g1 !== 'undefined') {
+                const matchIndex = Number.parseInt(g1, 10);
+                if (matchIndex >= 1 && matchIndex <= match.length) {
+                    return match[matchIndex];
+                }
+            } else if (typeof g2 !== 'undefined') {
+                const {groups} = match;
+                if (typeof groups === 'object' && groups !== null && Object.prototype.hasOwnProperty.call(groups, g2)) {
+                    return groups[g2];
+                }
+            } else {
+                switch (g0) {
+                    case '$': return '$';
+                    case '&': return match[0];
+                    case '`': return replacement.substring(0, match.index);
+                    case '\'': return replacement.substring(match.index + g0.length);
+                }
+            }
+            return g0;
+        });
+    }
+}
+
+// eslint-disable-next-line no-underscore-dangle
+RegexUtil._matchReplacementPattern = /\$(?:\$|&|`|'|(\d\d?)|<([^>]*)>)/g;

ext/js/language/translator.js

@@ -17,6 +17,7 @@
 
 /* global
  * Deinflector
+ * RegexUtil
  * TextSourceMap
  */
 
@@ -534,24 +535,6 @@ class Translator {
         );
     }
 
-    _removeUsedDefinitions(definitions, termInfoMap, usedDefinitions) {
-        for (let i = 0, ii = definitions.length; i < ii; ++i) {
-            const definition = definitions[i];
-            const {expression, reading} = definition;
-            const expressionMap = termInfoMap.get(expression);
-            if (
-                typeof expressionMap !== 'undefined' &&
-                typeof expressionMap.get(reading) !== 'undefined'
-            ) {
-                usedDefinitions.add(definition);
-            } else {
-                definitions.splice(i, 1);
-                --i;
-                --ii;
-            }
-        }
-    }
-
     _getUniqueDefinitionTags(definitions) {
         const definitionTagsMap = new Map();
         for (const {definitionTags} of definitions) {
@@ -794,16 +777,6 @@ class Translator {
 
     // Simple helpers
 
-    _scoreToTermFrequency(score) {
-        if (score > 0) {
-            return 'popular';
-        } else if (score < 0) {
-            return 'rare';
-        } else {
-            return 'normal';
-        }
-    }
-
     _getNameBase(name) {
         const pos = name.indexOf(':');
         return (pos >= 0 ? name.substring(0, pos) : name);
@@ -974,14 +947,6 @@ class Translator {
 
     // Reduction functions
 
-    _getTermTagsScoreSum(termTags) {
-        let result = 0;
-        for (const {score} of termTags) {
-            result += score;
-        }
-        return result;
-    }
-
     _getSourceTermMatchCountSum(definitions) {
         let result = 0;
         for (const {sourceTermExactMatchCount} of definitions) {
@@ -1408,59 +1373,8 @@ class Translator {
 
     _applyTextReplacements(text, sourceMap, replacements) {
         for (const {pattern, replacement} of replacements) {
-            text = this._applyTextReplacement(text, sourceMap, pattern, replacement);
+            text = RegexUtil.applyTextReplacement(text, sourceMap, pattern, replacement);
         }
         return text;
     }
-
-    _applyTextReplacement(text, sourceMap, pattern, replacement) {
-        const isGlobal = pattern.global;
-        if (isGlobal) { pattern.lastIndex = 0; }
-        for (let loop = true; loop; loop = isGlobal) {
-            const match = pattern.exec(text);
-            if (match === null) { break; }
-
-            const matchText = match[0];
-            const index = match.index;
-            const actualReplacement = this._applyMatchReplacement(replacement, match);
-            const actualReplacementLength = actualReplacement.length;
-            const delta = actualReplacementLength - (matchText.length > 0 ? matchText.length : -1);
-
-            text = `${text.substring(0, index)}${actualReplacement}${text.substring(index + matchText.length)}`;
-            pattern.lastIndex += delta;
-
-            if (actualReplacementLength > 0) {
-                sourceMap.insert(index, ...(new Array(actualReplacementLength).fill(0)));
-                sourceMap.combine(index - 1 + actualReplacementLength, matchText.length);
-            } else {
-                sourceMap.combine(index, matchText.length);
-            }
-        }
-        return text;
-    }
-
-    _applyMatchReplacement(replacement, match) {
-        const pattern = /\$(?:\$|&|`|'|(\d\d?)|<([^>]*)>)/g;
-        return replacement.replace(pattern, (g0, g1, g2) => {
-            if (typeof g1 !== 'undefined') {
-                const matchIndex = Number.parseInt(g1, 10);
-                if (matchIndex >= 1 && matchIndex <= match.length) {
-                    return match[matchIndex];
-                }
-            } else if (typeof g2 !== 'undefined') {
-                const {groups} = match;
-                if (typeof groups === 'object' && groups !== null && Object.prototype.hasOwnProperty.call(groups, g2)) {
-                    return groups[g2];
-                }
-            } else {
-                switch (g0) {
-                    case '$': return '$';
-                    case '&': return match[0];
-                    case '`': return replacement.substring(0, match.index);
-                    case '\'': return replacement.substring(match.index + g0.length);
-                }
-            }
-            return g0;
-        });
-    }
 }

ext/sw.js

@@ -36,6 +36,7 @@ self.importScripts(
     '/js/dom/simple-dom-parser.js',
     '/js/general/cache-map.js',
     '/js/general/object-property-accessor.js',
+    '/js/general/regex-util.js',
     '/js/general/text-source-map.js',
     '/js/language/deinflector.js',
     '/js/language/dictionary-data-util.js',

test/test-translator.js

@@ -44,6 +44,7 @@ async function createVM() {
         'js/language/dictionary-importer.js',
         'js/data/database.js',
         'js/language/dictionary-database.js',
+        'js/general/regex-util.js',
         'js/general/text-source-map.js',
         'js/language/deinflector.js',
         'js/language/translator.js'