alex/yomichan
1
Fork 0
Code Releases Activity

Clean up translator (#1505)

Browse Source 
* Remove unused: _removeUsedDefinitions

* Remove unused: _scoreToTermFrequency

* Remove unused: _getTermTagsScoreSum

* Add RegexUtil

* Update Translator to use RegexUtil

* Update sw.js

* Update tests
This commit is contained in:
toasted-nutbread 2021-03-08 21:01:55 -05:00 committed by GitHub
parent 643dbfb12a
commit c6f4144fda
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 99 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.eslintrc.json
View File

@ -208,6 +208,7 @@
"ext/js/dom/simple-dom-parser.js",
"ext/js/general/cache-map.js",
"ext/js/general/object-property-accessor.js",
"ext/js/general/regex-util.js",
"ext/js/general/text-source-map.js",
"ext/js/language/deinflector.js",
"ext/js/language/dictionary-data-util.js",

1
ext/background.html
View File

@ -39,6 +39,7 @@
<script src="/js/dom/native-simple-dom-parser.js"></script>
<script src="/js/general/cache-map.js"></script>
<script src="/js/general/object-property-accessor.js"></script>
<script src="/js/general/regex-util.js"></script>
<script src="/js/general/text-source-map.js"></script>
<script src="/js/language/deinflector.js"></script>
<script src="/js/language/dictionary-data-util.js"></script>

93
ext/js/general/regex-util.js Normal file
View File

@ -0,0 +1,93 @@
/*
* Copyright (C) 2021 Yomichan Authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
/**
* This class provides some general utility functions for regular expressions.
*/
class RegexUtil {
/**
* Applies string.replace using a regular expression and replacement string as arguments.
* A source map of the changes is also maintained.
* @param text A string of the text to replace.
* @param sourceMap An instance of `TextSourceMap` which corresponds to `text`.
* @param pattern A regular expression to use as the replacement.
* @param replacement A replacement string that follows the format of the standard
* JavaScript regular expression replacement string.
* @return A new string with the pattern replacements applied and the source map updated.
*/
static applyTextReplacement(text, sourceMap, pattern, replacement) {
const isGlobal = pattern.global;
if (isGlobal) { pattern.lastIndex = 0; }
for (let loop = true; loop; loop = isGlobal) {
const match = pattern.exec(text);
if (match === null) { break; }
const matchText = match[0];
const index = match.index;
const actualReplacement = this.applyMatchReplacement(replacement, match);
const actualReplacementLength = actualReplacement.length;
const delta = actualReplacementLength - (matchText.length > 0 ? matchText.length : -1);
text = `${text.substring(0, index)}${actualReplacement}${text.substring(index + matchText.length)}`;
pattern.lastIndex += delta;
if (actualReplacementLength > 0) {
sourceMap.insert(index, ...(new Array(actualReplacementLength).fill(0)));
sourceMap.combine(index - 1 + actualReplacementLength, matchText.length);
} else {
sourceMap.combine(index, matchText.length);
}
}
return text;
}
/**
* Applies the replacement string for a given regular expression match.
* @param replacement The replacement string that follows the format of the standard
* JavaScript regular expression replacement string.
* @param match A match object returned from RegExp.match.
* @return A new string with the pattern replacement applied.
*/
static applyMatchReplacement(replacement, match) {
const pattern = this._matchReplacementPattern;
pattern.lastIndex = 0;
return replacement.replace(pattern, (g0, g1, g2) => {
if (typeof g1 !== 'undefined') {
const matchIndex = Number.parseInt(g1, 10);
if (matchIndex >= 1 && matchIndex <= match.length) {
return match[matchIndex];
}
} else if (typeof g2 !== 'undefined') {
const {groups} = match;
if (typeof groups === 'object' && groups !== null && Object.prototype.hasOwnProperty.call(groups, g2)) {
return groups[g2];
}
} else {
switch (g0) {
case '$': return '$';
case '&': return match[0];
case '`': return replacement.substring(0, match.index);
case '\'': return replacement.substring(match.index + g0.length);
}
}
return g0;
});
}
}
// eslint-disable-next-line no-underscore-dangle
RegexUtil._matchReplacementPattern = /\$(?:\$|&|`|'|(\d\d?)|<([^>]*)>)/g;

90
ext/js/language/translator.js
View File

@ -17,6 +17,7 @@
/* global
* Deinflector
* RegexUtil
* TextSourceMap
*/
@ -534,24 +535,6 @@ class Translator {
);
}
_removeUsedDefinitions(definitions, termInfoMap, usedDefinitions) {
for (let i = 0, ii = definitions.length; i < ii; ++i) {
const definition = definitions[i];
const {expression, reading} = definition;
const expressionMap = termInfoMap.get(expression);
if (
typeof expressionMap !== 'undefined' &&
typeof expressionMap.get(reading) !== 'undefined'
) {
usedDefinitions.add(definition);
} else {
definitions.splice(i, 1);
--i;
--ii;
}
}
}
_getUniqueDefinitionTags(definitions) {
const definitionTagsMap = new Map();
for (const {definitionTags} of definitions) {
@ -794,16 +777,6 @@ class Translator {
// Simple helpers
_scoreToTermFrequency(score) {
if (score > 0) {
return 'popular';
} else if (score < 0) {
return 'rare';
} else {
return 'normal';
}
}
_getNameBase(name) {
const pos = name.indexOf(':');
return (pos >= 0 ? name.substring(0, pos) : name);
@ -974,14 +947,6 @@ class Translator {
// Reduction functions
_getTermTagsScoreSum(termTags) {
let result = 0;
for (const {score} of termTags) {
result += score;
}
return result;
}
_getSourceTermMatchCountSum(definitions) {
let result = 0;
for (const {sourceTermExactMatchCount} of definitions) {
@ -1408,59 +1373,8 @@ class Translator {
_applyTextReplacements(text, sourceMap, replacements) {
for (const {pattern, replacement} of replacements) {
text = this._applyTextReplacement(text, sourceMap, pattern, replacement);
text = RegexUtil.applyTextReplacement(text, sourceMap, pattern, replacement);
}
return text;
}
_applyTextReplacement(text, sourceMap, pattern, replacement) {
const isGlobal = pattern.global;
if (isGlobal) { pattern.lastIndex = 0; }
for (let loop = true; loop; loop = isGlobal) {
const match = pattern.exec(text);
if (match === null) { break; }
const matchText = match[0];
const index = match.index;
const actualReplacement = this._applyMatchReplacement(replacement, match);
const actualReplacementLength = actualReplacement.length;
const delta = actualReplacementLength - (matchText.length > 0 ? matchText.length : -1);
text = `${text.substring(0, index)}${actualReplacement}${text.substring(index + matchText.length)}`;
pattern.lastIndex += delta;
if (actualReplacementLength > 0) {
sourceMap.insert(index, ...(new Array(actualReplacementLength).fill(0)));
sourceMap.combine(index - 1 + actualReplacementLength, matchText.length);
} else {
sourceMap.combine(index, matchText.length);
}
}
return text;
}
_applyMatchReplacement(replacement, match) {
const pattern = /\$(?:\$|&|`|'|(\d\d?)|<([^>]*)>)/g;
return replacement.replace(pattern, (g0, g1, g2) => {
if (typeof g1 !== 'undefined') {
const matchIndex = Number.parseInt(g1, 10);
if (matchIndex >= 1 && matchIndex <= match.length) {
return match[matchIndex];
}
} else if (typeof g2 !== 'undefined') {
const {groups} = match;
if (typeof groups === 'object' && groups !== null && Object.prototype.hasOwnProperty.call(groups, g2)) {
return groups[g2];
}
} else {
switch (g0) {
case '$': return '$';
case '&': return match[0];
case '`': return replacement.substring(0, match.index);
case '\'': return replacement.substring(match.index + g0.length);
}
}
return g0;
});
}
}

1
ext/sw.js
View File

@ -36,6 +36,7 @@ self.importScripts(
'/js/dom/simple-dom-parser.js',
'/js/general/cache-map.js',
'/js/general/object-property-accessor.js',
'/js/general/regex-util.js',
'/js/general/text-source-map.js',
'/js/language/deinflector.js',
'/js/language/dictionary-data-util.js',

1
test/test-translator.js
View File

@ -44,6 +44,7 @@ async function createVM() {
'js/language/dictionary-importer.js',
'js/data/database.js',
'js/language/dictionary-database.js',
'js/general/regex-util.js',
'js/general/text-source-map.js',
'js/language/deinflector.js',
'js/language/translator.js'