alex/yomichan
1
Fork 0
Code Releases Activity

TranslatorVM (#1548)

Browse Source 
* Add TranslatorVM

* Update test-translator.js
This commit is contained in:
toasted-nutbread 2021-03-22 22:54:24 -04:00 committed by GitHub
parent 89ec1c7572
commit 7a8d359aa2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 186 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

175
dev/translator-vm.js Normal file
View File

@ -0,0 +1,175 @@
/*
* Copyright (C) 2021 Yomichan Authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const fs = require('fs');
const path = require('path');
const assert = require('assert');
const {DatabaseVM} = require('./database-vm');
const {createDictionaryArchive} = require('./util');
function clone(value) {
return JSON.parse(JSON.stringify(value));
}
class TranslatorVM extends DatabaseVM {
constructor() {
super();
this._japaneseUtil = null;
this._translator = null;
this._AnkiNoteData = null;
this._dictionaryName = null;
}
get translator() {
return this._translator;
}
async prepare(dictionaryDirectory, dictionaryName) {
this.execute([
'js/core.js',
'js/data/anki-note-data.js',
'js/data/database.js',
'js/data/json-schema.js',
'js/general/cache-map.js',
'js/general/regex-util.js',
'js/general/text-source-map.js',
'js/language/deinflector.js',
'js/language/dictionary-data-util.js',
'js/language/dictionary-importer.js',
'js/language/dictionary-database.js',
'js/language/japanese-util.js',
'js/language/translator.js',
'js/media/media-util.js'
]);
const [
DictionaryImporter,
DictionaryDatabase,
JapaneseUtil,
Translator,
AnkiNoteData
] = this.get([
'DictionaryImporter',
'DictionaryDatabase',
'JapaneseUtil',
'Translator',
'AnkiNoteData'
]);
// Dictionary
this._dictionaryName = dictionaryName;
const testDictionary = createDictionaryArchive(dictionaryDirectory, dictionaryName);
const testDictionaryContent = await testDictionary.generateAsync({type: 'string'});
// Setup database
const dictionaryImporter = new DictionaryImporter();
const dictionaryDatabase = new DictionaryDatabase();
await dictionaryDatabase.prepare();
const {errors} = await dictionaryImporter.importDictionary(
dictionaryDatabase,
testDictionaryContent,
{prefixWildcardsSupported: true},
() => {}
);
assert.deepStrictEqual(errors.length, 0);
// Setup translator
this._japaneseUtil = new JapaneseUtil(null);
this._translator = new Translator({
japaneseUtil: this._japaneseUtil,
database: dictionaryDatabase
});
const deinflectionReasions = JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'ext', 'data/deinflect.json')));
this._translator.prepare(deinflectionReasions);
// Assign properties
this._AnkiNoteData = AnkiNoteData;
}
createTestAnkiNoteData(definition, mode) {
const marker = '{marker}';
const data = {
definition,
resultOutputMode: mode,
mode: 'mode',
glossaryLayoutMode: 'default',
compactTags: false,
context: {
url: 'url:',
sentence: {text: '', offset: 0},
documentTitle: 'title'
},
injectedMedia: null
};
const AnkiNoteData = this._AnkiNoteData;
return new AnkiNoteData(this._japaneseUtil, marker, data).createPublic();
}
buildOptions(optionsPresets, optionsArray) {
const dictionaryName = this._dictionaryName;
const options = {};
if (!Array.isArray(optionsArray)) { optionsArray = [optionsArray]; }
for (const entry of optionsArray) {
switch (typeof entry) {
case 'string':
if (!Object.prototype.hasOwnProperty.call(optionsPresets, entry)) {
throw new Error('Invalid options preset');
}
Object.assign(options, clone(optionsPresets[entry]));
break;
case 'object':
Object.assign(options, clone(entry));
break;
default:
throw new Error('Invalid options type');
}
}
// Construct regex
if (Array.isArray(options.textReplacements)) {
options.textReplacements = options.textReplacements.map((value) => {
if (Array.isArray(value)) {
value = value.map(({pattern, flags, replacement}) => ({pattern: new RegExp(pattern, flags), replacement}));
}
return value;
});
}
// Update structure
const placeholder = '${title}';
if (options.mainDictionary === placeholder) {
options.mainDictionary = dictionaryName;
}
let {enabledDictionaryMap} = options;
if (Array.isArray(enabledDictionaryMap)) {
for (const entry of enabledDictionaryMap) {
if (entry[0] === placeholder) {
entry[0] = dictionaryName;
}
}
enabledDictionaryMap = new Map(enabledDictionaryMap);
options.enabledDictionaryMap = enabledDictionaryMap;
}
return options;
}
}
module.exports = {
TranslatorVM
};

151
test/test-translator.js
View File

@ -18,150 +18,21 @@
const fs = require('fs');
const path = require('path');
const assert = require('assert');
const {createDictionaryArchive, testMain} = require('../dev/util');
const {DatabaseVM} = require('../dev/database-vm');
const {testMain} = require('../dev/util');
const {TranslatorVM} = require('../dev/translator-vm');
function createTestDictionaryArchive(dictionary, dictionaryName) {
const dictionaryDirectory = path.join(__dirname, 'data', 'dictionaries', dictionary);
return createDictionaryArchive(dictionaryDirectory, dictionaryName);
}
function clone(value) {
return JSON.parse(JSON.stringify(value));
}
async function createVM() {
// Set up VM
const vm = new DatabaseVM();
vm.execute([
'js/core.js',
'js/data/anki-note-data.js',
'js/data/database.js',
'js/data/json-schema.js',
'js/general/cache-map.js',
'js/general/regex-util.js',
'js/general/text-source-map.js',
'js/language/deinflector.js',
'js/language/dictionary-data-util.js',
'js/language/dictionary-importer.js',
'js/language/dictionary-database.js',
'js/language/japanese-util.js',
'js/language/translator.js',
'js/media/media-util.js'
]);
const [
DictionaryImporter,
DictionaryDatabase,
JapaneseUtil,
Translator,
AnkiNoteData
] = vm.get([
'DictionaryImporter',
'DictionaryDatabase',
'JapaneseUtil',
'Translator',
'AnkiNoteData'
]);
// Dictionary
const testDictionary = createTestDictionaryArchive('valid-dictionary2');
const testDictionaryContent = await testDictionary.generateAsync({type: 'string'});
// Setup database
const dictionaryImporter = new DictionaryImporter();
const dictionaryDatabase = new DictionaryDatabase();
await dictionaryDatabase.prepare();
const {result, errors} = await dictionaryImporter.importDictionary(
dictionaryDatabase,
testDictionaryContent,
{prefixWildcardsSupported: true},
() => {}
);
assert.deepStrictEqual(errors.length, 0);
// Setup translator
const japaneseUtil = new JapaneseUtil(null);
const translator = new Translator({japaneseUtil, database: dictionaryDatabase});
const deinflectionReasions = JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'ext', 'data/deinflect.json')));
translator.prepare(deinflectionReasions);
// Note data creation
const createPublicAnkiNoteData = (marker, data) => new AnkiNoteData(japaneseUtil, marker, data).createPublic();
// Done
return {vm, translator, dictionary: result, createPublicAnkiNoteData};
}
function buildOptions(optionsPresets, optionsArray, dictionaryTitle) {
const options = {};
if (!Array.isArray(optionsArray)) { optionsArray = [optionsArray]; }
for (const entry of optionsArray) {
switch (typeof entry) {
case 'string':
if (!Object.prototype.hasOwnProperty.call(optionsPresets, entry)) {
throw new Error('Invalid options preset');
}
Object.assign(options, clone(optionsPresets[entry]));
break;
case 'object':
Object.assign(options, clone(entry));
break;
default:
throw new Error('Invalid options type');
}
}
// Construct regex
if (Array.isArray(options.textReplacements)) {
options.textReplacements = options.textReplacements.map((value) => {
if (Array.isArray(value)) {
value = value.map(({pattern, flags, replacement}) => ({pattern: new RegExp(pattern, flags), replacement}));
}
return value;
});
}
// Update structure
const placeholder = '${title}';
if (options.mainDictionary === placeholder) {
options.mainDictionary = dictionaryTitle;
}
let {enabledDictionaryMap} = options;
if (Array.isArray(enabledDictionaryMap)) {
for (const entry of enabledDictionaryMap) {
if (entry[0] === placeholder) {
entry[0] = dictionaryTitle;
}
}
enabledDictionaryMap = new Map(enabledDictionaryMap);
options.enabledDictionaryMap = enabledDictionaryMap;
}
return options;
}
async function main() {
const write = (process.argv[2] === '--write');
const {translator, dictionary: {title}, createPublicAnkiNoteData} = await createVM();
const createTestAnkiNoteData = (definition, mode) => createPublicAnkiNoteData('{marker}', {
definition,
resultOutputMode: mode,
mode: 'mode',
glossaryLayoutMode: 'default',
compactTags: false,
context: {
url: 'url:',
sentence: {text: '', offset: 0},
documentTitle: 'title'
},
injectedMedia: null
});
const translatorVM = new TranslatorVM();
const dictionaryDirectory = path.join(__dirname, 'data', 'dictionaries', 'valid-dictionary2');
await translatorVM.prepare(dictionaryDirectory, 'Test Dictionary 2');
const testInputsFilePath = path.join(__dirname, 'data', 'translator-test-inputs.json');
const {optionsPresets, tests} = JSON.parse(fs.readFileSync(testInputsFilePath, {encoding: 'utf8'}));
@ -182,9 +53,9 @@ async function main() {
case 'findTerms':
{
const {name, mode, text} = test;
const options = buildOptions(optionsPresets, test.options, title);
const [definitions, length] = clone(await translator.findTerms(mode, text, options));
const noteDataList = mode !== 'simple' ? clone(definitions.map((definition) => createTestAnkiNoteData(clone(definition), mode))) : null;
const options = translatorVM.buildOptions(optionsPresets, test.options);
const [definitions, length] = clone(await translatorVM.translator.findTerms(mode, text, options));
const noteDataList = mode !== 'simple' ? clone(definitions.map((definition) => translatorVM.createTestAnkiNoteData(clone(definition), mode))) : null;
actualResults1.push({name, length, definitions});
actualResults2.push({name, noteDataList});
if (!write) {
@ -197,9 +68,9 @@ async function main() {
case 'findKanji':
{
const {name, text} = test;
const options = buildOptions(optionsPresets, test.options, title);
const definitions = clone(await translator.findKanji(text, options));
const noteDataList = clone(definitions.map((definition) => createTestAnkiNoteData(clone(definition), null)));
const options = translatorVM.buildOptions(optionsPresets, test.options);
const definitions = clone(await translatorVM.translator.findKanji(text, options));
const noteDataList = clone(definitions.map((definition) => translatorVM.createTestAnkiNoteData(clone(definition), null)));
actualResults1.push({name, definitions});
actualResults2.push({name, noteDataList});
if (!write) {