Improve translator test data (#1531)
* Organize files * Split translator test data into two files * Add test for AnkiNoteData
This commit is contained in:
toasted-nutbread
GitHub
parent
9ad9961ca8
commit
cba45b5e30
File diff suppressed because it is too large
Load Diff
295
test/data/translator-test-inputs.json
Normal file
295
test/data/translator-test-inputs.json
Normal file
@ -0,0 +1,295 @@
|
|||||||
|
{
|
||||||
|
"optionsPresets": {
|
||||||
|
"kanji": {
|
||||||
|
"enabledDictionaryMap": [
|
||||||
|
[
|
||||||
|
"${title}",
|
||||||
|
{
|
||||||
|
"index": 0,
|
||||||
|
"priority": 0
|
||||||
|
}
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"default": {
|
||||||
|
"wildcard": null,
|
||||||
|
"mainDictionary": "${title}",
|
||||||
|
"alphanumeric": false,
|
||||||
|
"convertHalfWidthCharacters": false,
|
||||||
|
"convertNumericCharacters": false,
|
||||||
|
"convertAlphabeticCharacters": false,
|
||||||
|
"convertHiraganaToKatakana": false,
|
||||||
|
"convertKatakanaToHiragana": false,
|
||||||
|
"collapseEmphaticSequences": false,
|
||||||
|
"textReplacements": [
|
||||||
|
null
|
||||||
|
],
|
||||||
|
"enabledDictionaryMap": [
|
||||||
|
[
|
||||||
|
"${title}",
|
||||||
|
{
|
||||||
|
"index": 0,
|
||||||
|
"priority": 0,
|
||||||
|
"allowSecondarySearches": false
|
||||||
|
}
|
||||||
|
]
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"tests": [
|
||||||
|
{
|
||||||
|
"name": "Basic kanji test 1",
|
||||||
|
"func": "findKanji",
|
||||||
|
"text": "打",
|
||||||
|
"options": "kanji"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Basic kanji test 2",
|
||||||
|
"func": "findKanji",
|
||||||
|
"text": "込",
|
||||||
|
"options": "kanji"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Missing kanji test",
|
||||||
|
"func": "findKanji",
|
||||||
|
"text": "画",
|
||||||
|
"options": "kanji"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Find term using expression text 1",
|
||||||
|
"func": "findTerms",
|
||||||
|
"mode": "split",
|
||||||
|
"text": "打",
|
||||||
|
"options": "default"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Find term using expression text 2",
|
||||||
|
"func": "findTerms",
|
||||||
|
"mode": "split",
|
||||||
|
"text": "打つ",
|
||||||
|
"options": "default"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Find term using expression text 3",
|
||||||
|
"func": "findTerms",
|
||||||
|
"mode": "split",
|
||||||
|
"text": "打ち込む",
|
||||||
|
"options": "default"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Find term using expression text 4",
|
||||||
|
"func": "findTerms",
|
||||||
|
"mode": "split",
|
||||||
|
"text": "画像",
|
||||||
|
"options": "default"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Find term using reading 1",
|
||||||
|
"func": "findTerms",
|
||||||
|
"mode": "split",
|
||||||
|
"text": "だ",
|
||||||
|
"options": "default"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Find term using reading 2",
|
||||||
|
"func": "findTerms",
|
||||||
|
"mode": "split",
|
||||||
|
"text": "ダース",
|
||||||
|
"options": "default"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Find term using reading 3",
|
||||||
|
"func": "findTerms",
|
||||||
|
"mode": "split",
|
||||||
|
"text": "うつ",
|
||||||
|
"options": "default"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Find term using reading 4",
|
||||||
|
"func": "findTerms",
|
||||||
|
"mode": "split",
|
||||||
|
"text": "ぶつ",
|
||||||
|
"options": "default"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Find term using reading 5",
|
||||||
|
"func": "findTerms",
|
||||||
|
"mode": "split",
|
||||||
|
"text": "うちこむ",
|
||||||
|
"options": "default"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Find term using reading 6",
|
||||||
|
"func": "findTerms",
|
||||||
|
"mode": "split",
|
||||||
|
"text": "ぶちこむ",
|
||||||
|
"options": "default"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Find term using reading 7",
|
||||||
|
"func": "findTerms",
|
||||||
|
"mode": "split",
|
||||||
|
"text": "がぞう",
|
||||||
|
"options": "default"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Missing term 1",
|
||||||
|
"func": "findTerms",
|
||||||
|
"mode": "split",
|
||||||
|
"text": "為る",
|
||||||
|
"options": "default"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Missing term ２",
|
||||||
|
"func": "findTerms",
|
||||||
|
"mode": "split",
|
||||||
|
"text": "する",
|
||||||
|
"options": "default"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Search using different modes",
|
||||||
|
"func": "findTerms",
|
||||||
|
"mode": "simple",
|
||||||
|
"text": "打ち込む",
|
||||||
|
"options": "default"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Search using different modes",
|
||||||
|
"func": "findTerms",
|
||||||
|
"mode": "group",
|
||||||
|
"text": "打ち込む",
|
||||||
|
"options": "default"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Search using different modes",
|
||||||
|
"func": "findTerms",
|
||||||
|
"mode": "merge",
|
||||||
|
"text": "打ち込む",
|
||||||
|
"options": "default"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Search inflected term",
|
||||||
|
"func": "findTerms",
|
||||||
|
"mode": "split",
|
||||||
|
"text": "打ち込んでいませんでした",
|
||||||
|
"options": "default"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Ignore text inside parentheses",
|
||||||
|
"func": "findTerms",
|
||||||
|
"mode": "split",
|
||||||
|
"text": "打(う)ち込(こ)む",
|
||||||
|
"options": [
|
||||||
|
"default",
|
||||||
|
{
|
||||||
|
"alphanumeric": true,
|
||||||
|
"textReplacements": [
|
||||||
|
null,
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"pattern": "\\(([^)]*)(?:\\)|$)",
|
||||||
|
"flags": "g",
|
||||||
|
"replacement": ""
|
||||||
|
}
|
||||||
|
]
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Remove parentheses around text",
|
||||||
|
"func": "findTerms",
|
||||||
|
"mode": "split",
|
||||||
|
"text": "(打)(ち)(込)(む)",
|
||||||
|
"options": [
|
||||||
|
"default",
|
||||||
|
{
|
||||||
|
"alphanumeric": true,
|
||||||
|
"textReplacements": [
|
||||||
|
null,
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"pattern": "\\(([^)]*)(?:\\)|$)",
|
||||||
|
"flags": "g",
|
||||||
|
"replacement": "$1"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Test non-empty replacement",
|
||||||
|
"func": "findTerms",
|
||||||
|
"mode": "split",
|
||||||
|
"text": "test",
|
||||||
|
"options": [
|
||||||
|
"default",
|
||||||
|
{
|
||||||
|
"alphanumeric": true,
|
||||||
|
"textReplacements": [
|
||||||
|
null,
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"pattern": "test",
|
||||||
|
"flags": "g",
|
||||||
|
"replacement": "よみ"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Test non-empty replacement at end",
|
||||||
|
"func": "findTerms",
|
||||||
|
"mode": "split",
|
||||||
|
"text": "つtest",
|
||||||
|
"options": [
|
||||||
|
"default",
|
||||||
|
{
|
||||||
|
"alphanumeric": true,
|
||||||
|
"textReplacements": [
|
||||||
|
null,
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"pattern": "test",
|
||||||
|
"flags": "g",
|
||||||
|
"replacement": "よみ"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Test non-empty replacement at start",
|
||||||
|
"func": "findTerms",
|
||||||
|
"mode": "split",
|
||||||
|
"text": "testました",
|
||||||
|
"options": [
|
||||||
|
"default",
|
||||||
|
{
|
||||||
|
"alphanumeric": true,
|
||||||
|
"textReplacements": [
|
||||||
|
null,
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"pattern": "test",
|
||||||
|
"flags": "g",
|
||||||
|
"replacement": "よみ"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Search merged mode with non-primary definitions",
|
||||||
|
"func": "findTerms",
|
||||||
|
"mode": "merge",
|
||||||
|
"text": "うちこむ",
|
||||||
|
"options": "default"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
30799
test/data/translator-test-results-note-data1.json
Normal file
30799
test/data/translator-test-results-note-data1.json
Normal file
File diff suppressed because it is too large
Load Diff
23894
test/data/translator-test-results.json
Normal file
23894
test/data/translator-test-results.json
Normal file
File diff suppressed because it is too large
Load Diff
@ -37,28 +37,32 @@ async function createVM() {
|
|||||||
const vm = new DatabaseVM();
|
const vm = new DatabaseVM();
|
||||||
vm.execute([
|
vm.execute([
|
||||||
'js/core.js',
|
'js/core.js',
|
||||||
'js/general/cache-map.js',
|
'js/data/anki-note-data.js',
|
||||||
'js/language/japanese-util.js',
|
|
||||||
'js/data/json-schema.js',
|
|
||||||
'js/media/media-util.js',
|
|
||||||
'js/language/dictionary-importer.js',
|
|
||||||
'js/data/database.js',
|
'js/data/database.js',
|
||||||
'js/language/dictionary-database.js',
|
'js/data/json-schema.js',
|
||||||
|
'js/general/cache-map.js',
|
||||||
'js/general/regex-util.js',
|
'js/general/regex-util.js',
|
||||||
'js/general/text-source-map.js',
|
'js/general/text-source-map.js',
|
||||||
'js/language/deinflector.js',
|
'js/language/deinflector.js',
|
||||||
'js/language/translator.js'
|
'js/language/dictionary-data-util.js',
|
||||||
|
'js/language/dictionary-importer.js',
|
||||||
|
'js/language/dictionary-database.js',
|
||||||
|
'js/language/japanese-util.js',
|
||||||
|
'js/language/translator.js',
|
||||||
|
'js/media/media-util.js'
|
||||||
]);
|
]);
|
||||||
const [
|
const [
|
||||||
DictionaryImporter,
|
DictionaryImporter,
|
||||||
DictionaryDatabase,
|
DictionaryDatabase,
|
||||||
JapaneseUtil,
|
JapaneseUtil,
|
||||||
Translator
|
Translator,
|
||||||
|
AnkiNoteData
|
||||||
] = vm.get([
|
] = vm.get([
|
||||||
'DictionaryImporter',
|
'DictionaryImporter',
|
||||||
'DictionaryDatabase',
|
'DictionaryDatabase',
|
||||||
'JapaneseUtil',
|
'JapaneseUtil',
|
||||||
'Translator'
|
'Translator',
|
||||||
|
'AnkiNoteData'
|
||||||
]);
|
]);
|
||||||
|
|
||||||
// Dictionary
|
// Dictionary
|
||||||
@ -85,8 +89,11 @@ async function createVM() {
|
|||||||
const deinflectionReasions = JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'ext', 'data/deinflect.json')));
|
const deinflectionReasions = JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'ext', 'data/deinflect.json')));
|
||||||
translator.prepare(deinflectionReasions);
|
translator.prepare(deinflectionReasions);
|
||||||
|
|
||||||
|
// Note data creation
|
||||||
|
const createPublicAnkiNoteData = (marker, data) => new AnkiNoteData(japaneseUtil, marker, data).createPublic();
|
||||||
|
|
||||||
// Done
|
// Done
|
||||||
return {vm, translator, dictionary: result};
|
return {vm, translator, dictionary: result, createPublicAnkiNoteData};
|
||||||
}
|
}
|
||||||
|
|
||||||
function buildOptions(optionsPresets, optionsArray, dictionaryTitle) {
|
function buildOptions(optionsPresets, optionsArray, dictionaryTitle) {
|
||||||
@ -140,37 +147,64 @@ function buildOptions(optionsPresets, optionsArray, dictionaryTitle) {
|
|||||||
|
|
||||||
async function main() {
|
async function main() {
|
||||||
const write = (process.argv[2] === '--write');
|
const write = (process.argv[2] === '--write');
|
||||||
const {translator, dictionary: {title}} = await createVM();
|
const {translator, dictionary: {title}, createPublicAnkiNoteData} = await createVM();
|
||||||
|
|
||||||
const dataFilePath = path.join(__dirname, 'data', 'test-translator-data.json');
|
const createTestAnkiNoteData = (definition, mode) => createPublicAnkiNoteData('{marker}', {
|
||||||
const data = JSON.parse(fs.readFileSync(dataFilePath, {encoding: 'utf8'}));
|
definition,
|
||||||
const {optionsPresets, tests} = data;
|
resultOutputMode: mode,
|
||||||
for (const test of tests) {
|
mode: 'mode',
|
||||||
|
glossaryLayoutMode: 'default',
|
||||||
|
compactTags: false,
|
||||||
|
context: {
|
||||||
|
url: 'url:',
|
||||||
|
sentence: {text: '', offset: 0},
|
||||||
|
documentTitle: 'title'
|
||||||
|
},
|
||||||
|
injectedMedia: null
|
||||||
|
});
|
||||||
|
|
||||||
|
const testInputsFilePath = path.join(__dirname, 'data', 'translator-test-inputs.json');
|
||||||
|
const {optionsPresets, tests} = JSON.parse(fs.readFileSync(testInputsFilePath, {encoding: 'utf8'}));
|
||||||
|
|
||||||
|
const testResults1FilePath = path.join(__dirname, 'data', 'translator-test-results.json');
|
||||||
|
const expectedResults1 = JSON.parse(fs.readFileSync(testResults1FilePath, {encoding: 'utf8'}));
|
||||||
|
const actualResults1 = [];
|
||||||
|
|
||||||
|
const testResults2FilePath = path.join(__dirname, 'data', 'translator-test-results-note-data1.json');
|
||||||
|
const expectedResults2 = JSON.parse(fs.readFileSync(testResults2FilePath, {encoding: 'utf8'}));
|
||||||
|
const actualResults2 = [];
|
||||||
|
|
||||||
|
for (let i = 0, ii = tests.length; i < ii; ++i) {
|
||||||
|
const test = tests[i];
|
||||||
|
const expected1 = expectedResults1[i];
|
||||||
|
const expected2 = expectedResults2[i];
|
||||||
switch (test.func) {
|
switch (test.func) {
|
||||||
case 'findTerms':
|
case 'findTerms':
|
||||||
{
|
{
|
||||||
const {mode, text} = test;
|
const {name, mode, text} = test;
|
||||||
const options = buildOptions(optionsPresets, test.options, title);
|
const options = buildOptions(optionsPresets, test.options, title);
|
||||||
const [definitions, length] = clone(await translator.findTerms(mode, text, options));
|
const [definitions, length] = clone(await translator.findTerms(mode, text, options));
|
||||||
if (write) {
|
const noteDataList = clone(definitions.map((definition) => createTestAnkiNoteData(clone(definition), null)));
|
||||||
test.expected = {length, definitions};
|
actualResults1.push({name, length, definitions});
|
||||||
} else {
|
actualResults2.push({name, noteDataList});
|
||||||
const {expected} = test;
|
if (!write) {
|
||||||
assert.deepStrictEqual(length, expected.length);
|
assert.deepStrictEqual(length, expected1.length);
|
||||||
assert.deepStrictEqual(definitions, expected.definitions);
|
assert.deepStrictEqual(definitions, expected1.definitions);
|
||||||
|
assert.deepStrictEqual(noteDataList, expected2.noteDataList);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
case 'findKanji':
|
case 'findKanji':
|
||||||
{
|
{
|
||||||
const {text} = test;
|
const {name, text} = test;
|
||||||
const options = buildOptions(optionsPresets, test.options, title);
|
const options = buildOptions(optionsPresets, test.options, title);
|
||||||
const definitions = clone(await translator.findKanji(text, options));
|
const definitions = clone(await translator.findKanji(text, options));
|
||||||
if (write) {
|
const noteDataList = clone(definitions.map((definition) => createTestAnkiNoteData(clone(definition), null)));
|
||||||
test.expected = {definitions};
|
actualResults1.push({name, definitions});
|
||||||
} else {
|
actualResults2.push({name, noteDataList});
|
||||||
const {expected} = test;
|
if (!write) {
|
||||||
assert.deepStrictEqual(definitions, expected.definitions);
|
assert.deepStrictEqual(definitions, expected1.definitions);
|
||||||
|
assert.deepStrictEqual(noteDataList, expected2.noteDataList);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
@ -178,7 +212,9 @@ async function main() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (write) {
|
if (write) {
|
||||||
fs.writeFileSync(dataFilePath, JSON.stringify(data, null, 4), {encoding: 'utf8'});
|
// Use 2 indent instead of 4 to save a bit of file size
|
||||||
|
fs.writeFileSync(testResults1FilePath, JSON.stringify(actualResults1, null, 2), {encoding: 'utf8'});
|
||||||
|
fs.writeFileSync(testResults2FilePath, JSON.stringify(actualResults2, null, 2), {encoding: 'utf8'});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user