alex/yomichan
1
Fork 0
Code Releases Activity

Json schema validation improvements (#1697)

Browse Source 
* Create new JsonSchema class

* Add proxy handler

* Update tests

* Update validation scripts

* Update backend

* Update audio downloader

* Update options util

* Update dictionary importer

* Update json schema file reference

* Remove old json-schema.js

* Rename new json-schema.js

* Update file names

* Rename class
This commit is contained in:
toasted-nutbread 2021-05-22 15:45:20 -04:00 committed by GitHub
parent b48052ff32
commit d16739a83a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 794 additions and 709 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
dev/dictionary-validate.js
View File

@ -27,7 +27,7 @@ vm.execute([
'js/general/cache-map.js',
'js/data/json-schema.js'
]);
const JsonSchemaValidator = vm.get('JsonSchemaValidator');
const JsonSchema = vm.get('JsonSchema');
function readSchema(relativeFileName) {
@ -46,7 +46,7 @@ async function validateDictionaryBanks(zip, fileNameFormat, schema) {
if (!file) { break; }
const data = JSON.parse(await file.async('string'));
new JsonSchemaValidator().validate(data, schema);
new JsonSchema(schema).validate(data);
++index;
}
@ -61,7 +61,7 @@ async function validateDictionary(archive, schemas) {
const index = JSON.parse(await indexFile.async('string'));
const version = index.format || index.version;
new JsonSchemaValidator().validate(index, schemas.index);
new JsonSchema(schemas.index).validate(index);
await validateDictionaryBanks(archive, 'term_bank_?.json', version === 1 ? schemas.termBankV1 : schemas.termBankV3);
await validateDictionaryBanks(archive, 'term_meta_bank_?.json', schemas.termMetaBankV3);

4
dev/schema-validate.js
View File

@ -25,7 +25,7 @@ vm.execute([
'js/general/cache-map.js',
'js/data/json-schema.js'
]);
const JsonSchemaValidator = vm.get('JsonSchemaValidator');
const JsonSchema = vm.get('JsonSchema');
function main() {
@ -47,7 +47,7 @@ function main() {
console.log(`Validating ${dataFileName}...`);
const dataSource = fs.readFileSync(dataFileName, {encoding: 'utf8'});
const data = JSON.parse(dataSource);
new JsonSchemaValidator().validate(data, schema);
new JsonSchema(schema).validate(data);
const end = performance.now();
console.log(`No issues detected (${((end - start) / 1000).toFixed(2)}s)`);
} catch (e) {

5
ext/js/background/backend.js
View File

@ -24,7 +24,6 @@
* DictionaryDatabase
* Environment
* JapaneseUtil
* JsonSchemaValidator
* Mecab
* MediaUtil
* ObjectPropertyAccessor
@ -58,7 +57,6 @@ class Backend {
clipboardReader: this._clipboardReader
});
this._options = null;
this._profileConditionsSchemaValidator = new JsonSchemaValidator();
this._profileConditionsSchemaCache = [];
this._profileConditionsUtil = new ProfileConditionsUtil();
this._defaultAnkiFieldTemplates = null;
@ -1018,7 +1016,7 @@ class Backend {
this._profileConditionsSchemaCache.push(schema);
}
if (conditionGroups.length > 0 && this._profileConditionsSchemaValidator.isValid(optionsContext, schema)) {
if (conditionGroups.length > 0 && schema.isValid(optionsContext)) {
return profile;
}
++index;
@ -1029,7 +1027,6 @@ class Backend {
_clearProfileConditionsSchemaCache() {
this._profileConditionsSchemaCache = [];
this._profileConditionsSchemaValidator.clearCache();
}
_checkLastError() {

12
ext/js/background/profile-conditions-util.js
View File

@ -15,6 +15,10 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
/* global
* JsonSchema
*/
/**
* Utility class to help processing profile conditions.
*/
@ -109,11 +113,13 @@ class ProfileConditionsUtil {
default: anyOf.push({allOf}); break;
}
}
let schema;
switch (anyOf.length) {
case 0: return {};
case 1: return anyOf[0];
default: return {anyOf};
case 0: schema = {}; break;
case 1: schema = anyOf[0]; break;
default: schema = {anyOf}; break;
}
return new JsonSchema(schema);
}
/**

1407
ext/js/data/json-schema.js
View File

File diff suppressed because it is too large Load Diff

14
ext/js/data/options-util.js
View File

@ -16,19 +16,19 @@
*/
/* global
* JsonSchemaValidator
* JsonSchema
* TemplatePatcher
*/
class OptionsUtil {
constructor() {
this._schemaValidator = new JsonSchemaValidator();
this._templatePatcher = null;
this._optionsSchema = null;
}
async prepare() {
this._optionsSchema = await this._fetchAsset('/data/schemas/options-schema.json', true);
const schema = await this._fetchAsset('/data/schemas/options-schema.json', true);
this._optionsSchema = new JsonSchema(schema);
}
async update(options) {
@ -87,7 +87,7 @@ class OptionsUtil {
options = await this._applyUpdates(options, this._getVersionUpdates());
// Validation
options = this._schemaValidator.getValidValueOrDefault(this._optionsSchema, options);
options = this._optionsSchema.getValidValueOrDefault(options);
// Result
return options;
@ -135,17 +135,17 @@ class OptionsUtil {
getDefault() {
const optionsVersion = this._getVersionUpdates().length;
const options = this._schemaValidator.getValidValueOrDefault(this._optionsSchema);
const options = this._optionsSchema.getValidValueOrDefault();
options.version = optionsVersion;
return options;
}
createValidatingProxy(options) {
return this._schemaValidator.createProxy(options, this._optionsSchema);
return this._optionsSchema.createProxy(options);
}
validate(options) {
return this._schemaValidator.validate(options, this._optionsSchema);
return this._optionsSchema.validate(options);
}
// Legacy profile updating

24
ext/js/language/dictionary-importer.js
View File

@ -17,14 +17,13 @@
/* global
* JSZip
* JsonSchemaValidator
* JsonSchema
* MediaUtil
*/
class DictionaryImporter {
constructor() {
this._schemas = new Map();
this._jsonSchemaValidator = new JsonSchemaValidator();
}
async importDictionary(dictionaryDatabase, archiveSource, details, onProgress) {
@ -235,22 +234,27 @@ class DictionaryImporter {
return schemaPromise;
}
schemaPromise = this._fetchJsonAsset(fileName);
schemaPromise = this._createSchema(fileName);
this._schemas.set(fileName, schemaPromise);
return schemaPromise;
}
async _createSchema(fileName) {
const schema = await this._fetchJsonAsset(fileName);
return new JsonSchema(schema);
}
_validateJsonSchema(value, schema, fileName) {
try {
this._jsonSchemaValidator.validate(value, schema);
schema.validate(value);
} catch (e) {
throw this._formatSchemaError(e, fileName);
}
}
_formatSchemaError(e, fileName) {
const valuePathString = this._getSchemaErrorPathString(e.info.valuePath, 'dictionary');
const schemaPathString = this._getSchemaErrorPathString(e.info.schemaPath, 'schema');
const valuePathString = this._getSchemaErrorPathString(e.valuePath, 'dictionary');
const schemaPathString = this._getSchemaErrorPathString(e.schemaPath, 'schema');
const e2 = new Error(`Dictionary has invalid data in '${fileName}' for value '${valuePathString}', validated against '${schemaPathString}': ${e.message}`);
e2.data = e;
@ -260,16 +264,16 @@ class DictionaryImporter {
_getSchemaErrorPathString(infoList, base='') {
let result = base;
for (const [part] of infoList) {
switch (typeof part) {
for (const {path} of infoList) {
switch (typeof path) {
case 'string':
if (result.length > 0) {
result += '.';
}
result += part;
result += path;
break;
case 'number':
result += `[${part}]`;
result += `[${path}]`;
break;
}
}

12
ext/js/media/audio-downloader.js
View File

@ -16,7 +16,7 @@
*/
/* global
* JsonSchemaValidator
* JsonSchema
* NativeSimpleDOMParser
* SimpleDOMParser
*/
@ -26,7 +26,7 @@ class AudioDownloader {
this._japaneseUtil = japaneseUtil;
this._requestBuilder = requestBuilder;
this._customAudioListSchema = null;
this._schemaValidator = null;
this._customAudioListSchema = null;
this._getInfoHandlers = new Map([
['jpod101', this._getInfoJpod101.bind(this)],
['jpod101-alternate', this._getInfoJpod101Alternate.bind(this)],
@ -222,11 +222,11 @@ class AudioDownloader {
const responseJson = await response.json();
const schema = await this._getCustomAudioListSchema();
if (this._schemaValidator === null) {
this._schemaValidator = new JsonSchemaValidator();
if (this._customAudioListSchema === null) {
const schema = await this._getCustomAudioListSchema();
this._customAudioListSchema = new JsonSchema(schema);
}
this._schemaValidator.validate(responseJson, schema);
this._customAudioListSchema.validate(responseJson);
const results = [];
for (const {url: url2, name} of responseJson.audioSources) {

8
test/test-json-schema.js
View File

@ -25,19 +25,19 @@ vm.execute([
'js/general/cache-map.js',
'js/data/json-schema.js'
]);
const JsonSchemaValidator = vm.get('JsonSchemaValidator');
const JsonSchema = vm.get('JsonSchema');
function schemaValidate(schema, value) {
return new JsonSchemaValidator().isValid(value, schema);
return new JsonSchema(schema).isValid(value);
}
function getValidValueOrDefault(schema, value) {
return new JsonSchemaValidator().getValidValueOrDefault(schema, value);
return new JsonSchema(schema).getValidValueOrDefault(value);
}
function createProxy(schema, value) {
return new JsonSchemaValidator().createProxy(value, schema);
return new JsonSchema(schema).createProxy(value);
}
function clone(value) {

11
test/test-profile-conditions-util.js
View File

@ -27,12 +27,7 @@ vm.execute([
'js/data/json-schema.js',
'js/background/profile-conditions-util.js'
]);
const [JsonSchemaValidator, ProfileConditionsUtil] = vm.get(['JsonSchemaValidator', 'ProfileConditionsUtil']);
function schemaValidate(value, schema) {
return new JsonSchemaValidator().isValid(value, schema);
}
const [ProfileConditionsUtil] = vm.get(['ProfileConditionsUtil']);
function testNormalizeContext() {
@ -1081,12 +1076,12 @@ function testSchemas() {
const profileConditionsUtil = new ProfileConditionsUtil();
const schema = profileConditionsUtil.createSchema(conditionGroups);
if (typeof expectedSchema !== 'undefined') {
vm.assert.deepStrictEqual(schema, expectedSchema);
vm.assert.deepStrictEqual(schema.schema, expectedSchema);
}
if (Array.isArray(inputs)) {
for (const {expected, context} of inputs) {
const normalizedContext = profileConditionsUtil.normalizeContext(context);
const actual = schemaValidate(normalizedContext, schema);
const actual = schema.isValid(normalizedContext);
assert.strictEqual(actual, expected);
}
}