Json schema validation improvements (#1697)

* Create new JsonSchema class

* Add proxy handler

* Update tests

* Update validation scripts

* Update backend

* Update audio downloader

* Update options util

* Update dictionary importer

* Update json schema file reference

* Remove old json-schema.js

* Rename new json-schema.js

* Update file names

* Rename class
This commit is contained in:
toasted-nutbread 2021-05-22 15:45:20 -04:00 committed by GitHub
parent b48052ff32
commit d16739a83a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 794 additions and 709 deletions

View File

@ -27,7 +27,7 @@ vm.execute([
'js/general/cache-map.js', 'js/general/cache-map.js',
'js/data/json-schema.js' 'js/data/json-schema.js'
]); ]);
const JsonSchemaValidator = vm.get('JsonSchemaValidator'); const JsonSchema = vm.get('JsonSchema');
function readSchema(relativeFileName) { function readSchema(relativeFileName) {
@ -46,7 +46,7 @@ async function validateDictionaryBanks(zip, fileNameFormat, schema) {
if (!file) { break; } if (!file) { break; }
const data = JSON.parse(await file.async('string')); const data = JSON.parse(await file.async('string'));
new JsonSchemaValidator().validate(data, schema); new JsonSchema(schema).validate(data);
++index; ++index;
} }
@ -61,7 +61,7 @@ async function validateDictionary(archive, schemas) {
const index = JSON.parse(await indexFile.async('string')); const index = JSON.parse(await indexFile.async('string'));
const version = index.format || index.version; const version = index.format || index.version;
new JsonSchemaValidator().validate(index, schemas.index); new JsonSchema(schemas.index).validate(index);
await validateDictionaryBanks(archive, 'term_bank_?.json', version === 1 ? schemas.termBankV1 : schemas.termBankV3); await validateDictionaryBanks(archive, 'term_bank_?.json', version === 1 ? schemas.termBankV1 : schemas.termBankV3);
await validateDictionaryBanks(archive, 'term_meta_bank_?.json', schemas.termMetaBankV3); await validateDictionaryBanks(archive, 'term_meta_bank_?.json', schemas.termMetaBankV3);

View File

@ -25,7 +25,7 @@ vm.execute([
'js/general/cache-map.js', 'js/general/cache-map.js',
'js/data/json-schema.js' 'js/data/json-schema.js'
]); ]);
const JsonSchemaValidator = vm.get('JsonSchemaValidator'); const JsonSchema = vm.get('JsonSchema');
function main() { function main() {
@ -47,7 +47,7 @@ function main() {
console.log(`Validating ${dataFileName}...`); console.log(`Validating ${dataFileName}...`);
const dataSource = fs.readFileSync(dataFileName, {encoding: 'utf8'}); const dataSource = fs.readFileSync(dataFileName, {encoding: 'utf8'});
const data = JSON.parse(dataSource); const data = JSON.parse(dataSource);
new JsonSchemaValidator().validate(data, schema); new JsonSchema(schema).validate(data);
const end = performance.now(); const end = performance.now();
console.log(`No issues detected (${((end - start) / 1000).toFixed(2)}s)`); console.log(`No issues detected (${((end - start) / 1000).toFixed(2)}s)`);
} catch (e) { } catch (e) {

View File

@ -24,7 +24,6 @@
* DictionaryDatabase * DictionaryDatabase
* Environment * Environment
* JapaneseUtil * JapaneseUtil
* JsonSchemaValidator
* Mecab * Mecab
* MediaUtil * MediaUtil
* ObjectPropertyAccessor * ObjectPropertyAccessor
@ -58,7 +57,6 @@ class Backend {
clipboardReader: this._clipboardReader clipboardReader: this._clipboardReader
}); });
this._options = null; this._options = null;
this._profileConditionsSchemaValidator = new JsonSchemaValidator();
this._profileConditionsSchemaCache = []; this._profileConditionsSchemaCache = [];
this._profileConditionsUtil = new ProfileConditionsUtil(); this._profileConditionsUtil = new ProfileConditionsUtil();
this._defaultAnkiFieldTemplates = null; this._defaultAnkiFieldTemplates = null;
@ -1018,7 +1016,7 @@ class Backend {
this._profileConditionsSchemaCache.push(schema); this._profileConditionsSchemaCache.push(schema);
} }
if (conditionGroups.length > 0 && this._profileConditionsSchemaValidator.isValid(optionsContext, schema)) { if (conditionGroups.length > 0 && schema.isValid(optionsContext)) {
return profile; return profile;
} }
++index; ++index;
@ -1029,7 +1027,6 @@ class Backend {
_clearProfileConditionsSchemaCache() { _clearProfileConditionsSchemaCache() {
this._profileConditionsSchemaCache = []; this._profileConditionsSchemaCache = [];
this._profileConditionsSchemaValidator.clearCache();
} }
_checkLastError() { _checkLastError() {

View File

@ -15,6 +15,10 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. * along with this program. If not, see <https://www.gnu.org/licenses/>.
*/ */
/* global
* JsonSchema
*/
/** /**
* Utility class to help processing profile conditions. * Utility class to help processing profile conditions.
*/ */
@ -109,11 +113,13 @@ class ProfileConditionsUtil {
default: anyOf.push({allOf}); break; default: anyOf.push({allOf}); break;
} }
} }
let schema;
switch (anyOf.length) { switch (anyOf.length) {
case 0: return {}; case 0: schema = {}; break;
case 1: return anyOf[0]; case 1: schema = anyOf[0]; break;
default: return {anyOf}; default: schema = {anyOf}; break;
} }
return new JsonSchema(schema);
} }
/** /**

File diff suppressed because it is too large Load Diff

View File

@ -16,19 +16,19 @@
*/ */
/* global /* global
* JsonSchemaValidator * JsonSchema
* TemplatePatcher * TemplatePatcher
*/ */
class OptionsUtil { class OptionsUtil {
constructor() { constructor() {
this._schemaValidator = new JsonSchemaValidator();
this._templatePatcher = null; this._templatePatcher = null;
this._optionsSchema = null; this._optionsSchema = null;
} }
async prepare() { async prepare() {
this._optionsSchema = await this._fetchAsset('/data/schemas/options-schema.json', true); const schema = await this._fetchAsset('/data/schemas/options-schema.json', true);
this._optionsSchema = new JsonSchema(schema);
} }
async update(options) { async update(options) {
@ -87,7 +87,7 @@ class OptionsUtil {
options = await this._applyUpdates(options, this._getVersionUpdates()); options = await this._applyUpdates(options, this._getVersionUpdates());
// Validation // Validation
options = this._schemaValidator.getValidValueOrDefault(this._optionsSchema, options); options = this._optionsSchema.getValidValueOrDefault(options);
// Result // Result
return options; return options;
@ -135,17 +135,17 @@ class OptionsUtil {
getDefault() { getDefault() {
const optionsVersion = this._getVersionUpdates().length; const optionsVersion = this._getVersionUpdates().length;
const options = this._schemaValidator.getValidValueOrDefault(this._optionsSchema); const options = this._optionsSchema.getValidValueOrDefault();
options.version = optionsVersion; options.version = optionsVersion;
return options; return options;
} }
createValidatingProxy(options) { createValidatingProxy(options) {
return this._schemaValidator.createProxy(options, this._optionsSchema); return this._optionsSchema.createProxy(options);
} }
validate(options) { validate(options) {
return this._schemaValidator.validate(options, this._optionsSchema); return this._optionsSchema.validate(options);
} }
// Legacy profile updating // Legacy profile updating

View File

@ -17,14 +17,13 @@
/* global /* global
* JSZip * JSZip
* JsonSchemaValidator * JsonSchema
* MediaUtil * MediaUtil
*/ */
class DictionaryImporter { class DictionaryImporter {
constructor() { constructor() {
this._schemas = new Map(); this._schemas = new Map();
this._jsonSchemaValidator = new JsonSchemaValidator();
} }
async importDictionary(dictionaryDatabase, archiveSource, details, onProgress) { async importDictionary(dictionaryDatabase, archiveSource, details, onProgress) {
@ -235,22 +234,27 @@ class DictionaryImporter {
return schemaPromise; return schemaPromise;
} }
schemaPromise = this._fetchJsonAsset(fileName); schemaPromise = this._createSchema(fileName);
this._schemas.set(fileName, schemaPromise); this._schemas.set(fileName, schemaPromise);
return schemaPromise; return schemaPromise;
} }
async _createSchema(fileName) {
const schema = await this._fetchJsonAsset(fileName);
return new JsonSchema(schema);
}
_validateJsonSchema(value, schema, fileName) { _validateJsonSchema(value, schema, fileName) {
try { try {
this._jsonSchemaValidator.validate(value, schema); schema.validate(value);
} catch (e) { } catch (e) {
throw this._formatSchemaError(e, fileName); throw this._formatSchemaError(e, fileName);
} }
} }
_formatSchemaError(e, fileName) { _formatSchemaError(e, fileName) {
const valuePathString = this._getSchemaErrorPathString(e.info.valuePath, 'dictionary'); const valuePathString = this._getSchemaErrorPathString(e.valuePath, 'dictionary');
const schemaPathString = this._getSchemaErrorPathString(e.info.schemaPath, 'schema'); const schemaPathString = this._getSchemaErrorPathString(e.schemaPath, 'schema');
const e2 = new Error(`Dictionary has invalid data in '${fileName}' for value '${valuePathString}', validated against '${schemaPathString}': ${e.message}`); const e2 = new Error(`Dictionary has invalid data in '${fileName}' for value '${valuePathString}', validated against '${schemaPathString}': ${e.message}`);
e2.data = e; e2.data = e;
@ -260,16 +264,16 @@ class DictionaryImporter {
_getSchemaErrorPathString(infoList, base='') { _getSchemaErrorPathString(infoList, base='') {
let result = base; let result = base;
for (const [part] of infoList) { for (const {path} of infoList) {
switch (typeof part) { switch (typeof path) {
case 'string': case 'string':
if (result.length > 0) { if (result.length > 0) {
result += '.'; result += '.';
} }
result += part; result += path;
break; break;
case 'number': case 'number':
result += `[${part}]`; result += `[${path}]`;
break; break;
} }
} }

View File

@ -16,7 +16,7 @@
*/ */
/* global /* global
* JsonSchemaValidator * JsonSchema
* NativeSimpleDOMParser * NativeSimpleDOMParser
* SimpleDOMParser * SimpleDOMParser
*/ */
@ -26,7 +26,7 @@ class AudioDownloader {
this._japaneseUtil = japaneseUtil; this._japaneseUtil = japaneseUtil;
this._requestBuilder = requestBuilder; this._requestBuilder = requestBuilder;
this._customAudioListSchema = null; this._customAudioListSchema = null;
this._schemaValidator = null; this._customAudioListSchema = null;
this._getInfoHandlers = new Map([ this._getInfoHandlers = new Map([
['jpod101', this._getInfoJpod101.bind(this)], ['jpod101', this._getInfoJpod101.bind(this)],
['jpod101-alternate', this._getInfoJpod101Alternate.bind(this)], ['jpod101-alternate', this._getInfoJpod101Alternate.bind(this)],
@ -222,11 +222,11 @@ class AudioDownloader {
const responseJson = await response.json(); const responseJson = await response.json();
if (this._customAudioListSchema === null) {
const schema = await this._getCustomAudioListSchema(); const schema = await this._getCustomAudioListSchema();
if (this._schemaValidator === null) { this._customAudioListSchema = new JsonSchema(schema);
this._schemaValidator = new JsonSchemaValidator();
} }
this._schemaValidator.validate(responseJson, schema); this._customAudioListSchema.validate(responseJson);
const results = []; const results = [];
for (const {url: url2, name} of responseJson.audioSources) { for (const {url: url2, name} of responseJson.audioSources) {

View File

@ -25,19 +25,19 @@ vm.execute([
'js/general/cache-map.js', 'js/general/cache-map.js',
'js/data/json-schema.js' 'js/data/json-schema.js'
]); ]);
const JsonSchemaValidator = vm.get('JsonSchemaValidator'); const JsonSchema = vm.get('JsonSchema');
function schemaValidate(schema, value) { function schemaValidate(schema, value) {
return new JsonSchemaValidator().isValid(value, schema); return new JsonSchema(schema).isValid(value);
} }
function getValidValueOrDefault(schema, value) { function getValidValueOrDefault(schema, value) {
return new JsonSchemaValidator().getValidValueOrDefault(schema, value); return new JsonSchema(schema).getValidValueOrDefault(value);
} }
function createProxy(schema, value) { function createProxy(schema, value) {
return new JsonSchemaValidator().createProxy(value, schema); return new JsonSchema(schema).createProxy(value);
} }
function clone(value) { function clone(value) {

View File

@ -27,12 +27,7 @@ vm.execute([
'js/data/json-schema.js', 'js/data/json-schema.js',
'js/background/profile-conditions-util.js' 'js/background/profile-conditions-util.js'
]); ]);
const [JsonSchemaValidator, ProfileConditionsUtil] = vm.get(['JsonSchemaValidator', 'ProfileConditionsUtil']); const [ProfileConditionsUtil] = vm.get(['ProfileConditionsUtil']);
function schemaValidate(value, schema) {
return new JsonSchemaValidator().isValid(value, schema);
}
function testNormalizeContext() { function testNormalizeContext() {
@ -1081,12 +1076,12 @@ function testSchemas() {
const profileConditionsUtil = new ProfileConditionsUtil(); const profileConditionsUtil = new ProfileConditionsUtil();
const schema = profileConditionsUtil.createSchema(conditionGroups); const schema = profileConditionsUtil.createSchema(conditionGroups);
if (typeof expectedSchema !== 'undefined') { if (typeof expectedSchema !== 'undefined') {
vm.assert.deepStrictEqual(schema, expectedSchema); vm.assert.deepStrictEqual(schema.schema, expectedSchema);
} }
if (Array.isArray(inputs)) { if (Array.isArray(inputs)) {
for (const {expected, context} of inputs) { for (const {expected, context} of inputs) {
const normalizedContext = profileConditionsUtil.normalizeContext(context); const normalizedContext = profileConditionsUtil.normalizeContext(context);
const actual = schemaValidate(normalizedContext, schema); const actual = schema.isValid(normalizedContext);
assert.strictEqual(actual, expected); assert.strictEqual(actual, expected);
} }
} }