diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 7059d6f23..dc71356bb 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,2 +1,4 @@ +# Mon Jul 29 19:08:32 2024 -0400 - effigies@gmail.com - chore(fmt): deno fmt [ignore-rev] +92cbef9624663b719b7b4c22b6104669eb518f09 # Sun Jul 28 22:01:32 2024 -0400 - effigies@gmail.com - chore(fmt): deno fmt [ignore-rev] 1bccd0a2d09f1b7ff6a26d99c7845bf59908c421 \ No newline at end of file diff --git a/bids-validator/src/deps/ajv.ts b/bids-validator/src/deps/ajv.ts new file mode 100644 index 000000000..d42c5ad69 --- /dev/null +++ b/bids-validator/src/deps/ajv.ts @@ -0,0 +1 @@ +export { Ajv, type JSONSchemaType, type ValidateFunction } from 'https://esm.sh/ajv@8.16.0' diff --git a/bids-validator/src/issues/list.ts b/bids-validator/src/issues/list.ts index 8d31a9cc8..8af740533 100644 --- a/bids-validator/src/issues/list.ts +++ b/bids-validator/src/issues/list.ts @@ -42,6 +42,10 @@ export const filenameIssues: IssueDefinitionRecord = { severity: 'warning', reason: 'A data files JSON sidecar is missing a key listed as recommended.', }, + JSON_SCHEMA_VALIDATION_ERROR: { + severity: 'error', + reason: 'Invalid JSON sidecar file. The sidecar is not formatted according the schema.', + }, TSV_ERROR: { severity: 'error', reason: 'generic place holder for errors from tsv files', diff --git a/bids-validator/src/schema/applyRules.ts b/bids-validator/src/schema/applyRules.ts index 0e1231ac7..27593b39e 100644 --- a/bids-validator/src/schema/applyRules.ts +++ b/bids-validator/src/schema/applyRules.ts @@ -165,13 +165,14 @@ function schemaObjectTypeCheck( if (value === 'n/a') { return true } + if ('anyOf' in schemaObject) { return schemaObject.anyOf.some((x) => schemaObjectTypeCheck(x, value, schema)) } if ('enum' in schemaObject && schemaObject.enum) { return schemaObject.enum.some((x) => x === value) } - // @ts-expect-error + const format = schemaObject.format // @ts-expect-error ? schema.objects.formats[schemaObject.format] @@ -415,7 +416,8 @@ function evalJsonCheck( for (const [key, requirement] of Object.entries(rule.fields)) { const severity = getFieldSeverity(requirement, context) // @ts-expect-error - const keyName = schema.objects.metadata[key].name + const metadataDef = schema.objects.metadata[key] + const keyName: string = metadataDef.name if (severity && severity !== 'ignore' && !(keyName in context.sidecar)) { if (requirement.issue?.code && requirement.issue?.message) { context.issues.add({ @@ -440,6 +442,53 @@ function evalJsonCheck( ]) } } + + /* Regardless of if key is required/recommended/optional, we do no + * further valdiation if it is not present in sidecar. + */ + if (!(keyName in context.sidecar)) { + return + } + + let originFileKey = '' + if (keyName in context.sidecarKeyOrigin) { + originFileKey = `${context.sidecarKeyOrigin[keyName]}:${keyName}` + } else { + logger.warning( + `sidecarKeyOrigin map failed to initialize for ${context.file.path} on key ${keyName}. Validation caching not active for this key.`, + ) + } + + if (context.dataset.sidecarKeyValidated.has(originFileKey)) { + return + } + + const validate = context.dataset.ajv.compile(metadataDef) + const result = validate(context.sidecar[keyName]) + if (result === false) { + const evidenceBase = `Failed for this file.key: ${originFileKey} Schema path: ${schemaPath}` + if (!validate.errors) { + context.issues.addNonSchemaIssue('JSON_SCHEMA_VALIDATION_ERROR', [ + { + ...context.file, + evidence: evidenceBase, + }, + ]) + } else { + for (let error of validate.errors) { + const message = 'message' in error ? `message: ${error['message']}` : '' + context.issues.addNonSchemaIssue('JSON_SCHEMA_VALIDATION_ERROR', [ + { + ...context.file, + evidence: `${evidenceBase} ${message}`, + }, + ]) + } + } + } + if (originFileKey) { + context.dataset.sidecarKeyValidated.add(originFileKey) + } } } diff --git a/bids-validator/src/schema/context.ts b/bids-validator/src/schema/context.ts index dd5660934..1328c0479 100644 --- a/bids-validator/src/schema/context.ts +++ b/bids-validator/src/schema/context.ts @@ -16,6 +16,9 @@ import { loadHeader } from '../files/nifti.ts' import { buildAssociations } from './associations.ts' import { ValidatorOptions } from '../setup/options.ts' import { logger } from '../utils/logger.ts' +import { Ajv, JSONSchemaType, ValidateFunction } from '../deps/ajv.ts' +import { memoize } from '../utils/memoize.ts' +import { Schema } from '../types/schema.ts' export class BIDSContextDataset implements ContextDataset { dataset_description: Record @@ -25,13 +28,22 @@ export class BIDSContextDataset implements ContextDataset { ignored: any[] modalities: any[] subjects?: ContextDatasetSubjects + ajv: Ajv + sidecarKeyValidated: Set - constructor(options?: ValidatorOptions, description = {}) { + constructor(options?: ValidatorOptions, schema?: Schema, description = {}) { this.dataset_description = description this.files = [] this.tree = {} this.ignored = [] this.modalities = [] + this.ajv = new Ajv({ strictSchema: false }) + // @ts-expect-error + this.ajv.compile = memoize(this.ajv.compile) + this.sidecarKeyValidated = new Set() + if (schema) { + this.setCustomAjvFormats(schema) + } if (options) { this.options = options } @@ -44,6 +56,27 @@ export class BIDSContextDataset implements ContextDataset { this.dataset_description.DatasetType = 'raw' } } + + setCustomAjvFormats(schema: Schema): void { + if (typeof schema.objects.formats !== 'object') { + // logger.warning( + console.log( + `schema.objects.formats missing from schema, format validation disabled.`, + ) + return + } + const schemaFormats = schema.objects.formats + for (let key of Object.keys(schemaFormats)) { + if (typeof schemaFormats[key]['pattern'] !== 'string') { + // logger.warning( + console.log( + `schema.objects.formats.${key} pattern missing or invalid. Skipping this format for addition to context json validator`, + ) + continue + } + this.ajv.addFormat(key, schemaFormats[key]['pattern']) + } + } } export class BIDSContextDatasetSubjects implements ContextDatasetSubjects { @@ -78,6 +111,7 @@ export class BIDSContext implements Context { datatype: string modality: string sidecar: Record + sidecarKeyOrigin: Record json: object columns: ColumnsMap associations: ContextAssociations @@ -102,6 +136,7 @@ export class BIDSContext implements Context { this.datatype = '' this.modality = '' this.sidecar = {} + this.sidecarKeyOrigin = {} this.columns = new ColumnsMap() this.json = {} this.associations = {} as ContextAssociations @@ -166,6 +201,7 @@ export class BIDSContext implements Context { .then((text) => JSON.parse(text)) .catch((error) => {}) this.sidecar = { ...this.sidecar, ...json } + Object.keys(json).map((x) => this.sidecarKeyOrigin[x] = validSidecars[0].path) } const nextDir = fileTree.directories.find((directory) => { return this.file.path.startsWith(`${directory.path}/`) diff --git a/bids-validator/src/types/context.ts b/bids-validator/src/types/context.ts index e9c1c7843..4cf43022c 100644 --- a/bids-validator/src/types/context.ts +++ b/bids-validator/src/types/context.ts @@ -1,4 +1,5 @@ import { ValidatorOptions } from '../setup/options.ts' +import { Ajv } from '../deps/ajv.ts' export interface ContextDatasetSubjects { sub_dirs: string[] @@ -13,6 +14,8 @@ export interface ContextDataset { modalities: any[] subjects?: ContextDatasetSubjects options?: ValidatorOptions + ajv: Ajv + sidecarKeyValidated: Set } export interface ContextSubjectSessions { ses_dirs: string[] @@ -98,6 +101,7 @@ export interface Context { extension: string modality: string sidecar: Record + sidecarKeyOrigin: Record associations: ContextAssociations columns: object json: object diff --git a/bids-validator/src/types/schema.ts b/bids-validator/src/types/schema.ts index 6862cd527..b8a4f86f1 100644 --- a/bids-validator/src/types/schema.ts +++ b/bids-validator/src/types/schema.ts @@ -71,6 +71,7 @@ export interface GenericRule { format?: string required?: string index_columns?: string[] + metadata?: Record } export interface SchemaFields { @@ -81,7 +82,11 @@ export interface SchemaFields { interface SchemaType { type: string + format?: string enum?: string[] + items?: SchemaType[] + minItems?: number + maxItems?: number } interface AnyOf { diff --git a/bids-validator/src/utils/objectPathHandler.ts b/bids-validator/src/utils/objectPathHandler.ts index 91736280a..8abaa525e 100644 --- a/bids-validator/src/utils/objectPathHandler.ts +++ b/bids-validator/src/utils/objectPathHandler.ts @@ -9,6 +9,9 @@ export const hasProp = ( export const objectPathHandler = { get(target: unknown, property: string) { let res = target + if (typeof property === 'symbol') { + return res + } for (const prop of property.split('.')) { if (hasProp(res, prop)) { res = res[prop] diff --git a/bids-validator/src/validators/bids.ts b/bids-validator/src/validators/bids.ts index 1f2720011..ca448c99b 100644 --- a/bids-validator/src/validators/bids.ts +++ b/bids-validator/src/validators/bids.ts @@ -49,9 +49,9 @@ export async function validate( if (ddFile) { const description = await ddFile.text().then((text) => JSON.parse(text)) summary.dataProcessed = description.DatasetType === 'derivative' - dsContext = new BIDSContextDataset(options, description) + dsContext = new BIDSContextDataset(options, schema, description) } else { - dsContext = new BIDSContextDataset(options) + dsContext = new BIDSContextDataset(options, schema) issues.addNonSchemaIssue('MISSING_DATASET_DESCRIPTION', [] as IssueFile[]) } diff --git a/bids-validator/tests/data/bids-examples b/bids-validator/tests/data/bids-examples index ea3f7f65d..401f4cfc0 160000 --- a/bids-validator/tests/data/bids-examples +++ b/bids-validator/tests/data/bids-examples @@ -1 +1 @@ -Subproject commit ea3f7f65d68850630ab7c550cbe62aac9fceec47 +Subproject commit 401f4cfc0709be0e39c32766b79894160a8a3e45