diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5229fb2a..17b23239 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ jobs: - name: Download dependencies run: npm ci - name: Test & publish code coverage - uses: paambaati/codeclimate-action@v8.0.0 + uses: paambaati/codeclimate-action@v9.0.0 env: CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} with: diff --git a/.gitignore b/.gitignore index 55d6a413..0a1f28ec 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ dist/ spec_tests/*.txt spec_tests/temp*.json spec_tests/temp.spec.js +tests/temp.spec.js # Unit test / coverage reports htmlcov/ diff --git a/bids/validator/bidsHedTsvValidator.js b/bids/validator/bidsHedTsvValidator.js index 231ea14b..74bf9da2 100644 --- a/bids/validator/bidsHedTsvValidator.js +++ b/bids/validator/bidsHedTsvValidator.js @@ -82,6 +82,10 @@ export class BidsHedTsvValidator { * @private */ _validateHedColumn() { + if (this.tsvFile.hedColumnHedStrings.length === 0) { + // no HED column strings to validate + return [] + } return this.tsvFile.hedColumnHedStrings.flatMap((hedString, rowIndexMinusTwo) => this._validateHedColumnString(hedString, rowIndexMinusTwo + 2), ) @@ -230,13 +234,14 @@ export class BidsHedTsvParser { */ _parseHedRows(tsvHedRows) { const hedStrings = [] - - tsvHedRows.forEach((row, index) => { - const hedString = this._parseHedRow(row, index + 2) - if (hedString !== null) { - hedStrings.push(hedString) - } - }) + if (tsvHedRows.size > 0) { + tsvHedRows.forEach((row, index) => { + const hedString = this._parseHedRow(row, index + 2) + if (hedString !== null) { + hedStrings.push(hedString) + } + }) + } return hedStrings } @@ -248,13 +253,15 @@ export class BidsHedTsvParser { * @private */ _mergeEventRows(rowStrings) { - const groupedTsvRows = groupBy(rowStrings, (rowString) => rowString.onset) - const sortedOnsetTimes = Array.from(groupedTsvRows.keys()).sort((a, b) => a - b) const eventStrings = [] - for (const onset of sortedOnsetTimes) { - const onsetRows = groupedTsvRows.get(onset) - const onsetEventString = new BidsTsvEvent(this.tsvFile, onsetRows) - eventStrings.push(onsetEventString) + if (rowStrings.length > 0) { + const groupedTsvRows = groupBy(rowStrings, (rowString) => rowString.onset) + const sortedOnsetTimes = Array.from(groupedTsvRows.keys()).sort((a, b) => a - b) + for (const onset of sortedOnsetTimes) { + const onsetRows = groupedTsvRows.get(onset) + const onsetEventString = new BidsTsvEvent(this.tsvFile, onsetRows) + eventStrings.push(onsetEventString) + } } return eventStrings } diff --git a/common/issues/data.js b/common/issues/data.js index 2ef06151..2534ffbf 100644 --- a/common/issues/data.js +++ b/common/issues/data.js @@ -43,11 +43,26 @@ export default { level: 'error', message: stringTemplate`Invalid tag - "${'tag'}".`, }, + extraSlash: { + hedCode: 'TAG_INVALID', + level: 'error', + message: stringTemplate`Tag has extra slash at index ${'index'} of string "${'string'}".`, + }, + extraBlank: { + hedCode: 'TAG_INVALID', + level: 'error', + message: stringTemplate`Tag has extra blank at index ${'index'} of string "${'string'}".`, + }, extraCommaOrInvalid: { hedCode: 'TAG_INVALID', level: 'error', message: stringTemplate`Either "${'previousTag'}" contains a comma when it should not or "${'tag'}" is not a valid tag.`, }, + invalidTagPrefix: { + hedCode: 'TAG_NAMESPACE_PREFIX_INVALID', + level: 'error', + message: stringTemplate`Either tag prefix at index ${'index'} contains non-alphabetic characters or does not have an associated schema.`, + }, multipleUniqueTags: { hedCode: 'TAG_NOT_UNIQUE', level: 'error', diff --git a/parser/tokenizer.js b/parser/tokenizer.js index b308a9d9..58e20c21 100644 --- a/parser/tokenizer.js +++ b/parser/tokenizer.js @@ -1,28 +1,41 @@ +import { replaceTagNameWithPound } from '../utils/hedStrings' import { unicodeName } from 'unicode-name' - import { generateIssue } from '../common/issues/issues' -import { stringIsEmpty } from '../utils/string' -import { replaceTagNameWithPound } from '../utils/hedStrings' -const openingGroupCharacter = '(' -const closingGroupCharacter = ')' -const openingColumnCharacter = '{' -const closingColumnCharacter = '}' -const commaCharacter = ',' -const colonCharacter = ':' -const slashCharacter = '/' +const CHARACTERS = { + BLANK: ' ', + OPENING_GROUP: '(', + CLOSING_GROUP: ')', + OPENING_COLUMN: '{', + CLOSING_COLUMN: '}', + COMMA: ',', + COLON: ':', + SLASH: '/', +} + +function getTrimmedBounds(originalString) { + const start = originalString.search(/\S/) + const end = originalString.search(/\S\s*$/) + + if (start === -1) { + // The string contains only whitespace + return null + } + + return [start, end + 1] +} const invalidCharacters = new Set(['[', ']', '~', '"']) -const invalidCharactersOutsideOfValues = new Set([':']) -// C0 control codes +// Add control codes to invalidCharacters for (let i = 0x00; i <= 0x1f; i++) { invalidCharacters.add(String.fromCodePoint(i)) } -// DEL and C1 control codes for (let i = 0x7f; i <= 0x9f; i++) { invalidCharacters.add(String.fromCodePoint(i)) } +const invalidCharactersOutsideOfValues = new Set([':']) + /** * A specification for a tokenized substring. */ @@ -71,10 +84,10 @@ export class GroupSpec extends SubstringSpec { */ children - constructor(start, end) { + constructor(start, end, children) { super(start, end) - this.children = [] + this.children = children } } @@ -95,41 +108,27 @@ export class ColumnSpliceSpec extends SubstringSpec { } } +class TokenizerState { + constructor() { + this.currentToken = '' // Characters in the token currently being parsed + this.groupDepth = 0 + this.startingIndex = 0 // Starting index of this token + this.lastDelimiter = [undefined, -1] // Type and position of the last delimiter + this.librarySchema = '' + this.lastSlash = -1 // Position of the last slash in current token + this.currentGroupStack = [[]] + this.parenthesesStack = [] + } +} + /** * Class for tokenizing HED strings. */ export class HedStringTokenizer { - /** - * The HED string being parsed. - * @type {string} - */ - hedString - - syntaxIssues - - /** - * The current substring being parsed. - * @type {string} - */ - currentTag - - /** - * Whether we are currently closing a group. - * @type {boolean} - */ - closingGroup - - groupDepth - startingIndex - resetStartingIndex - slashFound - librarySchema - currentGroupStack - parenthesesStack - ignoringCharacters - constructor(hedString) { this.hedString = hedString + this.issues = [] + this.state = null } /** @@ -139,247 +138,260 @@ export class HedStringTokenizer { */ tokenize() { this.initializeTokenizer() - + // Empty strings cannot be tokenized + if (this.hedString.trim().length === 0) { + this.pushIssue('emptyTagFound', 0) + return [null, null, { syntax: this.issues }] + } for (let i = 0; i < this.hedString.length; i++) { const character = this.hedString.charAt(i) - this.tokenizeCharacter(i, character) - if (this.resetStartingIndex) { - this.resetStartingIndex = false - this.startingIndex = i + 1 - this.currentTag = '' + this.handleCharacter(i, character) + //this.tokenizeCharacter(i, character) + if (this.issues.length > 0) { + return [null, null, { syntax: this.issues }] } } - this.pushTag(this.hedString.length, true) - - if (this.columnSpliceIndex >= 0) { - this._pushSyntaxIssue('unclosedCurlyBrace', this.columnSpliceIndex) + this.finalizeTokenizer() + if (this.issues.length > 0) { + return [null, null, { syntax: this.issues }] + } else { + return [this.state.currentGroupStack.pop(), this.state.parenthesesStack.pop(), { syntax: [] }] } + } - this.unwindGroupStack() + resetToken(i) { + this.state.startingIndex = i + 1 + this.state.currentToken = '' + this.state.librarySchema = '' + this.state.lastSlash = '-1' + } - const tagSpecs = this.currentGroupStack.pop() - const groupSpecs = this.parenthesesStack.pop() - const issues = { - syntax: this.syntaxIssues, - conversion: [], + finalizeTokenizer() { + if (this.state.lastDelimiter[0] === CHARACTERS.OPENING_COLUMN) { + // Extra opening brace + this.pushIssue('unclosedCurlyBrace', this.state.lastDelimiter[1]) + } else if (this.state.lastDelimiter[0] === CHARACTERS.OPENING_GROUP) { + // Extra opening parenthesis + this.pushIssue('unclosedParentheses', this.state.lastDelimiter[1]) + } else if ( + this.state.lastDelimiter[0] === CHARACTERS.COMMA && + this.hedString.slice(this.state.lastDelimiter[1] + 1).trim().length === 0 + ) { + this.pushIssue('emptyTagFound', this.state.lastDelimiter[1]) // Extra comma + } else if (this.state.lastSlash >= 0 && this.hedString.slice(this.state.lastSlash + 1).trim().length === 0) { + this.pushIssue('extraSlash', this.state.lastSlash) // Extra slash + } else { + if (this.state.currentToken.trim().length > 0) { + this.pushTag(this.hedString.length) + } + this.unwindGroupStack() } - return [tagSpecs, groupSpecs, issues] } initializeTokenizer() { - this.syntaxIssues = [] - - this.currentTag = '' - this.groupDepth = 0 - this.startingIndex = 0 - this.resetStartingIndex = false - this.slashFound = false - this.librarySchema = '' - this.columnSpliceIndex = -1 - this.currentGroupStack = [[]] - this.parenthesesStack = [new GroupSpec(0, this.hedString.length)] - this.ignoringCharacters = false - this.closingGroup = false + this.issues = [] + this.state = new TokenizerState() + this.state.parenthesesStack = [new GroupSpec(0, this.hedString.length, [])] } - tokenizeCharacter(i, character) { - let dispatchTable - if (this.ignoringCharacters) { - dispatchTable = { - [closingGroupCharacter]: (i /* character */) => { - this.clearTag() - this.closingGroupCharacter(i) - }, - [commaCharacter]: (/*i, character */) => this.clearTag(), - } - } else { - dispatchTable = { - [openingGroupCharacter]: (i /* character */) => this.openingGroupCharacter(i), - [closingGroupCharacter]: (i /* character */) => { - this.pushTag(i, false) - this.closingGroupCharacter(i) - }, - [openingColumnCharacter]: (i /* character */) => this.openingColumnCharacter(i), - [closingColumnCharacter]: (i /* character */) => this.closingColumnCharacter(i), - [commaCharacter]: (i /* character */) => this.pushTag(i, false), - [colonCharacter]: (i, character) => this.colonCharacter(character), - [slashCharacter]: (i, character) => this.slashCharacter(character), - } - } - const characterHandler = dispatchTable[character] + handleCharacter(i, character) { + const characterHandler = { + [CHARACTERS.OPENING_GROUP]: () => this.handleOpeningGroup(i), + [CHARACTERS.CLOSING_GROUP]: () => this.handleClosingGroup(i), + [CHARACTERS.OPENING_COLUMN]: () => this.handleOpeningColumn(i), + [CHARACTERS.CLOSING_COLUMN]: () => this.handleClosingColumn(i), + [CHARACTERS.COMMA]: () => this.handleComma(i), + [CHARACTERS.COLON]: () => this.handleColon(i), + [CHARACTERS.SLASH]: () => this.handleSlash(i), + }[character] // Selects the character handler based on the value of character + if (characterHandler) { - characterHandler(i, character) + characterHandler() } else if (invalidCharacters.has(character)) { - this._pushInvalidCharacterIssue(character, i) + this.pushInvalidCharacterIssue(character, i) } else { - this.otherCharacter(character) + this.state.currentToken += character } } - openingGroupCharacter(i) { - this.currentGroupStack.push([]) - this.parenthesesStack.push(new GroupSpec(i)) - this.resetStartingIndex = true - this.groupDepth++ - } - - closingGroupCharacter(i) { - this.closingGroup = true - if (this.groupDepth <= 0) { - this._pushSyntaxIssue('unopenedParenthesis', i) + handleComma(i) { + if (this.state.lastDelimiter[0] === undefined && this.hedString.slice(0, i).length === 0) { + // Start of string empty + this.pushIssue('emptyTagFound', i) return } - this.closeGroup(i) + const trimmed = this.hedString.slice(this.state.lastDelimiter[1] + 1, i).trim() + if (this.state.lastDelimiter[0] === CHARACTERS.COMMA && trimmed.length === 0) { + // empty token after a previous comma + this.pushIssue('emptyTagFound', this.state.lastDelimiter[1]) // Check for empty group between commas + } else if (this.state.lastDelimiter[0] === CHARACTERS.OPENING_COLUMN) { + // Unclosed curly brace + this.pushIssue('unclosedCurlyBrace', this.state.lastDelimiter[1]) + } + if ( + [CHARACTERS.CLOSING_GROUP, CHARACTERS.CLOSING_COLUMN].includes(this.state.lastDelimiter[0]) && + trimmed.length > 0 + ) { + this.pushIssue('invalidTag', i, trimmed) + } else if (trimmed.length > 0) { + this.pushTag(i) + } else { + this.resetToken(i) + } + this.state.lastDelimiter = [CHARACTERS.COMMA, i] } - openingColumnCharacter(i) { - if (this.currentTag.length > 0) { - this._pushInvalidCharacterIssue(openingColumnCharacter, i) - this.ignoringCharacters = true - return + handleSlash(i) { + if (this.hedString.slice(0, i).trim().length === 0) { + // Slash at beginning of tag. + this.pushIssue('extraSlash', i) + } else if (this.state.lastSlash >= 0 && this.hedString.slice(this.state.lastSlash + 1, i).trim().length === 0) { + this.pushIssue('extraSlash', i) // Slashes with only blanks between + } else if (i > 0 && this.hedString.charAt(i - 1) === CHARACTERS.BLANK) { + // Blank before slash + this.pushIssue('extraBlank', i - 1) + } else if (i < this.hedString.length - 1 && this.hedString.charAt(i + 1) === CHARACTERS.BLANK) { + //Blank after + this.pushIssue('extraBlank', i + 1) + } else if (this.hedString.slice(i).trim().length === 0) { + this.pushIssue('extraSlash', this.state.startingIndex) + } else { + this.state.currentToken += CHARACTERS.SLASH + this.state.lastSlash = i } - if (this.columnSpliceIndex >= 0) { - this._pushSyntaxIssue('nestedCurlyBrace', i) + } + + handleOpeningGroup(i) { + if (this.state.lastDelimiter[0] === CHARACTERS.OPENING_COLUMN) { + this.pushIssue('unclosedCurlyBrace', this.state.lastDelimiter[1]) + } else { + this.state.currentGroupStack.push([]) + this.state.parenthesesStack.push(new GroupSpec(i, undefined, [])) + this.resetToken(i) + this.state.groupDepth++ + this.state.lastDelimiter = [CHARACTERS.OPENING_GROUP, i] } - this.columnSpliceIndex = i } - closingColumnCharacter(i) { - this.closingGroup = true - if (this.columnSpliceIndex < 0) { - this._pushSyntaxIssue('unopenedCurlyBrace', i) - return + handleClosingGroup(i) { + if ([CHARACTERS.OPENING_GROUP, CHARACTERS.COMMA].includes(this.state.lastDelimiter[0])) { + this.pushTag(i) } - if (!stringIsEmpty(this.currentTag)) { - this.currentGroupStack[this.groupDepth].push(new ColumnSpliceSpec(this.currentTag.trim(), this.startingIndex, i)) + if (this.state.groupDepth <= 0) { + // If the group depth is <= 0, it means there's no corresponding opening group. + this.pushIssue('unopenedParenthesis', i) + } else if (this.state.lastDelimiter[0] === CHARACTERS.OPENING_COLUMN) { + this.pushIssue('unclosedCurlyBrace', this.state.lastDelimiter[1]) } else { - this.syntaxIssues.push( - generateIssue('emptyCurlyBrace', { - string: this.hedString, - }), - ) + // Close the group by updating its bounds and moving it to the parent group. + this.closeGroup(i) + this.state.lastDelimiter = [CHARACTERS.CLOSING_GROUP, i] } - this.columnSpliceIndex = -1 - this.resetStartingIndex = true - this.slashFound = false } - colonCharacter(character) { - if (!this.slashFound && !this.librarySchema) { - this.librarySchema = this.currentTag - this.resetStartingIndex = true + handleOpeningColumn(i) { + if (this.state.currentToken.trim().length > 0) { + // In the middle of a token -- can't have an opening brace + this.pushInvalidCharacterIssue(CHARACTERS.OPENING_COLUMN, i) + } else if (this.state.lastDelimiter[0] === CHARACTERS.OPENING_COLUMN) { + // + this.pushIssue('nestedCurlyBrace', i) } else { - this.currentTag += character + this.state.lastDelimiter = [CHARACTERS.OPENING_COLUMN, i] } } - slashCharacter(character) { - this.slashFound = true - this.currentTag += character + handleClosingColumn(i) { + if (this.state.lastDelimiter[0] !== CHARACTERS.OPENING_COLUMN) { + // Column splice not in progress + this.pushIssue('unopenedCurlyBrace', i) + } else if (!this.state.currentToken.trim()) { + // Column slice cannot be empty + this.pushIssue('emptyCurlyBrace', i) + } else { + // Close column by updating bounds and moving it to the parent group, push a column splice on the stack. + this.state.currentGroupStack[this.state.groupDepth].push( + new ColumnSpliceSpec(this.state.currentToken.trim(), this.state.lastDelimiter[1], i), + ) + this.resetToken(i) + this.state.lastDelimiter = [CHARACTERS.CLOSING_COLUMN, i] + } } - otherCharacter(character) { - if (this.ignoringCharacters) { - return + handleColon(i) { + if (this.state.librarySchema || this.state.currentToken.trim().includes(CHARACTERS.BLANK)) { + // If colon has not been seen, it is a library. Ignore other colons. + this.state.currentToken += CHARACTERS.COLON + } else if (/[^A-Za-z]/.test(this.state.currentToken.trim())) { + this.pushIssue('invalidTagPrefix', i) + } else { + const lib = this.state.currentToken.trimStart() + this.resetToken(i) + this.state.librarySchema = lib } - this.currentTag += character - this.resetStartingIndex = stringIsEmpty(this.currentTag) } unwindGroupStack() { - // groupDepth is decremented in closeGroup. - // eslint-disable-next-line no-unmodified-loop-condition - while (this.groupDepth > 0) { - this._pushSyntaxIssue('unclosedParenthesis', this.parenthesesStack[this.parenthesesStack.length - 1].bounds[0]) + while (this.state.groupDepth > 0) { + this.pushIssue( + 'unclosedParenthesis', + this.state.parenthesesStack[this.state.parenthesesStack.length - 1].bounds[0], + ) this.closeGroup(this.hedString.length) } } - /** - * Push a tag to the current group. - * - * @param {number} i The current index. - * @param {boolean} isEndOfString Whether we are at the end of the string. - */ - pushTag(i, isEndOfString) { - if (stringIsEmpty(this.currentTag) && isEndOfString) { - return - } else if (this.closingGroup) { - this.closingGroup = false - } else if (stringIsEmpty(this.currentTag)) { - this.syntaxIssues.push(generateIssue('emptyTagFound', { index: i })) - } else if (this.columnSpliceIndex < 0) { - this._checkValueTagForInvalidCharacters() - this.currentGroupStack[this.groupDepth].push( - new TagSpec(this.currentTag.trim(), this.startingIndex, i, this.librarySchema), + pushTag(i) { + if (this.state.currentToken.trim().length == 0) { + this.pushIssue('emptyTagFound', i) + } else { + const bounds = getTrimmedBounds(this.state.currentToken) + this.state.currentGroupStack[this.state.groupDepth].push( + new TagSpec( + this.state.currentToken.trim(), + this.state.startingIndex + bounds[0], + this.state.startingIndex + bounds[1], + this.state.librarySchema, + ), ) + this.resetToken(i) } - this.resetStartingIndex = true - this.slashFound = false - this.librarySchema = '' - } - - clearTag() { - this.ignoringCharacters = false - this.resetStartingIndex = true - this.slashFound = false - this.librarySchema = '' } closeGroup(i) { - const groupSpec = this.parenthesesStack.pop() + const groupSpec = this.state.parenthesesStack.pop() groupSpec.bounds[1] = i + 1 - this.parenthesesStack[this.groupDepth - 1].children.push(groupSpec) - this.currentGroupStack[this.groupDepth - 1].push(this.currentGroupStack.pop()) - this.groupDepth-- + if (this.hedString.slice(groupSpec.bounds[0] + 1, i).trim().length === 0) { + //The group is empty + this.pushIssue('emptyTagFound', i) + } + this.state.parenthesesStack[this.state.groupDepth - 1].children.push(groupSpec) + this.state.currentGroupStack[this.state.groupDepth - 1].push(this.state.currentGroupStack.pop()) + this.state.groupDepth-- + //this.resetToken(i) } - /** - * Check an individual tag for invalid characters. - * - * @private - */ - _checkValueTagForInvalidCharacters() { - const formToCheck = replaceTagNameWithPound(this.currentTag) + checkValueTagForInvalidCharacters() { + const formToCheck = replaceTagNameWithPound(this.state.currentToken) for (let i = 0; i < formToCheck.length; i++) { const character = formToCheck.charAt(i) - if (!invalidCharactersOutsideOfValues.has(character)) { - continue + if (invalidCharactersOutsideOfValues.has(character)) { + this.pushInvalidCharacterIssue(character, this.state.startingIndex + i) } - this._pushInvalidCharacterIssue(character, this.startingIndex + i) } } - /** - * Push an issue to the syntax issue list. - * - * @param {string} issueCode The internal code of the issue to be pushed. - * @param {number} index The location of the issue. - * @private - */ - _pushSyntaxIssue(issueCode, index) { - this.syntaxIssues.push( - generateIssue(issueCode, { - index: index, - string: this.hedString, - }), - ) + pushIssue(issueCode, index) { + this.issues.push(generateIssue(issueCode, { index, string: this.hedString })) } - /** - * Push an invalid character issue to the syntax issue list. - * - * @param {string} character The illegal character to be reported. - * @param {number} index The location of the character. - * @private - */ - _pushInvalidCharacterIssue(character, index) { - this.syntaxIssues.push( - generateIssue('invalidCharacter', { - character: unicodeName(character), - index: index, - string: this.hedString, - }), + pushInvalidTag(issueCode, index, tag) { + this.issues.push(generateIssue(issueCode, { index, tag: tag, string: this.hedString })) + } + + pushInvalidCharacterIssue(character, index) { + this.issues.push( + generateIssue('invalidCharacter', { character: unicodeName(character), index, string: this.hedString }), ) } } diff --git a/parser/tokenizerOriginal.js b/parser/tokenizerOriginal.js new file mode 100644 index 00000000..068f29c9 --- /dev/null +++ b/parser/tokenizerOriginal.js @@ -0,0 +1,385 @@ +import { unicodeName } from 'unicode-name' + +import { generateIssue } from '../common/issues/issues' +import { stringIsEmpty } from '../utils/string' +import { replaceTagNameWithPound } from '../utils/hedStrings' + +const openingGroupCharacter = '(' +const closingGroupCharacter = ')' +const openingColumnCharacter = '{' +const closingColumnCharacter = '}' +const commaCharacter = ',' +const colonCharacter = ':' +const slashCharacter = '/' + +const invalidCharacters = new Set(['[', ']', '~', '"']) +const invalidCharactersOutsideOfValues = new Set([':']) +// C0 control codes +for (let i = 0x00; i <= 0x1f; i++) { + invalidCharacters.add(String.fromCodePoint(i)) +} +// DEL and C1 control codes +for (let i = 0x7f; i <= 0x9f; i++) { + invalidCharacters.add(String.fromCodePoint(i)) +} + +/** + * A specification for a tokenized substring. + */ +class SubstringSpec { + /** + * The starting and ending bounds of the substring. + * @type {number[]} + */ + bounds + + constructor(start, end) { + this.bounds = [start, end] + } +} + +/** + * A specification for a tokenized tag. + */ +class TagSpec extends SubstringSpec { + /** + * The tag this spec represents. + * @type {string} + */ + tag + /** + * The schema prefix for this tag, if any. + * @type {string} + */ + library + + constructor(tag, start, end, librarySchema) { + super(start, end) + + this.tag = tag.trim() + this.library = librarySchema + } +} + +/** + * A specification for a tokenized tag group. + */ +class GroupSpec extends SubstringSpec { + /** + * The child group specifications. + * @type {GroupSpec[]} + */ + children + + constructor(start, end, children) { + super(start, end) + + this.children = children + } +} + +/** + * A specification for a tokenized column splice template. + */ +class ColumnSpliceSpec extends SubstringSpec { + /** + * The column name this spec refers to. + * @type {string} + */ + columnName + + constructor(name, start, end) { + super(start, end) + + this.columnName = name.trim() + } +} + +/** + * Class for tokenizing HED strings. + */ +export class HedStringTokenizerOriginal { + /** + * The HED string being parsed. + * @type {string} + */ + hedString + + syntaxIssues + + /** + * The current substring being parsed. + * @type {string} + */ + currentTag + + /** + * Whether we are currently closing a group. + * @type {boolean} + */ + closingGroup + + groupDepth + startingIndex + resetStartingIndex + slashFound + librarySchema + currentGroupStack + parenthesesStack + ignoringCharacters + + constructor(hedString) { + this.hedString = hedString + } + + /** + * Split the HED string into delimiters and tags. + * + * @returns {[TagSpec[], GroupSpec, Object]} The tag specifications, group bounds, and any issues found. + */ + tokenize() { + this.initializeTokenizer() + + for (let i = 0; i < this.hedString.length; i++) { + const character = this.hedString.charAt(i) + this.tokenizeCharacter(i, character) + if (this.resetStartingIndex) { + this.resetStartingIndex = false + this.startingIndex = i + 1 + this.currentTag = '' + } + } + this.pushTag(this.hedString.length, true) + + if (this.columnSpliceIndex >= 0) { + this._pushSyntaxIssue('unclosedCurlyBrace', this.columnSpliceIndex) + } + + this.unwindGroupStack() + + const tagSpecs = this.currentGroupStack.pop() + const groupSpecs = this.parenthesesStack.pop() + const issues = { + syntax: this.syntaxIssues, + conversion: [], + } + return [tagSpecs, groupSpecs, issues] + } + + initializeTokenizer() { + this.syntaxIssues = [] + + this.currentTag = '' + this.groupDepth = 0 + this.startingIndex = 0 + this.resetStartingIndex = false + this.slashFound = false + this.librarySchema = '' + this.columnSpliceIndex = -1 + this.currentGroupStack = [[]] + this.parenthesesStack = [new GroupSpec(0, this.hedString.length, [])] + this.ignoringCharacters = false + this.closingGroup = false + } + + tokenizeCharacter(i, character) { + let dispatchTable + if (this.ignoringCharacters) { + dispatchTable = { + [closingGroupCharacter]: (i /* character */) => { + this.clearTag() + this.closingGroupCharacter(i) + }, + [commaCharacter]: (/*i, character */) => this.clearTag(), + } + } else { + dispatchTable = { + [openingGroupCharacter]: (i /* character */) => this.openingGroupCharacter(i), + [closingGroupCharacter]: (i /* character */) => { + this.pushTag(i, false) + this.closingGroupCharacter(i) + }, + [openingColumnCharacter]: (i /* character */) => this.openingColumnCharacter(i), + [closingColumnCharacter]: (i /* character */) => this.closingColumnCharacter(i), + [commaCharacter]: (i /* character */) => this.pushTag(i, false), + [colonCharacter]: (i, character) => this.colonCharacter(character), + [slashCharacter]: (i, character) => this.slashCharacter(character), + } + } + const characterHandler = dispatchTable[character] + if (characterHandler) { + characterHandler(i, character) + } else if (invalidCharacters.has(character)) { + this._pushInvalidCharacterIssue(character, i) + } else { + this.otherCharacter(character) + } + } + + openingGroupCharacter(i) { + this.currentGroupStack.push([]) + this.parenthesesStack.push(new GroupSpec(i, undefined, [])) + this.resetStartingIndex = true + this.groupDepth++ + } + + closingGroupCharacter(i) { + this.closingGroup = true + if (this.groupDepth <= 0) { + this._pushSyntaxIssue('unopenedParenthesis', i) + return + } + this.closeGroup(i) + } + + openingColumnCharacter(i) { + if (this.currentTag.length > 0) { + this._pushInvalidCharacterIssue(openingColumnCharacter, i) + this.ignoringCharacters = true + return + } + if (this.columnSpliceIndex >= 0) { + this._pushSyntaxIssue('nestedCurlyBrace', i) + } + this.columnSpliceIndex = i + } + + closingColumnCharacter(i) { + this.closingGroup = true + if (this.columnSpliceIndex < 0) { + this._pushSyntaxIssue('unopenedCurlyBrace', i) + return + } + if (!stringIsEmpty(this.currentTag)) { + this.currentGroupStack[this.groupDepth].push(new ColumnSpliceSpec(this.currentTag.trim(), this.startingIndex, i)) + } else { + this.syntaxIssues.push( + generateIssue('emptyCurlyBrace', { + string: this.hedString, + }), + ) + } + this.columnSpliceIndex = -1 + this.resetStartingIndex = true + this.slashFound = false + } + + colonCharacter(character) { + if (!this.slashFound && !this.librarySchema) { + this.librarySchema = this.currentTag + this.resetStartingIndex = true + } else { + this.currentTag += character + } + } + + slashCharacter(character) { + this.slashFound = true + this.currentTag += character + } + + otherCharacter(character) { + if (this.ignoringCharacters) { + return + } + this.currentTag += character + this.resetStartingIndex = stringIsEmpty(this.currentTag) + } + + unwindGroupStack() { + // groupDepth is decremented in closeGroup. + // eslint-disable-next-line no-unmodified-loop-condition + while (this.groupDepth > 0) { + this._pushSyntaxIssue('unclosedParenthesis', this.parenthesesStack[this.parenthesesStack.length - 1].bounds[0]) + this.closeGroup(this.hedString.length) + } + } + + /** + * Push a tag to the current group. + * + * @param {number} i The current index. + * @param {boolean} isEndOfString Whether we are at the end of the string. + */ + pushTag(i, isEndOfString) { + if (stringIsEmpty(this.currentTag) && isEndOfString) { + return + } else if (this.closingGroup) { + this.closingGroup = false + } else if (stringIsEmpty(this.currentTag)) { + this.syntaxIssues.push(generateIssue('emptyTagFound', { index: i })) + } else if (this.columnSpliceIndex < 0) { + this._checkValueTagForInvalidCharacters() + this.currentGroupStack[this.groupDepth].push( + new TagSpec(this.currentTag.trim(), this.startingIndex, i, this.librarySchema), + ) + } + this.resetStartingIndex = true + this.slashFound = false + this.librarySchema = '' + } + + clearTag() { + this.ignoringCharacters = false + this.resetStartingIndex = true + this.slashFound = false + this.librarySchema = '' + } + + closeGroup(i) { + const groupSpec = this.parenthesesStack.pop() + groupSpec.bounds[1] = i + 1 + this.parenthesesStack[this.groupDepth - 1].children.push(groupSpec) + this.currentGroupStack[this.groupDepth - 1].push(this.currentGroupStack.pop()) + this.groupDepth-- + } + + /** + * Check an individual tag for invalid characters. + * + * @private + */ + _checkValueTagForInvalidCharacters() { + const formToCheck = replaceTagNameWithPound(this.currentTag) + for (let i = 0; i < formToCheck.length; i++) { + const character = formToCheck.charAt(i) + if (!invalidCharactersOutsideOfValues.has(character)) { + continue + } + this._pushInvalidCharacterIssue(character, this.startingIndex + i) + } + } + + /** + * Push an issue to the syntax issue list. + * + * @param {string} issueCode The internal code of the issue to be pushed. + * @param {number} index The location of the issue. + * @private + */ + _pushSyntaxIssue(issueCode, index) { + this.syntaxIssues.push( + generateIssue(issueCode, { + index: index, + string: this.hedString, + }), + ) + } + + /** + * Push an invalid character issue to the syntax issue list. + * + * @param {string} character The illegal character to be reported. + * @param {number} index The location of the character. + * @private + */ + _pushInvalidCharacterIssue(character, index) { + this.syntaxIssues.push( + generateIssue('invalidCharacter', { + character: unicodeName(character), + index: index, + string: this.hedString, + }), + ) + } +} diff --git a/spec_tests/javascriptTests.json b/spec_tests/javascriptTests.json index 1272aec9..ffbe300c 100644 --- a/spec_tests/javascriptTests.json +++ b/spec_tests/javascriptTests.json @@ -9,7 +9,7 @@ "definitions": ["(Definition/Acc/#, (Acceleration/#, Red))", "(Definition/MyColor, (Label/Pie))"], "tests": { "string_tests": { - "fails": ["Item/Bl\b"], + "fails": ["Item/Bl\b", "Item/ABC\u009e"], "passes": ["Red, Blue, Description/Red", "Description/This is a \u00ca\u00b0 good character"] }, "sidecar_tests": { @@ -37,6 +37,10 @@ [ ["onset", "duration", "HED"], [4.5, 0, "Item/Bl\b"] + ], + [ + ["onset", "duration", "HED"], + [4.5, 0, "Item/{abc}"] ] ], "passes": [ diff --git a/tests/bids.spec.data.js b/tests/bids.spec.data.js index dd9183f0..a55f7b68 100644 --- a/tests/bids.spec.data.js +++ b/tests/bids.spec.data.js @@ -90,7 +90,7 @@ const sidecars = [ }, { multiple_value_tags: { - HED: 'Duration/# s, RGB-blue/#', + HED: 'Label/#, Description/#', }, }, { @@ -340,7 +340,7 @@ const sidecars = [ event_code: { HED: { face: '(Red, Blue), (Green, (Yellow)), {HED}', - ball: '{response_time}, (Def/Acc/3.5 m-per-s^2)', + ball: '(Def/Acc/3.5 m-per-s^2)', dog: 'Orange, {event_type}', }, }, @@ -352,7 +352,7 @@ const sidecars = [ }, event_type: { HED: { - banana: 'Blue, {response_time}', + banana: 'Blue, {event_code}', apple: 'Green', }, }, @@ -546,11 +546,11 @@ const tsvFiles = [ ], // sub03 - Valid combined sidecar/TSV data [ - [sidecars[2][0], 'onset\tduration\n' + '7\tsomething'], - [sidecars[0][0], 'onset\tduration\tcolor\n' + '7\tsomething\tred'], - [sidecars[0][1], 'onset\tduration\tspeed\n' + '7\tsomething\t60'], - [sidecars[2][0], hedColumnOnlyHeader + '7\tsomething\tLaptop-computer'], - [sidecars[0][0], 'onset\tduration\tcolor\tHED\n' + '7\tsomething\tgreen\tLaptop-computer'], + [sidecars[2][0], 'onset\tduration\n' + '7\t4'], + [sidecars[0][0], 'onset\tduration\tcolor\n' + '7\t4\tred'], + [sidecars[0][1], 'onset\tduration\tspeed\n' + '7\t4\t60'], + [sidecars[2][0], hedColumnOnlyHeader + '7\t4\tLaptop-computer'], + [sidecars[0][0], 'onset\tduration\tcolor\tHED\n' + '7\t4\tgreen\tLaptop-computer'], [ Object.assign({}, sidecars[0][0], sidecars[0][1]), 'onset\tduration\tcolor\tvehicle\tspeed\n' + '7\tsomething\tblue\ttrain\t150', @@ -706,39 +706,39 @@ const tsvFiles = [ const datasetDescriptions = [ // Good datasetDescription.json files [ - { Name: 'OnlyBase', BIDSVersion: '1.7.0', HEDVersion: '8.1.0' }, - { Name: 'BaseAndTest', BIDSVersion: '1.7.0', HEDVersion: ['8.1.0', 'ts:testlib_1.0.2'] }, - { Name: 'OnlyTestAsLib', BIDSVersion: '1.7.0', HEDVersion: ['ts:testlib_1.0.2'] }, - { Name: 'BaseAndTwoTests', BIDSVersion: '1.7.0', HEDVersion: ['8.1.0', 'ts:testlib_1.0.2', 'bg:testlib_1.0.2'] }, - { Name: 'TwoTests', BIDSVersion: '1.7.0', HEDVersion: ['ts:testlib_1.0.2', 'bg:testlib_1.0.2'] }, - { Name: 'OnlyScoreAsBase', BIDSVersion: '1.7.0', HEDVersion: 'score_1.0.0' }, - { Name: 'OnlyScoreAsLib', BIDSVersion: '1.7.0', HEDVersion: 'sc:score_1.0.0' }, - { Name: 'OnlyTestAsBase', BIDSVersion: '1.7.0', HEDVersion: 'testlib_1.0.2' }, - { Name: 'GoodLazyPartneredSchemas', BIDSVersion: '1.7.0', HEDVersion: ['testlib_2.0.0', 'testlib_3.0.0'] }, + { Name: 'OnlyBase', BIDSVersion: '1.10.0', HEDVersion: '8.3.0' }, + { Name: 'BaseAndTest', BIDSVersion: '1.10.0', HEDVersion: ['8.3.0', 'ts:testlib_1.0.2'] }, + { Name: 'OnlyTestAsLib', BIDSVersion: '1.10.0', HEDVersion: ['ts:testlib_1.0.2'] }, + { Name: 'BaseAndTwoTests', BIDSVersion: '1.10.0', HEDVersion: ['8.3.0', 'ts:testlib_1.0.2', 'bg:testlib_1.0.2'] }, + { Name: 'TwoTests', BIDSVersion: '1.10.0', HEDVersion: ['ts:testlib_1.0.2', 'bg:testlib_1.0.2'] }, + { Name: 'OnlyScoreAsBase', BIDSVersion: '1.10.0', HEDVersion: 'score_1.0.0' }, + { Name: 'OnlyScoreAsLib', BIDSVersion: '1.10.0', HEDVersion: 'sc:score_1.0.0' }, + { Name: 'OnlyTestAsBase', BIDSVersion: '1.10.0', HEDVersion: 'testlib_1.0.2' }, + { Name: 'GoodLazyPartneredSchemas', BIDSVersion: '1.10.0', HEDVersion: ['testlib_2.0.0', 'testlib_3.0.0'] }, { Name: 'GoodLazyPartneredSchemasWithStandard', - BIDSVersion: '1.7.0', + BIDSVersion: '1.10.0', HEDVersion: ['testlib_2.0.0', 'testlib_3.0.0', '8.2.0'], }, ], // Bad datasetDescription.json files [ - { Name: 'NonExistentLibrary', BIDSVersion: '1.7.0', HEDVersion: ['8.1.0', 'ts:badlib_1.0.2'] }, - { Name: 'LeadingColon', BIDSVersion: '1.7.0', HEDVersion: [':testlib_1.0.2', '8.1.0'] }, - { Name: 'BadNickName', BIDSVersion: '1.7.0', HEDVersion: ['8.1.0', 't-s:testlib_1.0.2'] }, - { Name: 'MultipleColons1', BIDSVersion: '1.7.0', HEDVersion: ['8.1.0', 'ts::testlib_1.0.2'] }, - { Name: 'MultipleColons2', BIDSVersion: '1.7.0', HEDVersion: ['8.1.0', ':ts:testlib_1.0.2'] }, - { Name: 'NoLibraryName', BIDSVersion: '1.7.0', HEDVersion: ['8.1.0', 'ts:_1.0.2'] }, - { Name: 'BadVersion1', BIDSVersion: '1.7.0', HEDVersion: ['8.1.0', 'ts:testlib1.0.2'] }, - { Name: 'BadVersion2', BIDSVersion: '1.7.0', HEDVersion: ['8.1.0', 'ts:testlib_1.a.2'] }, - { Name: 'BadRemote1', BIDSVersion: '1.7.0', HEDVersion: ['8.1.0', 'ts:testlib_1.800.2'] }, - { Name: 'BadRemote2', BIDSVersion: '1.7.0', HEDVersion: '8.828.0' }, - { Name: 'NoHedVersion', BIDSVersion: '1.7.0' }, - { Name: 'BadLazyPartneredSchema1', BIDSVersion: '1.7.0', HEDVersion: ['testlib_2.0.0', 'testlib_2.1.0'] }, - { Name: 'BadLazyPartneredSchema2', BIDSVersion: '1.7.0', HEDVersion: ['testlib_2.1.0', 'testlib_3.0.0'] }, + { Name: 'NonExistentLibrary', BIDSVersion: '1.10.0', HEDVersion: ['8.3.0', 'ts:badlib_1.0.2'] }, + { Name: 'LeadingColon', BIDSVersion: '1.10.0', HEDVersion: [':testlib_1.0.2', '8.3.0'] }, + { Name: 'BadNickName', BIDSVersion: '1.10.0', HEDVersion: ['8.3.0', 't-s:testlib_1.0.2'] }, + { Name: 'MultipleColons1', BIDSVersion: '1.10.0', HEDVersion: ['8.3.0', 'ts::testlib_1.0.2'] }, + { Name: 'MultipleColons2', BIDSVersion: '1.10.0', HEDVersion: ['8.3.0', ':ts:testlib_1.0.2'] }, + { Name: 'NoLibraryName', BIDSVersion: '1.10.0', HEDVersion: ['8.3.0', 'ts:_1.0.2'] }, + { Name: 'BadVersion1', BIDSVersion: '1.10.0', HEDVersion: ['8.3.0', 'ts:testlib1.0.2'] }, + { Name: 'BadVersion2', BIDSVersion: '1.10.0', HEDVersion: ['8.3.0', 'ts:testlib_1.a.2'] }, + { Name: 'BadRemote1', BIDSVersion: '1.10.0', HEDVersion: ['8.3.0', 'ts:testlib_1.800.2'] }, + { Name: 'BadRemote2', BIDSVersion: '1.10.0', HEDVersion: '8.828.0' }, + { Name: 'NoHedVersion', BIDSVersion: '1.10.0' }, + { Name: 'BadLazyPartneredSchema1', BIDSVersion: '1.10.0', HEDVersion: ['testlib_2.0.0', 'testlib_2.1.0'] }, + { Name: 'BadLazyPartneredSchema2', BIDSVersion: '1.10.0', HEDVersion: ['testlib_2.1.0', 'testlib_3.0.0'] }, { Name: 'LazyPartneredSchemasWithWrongStandard', - BIDSVersion: '1.7.0', + BIDSVersion: '1.10.0', HEDVersion: ['testlib_2.0.0', 'testlib_3.0.0', '8.1.0'], }, ], diff --git a/tests/bids.spec.js b/tests/bids.spec.js index f62b3de9..33e8c867 100644 --- a/tests/bids.spec.js +++ b/tests/bids.spec.js @@ -16,16 +16,10 @@ describe('BIDS datasets', () => { * @type {SchemasSpec} */ let specs - /** - * @type {SchemasSpec} - */ - let specs2 beforeAll(() => { - const spec1 = new SchemaSpec('', '8.0.0') + const spec1 = new SchemaSpec('', '8.3.0') specs = new SchemasSpec().addSchemaSpec(spec1) - const spec2 = new SchemaSpec('', '7.2.0') - specs2 = new SchemasSpec().addSchemaSpec(spec2) }) /** @@ -66,125 +60,124 @@ describe('BIDS datasets', () => { }), ) } - - describe('Sidecar-only datasets', () => { - it('should validate non-placeholder HED strings in BIDS sidecars', () => { - const goodDatasets = bidsSidecars[0] - const testDatasets = { - single: new BidsDataset([], [bidsSidecars[0][0]]), - all_good: new BidsDataset([], goodDatasets), - warning_and_good: new BidsDataset([], goodDatasets.concat([bidsSidecars[1][0]])), - error_and_good: new BidsDataset([], goodDatasets.concat([bidsSidecars[1][1]])), - } - const expectedIssues = { - single: [], - all_good: [], - warning_and_good: [ - BidsHedIssue.fromHedIssue( - generateIssue('extension', { tag: 'Train/Maglev', sidecarKey: 'transport' }), - bidsSidecars[1][0].file, - ), - ], - error_and_good: [ - BidsHedIssue.fromHedIssue(generateIssue('invalidTag', { tag: 'Confused' }), bidsSidecars[1][1].file), - ], - } - validator(testDatasets, expectedIssues, specs) - }, 10000) - - it('should validate placeholders in BIDS sidecars', () => { - const placeholderDatasets = bidsSidecars[2] - const testDatasets = { - placeholders: new BidsDataset([], placeholderDatasets), - } - const expectedIssues = { - placeholders: [ - BidsHedIssue.fromHedIssue( - generateIssue('invalidPlaceholderInDefinition', { - definition: 'InvalidDefinitionGroup', - sidecarKey: 'invalid_definition_group', - }), - placeholderDatasets[2].file, - ), - BidsHedIssue.fromHedIssue( - generateIssue('invalidPlaceholderInDefinition', { - definition: 'InvalidDefinitionTag', - sidecarKey: 'invalid_definition_tag', - }), - placeholderDatasets[3].file, - ), - BidsHedIssue.fromHedIssue( - generateIssue('invalidPlaceholderInDefinition', { - definition: 'MultiplePlaceholdersInGroupDefinition', - sidecarKey: 'multiple_placeholders_in_group', - }), - placeholderDatasets[4].file, - ), - BidsHedIssue.fromHedIssue( - generateIssue('invalidPlaceholder', { tag: 'Duration/# s', sidecarKey: 'multiple_value_tags' }), - placeholderDatasets[5].file, - ), - BidsHedIssue.fromHedIssue( - generateIssue('invalidPlaceholder', { tag: 'RGB-blue/#', sidecarKey: 'multiple_value_tags' }), - placeholderDatasets[5].file, - ), - BidsHedIssue.fromHedIssue( - generateIssue('missingPlaceholder', { string: 'Sad', sidecarKey: 'no_value_tags' }), - placeholderDatasets[6].file, - ), - BidsHedIssue.fromHedIssue( - generateIssue('invalidPlaceholder', { tag: 'RGB-green/#', sidecarKey: 'value_in_categorical' }), - placeholderDatasets[7].file, - ), - ], - } - return validator(testDatasets, expectedIssues, specs) - }, 10000) - }) - - describe('TSV-only datasets', () => { - it('should validate HED strings in BIDS event files', () => { - const goodDatasets = bidsTsvFiles[0] - const badDatasets = bidsTsvFiles[1] - const testDatasets = { - all_good: new BidsDataset(goodDatasets, []), - all_bad: new BidsDataset(badDatasets, []), - } - const legalSpeedUnits = ['m-per-s', 'kph', 'mph'] - const speedIssue = generateIssue('unitClassInvalidUnit', { - tag: 'Speed/300 miles', - unitClassUnits: legalSpeedUnits.sort().join(','), - }) - const maglevError = generateIssue('invalidTag', { tag: 'Maglev' }) - const maglevWarning = generateIssue('extension', { tag: 'Train/Maglev' }) - const expectedIssues = { - all_good: [], - all_bad: [ - BidsHedIssue.fromHedIssue(cloneDeep(speedIssue), badDatasets[0].file, { tsvLine: 2 }), - BidsHedIssue.fromHedIssue(cloneDeep(maglevWarning), badDatasets[1].file, { tsvLine: 2 }), - BidsHedIssue.fromHedIssue(cloneDeep(speedIssue), badDatasets[2].file, { tsvLine: 3 }), - BidsHedIssue.fromHedIssue(cloneDeep(maglevError), badDatasets[3].file, { tsvLine: 2 }), - BidsHedIssue.fromHedIssue(cloneDeep(speedIssue), badDatasets[3].file, { tsvLine: 3 }), - BidsHedIssue.fromHedIssue(cloneDeep(maglevWarning), badDatasets[4].file, { tsvLine: 2 }), - BidsHedIssue.fromHedIssue(cloneDeep(speedIssue), badDatasets[4].file, { tsvLine: 3 }), - ], - } - return validator(testDatasets, expectedIssues, specs) - }, 10000) - }) + // + // describe('Sidecar-only datasets', () => { + // it('should validate non-placeholder HED strings in BIDS sidecars', () => { + // const goodDatasets = bidsSidecars[0] + // const testDatasets = { + // single: new BidsDataset([], [bidsSidecars[0][0]]), + // all_good: new BidsDataset([], goodDatasets), + // warning_and_good: new BidsDataset([], goodDatasets.concat([bidsSidecars[1][0]])), + // error_and_good: new BidsDataset([], goodDatasets.concat([bidsSidecars[1][1]])), + // } + // const expectedIssues = { + // single: [], + // all_good: [], + // warning_and_good: [ + // BidsHedIssue.fromHedIssue( + // generateIssue('extension', { tag: 'Train/Maglev', sidecarKey: 'transport' }), + // bidsSidecars[1][0].file, + // ), + // ], + // error_and_good: [ + // BidsHedIssue.fromHedIssue(generateIssue('invalidTag', { tag: 'Confused' }), bidsSidecars[1][1].file), + // ], + // } + // validator(testDatasets, expectedIssues, specs) + // }, 10000) + // + // it('should validate placeholders in BIDS sidecars', () => { + // const placeholderDatasets = bidsSidecars[2] + // const testDatasets = { + // placeholders: new BidsDataset([], placeholderDatasets), + // } + // const expectedIssues = { + // placeholders: [ + // BidsHedIssue.fromHedIssue( + // generateIssue('invalidPlaceholderInDefinition', { + // definition: 'InvalidDefinitionGroup', + // sidecarKey: 'invalid_definition_group', + // }), + // placeholderDatasets[2].file, + // ), + // BidsHedIssue.fromHedIssue( + // generateIssue('invalidPlaceholderInDefinition', { + // definition: 'InvalidDefinitionTag', + // sidecarKey: 'invalid_definition_tag', + // }), + // placeholderDatasets[3].file, + // ), + // BidsHedIssue.fromHedIssue( + // generateIssue('invalidPlaceholderInDefinition', { + // definition: 'MultiplePlaceholdersInGroupDefinition', + // sidecarKey: 'multiple_placeholders_in_group', + // }), + // placeholderDatasets[4].file, + // ), + // BidsHedIssue.fromHedIssue( + // generateIssue('invalidPlaceholder', { tag: 'Label/#', sidecarKey: 'multiple_value_tags' }), + // placeholderDatasets[5].file, + // ), + // BidsHedIssue.fromHedIssue( + // generateIssue('invalidPlaceholder', { tag: 'Description/#', sidecarKey: 'multiple_value_tags' }), + // placeholderDatasets[5].file, + // ), + // BidsHedIssue.fromHedIssue( + // generateIssue('missingPlaceholder', { string: 'Sad', sidecarKey: 'no_value_tags' }), + // placeholderDatasets[6].file, + // ), + // BidsHedIssue.fromHedIssue( + // generateIssue('invalidPlaceholder', { tag: 'RGB-green/#', sidecarKey: 'value_in_categorical' }), + // placeholderDatasets[7].file, + // ), + // ], + // } + // return validator(testDatasets, expectedIssues, specs) + // }, 10000) + // }) + // + // describe('TSV-only datasets', () => { + // it('should validate HED strings in BIDS event files', () => { + // const goodDatasets = bidsTsvFiles[0] + // const badDatasets = bidsTsvFiles[1] + // const testDatasets = { + // all_good: new BidsDataset(goodDatasets, []), + // all_bad: new BidsDataset(badDatasets, []), + // } + // const legalSpeedUnits = ['m-per-s', 'kph', 'mph'] + // const speedIssue = generateIssue('unitClassInvalidUnit', { + // tag: 'Speed/300 miles', + // unitClassUnits: legalSpeedUnits.sort().join(','), + // }) + // const maglevError = generateIssue('invalidTag', { tag: 'Maglev' }) + // const maglevWarning = generateIssue('extension', { tag: 'Train/Maglev' }) + // const expectedIssues = { + // all_good: [], + // all_bad: [ + // BidsHedIssue.fromHedIssue(cloneDeep(speedIssue), badDatasets[0].file, { tsvLine: 2 }), + // BidsHedIssue.fromHedIssue(cloneDeep(maglevWarning), badDatasets[1].file, { tsvLine: 2 }), + // BidsHedIssue.fromHedIssue(cloneDeep(speedIssue), badDatasets[2].file, { tsvLine: 3 }), + // BidsHedIssue.fromHedIssue(cloneDeep(maglevError), badDatasets[3].file, { tsvLine: 2 }), + // BidsHedIssue.fromHedIssue(cloneDeep(speedIssue), badDatasets[3].file, { tsvLine: 3 }), + // BidsHedIssue.fromHedIssue(cloneDeep(maglevWarning), badDatasets[4].file, { tsvLine: 2 }), + // BidsHedIssue.fromHedIssue(cloneDeep(speedIssue), badDatasets[4].file, { tsvLine: 3 }), + // ], + // } + // return validator(testDatasets, expectedIssues, specs) + // }, 10000) + // }) describe('Combined datasets', () => { it('should validate BIDS event files combined with JSON sidecar data', () => { const goodDatasets = bidsTsvFiles[2] const badDatasets = bidsTsvFiles[3] const testDatasets = { - all_good: new BidsDataset(goodDatasets, []), + /* all_good: new BidsDataset(goodDatasets, []),*/ all_bad: new BidsDataset(badDatasets, []), } const expectedIssues = { all_good: [], all_bad: [ - // BidsHedIssue.fromHedIssue(generateIssue('invalidTag', { tag: 'Confused' }), badDatasets[0].file), BidsHedIssue.fromHedIssue(generateIssue('invalidTag', { tag: 'Confused' }), badDatasets[0].file), // TODO: Catch warning in sidecar validation /* BidsHedIssue.fromHedIssue( @@ -226,6 +219,13 @@ describe('BIDS datasets', () => { badDatasets[3].file, { tsvLine: 2 }, ), + BidsHedIssue.fromHedIssue( + generateIssue('invalidTopLevelTagGroupTag', { + tag: 'Duration/ferry s', + }), + badDatasets[3].file, + { tsvLine: 2 }, + ), BidsHedIssue.fromHedIssue( generateIssue('sidecarKeyMissing', { key: 'purple', @@ -583,18 +583,11 @@ describe('BIDS datasets', () => { ), BidsHedIssue.fromHedIssue( generateIssue('recursiveCurlyBracesWithKey', { - column: 'response_time', + column: 'event_code', referrer: 'event_type', }), standaloneSidecars[7].file, ), - BidsHedIssue.fromHedIssue( - generateIssue('recursiveCurlyBracesWithKey', { - column: 'response_time', - referrer: 'event_code', - }), - standaloneSidecars[7].file, - ), BidsHedIssue.fromHedIssue( generateIssue('recursiveCurlyBracesWithKey', { column: 'response_time', @@ -625,6 +618,7 @@ describe('BIDS datasets', () => { ), BidsHedIssue.fromHedIssue( generateIssue('emptyCurlyBrace', { + index: 1, string: standaloneSidecars[9].hedData.get('event_code4').ball, }), standaloneSidecars[9].file, diff --git a/tests/bidsTests.data.js b/tests/bidsTests.data.js new file mode 100644 index 00000000..c8c3d4e4 --- /dev/null +++ b/tests/bidsTests.data.js @@ -0,0 +1,245 @@ +import { BidsHedIssue } from '../bids' +import { generateIssue } from '../common/issues/issues' + +export const bidsTestData = [ + { + name: 'valid-bids-datasets-with-limited-hed', + description: 'HED or data is missing in various places', + tests: [ + { + testname: 'no-hed-at-all-but-both-tsv-json-non-empty', + explanation: 'Neither the sidecar or tsv has HED but neither non-empty', + schemaVersion: '8.3.0', + sidecar: { + duration: { + description: 'Duration of the event in seconds.', + }, + }, + eventsString: 'onset\tduration\n' + '7\t4', + sidecarOnlyErrors: [], + eventsOnlyErrors: [], + comboErrors: [], + }, + { + testname: 'only-header-in-tsv-with-return', + explanation: 'TSV only has header and trailing return and white space', + schemaVersion: '8.3.0', + sidecar: { + duration: { + description: 'Duration of the event in seconds.', + }, + }, + eventsString: 'onset\tduration\n ', + sidecarOnlyErrors: [], + eventsOnlyErrors: [], + comboErrors: [], + }, + { + testname: 'empty-json-empty-tsv', + explanation: 'Both sidecar and tsv are empty except for white space', + schemaVersion: '8.3.0', + sidecar: {}, + eventsString: '\n \n', + sidecarOnlyErrors: [], + eventsOnlyErrors: [], + comboErrors: [], + }, + ], + }, + { + name: 'valid-json-invalid-tsv', + description: 'JSON is valid but tsv is invalid', + tests: [ + { + testname: 'valid-sidecar-bad-tag-tsv', + explanation: 'Unrelated sidecar is valid but HED column tag is invalid', + schemaVersion: '8.3.0', + sidecar: { + event_code: { + HED: { + face: '(Red, Blue), (Green, (Yellow))', + }, + }, + }, + eventsString: 'onset\tduration\tHED\n' + '7\t4\tBaloney', + sidecarOnlyErrors: [], + eventsOnlyErrors: [ + BidsHedIssue.fromHedIssue( + generateIssue('invalidTag', { tag: 'Baloney' }), + { relativePath: 'valid-sidecar-bad-tag-tsv.tsv' }, + { tsvLine: 2 }, + ), + ], + comboErrors: [ + BidsHedIssue.fromHedIssue( + generateIssue('invalidTag', { tag: 'Baloney' }), + { path: 'valid-sidecar-bad-tag-tsv.tsv', relativePath: 'valid-sidecar-bad-tag-tsv.tsv' }, + { tsvLine: 2 }, + ), + ], + }, + { + testname: 'valid-sidecar-tsv-curly-brace', + explanation: 'The sidecar is valid, but tsv HED column has braces}', + schemaVersion: '8.3.0', + sidecar: { + event_code: { + HED: { + face: '(Red, Blue), (Green, (Yellow))', + }, + }, + }, + eventsString: 'onset\tduration\tevent_code\tHED\n' + '7\t4\tface\tRed,{blue}', + sidecarOnlyErrors: [], + eventsOnlyErrors: [ + BidsHedIssue.fromHedIssue( + generateIssue('curlyBracesInHedColumn', { column: '{blue}' }), + { relativePath: 'valid-sidecar-tsv-curly-brace.tsv' }, + { tsvLine: 2 }, + ), + ], + comboErrors: [ + BidsHedIssue.fromHedIssue( + generateIssue('curlyBracesInHedColumn', { column: '{blue}' }), + { path: 'valid-sidecar-tsv-curly-brace.tsv', relativePath: 'valid-sidecar-tsv-curly-brace.tsv' }, + { tsvLine: 2 }, + ), + ], + }, + ], + }, + { + name: 'duplicate-tag-test', + description: 'Duplicate tags can appear in isolation or in combiantion', + tests: [ + { + testname: 'first-level-duplicate-json-tsv', + explanation: 'Each is okay but when combined, duplicate tag', + schemaVersion: '8.3.0', + sidecar: { + vehicle: { + HED: { + car: 'Car', + train: 'Train', + boat: 'Boat', + }, + }, + speed: { + HED: 'Speed/# mph', + }, + transport: { + HED: { + car: 'Car', + train: 'Train', + boat: 'Boat', + maglev: 'Vehicle', + }, + }, + }, + eventsString: 'onset\tduration\tvehicle\ttransport\tspeed\n' + '19\t6\tboat\tboat\t5\n', + sidecarOnlyErrors: [], + eventsOnlyErrors: [], + comboErrors: [ + BidsHedIssue.fromHedIssue( + generateIssue('duplicateTag', { tag: 'Boat' }), + { path: 'first-level-duplicate-json-tsv.tsv', relativePath: 'first-level-duplicate-json-tsv.tsv' }, + { tsvLine: 2 }, + ), + ], + }, + ], + }, + + { + name: 'curly-brace-tests', + description: 'Curly braces tested in various places', + tests: [ + { + testname: 'valid-curly-brace-in-sidecar-with-simple-splice', + explanation: 'Valid curly brace in sidecar and valid value is spliced in', + schemaVersion: '8.3.0', + sidecar: { + event_code: { + HED: { + face: '(Red, Blue), (Green, (Yellow))', + ball: '{ball_type}, Black', + }, + }, + ball_type: { + Description: 'Has description with HED', + HED: 'Label/#', + }, + }, + eventsString: 'onset\tduration\tevent_code\tball_type\n' + '19\t6\tball\tbig-one\n', + sidecarOnlyErrors: [], + eventsOnlyErrors: [], + comboErrors: [], + }, + { + testname: 'valid-curly-brace-in-sidecar-with-n/a-splice', + explanation: 'Valid curly brace in sidecar and but tsv splice entry is n/a', + schemaVersion: '8.3.0', + sidecar: { + event_code: { + HED: { + face: '(Red, Blue), (Green, (Yellow))', + ball: '{ball_type}, Black', + }, + }, + ball_type: { + Description: 'Has description with HED', + HED: 'Label/#', + }, + }, + eventsString: 'onset\tduration\tevent_code\tball_type\n' + '19\t6\tball\tn/a\n', + sidecarOnlyErrors: [], + eventsOnlyErrors: [], + comboErrors: [], + }, + { + testname: 'valid-curly-brace-in-sidecar-with-HED-column-splice', + explanation: 'Valid curly brace in sidecar with HED column splice', + schemaVersion: '8.3.0', + sidecar: { + event_code: { + HED: { + face: '(Red, Blue), (Green, (Yellow))', + ball: '{ball_type}, Black, ({HED})', + }, + }, + ball_type: { + Description: 'Has description with HED', + HED: 'Label/#', + }, + }, + eventsString: 'onset\tduration\tevent_code\tball_type\tHED\n' + '19\t6\tball\tn/a\tPurple\n', + sidecarOnlyErrors: [], + eventsOnlyErrors: [], + comboErrors: [], + }, + { + testname: 'invalid-curly-brace-column-slice-has-no hed', + explanation: 'A column name is used in a splice but does not have HED', + schemaVersion: '8.3.0', + sidecar: { + event_code: { + HED: { + face: '(Red, Blue), (Green, (Yellow))', + ball: '{ball_type}, Black', + }, + }, + }, + eventsString: 'onset\tduration\tevent_code\tball_type\n' + '19\t6\tball\tn/a\tPurple\n', + sidecarOnlyErrors: [], + eventsOnlyErrors: [], + comboErrors: [ + BidsHedIssue.fromHedIssue( + generateIssue('invalidTag', { tag: 'Baloney' }), + { relativePath: 'valid-sidecar-bad-tag-tsv.tsv' }, + { column: 'ball_type' }, + ), + ], + }, + ], + }, +] diff --git a/tests/bidsTests.spec.js b/tests/bidsTests.spec.js new file mode 100644 index 00000000..c9cb4b96 --- /dev/null +++ b/tests/bidsTests.spec.js @@ -0,0 +1,190 @@ +import chai from 'chai' +const assert = chai.assert +const difference = require('lodash/difference') +import { beforeAll, describe, afterAll } from '@jest/globals' +import path from 'path' +import { BidsHedIssue } from '../bids/types/issues' +import { buildSchemas } from '../validator/schema/init' +import { SchemaSpec, SchemasSpec } from '../common/schema/types' +import { BidsDataset, BidsEventFile, BidsHedTsvValidator, BidsSidecar, BidsTsvFile } from '../bids' +import { generateIssue, IssueError } from '../common/issues/issues' + +import { bidsTestData } from './bidsTests.data' +import parseTSV from '../bids/tsvParser' +const fs = require('fs') + +//const displayLog = process.env.DISPLAY_LOG === 'true' +const displayLog = true +const skippedTests = new Map() + +// Ability to select individual tests to run +const runAll = true +let onlyRun = new Map() +if (!runAll) { + onlyRun = new Map([['curly-brace-tests', ['invalid-curly-brace-column-slice-has-no hed']]]) +} + +function shouldRun(name, testname) { + if (onlyRun.size === 0) return true + if (onlyRun.get(name) === undefined) return false + + const cases = onlyRun.get(name) + if (cases.length === 0) return true + + if (cases.includes(testname)) { + return true + } else { + return false + } +} + +// Return an array of hedCode values extracted from an issues list. +function extractHedCodes(issues) { + const errors = [] + for (const issue of issues) { + if (issue instanceof BidsHedIssue) { + errors.push(`${issue.hedIssue.hedCode}`) + } else { + errors.push(`${issue.hedCode}`) + } + } + return errors +} + +describe('BIDS validation', () => { + const schemaMap = new Map([ + ['8.2.0', undefined], + ['8.3.0', undefined], + ]) + + const badLog = [] + let totalTests + let wrongErrors + let missingErrors + + beforeAll(async () => { + const spec2 = new SchemaSpec('', '8.2.0', '', path.join(__dirname, '../tests/data/HED8.2.0.xml')) + const specs2 = new SchemasSpec().addSchemaSpec(spec2) + const schemas2 = await buildSchemas(specs2) + const spec3 = new SchemaSpec('', '8.3.0', '', path.join(__dirname, '../tests/data/HED8.3.0.xml')) + const specs3 = new SchemasSpec().addSchemaSpec(spec3) + const schemas3 = await buildSchemas(specs3) + schemaMap.set('8.2.0', schemas2) + schemaMap.set('8.3.0', schemas3) + totalTests = 0 + wrongErrors = 0 + missingErrors = 0 + }) + + afterAll(() => { + const outBad = path.join(__dirname, 'runLog.txt') + const summary = `Total tests:${totalTests} Wrong errors:${wrongErrors} MissingErrors:${missingErrors}\n` + if (displayLog) { + fs.writeFileSync(outBad, summary + badLog.join('\n'), 'utf8') + } + }) + + describe.each(bidsTestData)('$name : $description', ({ name, description, tests }) => { + let itemLog + + const assertErrors = function (test, type, expectedErrors, issues, iLog) { + const status = expectedErrors.length > 0 ? 'Expect fail' : 'Expect pass' + const header = `[${name}:${test.testname}][${type}](${status})` + const log = [] + totalTests += 1 + + const errors = extractHedCodes(issues) + const errorString = errors.join(',') + if (errors.length > 0) { + log.push(`---has errors [${errorString}]`) + } + if (expectedErrors.length === 0 && errorString.length > 0) { + const hasErrors = `---expected no errors but got errors [${errorString}]` + log.push(hasErrors) + log.push(`Received issues: ${JSON.stringify(issues)}`) + iLog.push(header + '\n' + log.join('\n')) + wrongErrors += 1 + assert(errorString.length === 0, `${header}${hasErrors}]`) + } else { + const expectedErrorCodes = extractHedCodes(expectedErrors) + const wrong = difference(errors, expectedErrorCodes) + const missing = difference(expectedErrorCodes, errors) + let errorMessage = '' + if (wrong.length > 0) { + errorMessage = `---received unexpected errors ${wrong.join(',')}\n` + wrongErrors += 1 + } + if (missing.length > 0) { + errorMessage = errorMessage + `---did not receive expected errors ${missing.join(',')}` + missingErrors += 1 + } + + if (errorMessage.length > 0) { + log.push(errorMessage) + log.push(`Expected issues:\n${JSON.stringify(expectedErrors)}`) + log.push(`Received issues:\n${JSON.stringify(issues)}`) + iLog.push(header + '\n' + log.join('\n')) + } else { + iLog.push(header) + } + assert.sameDeepMembers(issues, expectedErrors, header) + } + } + + const validate = function (test, iLog) { + // Make sure that the schema is available + const header = `[${test.testname} (Expect pass)]` + iLog.push(header) + const thisSchema = schemaMap.get(test.schemaVersion) + assert(thisSchema !== undefined, `${test.schemaVersion} is not available in test ${test.name}`) + + // Validate the sidecar by itself + const sidecarName = test.testname + '.json' + const bidsSidecar = new BidsSidecar('thisOne', test.sidecar, { relativePath: sidecarName, path: sidecarName }) + assert(bidsSidecar instanceof BidsSidecar, 'Test') + const sidecarIssues = bidsSidecar.validate(thisSchema) + assertErrors(test, 'Sidecar only', test.sidecarOnlyErrors, sidecarIssues, iLog) + + // Parse the events file + const eventName = test.testname + '.tsv' + const parsedTsv = parseTSV(test.eventsString) + assert(parsedTsv instanceof Map, `${eventName} cannot be parsed`) + + // Validate the events file by itself + const bidsTsv = new BidsTsvFile(test.testname, parsedTsv, { relativePath: eventName }, [], {}) + const validator = new BidsHedTsvValidator(bidsTsv, thisSchema) + validator.validate() + assertErrors(test, 'Events only', test.eventsOnlyErrors, validator.issues, iLog) + + // Validate the events file with the sidecar + const bidsTsvSide = new BidsTsvFile( + test.name, + parsedTsv, + { relativePath: eventName, path: eventName }, + [], + test.sidecar, + ) + const validatorWithSide = new BidsHedTsvValidator(bidsTsvSide, thisSchema) + validatorWithSide.validate() + assertErrors(test, 'Events+side', test.comboErrors, validatorWithSide.issues, iLog) + } + + beforeAll(async () => { + itemLog = [] + }) + + afterAll(() => { + badLog.push(itemLog.join('\n')) + }) + + if (tests && tests.length > 0) { + test.each(tests)('$testname: $explanation ', (test) => { + if (shouldRun(name, test.testname)) { + validate(test, itemLog) + } else { + itemLog.push(`----Skipping ${name}: ${test.testname}`) + } + }) + } + }) +}) diff --git a/tests/event.spec.js b/tests/event.spec.js index dfc22f2e..88e22882 100644 --- a/tests/event.spec.js +++ b/tests/event.spec.js @@ -64,22 +64,19 @@ describe('HED string and event validation', () => { it('should not have mismatched parentheses', () => { const testStrings = { extraOpening: - '/Action/Reach/To touch,((/Attribute/Object side/Left,/Participant/Effect/Body part/Arm),/Attribute/Location/Screen/Top/70 px,/Attribute/Location/Screen/Left/23 px', + 'Action/Reach/To touch,((Attribute/Object side/Left,Participant/Effect/Body part/Arm),Attribute/Location/Screen/Top/70 px,Attribute/Location/Screen/Left/23 px', // The extra comma is needed to avoid a comma error. extraClosing: - '/Action/Reach/To touch,(/Attribute/Object side/Left,/Participant/Effect/Body part/Arm),),/Attribute/Location/Screen/Top/70 px,/Attribute/Location/Screen/Left/23 px', + 'Action/Reach/To touch,(Attribute/Object side/Left,Participant/Effect/Body part/Arm),),Attribute/Location/Screen/Top/70 px,Attribute/Location/Screen/Left/23 px', wrongOrder: - '/Action/Reach/To touch,((/Attribute/Object side/Left),/Participant/Effect/Body part/Arm),/Attribute/Location/Screen/Top/70 px),(/Attribute/Location/Screen/Left/23 px', + 'Action/Reach/To touch,((Attribute/Object side/Left),Participant/Effect/Body part/Arm),Attribute/Location/Screen/Top/70 px),(Attribute/Location/Screen/Left/23 px', valid: - '/Action/Reach/To touch,(/Attribute/Object side/Left,/Participant/Effect/Body part/Arm),/Attribute/Location/Screen/Top/70 px,/Attribute/Location/Screen/Left/23 px', + 'Action/Reach/To touch,(Attribute/Object side/Left,Participant/Effect/Body part/Arm),Attribute/Location/Screen/Top/70 px,Attribute/Location/Screen/Left/23 px', } const expectedIssues = { extraOpening: [generateIssue('parentheses', { opening: 2, closing: 1 })], extraClosing: [generateIssue('parentheses', { opening: 1, closing: 2 })], - wrongOrder: [ - generateIssue('unopenedParenthesis', { index: 125, string: testStrings.wrongOrder }), - generateIssue('unclosedParenthesis', { index: 127, string: testStrings.wrongOrder }), - ], + wrongOrder: [generateIssue('unopenedParenthesis', { index: 121, string: testStrings.wrongOrder })], valid: [], } // No-op function as this check is done during the parsing stage. @@ -90,31 +87,31 @@ describe('HED string and event validation', () => { it('should not have malformed delimiters', () => { const testStrings = { missingOpeningComma: - '/Action/Reach/To touch(/Attribute/Object side/Left,/Participant/Effect/Body part/Arm),/Attribute/Location/Screen/Top/70 px,/Attribute/Location/Screen/Left/23 px', + 'Action/Reach/To touch(Attribute/Object side/Left,Participant/Effect/Body part/Arm),Attribute/Location/Screen/Top/70 px,Attribute/Location/Screen/Left/23 px', missingClosingComma: - '/Action/Reach/To touch,(/Attribute/Object side/Left,/Participant/Effect/Body part/Arm)/Attribute/Location/Screen/Top/70 px,/Attribute/Location/Screen/Left/23 px', + 'Action/Reach/To touch,(Attribute/Object side/Left,Participant/Effect/Body part/Arm)Attribute/Location/Screen/Top/70 px,Attribute/Location/Screen/Left/23 px', extraOpeningComma: - ',/Action/Reach/To touch,(/Attribute/Object side/Left,/Participant/Effect/Body part/Arm),/Attribute/Location/Screen/Top/70 px,/Attribute/Location/Screen/Left/23 px', + ',Action/Reach/To touch,(Attribute/Object side/Left,Participant/Effect/Body part/Arm),Attribute/Location/Screen/Top/70 px,Attribute/Location/Screen/Left/23 px', extraClosingComma: - '/Action/Reach/To touch,(/Attribute/Object side/Left,/Participant/Effect/Body part/Arm),/Attribute/Location/Screen/Top/70 px,/Attribute/Location/Screen/Left/23 px,', + 'Action/Reach/To touch,(Attribute/Object side/Left,Participant/Effect/Body part/Arm),/Attribute/Location/Screen/Top/70 px,Attribute/Location/Screen/Left/23 px,', multipleExtraOpeningDelimiter: - ',,/Action/Reach/To touch,(/Attribute/Object side/Left,/Participant/Effect/Body part/Arm),/Attribute/Location/Screen/Top/70 px,/Attribute/Location/Screen/Left/23 px', + ',,Action/Reach/To touch,(Attribute/Object side/Left,Participant/Effect/Body part/Arm),Attribute/Location/Screen/Top/70 px,Attribute/Location/Screen/Left/23 px', multipleExtraClosingDelimiter: - '/Action/Reach/To touch,(/Attribute/Object side/Left,/Participant/Effect/Body part/Arm),/Attribute/Location/Screen/Top/70 px,/Attribute/Location/Screen/Left/23 px,,', + 'Action/Reach/To touch,(Attribute/Object side/Left,Participant/Effect/Body part/Arm),/Attribute/Location/Screen/Top/70 px,Attribute/Location/Screen/Left/23 px,,', multipleExtraMiddleDelimiter: - '/Action/Reach/To touch,,(/Attribute/Object side/Left,/Participant/Effect/Body part/Arm),/Attribute/Location/Screen/Top/70 px,,/Attribute/Location/Screen/Left/23 px', + 'Action/Reach/To touch,,(Attribute/Object side/Left,Participant/Effect/Body part/Arm),Attribute/Location/Screen/Top/70 px,,Attribute/Location/Screen/Left/23 px', valid: - '/Action/Reach/To touch,(/Attribute/Object side/Left,/Participant/Effect/Body part/Arm),/Attribute/Location/Screen/Top/70 px,/Attribute/Location/Screen/Left/23 px', + 'Action/Reach/To touch,(Attribute/Object side/Left,Participant/Effect/Body part/Arm),Attribute/Location/Screen/Top/70 px,Attribute/Location/Screen/Left/23 px', validDoubleOpeningParentheses: - '/Action/Reach/To touch,((/Attribute/Object side/Left,/Participant/Effect/Body part/Arm),/Attribute/Location/Screen/Top/70 px,/Attribute/Location/Screen/Left/23 px),Event/Duration/3 ms', + 'Action/Reach/To touch,((Attribute/Object side/Left,Participant/Effect/Body part/Arm),Attribute/Location/Screen/Top/70 px,Attribute/Location/Screen/Left/23 px),Event/Duration/3 ms', validDoubleClosingParentheses: - '/Action/Reach/To touch,(/Attribute/Object side/Left,/Participant/Effect/Body part/Arm,(/Attribute/Location/Screen/Top/70 px,/Attribute/Location/Screen/Left/23 px)),Event/Duration/3 ms', + 'Action/Reach/To touch,(Attribute/Object side/Left,Participant/Effect/Body part/Arm,(Attribute/Location/Screen/Top/70 px,Attribute/Location/Screen/Left/23 px)),Event/Duration/3 ms', } const expectedIssues = { - missingOpeningComma: [generateIssue('commaMissing', { tag: '/Action/Reach/To touch(' })], + missingOpeningComma: [generateIssue('commaMissing', { tag: 'Action/Reach/To touch(' })], missingClosingComma: [ generateIssue('commaMissing', { - tag: '/Participant/Effect/Body part/Arm)', + tag: 'Participant/Effect/Body part/Arm)', }), ], extraOpeningComma: [ @@ -158,12 +155,12 @@ describe('HED string and event validation', () => { multipleExtraMiddleDelimiter: [ generateIssue('extraDelimiter', { character: ',', - index: 23, + index: 22, string: testStrings.multipleExtraMiddleDelimiter, }), generateIssue('extraDelimiter', { character: ',', - index: 125, + index: 121, string: testStrings.multipleExtraMiddleDelimiter, }), ], @@ -178,68 +175,68 @@ describe('HED string and event validation', () => { it('should not have invalid characters', () => { const testStrings = { - openingBrace: '/Attribute/Object side/Left,/Participant/Effect{/Body part/Arm', - closingBrace: '/Attribute/Object side/Left,/Participant/Effect}/Body part/Arm', - openingBracket: '/Attribute/Object side/Left,/Participant/Effect[/Body part/Arm', - closingBracket: '/Attribute/Object side/Left,/Participant/Effect]/Body part/Arm', - tilde: '/Attribute/Object side/Left,/Participant/Effect~/Body part/Arm', - doubleQuote: '/Attribute/Object side/Left,/Participant/Effect"/Body part/Arm', - null: '/Attribute/Object side/Left,/Participant/Effect/Body part/Arm\0', - tab: '/Attribute/Object side/Left,/Participant/Effect/Body part/Arm\t', + openingBrace: 'Attribute/Object side/Left,Participant/Effect{Body part/Arm', + closingBrace: 'Attribute/Object side/Left,Participant/Effect}/Body part/Arm', + openingBracket: 'Attribute/Object side/Left,Participant/Effect[Body part/Arm', + closingBracket: 'Attribute/Object side/Left,Participant/Effect]Body part/Arm', + tilde: 'Attribute/Object side/Left,Participant/Effect~/Body part/Arm', + doubleQuote: 'Attribute/Object side/Left,Participant/Effect"/Body part/Arm', + null: 'Attribute/Object side/Left,Participant/Effect/Body part/Arm\0', + tab: 'Attribute/Object side/Left,Participant/Effect/Body part/Arm\t', } const expectedIssues = { openingBrace: [ generateIssue('invalidCharacter', { character: 'LEFT CURLY BRACKET', - index: 47, + index: 45, string: testStrings.openingBrace, }), ], closingBrace: [ generateIssue('unopenedCurlyBrace', { - index: 47, + index: 45, string: testStrings.closingBrace, }), ], openingBracket: [ generateIssue('invalidCharacter', { character: 'LEFT SQUARE BRACKET', - index: 47, + index: 45, string: testStrings.openingBracket, }), ], closingBracket: [ generateIssue('invalidCharacter', { character: 'RIGHT SQUARE BRACKET', - index: 47, + index: 45, string: testStrings.closingBracket, }), ], tilde: [ generateIssue('invalidCharacter', { character: 'TILDE', - index: 47, + index: 45, string: testStrings.tilde, }), ], doubleQuote: [ generateIssue('invalidCharacter', { character: 'QUOTATION MARK', - index: 47, + index: 45, string: testStrings.doubleQuote, }), ], null: [ generateIssue('invalidCharacter', { character: 'NULL', - index: 61, + index: 59, string: testStrings.null, }), ], tab: [ generateIssue('invalidCharacter', { character: 'CHARACTER TABULATION', - index: 61, + index: 59, string: testStrings.tab, }), ], @@ -351,491 +348,6 @@ describe('HED string and event validation', () => { }) }) - describe('HED-2G validation', () => { - describe('Later HED-2G schemas', () => { - const hedSchemaFile = 'tests/data/HED7.1.1.xml' - let hedSchemas - - beforeAll(async () => { - const spec1 = new SchemaSpec('', '7.1.1', '', hedSchemaFile) - const specs = new SchemasSpec().addSchemaSpec(spec1) - hedSchemas = await buildSchemas(specs) - }) - - /** - * HED 2 semantic validation base function. - * - * This base function uses the HED 2-specific {@link Hed2Validator} validator class. - * - * @param {Object} testStrings A mapping of test strings. - * @param {Object} expectedIssues The expected issues for each test string. - * @param {function(HedValidator): void} testFunction A test-specific function that executes the required validation check. - * @param {Object?} testOptions Any needed custom options for the validator. - */ - const validatorSemanticBase = function (testStrings, expectedIssues, testFunction, testOptions = {}) { - validatorBase(hedSchemas, Hed2Validator, testStrings, expectedIssues, testFunction, testOptions) - } - - describe('Full HED Strings', () => { - const validatorSemantic = validatorSemanticBase - - // TODO: Rewrite as HED 3 test - it.skip('should not validate strings with extensions that are valid node names', () => { - const testStrings = { - // Event/Duration/20 cm is an obviously invalid tag that should not be caught due to the first error. - red: 'Attribute/Red, Event/Duration/20 cm', - redAndBlue: 'Attribute/Red, Attribute/Blue, Event/Duration/20 cm', - } - const expectedIssues = { - red: [ - generateIssue('invalidParentNode', { - tag: 'Red', - parentTag: 'Attribute/Visual/Color/Red', - }), - ], - redAndBlue: [ - generateIssue('invalidParentNode', { - tag: 'Red', - parentTag: 'Attribute/Visual/Color/Red', - }), - generateIssue('invalidParentNode', { - tag: 'Blue', - parentTag: 'Attribute/Visual/Color/Blue', - }), - ], - } - // This is a no-op function since this is checked during string parsing. - return validatorSemantic( - testStrings, - expectedIssues, - // eslint-disable-next-line no-unused-vars - (validator) => {}, - ) - }) - }) - - describe('Individual HED Tags', () => { - /** - * HED 2 individual tag semantic validation base function. - * - * @param {Object} testStrings A mapping of test strings. - * @param {Object} expectedIssues The expected issues for each test string. - * @param {function(HedValidator, ParsedHedTag, ParsedHedTag): void} testFunction A test-specific function that executes the required validation check. - * @param {Object?} testOptions Any needed custom options for the validator. - */ - const validatorSemantic = function (testStrings, expectedIssues, testFunction, testOptions) { - return validatorSemanticBase( - testStrings, - expectedIssues, - (validator) => { - let previousTag = new ParsedHedTag('', '', [0, 0], validator.hedSchemas) - for (const tag of validator.parsedString.tags) { - testFunction(validator, tag, previousTag) - previousTag = tag - } - }, - testOptions, - ) - } - - it('should exist in the schema or be an allowed extension', () => { - const testStrings = { - takesValue: 'Event/Duration/3 ms', - full: 'Attribute/Object side/Left', - extensionAllowed: 'Item/Object/Person/Driver', - leafExtension: 'Event/Category/Initial context/Something', - nonExtensionAllowed: 'Event/Nonsense', - illegalComma: 'Event/Label/This is a label,This/Is/A/Tag', - placeholder: 'Item/Object/#', - } - const expectedIssues = { - takesValue: [], - full: [], - extensionAllowed: [generateIssue('extension', { tag: testStrings.extensionAllowed })], - leafExtension: [generateIssue('invalidTag', { tag: testStrings.leafExtension })], - nonExtensionAllowed: [ - generateIssue('invalidTag', { - tag: testStrings.nonExtensionAllowed, - }), - ], - illegalComma: [ - generateIssue('extraCommaOrInvalid', { - previousTag: 'Event/Label/This is a label', - tag: 'This/Is/A/Tag', - }), - ], - placeholder: [ - generateIssue('invalidTag', { - tag: testStrings.placeholder, - }), - ], - } - return validatorSemantic( - testStrings, - expectedIssues, - (validator, tag, previousTag) => { - validator.checkIfTagIsValid(tag, previousTag) - }, - { checkForWarnings: true }, - ) - }) - - it('should have a child when required', () => { - const testStrings = { - hasChild: 'Event/Category/Experimental stimulus', - missingChild: 'Event/Category', - } - const expectedIssues = { - hasChild: [], - missingChild: [generateIssue('childRequired', { tag: testStrings.missingChild })], - } - return validatorSemantic( - testStrings, - expectedIssues, - // eslint-disable-next-line no-unused-vars - (validator, tag, previousTag) => { - validator.checkIfTagRequiresChild(tag) - }, - { checkForWarnings: true }, - ) - }) - - it('should have a proper unit when required', () => { - const testStrings = { - correctUnit: 'Event/Duration/3 ms', - correctUnitScientific: 'Event/Duration/3.5e1 ms', - correctSingularUnit: 'Event/Duration/1 millisecond', - correctPluralUnit: 'Event/Duration/3 milliseconds', - correctNoPluralUnit: 'Attribute/Temporal rate/3 hertz', - correctPrefixUnit: 'Participant/Effect/Cognitive/Reward/$19.69', - correctNonSymbolCapitalizedUnit: 'Event/Duration/3 MilliSeconds', - correctSymbolCapitalizedUnit: 'Attribute/Temporal rate/3 kHz', - missingRequiredUnit: 'Event/Duration/3', - incorrectUnit: 'Event/Duration/3 cm', - incorrectNonNumericValue: 'Event/Duration/A ms', - incorrectPluralUnit: 'Attribute/Temporal rate/3 hertzs', - incorrectSymbolCapitalizedUnit: 'Attribute/Temporal rate/3 hz', - incorrectSymbolCapitalizedUnitModifier: 'Attribute/Temporal rate/3 KHz', - incorrectNonSIUnitModifier: 'Event/Duration/1 millihour', - incorrectNonSIUnitSymbolModifier: 'Attribute/Path/Velocity/100 Mkph', - notRequiredNumber: 'Attribute/Visual/Color/Red/0.5', - notRequiredScientific: 'Attribute/Visual/Color/Red/5e-1', - properTime: 'Item/2D shape/Clock face/08:30', - invalidTime: 'Item/2D shape/Clock face/54:54', - } - const legalTimeUnits = ['s', 'second', 'day', 'minute', 'hour'] - const legalFrequencyUnits = ['Hz', 'hertz'] - const legalSpeedUnits = ['m-per-s', 'kph', 'mph'] - const expectedIssues = { - correctUnit: [], - correctUnitScientific: [], - correctSingularUnit: [], - correctPluralUnit: [], - correctNoPluralUnit: [], - correctPrefixUnit: [], - correctNonSymbolCapitalizedUnit: [], - correctSymbolCapitalizedUnit: [], - missingRequiredUnit: [ - generateIssue('unitClassDefaultUsed', { - defaultUnit: 's', - tag: testStrings.missingRequiredUnit, - }), - ], - incorrectUnit: [ - generateIssue('unitClassInvalidUnit', { - tag: testStrings.incorrectUnit, - unitClassUnits: legalTimeUnits.sort().join(','), - }), - ], - incorrectNonNumericValue: [ - generateIssue('invalidValue', { - tag: testStrings.incorrectNonNumericValue, - }), - ], - incorrectPluralUnit: [ - generateIssue('unitClassInvalidUnit', { - tag: testStrings.incorrectPluralUnit, - unitClassUnits: legalFrequencyUnits.sort().join(','), - }), - ], - incorrectSymbolCapitalizedUnit: [ - generateIssue('unitClassInvalidUnit', { - tag: testStrings.incorrectSymbolCapitalizedUnit, - unitClassUnits: legalFrequencyUnits.sort().join(','), - }), - ], - incorrectSymbolCapitalizedUnitModifier: [ - generateIssue('unitClassInvalidUnit', { - tag: testStrings.incorrectSymbolCapitalizedUnitModifier, - unitClassUnits: legalFrequencyUnits.sort().join(','), - }), - ], - incorrectNonSIUnitModifier: [ - generateIssue('unitClassInvalidUnit', { - tag: testStrings.incorrectNonSIUnitModifier, - unitClassUnits: legalTimeUnits.sort().join(','), - }), - ], - incorrectNonSIUnitSymbolModifier: [ - generateIssue('unitClassInvalidUnit', { - tag: testStrings.incorrectNonSIUnitSymbolModifier, - unitClassUnits: legalSpeedUnits.sort().join(','), - }), - ], - notRequiredNumber: [], - notRequiredScientific: [], - properTime: [], - invalidTime: [ - generateIssue('invalidValue', { - tag: testStrings.invalidTime, - }), - ], - } - return validatorSemantic( - testStrings, - expectedIssues, - // eslint-disable-next-line no-unused-vars - (validator, tag, previousTag) => { - validator.checkIfTagUnitClassUnitsAreValid(tag) - }, - { checkForWarnings: true }, - ) - }) - }) - - describe('HED Tag Levels', () => { - /** - * HED 2 Tag level semantic validation base function. - * - * @param {Object} testStrings A mapping of test strings. - * @param {Object} expectedIssues The expected issues for each test string. - * @param {function(HedValidator, ParsedHedSubstring[]): void} testFunction A test-specific function that executes the required validation check. - * @param {Object?} testOptions Any needed custom options for the validator. - */ - const validatorSemantic = function (testStrings, expectedIssues, testFunction, testOptions = {}) { - return validatorSemanticBase( - testStrings, - expectedIssues, - (validator) => { - for (const tagGroup of validator.parsedString.tagGroups) { - for (const subGroup of tagGroup.subGroupArrayIterator()) { - testFunction(validator, subGroup) - } - } - testFunction(validator, validator.parsedString.parseTree) - }, - testOptions, - ) - } - - it('should not have multiple copies of a unique tag', () => { - const testStrings = { - legal: - 'Event/Description/Rail vehicles,Item/Object/Vehicle/Train,(Item/Object/Vehicle/Train,Event/Category/Experimental stimulus)', - multipleDesc: - 'Event/Description/Rail vehicles,Event/Description/Locomotive-pulled or multiple units,Item/Object/Vehicle/Train,(Item/Object/Vehicle/Train,Event/Category/Experimental stimulus)', - } - const expectedIssues = { - legal: [], - multipleDesc: [generateIssue('multipleUniqueTags', { tag: 'event/description' })], - } - return validatorSemantic(testStrings, expectedIssues, (validator, tagLevel) => { - validator.checkForMultipleUniqueTags(tagLevel) - }) - }) - }) - - describe('Top-level Tags', () => { - const validatorSemantic = validatorSemanticBase - - it('should include all required tags', () => { - const testStrings = { - complete: - 'Event/Label/Bus,Event/Category/Experimental stimulus,Event/Description/Shown a picture of a bus,Item/Object/Vehicle/Bus', - missingLabel: - 'Event/Category/Experimental stimulus,Event/Description/Shown a picture of a bus,Item/Object/Vehicle/Bus', - missingCategory: 'Event/Label/Bus,Event/Description/Shown a picture of a bus,Item/Object/Vehicle/Bus', - missingDescription: 'Event/Label/Bus,Event/Category/Experimental stimulus,Item/Object/Vehicle/Bus', - missingAllRequired: 'Item/Object/Vehicle/Bus', - } - const expectedIssues = { - complete: [], - missingLabel: [ - generateIssue('requiredPrefixMissing', { - tagPrefix: 'event/label', - }), - ], - missingCategory: [ - generateIssue('requiredPrefixMissing', { - tagPrefix: 'event/category', - }), - ], - missingDescription: [ - generateIssue('requiredPrefixMissing', { - tagPrefix: 'event/description', - }), - ], - missingAllRequired: [ - generateIssue('requiredPrefixMissing', { - tagPrefix: 'event/label', - }), - generateIssue('requiredPrefixMissing', { - tagPrefix: 'event/category', - }), - generateIssue('requiredPrefixMissing', { - tagPrefix: 'event/description', - }), - ], - } - return validatorSemantic( - testStrings, - expectedIssues, - (validator) => { - validator.checkForRequiredTags() - }, - { checkForWarnings: true }, - ) - }) - }) - }) - - describe('Pre-v7.1.0 HED schemas', () => { - const hedSchemaFile = 'tests/data/HED7.0.4.xml' - let hedSchemas - - beforeAll(async () => { - const spec2 = new SchemaSpec('', '7.0.4', '', hedSchemaFile) - const specs = new SchemasSpec().addSchemaSpec(spec2) - hedSchemas = await buildSchemas(specs) - }) - - /** - * HED 2 semantic validation base function. - * - * This base function uses the HED 2-specific {@link Hed2Validator} validator class. - * - * @param {Object} testStrings A mapping of test strings. - * @param {Object} expectedIssues The expected issues for each test string. - * @param {function(HedValidator): void} testFunction A test-specific function that executes the required validation check. - * @param {Object?} testOptions Any needed custom options for the validator. - */ - const validatorSemanticBase = function (testStrings, expectedIssues, testFunction, testOptions = {}) { - validatorBase(hedSchemas, Hed2Validator, testStrings, expectedIssues, testFunction, testOptions) - } - - describe('Individual HED Tags', () => { - /** - * HED 2 individual tag semantic validation base function. - * - * @param {Object} testStrings A mapping of test strings. - * @param {Object} expectedIssues The expected issues for each test string. - * @param {function(HedValidator, ParsedHedTag, ParsedHedTag): void} testFunction A test-specific function that executes the required validation check. - * @param {Object?} testOptions Any needed custom options for the validator. - */ - const validatorSemantic = function (testStrings, expectedIssues, testFunction, testOptions) { - return validatorSemanticBase( - testStrings, - expectedIssues, - (validator) => { - let previousTag = new ParsedHedTag('', '', [0, 0], validator.hedSchemas) - for (const tag of validator.parsedString.tags) { - testFunction(validator, tag, previousTag) - previousTag = tag - } - }, - testOptions, - ) - } - - it('should have a proper unit when required', () => { - const testStrings = { - correctUnit: 'Event/Duration/3 ms', - correctUnitWord: 'Event/Duration/3 milliseconds', - correctUnitScientific: 'Event/Duration/3.5e1 ms', - missingRequiredUnit: 'Event/Duration/3', - incorrectUnit: 'Event/Duration/3 cm', - incorrectNonNumericValue: 'Event/Duration/A ms', - incorrectUnitWord: 'Event/Duration/3 nanoseconds', - incorrectModifier: 'Event/Duration/3 ns', - notRequiredNumber: 'Attribute/Visual/Color/Red/0.5', - notRequiredScientific: 'Attribute/Visual/Color/Red/5e-1', - properTime: 'Item/2D shape/Clock face/08:30', - invalidTime: 'Item/2D shape/Clock face/54:54', - } - const legalTimeUnits = [ - 's', - 'second', - 'seconds', - 'centiseconds', - 'centisecond', - 'cs', - 'hour:min', - 'day', - 'days', - 'ms', - 'milliseconds', - 'millisecond', - 'minute', - 'minutes', - 'hour', - 'hours', - ] - const expectedIssues = { - correctUnit: [], - correctUnitWord: [], - correctUnitScientific: [], - missingRequiredUnit: [ - generateIssue('unitClassDefaultUsed', { - defaultUnit: 's', - tag: testStrings.missingRequiredUnit, - }), - ], - incorrectUnit: [ - generateIssue('unitClassInvalidUnit', { - tag: testStrings.incorrectUnit, - unitClassUnits: legalTimeUnits.sort().join(','), - }), - ], - incorrectNonNumericValue: [ - generateIssue('invalidValue', { - tag: testStrings.incorrectNonNumericValue, - }), - ], - incorrectUnitWord: [ - generateIssue('unitClassInvalidUnit', { - tag: testStrings.incorrectUnitWord, - unitClassUnits: legalTimeUnits.sort().join(','), - }), - ], - incorrectModifier: [ - generateIssue('unitClassInvalidUnit', { - tag: testStrings.incorrectModifier, - unitClassUnits: legalTimeUnits.sort().join(','), - }), - ], - notRequiredNumber: [], - notRequiredScientific: [], - properTime: [], - invalidTime: [ - generateIssue('invalidValue', { - tag: testStrings.invalidTime, - }), - ], - } - return validatorSemantic( - testStrings, - expectedIssues, - // eslint-disable-next-line no-unused-vars - (validator, tag, previousTag) => { - validator.checkIfTagUnitClassUnitsAreValid(tag) - }, - { checkForWarnings: true }, - ) - }) - }) - }) - }) - describe('HED-3G validation', () => { const hedSchemaFile = 'tests/data/HED8.2.0.xml' let hedSchemas diff --git a/tests/event2G.spec.js b/tests/event2G.spec.js new file mode 100644 index 00000000..fb462dee --- /dev/null +++ b/tests/event2G.spec.js @@ -0,0 +1,530 @@ +import chai from 'chai' +const assert = chai.assert +import { beforeAll, describe, it } from '@jest/globals' + +import * as hed from '../validator/event' +import { buildSchemas } from '../validator/schema/init' +import { parseHedString } from '../parser/parser' +import { ParsedHedTag } from '../parser/parsedHedTag' +import { HedValidator, Hed2Validator, Hed3Validator } from '../validator/event' +import { generateIssue } from '../common/issues/issues' +import { Schemas, SchemaSpec, SchemasSpec } from '../common/schema/types' + +describe('HED string and event validation', () => { + /** + * Validation base function. + * + * @param {Schemas} hedSchemas The HED schema collection used for testing. + * @param {typeof HedValidator} ValidatorClass A subclass of {@link HedValidator} to use for validation. + * @param {Object} testStrings A mapping of test strings. + * @param {Object} expectedIssues The expected issues for each test string. + * @param {function(HedValidator): void} testFunction A test-specific function that executes the required validation check. + * @param {Object?} testOptions Any needed custom options for the validator. + */ + const validatorBase = function ( + hedSchemas, + ValidatorClass, + testStrings, + expectedIssues, + testFunction, + testOptions = {}, + ) { + for (const [testStringKey, testString] of Object.entries(testStrings)) { + assert.property(expectedIssues, testStringKey, testStringKey + ' is not in expectedIssues') + const [parsedTestString, parsingIssues] = parseHedString(testString, hedSchemas) + const validator = new ValidatorClass(parsedTestString, hedSchemas, testOptions) + const flattenedParsingIssues = Object.values(parsingIssues).flat() + if (flattenedParsingIssues.length === 0) { + testFunction(validator) + } + const issues = [].concat(flattenedParsingIssues, validator.issues) + assert.sameDeepMembers(issues, expectedIssues[testStringKey], testString) + } + } + + describe.skip('HED-2G validation', () => { + describe('Later HED-2G schemas', () => { + const hedSchemaFile = 'tests/data/HED7.1.1.xml' + let hedSchemas + + beforeAll(async () => { + const spec1 = new SchemaSpec('', '7.1.1', '', hedSchemaFile) + const specs = new SchemasSpec().addSchemaSpec(spec1) + hedSchemas = await buildSchemas(specs) + }) + + /** + * HED 2 semantic validation base function. + * + * This base function uses the HED 2-specific {@link Hed2Validator} validator class. + * + * @param {Object} testStrings A mapping of test strings. + * @param {Object} expectedIssues The expected issues for each test string. + * @param {function(HedValidator): void} testFunction A test-specific function that executes the required validation check. + * @param {Object?} testOptions Any needed custom options for the validator. + */ + const validatorSemanticBase = function (testStrings, expectedIssues, testFunction, testOptions = {}) { + validatorBase(hedSchemas, Hed2Validator, testStrings, expectedIssues, testFunction, testOptions) + } + + describe('Full HED Strings', () => { + const validatorSemantic = validatorSemanticBase + + // TODO: Rewrite as HED 3 test + it.skip('should not validate strings with extensions that are valid node names', () => { + const testStrings = { + // Event/Duration/20 cm is an obviously invalid tag that should not be caught due to the first error. + red: 'Attribute/Red, Event/Duration/20 cm', + redAndBlue: 'Attribute/Red, Attribute/Blue, Event/Duration/20 cm', + } + const expectedIssues = { + red: [ + generateIssue('invalidParentNode', { + tag: 'Red', + parentTag: 'Attribute/Visual/Color/Red', + }), + ], + redAndBlue: [ + generateIssue('invalidParentNode', { + tag: 'Red', + parentTag: 'Attribute/Visual/Color/Red', + }), + generateIssue('invalidParentNode', { + tag: 'Blue', + parentTag: 'Attribute/Visual/Color/Blue', + }), + ], + } + // This is a no-op function since this is checked during string parsing. + return validatorSemantic( + testStrings, + expectedIssues, + // eslint-disable-next-line no-unused-vars + (validator) => {}, + ) + }) + }) + + describe('Individual HED Tags', () => { + /** + * HED 2 individual tag semantic validation base function. + * + * @param {Object} testStrings A mapping of test strings. + * @param {Object} expectedIssues The expected issues for each test string. + * @param {function(HedValidator, ParsedHedTag, ParsedHedTag): void} testFunction A test-specific function that executes the required validation check. + * @param {Object?} testOptions Any needed custom options for the validator. + */ + const validatorSemantic = function (testStrings, expectedIssues, testFunction, testOptions) { + return validatorSemanticBase( + testStrings, + expectedIssues, + (validator) => { + let previousTag = new ParsedHedTag('', '', [0, 0], validator.hedSchemas) + for (const tag of validator.parsedString.tags) { + testFunction(validator, tag, previousTag) + previousTag = tag + } + }, + testOptions, + ) + } + //TODO: Rewrite for HED-3 + it('should exist in the schema or be an allowed extension', () => { + const testStrings = { + takesValue: 'Event/Duration/3 ms', + full: 'Attribute/Object side/Left', + extensionAllowed: 'Item/Object/Person/Driver', + leafExtension: 'Event/Category/Initial context/Something', + nonExtensionAllowed: 'Event/Nonsense', + illegalComma: 'Event/Label/This is a label,This/Is/A/Tag', + placeholder: 'Item/Object/#', + } + const expectedIssues = { + takesValue: [], + full: [], + extensionAllowed: [generateIssue('extension', { tag: testStrings.extensionAllowed })], + leafExtension: [generateIssue('invalidTag', { tag: testStrings.leafExtension })], + nonExtensionAllowed: [ + generateIssue('invalidTag', { + tag: testStrings.nonExtensionAllowed, + }), + ], + illegalComma: [ + generateIssue('extraCommaOrInvalid', { + previousTag: 'Event/Label/This is a label', + tag: 'This/Is/A/Tag', + }), + ], + placeholder: [ + generateIssue('invalidTag', { + tag: testStrings.placeholder, + }), + ], + } + return validatorSemantic( + testStrings, + expectedIssues, + (validator, tag, previousTag) => { + validator.checkIfTagIsValid(tag, previousTag) + }, + { checkForWarnings: true }, + ) + }) + + it('should have a child when required', () => { + const testStrings = { + hasChild: 'Event/Category/Experimental stimulus', + missingChild: 'Event/Category', + } + const expectedIssues = { + hasChild: [], + missingChild: [generateIssue('childRequired', { tag: testStrings.missingChild })], + } + return validatorSemantic( + testStrings, + expectedIssues, + // eslint-disable-next-line no-unused-vars + (validator, tag, previousTag) => { + validator.checkIfTagRequiresChild(tag) + }, + { checkForWarnings: true }, + ) + }) + + it('should have a proper unit when required', () => { + const testStrings = { + correctUnit: 'Event/Duration/3 ms', + correctUnitScientific: 'Event/Duration/3.5e1 ms', + correctSingularUnit: 'Event/Duration/1 millisecond', + correctPluralUnit: 'Event/Duration/3 milliseconds', + correctNoPluralUnit: 'Attribute/Temporal rate/3 hertz', + correctPrefixUnit: 'Participant/Effect/Cognitive/Reward/$19.69', + correctNonSymbolCapitalizedUnit: 'Event/Duration/3 MilliSeconds', + correctSymbolCapitalizedUnit: 'Attribute/Temporal rate/3 kHz', + missingRequiredUnit: 'Event/Duration/3', + incorrectUnit: 'Event/Duration/3 cm', + incorrectNonNumericValue: 'Event/Duration/A ms', + incorrectPluralUnit: 'Attribute/Temporal rate/3 hertzs', + incorrectSymbolCapitalizedUnit: 'Attribute/Temporal rate/3 hz', + incorrectSymbolCapitalizedUnitModifier: 'Attribute/Temporal rate/3 KHz', + incorrectNonSIUnitModifier: 'Event/Duration/1 millihour', + incorrectNonSIUnitSymbolModifier: 'Attribute/Path/Velocity/100 Mkph', + notRequiredNumber: 'Attribute/Visual/Color/Red/0.5', + notRequiredScientific: 'Attribute/Visual/Color/Red/5e-1', + properTime: 'Item/2D shape/Clock face/08:30', + invalidTime: 'Item/2D shape/Clock face/54:54', + } + const legalTimeUnits = ['s', 'second', 'day', 'minute', 'hour'] + const legalFrequencyUnits = ['Hz', 'hertz'] + const legalSpeedUnits = ['m-per-s', 'kph', 'mph'] + const expectedIssues = { + correctUnit: [], + correctUnitScientific: [], + correctSingularUnit: [], + correctPluralUnit: [], + correctNoPluralUnit: [], + correctPrefixUnit: [], + correctNonSymbolCapitalizedUnit: [], + correctSymbolCapitalizedUnit: [], + missingRequiredUnit: [ + generateIssue('unitClassDefaultUsed', { + defaultUnit: 's', + tag: testStrings.missingRequiredUnit, + }), + ], + incorrectUnit: [ + generateIssue('unitClassInvalidUnit', { + tag: testStrings.incorrectUnit, + unitClassUnits: legalTimeUnits.sort().join(','), + }), + ], + incorrectNonNumericValue: [ + generateIssue('invalidValue', { + tag: testStrings.incorrectNonNumericValue, + }), + ], + incorrectPluralUnit: [ + generateIssue('unitClassInvalidUnit', { + tag: testStrings.incorrectPluralUnit, + unitClassUnits: legalFrequencyUnits.sort().join(','), + }), + ], + incorrectSymbolCapitalizedUnit: [ + generateIssue('unitClassInvalidUnit', { + tag: testStrings.incorrectSymbolCapitalizedUnit, + unitClassUnits: legalFrequencyUnits.sort().join(','), + }), + ], + incorrectSymbolCapitalizedUnitModifier: [ + generateIssue('unitClassInvalidUnit', { + tag: testStrings.incorrectSymbolCapitalizedUnitModifier, + unitClassUnits: legalFrequencyUnits.sort().join(','), + }), + ], + incorrectNonSIUnitModifier: [ + generateIssue('unitClassInvalidUnit', { + tag: testStrings.incorrectNonSIUnitModifier, + unitClassUnits: legalTimeUnits.sort().join(','), + }), + ], + incorrectNonSIUnitSymbolModifier: [ + generateIssue('unitClassInvalidUnit', { + tag: testStrings.incorrectNonSIUnitSymbolModifier, + unitClassUnits: legalSpeedUnits.sort().join(','), + }), + ], + notRequiredNumber: [], + notRequiredScientific: [], + properTime: [], + invalidTime: [ + generateIssue('invalidValue', { + tag: testStrings.invalidTime, + }), + ], + } + return validatorSemantic( + testStrings, + expectedIssues, + // eslint-disable-next-line no-unused-vars + (validator, tag, previousTag) => { + validator.checkIfTagUnitClassUnitsAreValid(tag) + }, + { checkForWarnings: true }, + ) + }) + }) + + //TODO: Replace with HED-3 + describe('HED Tag Levels', () => { + /** + * HED 2 Tag level semantic validation base function. + * + * @param {Object} testStrings A mapping of test strings. + * @param {Object} expectedIssues The expected issues for each test string. + * @param {function(HedValidator, ParsedHedSubstring[]): void} testFunction A test-specific function that executes the required validation check. + * @param {Object?} testOptions Any needed custom options for the validator. + */ + const validatorSemantic = function (testStrings, expectedIssues, testFunction, testOptions = {}) { + return validatorSemanticBase( + testStrings, + expectedIssues, + (validator) => { + for (const tagGroup of validator.parsedString.tagGroups) { + for (const subGroup of tagGroup.subGroupArrayIterator()) { + testFunction(validator, subGroup) + } + } + testFunction(validator, validator.parsedString.parseTree) + }, + testOptions, + ) + } + + it('should not have multiple copies of a unique tag', () => { + const testStrings = { + legal: + 'Event/Description/Rail vehicles,Item/Object/Vehicle/Train,(Item/Object/Vehicle/Train,Event/Category/Experimental stimulus)', + multipleDesc: + 'Event/Description/Rail vehicles,Event/Description/Locomotive-pulled or multiple units,Item/Object/Vehicle/Train,(Item/Object/Vehicle/Train,Event/Category/Experimental stimulus)', + } + const expectedIssues = { + legal: [], + multipleDesc: [generateIssue('multipleUniqueTags', { tag: 'event/description' })], + } + return validatorSemantic(testStrings, expectedIssues, (validator, tagLevel) => { + validator.checkForMultipleUniqueTags(tagLevel) + }) + }) + }) + + describe('Top-level Tags', () => { + const validatorSemantic = validatorSemanticBase + + it('should include all required tags', () => { + const testStrings = { + complete: + 'Event/Label/Bus,Event/Category/Experimental stimulus,Event/Description/Shown a picture of a bus,Item/Object/Vehicle/Bus', + missingLabel: + 'Event/Category/Experimental stimulus,Event/Description/Shown a picture of a bus,Item/Object/Vehicle/Bus', + missingCategory: 'Event/Label/Bus,Event/Description/Shown a picture of a bus,Item/Object/Vehicle/Bus', + missingDescription: 'Event/Label/Bus,Event/Category/Experimental stimulus,Item/Object/Vehicle/Bus', + missingAllRequired: 'Item/Object/Vehicle/Bus', + } + const expectedIssues = { + complete: [], + missingLabel: [ + generateIssue('requiredPrefixMissing', { + tagPrefix: 'event/label', + }), + ], + missingCategory: [ + generateIssue('requiredPrefixMissing', { + tagPrefix: 'event/category', + }), + ], + missingDescription: [ + generateIssue('requiredPrefixMissing', { + tagPrefix: 'event/description', + }), + ], + missingAllRequired: [ + generateIssue('requiredPrefixMissing', { + tagPrefix: 'event/label', + }), + generateIssue('requiredPrefixMissing', { + tagPrefix: 'event/category', + }), + generateIssue('requiredPrefixMissing', { + tagPrefix: 'event/description', + }), + ], + } + return validatorSemantic( + testStrings, + expectedIssues, + (validator) => { + validator.checkForRequiredTags() + }, + { checkForWarnings: true }, + ) + }) + }) + }) + + describe('Pre-v7.1.0 HED schemas', () => { + const hedSchemaFile = 'tests/data/HED7.0.4.xml' + let hedSchemas + + beforeAll(async () => { + const spec2 = new SchemaSpec('', '7.0.4', '', hedSchemaFile) + const specs = new SchemasSpec().addSchemaSpec(spec2) + hedSchemas = await buildSchemas(specs) + }) + + /** + * HED 2 semantic validation base function. + * + * This base function uses the HED 2-specific {@link Hed2Validator} validator class. + * + * @param {Object} testStrings A mapping of test strings. + * @param {Object} expectedIssues The expected issues for each test string. + * @param {function(HedValidator): void} testFunction A test-specific function that executes the required validation check. + * @param {Object?} testOptions Any needed custom options for the validator. + */ + const validatorSemanticBase = function (testStrings, expectedIssues, testFunction, testOptions = {}) { + validatorBase(hedSchemas, Hed2Validator, testStrings, expectedIssues, testFunction, testOptions) + } + + describe('Individual HED Tags', () => { + /** + * HED 2 individual tag semantic validation base function. + * + * @param {Object} testStrings A mapping of test strings. + * @param {Object} expectedIssues The expected issues for each test string. + * @param {function(HedValidator, ParsedHedTag, ParsedHedTag): void} testFunction A test-specific function that executes the required validation check. + * @param {Object?} testOptions Any needed custom options for the validator. + */ + const validatorSemantic = function (testStrings, expectedIssues, testFunction, testOptions) { + return validatorSemanticBase( + testStrings, + expectedIssues, + (validator) => { + let previousTag = new ParsedHedTag('', '', [0, 0], validator.hedSchemas) + for (const tag of validator.parsedString.tags) { + testFunction(validator, tag, previousTag) + previousTag = tag + } + }, + testOptions, + ) + } + + it('should have a proper unit when required', () => { + const testStrings = { + correctUnit: 'Event/Duration/3 ms', + correctUnitWord: 'Event/Duration/3 milliseconds', + correctUnitScientific: 'Event/Duration/3.5e1 ms', + missingRequiredUnit: 'Event/Duration/3', + incorrectUnit: 'Event/Duration/3 cm', + incorrectNonNumericValue: 'Event/Duration/A ms', + incorrectUnitWord: 'Event/Duration/3 nanoseconds', + incorrectModifier: 'Event/Duration/3 ns', + notRequiredNumber: 'Attribute/Visual/Color/Red/0.5', + notRequiredScientific: 'Attribute/Visual/Color/Red/5e-1', + properTime: 'Item/2D shape/Clock face/08:30', + invalidTime: 'Item/2D shape/Clock face/54:54', + } + const legalTimeUnits = [ + 's', + 'second', + 'seconds', + 'centiseconds', + 'centisecond', + 'cs', + 'hour:min', + 'day', + 'days', + 'ms', + 'milliseconds', + 'millisecond', + 'minute', + 'minutes', + 'hour', + 'hours', + ] + const expectedIssues = { + correctUnit: [], + correctUnitWord: [], + correctUnitScientific: [], + missingRequiredUnit: [ + generateIssue('unitClassDefaultUsed', { + defaultUnit: 's', + tag: testStrings.missingRequiredUnit, + }), + ], + incorrectUnit: [ + generateIssue('unitClassInvalidUnit', { + tag: testStrings.incorrectUnit, + unitClassUnits: legalTimeUnits.sort().join(','), + }), + ], + incorrectNonNumericValue: [ + generateIssue('invalidValue', { + tag: testStrings.incorrectNonNumericValue, + }), + ], + incorrectUnitWord: [ + generateIssue('unitClassInvalidUnit', { + tag: testStrings.incorrectUnitWord, + unitClassUnits: legalTimeUnits.sort().join(','), + }), + ], + incorrectModifier: [ + generateIssue('unitClassInvalidUnit', { + tag: testStrings.incorrectModifier, + unitClassUnits: legalTimeUnits.sort().join(','), + }), + ], + notRequiredNumber: [], + notRequiredScientific: [], + properTime: [], + invalidTime: [ + generateIssue('invalidValue', { + tag: testStrings.invalidTime, + }), + ], + } + return validatorSemantic( + testStrings, + expectedIssues, + // eslint-disable-next-line no-unused-vars + (validator, tag, previousTag) => { + validator.checkIfTagUnitClassUnitsAreValid(tag) + }, + { checkForWarnings: true }, + ) + }) + }) + }) + }) +}) diff --git a/tests/runLog.txt b/tests/runLog.txt new file mode 100644 index 00000000..2b7c1ad7 --- /dev/null +++ b/tests/runLog.txt @@ -0,0 +1,47 @@ +Total tests:28 Wrong errors:1 MissingErrors:1 +[no-hed-at-all-but-both-tsv-json-non-empty (Expect pass)] +[valid-bids-datasets-with-limited-hed:no-hed-at-all-but-both-tsv-json-non-empty][Sidecar only](Expect pass) +[valid-bids-datasets-with-limited-hed:no-hed-at-all-but-both-tsv-json-non-empty][Events only](Expect pass) +[valid-bids-datasets-with-limited-hed:no-hed-at-all-but-both-tsv-json-non-empty][Events+side](Expect pass) +[only-header-in-tsv-with-return (Expect pass)] +[valid-bids-datasets-with-limited-hed:only-header-in-tsv-with-return][Sidecar only](Expect pass) +[valid-bids-datasets-with-limited-hed:only-header-in-tsv-with-return][Events only](Expect pass) +[valid-bids-datasets-with-limited-hed:only-header-in-tsv-with-return][Events+side](Expect pass) +[empty-json-empty-tsv (Expect pass)] +[valid-bids-datasets-with-limited-hed:empty-json-empty-tsv][Sidecar only](Expect pass) +[valid-bids-datasets-with-limited-hed:empty-json-empty-tsv][Events only](Expect pass) +[valid-bids-datasets-with-limited-hed:empty-json-empty-tsv][Events+side](Expect pass) +[valid-sidecar-bad-tag-tsv (Expect pass)] +[valid-json-invalid-tsv:valid-sidecar-bad-tag-tsv][Sidecar only](Expect pass) +[valid-json-invalid-tsv:valid-sidecar-bad-tag-tsv][Events only](Expect fail) +[valid-json-invalid-tsv:valid-sidecar-bad-tag-tsv][Events+side](Expect fail) +[valid-sidecar-tsv-curly-brace (Expect pass)] +[valid-json-invalid-tsv:valid-sidecar-tsv-curly-brace][Sidecar only](Expect pass) +[valid-json-invalid-tsv:valid-sidecar-tsv-curly-brace][Events only](Expect fail) +[valid-json-invalid-tsv:valid-sidecar-tsv-curly-brace][Events+side](Expect fail) +[first-level-duplicate-json-tsv (Expect pass)] +[duplicate-tag-test:first-level-duplicate-json-tsv][Sidecar only](Expect pass) +[duplicate-tag-test:first-level-duplicate-json-tsv][Events only](Expect pass) +[duplicate-tag-test:first-level-duplicate-json-tsv][Events+side](Expect fail) +---did not receive expected errors TAG_EXPRESSION_REPEATED +Expected issues: +[{"code":104,"file":{"path":"first-level-duplicate-json-tsv.tsv","relativePath":"first-level-duplicate-json-tsv.tsv"},"evidence":"ERROR: [TAG_EXPRESSION_REPEATED] Duplicate tag - \"Boat\". TSV line: 2. (For more information on this HED error, see https://hed-specification.readthedocs.io/en/latest/Appendix_B.html#tag-expression-repeated.)","hedIssue":{"internalCode":"duplicateTag","code":"duplicateTag","hedCode":"TAG_EXPRESSION_REPEATED","level":"error","message":"ERROR: [TAG_EXPRESSION_REPEATED] Duplicate tag - \"Boat\". TSV line: 2. (For more information on this HED error, see https://hed-specification.readthedocs.io/en/latest/Appendix_B.html#tag-expression-repeated.)","parameters":{"tag":"Boat","tsvLine":"2"}}}] +Received issues: +[] +[valid-curly-brace-in-sidecar-with-simple-splice (Expect pass)] +[curly-brace-tests:valid-curly-brace-in-sidecar-with-simple-splice][Sidecar only](Expect pass) +[curly-brace-tests:valid-curly-brace-in-sidecar-with-simple-splice][Events only](Expect pass) +[curly-brace-tests:valid-curly-brace-in-sidecar-with-simple-splice][Events+side](Expect pass) +[valid-curly-brace-in-sidecar-with-n/a-splice (Expect pass)] +[curly-brace-tests:valid-curly-brace-in-sidecar-with-n/a-splice][Sidecar only](Expect pass) +[curly-brace-tests:valid-curly-brace-in-sidecar-with-n/a-splice][Events only](Expect pass) +[curly-brace-tests:valid-curly-brace-in-sidecar-with-n/a-splice][Events+side](Expect pass) +[valid-curly-brace-in-sidecar-with-HED-column-splice (Expect pass)] +[curly-brace-tests:valid-curly-brace-in-sidecar-with-HED-column-splice][Sidecar only](Expect pass) +[curly-brace-tests:valid-curly-brace-in-sidecar-with-HED-column-splice][Events only](Expect pass) +[curly-brace-tests:valid-curly-brace-in-sidecar-with-HED-column-splice][Events+side](Expect pass) +[invalid-curly-brace-column-slice-has-no hed (Expect pass)] +[curly-brace-tests:invalid-curly-brace-column-slice-has-no hed][Sidecar only](Expect pass) +---has errors [SIDECAR_BRACES_INVALID] +---expected no errors but got errors [SIDECAR_BRACES_INVALID] +Received issues: [{"code":104,"file":{"relativePath":"invalid-curly-brace-column-slice-has-no hed.json","path":"invalid-curly-brace-column-slice-has-no hed.json"},"evidence":"ERROR: [SIDECAR_BRACES_INVALID] Column name \"ball_type\", used in curly braces, is not mapped to a defined column. (For more information on this HED error, see https://hed-specification.readthedocs.io/en/latest/Appendix_B.html#sidecar-braces-invalid.)","hedIssue":{"internalCode":"undefinedCurlyBraces","code":"undefinedCurlyBraces","hedCode":"SIDECAR_BRACES_INVALID","level":"error","message":"ERROR: [SIDECAR_BRACES_INVALID] Column name \"ball_type\", used in curly braces, is not mapped to a defined column. (For more information on this HED error, see https://hed-specification.readthedocs.io/en/latest/Appendix_B.html#sidecar-braces-invalid.)","parameters":{"column":"ball_type"}}}] \ No newline at end of file diff --git a/tests/stringParser.spec.js b/tests/stringParser.spec.js index c9335456..0a18d024 100644 --- a/tests/stringParser.spec.js +++ b/tests/stringParser.spec.js @@ -81,7 +81,6 @@ describe('HED string parsing', () => { } const expectedIssues = { openingSquare: { - conversion: [], syntax: [ generateIssue('invalidCharacter', { character: 'LEFT SQUARE BRACKET', @@ -91,7 +90,6 @@ describe('HED string parsing', () => { ], }, closingSquare: { - conversion: [], syntax: [ generateIssue('invalidCharacter', { character: 'RIGHT SQUARE BRACKET', @@ -101,7 +99,6 @@ describe('HED string parsing', () => { ], }, tilde: { - conversion: [], syntax: [ generateIssue('invalidCharacter', { character: 'TILDE', @@ -146,39 +143,42 @@ describe('HED string parsing', () => { it('should include each group as its own single element', () => { const hedString = - '/Action/Move/Flex,(Relation/Spatial-relation/Left-side-of,/Action/Move/Bend,/Upper-extremity/Elbow),/Position/X-position/70 px,/Position/Y-position/23 px' + 'Action/Move/Flex,(Relation/Spatial-relation/Left-side-of,Action/Move/Bend,Upper-extremity/Elbow),Position/X-position/70 px,Position/Y-position/23 px' const [result, issues] = splitHedString(hedString, nullSchema) assert.isEmpty(Object.values(issues).flat(), 'Parsing issues occurred') assert.deepStrictEqual(result, [ - new ParsedHedTag('/Action/Move/Flex', [0, 17]), + new ParsedHedTag('Action/Move/Flex', [0, 16]), new ParsedHedGroup( [ - new ParsedHedTag('Relation/Spatial-relation/Left-side-of', [19, 57]), - new ParsedHedTag('/Action/Move/Bend', [58, 75]), - new ParsedHedTag('/Upper-extremity/Elbow', [76, 98]), + new ParsedHedTag('Relation/Spatial-relation/Left-side-of', [18, 56]), + new ParsedHedTag('Action/Move/Bend', [57, 73]), + new ParsedHedTag('Upper-extremity/Elbow', [74, 95]), ], nullSchema, hedString, - [18, 99], + [17, 96], ), - new ParsedHedTag('/Position/X-position/70 px', [100, 126]), - new ParsedHedTag('/Position/Y-position/23 px', [127, 153]), + new ParsedHedTag('Position/X-position/70 px', [97, 122]), + new ParsedHedTag('Position/Y-position/23 px', [123, 148]), ]) }) it('should not include blanks', () => { const testStrings = { - trailingBlank: '/Item/Object/Man-made-object/Vehicle/Car, /Action/Perform/Operate,', + okay: 'Item/Object/Man-made-object/Vehicle/Car, Action/Perform/Operate', + internalBlank: 'Item Object', } const expectedList = [ - new ParsedHedTag('/Item/Object/Man-made-object/Vehicle/Car', [0, 40]), - new ParsedHedTag('/Action/Perform/Operate', [42, 65]), + new ParsedHedTag('Item/Object/Man-made-object/Vehicle/Car', [0, 39]), + new ParsedHedTag('Action/Perform/Operate', [41, 63]), ] const expectedResults = { - trailingBlank: expectedList, + okay: expectedList, + internalBlank: [new ParsedHedTag('Item Object', [0, 11])], } const expectedIssues = { - trailingBlank: {}, + okay: {}, + internalBlank: {}, } validatorWithIssues(testStrings, expectedResults, expectedIssues, (string) => { return splitHedString(string, nullSchema) @@ -228,31 +228,31 @@ describe('HED string parsing', () => { describe('Parsed HED strings', () => { it('must have the correct number of tags, top-level tags, and groups', () => { const hedString = - '/Action/Move/Flex,(Relation/Spatial-relation/Left-side-of,/Action/Move/Bend,/Upper-extremity/Elbow),/Position/X-position/70 px,/Position/Y-position/23 px' + 'Action/Move/Flex,(Relation/Spatial-relation/Left-side-of,Action/Move/Bend,Upper-extremity/Elbow),Position/X-position/70 px,Position/Y-position/23 px' const [parsedString, issues] = parseHedString(hedString, nullSchema) assert.isEmpty(Object.values(issues).flat(), 'Parsing issues occurred') assert.sameDeepMembers(parsedString.tags.map(originalMap), [ - '/Action/Move/Flex', + 'Action/Move/Flex', 'Relation/Spatial-relation/Left-side-of', - '/Action/Move/Bend', - '/Upper-extremity/Elbow', - '/Position/X-position/70 px', - '/Position/Y-position/23 px', + 'Action/Move/Bend', + 'Upper-extremity/Elbow', + 'Position/X-position/70 px', + 'Position/Y-position/23 px', ]) assert.sameDeepMembers(parsedString.topLevelTags.map(originalMap), [ - '/Action/Move/Flex', - '/Position/X-position/70 px', - '/Position/Y-position/23 px', + 'Action/Move/Flex', + 'Position/X-position/70 px', + 'Position/Y-position/23 px', ]) assert.sameDeepMembers( parsedString.tagGroups.map((group) => group.tags.map(originalMap)), - [['Relation/Spatial-relation/Left-side-of', '/Action/Move/Bend', '/Upper-extremity/Elbow']], + [['Relation/Spatial-relation/Left-side-of', 'Action/Move/Bend', 'Upper-extremity/Elbow']], ) }) it('must include properly formatted tags', () => { const hedString = - '/Action/Move/Flex,(Relation/Spatial-relation/Left-side-of,/Action/Move/Bend,/Upper-extremity/Elbow),/Position/X-position/70 px,/Position/Y-position/23 px' + 'Action/Move/Flex,(Relation/Spatial-relation/Left-side-of,Action/Move/Bend,/Upper-extremity/Elbow),Position/X-position/70 px,Position/Y-position/23 px' const formattedHedString = 'action/move/flex,(relation/spatial-relation/left-side-of,action/move/bend,upper-extremity/elbow),position/x-position/70 px,position/y-position/23 px' const [parsedString, issues] = parseHedString(hedString, nullSchema) diff --git a/tests/tockenizerErrorTests.spec.js b/tests/tockenizerErrorTests.spec.js new file mode 100644 index 00000000..b84ac6c2 --- /dev/null +++ b/tests/tockenizerErrorTests.spec.js @@ -0,0 +1,166 @@ +import chai from 'chai' +const assert = chai.assert +import { beforeAll, describe, afterAll } from '@jest/globals' +import path from 'path' +import { HedStringTokenizer } from '../parser/tokenizer' +import { HedStringTokenizerOriginal } from '../parser/tokenizerOriginal' +import { errorTests } from './tokenizerErrorData' +const displayLog = process.env.DISPLAY_LOG === 'true' +const fs = require('fs') + +const skippedErrors = {} + +//const testInfo = loadTestData() +console.log(errorTests) + +describe('Tokenizer validation using JSON tests', () => { + const badLog = [] + let totalTests = 0 + let wrongErrors = 0 + let unexpectedErrors = 0 + + beforeAll(async () => {}) + + afterAll(() => { + const outBad = path.join(__dirname, 'runLog.txt') + const summary = `Total tests:${totalTests} Wrong error codes:${wrongErrors} Unexpected errors:${unexpectedErrors}\n` + if (displayLog) { + fs.writeFileSync(outBad, summary + badLog.join('\n'), 'utf8') + } + }) + + describe.each(errorTests)('$name : $description', ({ tests }) => { + let itemLog + + const assertErrors = function (eHedCode, eCode, expectError, iLog, header, issues) { + const log = [header] + totalTests += 1 + + let errors = [] + if (issues.length > 0) { + errors = issues.map((dict) => dict.hedCode) // list of hedCodes in the issues + } + const errorString = errors.join(',') + if (errors.length > 0) { + log.push(`---has errors [${errorString}]`) + } + const expectedError = eCode + const wrongError = `---expected ${eHedCode} but got errors [${errorString}]` + const hasErrors = `---expected no errors but got errors [${errorString}]` + if (expectError && !errors.includes(eHedCode)) { + log.push(wrongError) + iLog.push(log.join('\n')) + wrongErrors += 1 + assert.strictEqual( + errors.includes(eHedCode), + true, + `${header}---expected ${eHedCode} and got errors [${errorString}]`, + ) + } else if (!expectError && errors.length > 0) { + log.push(hasErrors) + iLog.push(log.join('\n')) + unexpectedErrors += 1 + assert(errors.length === 0, `${header}---expected no errors but got errors [${errorString}]`) + } + } + + const stringTokenizer = function (eHedCode, eCode, eName, tokenizer, expectError, iLog) { + const status = expectError ? 'Expect fail' : 'Expect pass' + const tokType = tokenizer instanceof HedStringTokenizer ? 'New tokenizer' : 'Original tokenizer' + const header = `\n[${eHedCode} ${eName} ${tokType}](${status})\tSTRING: "${tokenizer.hedString}"` + const [tagSpecs, groupBounds, tokenizingIssues] = tokenizer.tokenize() + const issues = Object.values(tokenizingIssues).flat() + assertErrors(eHedCode, eCode, expectError, iLog, header, issues) + } + + beforeAll(async () => { + itemLog = [] + }) + + afterAll(() => { + badLog.push(itemLog.join('\n')) + }) + + if (tests && tests.length > 0) { + test.each(tests)('Tokenizer: %s ', (ex) => { + stringTokenizer(ex.hedCode, ex.code, ex.name, new HedStringTokenizer(ex.string), true, itemLog) + }) + } + }) +}) + +describe('Original tokenizer validation using JSON tests', () => { + const badLog = [] + let totalTests = 0 + let wrongErrors = 0 + let unexpectedErrors = 0 + + beforeAll(async () => {}) + + afterAll(() => { + const outBad = path.join(__dirname, 'runLogOriginal.txt') + const summary = `Total tests:${totalTests} Wrong error codes:${wrongErrors} Unexpected errors:${unexpectedErrors}\n` + if (displayLog) { + fs.writeFileSync(outBad, summary + badLog.join('\n'), 'utf8') + } + }) + + describe.each(errorTests)('$name : $description', ({ tests }) => { + let itemLog + + const assertErrors = function (eHedCode, eCode, expectError, iLog, header, issues) { + const log = [header] + totalTests += 1 + + let errors = [] + if (issues.length > 0) { + errors = issues.map((dict) => dict.hedCode) // list of hedCodes in the issues + } + const errorString = errors.join(',') + if (errors.length > 0) { + log.push(`---has errors [${errorString}]`) + } + const expectedError = eCode + const wrongError = `---expected ${eHedCode} but got errors [${errorString}]` + const hasErrors = `---expected no errors but got errors [${errorString}]` + if (expectError && !errors.includes(eHedCode)) { + log.push(wrongError) + iLog.push(log.join('\n')) + wrongErrors += 1 + assert.strictEqual( + errors.includes(eHedCode), + true, + `${header}---expected ${eHedCode} and got errors [${errorString}]`, + ) + } else if (!expectError && errors.length > 0) { + log.push(hasErrors) + iLog.push(log.join('\n')) + unexpectedErrors += 1 + assert(errors.length === 0, `${header}---expected no errors but got errors [${errorString}]`) + } + } + + const stringTokenizer = function (eHedCode, eCode, eName, tokenizer, expectError, iLog) { + const status = expectError ? 'Expect fail' : 'Expect pass' + const tokType = tokenizer instanceof HedStringTokenizer ? 'New tokenizer' : 'Original tokenizer' + const header = `\n[${eHedCode} ${eName} ${tokType}](${status})\tSTRING: "${tokenizer.hedString}"` + const [tagSpecs, groupBounds, tokenizingIssues] = tokenizer.tokenize() + const issues = Object.values(tokenizingIssues).flat() + assertErrors(eHedCode, eCode, expectError, iLog, header, issues) + } + + beforeAll(async () => { + itemLog = [] + }) + + afterAll(() => { + badLog.push(itemLog.join('\n')) + }) + + if (tests && tests.length > 0) { + test.each(tests)('Original tokenizer: %s ', (ex) => { + stringTokenizer(ex.hedCode, ex.code, ex.name, new HedStringTokenizerOriginal(ex.string), true, itemLog) + }) + } + }) +}) diff --git a/tests/tokenizerErrorData.js b/tests/tokenizerErrorData.js new file mode 100644 index 00000000..17114f3f --- /dev/null +++ b/tests/tokenizerErrorData.js @@ -0,0 +1,137 @@ +export const errorTests = [ + { + name: 'empty-tag-in-various-places', + description: 'Empty tags in various places (empty groups are allowed).', + tests: [ + { + name: 'end-in-comma', + string: 'x,y,', + issueCount: 1, + hedCode: 'TAG_EMPTY', + code: 'emptyTagFound', + warning: false, + explanation: 'Cannot end in a comma', + }, + { + name: 'double-in-comma', + string: 'x,,y,', + issueCount: 1, + hedCode: 'TAG_EMPTY', + code: 'emptyTagFound', + warning: false, + explanation: 'Cannot have double commas', + }, + { + name: 'leading-comma', + string: ',x,y', + issueCount: 1, + hedCode: 'TAG_EMPTY', + code: 'emptyTagFound', + warning: false, + explanation: 'Cannot have a leading comma', + }, + ], + }, + { + name: 'extra-slash-in-various-places', + description: 'Tags cannot have leading or trailing, or extra slashes', + tests: [ + { + name: 'leading-slash', + string: '/x', + issueCount: 1, + hedCode: 'TAG_INVALID', + code: 'extraSlash', + warning: false, + explanation: 'Cannot have a leading slash', + }, + { + name: 'double-slash', + string: 'x//y', + issueCount: 1, + hedCode: 'TAG_INVALID', + code: 'extraSlash', + warning: false, + explanation: 'Cannot have double slash', + }, + { + name: 'triple-slash', + string: 'x///y', + issueCount: 1, + hedCode: 'TAG_INVALID', + code: 'extraSlash', + warning: false, + explanation: 'Cannot have double slash', + }, + { + name: 'trailing-slash', + string: 'x/y/', + issueCount: 1, + hedCode: 'TAG_INVALID', + code: 'extraSlash', + warning: false, + explanation: 'Cannot have ending slash', + }, + { + name: 'value-slash', + string: 'x /y', + issueCount: 1, + hedCode: 'TAG_INVALID', + code: 'extraBlank', + warning: false, + explanation: 'Cannot extra blanks before or after slashes', + }, + ], + }, + { + name: 'improper-curly-braces', + description: 'Curly braces cannot have commas or parentheses or other curly braces', + tests: [ + { + name: 'leading-close-brace', + string: '}x', + issueCount: 1, + hedCode: 'SIDECAR_BRACES_INVALID', + code: 'extraSlash', + warning: false, + explanation: 'Cannot have a leading slash', + }, + { + name: 'parenthesis-after-open-brace', + string: 'x, {y(z)}', + issueCount: 1, + hedCode: 'SIDECAR_BRACES_INVALID', + code: 'unclosedCurlyBrace', + warning: false, + explanation: 'Cannot parentheses inside curly braces', + }, + { + name: 'comma-inside-curly-brace', + string: 'x, {y,z}', + issueCount: 1, + hedCode: 'SIDECAR_BRACES_INVALID', + code: 'unclosedCurlyBrace', + warning: false, + explanation: 'Cannot have a comma inside curly brace', + }, + { + name: 'unclosed-curly-brace', + string: 'x, {y, z', + issueCount: 1, + hedCode: 'SIDECAR_BRACES_INVALID', + code: 'unclosedCurlyBrace', + warning: false, + explanation: 'Open curly braces must be matched with closing curly braces', + }, + { + name: 'nested-curly-brace', + string: '{x}, {{y, z}}', + issueCount: 1, + hedCode: 'SIDECAR_BRACES_INVALID', + code: 'nestedCurlyBrace', + warning: false, + explanation: 'Curly braces cannot be nested', + }, + ], + }, +] diff --git a/tests/tokenizerPassingData.js b/tests/tokenizerPassingData.js new file mode 100644 index 00000000..7bb7cf21 --- /dev/null +++ b/tests/tokenizerPassingData.js @@ -0,0 +1,239 @@ +//import { TagSpec, GroupSpec, ColumnSpliceSpec } from '../parser/tokenizerNew' +import { TagSpec, GroupSpec, ColumnSpliceSpec } from '../parser/tokenizer' + +export const passingTests = [ + { + name: 'valid-single-tags', + description: 'Single tags with no groups.', + warning: false, + tests: [ + { + name: 'simple-tag-no-blanks', + string: 'xy', + explanation: 'Should have bounds 0, 2', + tagSpecs: [new TagSpec('xy', 0, 2, '')], + groupSpec: new GroupSpec(0, 2, []), + }, + { + name: 'internal-blank', + string: 'x y', + explanation: 'Can have internal blank', + tagSpecs: [new TagSpec('x y', 0, 3, '')], + groupSpec: new GroupSpec(0, 3, []), + }, + { + name: 'extra-blanks-simple', + string: ' xy ', + explanation: 'Can have extra blanks', + tagSpecs: [new TagSpec('xy', 1, 3, '')], + groupSpec: new GroupSpec(0, 5, []), + }, + { + name: 'tag-with-slashes', + string: 'x/y/z', + explanation: 'Can have multiple slashes', + tagSpecs: [new TagSpec('x/y/z', 0, 5, '')], + groupSpec: new GroupSpec(0, 5, []), + }, + { + name: 'tag-in-column-spec', + string: '{xy}', + explanation: 'Single column spec', + tagSpecs: [new ColumnSpliceSpec('xy', 0, 3, '')], + groupSpec: new GroupSpec(0, 4, []), + }, + { + name: 'tag-in-column-spec-multiple-blanks', + string: ' { xy } ', + explanation: 'Single column spec with multiple blanks', + tagSpecs: [new ColumnSpliceSpec('xy', 2, 8, '')], + groupSpec: new GroupSpec(0, 10, []), + }, + { + name: 'tag-with-colons-no-blanks', + string: 'xy:wz', + explanation: 'Tag with a single colon and no blanks', + tagSpecs: [new TagSpec('wz', 3, 5, 'xy')], + groupSpec: new GroupSpec(0, 5, []), + }, + { + name: 'tag-with-multiple-colons', + string: 'xy:wz x:y', + explanation: 'Tag with one colon marking library and another as part of a value', + tagSpecs: [new TagSpec('wz x:y', 3, 9, 'xy')], + groupSpec: new GroupSpec(0, 9, []), + }, + { + name: 'tags-with-one-value column', + string: 'xy x:y', + explanation: 'Tag with one colon as part of a value', + tagSpecs: [new TagSpec('xy x:y', 0, 6, '')], + groupSpec: new GroupSpec(0, 6, []), + }, + ], + }, + { + name: 'multiple-tags-no-groups', + description: 'multiple tags with no groups.', + warning: false, + tests: [ + { + name: 'multiple-tags', + string: 'xy,zy,wy', + explanation: 'Multiple tags with no blanks', + tagSpecs: [new TagSpec('xy', 0, 2, ''), new TagSpec('zy', 3, 5, ''), new TagSpec('wy', 6, 8, '')], + groupSpec: new GroupSpec(0, 8, []), + }, + { + name: 'multiple-tags-with-blanks', + string: ' xy, zy , wy ', + explanation: 'Can have extra blanks', + tagSpecs: [new TagSpec('xy', 1, 3, ''), new TagSpec('zy', 6, 8, ''), new TagSpec('wy', 11, 13, '')], + groupSpec: new GroupSpec(0, 15, []), + }, + { + name: 'multiple-tags-with-blanks', + string: ' xy, zy , wy ', + explanation: 'Can have extra blanks', + tagSpecs: [new TagSpec('xy', 1, 3, ''), new TagSpec('zy', 6, 8, ''), new TagSpec('wy', 11, 13, '')], + groupSpec: new GroupSpec(0, 15, []), + }, + ], + }, + { + name: 'un-nested-groups', + description: 'Groups with no nesting', + warning: false, + tests: [ + { + name: 'single-non-empty-group-no-blanks', + string: '(xy)', + explanation: 'Single group', + tagSpecs: [[new TagSpec('xy', 1, 3, '')]], + groupSpec: new GroupSpec(0, 4, [new GroupSpec(0, 4, [])]), + }, + { + name: 'tag-after-group', + string: '(x), p', + explanation: 'A tag after a group.', + tagSpecs: [[new TagSpec('x', 1, 2, '')], new TagSpec('p', 5, 6, '')], + groupSpec: new GroupSpec(0, 6, [new GroupSpec(0, 3, [])]), + }, + { + name: 'multiple-tags-in-group', + string: '(x,y)', + explanation: 'Multiple tags in one group.', + tagSpecs: [[new TagSpec('x', 1, 2, ''), new TagSpec('y', 3, 4, '')]], + groupSpec: new GroupSpec(0, 5, [new GroupSpec(0, 5, [])]), + }, + { + name: 'multiple-unnested-groups', + string: 'q, (xy), (zw, uv), p', + explanation: 'Multiple unnested tag groups and tags.', + tagSpecs: [ + new TagSpec('q', 0, 1, ''), + [new TagSpec('xy', 4, 6, '')], + [new TagSpec('zw', 10, 12, ''), new TagSpec('uv', 14, 16, '')], + new TagSpec('p', 19, 20, ''), + ], + groupSpec: new GroupSpec(0, 20, [new GroupSpec(3, 7, []), new GroupSpec(9, 17, [])]), + }, + { + name: 'tag-after-group', + string: 'x/y,(r,v)', + explanation: 'A tag after a group.', + tagSpecs: [new TagSpec('x/y', 0, 3, ''), [new TagSpec('r', 5, 6, ''), new TagSpec('v', 7, 8, '')]], + groupSpec: new GroupSpec(0, 9, [new GroupSpec(4, 9, [])]), + }, + ], + }, + { + name: 'Nested groups', + description: 'Nested groups with complex nesting', + warning: false, + tests: [ + { + name: 'Single-multi-nested-group', + string: '(((xy)))', + explanation: 'Single group with deep nesting', + tagSpecs: [[[[new TagSpec('xy', 3, 5, '')]]]], + groupSpec: new GroupSpec(0, 8, [new GroupSpec(0, 8, [new GroupSpec(1, 7, [new GroupSpec(2, 6, [])])])]), + }, + { + name: 'Single-nested-group-with-extra-tag', + string: '((xy)), g', + explanation: 'Nested group with trailing tag', + tagSpecs: [[[new TagSpec('xy', 2, 4, '')]], new TagSpec('g', 8, 9, '')], + groupSpec: new GroupSpec(0, 9, [new GroupSpec(0, 6, [new GroupSpec(1, 5, [])])]), + }, + { + name: 'Single-nested-group-with-splice', + string: '((({xy})))', + explanation: 'A single nested group with a column splice.', + tagSpecs: [[[[new ColumnSpliceSpec('xy', 3, 6)]]]], + groupSpec: new GroupSpec(0, 10, [new GroupSpec(0, 10, [new GroupSpec(1, 9, [new GroupSpec(2, 8, [])])])]), + }, + { + name: 'Complex-nested-group-1', + string: '((xy), ( h:p, ((q, r ))))', + explanation: 'Single group', + tagSpecs: [ + [ + [new TagSpec('xy', 2, 4, '')], + [new TagSpec('p', 11, 12, 'h'), [[new TagSpec('q', 16, 17, ''), new TagSpec('r', 19, 20, '')]]], + ], + ], + groupSpec: new GroupSpec(0, 25, [ + new GroupSpec(0, 25, [ + new GroupSpec(1, 5, []), + new GroupSpec(7, 24, [new GroupSpec(14, 23, [new GroupSpec(15, 22, [])])]), + ]), + ]), + }, + { + name: 'Complex-nested-group-2', + string: '((xy), g), h', + explanation: 'Nested groups with tags', + tagSpecs: [[[new TagSpec('xy', 2, 4, '')], new TagSpec('g', 7, 8, '')], new TagSpec('h', 11, 12, '')], + groupSpec: new GroupSpec(0, 12, [new GroupSpec(0, 9, [new GroupSpec(1, 5, [])])]), + }, + { + name: 'Complex-nested-group-3', + string: '((xy), ( h:p, ((q, r ))), g)', + explanation: 'Single group', + tagSpecs: [ + [ + [new TagSpec('xy', 2, 4, '')], + [new TagSpec('p', 11, 12, 'h'), [[new TagSpec('q', 16, 17, ''), new TagSpec('r', 19, 20, '')]]], + new TagSpec('g', 26, 27, ''), + ], + ], + groupSpec: new GroupSpec(0, 28, [ + new GroupSpec(0, 28, [ + new GroupSpec(1, 5, []), + new GroupSpec(7, 24, [new GroupSpec(14, 23, [new GroupSpec(15, 22, [])])]), + ]), + ]), + }, + { + name: 'Complex-nested-group-4', + string: '((xy), ( h:p, ((q, r ))), g), h', + explanation: 'Single group', + tagSpecs: [ + [ + [new TagSpec('xy', 2, 4, '')], + [new TagSpec('p', 11, 12, 'h'), [[new TagSpec('q', 16, 17, ''), new TagSpec('r', 19, 20, '')]]], + new TagSpec('g', 26, 27, ''), + ], + new TagSpec('h', 30, 31, ''), + ], + groupSpec: new GroupSpec(0, 31, [ + new GroupSpec(0, 28, [ + new GroupSpec(1, 5, []), + new GroupSpec(7, 24, [new GroupSpec(14, 23, [new GroupSpec(15, 22, [])])]), + ]), + ]), + }, + ], + }, +] diff --git a/tests/tokenizerPassingTests.spec.js b/tests/tokenizerPassingTests.spec.js new file mode 100644 index 00000000..c91ac41b --- /dev/null +++ b/tests/tokenizerPassingTests.spec.js @@ -0,0 +1,153 @@ +import chai from 'chai' +const assert = chai.assert +import { beforeAll, describe, afterAll } from '@jest/globals' +import path from 'path' +import { HedStringTokenizerOriginal } from '../parser/tokenizerOriginal' +import { HedStringTokenizer } from '../parser/tokenizer' +import { passingTests } from './tokenizerPassingData' +const fs = require('fs') + +const displayLog = process.env.DISPLAY_LOG === 'true' + +const skippedErrors = {} + +describe('HED tokenizer validation', () => { + describe('Tokenizer validation - validData', () => { + const badLog = [] + let totalTests = 0 + let unexpectedErrors = 0 + + beforeAll(async () => {}) + + afterAll(() => { + const outBad = path.join(__dirname, 'runLog.txt') + const summary = `Total tests:${totalTests} Unexpected errors:${unexpectedErrors}\n` + if (displayLog) { + fs.writeFileSync(outBad, summary + badLog.join('\n'), 'utf8') + } + }) + + describe.each(passingTests)('$name : $description', ({ tests }) => { + let itemLog + const assertErrors = function (header, issues, iLog) { + iLog.push(`${header}\n`) + totalTests += 1 + + let errors = [] + if (issues.length > 0) { + errors = issues.map((dict) => dict.hedCode) // list of hedCodes in the issues + } + const errorString = errors.join(',') + if (errors.length > 0) { + iLog.push(`---expected no errors but got errors [${errorString}]\n`) + unexpectedErrors += 1 + assert(errors.length === 0, `${header}---expected no errors but got errors [${errorString}]`) + } + } + + const stringTokenizer = function (eName, tokenizer, tSpecs, gSpec, explanation, iLog) { + const status = 'Expect pass' + const tokType = tokenizer instanceof HedStringTokenizer ? 'Tokenizer' : 'Original tokenizer' + const header = `\n[${tokType}](${status})\tSTRING: "${tokenizer.hedString}"` + const [tagSpecs, groupSpec, tokenizingIssues] = tokenizer.tokenize() + // Test for no errors + const issues = Object.values(tokenizingIssues).flat() + assertErrors(header, issues, iLog) + assert.sameDeepMembers(tagSpecs, tSpecs, explanation) + assert.deepEqual(groupSpec, gSpec, explanation) + //assert.sameDeepMembers(groupSpec, gSpec, explanation) + } + + beforeAll(async () => { + itemLog = [] + }) + + afterAll(() => { + badLog.push(itemLog.join('\n')) + }) + + if (tests && tests.length > 0) { + test.each(tests)('Tokenizer: %s ', (ex) => { + stringTokenizer( + ex.name, + new HedStringTokenizer(ex.string), + ex.tagSpecs, + ex.groupSpec, + ex.explanation, + itemLog, + ) + }) + } + }) + }) + + describe('Original tokenizer validation - validData', () => { + const badLog = [] + let totalTests = 0 + let unexpectedErrors = 0 + + beforeAll(async () => {}) + + afterAll(() => { + const outBad = path.join(__dirname, 'runLogOriginal.txt') + const summary = `Total tests:${totalTests} Unexpected errors:${unexpectedErrors}\n` + if (displayLog) { + fs.writeFileSync(outBad, summary + badLog.join('\n'), 'utf8') + } + }) + + describe.each(passingTests)('$name : $description', ({ tests }) => { + let itemLog + + const assertErrors = function (header, issues, iLog) { + iLog.push(`${header}\n`) + totalTests += 1 + + let errors = [] + if (issues.length > 0) { + errors = issues.map((dict) => dict.hedCode) // list of hedCodes in the issues + } + const errorString = errors.join(',') + if (errors.length > 0) { + iLog.push(`---expected no errors but got errors [${errorString}]\n`) + unexpectedErrors += 1 + assert(errors.length === 0, `${header}---expected no errors but got errors [${errorString}]`) + } + } + + const stringTokenizer = function (eName, tokenizer, tSpecs, gSpec, explanation, iLog) { + const status = 'Expect pass' + const tokType = tokenizer instanceof HedStringTokenizer ? 'Tokenizer' : 'Original tokenizer' + const header = `\n[${tokType}](${status})\tSTRING: "${tokenizer.hedString}"` + const [tagSpecs, groupSpec, tokenizingIssues] = tokenizer.tokenize() + // Test for no errors + const issues = Object.values(tokenizingIssues).flat() + assertErrors(header, issues, iLog) + assert.sameDeepMembers(tagSpecs, tSpecs, explanation) + assert.deepEqual(groupSpec, gSpec, explanation) + //assert.sameDeepMembers(groupSpec, gSpec, explanation) + } + + beforeAll(async () => { + itemLog = [] + }) + + afterAll(() => { + badLog.push(itemLog.join('\n')) + }) + + if (tests && tests.length > 0) { + test.each(tests)('Original tokenizer: %s ', (ex) => { + stringTokenizer( + ex.name, + new HedStringTokenizerOriginal(ex.string), + ex.tagSpecs, + ex.groupSpec, + ex.explanation, + itemLog, + ) + }) + } + }) + }) +}) diff --git a/validator/event/specialTags.json b/validator/event/specialTags.json index 7190340a..3f2b52f0 100644 --- a/validator/event/specialTags.json +++ b/validator/event/specialTags.json @@ -1,121 +1,139 @@ { - "Definition": { - "child": true, - "requireChild": true, + "Def": { + "allowValue": true, + "allowTwoLevelValue": true, + "requireValue": true, + "tagGroup": false, + "topLevelTagGroup": false, + "maxNumberSubgroups": -1, + "minNumberSubgroups": -1, + "ERROR_CODE": "DEF_INVALID", + "forbiddenSubgroupTags": [], + "defTagRequired": false, + "otherAllowedTags": [] + }, + "Def-expand": { + "allowValue": true, + "allowTwoLevelValue": true, + "requireValue": true, "tagGroup": true, - "topLevelTagGroup": true, + "topLevelTagGroup": false, "maxNumberSubgroups": 1, "minNumberSubgroups": 0, - "ERROR_CODE": "DEFINITION_INVALID", - "subgroupTagsNotAllowed": [ + "ERROR_CODE": "DEF_EXPAND_INVALID", + "forbiddenSubgroupTags": [ "Def", "Def-expand", - "Event-context", "Definition", - "Onset", + "Delay", + "Duration", + "Event-context", "Inset", "Offset", - "Delay", - "Duration" + "Onset" ], "defTagRequired": false, "otherAllowedTags": [] }, - "Def": { - "child": true, - "tagGroup": false, - "topLevelTagGroup": false, - "maxNumberSubgroups": null, - "minNumberSubgroups": null, - "ERROR_CODE": "DEF_INVALID", - "subgroupTagsNotAllowed": [], - "defTagRequired": false, - "otherAllowedTags": null - }, - "Def-expand": { - "child": true, + "Definition": { + "allowValue": true, + "allowTwoLevelValue": true, + "requireValue": true, "tagGroup": true, - "topLevelTagGroup": false, + "topLevelTagGroup": true, "maxNumberSubgroups": 1, "minNumberSubgroups": 0, - "ERROR_CODE": "DEF_EXPAND_INVALID", - "subgroupTagsNotAllowed": [ + "ERROR_CODE": "DEFINITION_INVALID", + "forbiddenSubgroupTags": [ "Def", "Def-expand", - "Event-context", "Definition", - "Onset", + "Delay", + "Duration", + "Event-context", "Inset", "Offset", - "Delay", - "Duration" + "Onset" ], "defTagRequired": false, "otherAllowedTags": [] }, - "Onset": { - "child": false, + "Delay": { + "allowValue": true, + "allowTwoLevelValue": false, + "requireValue": true, "tagGroup": true, "topLevelTagGroup": true, "maxNumberSubgroups": 1, - "minNumberSubgroups": 0, + "minNumberSubgroups": 1, "ERROR_CODE": "TEMPORAL_TAG_ERROR", - "subgroupTagsNotAllowed": ["Event-context", "Definition", "Onset", "Inset", "Offset", "Delay", "Duration"], - "defTagRequired": true, + "forbiddenSubgroupTags": ["Definition", "Delay", "Duration", "Event-context", "Inset", "Offset", "Onset"], + "defTagRequired": false, + "otherAllowedTags": ["Duration"] + }, + "Duration": { + "allowValue": true, + "allowTwoLevelValue": false, + "requireValue": true, + "tagGroup": true, + "topLevelTagGroup": true, + "maxNumberSubgroups": 1, + "minNumberSubgroups": 1, + "ERROR_CODE": "TEMPORAL_TAG_ERROR", + "forbiddenSubgroupTags": ["Definition", "Delay", "Duration", "Event-context", "Inset", "Offset", "Onset"], + "defTagRequired": false, + "otherAllowedTags": ["Delay"] + }, + "Event-context": { + "allowValue": false, + "allowTwoLevelValue": false, + "requireValue": false, + "tagGroup": true, + "topLevelTagGroup": true, + "maxNumberSubgroups": null, + "minNumberSubgroups": 0, + "ERROR_CODE": "TAG_GROUP_ERROR", + "forbiddenSubgroupTags": ["Event-context", "Definition", "Onset", "Inset", "Offset", "Delay", "Duration"], + "defTagRequired": false, "otherAllowedTags": [] }, "Inset": { - "child": false, + "allowValue": false, + "allowTwoLevelValue": false, + "requireValue": false, "tagGroup": true, "topLevelTagGroup": true, "maxNumberSubgroups": 1, "minNumberSubgroups": 0, "ERROR_CODE": "TEMPORAL_TAG_ERROR", - "subgroupTagsNotAllowed": ["Event-context", "Definition", "Onset", "Inset", "Offset", "Delay", "Duration"], + "forbiddenSubgroupTags": ["Definition", "Delay", "Duration", "Event-context", "Inset", "Offset", "Onset"], "defTagRequired": true, "otherAllowedTags": [] }, "Offset": { - "child": false, + "allowValue": false, + "allowTwoLevelValue": false, + "requireValue": false, "tagGroup": true, "topLevelTagGroup": true, "maxNumberSubgroups": 0, "minNumberSubgroups": 0, "ERROR_CODE": "TEMPORAL_TAG_ERROR", - "subgroupTagsNotAllowed": [], + "forbiddenSubgroupTags": [], "defTagRequired": true, "otherAllowedTags": [] }, - "Delay": { - "child": true, + "Onset": { + "allowValue": false, + "allowTwoLevelValue": false, + "requireValue": false, "tagGroup": true, "topLevelTagGroup": true, "maxNumberSubgroups": 1, - "minNumberSubgroups": 1, - "ERROR_CODE": "TEMPORAL_TAG_ERROR", - "subgroupTagsNotAllowed": ["Event-context", "Definition", "Onset", "Inset", "Offset", "Delay", "Duration"], - "defTagRequired": false, - "otherAllowedTags": ["Duration"] - }, - "Duration": { - "child": true, - "topLevelTagGroup": true, - "maxNumberSubgroups": 1, - "minNumberSubgroups": 1, - "ERROR_CODE": "TEMPORAL_TAG_ERROR", - "subgroupTagsNotAllowed": ["Event-context", "Definition", "Onset", "Inset", "Offset", "Delay", "Duration"], - "defTagRequired": false, - "otherAllowedTags": ["Delay"] - }, - "Event-context": { - "child": false, - "tagGroup": true, - "topLevelTagGroup": true, - "maxNumberSubgroups": null, "minNumberSubgroups": 0, - "ERROR_CODE": "TAG_GROUP_ERROR", - "subgroupTagsNotAllowed": ["Event-context", "Definition", "Onset", "Inset", "Offset", "Delay", "Duration"], - "defTagRequired": false, + "ERROR_CODE": "TEMPORAL_TAG_ERROR", + "forbiddenSubgroupTags": ["Definition", "Delay", "Duration", "Event-context", "Inset", "Offset", "Onset"], + "defTagRequired": true, "otherAllowedTags": [] } }