Merge pull request #225 from VisLab/update-tokenizer

Updated the validator -- now ready to work on definitions
hed-standard · Nov 14, 2024 · 08b87b9 · 08b87b9
2 parents 22e0007 + f33fd5b
commit 08b87b9
Show file tree

Hide file tree

Showing 8 changed files with 120 additions and 12 deletions.
diff --git a/bids/types/json.js b/bids/types/json.js
@@ -71,7 +71,8 @@ export class BidsSidecar extends BidsJsonFile {
    */
   constructor(name, sidecarData = {}, file) {
     super(name, sidecarData, file)
-
+    this.columnSpliceMapping = new Map()
+    this.columnSpliceReferences = new Set()
     this._filterHedStrings()
     this._categorizeHedStrings()
   }
@@ -167,9 +168,6 @@ export class BidsSidecar extends BidsJsonFile {
    * @private
    */
   _generateSidecarColumnSpliceMap() {
-    this.columnSpliceMapping = new Map()
-    this.columnSpliceReferences = new Set()
-
     for (const [sidecarKey, hedData] of this.parsedHedData) {
       if (hedData === null) {
         // Skipped

diff --git a/common/issues/data.js b/common/issues/data.js
@@ -215,6 +215,11 @@ export default {
     level: 'error',
     message: stringTemplate`Empty tag at index ${'index'} cannot be converted.`,
   },
+  invalidTagString: {
+    hedCode: 'TAG_INVALID',
+    level: 'error',
+    message: stringTemplate`Tag string is null or undefined.`,
+  },
   duplicateTagsInSchema: {
     hedCode: 'SCHEMA_DUPLICATE_NODE',
     level: 'error',

diff --git a/parser/splitter.js b/parser/splitter.js
@@ -48,6 +48,12 @@ export default class HedStringSplitter {
    * @returns {[ParsedHedSubstring[], Object<string, Issue[]>]} The parsed HED string data and any issues found.
    */
   splitHedString() {
+    if (this.hedString === null || this.hedString === undefined || typeof this.hedString !== 'string') {
+      return [null, { syntax: [generateIssue('invalidTagString', {})] }]
+    }
+    if (this.hedString.length === 0) {
+      return [[], {}]
+    }
     const [tagSpecs, groupBounds, tokenizingIssues] = new HedStringTokenizer(this.hedString).tokenize()
     if (tokenizingIssues.syntax.length > 0) {
       return [null, tokenizingIssues]

diff --git a/spec_tests/jsonTests.spec.js b/spec_tests/jsonTests.spec.js
@@ -182,7 +182,7 @@ describe('HED validation using JSON tests', () => {
       }
 
       const stringValidator = function (str, expectedErrors) {
-        const status = expectedErrors.size === 0 ? 'Expect fail' : 'Expect pass'
+        const status = expectedErrors.size === 0 ? 'Expect pass' : 'Expect fail'
         const header = `\n[${error_code} ${name}](${status})\tSTRING: "${str}"`
         const hTsv = `HED\n${str}\n`
         let stringIssues = []
@@ -211,7 +211,11 @@ describe('HED validation using JSON tests', () => {
 
       beforeAll(async () => {
         hedSchema = schemaMap.get(schema)
-        defs = { definitions: { HED: { defList: definitions.join(',') } } }
+        if (definitions.length === 0) {
+          defs = {}
+        } else {
+          defs = { definitions: { HED: { defList: definitions.join(',') } } }
+        }
         expectedErrors = new Set(alt_codes)
         expectedErrors.add(error_code)
       })

diff --git a/tests/tagParserTests.spec.js b/tests/tagParserTests.spec.js
@@ -11,11 +11,12 @@ import { buildSchemas } from '../schema/init'
 import { SchemaTag, SchemaValueTag } from '../schema/entries'
 import { TagSpec } from '../parser/tokenizer'
 import TagConverter from '../parser/tagConverter'
+import { BidsTsvFile } from '../bids'
 
 // Ability to select individual tests to run
 const skipMap = new Map()
 const runAll = true
-const runMap = new Map([['invalid-tags', ['invalid-tag-value-no-units']]])
+const runMap = new Map([['valid-tags', ['valid-numeric-scientific-value']]])
 
 describe('TagSpec converter tests using JSON tests', () => {
   const schemaMap = new Map([
@@ -36,6 +37,22 @@ describe('TagSpec converter tests using JSON tests', () => {
 
   afterAll(() => {})
 
+  describe('BIDS experiments', () => {
+    it('should be able to convert', () => {
+      const thisSchema = schemaMap.get('8.3.0')
+      assert.isDefined(thisSchema, 'yes')
+      const hTsv = `HED\nRed\n`
+      let stringIssues = []
+      try {
+        const bidsTsv = new BidsTsvFile(`events`, hTsv, { relativePath: 'string test tsv' }, [], {})
+        stringIssues = bidsTsv.validate(thisSchema)
+        console.log(stringIssues)
+      } catch (e) {
+        console.log(stringIssues)
+      }
+    })
+  })
+
   // TODO: Remove after refactoring of validation complete
   describe.skip('TagConverter experiments', () => {
     it('should be able to convert', () => {

diff --git a/tests/testData/stringParserTests.data.js b/tests/testData/stringParserTests.data.js
@@ -347,8 +347,8 @@ export const parseTestData = [
     ],
   },
   {
-    name: 'multiple-tags',
-    description: 'Multiple tags',
+    name: 'tag-strings',
+    description: 'Tag strings in various forms including empty and many tags',
     tests: [
       {
         testname: 'multiple-valid-tags',
@@ -360,6 +360,58 @@ export const parseTestData = [
         errors: [],
         warnings: [],
       },
+      {
+        testname: 'valid-empty-string',
+        explanation: '"" is a valid HED string',
+        schemaVersion: '8.3.0',
+        stringIn: '',
+        stringLong: '',
+        stringShort: '',
+        errors: [],
+        warnings: [],
+      },
+      {
+        testname: 'valid-string-with-groups',
+        explanation: '"Red, (Blue,Green)" is a valid HED string',
+        schemaVersion: '8.3.0',
+        stringIn: 'Red, (Blue,Green)',
+        stringLong:
+          'Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/CSS-color/Red-color/Red, (Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/CSS-color/Blue-color/Blue, Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/CSS-color/Green-color/Green)',
+        stringShort: 'Red, (Blue, Green)',
+        errors: [],
+        warnings: [],
+      },
+      {
+        testname: 'valid-string-with-nested-groups',
+        explanation: '"Red, (Blue,(Green))" is a valid HED string',
+        schemaVersion: '8.3.0',
+        stringIn: 'Red, (Blue,(Green))',
+        stringLong:
+          'Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/CSS-color/Red-color/Red, (Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/CSS-color/Blue-color/Blue, (Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/CSS-color/Green-color/Green))',
+        stringShort: 'Red, (Blue, (Green))',
+        errors: [],
+        warnings: [],
+      },
+      {
+        testname: 'invalid-null-string',
+        explanation: '"null" is not a valid HED string',
+        schemaVersion: '8.3.0',
+        stringIn: null,
+        stringLong: null,
+        stringShort: null,
+        errors: [generateIssue('invalidTagString', {})],
+        warnings: [],
+      },
+      {
+        testname: 'invalid-undefined-string',
+        explanation: '"undefined" is not a valid HED string',
+        schemaVersion: '8.3.0',
+        stringIn: undefined,
+        stringLong: null,
+        stringShort: null,
+        errors: [generateIssue('invalidTagString', {})],
+        warnings: [],
+      },
     ],
   },
 ]
diff --git a/tests/testData/tagParserTests.data.js b/tests/testData/tagParserTests.data.js
@@ -98,6 +98,32 @@ export const parsedHedTagTests = [
         takesValue: true,
         error: null,
       },
+      {
+        testname: 'valid-numeric-value-tag-no-units',
+        explanation: '"Weight/30" is a valid numeric value tag.',
+        schemaVersion: '8.3.0',
+        fullString: 'Weight/30',
+        tagSpec: new TagSpec('Weight/30', 0, 9, ''),
+        tagLong: 'Property/Data-property/Data-value/Physical-value/Weight/30',
+        tagShort: 'Weight/30',
+        formattedTag: 'property/data-property/data-value/physical-value/weight/30',
+        canonicalTag: 'Property/Data-property/Data-value/Physical-value/Weight/30',
+        takesValue: true,
+        error: null,
+      },
+      {
+        testname: 'valid-numeric-scientific-value',
+        explanation: '"Weight/3.0e10 kg" is a valid numeric value tag with scientific notation.',
+        schemaVersion: '8.3.0',
+        fullString: 'Weight/30',
+        tagSpec: new TagSpec('Weight/3.0e10 kg', 0, 16, ''),
+        tagLong: 'Property/Data-property/Data-value/Physical-value/Weight/3.0e10 kg',
+        tagShort: 'Weight/3.0e10 kg',
+        formattedTag: 'property/data-property/data-value/physical-value/weight/3.0e10 kg',
+        canonicalTag: 'Property/Data-property/Data-value/Physical-value/Weight/3.0e10 kg',
+        takesValue: true,
+        error: null,
+      },
     ],
   },
   {

diff --git a/validator/event/validator.js b/validator/event/validator.js
@@ -6,8 +6,8 @@ import ParsedHedGroup from '../../parser/parsedHedGroup'
 import ParsedHedTag from '../../parser/parsedHedTag'
 import ParsedHedColumnSplice from '../../parser/parsedHedColumnSplice'
 import { getParsedParentTags } from '../../utils/hedData'
-import { getParentTag, getTagName, hedStringIsAGroup, replaceTagNameWithPound } from '../../utils/hedStrings'
-import { getCharacterCount, isNumber } from '../../utils/string'
+import { hedStringIsAGroup, replaceTagNameWithPound } from '../../utils/hedStrings'
+import { getCharacterCount } from '../../utils/string'
 
 const tagGroupType = 'tagGroup'
 const topLevelTagGroupType = 'topLevelTagGroup'
@@ -249,7 +249,7 @@ export default class HedValidator {
   }
 
   // TODO: Checking for extensions is being removed temporarily -- well have to add it back eventually.
-  /*  /!**
+  /*
    * Check if an individual HED tag is in the schema or is an allowed extension.
    *!/
   checkIfTagIsValid(tag) {