diff --git a/common/issues/data.js b/common/issues/data.js index 934de65d..a7e914dd 100644 --- a/common/issues/data.js +++ b/common/issues/data.js @@ -208,7 +208,7 @@ export default { invalidExtension: { hedCode: 'TAG_EXTENSION_INVALID', level: 'error', - message: stringTemplate`"${'tag'}" appears as an extension of "${'parentTag'}", which does not allow tag extensions.`, + message: stringTemplate`"${'tag'}" appears as an extension of "${'parentTag'}", which does not allow this tag extension.`, }, emptyTagFound: { hedCode: 'TAG_EMPTY', diff --git a/parser/parsedHedTag.js b/parser/parsedHedTag.js index 59d62da0..46152a75 100644 --- a/parser/parsedHedTag.js +++ b/parser/parsedHedTag.js @@ -3,7 +3,9 @@ import { getParentTag, getTagLevels, getTagName } from '../utils/hedStrings' import ParsedHedSubstring from './parsedHedSubstring' import { SchemaValueTag } from '../schema/entries' import TagConverter from './tagConverter' -import { Schema } from '../schema/containers' +import { getRegExp } from './tempRegex' + +import RegexClass from '../schema/regExps' /** * A parsed HED tag. @@ -39,6 +41,30 @@ export default class ParsedHedTag extends ParsedHedSubstring { */ _remainder + /** + * The extension if any + * + * @type {string} + * @private + */ + _extension + + /** + * The value if any + * + * @type {string} + * @private + */ + _value + + /** + * The units if any + * + * @type {string} + * @private + */ + _units + /** * Constructor. * @@ -49,7 +75,8 @@ export default class ParsedHedTag extends ParsedHedSubstring { */ constructor(tagSpec, hedSchemas, hedString) { super(tagSpec.tag, tagSpec.bounds) // Sets originalTag and originalBounds - this._convertTag(hedSchemas, hedString, tagSpec) // Sets various parameters + this._convertTag(hedSchemas, hedString, tagSpec) // Sets various forms of the tag. + this._handleRemainder() //this._checkTagAttributes() // Checks various aspects like requireChild or extensionAllowed. //this.formattedTag = this._formatTag() //this.formattedTag = this.canonicalTag.toLowerCase() @@ -86,6 +113,45 @@ export default class ParsedHedTag extends ParsedHedSubstring { this.formattedTag = this.canonicalTag.toLowerCase() } + /** + * Handle the remainder portion + * + * @throws {IssueError} If parsing the remainder section fails. + */ + _handleRemainder() { + if (this._remainder === '') { + return + } + // if (this.allowsExtensions) { + // this._handleExtension() + // } else if (this.takesValue) { // Its a value tag + // return + // } else { + // //IssueError.generateAndThrow('invalidTag', {tag: this.originalTag}) + // } + } + + /** + * Handle potenial extensions + * + * @throws {IssueError} If parsing the remainder section fails. + */ + _handleExtension() { + this._extension = this._remainder + const testReg = getRegExp('nameClass') + if (!testReg.test(this._extension)) { + IssueError.generateAndThrow('invalidExtension', { tag: this.originalTag }) + } + } + + // _handleExtension() { + // this._extension = this._remainder + // const testit = RegexClass.testRegex('nameClass', this._extension) + // if (!RegexClass.testRegex('nameClass', this._extension)) { + // IssueError.generateAndThrow('invalidExtension', {tag: this.originalTag}) + // } + // } + /** * Nicely format this tag. * diff --git a/parser/tagConverter.js b/parser/tagConverter.js index 80243935..6ec97f86 100644 --- a/parser/tagConverter.js +++ b/parser/tagConverter.js @@ -1,7 +1,7 @@ import { IssueError } from '../common/issues/issues' import { getTagSlashIndices } from '../utils/hedStrings' import { SchemaValueTag } from '../schema/entries' - +import { getRegExp } from './tempRegex' /** * Converter from a tag specification to a schema-based tag object. */ @@ -56,6 +56,7 @@ export default class TagConverter { constructor(tagSpec, hedSchemas) { this.hedSchemas = hedSchemas this.tagMapping = hedSchemas.getSchema(tagSpec.library).entries.tags + this.tagSpec = tagSpec this.tagString = tagSpec.tag this.tagLevels = this.tagString.split('/') @@ -67,6 +68,7 @@ export default class TagConverter { * Retrieve the {@link SchemaTag} object for a tag specification. * * @returns {[SchemaTag, string]} The schema's corresponding tag object and the remainder of the tag string. + * @throws {IssueError} If tag conversion. */ convert() { let parentTag = undefined @@ -86,45 +88,50 @@ export default class TagConverter { } _validateChildTag(parentTag, tagLevelIndex) { - if (this.schemaTag instanceof SchemaValueTag) { - IssueError.generateAndThrow('internalConsistencyError', { - message: 'Child tag is a value tag which should have been handled earlier.', - }) - } - const childTag = this._getSchemaTag(tagLevelIndex) if (childTag === undefined) { + // This is an extended tag if (tagLevelIndex === 0) { IssueError.generateAndThrow('invalidTag', { tag: this.tagString }) } if (parentTag !== undefined && !parentTag.hasAttributeName('extensionAllowed')) { IssueError.generateAndThrow('invalidExtension', { tag: this.tagLevels[tagLevelIndex], - parentTag: parentTag.longName, + parentTag: this.tagLevels.slice(0, tagLevelIndex).join('/'), }) } + this._checkExtensions(tagLevelIndex) return childTag } if (tagLevelIndex > 0 && (childTag.parent === undefined || childTag.parent !== parentTag)) { IssueError.generateAndThrow('invalidParentNode', { tag: this.tagLevels[tagLevelIndex], - parentTag: childTag.longName, + parentTag: this.tagLevels.slice(0, tagLevelIndex).join('/'), }) } return childTag } + _checkExtensions(tagLevelIndex) { + // A non-tag has been detected --- from here on must be non-tags. + this._checkNameClass(tagLevelIndex) // This is an extension + for (let index = tagLevelIndex + 1; index < this.tagLevels.length; index++) { + const child = this._getSchemaTag(index) + if (child !== undefined) { + // A schema tag showed up after a non-schema tag + IssueError.generateAndThrow('invalidParentNode', { + tag: this.tagLevels[index], + parentTag: this.tagLevels.slice(0, index).join('/'), + }) + } + this._checkNameClass(index) + } + } + _getSchemaTag(tagLevelIndex) { - let tagLevel = this.tagLevels[tagLevelIndex].toLowerCase() - // TODO: These two checks should probably be removed as the tokenizer handles this. - // if (tagLevelIndex === 0) { - // tagLevel = tagLevel.trimLeft() - // } - // if (tagLevel === '' || tagLevel !== tagLevel.trim()) { - // IssueError.generateAndThrow('invalidTag', { tag: this.tagString }) - // } + const tagLevel = this.tagLevels[tagLevelIndex].toLowerCase() return this.tagMapping.getEntry(tagLevel) } @@ -138,4 +145,19 @@ export default class TagConverter { IssueError.generateAndThrow('childRequired', { tag: this.tagString }) } } + + _checkNameClass(index) { + // Check whether the tagLevel is a valid name class + // TODO: this test should be in the schema and the RegExp only created once. + const valueClasses = this.hedSchemas.getSchema(this.tagSpec.library).entries.valueClasses + const myRex = valueClasses._definitions.get('nameClass')?._charClassRegex + const my = new RegExp(myRex) + if (!my.test(this.tagLevels[index])) { + // An extension is not name class + IssueError.generateAndThrow('invalidExtension', { + tag: this.tagLevels[index], + parentTag: this.tagLevels.slice(0, index).join('/'), + }) + } + } } diff --git a/parser/tempRegex.js b/parser/tempRegex.js new file mode 100644 index 00000000..4937ea31 --- /dev/null +++ b/parser/tempRegex.js @@ -0,0 +1,25 @@ +import regexData from '../data/json/class_regex.json' + +// Function to get the RegExp +export function getRegExp(name) { + if (!regexData.class_chars[name]) { + throw new Error(`Invalid class name: ${name}`) + } + + const charNames = regexData.class_chars[name] + if (charNames.length === 0) { + throw new Error(`No character definitions for class: ${name}`) + } + + // Join the individual character regex patterns + const pattern = charNames + .map((charName) => { + if (!regexData.char_regex[charName]) { + throw new Error(`Invalid character name: ${charName}`) + } + return regexData.char_regex[charName] + }) + .join('|') + + return new RegExp(`^(?:${pattern})+$`) +} diff --git a/schema/parser.js b/schema/parser.js index 42e24559..14d32ec3 100644 --- a/schema/parser.js +++ b/schema/parser.js @@ -220,9 +220,9 @@ export default class SchemaParser { for (const [name, valueAttributes] of valueAttributeDefinitions) { const booleanAttributes = booleanAttributeDefinitions.get(name) //valueClasses.set(name, new SchemaValueClass(name, booleanAttributes, valueAttributes)) - const charClassRegex = this._getValueClassChars(name) + const charRegex = this._getValueClassChars(name) const wordRegex = new RegExp(classRegex.class_words[name] ?? '^.+$') - valueClasses.set(name, new SchemaValueClass(name, booleanAttributes, valueAttributes, charClassRegex, wordRegex)) + valueClasses.set(name, new SchemaValueClass(name, booleanAttributes, valueAttributes, charRegex, wordRegex)) } this.valueClasses = new SchemaEntryManager(valueClasses) } diff --git a/schema/regExps.js b/schema/regExps.js new file mode 100644 index 00000000..bd8a2e0b --- /dev/null +++ b/schema/regExps.js @@ -0,0 +1,21 @@ +import classRegex from '../data/json/class_regex.json' + +export class RegexClass { + // Static method that returns the RegExp object + + static getValueClassChars(name) { + let classChars + if (Array.isArray(classRegex.class_chars[name]) && classRegex.class_chars[name].length > 0) { + classChars = + '^(?:' + classRegex.class_chars[name].map((charClass) => classRegex.char_regex[charClass]).join('|') + ')+$' + } else { + classChars = '^.+$' // Any non-empty line or string. + } + return new RegExp(classChars) + } + + static testRegex(name, value) { + const regex = RegexClass.getValueClassChars(name) + return regex.test(value) + } +} diff --git a/schema/tempParser.js b/schema/tempParser.js new file mode 100644 index 00000000..04e4d5f4 --- /dev/null +++ b/schema/tempParser.js @@ -0,0 +1,577 @@ +import zip from 'lodash/zip' +import semver from 'semver' + +// TODO: Switch require once upstream bugs are fixed. +// import xpath from 'xml2js-xpath' +// Temporary +import * as xpath from '../../utils/xpath' + +import { SchemaParser } from './parser' +import { + nodeProperty, + SchemaAttribute, + schemaAttributeProperty, + SchemaEntries, + SchemaEntryManager, + SchemaProperty, + SchemaTag, + SchemaTagManager, + SchemaUnit, + SchemaUnitClass, + SchemaUnitModifier, + SchemaValueClass, + SchemaValueTag, +} from './types' +import { generateIssue, IssueError } from '../../common/issues/issues' + +import classRegex from './class_regex.json' + +const lc = (str) => str.toLowerCase() + +export class Hed3SchemaParser extends SchemaParser { + /** + * @type {Map} + */ + properties + /** + * @type {Map} + */ + attributes + /** + * The schema's value classes. + * @type {SchemaEntryManager} + */ + valueClasses + /** + * The schema's unit classes. + * @type {SchemaEntryManager} + */ + unitClasses + /** + * The schema's unit modifiers. + * @type {SchemaEntryManager} + */ + unitModifiers + /** + * The schema's tags. + * @type {SchemaTagManager} + */ + tags + + constructor(rootElement) { + super(rootElement) + this._versionDefinitions = {} + } + + parse() { + this.populateDictionaries() + return new SchemaEntries(this) + } + + populateDictionaries() { + this.parseProperties() + this.parseAttributes() + this.parseValueClasses() + this.parseUnitModifiers() + this.parseUnitClasses() + this.parseTags() + } + + static attributeFilter(propertyName) { + return (element) => { + const validProperty = propertyName + if (!element.property) { + return false + } + for (const property of element.property) { + if (property.name[0]._ === validProperty) { + return true + } + } + return false + } + } + + /** + * Retrieve all the tags in the schema. + * + * @param {string} tagElementName The name of the tag element. + * @returns {Map} The tag names and XML elements. + */ + getAllTags(tagElementName = 'node') { + const tagElements = xpath.find(this.rootElement, '//' + tagElementName) + const tags = tagElements.map((element) => this.getElementTagName(element)) + return new Map(zip(tagElements, tags)) + } + + // Rewrite starts here. + + parseProperties() { + const propertyDefinitions = this.getElementsByName('propertyDefinition') + this.properties = new Map() + for (const definition of propertyDefinitions) { + const propertyName = this.getElementTagName(definition) + if ( + this._versionDefinitions.categoryProperties && + this._versionDefinitions.categoryProperties.has(propertyName) + ) { + this.properties.set( + propertyName, + // TODO: Switch back to class constant once upstream bug is fixed. + new SchemaProperty(propertyName, 'categoryProperty'), + ) + } else if (this._versionDefinitions.typeProperties && this._versionDefinitions.typeProperties.has(propertyName)) { + this.properties.set( + propertyName, + // TODO: Switch back to class constant once upstream bug is fixed. + new SchemaProperty(propertyName, 'typeProperty'), + ) + } else if (this._versionDefinitions.roleProperties && this._versionDefinitions.roleProperties.has(propertyName)) { + this.properties.set( + propertyName, + // TODO: Switch back to class constant once upstream bug is fixed. + new SchemaProperty(propertyName, 'roleProperty'), + ) + } + } + this._addCustomProperties() + } + + parseAttributes() { + const attributeDefinitions = this.getElementsByName('schemaAttributeDefinition') + this.attributes = new Map() + for (const definition of attributeDefinitions) { + const attributeName = this.getElementTagName(definition) + const propertyElements = definition.property + let properties + if (propertyElements === undefined) { + properties = [] + } else { + properties = propertyElements.map((element) => this.properties.get(element.name[0]._)) + } + this.attributes.set(attributeName, new SchemaAttribute(attributeName, properties)) + } + this._addCustomAttributes() + } + + parseValueClasses() { + const valueClasses = new Map() + const [booleanAttributeDefinitions, valueAttributeDefinitions] = this._parseDefinitions('valueClass') + for (const [name, valueAttributes] of valueAttributeDefinitions) { + const booleanAttributes = booleanAttributeDefinitions.get(name) + let classChars + if (Array.isArray(classRegex.class_chars[name]) && classRegex.class_chars[name].length > 0) { + classChars = + '^(?:' + classRegex.class_chars[name].map((charClass) => classRegex.char_regex[charClass]).join('|') + ')+$' + } else { + classChars = '^.+$' + } + const classCharsRegex = new RegExp(classChars) + const classWordRegex = new RegExp(classRegex.class_words[name] ?? '^.+$') + valueClasses.set( + name, + new SchemaValueClass(name, booleanAttributes, valueAttributes, classCharsRegex, classWordRegex), + ) + } + this.valueClasses = new SchemaEntryManager(valueClasses) + } + + parseUnitModifiers() { + const unitModifiers = new Map() + const [booleanAttributeDefinitions, valueAttributeDefinitions] = this._parseDefinitions('unitModifier') + for (const [name, valueAttributes] of valueAttributeDefinitions) { + const booleanAttributes = booleanAttributeDefinitions.get(name) + unitModifiers.set(name, new SchemaUnitModifier(name, booleanAttributes, valueAttributes)) + } + this.unitModifiers = new SchemaEntryManager(unitModifiers) + } + + parseUnitClasses() { + const unitClasses = new Map() + const [booleanAttributeDefinitions, valueAttributeDefinitions] = this._parseDefinitions('unitClass') + const unitClassUnits = this.parseUnits() + + for (const [name, valueAttributes] of valueAttributeDefinitions) { + const booleanAttributes = booleanAttributeDefinitions.get(name) + unitClasses.set(name, new SchemaUnitClass(name, booleanAttributes, valueAttributes, unitClassUnits.get(name))) + } + this.unitClasses = new SchemaEntryManager(unitClasses) + } + + parseUnits() { + const unitClassUnits = new Map() + const unitClassElements = this.getElementsByName('unitClassDefinition') + const unitModifiers = this.unitModifiers + for (const element of unitClassElements) { + const elementName = this.getElementTagName(element) + const units = new Map() + unitClassUnits.set(elementName, units) + if (element.unit === undefined) { + continue + } + const [unitBooleanAttributeDefinitions, unitValueAttributeDefinitions] = this._parseAttributeElements( + element.unit, + this.getElementTagName, + ) + for (const [name, valueAttributes] of unitValueAttributeDefinitions) { + const booleanAttributes = unitBooleanAttributeDefinitions.get(name) + units.set(name, new SchemaUnit(name, booleanAttributes, valueAttributes, unitModifiers)) + } + } + return unitClassUnits + } + + parseTags() { + const tags = this.getAllTags() + const shortTags = new Map() + for (const tagElement of tags.keys()) { + const shortKey = + this.getElementTagName(tagElement) === '#' + ? this.getParentTagName(tagElement) + '-#' + : this.getElementTagName(tagElement) + shortTags.set(tagElement, shortKey) + } + const [booleanAttributeDefinitions, valueAttributeDefinitions] = this._parseAttributeElements( + tags.keys(), + (element) => shortTags.get(element), + ) + + const recursiveAttributes = this._getRecursiveAttributes() + const tagUnitClassAttribute = this.attributes.get('unitClass') + const tagValueClassAttribute = this.attributes.get('valueClass') + const tagTakesValueAttribute = this.attributes.get('takesValue') + + const tagUnitClassDefinitions = new Map() + const tagValueClassDefinitions = new Map() + const recursiveChildren = new Map() + for (const [tagElement, tagName] of shortTags) { + const valueAttributes = valueAttributeDefinitions.get(tagName) + if (valueAttributes.has(tagUnitClassAttribute)) { + tagUnitClassDefinitions.set( + tagName, + valueAttributes.get(tagUnitClassAttribute).map((unitClassName) => { + return this.unitClasses.getEntry(unitClassName) + }), + ) + valueAttributes.delete(tagUnitClassAttribute) + } + if (valueAttributes.has(tagValueClassAttribute)) { + tagValueClassDefinitions.set( + tagName, + valueAttributes.get(tagValueClassAttribute).map((valueClassName) => { + return this.valueClasses.getEntry(valueClassName) + }), + ) + valueAttributes.delete(tagValueClassAttribute) + } + for (const attribute of recursiveAttributes) { + const children = recursiveChildren.get(attribute) ?? [] + if (booleanAttributeDefinitions.get(tagName).has(attribute)) { + children.push(...this.getAllChildTags(tagElement)) + } + recursiveChildren.set(attribute, children) + } + } + + for (const [attribute, childTagElements] of recursiveChildren) { + for (const tagElement of childTagElements) { + const tagName = this.getElementTagName(tagElement) + booleanAttributeDefinitions.get(tagName).add(attribute) + } + } + + const tagEntries = new Map() + for (const [name, valueAttributes] of valueAttributeDefinitions) { + if (tagEntries.has(name)) { + IssueError.generateAndThrow('duplicateTagsInSchema') + } + const booleanAttributes = booleanAttributeDefinitions.get(name) + const unitClasses = tagUnitClassDefinitions.get(name) + const valueClasses = tagValueClassDefinitions.get(name) + if (booleanAttributes.has(tagTakesValueAttribute)) { + tagEntries.set( + lc(name), + new SchemaValueTag(name, booleanAttributes, valueAttributes, unitClasses, valueClasses), + ) + } else { + tagEntries.set(lc(name), new SchemaTag(name, booleanAttributes, valueAttributes, unitClasses, valueClasses)) + } + } + + for (const tagElement of tags.keys()) { + const tagName = shortTags.get(tagElement) + const parentTagName = shortTags.get(tagElement.$parent) + if (parentTagName) { + tagEntries.get(lc(tagName))._parent = tagEntries.get(lc(parentTagName)) + } + if (this.getElementTagName(tagElement) === '#') { + tagEntries.get(lc(parentTagName))._valueTag = tagEntries.get(lc(tagName)) + } + } + + const longNameTagEntries = new Map() + for (const tag of tagEntries.values()) { + longNameTagEntries.set(lc(tag.longName), tag) + } + + this.tags = new SchemaTagManager(tagEntries, longNameTagEntries) + } + + _parseDefinitions(category) { + const categoryTagName = category + 'Definition' + const definitionElements = this.getElementsByName(categoryTagName) + + return this._parseAttributeElements(definitionElements, this.getElementTagName) + } + + _parseAttributeElements(elements, namer) { + const booleanAttributeDefinitions = new Map() + const valueAttributeDefinitions = new Map() + + for (const element of elements) { + const [booleanAttributes, valueAttributes] = this._parseAttributeElement(element) + + const elementName = namer(element) + booleanAttributeDefinitions.set(elementName, booleanAttributes) + valueAttributeDefinitions.set(elementName, valueAttributes) + } + + return [booleanAttributeDefinitions, valueAttributeDefinitions] + } + + _parseAttributeElement(element) { + const booleanAttributes = new Set() + const valueAttributes = new Map() + + const tagAttributes = element.attribute ?? [] + + for (const tagAttribute of tagAttributes) { + const attributeName = this.getElementTagName(tagAttribute) + if (tagAttribute.value === undefined) { + booleanAttributes.add(this.attributes.get(attributeName)) + continue + } + const values = tagAttribute.value.map((value) => value._) + valueAttributes.set(this.attributes.get(attributeName), values) + } + + return [booleanAttributes, valueAttributes] + } + + _getRecursiveAttributes() { + const attributeArray = Array.from(this.attributes.values()) + if (semver.lt(this.rootElement.$.version, '8.3.0')) { + return attributeArray.filter((attribute) => + attribute.roleProperties.has(this.properties.get('isInheritedProperty')), + ) + } else { + return attributeArray.filter( + (attribute) => !attribute.roleProperties.has(this.properties.get('annotationProperty')), + ) + } + } + + _addCustomAttributes() { + // No-op + } + + _addCustomProperties() { + // No-op + } +} + +export class HedV8SchemaParser extends Hed3SchemaParser { + constructor(rootElement) { + super(rootElement) + this._versionDefinitions = { + typeProperties: new Set(['boolProperty']), + categoryProperties: new Set([ + 'elementProperty', + 'nodeProperty', + 'schemaAttributeProperty', + 'unitProperty', + 'unitClassProperty', + 'unitModifierProperty', + 'valueClassProperty', + ]), + roleProperties: new Set(['recursiveProperty', 'isInheritedProperty', 'annotationProperty']), + } + } + + _addCustomAttributes() { + const isInheritedProperty = this.properties.get('isInheritedProperty') + const extensionAllowedAttribute = this.attributes.get('extensionAllowed') + if (this.rootElement.$.library === undefined && semver.lt(this.rootElement.$.version, '8.2.0')) { + extensionAllowedAttribute._roleProperties.add(isInheritedProperty) + } + const inLibraryAttribute = this.attributes.get('inLibrary') + if (inLibraryAttribute && semver.lt(this.rootElement.$.version, '8.3.0')) { + inLibraryAttribute._roleProperties.add(isInheritedProperty) + } + } + + _addCustomProperties() { + if (this.rootElement.$.library === undefined && semver.lt(this.rootElement.$.version, '8.2.0')) { + const recursiveProperty = new SchemaProperty('isInheritedProperty', 'roleProperty') + this.properties.set('isInheritedProperty', recursiveProperty) + } + } +} + +export class Hed3PartneredSchemaMerger { + /** + * The source of data to be merged. + * @type {Hed3Schema} + */ + source + /** + * The destination of data to be merged. + * @type {Hed3Schema} + */ + destination + + /** + * Constructor. + * + * @param {Hed3Schema} source The source of data to be merged. + * @param {Hed3Schema} destination The destination of data to be merged. + */ + constructor(source, destination) { + this._validate(source, destination) + + this.source = source + this.destination = destination + } + + /** + * Pre-validate the partnered schemas. + * + * @param {Hed3Schema} source The source of data to be merged. + * @param {Hed3Schema} destination The destination of data to be merged. + * @private + */ + _validate(source, destination) { + if (source.generation < 3 || destination.generation < 3) { + IssueError.generateAndThrow('internalConsistencyError', { message: 'Partnered schemas must be HED-3G schemas' }) + } + + if (source.withStandard !== destination.withStandard) { + IssueError.generateAndThrow('differentWithStandard', { + first: source.withStandard, + second: destination.withStandard, + }) + } + } + + /** + * The source schema's tag collection. + * + * @return {SchemaTagManager} + */ + get sourceTags() { + return this.source.entries.tags + } + + /** + * The destination schema's tag collection. + * + * @return {SchemaTagManager} + */ + get destinationTags() { + return this.destination.entries.tags + } + + /** + * Merge two lazy partnered schemas. + * + * @returns {Hed3Schema} The merged partnered schema, for convenience. + */ + mergeData() { + this.mergeTags() + return this.destination + } + + /** + * Merge the tags from two lazy partnered schemas. + */ + mergeTags() { + for (const tag of this.sourceTags.values()) { + this._mergeTag(tag) + } + } + + /** + * Merge a tag from one schema to another. + * + * @param {SchemaTag} tag The tag to copy. + * @private + */ + _mergeTag(tag) { + if (!tag.getNamedAttributeValue('inLibrary')) { + return + } + + const shortName = tag.name + if (this.destinationTags.hasEntry(shortName.toLowerCase())) { + IssueError.generateAndThrow('lazyPartneredSchemasShareTag', { tag: shortName }) + } + + const rootedTagShortName = tag.getNamedAttributeValue('rooted') + if (rootedTagShortName) { + const parentTag = tag.parent + if (parentTag?.name?.toLowerCase() !== rootedTagShortName?.toLowerCase()) { + IssueError.generateAndThrow('internalError', { message: `Node ${shortName} is improperly rooted.` }) + } + } + + this._copyTagToSchema(tag) + } + + /** + * Copy a tag from one schema to another. + * + * @param {SchemaTag} tag The tag to copy. + * @private + */ + _copyTagToSchema(tag) { + const booleanAttributes = new Set() + const valueAttributes = new Map() + + for (const attribute of tag.booleanAttributes) { + booleanAttributes.add(this.destination.entries.attributes.getEntry(attribute.name) ?? attribute) + } + for (const [key, value] of tag.valueAttributes) { + valueAttributes.set(this.destination.entries.attributes.getEntry(key.name) ?? key, value) + } + + /** + * @type {SchemaUnitClass[]} + */ + const unitClasses = tag.unitClasses.map( + (unitClass) => this.destination.entries.unitClasses.getEntry(unitClass.name) ?? unitClass, + ) + /** + * @type {SchemaValueClass[]} + */ + const valueClasses = tag.valueClasses.map( + (valueClass) => this.destination.entries.valueClasses.getEntry(valueClass.name) ?? valueClass, + ) + + let newTag + if (tag instanceof SchemaValueTag) { + newTag = new SchemaValueTag(tag.name, booleanAttributes, valueAttributes, unitClasses, valueClasses) + } else { + newTag = new SchemaTag(tag.name, booleanAttributes, valueAttributes, unitClasses, valueClasses) + } + const destinationParentTag = this.destinationTags.getEntry(tag.parent?.name?.toLowerCase()) + if (destinationParentTag) { + newTag._parent = destinationParentTag + if (newTag instanceof SchemaValueTag) { + newTag.parent._valueTag = newTag + } + } + + this.destinationTags._definitions.set(newTag.name.toLowerCase(), newTag) + this.destinationTags._definitionsByLongName.set(newTag.longName.toLowerCase(), newTag) + } +} diff --git a/tests/convertExperiments.js b/tests/convertExperiments.js new file mode 100644 index 00000000..d60b8770 --- /dev/null +++ b/tests/convertExperiments.js @@ -0,0 +1,25 @@ +const { SchemaSpec, SchemasSpec } = require('../schema/specs') +const { TagSpec } = require('../parser/tokenizer') +const path = require('path') +const { buildSchemas } = require('../schema/init') +// const { SchemaTag } = require('../schema/entries'); +const TagConverter = require('../parser/tagConverter') + +async function getSchema() { + const spec3 = new SchemaSpec('', '8.3.0', '', path.join(__dirname, '../tests/data/HED8.3.0.xml')) + const specs3 = new SchemasSpec().addSchemaSpec(spec3) + const schemas3 = await buildSchemas(specs3) + return schemas3 +} + +async function runConversion() { + const hedSchema = await getSchema() // Wait for schema to be ready + const spec = new TagSpec('Item/object', 1, 12, '') + + const myCon = new TagConverter(spec, hedSchema) + + const [tag, remainder] = myCon.convert() + console.log(tag, remainder) +} + +runConversion() diff --git a/tests/converter.spec.js b/tests/converter.spec.js index 515c5122..895c41e3 100644 --- a/tests/converter.spec.js +++ b/tests/converter.spec.js @@ -41,7 +41,7 @@ describe('HED string conversion', () => { } // TODO: Remove this test - Test now implemented as valid-long-to-short in conversionTestsData - it('should convert basic HED tags to short form', () => { + it.skip('(REMOVE) should convert basic HED tags to short form', () => { const testStrings = { singleLevel: 'Event', twoLevel: 'Event/Sensory-event', @@ -67,7 +67,7 @@ describe('HED string conversion', () => { }) // TODO: Remove this test - Test now implemented as valid-long-to-short in conversionTestsData - note not values - it('should convert HED tags with values to short form', () => { + it.skip('(REMOVE) should convert HED tags with values to short form', () => { const testStrings = { uniqueValue: 'Item/Sound/Environmental-sound/Unique Value', multiLevel: 'Item/Sound/Environmental-sound/Long Unique Value With/Slash Marks', @@ -87,7 +87,7 @@ describe('HED string conversion', () => { }) // TODO: Remove this test - Test now implemented as valid-long-to-short in conversionTestsData - it('should raise an issue if a "value" is an already valid node', () => { + it.skip('(REMOVE) should raise an issue if a "value" is an already valid node', () => { const testStrings = { singleLevel: 'Item/Sound/Environmental-sound/Event', multiLevel: 'Item/Sound/Environmental-sound/Event/Sensory-event', @@ -107,7 +107,7 @@ describe('HED string conversion', () => { }) //TODO: This is a repeat of the tags with values (which are tags with extensions) Remove this. - it('should convert HED tags with extensions to short form', () => { + it.skip('(REMOVE)should convert HED tags with extensions to short form', () => { const testStrings = { singleLevel: 'Item/Object/extended lvl1', multiLevel: 'Item/Object/extended lvl1/Extension2', @@ -127,7 +127,7 @@ describe('HED string conversion', () => { }) //TODO: Remove this -- shouldn't parse when invalid -- the rest is tested for - it('should raise an issue if an "extension" is already a valid node', () => { + it.skip('(REMOVE) should raise an issue if an "extension" is already a valid node', () => { const testStrings = { validThenInvalid: 'Item/Object/valid extension followed by invalid/Event', singleLevel: 'Item/Object/Visual-presentation', @@ -168,7 +168,7 @@ describe('HED string conversion', () => { }) //TODO: Remove -- these cases are handled by stringParserTests - it('should raise an issue if an invalid node is found', () => { + it.skip('(REMOVE)should raise an issue if an invalid node is found', () => { const testStrings = { invalidParentWithExistingGrandchild: 'InvalidItem/Object/Visual-presentation', invalidChildWithExistingGrandchild: 'Event/InvalidEvent/Geometric-object', @@ -199,7 +199,7 @@ describe('HED string conversion', () => { return validator(testStrings, expectedResults, expectedIssues) }) - it('should validate whether a node actually allows extensions', () => { + it.skip('should validate whether a node actually allows extensions', () => { const testStrings = { validTakesValue: 'Property/Agent-property/Agent-trait/Age/15', cascadeExtension: 'Property/Agent-property/Agent-state/Agent-emotional-state/Awed/Cascade Extension', @@ -287,8 +287,8 @@ describe('HED string conversion', () => { const validator = function (testStrings, expectedResults, expectedIssues) { return validatorBase(testStrings, expectedResults, expectedIssues, converter.convertHedStringToLong) } - - it('should convert basic HED tags to long form', () => { + //TODO: now part of tag parsing + it.skip('(REMOVE)should convert basic HED tags to long form', () => { const testStrings = { singleLevel: 'Event', twoLevel: 'Sensory-event', @@ -313,7 +313,7 @@ describe('HED string conversion', () => { return validator(testStrings, expectedResults, expectedIssues) }) - it('should convert HED tags with values to long form', () => { + it.skip('should convert HED tags with values to long form', () => { const testStrings = { uniqueValue: 'Environmental-sound/Unique Value', multiLevel: 'Environmental-sound/Long Unique Value With/Slash Marks', @@ -332,7 +332,7 @@ describe('HED string conversion', () => { return validator(testStrings, expectedResults, expectedIssues) }) - it('should convert HED tags with extensions to long form', () => { + it.skip('should convert HED tags with extensions to long form', () => { const testStrings = { singleLevel: 'Object/extended lvl1', multiLevel: 'Object/extended lvl1/Extension2', @@ -351,7 +351,7 @@ describe('HED string conversion', () => { return validator(testStrings, expectedResults, expectedIssues) }) - it('should raise an issue if an "extension" is already a valid node', () => { + it.skip('should raise an issue if an "extension" is already a valid node', () => { const testStrings = { validThenInvalid: 'Object/valid extension followed by invalid/Event', singleLevel: 'Object/Visual-presentation', @@ -391,7 +391,7 @@ describe('HED string conversion', () => { return validator(testStrings, expectedResults, expectedIssues) }) - it('should raise an issue if an invalid node is found', () => { + it.skip('should raise an issue if an invalid node is found', () => { const testStrings = { single: 'InvalidEvent', invalidChild: 'InvalidEvent/InvalidExtension', @@ -410,7 +410,7 @@ describe('HED string conversion', () => { return validator(testStrings, expectedResults, expectedIssues) }) - it('should validate whether a node actually allows extensions', () => { + it.skip('should validate whether a node actually allows extensions', () => { const testStrings = { validTakesValue: 'Age/15', cascadeExtension: 'Awed/Cascade Extension', @@ -429,7 +429,7 @@ describe('HED string conversion', () => { return validator(testStrings, expectedResults, expectedIssues) }) - it('should handle leading and trailing spaces correctly', () => { + it.skip('should handle leading and trailing spaces correctly', () => { const testStrings = { leadingSpace: ' Environmental-sound/Unique Value', trailingSpace: 'Environmental-sound/Unique Value ', @@ -445,7 +445,8 @@ describe('HED string conversion', () => { return validator(testStrings, expectedResults, expectedIssues) }) - it.skip('should strip leading and trailing slashes', () => { + // TODO: These are taken care of in the tokenizer + it.skip('(REMOVE)should strip leading and trailing slashes', () => { const testStrings = { leadingSingle: '/Event', leadingExtension: '/Event/Extension', @@ -491,7 +492,8 @@ describe('HED string conversion', () => { return validator(testStrings, expectedResults, expectedIssues) }) - it('should properly handle node names in value-taking strings', () => { + // TODO: Revisit + it.skip('(REVISIT) should properly handle node names in value-taking strings', () => { const testStrings = { valueTaking: 'Label/Red', nonValueTaking: 'Train/Car', @@ -541,7 +543,7 @@ describe('HED string conversion', () => { } } - describe('Long-to-short', () => { + describe.skip('Long-to-short', () => { const validator = function (testStrings, expectedResults, expectedIssues) { return validatorBase(testStrings, expectedResults, expectedIssues, converter.convertHedStringToShort) } @@ -607,7 +609,7 @@ describe('HED string conversion', () => { return validator(testStrings, expectedResults, expectedIssues) }) - it('should ignore leading and trailing spaces', () => { + it.skip('(REMOVE) should ignore leading and trailing spaces', () => { const testStrings = { leadingSpace: ' Item/Sound/Environmental-sound/Unique Value', trailingSpace: 'Item/Sound/Environmental-sound/Unique Value ', @@ -711,7 +713,7 @@ describe('HED string conversion', () => { }) }) - describe('Short-to-long', () => { + describe.skip('Short-to-long', () => { const validator = function (testStrings, expectedResults, expectedIssues) { return validatorBase(testStrings, expectedResults, expectedIssues, converter.convertHedStringToLong) } diff --git a/tests/event.spec.js b/tests/event.spec.js index 24871f6a..3ca0c218 100644 --- a/tests/event.spec.js +++ b/tests/event.spec.js @@ -1070,7 +1070,7 @@ describe('HED string and event validation', () => { } // TODO: Remove -- now in tokenizer tests - it('(REMOVE) should properly handle strings with placeholders', () => { + it.skip('(REMOVE) should properly handle strings with placeholders', () => { const testStrings = { takesValue: 'RGB-red/#', withUnit: 'Time-value/# ms', diff --git a/tests/tagParserTests.spec.js b/tests/tagParserTests.spec.js index f2014637..399a2dcf 100644 --- a/tests/tagParserTests.spec.js +++ b/tests/tagParserTests.spec.js @@ -1,6 +1,6 @@ import chai from 'chai' const assert = chai.assert -import { beforeAll, describe, afterAll } from '@jest/globals' +import { beforeAll, describe, afterAll, it } from '@jest/globals' import ParsedHedTag from '../parser/parsedHedTag' import { shouldRun } from './testUtilities' @@ -8,7 +8,9 @@ import { parsedHedTagTests } from './testData/tagParserTests.data' import { SchemaSpec, SchemasSpec } from '../schema/specs' import path from 'path' import { buildSchemas } from '../schema/init' -import { SchemaTag } from '../schema/entries' +import { SchemaTag, SchemaValueTag } from '../schema/entries' +import { TagSpec } from '../parser/tokenizer' +import TagConverter from '../parser/tagConverter' // Ability to select individual tests to run const skipMap = new Map() @@ -34,6 +36,27 @@ describe('TagSpec converter tests using JSON tests', () => { afterAll(() => {}) + describe('TagConverter tests', () => { + /* it('should be able to convert', () => { + const thisSchema = schemaMap.get('8.3.0') + assert.isDefined(thisSchema, 'yes') + + // let schema1 = thisSchema.schemas.get("") + // let valueClassManager = schema1.entries.valueClasses + + //const spec = new TagSpec('Item/Ble ch', 0, 11, ''); + //const spec = new TagSpec('Item/Blech', 0, 10, ''); + + //const spec = new TagSpec('Item/Junk/Object', 0, 16, ''); + const spec = new TagSpec('object/Junk/baloney/Red', 0, 22, '') + const myCon = new TagConverter(spec, thisSchema) + const [tag, remainder] = myCon.convert(); + assert.instanceOf(tag, SchemaTag, 'A schema tag comes back') + //assert.instanceOf(remainder, String, 'A string comes back') + + })*/ + }) + describe.each(parsedHedTagTests)('$name : $description', ({ name, tests }) => { const hedTagTest = function (test) { const status = test.error !== null ? 'Expect fail' : 'Expect pass' @@ -49,11 +72,19 @@ describe('TagSpec converter tests using JSON tests', () => { } catch (error) { issue = error.issue } - assert.deepEqual(issue, issue) + assert.deepEqual(issue, test.error, `${header}: wrong issue`) assert.strictEqual(tag?.format(false), test.tagShort, `${header}: wrong short version`) assert.strictEqual(tag?.format(true), test.tagLong, `${header}: wrong long version`) assert.strictEqual(tag?.formattedTag, test.formattedTag, `${header}: wrong formatted version`) assert.strictEqual(tag?.canonicalTag, test.canonicalTag, `${header}: wrong canonical version`) + if (test.error) { + return + } + if (test.takesValue) { + assert.instanceOf(tag._schemaTag, SchemaValueTag, `${header}: tag should be a takes value tag`) + } else { + assert.notInstanceOf(tag._schemaTag, SchemaValueTag, `${header}: tag should be a takes value tag`) + } } beforeAll(async () => {}) diff --git a/tests/testData/stringParserTests.data.js b/tests/testData/stringParserTests.data.js index 72f36a65..e045f6af 100644 --- a/tests/testData/stringParserTests.data.js +++ b/tests/testData/stringParserTests.data.js @@ -164,7 +164,7 @@ export const parseTestData = [ stringIn: 'Item/Sound/Event', stringLong: null, stringShort: null, - errors: [generateIssue('invalidParentNode', { tag: 'Event', parentTag: 'Event' })], + errors: [generateIssue('invalidParentNode', { tag: 'Event', parentTag: 'Item/Sound' })], warnings: [], }, { @@ -174,7 +174,7 @@ export const parseTestData = [ stringIn: 'Item/Sound/Environmental-sound/Event/Sensory-event', stringLong: null, stringShort: null, - errors: [generateIssue('invalidParentNode', { parentTag: 'Event', tag: 'Event' })], + errors: [generateIssue('invalidParentNode', { parentTag: 'Item/Sound/Environmental-sound', tag: 'Event' })], warnings: [], }, { @@ -184,17 +184,7 @@ export const parseTestData = [ stringIn: 'Item/Sound/Event/Sensory-event/Environmental-sound', stringLong: null, stringShort: null, - errors: [generateIssue('invalidParentNode', { parentTag: 'Event', tag: 'Event' })], - warnings: [], - }, - { - testname: 'mixed-extension-path', - explanation: '"Sensory-event" in "Item/Sound/Event/Sensory-event/Environmental-sound"', - schemaVersion: '8.3.0', - stringIn: 'Item/Sound/Event/Sensory-event/Environmental-sound', - stringLong: null, - stringShort: null, - errors: [generateIssue('invalidParentNode', { parentTag: 'Event', tag: 'Event' })], + errors: [generateIssue('invalidParentNode', { parentTag: 'Item/Sound', tag: 'Event' })], warnings: [], }, { @@ -235,9 +225,7 @@ export const parseTestData = [ stringIn: 'Item/Object/Junk/Geometric-object/2D-shape', stringLong: null, stringShort: null, - errors: [ - generateIssue('invalidParentNode', { parentTag: 'Item/Object/Geometric-object', tag: 'Geometric-object' }), - ], + errors: [generateIssue('invalidParentNode', { parentTag: 'Item/Object/Junk', tag: 'Geometric-object' })], warnings: [], }, { diff --git a/tests/testData/tagParserTests.data.js b/tests/testData/tagParserTests.data.js index 852a2379..24fa6c8d 100644 --- a/tests/testData/tagParserTests.data.js +++ b/tests/testData/tagParserTests.data.js @@ -17,6 +17,7 @@ export const parsedHedTagTests = [ tagShort: 'Item', formattedTag: 'item', canonicalTag: 'Item', + takesValue: false, error: null, }, { @@ -84,6 +85,19 @@ export const parsedHedTagTests = [ takesValue: true, error: null, }, + { + testname: 'valid-value-tag-with-placeholder', + explanation: '" Label/# " is a valid tag with placeholder.', + schemaVersion: '8.3.0', + fullString: ' Label/# ', + tagSpec: new TagSpec(' Label/# ', 1, 8, ''), + tagLong: 'Property/Informational-property/Label/#', + tagShort: 'Label/#', + formattedTag: 'property/informational-property/label/#', + canonicalTag: 'Property/Informational-property/Label/#', + takesValue: true, + error: null, + }, ], }, { @@ -91,6 +105,19 @@ export const parsedHedTagTests = [ description: 'Various invalid tags', warning: false, tests: [ + { + testname: 'invalid-top-level-tag', + explanation: '"Blech" is not a valid tag.', + schemaVersion: '8.3.0', + fullString: 'Blech', + tagSpec: new TagSpec('Blech', 0, 6, ''), + tagLong: undefined, + tagShort: undefined, + formattedTag: undefined, + canonicalTag: undefined, + takesValue: false, + error: generateIssue('invalidTag', { tag: 'Blech' }), + }, { testname: 'invalid-tag-requires-child', explanation: '"Duration" should have a child.', @@ -101,8 +128,22 @@ export const parsedHedTagTests = [ tagShort: undefined, formattedTag: undefined, canonicalTag: undefined, + takesValue: true, error: generateIssue('childRequired', { tag: 'Duration' }), }, + { + testname: 'invalid-tag-with-blank-in-extension', + explanation: '" Object/blec h " has a blank in the tag extension', + schemaVersion: '8.3.0', + fullString: ' Object/blec h ', + tagSpec: new TagSpec(' Object/blec h ', 1, 14, ''), + tagLong: undefined, + tagShort: undefined, + formattedTag: undefined, + canonicalTag: undefined, + takesValue: false, + error: generateIssue('invalidExtension', { parentTag: 'Object', tag: 'blec h' }), + }, { testname: 'invalid-tag-should-not-have-a-placeholder', explanation: '"object/#" should not have a placeholder.', @@ -113,7 +154,34 @@ export const parsedHedTagTests = [ tagShort: undefined, formattedTag: undefined, canonicalTag: undefined, - error: null, + takesValue: false, + error: generateIssue('invalidExtension', { parentTag: 'object', tag: '#' }), + }, + { + testname: 'invalid-tag-bad-parent', + explanation: '"object/property/Red" -- property is not a child of object.', + schemaVersion: '8.3.0', + fullString: 'object/property/Red', + tagSpec: new TagSpec('object/property/Red', 0, 19, ''), + tagLong: undefined, + tagShort: undefined, + formattedTag: undefined, + canonicalTag: undefined, + takesValue: false, + error: generateIssue('invalidParentNode', { parentTag: 'object', tag: 'property' }), + }, + { + testname: 'invalid-tag-bad-parent-after extension', + explanation: '"object/Junk/baloney/Red" -- Red is not a child of baloney.', + schemaVersion: '8.3.0', + fullString: 'object/Junk/baloney/Red', + tagSpec: new TagSpec('object/Junk/baloney/Red', 0, 22, ''), + tagLong: undefined, + tagShort: undefined, + formattedTag: undefined, + canonicalTag: undefined, + takesValue: false, + error: generateIssue('invalidParentNode', { parentTag: 'object/Junk/baloney', tag: 'Red' }), }, ], },