From 15548a68a45f6117477f403405177bf0d1f88a4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Berkefeld?= Date: Wed, 30 Aug 2023 16:02:10 +0200 Subject: [PATCH] #9: switch to folder-based collection retrieve; resolve all filterDefinition details; split off hidden ones, resolve filterDefinition in filter --- docs/dist/documentation.md | 108 +++- lib/MetadataTypeDefinitions.js | 1 + lib/MetadataTypeInfo.js | 1 + lib/metadataTypes/DataExtension.js | 2 +- lib/metadataTypes/Filter.js | 50 +- lib/metadataTypes/FilterDefinition.js | 493 ++++++++++++++---- lib/metadataTypes/FilterDefinitionHidden.js | 24 + .../definitions/Filter.definition.js | 19 +- .../FilterDefinition.definition.js | 66 ++- .../FilterDefinitionHidden.definition.js | 156 ++++++ 10 files changed, 751 insertions(+), 169 deletions(-) create mode 100644 lib/metadataTypes/FilterDefinitionHidden.js create mode 100644 lib/metadataTypes/definitions/FilterDefinitionHidden.definition.js diff --git a/docs/dist/documentation.md b/docs/dist/documentation.md index 319125d8c..0df8883e8 100644 --- a/docs/dist/documentation.md +++ b/docs/dist/documentation.md @@ -70,6 +70,9 @@ as this is a configuration in the EID

FilterDefinition ⇐ MetadataType

FilterDefinition MetadataType

+
FilterDefinitionHidden ⇐ FilterDefinitionHidden
+

FilterDefinitionHidden MetadataType

+
Folder ⇐ MetadataType

Folder MetadataType

@@ -2884,8 +2887,7 @@ Filter MetadataType * [Filter](#Filter) ⇐ [MetadataType](#MetadataType) * [.retrieve(retrieveDir, [_], [__], [key])](#Filter.retrieve) ⇒ Promise.<{metadata: TYPE.FilterMap, type: string}> - * [.postRetrieveTasks(item)](#Filter.postRetrieveTasks) ⇒ TYPE.FilterItem - * [.parseMetadata(metadata)](#Filter.parseMetadata) ⇒ TYPE.FilterItem + * [.postRetrieveTasks(metadata)](#Filter.postRetrieveTasks) ⇒ TYPE.FilterItem * [.preDeployTasks(metadata)](#Filter.preDeployTasks) ⇒ Promise.<TYPE.FilterItem> @@ -2908,19 +2910,7 @@ Filters with the endpoint /automation/v1/filters/{id} -### Filter.postRetrieveTasks(item) ⇒ TYPE.FilterItem -manages post retrieve steps - -**Kind**: static method of [Filter](#Filter) -**Returns**: TYPE.FilterItem - parsed metadata definition - -| Param | Type | Description | -| --- | --- | --- | -| item | TYPE.FilterItem | a single record | - - - -### Filter.parseMetadata(metadata) ⇒ TYPE.FilterItem +### Filter.postRetrieveTasks(metadata) ⇒ TYPE.FilterItem parses retrieved Metadata before saving **Kind**: static method of [Filter](#Filter) @@ -2952,8 +2942,15 @@ FilterDefinition MetadataType * [FilterDefinition](#FilterDefinition) ⇐ [MetadataType](#MetadataType) * [.retrieve(retrieveDir, [_], [__], [key])](#FilterDefinition.retrieve) ⇒ Promise.<{metadata: TYPE.FilterDefinitionMap, type: string}> + * [.getFilterFolderIds([hidden])](#FilterDefinition.getFilterFolderIds) ⇒ Array.<number> + * [.getMeasureFolderIds()](#FilterDefinition.getMeasureFolderIds) ⇒ Array.<number> + * [.cacheDeFields(metadataTypeMapObj)](#FilterDefinition.cacheDeFields) + * [.cacheContactAttributes(metadataTypeMapObj)](#FilterDefinition.cacheContactAttributes) + * [.cacheMeasures(metadataTypeMapObj)](#FilterDefinition.cacheMeasures) * [.retrieveForCache()](#FilterDefinition.retrieveForCache) ⇒ Promise.<{metadata: TYPE.FilterDefinitionMap, type: string}> * [.postRetrieveTasks(metadata)](#FilterDefinition.postRetrieveTasks) ⇒ TYPE.FilterDefinitionItem + * [.resolveFieldIds(metadata, [fieldCache], [filter])](#FilterDefinition.resolveFieldIds) ⇒ void + * [.resolveAttributeIds(metadata, [filter])](#FilterDefinition.resolveAttributeIds) ⇒ void * [.preDeployTasks(metadata)](#FilterDefinition.preDeployTasks) ⇒ Promise.<TYPE.FilterDefinitionItem> * [.create(metadata)](#FilterDefinition.create) ⇒ Promise.<TYPE.FilterDefinitionItem> * [.update(metadata)](#FilterDefinition.update) ⇒ Promise.<TYPE.FilterDefinitionItem> @@ -2973,6 +2970,52 @@ Retrieves all records and saves it to disk | [__] | void | unused parameter | | [key] | string | customer key of single item to retrieve | + + +### FilterDefinition.getFilterFolderIds([hidden]) ⇒ Array.<number> +helper for [retrieve](#FilterDefinition.retrieve) + +**Kind**: static method of [FilterDefinition](#FilterDefinition) +**Returns**: Array.<number> - Array of folder IDs + +| Param | Type | Default | Description | +| --- | --- | --- | --- | +| [hidden] | boolean | false | used to filter out hidden or non-hidden filterDefinitions | + + + +### FilterDefinition.getMeasureFolderIds() ⇒ Array.<number> +helper for [retrieve](#FilterDefinition.retrieve) + +**Kind**: static method of [FilterDefinition](#FilterDefinition) +**Returns**: Array.<number> - Array of folder IDs + + +### FilterDefinition.cacheDeFields(metadataTypeMapObj) +**Kind**: static method of [FilterDefinition](#FilterDefinition) + +| Param | Type | Description | +| --- | --- | --- | +| metadataTypeMapObj | TYPE.MultiMetadataTypeMap | - | + + + +### FilterDefinition.cacheContactAttributes(metadataTypeMapObj) +**Kind**: static method of [FilterDefinition](#FilterDefinition) + +| Param | Type | Description | +| --- | --- | --- | +| metadataTypeMapObj | TYPE.MultiMetadataTypeMap | - | + + + +### FilterDefinition.cacheMeasures(metadataTypeMapObj) +**Kind**: static method of [FilterDefinition](#FilterDefinition) + +| Param | Type | Description | +| --- | --- | --- | +| metadataTypeMapObj | TYPE.MultiMetadataTypeMap | - | + ### FilterDefinition.retrieveForCache() ⇒ Promise.<{metadata: TYPE.FilterDefinitionMap, type: string}> @@ -2992,6 +3035,27 @@ parses retrieved Metadata before saving | --- | --- | --- | | metadata | TYPE.FilterDefinitionItem | a single record | + + +### FilterDefinition.resolveFieldIds(metadata, [fieldCache], [filter]) ⇒ void +**Kind**: static method of [FilterDefinition](#FilterDefinition) + +| Param | Type | Description | +| --- | --- | --- | +| metadata | TYPE.FilterDefinitionItem | - | +| [fieldCache] | Array.<object> | - | +| [filter] | object | - | + + + +### FilterDefinition.resolveAttributeIds(metadata, [filter]) ⇒ void +**Kind**: static method of [FilterDefinition](#FilterDefinition) + +| Param | Type | Description | +| --- | --- | --- | +| metadata | TYPE.FilterDefinitionItem | - | +| [filter] | object | - | + ### FilterDefinition.preDeployTasks(metadata) ⇒ Promise.<TYPE.FilterDefinitionItem> @@ -3028,6 +3092,20 @@ Updates a single item | --- | --- | --- | | metadata | TYPE.FilterDefinitionItem | a single item | + + +## FilterDefinitionHidden ⇐ [FilterDefinitionHidden](#FilterDefinitionHidden) +FilterDefinitionHidden MetadataType + +**Kind**: global class +**Extends**: [FilterDefinitionHidden](#FilterDefinitionHidden) + + +### FilterDefinitionHidden.getFilterFolderIds() ⇒ Array.<number> +helper for [retrieve](#FilterDefinition.retrieve) + +**Kind**: static method of [FilterDefinitionHidden](#FilterDefinitionHidden) +**Returns**: Array.<number> - Array of folder IDs ## Folder ⇐ [MetadataType](#MetadataType) diff --git a/lib/MetadataTypeDefinitions.js b/lib/MetadataTypeDefinitions.js index cba29b1db..68da505fa 100644 --- a/lib/MetadataTypeDefinitions.js +++ b/lib/MetadataTypeDefinitions.js @@ -23,6 +23,7 @@ const MetadataTypeDefinitions = { fileTransfer: require('./metadataTypes/definitions/FileTransfer.definition'), filter: require('./metadataTypes/definitions/Filter.definition'), filterDefinition: require('./metadataTypes/definitions/FilterDefinition.definition'), + filterDefinitionHidden: require('./metadataTypes/definitions/FilterDefinitionHidden.definition'), folder: require('./metadataTypes/definitions/Folder.definition'), importFile: require('./metadataTypes/definitions/ImportFile.definition'), journey: require('./metadataTypes/definitions/Journey.definition'), diff --git a/lib/MetadataTypeInfo.js b/lib/MetadataTypeInfo.js index f88326649..6f4112bc4 100644 --- a/lib/MetadataTypeInfo.js +++ b/lib/MetadataTypeInfo.js @@ -23,6 +23,7 @@ const MetadataTypeInfo = { fileTransfer: require('./metadataTypes/FileTransfer'), filter: require('./metadataTypes/Filter'), filterDefinition: require('./metadataTypes/FilterDefinition'), + filterDefinitionHidden: require('./metadataTypes/FilterDefinitionHidden'), folder: require('./metadataTypes/Folder'), importFile: require('./metadataTypes/ImportFile'), journey: require('./metadataTypes/Journey'), diff --git a/lib/metadataTypes/DataExtension.js b/lib/metadataTypes/DataExtension.js index 6e96f463b..9676723ae 100644 --- a/lib/metadataTypes/DataExtension.js +++ b/lib/metadataTypes/DataExtension.js @@ -1341,7 +1341,7 @@ class DataExtension extends MetadataType { * @returns {Promise.<{metadata: TYPE.DataExtensionMap, type: string}>} Promise */ static async retrieveForCache() { - return this.retrieve(null, ['ObjectID', 'CustomerKey', 'Name'], this.buObject, null, null); + return this.retrieve(null, ['ObjectID', 'CustomerKey', 'Name']); } /** * Retrieves dataExtension metadata in template format. diff --git a/lib/metadataTypes/Filter.js b/lib/metadataTypes/Filter.js index 4a09dc1fa..214a6c3c6 100644 --- a/lib/metadataTypes/Filter.js +++ b/lib/metadataTypes/Filter.js @@ -33,32 +33,17 @@ class Filter extends MetadataType { static async retrieve(retrieveDir, _, __, key) { return super.retrieveREST(retrieveDir, '/automation/v1/filters/', null, key); } - /** - * manages post retrieve steps - * - * @param {TYPE.FilterItem} item a single record - * @returns {TYPE.FilterItem} parsed metadata definition - */ - static postRetrieveTasks(item) { - return this.parseMetadata(item); - } /** * parses retrieved Metadata before saving * * @param {TYPE.FilterItem} metadata a single record * @returns {TYPE.FilterItem} parsed metadata definition */ - static parseMetadata(metadata) { - try { - // folder - metadata.r__folder_Path = cache.searchForField( - 'folder', - metadata.categoryId, - 'ID', - 'Path' - ); - delete metadata.categoryId; + static postRetrieveTasks(metadata) { + // folder + this.setFolderPath(metadata); + try { // filterDefinition metadata.r__filterDefinition_CustomerKey = cache.searchForField( 'filterDefinition', @@ -67,7 +52,21 @@ class Filter extends MetadataType { 'key' ); delete metadata.filterDefinitionId; - + } catch { + try { + // filterDefinition + metadata.r__filterDefinition_CustomerKey = cache.searchForField( + 'filterDefinitionHidden', + metadata.filterDefinitionId, + 'id', + 'key' + ); + delete metadata.filterDefinitionId; + } catch { + // ignore + } + } + try { // source if (metadata.sourceTypeId === 1) { // list @@ -90,7 +89,12 @@ class Filter extends MetadataType { }` ); } - + } catch (ex) { + Util.logger.warn( + ` - filter '${metadata.name}' (${metadata.customerKey}): Destination not found (${ex.message})` + ); + } + try { // target if (metadata.destinationTypeId === 1) { // list @@ -106,7 +110,7 @@ class Filter extends MetadataType { delete metadata.destinationTypeId; } else { Util.logger.warn( - ` - Filter '${metadata.name}' (${ + ` - filter '${metadata.name}' (${ metadata.customerKey }): Unsupported destination type ${metadata.destinationTypeId}=${ dataTypes[metadata.destinationTypeId] @@ -115,7 +119,7 @@ class Filter extends MetadataType { } } catch (ex) { Util.logger.warn( - ` - Filter '${metadata.name}' (${metadata.customerKey}): ${ex.message}` + ` - filter '${metadata.name}' (${metadata.customerKey}): Source not found (${ex.message})` ); } return metadata; diff --git a/lib/metadataTypes/FilterDefinition.js b/lib/metadataTypes/FilterDefinition.js index 5422e72eb..6c480fed2 100644 --- a/lib/metadataTypes/FilterDefinition.js +++ b/lib/metadataTypes/FilterDefinition.js @@ -2,6 +2,8 @@ const TYPE = require('../../types/mcdev.d'); const MetadataType = require('./MetadataType'); +const DataExtensionField = require('./DataExtensionField'); +const Folder = require('./Folder'); const Util = require('../util/util'); const cache = require('../util/cache'); const { XMLBuilder, XMLParser } = require('fast-xml-parser'); @@ -12,6 +14,7 @@ const { XMLBuilder, XMLParser } = require('fast-xml-parser'); * @augments MetadataType */ class FilterDefinition extends MetadataType { + static cache = {}; // type internal cache for various things /** * Retrieves all records and saves it to disk * @@ -22,74 +25,223 @@ class FilterDefinition extends MetadataType { * @returns {Promise.<{metadata: TYPE.FilterDefinitionMap, type: string}>} Promise of items */ static async retrieve(retrieveDir, _, __, key) { - // #1 get the list via SOAP cause the corresponding REST call has no BU filter apparently - // for reference the rest path: '/automation/v1/filterdefinitions?view=categoryinfo' - - const soapFields = ['DataFilter', 'ObjectID', 'CustomerKey', 'Name']; - let requestParams; - if (key) { - requestParams = { - filter: { - leftOperand: 'CustomerKey', - operator: 'equals', - rightOperand: key, - }, - }; - } - - /** - * @type {TYPE.FilterDefinitionSOAPItemMap[]} - */ - const responseSOAP = await this.client.soap.retrieveBulk( - this.definition.type, - soapFields, - requestParams - ); + const filterFolders = await this.getFilterFolderIds(); - // backup REST value of the keyField - const keyFieldBak = this.definition.keyField; - this.definition.keyField = 'CustomerKey'; - const responseSOAPMap = this.parseResponseBody(responseSOAP, key); - // restore the keyField to its REST value - this.definition.keyField = keyFieldBak; - - /** - * @type {TYPE.FilterDefinitionSOAPItem[]} - */ - const responseSOAPList = responseSOAP.Results.filter((item) => { - if (item.ObjectState) { - Util.logger.debug(`Filtered filterDefinition ${item.Name}: ${item.ObjectState}`); - return false; - } else { - return true; - } - }); - - // #2 - // /automation/v1/filterdefinitions/ - const metadataMap = ( - await super.retrieveRESTcollection( - responseSOAPList.map((item) => ({ - id: item.ObjectID, - uri: '/email/v1/filters/filterdefinition/' + item.ObjectID, - })) - ) - ).metadata; - for (const item of Object.values(metadataMap)) { - // description is not returned when empty + const metadataTypeMapObj = { metadata: {}, type: this.definition.type }; + for (const folderId of filterFolders) { + const metadataMapFolder = await super.retrieveREST( + null, + 'email/v1/filters/filterdefinition/category/' + + folderId + + '?derivedFromType=1,2,3,4&', + null, + key + ); + if (Object.keys(metadataMapFolder.metadata).length) { + metadataTypeMapObj.metadata = { + ...metadataTypeMapObj.metadata, + ...metadataMapFolder.metadata, + }; + if (key) { + // if key was found we can stop checking other folders + break; + } + } + } + // console.log('metadataMap', metadataMap); + + for (const item of Object.values(metadataTypeMapObj.metadata)) { + // description is not returned when emptyg item.description ||= ''; - // add extra info from XML - item.c__soap_DataFilter = responseSOAPMap[item.key].DataFilter; } if (retrieveDir) { - const savedMetadata = await this.saveResults(metadataMap, retrieveDir); + // custom dataExtensionField caching + await this.cacheDeFields(metadataTypeMapObj); + await this.cacheContactAttributes(metadataTypeMapObj); + await this.cacheMeasures(metadataTypeMapObj); + + const savedMetadata = await this.saveResults(metadataTypeMapObj.metadata, retrieveDir); Util.logger.info( - `Downloaded: ${this.definition.type} (${Object.keys(savedMetadata).length})` + `Downloaded: ${this.definition.type} (${Object.keys(savedMetadata).length})` + + Util.getKeysString(key) ); } - return { metadata: metadataMap, type: this.definition.type }; + return metadataTypeMapObj; + } + /** + * helper for {@link FilterDefinition.retrieve} + * + * @param {boolean} [hidden] used to filter out hidden or non-hidden filterDefinitions + * @returns {number[]} Array of folder IDs + */ + static async getFilterFolderIds(hidden = false) { + const fromCache = + this.cache.folderFilter || cache.getCache().folder + ? Object.values(this.cache.folderFilter || cache.getCache().folder) + .filter((item) => item.ContentType === 'filterdefinition') + .filter( + (item) => + (!hidden && item.Path.startsWith('Data Filters')) || + (hidden && !item.Path.startsWith('Data Filters')) + ) // only retrieve from Data Filters folder + .map((item) => item.ID) + : []; + if (fromCache.length) { + return fromCache; + } + + const subTypeArr = ['hidden', 'filterdefinition']; + Util.logger.info(` - Caching dependent Metadata: folder`); + Util.logSubtypes(subTypeArr); + + Folder.client = this.client; + Folder.buObject = this.buObject; + Folder.properties = this.properties; + this.cache.folderFilter = (await Folder.retrieveForCache(null, subTypeArr)).metadata; + return this.getFilterFolderIds(hidden); + } + /** + * helper for {@link FilterDefinition.retrieve} + * + * @returns {number[]} Array of folder IDs + */ + static async getMeasureFolderIds() { + const fromCache = + this.cache.folderMeasure?.[this.buObject.mid] || cache.getCache().folder + ? Object.values( + this.cache.folderMeasure?.[this.buObject.mid] || cache.getCache().folder + ) + .filter((item) => item.ContentType === 'measure') + .map((item) => item.ID) + : []; + if (fromCache.length) { + return fromCache; + } + + const subTypeArr = ['measure']; + Util.logger.info(` - Caching dependent Metadata: folder`); + Util.logSubtypes(subTypeArr); + + Folder.client = this.client; + Folder.buObject = this.buObject; + Folder.properties = this.properties; + this.cache.folderMeasure ||= {}; + this.cache.folderMeasure[this.buObject.mid] = ( + await Folder.retrieveForCache(null, subTypeArr) + ).metadata; + return this.getMeasureFolderIds(); + } + + /** + * + * @param {TYPE.MultiMetadataTypeMap} metadataTypeMapObj - + */ + static async cacheDeFields(metadataTypeMapObj) { + const deKeys = Object.values(metadataTypeMapObj.metadata) + .filter((item) => item.derivedFromObjectTypeName === 'DataExtension') + .filter((item) => item.derivedFromObjectId) + .map((item) => { + try { + const deKey = cache.searchForField( + 'dataExtension', + item.derivedFromObjectId, + 'ObjectID', + 'CustomerKey' + ); + if (deKey) { + this.deIdKeyMap ||= {}; + this.deIdKeyMap[item.derivedFromObjectId] = deKey; + return deKey; + } + } catch { + return null; + } + }) + .filter(Boolean); + if (deKeys.length) { + Util.logger.info(' - Caching dependent Metadata: dataExtensionField'); + // only proceed with the download if we have dataExtension keys + const fieldOptions = {}; + for (const deKey of deKeys) { + fieldOptions.filter = fieldOptions.filter + ? { + leftOperand: { + leftOperand: 'DataExtension.CustomerKey', + operator: 'equals', + rightOperand: deKey, + }, + operator: 'OR', + rightOperand: fieldOptions.filter, + } + : { + leftOperand: 'DataExtension.CustomerKey', + operator: 'equals', + rightOperand: deKey, + }; + } + DataExtensionField.buObject = this.buObject; + DataExtensionField.client = this.client; + DataExtensionField.properties = this.properties; + this.dataExtensionFieldCache = ( + await DataExtensionField.retrieveForCache(fieldOptions, ['Name', 'ObjectID']) + ).metadata; + } } + /** + * + * @param {TYPE.MultiMetadataTypeMap} metadataTypeMapObj - + */ + static async cacheContactAttributes(metadataTypeMapObj) { + if (this.cache.contactAttributes?.[this.buObject.mid]) { + return; + } + const subscriberFilters = Object.values(metadataTypeMapObj.metadata) + .filter((item) => item.derivedFromObjectTypeName === 'SubscriberAttributes') + .filter((item) => item.derivedFromObjectId); + if (subscriberFilters.length) { + Util.logger.info(' - Caching dependent Metadata: contactAttributes'); + const response = await this.client.rest.get('/email/v1/Contacts/Attributes/'); + const keyFieldBackup = this.definition.keyField; + this.definition.keyField = 'id'; + this.cache.contactAttributes ||= {}; + this.cache.contactAttributes[this.buObject.mid] = this.parseResponseBody(response); + this.definition.keyField = keyFieldBackup; + } + } + /** + * + * @param {TYPE.MultiMetadataTypeMap} metadataTypeMapObj - + */ + static async cacheMeasures(metadataTypeMapObj) { + if (this.cache.measures?.[this.buObject.mid]) { + return; + } + const subscriberFilters = Object.values(metadataTypeMapObj.metadata) + .filter((item) => item.derivedFromObjectTypeName === 'SubscriberAttributes') + .filter((item) => item.derivedFromObjectId); + const measureFolders = await this.getMeasureFolderIds(); + if (subscriberFilters.length) { + Util.logger.info(' - Caching dependent Metadata: measure'); + const response = { items: [] }; + for (const folderId of measureFolders) { + const metadataMapFolder = await this.client.rest.getBulk( + 'email/v1/Measures/category/' + folderId + '/', + 250 // 250 is what the GUI is using + ); + if (Object.keys(metadataMapFolder.items).length) { + response.items.push(...metadataMapFolder.items); + } + } + + const keyFieldBackup = this.definition.keyField; + this.definition.keyField = 'measureID'; + this.cache.measures ||= {}; + this.cache.measures[this.buObject.mid] = this.parseResponseBody(response); + this.definition.keyField = keyFieldBackup; + } + } + /** * Retrieves all records for caching * @@ -111,57 +263,200 @@ class FilterDefinition extends MetadataType { // type 6 seems to be journey related. Maybe we need to change that again in the future return; } - try { - // folder - this.setFolderPath(metadata); + // folder + this.setFolderPath(metadata); - switch (metadata.derivedFromType) { - case 1: { - // SubscriberAttributes - // TODO - break; + // parse XML filter for further processing in JSON format + const xmlToJson = new XMLParser({ ignoreAttributes: false }); + metadata.c__filterDefinition = xmlToJson.parse( + metadata.filterDefinitionXml + )?.FilterDefinition; + delete metadata.filterDefinitionXml; + + switch (metadata.derivedFromType) { + case 1: { + if (metadata.c__filterDefinition['@_Source'] === 'SubscriberAttribute') { + if ( + metadata.derivedFromObjectId && + metadata.derivedFromObjectId !== '00000000-0000-0000-0000-000000000000' + ) { + // Lists + try { + metadata.r__source_list_PathName = cache.getListPathName( + metadata.derivedFromObjectId, + 'ObjectID' + ); + } catch { + Util.logger.warn( + ` - skipping ${this.definition.type} ${metadata.key}: list ${metadata.derivedFromObjectId} not found on current or Parent BU` + ); + // return; + } + } else { + // SubscriberAttributes + // - nothing to do + } } - case 2: { + + break; + } + case 2: { + // DataExtension + XXX? + if ( + metadata.c__filterDefinition['@_Source'] === 'Meta' || + metadata.derivedFromObjectId === '00000000-0000-0000-0000-000000000000' + ) { + // TODO - weird so far not understood case of Source=Meta + // sample: + } else if (metadata.c__filterDefinition['@_Source'] === 'DataExtension') { // DataExtension - metadata.r__dataExtension_CustomerKey = cache.searchForField( - 'dataExtension', - metadata.derivedFromObjectId, - 'ObjectID', - 'CustomerKey' - ); + try { + metadata.r__source_dataExtension_CustomerKey = + this.deIdKeyMap?.[metadata.derivedFromObjectId] || + cache.searchForField( + 'dataExtension', + metadata.derivedFromObjectId, + 'ObjectID', + 'CustomerKey' + ); + } catch { + Util.logger.debug( + ` - skipping ${this.definition.type} ${metadata.key}: dataExtension ${metadata.derivedFromObjectId} not found on BU` + ); + return; + } + } + break; + } + case 3: { + // TODO + break; + } + case 4: { + // TODO + break; + } + case 5: { + // TODO + break; + } + case 6: { + // TODO + break; + } + } + + // map Condition ID to fields ID + switch (metadata.derivedFromType) { + case 1: { + // SubscriberAttributes + this.resolveAttributeIds(metadata); + delete metadata.derivedFromObjectId; + delete metadata.derivedFromType; + delete metadata.c__filterDefinition['@_Source']; + break; + } + case 2: { + if (metadata.c__filterDefinition['@_Source'] === 'Meta') { + // TODO - weird so far not understood case of Source=Meta + // sample: + } else if (metadata.c__filterDefinition['@_Source'] === 'DataExtension') { + // DataExtension + this.resolveFieldIds(metadata); delete metadata.derivedFromObjectId; delete metadata.derivedFromType; - break; - } - case 3: { - // TODO - break; - } - case 4: { - // TODO - break; - } - case 5: { - // TODO - break; - } - case 6: { - // TODO - break; + delete metadata.c__filterDefinition['@_Source']; + delete metadata.c__filterDefinition['@_SourceID']; } + break; + } + case 3: { + // TODO + break; + } + case 4: { + // TODO + break; } + case 5: { + // TODO + break; + } + case 6: { + // TODO + break; + } + } + return metadata; + } - const xmlToJson = new XMLParser({ ignoreAttributes: false }); - metadata.c__filterDefinition = xmlToJson.parse(metadata.filterDefinitionXml); - // TODO map Condition ID to DataExtensionField ID - delete metadata.filterDefinitionXml; - } catch (ex) { - Util.logger.error( - `FilterDefinition '${metadata.name}' (${metadata.key}): ${ex.message}` + /** + * + * @param {TYPE.FilterDefinitionItem} metadata - + * @param {object[]} [fieldCache] - + * @param {object} [filter] - + * @returns {void} + */ + static resolveFieldIds(metadata, fieldCache, filter) { + if (!filter) { + return this.resolveFieldIds( + metadata, + Object.values(this.dataExtensionFieldCache), + metadata.c__filterDefinition?.ConditionSet ); } - return metadata; + const conditionsArr = Array.isArray(filter.Condition) + ? filter.Condition + : [filter.Condition]; + for (const condition of conditionsArr) { + condition.r__dataExtensionField = fieldCache.find( + (field) => field.ObjectID === condition['@_ID'] + )?.Name; + delete condition['@_ID']; + if (['IsEmpty', 'IsNotEmpty'].includes(condition['@_Operator'])) { + delete condition.Value; + } + } + if (filter.ConditionSet) { + this.resolveFieldIds(metadata, fieldCache, filter.ConditionSet); + } } + /** + * + * @param {TYPE.FilterDefinitionItem} metadata - + * @param {object} [filter] - + * @returns {void} + */ + static resolveAttributeIds(metadata, filter) { + if (!filter) { + return this.resolveAttributeIds(metadata, metadata.c__filterDefinition?.ConditionSet); + } + const contactAttributes = this.cache.contactAttributes[this.buObject.mid]; + const measures = this.cache.measures[this.buObject.mid]; + const conditionsArr = Array.isArray(filter.Condition) + ? filter.Condition + : [filter.Condition]; + for (const condition of conditionsArr) { + condition['@_ID'] += ''; + if (condition['@_SourceType'] === 'Measure' && measures[condition['@_ID']]) { + condition.r__measure = measures[condition['@_ID']]?.name; + delete condition['@_ID']; + } else if ( + condition['@_SourceType'] !== 'Measure' && + contactAttributes[condition['@_ID']] + ) { + condition.r__contactAttribute = contactAttributes[condition['@_ID']]?.name; + delete condition['@_ID']; + } + if (['IsEmpty', 'IsNotEmpty'].includes(condition['@_Operator'])) { + delete condition.Value; + } + } + if (filter.ConditionSet) { + this.resolveAttributeIds(metadata, filter.ConditionSet); + } + } + /** * prepares a item for deployment * diff --git a/lib/metadataTypes/FilterDefinitionHidden.js b/lib/metadataTypes/FilterDefinitionHidden.js new file mode 100644 index 000000000..634fb9a44 --- /dev/null +++ b/lib/metadataTypes/FilterDefinitionHidden.js @@ -0,0 +1,24 @@ +'use strict'; + +// const TYPE = require('../../types/mcdev.d'); +const FilterDefinition = require('./FilterDefinition'); + +/** + * FilterDefinitionHidden MetadataType + * + * @augments FilterDefinitionHidden + */ +class FilterDefinitionHidden extends FilterDefinition { + /** + * helper for {@link FilterDefinition.retrieve} + * + * @returns {number[]} Array of folder IDs + */ + static async getFilterFolderIds() { + return super.getFilterFolderIds(true); + } +} +// Assign definition to static attributes +FilterDefinitionHidden.definition = require('../MetadataTypeDefinitions').filterDefinitionHidden; + +module.exports = FilterDefinitionHidden; diff --git a/lib/metadataTypes/definitions/Filter.definition.js b/lib/metadataTypes/definitions/Filter.definition.js index 9f442bda1..51f50508b 100644 --- a/lib/metadataTypes/definitions/Filter.definition.js +++ b/lib/metadataTypes/definitions/Filter.definition.js @@ -1,8 +1,15 @@ module.exports = { bodyIteratorField: 'items', - dependencies: ['filterDefinition', 'list', 'dataExtension', 'folder'], + dependencies: [ + 'filterDefinition', + 'filterDefinitionHidden', + 'list', + 'dataExtension', + 'folder-filteractivity', + 'folder-hidden', + ], hasExtended: false, - idField: 'id', + idField: 'filterActivityId', keyIsFixed: null, keyField: 'customerKey', nameField: 'name', @@ -56,10 +63,10 @@ module.exports = { template: true, }, filterActivityId: { - isCreateable: null, - isUpdateable: null, - retrieving: true, - template: true, + isCreateable: false, + isUpdateable: true, + retrieving: false, + template: false, }, filterDefinitionId: { isCreateable: true, diff --git a/lib/metadataTypes/definitions/FilterDefinition.definition.js b/lib/metadataTypes/definitions/FilterDefinition.definition.js index 83f51d78a..b8299c046 100644 --- a/lib/metadataTypes/definitions/FilterDefinition.definition.js +++ b/lib/metadataTypes/definitions/FilterDefinition.definition.js @@ -1,5 +1,5 @@ module.exports = { - bodyIteratorField: 'Results', + bodyIteratorField: 'items', dependencies: ['folder-filterdefinition', 'folder-hidden', 'dataExtension'], filter: {}, hasExtended: false, @@ -12,10 +12,10 @@ module.exports = { createdNameField: 'createdBy', lastmodDateField: 'lastUpdated', lastmodNameField: 'lastUpdatedBy', - restPagination: false, + restPagination: true, + restPageSize: 100, type: 'filterDefinition', - typeDescription: - 'Defines an audience based on specified rules. Used by Filter Activities and Filtered DEs.', + typeDescription: 'Defines an audience based on specified rules. Used by Filter Activities.', typeRetrieveByDefault: true, typeName: 'Filter Definition', fields: { @@ -44,12 +44,13 @@ module.exports = { retrieving: false, template: false, }, - // createdByName: { - // isCreateable: false, - // isUpdateable: false, - // retrieving: false, - // template: false, - // }, + createdByName: { + // actual name of user indicated by id in createdBy + isCreateable: false, + isUpdateable: false, + retrieving: false, + template: false, + }, lastUpdated: { isCreateable: false, isUpdateable: false, @@ -62,12 +63,13 @@ module.exports = { retrieving: false, template: false, }, - // lastUpdatedName: { - // isCreateable: false, - // isUpdateable: false, - // retrieving: false, - // template: false, - // }, + lastUpdatedByName: { + // actual name of user indicated by id in lastUpdatedBy + isCreateable: false, + isUpdateable: false, + retrieving: false, + template: false, + }, name: { isCreateable: true, isUpdateable: true, @@ -81,13 +83,12 @@ module.exports = { retrieving: true, template: true, }, - // CategoryId: { - // // used by UPDATE payload - // isCreateable: false, - // isUpdateable: true, - // retrieving: false, - // template: false, - // }, + description: { + isCreateable: true, + isUpdateable: true, + retrieving: true, + template: true, + }, filterDefinitionXml: { isCreateable: true, isUpdateable: true, @@ -126,8 +127,8 @@ module.exports = { // dataExtension name; field only returned by GET-API isCreateable: false, isUpdateable: false, - retrieving: false, - template: false, + retrieving: true, + template: true, }, isSendable: { isCreateable: false, // automatically set during create @@ -135,5 +136,20 @@ module.exports = { retrieving: true, template: true, }, + r__dataExtension_CustomerKey: { + isCreateable: false, + isUpdateable: false, + retrieving: true, + template: true, + }, + c__filterDefinition: { + skipValidation: true, + }, + r__folder_Path: { + isCreateable: false, + isUpdateable: false, + retrieving: true, + template: true, + }, }, }; diff --git a/lib/metadataTypes/definitions/FilterDefinitionHidden.definition.js b/lib/metadataTypes/definitions/FilterDefinitionHidden.definition.js new file mode 100644 index 000000000..60b8a7c70 --- /dev/null +++ b/lib/metadataTypes/definitions/FilterDefinitionHidden.definition.js @@ -0,0 +1,156 @@ +module.exports = { + bodyIteratorField: 'items', + dependencies: ['folder-filterdefinition', 'folder-hidden', 'dataExtension', 'list'], + filter: {}, + hasExtended: false, + idField: 'id', + keyField: 'key', + nameField: 'name', + folderType: 'filterdefinition', + folderIdField: 'categoryId', + createdDateField: 'createdDate', + createdNameField: 'createdBy', + lastmodDateField: 'lastUpdated', + lastmodNameField: 'lastUpdatedBy', + restPagination: true, + restPageSize: 100, + type: 'filterDefinitionHidden', + typeDescription: + 'Defines an audience based on specified rules. Used by filtered DEs and filtered Lists.', + typeRetrieveByDefault: false, + typeName: 'Filter Definition for filtered Lists && Data Extensions', + fields: { + // the GUI seems to ONLY send fields during update that are actually changed. It has yet to be tested if that also works with sending other fiedls as well + id: { + isCreateable: false, + isUpdateable: false, // included in URL + retrieving: false, + template: false, + }, + key: { + isCreateable: true, + isUpdateable: true, + retrieving: true, + template: true, + }, + createdDate: { + isCreateable: false, + isUpdateable: false, + retrieving: false, + template: false, + }, + createdBy: { + isCreateable: false, + isUpdateable: false, + retrieving: false, + template: false, + }, + createdByName: { + // actual name of user indicated by id in createdBy + isCreateable: false, + isUpdateable: false, + retrieving: false, + template: false, + }, + lastUpdated: { + isCreateable: false, + isUpdateable: false, + retrieving: false, + template: false, + }, + lastUpdatedBy: { + isCreateable: false, + isUpdateable: false, + retrieving: false, + template: false, + }, + lastUpdatedByName: { + // actual name of user indicated by id in lastUpdatedBy + isCreateable: false, + isUpdateable: false, + retrieving: false, + template: false, + }, + name: { + isCreateable: true, + isUpdateable: true, + retrieving: true, + template: true, + }, + categoryId: { + // returned by GET / CREATE / UPDATE; used in CREATE payload + isCreateable: true, + isUpdateable: true, + retrieving: true, + template: true, + }, + description: { + isCreateable: true, + isUpdateable: true, + retrieving: true, + template: true, + }, + filterDefinitionXml: { + isCreateable: true, + isUpdateable: true, + retrieving: true, + template: true, + }, + // DerivedFromType: { + // // this upper-cased spelling is used by GUI when creating a dataExtension based filterDefintion + // isCreateable: true, + // isUpdateable: false, // cannot be updated + // retrieving: false, + // template: false, + // }, + derivedFromType: { + // 1: SubscriberAttributes, 2: DataExtension, 6: EntryCriteria; + isCreateable: true, + isUpdateable: false, // cannot be updated + retrieving: true, + template: true, + }, + derivedFromObjectId: { + // dataExtension ID or '00000000-0000-0000-0000-000000000000' for lists + isCreateable: true, + isUpdateable: false, // cannot be updated + retrieving: true, + template: true, + }, + derivedFromObjectTypeName: { + // "SubscriberAttributes" | "DataExtension" | "EntryCriteria" ...; only returned by GET API + isCreateable: false, + isUpdateable: false, + retrieving: true, + template: true, + }, + derivedFromObjectName: { + // dataExtension name; field only returned by GET-API + isCreateable: false, + isUpdateable: false, + retrieving: true, + template: true, + }, + isSendable: { + isCreateable: false, // automatically set during create + isUpdateable: false, + retrieving: true, + template: true, + }, + r__dataExtension_CustomerKey: { + isCreateable: false, + isUpdateable: false, + retrieving: true, + template: true, + }, + c__filterDefinition: { + skipValidation: true, + }, + r__folder_Path: { + isCreateable: false, + isUpdateable: false, + retrieving: true, + template: true, + }, + }, +};