diff --git a/lib/MetadataTypeDefinitions.js b/lib/MetadataTypeDefinitions.js index d6b931e75..01a08aeb4 100644 --- a/lib/MetadataTypeDefinitions.js +++ b/lib/MetadataTypeDefinitions.js @@ -20,6 +20,7 @@ const MetadataTypeDefinitions = { eventDefinition: require('./metadataTypes/definitions/EventDefinition.definition'), fileTransfer: require('./metadataTypes/definitions/FileTransfer.definition'), filter: require('./metadataTypes/definitions/Filter.definition'), + filterDefinition: require('./metadataTypes/definitions/FilterDefinition.definition'), folder: require('./metadataTypes/definitions/Folder.definition'), ftpLocation: require('./metadataTypes/definitions/FtpLocation.definition'), importFile: require('./metadataTypes/definitions/ImportFile.definition'), diff --git a/lib/MetadataTypeInfo.js b/lib/MetadataTypeInfo.js index 594af4d4e..8e0d19ed2 100644 --- a/lib/MetadataTypeInfo.js +++ b/lib/MetadataTypeInfo.js @@ -20,6 +20,7 @@ const MetadataTypeInfo = { eventDefinition: require('./metadataTypes/EventDefinition'), fileTransfer: require('./metadataTypes/FileTransfer'), filter: require('./metadataTypes/Filter'), + filterDefinition: require('./metadataTypes/FilterDefinition'), folder: require('./metadataTypes/Folder'), ftpLocation: require('./metadataTypes/FtpLocation'), importFile: require('./metadataTypes/ImportFile'), diff --git a/lib/metadataTypes/Filter.js b/lib/metadataTypes/Filter.js index 658fa493d..b45000ff4 100644 --- a/lib/metadataTypes/Filter.js +++ b/lib/metadataTypes/Filter.js @@ -1,6 +1,32 @@ 'use strict'; +/** + * @typedef {Object} FilterItem + * @property {number} categoryId folder id + * @property {string} [createdDate] - + * @property {string} customerKey key + * @property {string} destinationObjectId DE/List ID + * @property {1|2|3|4} destinationTypeId 1:SubscriberList, 2:DataExtension, 3:GroupWizard, 4:BehavioralData + * @property {string} filterActivityId ? + * @property {string} filterDefinitionId ObjectID of filterDefinition + * @property {string} modifiedDate - + * @property {string} name name + * @property {string} sourceObjectId DE/List ID + * @property {1|2|3|4} sourceTypeId 1:SubscriberList, 2:DataExtension, 3:GroupWizard, 4:BehavioralData + * @property {number} statusId ? + * + * @typedef {Object.} FilterMap + */ + const MetadataType = require('./MetadataType'); +const Util = require('../util/util'); + +const dataTypes = { + 1: 'List', + 2: 'DataExtension', + 3: 'Group Wizard', + 4: 'Behavioral Data', +}; /** * Filter MetadataType @@ -13,11 +39,176 @@ class Filter extends MetadataType { * but only with some of the fields. So it is needed to loop over * Filters with the endpoint /automation/v1/filters/{id} * @param {String} retrieveDir Directory where retrieved metadata directory will be saved - * @returns {Promise} Promise + * @returns {Promise<{metadata:FilterMap,type:string}>} Promise of items */ static async retrieve(retrieveDir) { return super.retrieveREST(retrieveDir, '/automation/v1/filters/', null); } + /** + * manages post retrieve steps + * @param {FilterItem} item a single record + * @returns {FilterItem} parsed metadata definition + */ + static postRetrieveTasks(item) { + return this.parseMetadata(item); + } + /** + * parses retrieved Metadata before saving + * @param {FilterItem} metadata a single record + * @returns {FilterItem} parsed metadata definition + */ + static parseMetadata(metadata) { + try { + // folder + metadata.r__folder_Path = Util.getFromCache( + this.cache, + 'folder', + metadata.categoryId, + 'ID', + 'Path' + ); + delete metadata.categoryId; + + // filterDefinition + metadata.r__filterDefinition_CustomerKey = Util.getFromCache( + this.cache, + 'filterDefinition', + metadata.filterDefinitionId, + 'id', + 'key' + ); + delete metadata.filterDefinitionId; + + // source + if (metadata.sourceTypeId === 1) { + // list + } else if (metadata.sourceTypeId === 2) { + // dataExtension + metadata.r__source_dataExtension_CustomerKey = Util.getFromCache( + this.cache, + 'dataExtension', + metadata.sourceObjectId, + 'ObjectID', + 'CustomerKey' + ); + delete metadata.sourceObjectId; + delete metadata.sourceTypeId; + } else { + Util.logger.error( + `Filter '${metadata.name}' (${metadata.customerKey}): Unsupported source type ${ + metadata.sourceTypeId + }=${dataTypes[metadata.sourceTypeId]}` + ); + } + + // target + if (metadata.destinationTypeId === 1) { + // list + } else if (metadata.destinationTypeId === 2) { + // dataExtension + metadata.r__destination_dataExtension_CustomerKey = Util.getFromCache( + this.cache, + 'dataExtension', + metadata.destinationObjectId, + 'ObjectID', + 'CustomerKey' + ); + delete metadata.destinationObjectId; + delete metadata.destinationTypeId; + } else { + Util.logger.error( + `Filter '${metadata.name}' (${ + metadata.customerKey + }): Unsupported destination type ${metadata.destinationTypeId}=${ + dataTypes[metadata.destinationTypeId] + }` + ); + } + } catch (ex) { + Util.logger.error(`Filter '${metadata.name}' (${metadata.customerKey}): ${ex.message}`); + } + return metadata; + } + /** + * prepares a record for deployment + * @param {FilterItem} metadata a single record + * @returns {Promise} Promise of updated single record + */ + static async preDeployTasks(metadata) { + // folder + if (metadata.r__folder_Path) { + metadata.categoryId = Util.getFromCache( + this.cache, + 'folder', + metadata.r__folder_Path, + 'Path', + 'ID' + ); + delete metadata.r__folder_Path; + } + + // filterDefinition + if (metadata.r__filterDefinition_CustomerKey) { + metadata.filterDefinitionId = Util.getFromCache( + this.cache, + 'filterDefinition', + metadata.r__filterDefinition_CustomerKey, + 'CustomerKey', + 'ObjectID' + ); + delete metadata.r__filterDefinition_CustomerKey; + } + + // source + if (metadata.sourceTypeId === 1) { + // list + } else if (metadata.r__source_dataExtension_CustomerKey) { + // dataExtension + metadata.sourceObjectId = Util.getFromCache( + this.cache, + 'dataExtension', + metadata.r__source_dataExtension_CustomerKey, + 'CustomerKey', + 'ObjectID' + ); + metadata.sourceTypeId = 2; + delete metadata.r__source_dataExtension_CustomerKey; + } else { + // assume the type id is still in the metadata + throw new Error( + `Filter '${metadata.name}' (${metadata.customerKey}): Unsupported source type ${ + metadata.sourceTypeId + }=${dataTypes[metadata.sourceTypeId]}` + ); + } + + // target + if (metadata.destinationTypeId === 1) { + // list + } else if (metadata.r__destination_dataExtension_CustomerKey) { + // dataExtension + metadata.destinationObjectId = Util.getFromCache( + this.cache, + 'dataExtension', + metadata.r__destination_dataExtension_CustomerKey, + 'CustomerKey', + 'ObjectID' + ); + metadata.destinationTypeId = 2; + delete metadata.r__destination_dataExtension_CustomerKey; + } else { + // assume the type id is still in the metadata + throw new Error( + `Filter '${metadata.name}' (${ + metadata.customerKey + }): Unsupported destination type ${metadata.destinationTypeId}=${ + dataTypes[metadata.destinationTypeId] + }` + ); + } + + return metadata; + } } // Assign definition to static attributes diff --git a/lib/metadataTypes/FilterDefinition.js b/lib/metadataTypes/FilterDefinition.js new file mode 100644 index 000000000..2eeadd855 --- /dev/null +++ b/lib/metadataTypes/FilterDefinition.js @@ -0,0 +1,279 @@ +'use strict'; + +/** + * @typedef {Object} FilterDefinitionSOAPItem + * @property {string} ObjectID id + * @property {string} CustomerKey key + * @property {Object} [DataFilter] most relevant part that defines the filter + * @property {Object} DataFilter.LeftOperand - + * @property {string} DataFilter.LeftOperand.Property - + * @property {string} DataFilter.LeftOperand.SimpleOperator - + * @property {string} DataFilter.LeftOperand.Value - + * @property {string} DataFilter.LogicalOperator - + * @property {Object} [DataFilter.RightOperand] - + * @property {string} DataFilter.RightOperand.Property - + * @property {string} DataFilter.RightOperand.SimpleOperator - + * @property {string} DataFilter.RightOperand.Value - + * @property {string} Name name + * @property {string} Description - + * @property {string} [ObjectState] returned from SOAP API; used to return error messages + * + * @typedef {Object.} FilterDefinitionSOAPItemMap + + * + * /automation/v1/filterdefinitions/ (not used) + * @typedef {Object} AutomationFilterDefinitionItem + * @property {string} id object id + * @property {string} key external key + * @property {string} createdDate - + * @property {number} createdBy user id + * @property {string} createdName - + * @property {string} [description] (omitted by API if empty) + * @property {string} modifiedDate - + * @property {number} modifiedBy user id + * @property {string} modifiedName - + * @property {string} name name + * @property {string} categoryId folder id + * @property {string} filterDefinitionXml from REST API defines the filter in XML form + * @property {1|2} derivedFromType 1:list/profile attributes/measures, 2: dataExtension + * @property {boolean} isSendable ? + * @property {Object} [soap__DataFilter] copied from SOAP API, defines the filter in readable form + * @property {Object} soap__DataFilter.LeftOperand - + * @property {string} soap__DataFilter.LeftOperand.Property - + * @property {string} soap__DataFilter.LeftOperand.SimpleOperator - + * @property {string} soap__DataFilter.LeftOperand.Value - + * @property {string} soap__DataFilter.LogicalOperator - + * @property {Object} [soap__DataFilter.RightOperand] - + * @property {string} soap__DataFilter.RightOperand.Property - + * @property {string} soap__DataFilter.RightOperand.SimpleOperator - + * @property {string} soap__DataFilter.RightOperand.Value - + * + * /email/v1/filters/filterdefinition/ + * @typedef {Object} FilterDefinitionItem + * @property {string} id object id + * @property {string} key external key + * @property {string} createdDate date + * @property {number} createdBy user id + * @property {string} createdName name + * @property {string} [description] (omitted by API if empty) + * @property {string} lastUpdated date + * @property {number} lastUpdatedBy user id + * @property {string} lastUpdatedName name + * @property {string} name name + * @property {string} categoryId folder id + * @property {string} filterDefinitionXml from REST API defines the filter in XML form + * @property {1|2} derivedFromType 1:list/profile attributes/measures, 2: dataExtension + * @property {string} derivedFromObjectId Id of DataExtension - present if derivedFromType=2 + * @property {'DataExtension'|'SubscriberAttributes'} derivedFromObjectTypeName - + * @property {string} [derivedFromObjectName] name of DataExtension + * @property {boolean} isSendable ? + * @property {Object} [soap__DataFilter] copied from SOAP API, defines the filter in readable form + * @property {Object} soap__DataFilter.LeftOperand - + * @property {string} soap__DataFilter.LeftOperand.Property - + * @property {string} soap__DataFilter.LeftOperand.SimpleOperator - + * @property {string} soap__DataFilter.LeftOperand.Value - + * @property {string} soap__DataFilter.LogicalOperator - + * @property {Object} [soap__DataFilter.RightOperand] - + * @property {string} soap__DataFilter.RightOperand.Property - + * @property {string} soap__DataFilter.RightOperand.SimpleOperator - + * @property {string} soap__DataFilter.RightOperand.Value - + + * + * @typedef {Object.} FilterDefinitionMap + */ + +const MetadataType = require('./MetadataType'); +const Util = require('../util/util'); +const xml2js = require('xml2js'); + +/** + * FilterDefinition MetadataType + * @augments MetadataType + */ +class FilterDefinition extends MetadataType { + /** + * Retrieves all records and saves it to disk + * @param {string} retrieveDir Directory where retrieved metadata directory will be saved + * @returns {Promise<{metadata:FilterDefinitionMap,type:string}>} Promise of items + */ + static async retrieve(retrieveDir) { + // #1 get the list via SOAP cause the corresponding REST call has no BU filter apparently + // for reference the rest path: '/automation/v1/filterdefinitions?view=categoryinfo' + const keyFieldBak = this.definition.keyField; + const soapFields = ['DataFilter', 'ObjectID', 'CustomerKey', 'Description', 'Name']; + this.definition.keyField = 'CustomerKey'; + /** + * @type {FilterDefinitionSOAPItemMap[]} + */ + const responseObject = await this.retrieveSOAPBody(soapFields); + this.definition.keyField = keyFieldBak; + + // convert back to array + /** + * @type {FilterDefinitionSOAPItem[]} + */ + const listResponse = Object.keys(responseObject) + .map((key) => responseObject[key]) + .filter((item) => { + if (item.ObjectState) { + Util.logger.debug( + `Filtered filterDefinition ${item.name}: ${item.ObjectState}` + ); + return false; + } else { + return true; + } + }); + + // #2 + // /automation/v1/filterdefinitions/ + const response = ( + await Promise.all( + listResponse.map((item) => + this.client.RestClient.get({ + uri: '/email/v1/filters/filterdefinition/' + item.ObjectID, + }) + ) + ) + ) + .map((item) => item.body) + .map((item) => { + // description is not returned when empty + item.description = item.description || ''; + // add extra info from XML + item.c__soap_DataFilter = responseObject[item.key].DataFilter; + return item; + }); + const results = this.parseResponseBody({ Results: response }); + if (retrieveDir) { + const savedMetadata = await this.saveResults(results, retrieveDir, null, null); + Util.logger.info( + `Downloaded: ${this.definition.type} (${Object.keys(savedMetadata).length})` + ); + } + + return { metadata: results, type: this.definition.type }; + + // return super.retrieveSOAPgeneric(retrieveDir); + } + /** + * Retrieves all records for caching + * @returns {Promise<{metadata:FilterDefinitionMap,type:string}>} Promise of items + */ + static async retrieveForCache() { + return this.retrieve(null); + } + + /** + * manages post retrieve steps + * @param {FilterDefinitionItem} item a single record + * @returns {FilterDefinitionItem} parsed metadata definition + */ + static async postRetrieveTasks(item) { + return this.parseMetadata(item); + } + /** + * parses retrieved Metadata before saving + * @param {FilterDefinitionItem} metadata a single record + * @returns {FilterDefinitionItem} parsed metadata definition + */ + static async parseMetadata(metadata) { + try { + // folder + metadata.r__folder_Path = Util.getFromCache( + this.cache, + 'folder', + metadata.categoryId, + 'ID', + 'Path' + ); + delete metadata.categoryId; + + if (metadata.derivedFromType === 2) { + // DataExtension + metadata.r__dataExtension_CustomerKey = Util.getFromCache( + this.cache, + 'dataExtension', + metadata.derivedFromObjectId, + 'ObjectID', + 'CustomerKey' + ); + } + delete metadata.derivedFromObjectId; + delete metadata.derivedFromType; + metadata.c__filterDefinition = await xml2js.parseStringPromise( + metadata.filterDefinitionXml /* , options */ + ); + + // TODO check if Condition ID needs to be resolved or can be ignored + } catch (ex) { + Util.logger.error( + `FilterDefinition '${metadata.name}' (${metadata.key}): ${ex.message}` + ); + } + return metadata; + } + /** + * prepares a item for deployment + * @param {FilterDefinitionItem} metadata a single record + * @returns {Promise} Promise of updated single item + */ + static async preDeployTasks(metadata) { + // folder + metadata.categoryId = Util.getFromCache( + this.cache, + 'folder', + metadata.r__folder_Path, + 'Path', + 'ID' + ); + delete metadata.r__folder_Path; + + if (metadata.derivedFromObjectTypeName === 'SubscriberAttributes') { + // List + metadata.derivedFromType = 1; + metadata.derivedFromObjectId = '00000000-0000-0000-0000-000000000000'; + } else { + // DataExtension + metadata.derivedFromType = 2; + + if (metadata.r__dataExtension_CustomerKey) { + metadata.derivedFromObjectId = Util.getFromCache( + this.cache, + 'dataExtension', + metadata.r__dataExtension_CustomerKey, + 'CustomerKey', + 'ObjectID' + ); + delete metadata.r__dataExtension_CustomerKey; + } + } + delete metadata.c__filterDefinition; + delete metadata.c__soap_DataFilter; + + return metadata; + } + /** + * Creates a single item + * @param {FilterDefinitionItem} metadata a single item + * @returns {Promise} Promise + */ + static create(metadata) { + // TODO test the create + return super.createREST(metadata, '/email/v1/filters/filterdefinition/'); + } + /** + * Updates a single item + * @param {FilterDefinitionItem} metadata a single item + * @returns {Promise} Promise + */ + static update(metadata) { + // TODO test the update + // TODO figure out how to get the ID on the fly + return super.updateREST(metadata, '/email/v1/filters/filterdefinition/' + metadata.Id); + } +} +// Assign definition to static attributes +FilterDefinition.definition = require('../MetadataTypeDefinitions').filterDefinition; + +module.exports = FilterDefinition; diff --git a/lib/metadataTypes/definitions/Filter.definition.js b/lib/metadataTypes/definitions/Filter.definition.js index b9503571f..8f1686f6c 100644 --- a/lib/metadataTypes/definitions/Filter.definition.js +++ b/lib/metadataTypes/definitions/Filter.definition.js @@ -1,6 +1,6 @@ module.exports = { bodyIteratorField: 'items', - dependencies: [], + dependencies: ['filterDefinition', 'list', 'dataExtension', 'folder'], hasExtended: false, idField: 'id', keyField: 'customerKey', @@ -8,56 +8,58 @@ module.exports = { restPagination: true, type: 'filter', typeDescription: - 'BETA: Part of how filtered Data Extensions are created. Depends on type "FilterDefinitions".', - typeRetrieveByDefault: false, + 'Used in automations to filter lists and DEs. Depends on type "FilterDefinitions".', + typeRetrieveByDefault: true, typeName: 'Automation: Filter Activity', fields: { + // https://developer.salesforce.com/docs/atlas.en-us.noversion.mc-apis.meta/mc-apis/filteractivity.htm categoryId: { - isCreateable: false, - isUpdateable: false, + isCreateable: true, + isUpdateable: true, retrieving: true, - template: false, + template: true, }, createdDate: { isCreateable: false, isUpdateable: false, - retrieving: true, + retrieving: false, template: false, }, customerKey: { - isCreateable: null, - isUpdateable: false, + isCreateable: true, + isUpdateable: true, retrieving: true, - template: false, + template: true, }, description: { - isCreateable: false, - isUpdateable: false, + isCreateable: true, + isUpdateable: true, retrieving: true, - template: false, + template: true, }, destinationObjectId: { - isCreateable: false, - isUpdateable: false, + isCreateable: true, + isUpdateable: true, retrieving: true, - template: false, + template: true, }, destinationTypeId: { - isCreateable: false, - isUpdateable: false, + isCreateable: true, + isUpdateable: true, retrieving: true, - template: false, + template: true, }, filterActivityId: { isCreateable: null, isUpdateable: null, retrieving: true, + template: true, }, filterDefinitionId: { - isCreateable: false, - isUpdateable: false, + isCreateable: true, + isUpdateable: true, retrieving: true, - template: false, + template: true, }, modifiedDate: { isCreateable: false, @@ -66,28 +68,28 @@ module.exports = { template: false, }, name: { - isCreateable: null, - isUpdateable: false, + isCreateable: true, + isUpdateable: true, retrieving: true, - template: false, + template: true, }, sourceObjectId: { - isCreateable: false, - isUpdateable: false, + isCreateable: true, + isUpdateable: true, retrieving: true, - template: false, + template: true, }, sourceTypeId: { - isCreateable: false, - isUpdateable: false, + isCreateable: true, + isUpdateable: true, retrieving: true, - template: false, + template: true, }, statusId: { - isCreateable: false, - isUpdateable: false, + isCreateable: true, + isUpdateable: true, retrieving: true, - template: false, + template: true, }, }, }; diff --git a/lib/metadataTypes/definitions/FilterDefinition.definition.js b/lib/metadataTypes/definitions/FilterDefinition.definition.js new file mode 100644 index 000000000..77f1c62b7 --- /dev/null +++ b/lib/metadataTypes/definitions/FilterDefinition.definition.js @@ -0,0 +1,113 @@ +module.exports = { + bodyIteratorField: 'Results', + dependencies: ['folder', 'dataExtension'], + filter: {}, + hasExtended: false, + idField: 'id', + keyField: 'key', + nameField: 'name', + restPagination: false, + type: 'filterDefinition', + typeDescription: + 'Defines an audience based on specified rules. Used by Filter Activities and Filtered DEs.', + typeRetrieveByDefault: true, + typeName: 'Filter Definition', + fields: { + id: { + isCreateable: false, + isUpdateable: false, + retrieving: false, + template: false, + }, + key: { + isCreateable: true, + isUpdateable: true, + retrieving: true, + template: true, + }, + createdDate: { + isCreateable: false, + isUpdateable: false, + retrieving: false, + template: false, + }, + createdBy: { + isCreateable: false, + isUpdateable: false, + retrieving: false, + template: false, + }, + createdByName: { + isCreateable: false, + isUpdateable: false, + retrieving: false, + template: false, + }, + lastUpdated: { + isCreateable: false, + isUpdateable: false, + retrieving: true, + template: false, + }, + lastUpdatedBy: { + isCreateable: false, + isUpdateable: false, + retrieving: false, + template: false, + }, + lastUpdatedName: { + isCreateable: false, + isUpdateable: false, + retrieving: false, + template: false, + }, + name: { + isCreateable: true, + isUpdateable: true, + retrieving: true, + template: true, + }, + categoryId: { + isCreateable: true, + isUpdateable: true, + retrieving: true, + template: true, + }, + filterDefinitionXml: { + isCreateable: true, + isUpdateable: true, + retrieving: true, + template: true, + }, + derivedFromType: { + isCreateable: true, + isUpdateable: true, + retrieving: true, + template: true, + }, + derivedFromObjectId: { + isCreateable: true, + isUpdateable: true, + retrieving: true, + template: true, + }, + derivedFromObjectTypeName: { + isCreateable: true, + isUpdateable: true, + retrieving: true, + template: true, + }, + derivedFromObjectName: { + isCreateable: false, + isUpdateable: false, + retrieving: true, + template: true, + }, + isSendable: { + isCreateable: true, + isUpdateable: true, + retrieving: true, + template: true, + }, + }, +};