Merge pull request #1303 from Accenture/feature/1302-automatically-ad…

…d-suffix-to-key

Feature/1302 + Bugfix/1298 automatically add suffix to key

lib/cli.js

@@ -87,37 +87,50 @@ yargs(hideBin(process.argv))
  describe:
  'type or type:key or type:i:id or type:n:name to deploy; if not provided, all metadata will be deploy',
  })
+ .option('keySuffix', {
+ type: 'string',
+ alias: 'ks',
+ group: 'Options for deploy:',
+ describe:
+ 'allows you to add a suffix to the key of the metadata to be deployed. Only works together with changeKeyField',
+ })
  .option('changeKeyField', {
  type: 'string',
+ alias: 'ckf',
  group: 'Options for deploy:',
  describe:
  'enables updating the key of the deployed metadata with the value in provided field (e.g. c__newKey). Can be used to sync name and key fields.',
  })
  .option('changeKeyValue', {
  type: 'string',
+ alias: 'ckv',
  group: 'Options for deploy:',
  describe:
  'allows updating the key of the metadata to the provided value. Only available if a single type and key is deployed',
  })
  .option('fromRetrieve', {
  type: 'boolean',
+ alias: 'fr',
  group: 'Options for deploy:',
  describe: 'optionally deploy from retrieve folder',
  })
  .option('refresh', {
  type: 'boolean',
+ alias: 'r',
  group: 'Options for deploy:',
  describe:
  'optional for asset-message: runs refresh command for related triggeredSends after deploy',
  })
  .option('execute', {
  type: 'boolean',
+ alias: 'e',
  group: 'Options for deploy:',
  describe:
  'optional: executes item after deploy; this will run the item once immediately',
  })
  .option('schedule', {
  type: 'boolean',
+ alias: 's',
  group: 'Options for deploy:',
  describe:
  'optionally start existing schedule instead of running item once immediately (only works for automations)',
@@ -127,7 +140,6 @@ yargs(hideBin(process.argv))
  describe:
  "optionally ensure that updates to shared DataExtensions become visible in child BU's data designer (SF Known issue W-11031095)",
  });
- // TODO: add option --metadata
  },
  handler: (argv) => {
  Mcdev.setOptions(argv);
@@ -644,14 +656,23 @@ yargs(hideBin(process.argv))
  describe:
  'filter metadata components (can include % as wildcard or _ for a single character)',
  })
+ .option('keySuffix', {
+ type: 'string',
+ alias: 'ks',
+ group: 'Options for fixKeys:',
+ describe:
+ 'allows you to add a suffix to the key of the metadata to be deployed.',
+ })
  .option('execute', {
  type: 'boolean',
+ alias: 'e',
  group: 'Options for fixKeys:',
  describe:
  'optional: executes item after deploy; this will run the item once immediately',
  })
  .option('schedule', {
  type: 'boolean',
+ alias: 's',
  group: 'Options for fixKeys:',
  describe:
  'optionally start existing schedule instead of running item once immediately (only works for automations)',

lib/index.js

@@ -77,6 +77,7 @@ class Mcdev {
  static setOptions(argv) {
  const knownOptions = [
  'api',
+ 'keySuffix',
  'changeKeyField',
  'changeKeyValue',
  'commitHistory',

lib/metadataTypes/MetadataType.js

@@ -738,12 +738,17 @@ class MetadataType {
  ` - --changeKeyField is set to the same value as the keyField for ${this.definition.type}. Skipping change.`
  );
  } else if (metadataMap[metadataKey][Util.OPTIONS.changeKeyField]) {
- // NOTE: trim twice while getting the newKey value to remove leading spaces before limiting the length
- const newKey = (metadataMap[metadataKey][Util.OPTIONS.changeKeyField] + '')
- .trim()
- .slice(0, maxKeyLength)
- .trim();
- if (metadataMap[metadataKey][Util.OPTIONS.changeKeyField] + '' > maxKeyLength) {
+ Util.OPTIONS.keySuffix = Util.OPTIONS.keySuffix
+ ? Util.OPTIONS.keySuffix.trim()
+ : '';
+ const newKey = this.getNewKey(metadataMap[metadataKey], maxKeyLength);
+
+ if (
+ metadataMap[metadataKey][Util.OPTIONS.changeKeyField] +
+ '' +
+ Util.OPTIONS.keySuffix >
+ maxKeyLength
+ ) {
  Util.logger.warn(
  `${this.definition.type} ${this.definition.keyField} may not exceed ${maxKeyLength} characters. Truncated the value in field ${Util.OPTIONS.changeKeyField} to ${newKey}`
  );
@@ -2166,30 +2171,31 @@ class MetadataType {
  Util.logger.info(
  `Searching for ${this.definition.type} keys among downloaded items that need fixing:`
  );
+ Util.OPTIONS.keySuffix = Util.OPTIONS.keySuffix ? Util.OPTIONS.keySuffix.trim() : '';
+ const maxKeyLength = this.definition.maxKeyLength || 36;
+
  for (const item of Object.values(metadataMap)) {
- if (item[this.definition.nameField].length > this.definition.maxKeyLength) {
+ if (
+ (item[this.definition.nameField].endsWith(Util.OPTIONS.keySuffix) &&
+ item[this.definition.nameField].length > maxKeyLength) ||
+ (!item[this.definition.nameField].endsWith(Util.OPTIONS.keySuffix) &&
+ item[this.definition.nameField].length + Util.OPTIONS.keySuffix.length >
+ maxKeyLength)
+ ) {
  Util.logger.warn(
- `Name of the item ${
- item[this.definition.keyField]
- } is too long for a key. Consider renaming your item. Key will be equal first ${
- this.definition.maxKeyLength
- } characters of the name`
- );
- item[this.definition.nameField] = item[this.definition.nameField].slice(
- 0,
- this.definition.maxKeyLength
+ `Name of the item ${item[this.definition.keyField]} (${
+ item[this.definition.nameField]
+ }) is too long for a key${Util.OPTIONS.keySuffix.length ? ' (including the suffix ' + Util.OPTIONS.keySuffix + ')' : ''}. Consider renaming your item. Key will be equal first ${maxKeyLength} characters of the name`
  );
  }
-
- if (
- item[this.definition.nameField] != item[this.definition.keyField] &&
- !this.definition.keyIsFixed
- ) {
- keysForDeploy.push(item[this.definition.keyField]);
+ const newKey = this.getNewKey(item, maxKeyLength);
+ if (newKey != item[this.definition.keyField] && !this.definition.keyIsFixed) {
+ // add key but make sure to turn it into string or else numeric keys will be filtered later
+ keysForDeploy.push(item[this.definition.keyField] + '');
  Util.logger.info(
  ` - added ${this.definition.type} to fixKey queue: ${
  item[this.definition.keyField]
- }`
+ } >> ${newKey}`
  );
  } else {
  Util.logger.info(
@@ -2205,6 +2211,29 @@ class MetadataType {
  }
  return keysForDeploy;
  }
+ /**
+ * helper for getKeysForFixing and createOrUpdate
+ *
+ * @param {MetadataTypeItem} metadataItem -
+ * @param {number} maxKeyLength -
+ * @returns {string} newKey
+ */
+ static getNewKey(metadataItem, maxKeyLength) {
+ let newKey;
+ newKey = (metadataItem[this.definition.nameField] + '')
+ .trim()
+ .slice(0, maxKeyLength)
+ .trim();
+ if (Util.OPTIONS.keySuffix.length && !newKey.endsWith(Util.OPTIONS.keySuffix)) {
+ newKey =
+ (metadataItem[this.definition.nameField] + '')
+ .trim()
+ .slice(0, maxKeyLength - Util.OPTIONS.keySuffix.length)
+ .trim() + Util.OPTIONS.keySuffix;
+ }
+
+ return newKey;
+ }
 }
 
 MetadataType.definition = {

lib/metadataTypes/definitions/DeliveryProfile.definition.js

@@ -5,6 +5,8 @@ export default {
  hasExtended: false,
  idField: 'id',
  keyField: 'key',
+ keyIsFixed: false,
+ maxKeyLength: 36, // confirmed max length
  nameField: 'name',
  createdDateField: 'createdDate',
  createdNameField: null,

lib/metadataTypes/definitions/SendClassification.definition.js

@@ -5,8 +5,9 @@ export default {
  filter: {},
  hasExtended: false,
  idField: 'ObjectID',
- keyIsFixed: null,
  keyField: 'CustomerKey',
+ keyIsFixed: false,
+ maxKeyLength: 36, // confirmed max length
  nameField: 'Name',
  createdDateField: 'CreatedDate',
  createdNameField: null,

lib/metadataTypes/definitions/SenderProfile.definition.js

@@ -4,8 +4,9 @@ export default {
  filter: {},
  hasExtended: false,
  idField: 'ObjectID',
- keyIsFixed: false,
  keyField: 'CustomerKey',
+ keyIsFixed: false,
+ maxKeyLength: 36, // confirmed max length
  nameField: 'Name',
  createdDateField: 'CreatedDate',
  createdNameField: null,

test/resources/9999999/automation/v1/queries/549f0568-607c-4940-afef-437965094dat_fixKeysSuffix/get-response.json

@@ -0,0 +1,17 @@
+{
+ "queryDefinitionId": "549f0568-607c-4940-afef-437965094dat_fixKeysSuffix",
+ "name": "testExisting_query_fixedKeys",
+ "key": "testExisting_query_fixKeysSuffix",
+ "description": "bla bla",
+ "queryText": "Select\n SubscriberKey as testField, Trim(last_name) AS name\nfrom\n _Subscribers\nwhere\n country in ('test')\n",
+ "targetName": "testExisting_dataExtension",
+ "targetKey": "testExisting_dataExtension",
+ "targetId": "21711373-72c1-ec11-b83b-48df37d1deb7",
+ "targetDescription": "",
+ "createdDate": "2022-04-26T15:21:16.453",
+ "modifiedDate": "2022-04-26T16:02:44.01",
+ "targetUpdateTypeId": 0,
+ "targetUpdateTypeName": "Overwrite",
+ "categoryId": 999,
+ "isFrozen": false
+}

test/resources/9999999/automation/v1/queries/549f0568-607c-4940-afef-437965094dat_fixKeysSuffix/patch-response.json

@@ -0,0 +1,18 @@
+{
+ "queryDefinitionId": "549f0568-607c-4940-afef-437965094dat_fixKeys",
+ "name": "testExisting_query_fixedKeys",
+ "key": "testExisting_query_fixedKeys_DEV",
+ "description": "updated on deploy",
+ "queryText": "SELECT\n SubscriberKey as testField\nFROM\n _Subscribers\nWHERE\n country IN ('test')\n",
+ "targetName": "testExisting_dataExtension",
+ "targetKey": "testExisting_dataExtension",
+ "targetId": "21711373-72c1-ec11-b83b-48df37d1deb7",
+ "targetDescription": "",
+ "createdDate": "2022-04-26T15:21:16.453",
+ "modifiedDate": "2022-04-26T16:04:15.88",
+ "targetUpdateTypeId": 0,
+ "targetUpdateTypeName": "Overwrite",
+ "validatedQueryText": "SET NOCOUNT ON; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;\r\n\r\nINSERT INTO C518001158.[testDataExtension] ([testField])\r\nSELECT querydef.[testField]\r\nFROM (SELECT SubscriberKey as testField FROM C518001158._Subscribers ) AS querydef \r\nSELECT @rcInsert = @@ROWCOUNT;;\r\n",
+ "categoryId": 999,
+ "isFrozen": false
+}

test/resources/9999999/automation/v1/queries/get-response.json

@@ -53,6 +53,23 @@
  "targetUpdateTypeName": "Overwrite",
  "categoryId": 999,
  "isFrozen": false
+ },
+ {
+ "queryDefinitionId": "549f0568-607c-4940-afef-437965094dat_fixKeysSuffix",
+ "name": "testExisting_query_fixedKeys",
+ "key": "testExisting_query_fixKeysSuffix",
+ "description": "bla bla",
+ "queryText": "Select\n SubscriberKey as testField, Trim(last_name) AS name\nfrom\n _Subscribers\nwhere\n country in ('test')\n",
+ "targetName": "testExisting_dataExtension",
+ "targetKey": "testExisting_dataExtension",
+ "targetId": "21711373-72c1-ec11-b83b-48df37d1deb7",
+ "targetDescription": "",
+ "createdDate": "2022-04-26T15:21:16.453",
+ "modifiedDate": "2022-04-26T16:02:44.01",
+ "targetUpdateTypeId": 0,
+ "targetUpdateTypeName": "Overwrite",
+ "categoryId": 999,
+ "isFrozen": false
  }
  ]
 }

test/resources/9999999/query/patch_fixKeysSuffix-expected.json

@@ -0,0 +1,11 @@
+{
+ "name": "testExisting_query_fixedKeys",
+ "key": "testExisting_query_fixedKeys_DEV",
+ "description": "updated on deploy",
+ "targetKey": "testExisting_dataExtension",
+ "createdDate": "2022-04-26T15:21:16.453",
+ "modifiedDate": "2022-04-26T16:04:15.88",
+ "targetUpdateTypeName": "Overwrite",
+ "isFrozen": false,
+ "r__folder_Path": "Query"
+}

test/resources/9999999/query/patch_fixKeysSuffix-expected.sql

@@ -0,0 +1,6 @@
+SELECT
+ SubscriberKey AS testField
+FROM
+ _Subscribers
+WHERE
+ country IN ('test')

test/resources/9999999/queryDefinition/retrieve-CustomerKey=testExisting_query_fixKeysSuffixANDStatus=Active-response.xml

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="utf-8"?>
+<soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:xsd="http://www.w3.org/2001/XMLSchema"
+ xmlns:wsa="http://schemas.xmlsoap.org/ws/2004/08/addressing"
+ xmlns:wsse="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd"
+ xmlns:wsu="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd">
+ <soap:Header>
+ <wsa:Action>RetrieveResponse</wsa:Action>
+ <wsa:MessageID>urn:uuid:7ef0345e-b559-4fc4-8986-47e54e1a8a58</wsa:MessageID>
+ <wsa:RelatesTo>urn:uuid:b2e814a6-517c-4882-9bbb-238bfce951ce</wsa:RelatesTo>
+ <wsa:To>http://schemas.xmlsoap.org/ws/2004/08/addressing/role/anonymous</wsa:To>
+ <wsse:Security>
+ <wsu:Timestamp wsu:Id="Timestamp-dfc4ae59-8642-4432-b505-554669b47186">
+ <wsu:Created>2023-04-11T16:33:48Z</wsu:Created>
+ <wsu:Expires>2023-04-11T16:38:48Z</wsu:Expires>
+ </wsu:Timestamp>
+ </wsse:Security>
+ </soap:Header>
+ <soap:Body>
+ <RetrieveResponseMsg xmlns="http://exacttarget.com/wsdl/partnerAPI">
+ <OverallStatus>OK</OverallStatus>
+ <RequestID>e8eb2988-2f43-4243-a6b0-6ab6b841a6ab</RequestID>
+ <Results xsi:type="QueryDefinition">
+ <PartnerKey xsi:nil="true" />
+ <ObjectID>549f0568-607c-4940-afef-437965094dat_fixKeysSuffix</ObjectID>
+ </Results>
+ </RetrieveResponseMsg>
+ </soap:Body>
+</soap:Envelope>