Merge pull request #1094 from Accenture/develop

Release 5.3

.github/ISSUE_TEMPLATE/bug.yml

@@ -39,6 +39,7 @@ body:
  label: Version
  description: What version of our software are you running? (mcdev --version)
  options:
+ - 5.3.0
  - 5.2.0
  - 5.1.0
  - 5.0.2

lib/MetadataTypeDefinitions.js

@@ -38,6 +38,7 @@ const MetadataTypeDefinitions = {
  transactionalSMS: require('./metadataTypes/definitions/TransactionalSMS.definition'),
  triggeredSend: require('./metadataTypes/definitions/TriggeredSend.definition'),
  user: require('./metadataTypes/definitions/User.definition'),
+ verification: require('./metadataTypes/definitions/Verification.definition'),
 };
 
 module.exports = MetadataTypeDefinitions;

lib/MetadataTypeInfo.js

@@ -38,6 +38,7 @@ const MetadataTypeInfo = {
  transactionalSMS: require('./metadataTypes/TransactionalSMS'),
  triggeredSend: require('./metadataTypes/TriggeredSend'),
  user: require('./metadataTypes/User'),
+ verification: require('./metadataTypes/Verification'),
 };
 
 module.exports = MetadataTypeInfo;

lib/cli.js

@@ -45,7 +45,7 @@ yargs
  },
  })
  .command({
- command: 'deploy [BU] [TYPE] [KEY] [--fromRetrieve] [--refresh]',
+ command: 'deploy [BU] [TYPE] [KEY]',
  aliases: ['d'],
  desc: 'deploys local metadata to a business unit',
  builder: (yargs) => {
@@ -97,6 +97,11 @@ yargs
  group: 'Options for deploy:',
  describe:
  'optionally start existing schedule instead of running item once immediately (only works for automations)',
+ })
+ .option('fixShared', {
+ group: 'Options for deploy:',
+ describe:
+ "optionally ensure that updates to shared DataExtensions become visible in child BU's data designer (SF Known issue W-11031095)",
  });
  },
  handler: (argv) => {

lib/index.js

@@ -56,12 +56,14 @@ class Mcdev {
  'commitHistory',
  'execute',
  'filter',
+ 'fixShared',
  'fromRetrieve',
  'json',
  'like',
  'noLogColors',
  'noLogFile',
  'refresh',
+ '_runningTest',
  'schedule',
  'skipInteraction',
  ];
@@ -310,7 +312,8 @@ class Mcdev {
  triggeredSend: 'triggeredSendDefinition',
  user: 'accountUser',
  };
- Util.logger.info(`:: Retrieving ${cred}/${bu}\n`);
+ Util.logger.info('');
+ Util.logger.info(`:: Retrieving ${cred}/${bu}`);
  const retrieveTypesArr = [];
  if (selectedTypesArr) {
  for (const selectedType of Array.isArray(selectedTypesArr)

lib/metadataTypes/AttributeSet.js

@@ -31,13 +31,7 @@ class AttributeSet extends MetadataType {
  const result = await AttributeGroup.retrieveForCache();
  cache.setMetadata('attributeGroup', result.metadata);
  }
- return super.retrieveREST(
- retrieveDir,
- '/hub/v1/contacts/schema/setDefinitions',
- null,
- null,
- key
- );
+ return super.retrieveREST(retrieveDir, '/hub/v1/contacts/schema/setDefinitions', null, key);
  }
  /**
  * Retrieves Metadata of schema set definitions for caching.
@@ -47,6 +41,110 @@ class AttributeSet extends MetadataType {
  static retrieveForCache() {
  return super.retrieveREST(null, '/hub/v1/contacts/schema/setDefinitions');
  }
+ /**
+ * used to identify updated shared data extensions that are used in attributeSets.
+ * helper for DataExtension.#fixShared_onBU
+ *
+ * @param {Object.<string, string>} sharedDataExtensionMap ID-Key relationship of shared data extensions
+ * @param {object} fixShared_fields DataExtensionField.fixShared_fields
+ * @returns {Promise.<string[]>} Promise of list of shared dataExtension IDs
+ */
+ static async fixShared_retrieve(sharedDataExtensionMap, fixShared_fields) {
+ if (!Object.keys(sharedDataExtensionMap).length) {
+ return [];
+ }
+ const result = await super.retrieveREST(null, '/hub/v1/contacts/schema/setDefinitions');
+ const metadataMap = result?.metadata;
+ if (metadataMap && Object.keys(metadataMap).length) {
+ const sharedDeIds = Object.keys(metadataMap)
+ .filter(
+ (asKey) =>
+ metadataMap[asKey].storageLogicalType === 'ExactTargetSchema' ||
+ metadataMap[asKey].storageLogicalType === 'DataExtension'
+ )
+ .filter((asKey) => {
+ // check if dataExtension ID is found on any attributeSet of this BU
+ if (sharedDataExtensionMap[metadataMap[asKey].storageReferenceID.value]) {
+ Util.logger.debug(
+ ` shared dataExtension ID ${metadataMap[asKey].storageReferenceID.value} found in attributeSet ${asKey}`
+ );
+ return true;
+ } else {
+ return false;
+ }
+ })
+ .filter((asKey) => {
+ // check if any of the dataExtension fields dont exist on the attributeSet or are out of date
+ const deKey =
+ sharedDataExtensionMap[metadataMap[asKey].storageReferenceID.value];
+ const asFields = metadataMap[asKey].valueDefinitions;
+ const deFields = Object.values(fixShared_fields[deKey]);
+ return deFields.some((deField) => {
+ const search = asFields.filter((asf) => asf.name === deField.Name);
+ if (!search.length) {
+ Util.logger.debug(
+ Util.getGrayMsg(
+ ` - Field ${deField.Name} not found in attributeSet; Note: only first recognized difference is printed to log`
+ )
+ );
+ return true;
+ }
+ const asField = search[0];
+ if (asField.dataType !== deField.FieldType) {
+ Util.logger.debug(
+ Util.getGrayMsg(
+ ` - Field ${deField.Name} FieldType changed (old: ${asField.dataType}; new: ${deField.FieldType}); Note: only first recognized difference is printed to log`
+ )
+ );
+ return true;
+ }
+ asField.defaultValue ||= '';
+ if (
+ (asField.defaultValue && deField.DefaultValue === '') ||
+ (deField.FieldType === 'Boolean' &&
+ deField.DefaultValue !== '' &&
+ (deField.DefaultValue
+ ? 'True'
+ : 'False' !== asField.defaultValue)) ||
+ (deField.FieldType !== 'Boolean' &&
+ deField.DefaultValue !== asField.defaultValue)
+ ) {
+ Util.logger.debug(
+ ` - Field ${deField.Name} DefaultValue changed (old: ${asField.defaultValue}; new: ${deField.DefaultValue}); Note: only first recognized difference is printed to log`
+ );
+ return true;
+ }
+ // some field types don't carry the length property. reset to 0 to ease comparison
+ asField.length ||= 0;
+ if (asField.length !== deField.MaxLength) {
+ Util.logger.debug(
+ ` - Field ${deField.Name} MaxLength changed (old: ${asField.length}; new: ${deField.MaxLength}); Note: only first recognized difference is printed to log`
+ );
+ return true;
+ }
+ if (asField.isNullable !== deField.IsRequired) {
+ Util.logger.debug(
+ ` - Field ${deField.Name} IsRequired changed (old: ${asField.isNullable}; new: ${deField.IsRequired}); Note: only first recognized difference is printed to log`
+ );
+ return true;
+ }
+ if (asField.isPrimaryKey !== deField.IsPrimaryKey) {
+ Util.logger.debug(
+ ` - Field ${deField.Name} IsPrimaryKey changed (old: ${asField.isPrimaryKey}; new: ${deField.IsPrimaryKey}); Note: only first recognized difference is printed to log`
+ );
+ return true;
+ }
+ return false;
+ });
+ })
+ .map((key) => metadataMap[key].storageReferenceID.value)
+ .filter(Boolean);
+ return sharedDeIds;
+ } else {
+ // nothing to do - return empty array
+ return [];
+ }
+ }
 
  /**
  * Builds map of metadata entries mapped to their keyfields
@@ -83,7 +181,7 @@ class AttributeSet extends MetadataType {
  switch (metadata.storageLogicalType) {
  case 'ExactTargetSchema': // synced / shared DEs
  case 'DataExtension': {
- // local DEs
+ // shared / local DEs
  try {
  metadata.r__dataExtension_CustomerKey = cache.searchForField(
  'dataExtension',
@@ -226,6 +324,12 @@ class AttributeSet extends MetadataType {
  // Member ID
  delete metadata.customObjectOwnerMID;
 
+ // remove duplicate ID fields (main field is definitionID)
+ delete metadata.setDefinitionID;
+ if (metadata.dataRetentionProperties?.setDefinitionID) {
+ delete metadata.dataRetentionProperties?.setDefinitionID;
+ }
+
  // connectingID.identifierType seems to be always set to 'FullyQualifiedName' - to be sure we check it here and remove it if it's the case
  if (metadata.connectingID?.identifierType === 'FullyQualifiedName') {
  // remove useless field
@@ -241,16 +345,19 @@ class AttributeSet extends MetadataType {
  * @returns {object[]} all system value definitions
  */
  static _getSystemValueDefinitions() {
- if (!this.systemValueDefinitions) {
- this.systemValueDefinitions = Object.values(cache.getCache()['attributeSet'])
+ this.systemValueDefinitions ||= {};
+ if (!this.systemValueDefinitions[this.buObject.mid]) {
+ this.systemValueDefinitions[this.buObject.mid] = Object.values(
+ cache.getCache()['attributeSet']
+ )
  .flatMap((item) => {
  if (item.isSystemDefined) {
  return item.valueDefinitions;
  }
  })
  .filter(Boolean);
  }
- return this.systemValueDefinitions;
+ return this.systemValueDefinitions[this.buObject.mid];
  }
 }