v1.6.16 - Continued Optimizations (#567)

* Starting new version, skipped one version for namespaced package so that the versions are now in sync * In flight changes performance tuning bulk full recalcs even further * Optimizations to full batch recalculations by minimizing timeout checks (which are apparently accounting for 50+% of recalc time with larger batch sizes) and standardized how logging is occurring in batch/queueable jobs * Further optimizing RollupLimits class * Making timeout check respect all full recalcs
jamessimone · Feb 26, 2024 · 665cec0 · 665cec0
1 parent 403fa58
commit 665cec0
Show file tree

Hide file tree

Showing 14 changed files with 23,804 additions and 11,993 deletions.
diff --git a/README.md b/README.md
@@ -24,12 +24,12 @@ As well, don't miss [the Wiki](../../wiki), which includes even more info for co
 
 ## Deployment & Setup
 
-<a href="https://login.salesforce.com/packaging/installPackage.apexp?p0=04t6g000008OaZ6AAK">
+<a href="https://login.salesforce.com/packaging/installPackage.apexp?p0=04t6g000008OaZVAA0">
   <img alt="Deploy to Salesforce"
        src="./media/deploy-package-to-prod.png">
 </a>
 
-<a href="https://test.salesforce.com/packaging/installPackage.apexp?p0=04t6g000008OaZ6AAK">
+<a href="https://test.salesforce.com/packaging/installPackage.apexp?p0=04t6g000008OaZVAA0">
   <img alt="Deploy to Salesforce Sandbox"
        src="./media/deploy-package-to-sandbox.png">
 </a>

diff --git a/extra-tests/classes/RollupFullRecalcTests.cls b/extra-tests/classes/RollupFullRecalcTests.cls
@@ -2226,7 +2226,7 @@ private class RollupFullRecalcTests {
   }
 
   @IsTest
-  static void addsCabooseForMultiplFullRecalcsToDifferentParents() {
+  static void addsCabooseForMultipleFullRecalcsToDifferentParents() {
     // when there are multiple children types passed to Rollup.performBulkFullRecalc (or any other method ending in a full recalc),
     // and there are multiple instances of the full batch recalcs that are present in the list of rollups
     Rollup.defaultControl = new RollupControl__mdt(MaxRollupRetries__c = 100, MaxLookupRowsBeforeBatching__c = 2, IsRollupLoggingEnabled__c = true);
@@ -2308,7 +2308,7 @@ private class RollupFullRecalcTests {
   }
 
   @IsTest
-  static void addsCabooseForMultiplFullRecalcsToDifferentParentsWithoutOtherRollups() {
+  static void addsCabooseForMultipleFullRecalcsToDifferentParentsWithoutOtherRollups() {
     // when there are multiple children types passed to Rollup.performBulkFullRecalc (or any other method ending in a full recalc),
     // and there are multiple instances of the full batch recalcs that are present in the list of rollups
     Rollup.defaultControl = new RollupControl__mdt(MaxRollupRetries__c = 100, MaxLookupRowsBeforeBatching__c = 2, IsRollupLoggingEnabled__c = true);

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "apex-rollup",
-  "version": "1.6.15",
+  "version": "1.6.16",
   "description": "Fast, configurable, elastically scaling custom rollup solution. Apex Invocable action, one-liner Apex trigger/CMDT-driven logic, and scheduled Apex-ready.",
   "repository": {
     "type": "git",
@@ -15,7 +15,7 @@
     "@ljharb/eslint-config": "21.1.0",
     "@lwc/eslint-plugin-lwc": "1.7.2",
     "@prettier/plugin-xml": "3.2.2",
-    "@salesforce/cli": "2.21.8",
+    "@salesforce/cli": "2.29.5",
     "@salesforce/eslint-config-lwc": "3.5.2",
     "@salesforce/eslint-plugin-lightning": "1.0.0",
     "@salesforce/sfdx-lwc-jest": "3.1.1",
@@ -45,6 +45,7 @@
     "create:package:logger": "pwsh -Command \"&{ . ./scripts/generatePackage.ps1; Generate -PackageName '\"Apex Rollup - Custom Logger\"' -ReadmePath \"./plugins/CustomObjectRollupLogger/README.md\" }\"",
     "create:package:callback": "pwsh -Command \"&{ . ./scripts/generatePackage.ps1; Generate -PackageName '\"Apex Rollup - Rollup Callback\"' -ReadmePath \"./plugins/RollupCallback/README.md\"\" }\"",
     "create:package:code-coverage": "pwsh -Command \"&{ . ./scripts/generatePackage.ps1; Generate -PackageName '\"Apex Rollup - Extra Code Coverage\"' -ReadmePath \"./plugins/ExtraCodeCoverage/README.md\" }\"",
+    "get:project:json": "pwsh -Command \"&{ . ./scripts/generatePackage.ps1; Update-SFDX-Project-JSON }\"",
     "husky:pre-commit": "lint-staged",
     "lint:verify": "sfdx scanner:run --target **/lwc/**/*.js,!node_modules/** --engine eslint-lwc --severity-threshold 3 --eslintconfig .eslintrc.json",
     "prepare": "husky install && sfdx plugins:link ./node_modules/@salesforce/sfdx-scanner && sfdx plugins:link ./node_modules/@jongpie/sfdx-bummer-plugin",

diff --git a/rollup-namespaced/README.md b/rollup-namespaced/README.md
@@ -18,12 +18,12 @@ For more info, see the base `README`.
 
 ## Deployment & Setup
 
-<a href="https://login.salesforce.com/packaging/installPackage.apexp?p0=04t6g000008OaZBAA0">
+<a href="https://login.salesforce.com/packaging/installPackage.apexp?p0=04t6g000008OaZaAAK">
   <img alt="Deploy to Salesforce"
        src="./media/deploy-package-to-prod.png">
 </a>
 
-<a href="https://test.salesforce.com/packaging/installPackage.apexp?p0=04t6g000008OaZBAA0">
+<a href="https://test.salesforce.com/packaging/installPackage.apexp?p0=04t6g000008OaZaAAK">
   <img alt="Deploy to Salesforce Sandbox"
        src="./media/deploy-package-to-sandbox.png">
 </a>
diff --git a/rollup-namespaced/sfdx-project.json b/rollup-namespaced/sfdx-project.json
@@ -5,7 +5,7 @@
             "package": "apex-rollup-namespaced",
             "path": "rollup-namespaced/source/rollup",
             "versionName": "Fixes an issue where rollups to multiple parents running in an unordered fashion could accidentally reset parents between bulk full recalc batches",
-            "versionNumber": "1.1.13.0",
+            "versionNumber": "1.1.16.0",
             "versionDescription": "Fast, configurable, elastically scaling custom rollup solution. Apex Invocable action, one-liner Apex trigger/CMDT-driven logic, and scheduled Apex-ready.",
             "releaseNotesUrl": "https://github.com/jamessimone/apex-rollup/releases/latest",
             "unpackagedMetadata": {
@@ -18,16 +18,12 @@
     ],
     "namespace": "please",
     "sfdcLoginUrl": "https://login.salesforce.com",
-    "sourceApiVersion": "58.0",
+    "sourceApiVersion": "60.0",
     "packageAliases": {
         "apex-rollup-namespaced": "0Ho6g000000PBMjCAO",
-        "apex-rollup-namespaced@1.1.7": "04t6g000008OaJQAA0",
-        "apex-rollup-namespaced@1.1.8": "04t6g000008OaJpAAK",
-        "apex-rollup-namespaced@1.1.9": "04t6g000008OaLHAA0",
-        "apex-rollup-namespaced@1.1.10": "04t6g000008OaOBAA0",
-        "apex-rollup-namespaced@1.1.11": "04t6g000008OaOzAAK",
         "apex-rollup-namespaced@1.1.12": "04t6g000008OaUZAA0",
         "apex-rollup-namespaced@1.1.13": "04t6g000008OaYwAAK",
-        "apex-rollup-namespaced@1.1.14": "04t6g000008OaZBAA0"
+        "apex-rollup-namespaced@1.1.14": "04t6g000008OaZBAA0",
+        "apex-rollup-namespaced@1.1.16": "04t6g000008OaZaAAK"
     }
 }
diff --git a/rollup/core/classes/Rollup.cls b/rollup/core/classes/Rollup.cls
@@ -26,7 +26,6 @@ global without sharing virtual class Rollup implements RollupLogger.ToStringObje
   @TestVisible
   private static Boolean shouldFlattenAsyncProcesses = false;
 
-  private static RollupLimits.Tester limitTester;
   private static List<Rollup__mdt> cachedMetadata;
   private static RollupControl__mdt cachedOrgDefault;
   private static Boolean isCDC = false;
@@ -1897,11 +1896,9 @@ global without sharing virtual class Rollup implements RollupLogger.ToStringObje
   }
 
   public static Boolean hasExceededCurrentRollupLimits(RollupControl__mdt control) {
-    if (limitTester == null) {
-      limitTester = new RollupLimits.Tester(control, isContextAsync());
-    }
-    Boolean hasExceededLimits = limitTester.hasExceededLimits && isDeferralAllowed;
-    if (hasExceededLimits) {
+    RollupLimits.Tester limitTester = new RollupLimits.Tester(control, isContextAsync());
+    Boolean hasExceededLimits = limitTester.hasExceededLimits() && isDeferralAllowed;
+    if (limitTester.hasExceededLimits() && isDeferralAllowed) {
       Boolean isLoggingCurrentlyDisabled = control.IsRollupLoggingEnabled__c == false;
       if (isLoggingCurrentlyDisabled) {
         control.IsRollupLoggingEnabled__c = true;

diff --git a/rollup/core/classes/RollupAsyncProcessor.cls b/rollup/core/classes/RollupAsyncProcessor.cls
@@ -13,6 +13,7 @@ global virtual without sharing class RollupAsyncProcessor extends Rollup impleme
   private Map<SObjectType, Set<String>> lookupObjectToUniqueFieldNames;
   private RollupRelationshipFieldFinder.Traversal traversal;
   private Boolean wasConvertedToFullRecalculation = false;
+  private Boolean isFromBatchExecute = false;
 
   protected Boolean isTimingOut = false;
   protected Boolean isProcessed = false;
@@ -39,6 +40,11 @@ global virtual without sharing class RollupAsyncProcessor extends Rollup impleme
     REPARENTED
   }
 
+  private enum TimeoutCheckLevel {
+    AT_ROLLUP,
+    AT_PARENT
+  }
+
   private Map<Schema.SObjectType, RollupFullRecalcProcessor> childToUnexpectedFullRecalc {
     get {
       if (childToUnexpectedFullRecalc == null) {
@@ -89,6 +95,16 @@ global virtual without sharing class RollupAsyncProcessor extends Rollup impleme
     set;
   }
 
+  private static Map<String, Map<String, CalcItemBag>> CACHED_CALC_ITEM_BAGS {
+    get {
+      if (CACHED_CALC_ITEM_BAGS == null) {
+        CACHED_CALC_ITEM_BAGS = new Map<String, Map<String, CalcItemBag>>();
+      }
+      return CACHED_CALC_ITEM_BAGS;
+    }
+    set;
+  }
+
   public static RollupAsyncProcessor getConductor(InvocationPoint invokePoint, List<SObject> calcItems, Map<Id, SObject> oldCalcItems) {
     return new QueueableProcessor(invokePoint, calcItems, oldCalcItems);
   }
@@ -228,26 +244,11 @@ global virtual without sharing class RollupAsyncProcessor extends Rollup impleme
   public virtual void execute(Database.BatchableContext context, List<SObject> scope) {
     this.logger.log('starting batch chunk', this, System.LoggingLevel.INFO);
 
-    if (this.getTypeName() == RollupAsyncProcessor.class.getName()) {
-      this.lookupItems = scope;
-      for (Integer index = this.rollups.size() - 1; index >= 0; index--) {
-        RollupAsyncProcessor roll = this.rollups[index];
-        if (scope.getSObjectType() != roll.lookupObj) {
-          this.rollups.remove(index);
-          this.deferredRollups.add(roll);
-        }
-      }
-    } else {
-      this.calcItems = scope;
-    }
-
-    this.performWork();
+    this.isFromBatchExecute = true;
+    this.innerBatchExecute(scope);
+    this.isFromBatchExecute = false;
 
-    BatchEndSnapshot snapshot = new BatchEndSnapshot();
-    snapshot.currentHeapUsage = Limits.getHeapSize();
-    snapshot.previouslyResetParentsSize = this.previouslyResetParents.size();
-    this.logger.log('batch chunk end', snapshot, System.LoggingLevel.INFO);
-    this.logger.save();
+    this.executeFinish(new Map<String, String>{ 'previouslyResetParents size' => '' + this.previouslyResetParents.size() });
   }
 
   public virtual void finish(Database.BatchableContext context) {
@@ -339,6 +340,23 @@ global virtual without sharing class RollupAsyncProcessor extends Rollup impleme
     return 'No process Id';
   }
 
+  protected virtual void innerBatchExecute(List<SObject> scope) {
+    if (this.getTypeName() == RollupAsyncProcessor.class.getName()) {
+      this.lookupItems = scope;
+      for (Integer index = this.rollups.size() - 1; index >= 0; index--) {
+        RollupAsyncProcessor roll = this.rollups[index];
+        if (scope.getSObjectType() != roll.lookupObj) {
+          this.rollups.remove(index);
+          this.deferredRollups.add(roll);
+        }
+      }
+    } else {
+      this.calcItems = scope;
+    }
+
+    this.performWork();
+  }
+
   protected virtual void performWork() {
     this.process(this.rollups);
   }
@@ -452,13 +470,17 @@ global virtual without sharing class RollupAsyncProcessor extends Rollup impleme
     }
 
     public void execute(System.QueueableContext qc) {
+      this.logger.log('Starting queueable', this, System.LoggingLevel.INFO);
       AdditionalContext context = new AdditionalContext(qc);
       this.setCurrentJobId(context.getJobId());
       System.attachFinalizer(new RollupFinalizer());
+
       this.performWork();
+
       if (this.fullRecalcProcessor?.isBatch() != true) {
         this.finish(context);
       }
+      this.executeFinish(null);
     }
   }
 
@@ -482,14 +504,21 @@ global virtual without sharing class RollupAsyncProcessor extends Rollup impleme
     }
   }
 
-  private class BatchEndSnapshot {
-    public Long currentHeapUsage { get; set; }
+  private class RollupEndSnapshot {
+    public RollupEndSnapshot(Map<String, String> additionalContext) {
+      this.additionalContext = additionalContext;
+    }
+    public Long currentHeapUsage {
+      get {
+        return Limits.getHeapSize();
+      }
+    }
     public Long heapLimit {
       get {
-        return 12000000;
+        return Limits.getLimitHeapSize();
       }
     }
-    public Long previouslyResetParentsSize { get; set; }
+    public Map<String, String> additionalContext { get; set; }
   }
 
   protected virtual String beginAsyncRollup() {
@@ -503,6 +532,9 @@ global virtual without sharing class RollupAsyncProcessor extends Rollup impleme
   }
 
   protected String startBatchProcessor() {
+    if (this.isFromBatchExecute) {
+      return new QueueableProcessor(this).startAsyncWork();
+    }
     return Database.executeBatch(this, this.rollupControl.BatchChunkSize__c.intValue());
   }
 
@@ -682,7 +714,7 @@ global virtual without sharing class RollupAsyncProcessor extends Rollup impleme
       roll.isProcessed = true;
       this.setupCalcItemData(roll);
       // for each iteration, ensure we're not operating beyond the bounds of our governor limits
-      if (this.getIsTimingOut(roll.rollupControl)) {
+      if (this.getIsTimingOut(roll.rollupControl, TimeoutCheckLevel.AT_ROLLUP)) {
         this.addProcessorToDeferredRollups(roll);
         continue;
       }
@@ -814,7 +846,10 @@ global virtual without sharing class RollupAsyncProcessor extends Rollup impleme
     }
   }
 
-  private Boolean getIsTimingOut(RollupControl__mdt control) {
+  private Boolean getIsTimingOut(RollupControl__mdt control, TimeoutCheckLevel timeCheckLevel) {
+    if (timeCheckLevel == TimeoutCheckLevel.AT_PARENT && this.fullRecalcProcessor != null) {
+      return false;
+    }
     if (this.isTimingOut == false && hasExceededCurrentRollupLimits(control)) {
       this.logger.log(this.getTypeName() + ': starting to time out ...', System.LoggingLevel.WARN);
       this.isTimingOut = true;
@@ -866,6 +901,16 @@ global virtual without sharing class RollupAsyncProcessor extends Rollup impleme
       return new Map<String, CalcItemBag>();
     }
 
+    String cacheKey = '' + roll.lookupObj + roll.lookupFieldOnCalcItem;
+    switch on this.invokePoint {
+      when FROM_FULL_RECALC_LWC, FROM_SINGULAR_PARENT_RECALC_LWC, FROM_FULL_RECALC_FLOW {
+        Map<String, CalcItemBag> possiblyCachedItems = CACHED_CALC_ITEM_BAGS.get(cacheKey);
+        if (possiblyCachedItems != null) {
+          return possiblyCachedItems;
+        }
+      }
+    }
+
     Map<String, CalcItemBag> lookupFieldToCalcItems = new Map<String, CalcItemBag>();
     if (String.isNotBlank(roll.metadata.GrandparentRelationshipFieldPath__c) || roll.metadata.RollupToUltimateParent__c) {
       if (roll.traversal == null) {
@@ -929,7 +974,7 @@ global virtual without sharing class RollupAsyncProcessor extends Rollup impleme
         }
       }
     }
-
+    CACHED_CALC_ITEM_BAGS.put(cacheKey, lookupFieldToCalcItems);
     return lookupFieldToCalcItems;
   }
 
@@ -1123,7 +1168,7 @@ global virtual without sharing class RollupAsyncProcessor extends Rollup impleme
       Boolean shouldRunSyncDeferred = this.getShouldRunSyncDeferred(rollup);
       Boolean couldRunSync =
         rollup.rollupControl.ShouldRunAs__c == RollupMetaPicklists.ShouldRunAs.Synchronous ||
-        (isRunningAsync && this.getIsTimingOut(rollup.rollupControl) == false) ||
+        (isRunningAsync && this.getIsTimingOut(rollup.rollupControl, TimeoutCheckLevel.AT_ROLLUP) == false) ||
         this.isSingleRecordSyncUpdate(rollup);
 
       if (limitTester.hasExceededOrgAsyncLimit()) {
@@ -1266,7 +1311,10 @@ global virtual without sharing class RollupAsyncProcessor extends Rollup impleme
         CalcItemBag bag = calcItemsByLookupField.get(key);
         List<SObject> localCalcItems = bag.getAll();
 
-        if (this.getIsTimingOut(this.rollupControl) || (this.isValidAdditionalCalcItemRetrieval(roll) && bag.hasQueriedForAdditionalItems == false)) {
+        if (
+          this.getIsTimingOut(this.rollupControl, TimeoutCheckLevel.AT_PARENT) ||
+          (this.isValidAdditionalCalcItemRetrieval(roll) && bag.hasQueriedForAdditionalItems == false)
+        ) {
           this.addProcessorToDeferredRollups(roll);
           lookupItems.remove(index);
           continue;
@@ -1447,8 +1495,15 @@ global virtual without sharing class RollupAsyncProcessor extends Rollup impleme
 
   private void cleanup() {
     RollupEvaluator.clearCache();
+    CACHED_CALC_ITEM_BAGS = null;
     this.uniqueParentFields.clear();
     this.cleanupStaticVars();
     this.getCachedApexOperations().clear();
   }
+
+  private void executeFinish(Map<String, String> additionalContext) {
+    RollupEndSnapshot snapshot = new RollupEndSnapshot(additionalContext);
+    this.logger.log(this.getTypeName() + ' execute end', snapshot, System.LoggingLevel.INFO);
+    this.logger.save();
+  }
 }
diff --git a/rollup/core/classes/RollupEvaluator.cls b/rollup/core/classes/RollupEvaluator.cls
@@ -581,8 +581,9 @@ public without sharing abstract class RollupEvaluator implements Rollup.Evaluato
     @SuppressWarnings('PMD.EmptyCatchBlock')
     public override Boolean matches(Object o) {
       String decisionKey = this.fieldName + this.criteria + this.originalValues + o.hashCode();
-      if (CACHED_WHERE_DECISIONS.containsKey(decisionKey)) {
-        return CACHED_WHERE_DECISIONS.get(decisionKey);
+      Boolean possiblePriorMatchOutcome = CACHED_WHERE_DECISIONS.get(decisionKey);
+      if (possiblePriorMatchOutcome != null) {
+        return possiblePriorMatchOutcome;
       }
 
       SObject item = (SObject) o;