From 943d00e3706b5b9d3a4a1ed4ab1a05bdcd65fd44 Mon Sep 17 00:00:00 2001
From: Luca Forstner <luca.forstner@sentry.io>
Date: Wed, 16 Aug 2023 10:32:45 +0200
Subject: [PATCH] ref: Run upload preparation with maximum concurrency (#379)

---
 .../src/debug-id-upload.ts                    | 30 +++++++++++++++----
 1 file changed, 24 insertions(+), 6 deletions(-)

diff --git a/packages/bundler-plugin-core/src/debug-id-upload.ts b/packages/bundler-plugin-core/src/debug-id-upload.ts
index b29b561d..690b7807 100644
--- a/packages/bundler-plugin-core/src/debug-id-upload.ts
+++ b/packages/bundler-plugin-core/src/debug-id-upload.ts
@@ -105,8 +105,11 @@ export function createDebugIdUploadFunction({
         const prepareSpan = artifactBundleUploadTransaction.startChild({
           description: "prepare-bundles",
         });
-        await Promise.all(
-          debugIdChunkFilePaths.map(async (chunkFilePath, chunkIndex): Promise<void> => {
+
+        // Preparing the bundles can be a lot of work and doing it all at once has the potential of nuking the heap so
+        // instead we do it with a maximum of 16 concurrent workers
+        const preparationTasks = debugIdChunkFilePaths.map(
+          (chunkFilePath, chunkIndex) => async () => {
             await prepareBundleForDebugIdUpload(
               chunkFilePath,
               tmpUploadFolder,
@@ -114,8 +117,22 @@ export function createDebugIdUploadFunction({
               logger,
               rewriteSourcesHook ?? defaultRewriteSourcesHook
             );
-          })
+          }
         );
+        const workers: Promise<void>[] = [];
+        const worker = async () => {
+          while (preparationTasks.length > 0) {
+            const task = preparationTasks.shift();
+            if (task) {
+              await task();
+            }
+          }
+        };
+        for (let workerIndex = 0; workerIndex < 16; workerIndex++) {
+          workers.push(worker());
+        }
+        await Promise.all(workers);
+
         prepareSpan.finish();
 
         const files = await fs.promises.readdir(tmpUploadFolder);
@@ -251,9 +268,9 @@ export async function prepareBundleForDebugIdUpload(
     bundleFilePath,
     bundleContent,
     logger
-  ).then(async (sourceMapPath): Promise<void> => {
+  ).then(async (sourceMapPath) => {
     if (sourceMapPath) {
-      return await prepareSourceMapForDebugIdUpload(
+      await prepareSourceMapForDebugIdUpload(
         sourceMapPath,
         path.join(uploadFolder, `${uniqueUploadName}.js.map`),
         debugId,
@@ -263,7 +280,8 @@ export async function prepareBundleForDebugIdUpload(
     }
   });
 
-  return Promise.all([writeSourceFilePromise, writeSourceMapFilePromise]);
+  await writeSourceFilePromise;
+  await writeSourceMapFilePromise;
 }
 
 /**