ref: Run upload preparation with maximum concurrency (#379)

getsentry · Aug 16, 2023 · 943d00e · 943d00e
1 parent a93ac85
commit 943d00e
Showing 1 changed file with 24 additions and 6 deletions.
diff --git a/packages/bundler-plugin-core/src/debug-id-upload.ts b/packages/bundler-plugin-core/src/debug-id-upload.ts
@@ -105,17 +105,34 @@ export function createDebugIdUploadFunction({
         const prepareSpan = artifactBundleUploadTransaction.startChild({
           description: "prepare-bundles",
         });
-        await Promise.all(
-          debugIdChunkFilePaths.map(async (chunkFilePath, chunkIndex): Promise<void> => {
+
+        // Preparing the bundles can be a lot of work and doing it all at once has the potential of nuking the heap so
+        // instead we do it with a maximum of 16 concurrent workers
+        const preparationTasks = debugIdChunkFilePaths.map(
+          (chunkFilePath, chunkIndex) => async () => {
             await prepareBundleForDebugIdUpload(
               chunkFilePath,
               tmpUploadFolder,
               chunkIndex,
               logger,
               rewriteSourcesHook ?? defaultRewriteSourcesHook
             );
-          })
+          }
         );
+        const workers: Promise<void>[] = [];
+        const worker = async () => {
+          while (preparationTasks.length > 0) {
+            const task = preparationTasks.shift();
+            if (task) {
+              await task();
+            }
+          }
+        };
+        for (let workerIndex = 0; workerIndex < 16; workerIndex++) {
+          workers.push(worker());
+        }
+        await Promise.all(workers);
+
         prepareSpan.finish();
 
         const files = await fs.promises.readdir(tmpUploadFolder);
@@ -251,9 +268,9 @@ export async function prepareBundleForDebugIdUpload(
     bundleFilePath,
     bundleContent,
     logger
-  ).then(async (sourceMapPath): Promise<void> => {
+  ).then(async (sourceMapPath) => {
     if (sourceMapPath) {
-      return await prepareSourceMapForDebugIdUpload(
+      await prepareSourceMapForDebugIdUpload(
         sourceMapPath,
         path.join(uploadFolder, `${uniqueUploadName}.js.map`),
         debugId,
@@ -263,7 +280,8 @@ export async function prepareBundleForDebugIdUpload(
     }
   });
 
-  return Promise.all([writeSourceFilePromise, writeSourceMapFilePromise]);
+  await writeSourceFilePromise;
+  await writeSourceMapFilePromise;
 }
 
 /**