From 3ea27a29a98f3485f8f69619499f8ed49a065e4b Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Mon, 29 Jan 2024 16:18:20 -0700 Subject: [PATCH] num tasks and num requests limits --- .../deephaven/engine/context/QueryCompiler.java | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java index 777712b12cd..7f4e15930af 100644 --- a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java +++ b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java @@ -47,7 +47,8 @@ public class QueryCompiler { /** A flag to externally disable parallel compilation. */ - public static volatile boolean DISABLE_PARALLEL_COMPILE = false; + public static volatile int PARALLELISM_FACTOR = ForkJoinPool.getCommonPoolParallelism(); + public static volatile int REQUESTS_PER_TASK = 0; public static volatile boolean DISABLE_SHARED_COMPILER = false; private static final Logger log = LoggerFactory.getLogger(QueryCompiler.class); @@ -835,10 +836,17 @@ private void maybeCreateClass( boolean exceptionCaught = false; try { long startTm = System.nanoTime(); - int parallelismFactor = ForkJoinPool.getCommonPoolParallelism(); - log.warn().append("Compiling with parallelism factor of: ").append(parallelismFactor).endl(); + int parallelismFactor; + if (REQUESTS_PER_TASK == 0) { + parallelismFactor = PARALLELISM_FACTOR; + } else { + parallelismFactor = (requests.length + REQUESTS_PER_TASK - 1) / REQUESTS_PER_TASK; + } + int requestsPerTask = Math.max(32, (requests.length + parallelismFactor - 1) / parallelismFactor); - if (DISABLE_PARALLEL_COMPILE || parallelismFactor == 1 || requestsPerTask >= requests.length) { + log.warn().append("Compiling with parallelismFactor = ").append(parallelismFactor) + .append(" requestsPerTask = ").append(requestsPerTask).endl(); + if (parallelismFactor == 1 || requestsPerTask >= requests.length) { maybeCreateClassHelper(compiler, fileManager, requests, rootPathAsString, tempDirAsString, 0, requests.length); } else {