Speed up task resolution and planning logic (#4132)

Fixes #4129 and #4026. We never really looked at performance at this stuff before, but in order to support `selective.prepare __` (the default of `selective.prepare`) we need to improve these bottlenecks * Introduces the `ResolveCore#Cache` class that holds mutable dictionaries to memo-ize various expensive parts of the task resolution logic * Lots of micro-optimizations in various places: cache `Segments#hashCode`, avoid calling `Segments#render` for sorting or `Segments#part` for taking the last part, etc. Brings down `time ./mill plan __` from ~60s to ~3s
com-lihaoyi · Dec 14, 2024 · 178a545 · 178a545
1 parent 28b07d9
commit 178a545
Show file tree

Hide file tree

Showing 25 changed files with 354 additions and 270 deletions.
diff --git a/example/thirdparty/mockito/build.mill b/example/thirdparty/mockito/build.mill
@@ -226,7 +226,7 @@ object `package` extends RootModule with MockitoModule {
 //      def testIvyDeps = Agg(libraries.junit4, libraries.osgi)
 //      def testRuntimeIvyDeps = Agg(libraries.equinox)
 //      trait BundleModule extends MockitoModule{
-//        def bundleName: String = millModuleSegments.parts.last
+//        def bundleName: String = millModuleSegments.last.value
 //        override def millSourcePath = `osgi-test`.millSourcePath
 //        def sources = Task.Sources(millSourcePath / "src" / bundleName / "java")
 //

diff --git a/example/thirdparty/netty/build.mill b/example/thirdparty/netty/build.mill
@@ -77,7 +77,7 @@ trait NettyModule extends NettyBaseModule {
     def ivyDeps = super.ivyDeps() ++ testIvyDeps()
     def forkWorkingDir = NettyModule.this.millSourcePath
     def forkArgs = super.forkArgs() ++ Seq(
-      "-Dnativeimage.handlerMetadataArtifactId=netty-" + NettyModule.this.millModuleSegments.parts.last
+      "-Dnativeimage.handlerMetadataArtifactId=netty-" + NettyModule.this.millModuleSegments.last.value
     )
 
   }

diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-compile-classpaths.json b/integration/ide/bsp-server/resources/snapshots/build-targets-compile-classpaths.json
@@ -1,13 +1,5 @@
 {
   "items": [
-    {
-      "target": {
-        "uri": "file:///workspace/hello-java"
-      },
-      "classpath": [
-        "file:///workspace/hello-java/compile-resources"
-      ]
-    },
     {
       "target": {
         "uri": "file:///workspace/hello-kotlin"
@@ -26,6 +18,14 @@
         "file:///coursier-cache/https/repo1.maven.org/maven2/org/scala-lang/scala-library/<scala-version>/scala-library-<scala-version>.jar",
         "file:///workspace/hello-scala/compile-resources"
       ]
+    },
+    {
+      "target": {
+        "uri": "file:///workspace/hello-java"
+      },
+      "classpath": [
+        "file:///workspace/hello-java/compile-resources"
+      ]
     }
   ]
 }
diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-dependency-modules.json b/integration/ide/bsp-server/resources/snapshots/build-targets-dependency-modules.json
@@ -1,11 +1,5 @@
 {
   "items": [
-    {
-      "target": {
-        "uri": "file:///workspace/hello-java"
-      },
-      "modules": []
-    },
     {
       "target": {
         "uri": "file:///workspace/hello-kotlin"
@@ -27,6 +21,12 @@
           "version": "<scala-version>"
         }
       ]
+    },
+    {
+      "target": {
+        "uri": "file:///workspace/hello-java"
+      },
+      "modules": []
     }
   ]
 }
diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-dependency-sources.json b/integration/ide/bsp-server/resources/snapshots/build-targets-dependency-sources.json
@@ -1,11 +1,5 @@
 {
   "items": [
-    {
-      "target": {
-        "uri": "file:///workspace/hello-java"
-      },
-      "sources": []
-    },
     {
       "target": {
         "uri": "file:///workspace/hello-kotlin"
@@ -22,6 +16,12 @@
       "sources": [
         "file:///coursier-cache/https/repo1.maven.org/maven2/org/scala-lang/scala-library/<scala-version>/scala-library-<scala-version>-sources.jar"
       ]
+    },
+    {
+      "target": {
+        "uri": "file:///workspace/hello-java"
+      },
+      "sources": []
     }
   ]
 }
diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-javac-options.json b/integration/ide/bsp-server/resources/snapshots/build-targets-javac-options.json
@@ -1,15 +1,5 @@
 {
   "items": [
-    {
-      "target": {
-        "uri": "file:///workspace/hello-java"
-      },
-      "options": [],
-      "classpath": [
-        "file:///workspace/hello-java/compile-resources"
-      ],
-      "classDirectory": "file:///workspace/out/hello-java/compile.dest/classes"
-    },
     {
       "target": {
         "uri": "file:///workspace/hello-kotlin"
@@ -32,6 +22,16 @@
         "file:///workspace/hello-scala/compile-resources"
       ],
       "classDirectory": "file:///workspace/out/hello-scala/compile.dest/classes"
+    },
+    {
+      "target": {
+        "uri": "file:///workspace/hello-java"
+      },
+      "options": [],
+      "classpath": [
+        "file:///workspace/hello-java/compile-resources"
+      ],
+      "classDirectory": "file:///workspace/out/hello-java/compile.dest/classes"
     }
   ]
 }
diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-jvm-run-environments.json b/integration/ide/bsp-server/resources/snapshots/build-targets-jvm-run-environments.json
@@ -1,19 +1,5 @@
 {
   "items": [
-    {
-      "target": {
-        "uri": "file:///workspace/hello-java"
-      },
-      "classpath": [
-        "file:///workspace/hello-java/compile-resources",
-        "file:///workspace/hello-java/resources",
-        "file:///workspace/out/hello-java/compile.dest/classes"
-      ],
-      "jvmOptions": [],
-      "workingDirectory": "/workspace",
-      "environmentVariables": {},
-      "mainClasses": []
-    },
     {
       "target": {
         "uri": "file:///workspace/hello-kotlin"
@@ -44,6 +30,20 @@
       "workingDirectory": "/workspace",
       "environmentVariables": {},
       "mainClasses": []
+    },
+    {
+      "target": {
+        "uri": "file:///workspace/hello-java"
+      },
+      "classpath": [
+        "file:///workspace/hello-java/compile-resources",
+        "file:///workspace/hello-java/resources",
+        "file:///workspace/out/hello-java/compile.dest/classes"
+      ],
+      "jvmOptions": [],
+      "workingDirectory": "/workspace",
+      "environmentVariables": {},
+      "mainClasses": []
     }
   ]
 }
diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-jvm-test-environments.json b/integration/ide/bsp-server/resources/snapshots/build-targets-jvm-test-environments.json
@@ -1,19 +1,5 @@
 {
   "items": [
-    {
-      "target": {
-        "uri": "file:///workspace/hello-java"
-      },
-      "classpath": [
-        "file:///workspace/hello-java/compile-resources",
-        "file:///workspace/hello-java/resources",
-        "file:///workspace/out/hello-java/compile.dest/classes"
-      ],
-      "jvmOptions": [],
-      "workingDirectory": "/workspace",
-      "environmentVariables": {},
-      "mainClasses": []
-    },
     {
       "target": {
         "uri": "file:///workspace/hello-kotlin"
@@ -44,6 +30,20 @@
       "workingDirectory": "/workspace",
       "environmentVariables": {},
       "mainClasses": []
+    },
+    {
+      "target": {
+        "uri": "file:///workspace/hello-java"
+      },
+      "classpath": [
+        "file:///workspace/hello-java/compile-resources",
+        "file:///workspace/hello-java/resources",
+        "file:///workspace/out/hello-java/compile.dest/classes"
+      ],
+      "jvmOptions": [],
+      "workingDirectory": "/workspace",
+      "environmentVariables": {},
+      "mainClasses": []
     }
   ]
 }
diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-output-paths.json b/integration/ide/bsp-server/resources/snapshots/build-targets-output-paths.json
@@ -2,19 +2,19 @@
   "items": [
     {
       "target": {
-        "uri": "file:///workspace/hello-java"
+        "uri": "file:///workspace/hello-kotlin"
       },
       "outputPaths": []
     },
     {
       "target": {
-        "uri": "file:///workspace/hello-kotlin"
+        "uri": "file:///workspace/hello-scala"
       },
       "outputPaths": []
     },
     {
       "target": {
-        "uri": "file:///workspace/hello-scala"
+        "uri": "file:///workspace/hello-java"
       },
       "outputPaths": []
     },

diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-resources.json b/integration/ide/bsp-server/resources/snapshots/build-targets-resources.json
@@ -2,19 +2,19 @@
   "items": [
     {
       "target": {
-        "uri": "file:///workspace/hello-java"
+        "uri": "file:///workspace/hello-kotlin"
       },
       "resources": []
     },
     {
       "target": {
-        "uri": "file:///workspace/hello-kotlin"
+        "uri": "file:///workspace/hello-scala"
       },
       "resources": []
     },
     {
       "target": {
-        "uri": "file:///workspace/hello-scala"
+        "uri": "file:///workspace/hello-java"
       },
       "resources": []
     },

diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-scalac-options.json b/integration/ide/bsp-server/resources/snapshots/build-targets-scalac-options.json
@@ -1,15 +1,5 @@
 {
   "items": [
-    {
-      "target": {
-        "uri": "file:///workspace/hello-java"
-      },
-      "options": [],
-      "classpath": [
-        "file:///workspace/hello-java/compile-resources"
-      ],
-      "classDirectory": "file:///workspace/out/hello-java/compile.dest/classes"
-    },
     {
       "target": {
         "uri": "file:///workspace/hello-kotlin"
@@ -32,6 +22,16 @@
         "file:///workspace/hello-scala/compile-resources"
       ],
       "classDirectory": "file:///workspace/out/hello-scala/compile.dest/classes"
+    },
+    {
+      "target": {
+        "uri": "file:///workspace/hello-java"
+      },
+      "options": [],
+      "classpath": [
+        "file:///workspace/hello-java/compile-resources"
+      ],
+      "classDirectory": "file:///workspace/out/hello-java/compile.dest/classes"
     }
   ]
 }
diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-sources.json b/integration/ide/bsp-server/resources/snapshots/build-targets-sources.json
@@ -2,35 +2,35 @@
   "items": [
     {
       "target": {
-        "uri": "file:///workspace/hello-java"
+        "uri": "file:///workspace/hello-kotlin"
       },
       "sources": [
         {
-          "uri": "file:///workspace/hello-java/src",
+          "uri": "file:///workspace/hello-kotlin/src",
           "kind": 2,
           "generated": false
         }
       ]
     },
     {
       "target": {
-        "uri": "file:///workspace/hello-kotlin"
+        "uri": "file:///workspace/hello-scala"
       },
       "sources": [
         {
-          "uri": "file:///workspace/hello-kotlin/src",
+          "uri": "file:///workspace/hello-scala/src",
           "kind": 2,
           "generated": false
         }
       ]
     },
     {
       "target": {
-        "uri": "file:///workspace/hello-scala"
+        "uri": "file:///workspace/hello-java"
       },
       "sources": [
         {
-          "uri": "file:///workspace/hello-scala/src",
+          "uri": "file:///workspace/hello-java/src",
           "kind": 2,
           "generated": false
         }