From c2bc74b3794fe387a3b0376096f5867c457eb2d2 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Mon, 16 Dec 2024 20:11:00 +0800 Subject: [PATCH] Fix edge cases in selective execution (#4140) * Fixes handling of private tasks and added tests for that and for renaming tasks. As private tasks and overridden tasks are assigned a `.super.xyz` suffix during planning, we need to ensure we use the `Terminal#render` label which includes this suffix rather than the `.ctx.segments.render` label which does not * Ensure we preserve all watches during `--watch`, even if some portion of the build graph was skipped due to selective execution. This ensures that inputs that were skipped can continue to be watched and still trigger downstream changes later * We should make sure in `selective.resolve` that we do not call `resolve` on the outcome of `diffMetadata`, as that may include private tasks that cannot be resolved externally. Instead we should only call `resolve` when there is no diff to be had, in which case we need to take the input selector and resolve it to concrete task names to display --- build.mill | 1 + .../out-dir/1-out-files/build.mill | 39 +++++++++- .../selective-execution/resources/build.mill | 3 +- .../src/SelectiveExecutionTests.scala | 77 +++++++++++++------ .../src/WatchSourceInputTests.scala | 13 ++-- main/codesig/src/Logger.scala | 2 +- main/codesig/src/ReachabilityAnalysis.scala | 8 +- main/codesig/test/src/CallGraphTests.scala | 8 +- main/eval/src/mill/eval/Evaluator.scala | 2 +- main/eval/src/mill/eval/EvaluatorImpl.scala | 9 ++- main/src/mill/main/MainModule.scala | 3 +- main/src/mill/main/RunScript.scala | 50 +++++++----- main/src/mill/main/SelectiveExecution.scala | 46 ++++++++--- .../mill/main/SelectiveExecutionModule.scala | 35 ++------- .../src/mill/runner/MillBuildBootstrap.scala | 6 +- testkit/src/mill/testkit/UnitTester.scala | 3 +- 16 files changed, 196 insertions(+), 109 deletions(-) diff --git a/build.mill b/build.mill index 0b891de175a..1b753bdd3f9 100644 --- a/build.mill +++ b/build.mill @@ -489,6 +489,7 @@ trait MillStableScalaModule extends MillPublishScalaModule with Mima { // so we have to put explicit ignores here (https://github.com/lightbend/mima/issues/771) ProblemFilter.exclude[Problem]("mill.eval.ProfileLogger*"), ProblemFilter.exclude[Problem]("mill.eval.GroupEvaluator*"), + ProblemFilter.exclude[Problem]("mill.eval.EvaluatorCore*"), ProblemFilter.exclude[Problem]("mill.eval.Tarjans*"), ProblemFilter.exclude[Problem]("mill.define.Ctx#Impl*"), ProblemFilter.exclude[Problem]("mill.resolve.ResolveNotFoundHandler*"), diff --git a/example/fundamentals/out-dir/1-out-files/build.mill b/example/fundamentals/out-dir/1-out-files/build.mill index 90516cb1971..9c119e2eecf 100644 --- a/example/fundamentals/out-dir/1-out-files/build.mill +++ b/example/fundamentals/out-dir/1-out-files/build.mill @@ -227,12 +227,49 @@ out/mill-server // Again, if there are multiple paths through which one task was invalidated by another, // one path is chosen arbitrarily to be shown in the spanning tree // +// == `methodCodeHashSignatures spanningInvalidationTree` +// // Sometimes invalidation can be caused by a code change in your `build.mill`/`package.mill` // files, rather than by a change in the project's source files or inputs. In such cases, // the root tasks in `mill-invalidation-tree.json` may not necessarily be inputs. In such -// cases, you can look at `out/mill-build/methodCodeHashSignatures.dest/current/spanningInvalidationForest.json` +// cases, you can look at `out/mill-build/methodCodeHashSignatures.dest/current/spanningInvalidationTree.json` // to see an invalidation tree for how code changes in specfic methods propagate throughout // the `build.mill` codebase. + +/** Usage + +> sed -i.bak 's/{}/{println(123)}/g' build.mill + +> ./mill foo.compile # compile after changing build.mill + +> cat out/mill-build/methodCodeHashSignatures.dest/current/spanningInvalidationTree.json +{ + "call scala.runtime.BoxesRunTime.boxToInteger(int)java.lang.Integer": {}, + "call scala.Predef$#println(java.lang.Object)void": { + "def build_.package_$foo$#(build_.package_)void": { + "call build_.package_$foo$!(build_.package_)void": { + "def build_.package_#foo$lzycompute$1()void": { + "call build_.package_!foo$lzycompute$1()void": { + "def build_.package_#foo()build_.package_$foo$": {} + } + } + } + } + } +} +*/ + +// In the `spanningInvalidationTree.json` above, we can see how to addition of the call +// to `scala.Predef.println` caused the `` constructor method of `build_.package_.foo` +// to invalidation, and ends up invalidating `def build_.package_#foo()` which is the method +// representing the `build.foo` task that will thus need to be re-evaluated. +// +// Mill's code-change invalidation analysis is _approximate_ and _conservative_. That means +// that it invalidates each task when any method it calls (transitively) is changed. This may +// sometimes invalidate _too many_ tasks, but it generally does not invalidate _too few_ tasks, +// except in code using Java Reflection or similar techniques which the code-change analysis +// does not understand. +// // // === `mill-build/` // diff --git a/integration/invalidation/selective-execution/resources/build.mill b/integration/invalidation/selective-execution/resources/build.mill index 36866986943..4b0bf9d6068 100644 --- a/integration/invalidation/selective-execution/resources/build.mill +++ b/integration/invalidation/selective-execution/resources/build.mill @@ -14,7 +14,8 @@ object foo extends Module { } object bar extends mill.define.TaskModule { - def barTask = Task.Source(millSourcePath / "bar.txt") + // make sure it works with private tasks as well + private def barTask = Task.Source(millSourcePath / "bar.txt") def barHelper(p: os.Path) = { "barHelper " + os.read(p) diff --git a/integration/invalidation/selective-execution/src/SelectiveExecutionTests.scala b/integration/invalidation/selective-execution/src/SelectiveExecutionTests.scala index 1e59861bda6..53753adfac0 100644 --- a/integration/invalidation/selective-execution/src/SelectiveExecutionTests.scala +++ b/integration/invalidation/selective-execution/src/SelectiveExecutionTests.scala @@ -14,7 +14,11 @@ object SelectiveExecutionTests extends UtestIntegrationTestSuite { test("changed-inputs") - integrationTest { tester => import tester._ - eval(("selective.prepare", "{foo.fooCommand,bar.barCommand}"), check = true) + eval( + ("selective.prepare", "{foo.fooCommand,bar.barCommand}"), + check = true, + stderr = os.Inherit + ) // no op val noOp = eval( @@ -151,14 +155,21 @@ object SelectiveExecutionTests extends UtestIntegrationTestSuite { eventually( output.contains("Computing fooCommand") && output.contains("Computing barCommand") ) + + // Make sure editing each individual input results in the corresponding downstream + // command being re-run, and watches on both are maintained even if in a prior run + // one set of tasks was ignored. output0 = Nil modifyFile(workspacePath / "bar/bar.txt", _ + "!") - eventually( - !output.contains("Computing fooCommand") && output.contains("Computing barCommand") - ) - eventually( + eventually { !output.contains("Computing fooCommand") && output.contains("Computing barCommand") - ) + } + + output0 = Nil + modifyFile(workspacePath / "foo/foo.txt", _ + "!") + eventually { + output.contains("Computing fooCommand") && !output.contains("Computing barCommand") + } } test("show-changed-inputs") - integrationTest { tester => import tester._ @@ -168,24 +179,26 @@ object SelectiveExecutionTests extends UtestIntegrationTestSuite { eval( ("--watch", "show", "{foo.fooCommand,bar.barCommand}"), check = true, - stderr = os.ProcessOutput.Readlines(line => output0 = output0 :+ line) + stderr = os.ProcessOutput.Readlines(line => output0 = output0 :+ line), + stdout = os.ProcessOutput.Readlines(line => output0 = output0 :+ line) ) } - eventually( + eventually { output.contains("Computing fooCommand") && output.contains("Computing barCommand") - ) + } output0 = Nil modifyFile(workspacePath / "bar/bar.txt", _ + "!") - // For now, selective execution doesn't work with `show`, and always runs all provided - // tasks. This is necessary because we need all specified tasks to be run in order to - // get their value to render as JSON at the end of `show` - eventually( - output.contains("Computing fooCommand") && output.contains("Computing barCommand") - ) - eventually( - output.contains("Computing fooCommand") && output.contains("Computing barCommand") - ) + + eventually { + !output.contains("Computing fooCommand") && output.contains("Computing barCommand") + } + + output0 = Nil + modifyFile(workspacePath / "foo/foo.txt", _ + "!") + eventually { + output.contains("Computing fooCommand") && !output.contains("Computing barCommand") + } } test("changed-code") - integrationTest { tester => @@ -197,22 +210,22 @@ object SelectiveExecutionTests extends UtestIntegrationTestSuite { eval( ("--watch", "{foo.fooCommand,bar.barCommand}"), check = true, - stdout = os.ProcessOutput.Readlines(line => output0 = output0 :+ line), + stdout = os.ProcessOutput.Readlines { line => output0 = output0 :+ line }, stderr = os.Inherit ) } - eventually( + eventually { output.contains("Computing fooCommand") && output.contains("Computing barCommand") - ) + } output0 = Nil // Check method body code changes correctly trigger downstream evaluation modifyFile(workspacePath / "build.mill", _.replace("\"barHelper \"", "\"barHelper! \"")) - eventually( + eventually { !output.contains("Computing fooCommand") && output.contains("Computing barCommand") - ) + } output0 = Nil // Check module body code changes correctly trigger downstream evaluation @@ -221,9 +234,9 @@ object SelectiveExecutionTests extends UtestIntegrationTestSuite { _.replace("object foo extends Module {", "object foo extends Module { println(123)") ) - eventually( + eventually { output.contains("Computing fooCommand") && !output.contains("Computing barCommand") - ) + } } } test("failures") { @@ -239,5 +252,19 @@ object SelectiveExecutionTests extends UtestIntegrationTestSuite { assert(cached.err.contains("`selective.run` can only be run after `selective.prepare`")) } } + test("renamed-tasks") - integrationTest { tester => + import tester._ + eval(("selective.prepare", "{foo,bar}._"), check = true) + + modifyFile(workspacePath / "build.mill", _.replace("fooTask", "fooTaskRenamed")) + modifyFile(workspacePath / "build.mill", _.replace("barCommand", "barCommandRenamed")) + + val cached = eval(("selective.resolve", "{foo,bar}._"), stderr = os.Pipe) + + assert( + cached.out.linesIterator.toList == + Seq("bar.barCommandRenamed", "foo.fooCommand", "foo.fooTaskRenamed") + ) + } } } diff --git a/integration/invalidation/watch-source-input/src/WatchSourceInputTests.scala b/integration/invalidation/watch-source-input/src/WatchSourceInputTests.scala index 60076bc07e4..c752783d3c2 100644 --- a/integration/invalidation/watch-source-input/src/WatchSourceInputTests.scala +++ b/integration/invalidation/watch-source-input/src/WatchSourceInputTests.scala @@ -43,8 +43,8 @@ object WatchSourceInputTests extends UtestIntegrationTestSuite { // Most of these are normal `println`s, so they go to `stdout` by // default unless you use `show` in which case they go to `stderr`. val expectedErr = if (show) mutable.Buffer.empty[String] else expectedOut - val expectedShows = mutable.Buffer.empty[String] - val res = f(expectedOut, expectedErr, expectedShows) + val expectedShows0 = mutable.Buffer.empty[String] + val res = f(expectedOut, expectedErr, expectedShows0) val (shows, out) = res.out.linesIterator.toVector.partition(_.startsWith("\"")) val err = res.err.linesIterator.toVector .filter(!_.contains("Compiling compiler interface...")) @@ -59,7 +59,8 @@ object WatchSourceInputTests extends UtestIntegrationTestSuite { if (show) assert(err == expectedErr) else assert(err.isEmpty) - if (show) assert(shows == expectedShows.map('"' + _ + '"')) + val expectedShows = expectedShows0.map('"' + _ + '"') + if (show) assert(shows == expectedShows) } def testWatchSource(tester: IntegrationTester, show: Boolean) = @@ -120,10 +121,8 @@ object WatchSourceInputTests extends UtestIntegrationTestSuite { // "Running qux foo contents edited-foo1 edited-foo2", // "Running qux bar contents edited-bar" ) - expectedShows.append( - "Running qux foo contents edited-foo1 edited-foo2 Running qux bar contents edited-bar" - ) + if (show) expectedOut.append("{}") os.write.over(workspacePath / "watchValue.txt", "exit") awaitCompletionMarker(tester, "initialized2") expectedOut.append("Setting up build.mill") @@ -174,10 +173,10 @@ object WatchSourceInputTests extends UtestIntegrationTestSuite { os.write.over(workspacePath / "watchValue.txt", "edited-watchValue") awaitCompletionMarker(tester, "initialized1") expectedOut.append("Setting up build.mill") - expectedShows.append("Running lol baz contents edited-baz") os.write.over(workspacePath / "watchValue.txt", "exit") awaitCompletionMarker(tester, "initialized2") + if (show) expectedOut.append("{}") expectedOut.append("Setting up build.mill") Await.result(evalResult, Duration.apply(maxDuration, SECONDS)) diff --git a/main/codesig/src/Logger.scala b/main/codesig/src/Logger.scala index b30de439433..669f60ceaf2 100644 --- a/main/codesig/src/Logger.scala +++ b/main/codesig/src/Logger.scala @@ -12,7 +12,7 @@ class Logger(mandatoryLogFolder: os.Path, logFolder: Option[os.Path]) { ): Unit = { os.write( p / s"$prefix${res.source}.json", - upickle.default.stream(res.value, indent = 4), + upickle.default.stream(res.value, indent = 2), createFolders = true ) count += 1 diff --git a/main/codesig/src/ReachabilityAnalysis.scala b/main/codesig/src/ReachabilityAnalysis.scala index f58c2fcf6ea..f307c8d4ecd 100644 --- a/main/codesig/src/ReachabilityAnalysis.scala +++ b/main/codesig/src/ReachabilityAnalysis.scala @@ -80,9 +80,9 @@ class CallGraphAnalysis( logger.mandatoryLog(transitiveCallGraphHashes0) logger.log(transitiveCallGraphHashes) - lazy val spanningInvalidationForest: Obj = prevTransitiveCallGraphHashesOpt() match { + lazy val spanningInvalidationTree: Obj = prevTransitiveCallGraphHashesOpt() match { case Some(prevTransitiveCallGraphHashes) => - CallGraphAnalysis.spanningInvalidationForest( + CallGraphAnalysis.spanningInvalidationTree( prevTransitiveCallGraphHashes, transitiveCallGraphHashes0, indexToNodes, @@ -91,7 +91,7 @@ class CallGraphAnalysis( case None => ujson.Obj() } - logger.mandatoryLog(spanningInvalidationForest) + logger.mandatoryLog(spanningInvalidationTree) } object CallGraphAnalysis { @@ -109,7 +109,7 @@ object CallGraphAnalysis { * typically are investigating why there's a path to a node at all where none * should exist, rather than trying to fully analyse all possible paths */ - def spanningInvalidationForest( + def spanningInvalidationTree( prevTransitiveCallGraphHashes: Map[String, Int], transitiveCallGraphHashes0: Array[(CallGraphAnalysis.Node, Int)], indexToNodes: Array[Node], diff --git a/main/codesig/test/src/CallGraphTests.scala b/main/codesig/test/src/CallGraphTests.scala index 95f1d8ebe6c..0b24a32beea 100644 --- a/main/codesig/test/src/CallGraphTests.scala +++ b/main/codesig/test/src/CallGraphTests.scala @@ -115,8 +115,8 @@ object CallGraphTests extends TestSuite { val foundCallGraph = simplifyCallGraph(codeSig, skipped) - val expectedCallGraphJson = write(expectedCallGraph, indent = 4) - val foundCallGraphJson = write(foundCallGraph, indent = 4) + val expectedCallGraphJson = write(expectedCallGraph, indent = 2) + val foundCallGraphJson = write(foundCallGraph, indent = 2) assert(expectedCallGraphJson == foundCallGraphJson) foundCallGraphJson @@ -156,8 +156,8 @@ object CallGraphTests extends TestSuite { .to(SortedMap) for (expectedTransitiveGraph <- expectedTransitiveGraphOpt) { - val expectedTransitiveGraphJson = upickle.default.write(expectedTransitiveGraph, indent = 4) - val transitiveGraphJson = upickle.default.write(transitiveGraph, indent = 4) + val expectedTransitiveGraphJson = upickle.default.write(expectedTransitiveGraph, indent = 2) + val transitiveGraphJson = upickle.default.write(transitiveGraph, indent = 2) assert(expectedTransitiveGraphJson == transitiveGraphJson) } } diff --git a/main/eval/src/mill/eval/Evaluator.scala b/main/eval/src/mill/eval/Evaluator.scala index 6eae8393495..22fcbfbdafe 100644 --- a/main/eval/src/mill/eval/Evaluator.scala +++ b/main/eval/src/mill/eval/Evaluator.scala @@ -19,6 +19,7 @@ trait Evaluator extends AutoCloseable { def rootModule: BaseModule def effectiveThreadCount: Int def outPath: os.Path + def selectiveExecution: Boolean = false def externalOutPath: os.Path def pathsResolver: EvaluatorPathsResolver def methodCodeHashSignatures: Map[String, Int] = Map.empty @@ -78,7 +79,6 @@ object Evaluator { def transitive: Agg[Task[_]] def failing: MultiBiMap[Terminal, Result.Failing[Val]] def results: collection.Map[Task[_], TaskResult[Val]] - def values: Seq[Val] = rawValues.collect { case Result.Success(v) => v } } diff --git a/main/eval/src/mill/eval/EvaluatorImpl.scala b/main/eval/src/mill/eval/EvaluatorImpl.scala index 507497cc135..92f3e8ed59d 100644 --- a/main/eval/src/mill/eval/EvaluatorImpl.scala +++ b/main/eval/src/mill/eval/EvaluatorImpl.scala @@ -34,7 +34,8 @@ private[mill] case class EvaluatorImpl( val systemExit: Int => Nothing, val exclusiveSystemStreams: SystemStreams, protected[eval] val chromeProfileLogger: ChromeProfileLogger, - protected[eval] val profileLogger: ProfileLogger + protected[eval] val profileLogger: ProfileLogger, + override val selectiveExecution: Boolean = false ) extends Evaluator with EvaluatorCore { import EvaluatorImpl._ @@ -93,7 +94,8 @@ private[mill] object EvaluatorImpl { disableCallgraph: Boolean, allowPositionalCommandArgs: Boolean, systemExit: Int => Nothing, - exclusiveSystemStreams: SystemStreams + exclusiveSystemStreams: SystemStreams, + selectiveExecution: Boolean ) = new EvaluatorImpl( home, workspace, @@ -114,7 +116,8 @@ private[mill] object EvaluatorImpl { systemExit, exclusiveSystemStreams, chromeProfileLogger = new ChromeProfileLogger(outPath / millChromeProfile), - profileLogger = new ProfileLogger(outPath / millProfile) + profileLogger = new ProfileLogger(outPath / millProfile), + selectiveExecution = selectiveExecution ) class EvalOrThrow(evaluator: Evaluator, exceptionFactory: Evaluator.Results => Throwable) diff --git a/main/src/mill/main/MainModule.scala b/main/src/mill/main/MainModule.scala index ebcad880e70..28c02867665 100644 --- a/main/src/mill/main/MainModule.scala +++ b/main/src/mill/main/MainModule.scala @@ -61,7 +61,8 @@ object MainModule { RunScript.evaluateTasksNamed( evaluator.withBaseLogger(redirectLogger), targets, - Separated + Separated, + selectiveExecution = evaluator.selectiveExecution ) match { case Left(err) => Result.Failure(err) case Right((watched, Left(err))) => diff --git a/main/src/mill/main/RunScript.scala b/main/src/mill/main/RunScript.scala index 4944a320697..67b2048281f 100644 --- a/main/src/mill/main/RunScript.scala +++ b/main/src/mill/main/RunScript.scala @@ -1,7 +1,7 @@ package mill.main import mill.define._ -import mill.eval.{Evaluator, EvaluatorPaths} +import mill.eval.{Evaluator, EvaluatorPaths, Plan} import mill.util.Watchable import mill.api.{PathRef, Result, Val} import mill.api.Strict.Agg @@ -30,8 +30,7 @@ object RunScript { evaluator: Evaluator, scriptArgs: Seq[String], selectMode: SelectMode, - selectiveExecution: Boolean = false, - selectiveExecutionSave: Boolean = false + selectiveExecution: Boolean = false ): Either[ String, (Seq[Watchable], Either[String, Seq[(Any, Option[(TaskName, ujson.Value)])]]) @@ -45,14 +44,14 @@ object RunScript { ) } for (targets <- resolved) - yield evaluateNamed(evaluator, Agg.from(targets), selectiveExecution, selectiveExecutionSave) + yield evaluateNamed(evaluator, Agg.from(targets), selectiveExecution) } def evaluateNamed( evaluator: Evaluator, targets: Agg[NamedTask[Any]] ): (Seq[Watchable], Either[String, Seq[(Any, Option[(TaskName, ujson.Value)])]]) = - evaluateNamed(evaluator, targets, selectiveExecution = false, selectiveExecutionSave = false) + evaluateNamed(evaluator, targets, selectiveExecution = false) /** * @param evaluator @@ -62,28 +61,39 @@ object RunScript { def evaluateNamed( evaluator: Evaluator, targets: Agg[NamedTask[Any]], - selectiveExecution: Boolean = false, - selectiveExecutionSave: Boolean = false + selectiveExecution: Boolean = false ): (Seq[Watchable], Either[String, Seq[(Any, Option[(TaskName, ujson.Value)])]]) = { + val (sortedGroups, transitive) = Plan.plan(targets) + val terminals = sortedGroups.keys.map(t => (t.task, t)).toMap + val selectiveExecutionEnabled = selectiveExecution && targets + .collectFirst { case c: Command[_] if c.exclusive => true } + .isEmpty + val selectedTargetsOrErr = - if (selectiveExecution && os.exists(evaluator.outPath / OutFiles.millSelectiveExecution)) { + if ( + selectiveExecutionEnabled && os.exists(evaluator.outPath / OutFiles.millSelectiveExecution) + ) { SelectiveExecution - .diffMetadata(evaluator, targets.map(_.ctx.segments.render).toSeq) - .map { selected => - targets.filter { - case c: Command[_] if c.exclusive => true - case t => selected(t.ctx.segments.render) - } + .diffMetadata(evaluator, targets.map(terminals(_).render).toSeq) + .map { x => + val (selected, results) = x + val selectedSet = selected.toSet + ( + targets.filter { + case c: Command[_] if c.exclusive => true + case t => selectedSet(terminals(t).render) + }, + results + ) } - } else Right(targets) + } else Right(targets -> Map.empty) selectedTargetsOrErr match { case Left(err) => (Nil, Left(err)) - case Right(selectedTargets) => + case Right((selectedTargets, selectiveResults)) => val evaluated: Results = evaluator.evaluate(selectedTargets, serialCommandExec = true) - val watched = evaluated.results - .iterator + val watched = (evaluated.results.iterator ++ selectiveResults) .collect { case (t: SourcesImpl, TaskResult(Result.Success(Val(ps: Seq[PathRef])), _)) => ps.map(Watchable.Path(_)) @@ -100,11 +110,11 @@ object RunScript { .iterator .collect { case (t: InputImpl[_], TaskResult(Result.Success(Val(value)), _)) => - (t.ctx.segments.render, value.##) + (terminals(t).render, value.##) } .toMap - if (selectiveExecutionSave) { + if (selectiveExecutionEnabled) { SelectiveExecution.saveMetadata( evaluator, SelectiveExecution.Metadata(allInputHashes, evaluator.methodCodeHashSignatures) diff --git a/main/src/mill/main/SelectiveExecution.scala b/main/src/mill/main/SelectiveExecution.scala index 36d4783cb3a..3daa3ab1389 100644 --- a/main/src/mill/main/SelectiveExecution.scala +++ b/main/src/mill/main/SelectiveExecution.scala @@ -1,6 +1,6 @@ package mill.main -import mill.api.Strict +import mill.api.{Strict, Val} import mill.define.{InputImpl, NamedTask, Task} import mill.eval.{CodeSigUtils, Evaluator, Plan, Terminal} import mill.main.client.OutFiles @@ -12,7 +12,10 @@ private[mill] object SelectiveExecution { implicit val rw: upickle.default.ReadWriter[Metadata] = upickle.default.macroRW object Metadata { - def compute(evaluator: Evaluator, tasks: Seq[String]): Either[String, Metadata] = { + def compute( + evaluator: Evaluator, + tasks: Seq[String] + ): Either[String, (Metadata, Map[Task[_], Evaluator.TaskResult[Val]])] = { for (transitive <- plan0(evaluator, tasks)) yield { val inputTasksToLabels: Map[Task[_], String] = transitive .collect { case Terminal.Labelled(task: InputImpl[_], segments) => @@ -32,7 +35,7 @@ private[mill] object SelectiveExecution { } .toMap, methodCodeHashSignatures = evaluator.methodCodeHashSignatures - ) + ) -> results.results.toMap } } } @@ -122,16 +125,41 @@ private[mill] object SelectiveExecution { ) } - def diffMetadata(evaluator: Evaluator, tasks: Seq[String]): Either[String, Set[String]] = { + def diffMetadata( + evaluator: Evaluator, + tasks: Seq[String] + ): Either[String, (Seq[String], Map[Task[_], Evaluator.TaskResult[Val]])] = { val oldMetadataTxt = os.read(evaluator.outPath / OutFiles.millSelectiveExecution) - if (oldMetadataTxt == "") Right(tasks.toSet) - else { + if (oldMetadataTxt == "") { + Resolve.Segments.resolve( + evaluator.rootModule, + tasks, + SelectMode.Separated, + evaluator.allowPositionalCommandArgs + ).map(_.map(_.render) -> Map.empty) + } else { val oldMetadata = upickle.default.read[SelectiveExecution.Metadata](oldMetadataTxt) - for (newMetadata <- SelectiveExecution.Metadata.compute(evaluator, tasks)) yield { + for (x <- SelectiveExecution.Metadata.compute(evaluator, tasks)) yield { + val (newMetadata, results) = x SelectiveExecution.computeDownstream(evaluator, tasks, oldMetadata, newMetadata) - .collect { case n: NamedTask[_] => n.ctx.segments.render } - .toSet + .collect { case n: NamedTask[_] => n.ctx.segments.render } -> results + } } } + + def resolve0(evaluator: Evaluator, tasks: Seq[String]): Either[String, Array[String]] = { + for { + resolved <- Resolve.Tasks.resolve(evaluator.rootModule, tasks, SelectMode.Separated) + x <- SelectiveExecution.diffMetadata(evaluator, tasks) + } yield { + val (newTasks, results) = x + resolved + .map(_.ctx.segments.render) + .toSet + .intersect(newTasks.toSet) + .toArray + .sorted + } + } } diff --git a/main/src/mill/main/SelectiveExecutionModule.scala b/main/src/mill/main/SelectiveExecutionModule.scala index e39abfe100f..96a75952747 100644 --- a/main/src/mill/main/SelectiveExecutionModule.scala +++ b/main/src/mill/main/SelectiveExecutionModule.scala @@ -4,8 +4,7 @@ import mill.api.Result import mill.define.{Command, Task} import mill.eval.Evaluator import mill.main.client.OutFiles -import mill.resolve.{Resolve, SelectMode} -import mill.resolve.SelectMode.Separated +import mill.resolve.SelectMode trait SelectiveExecutionModule extends mill.define.Module { @@ -18,7 +17,7 @@ trait SelectiveExecutionModule extends mill.define.Module { Task.Command(exclusive = true) { val res: Either[String, Unit] = SelectiveExecution.Metadata .compute(evaluator, if (tasks.isEmpty) Seq("__") else tasks) - .map(SelectiveExecution.saveMetadata(evaluator, _)) + .map(x => SelectiveExecution.saveMetadata(evaluator, x._1)) res match { case Left(err) => Result.Failure(err) @@ -34,25 +33,7 @@ trait SelectiveExecutionModule extends mill.define.Module { */ def resolve(evaluator: Evaluator, tasks: String*): Command[Array[String]] = Task.Command(exclusive = true) { - val result = for { - resolved <- Resolve.Tasks.resolve(evaluator.rootModule, tasks, SelectMode.Multi) - diffed <- SelectiveExecution.diffMetadata(evaluator, tasks) - resolvedDiffed <- { - if (diffed.isEmpty) Right(Nil) - else Resolve.Segments.resolve( - evaluator.rootModule, - diffed.toSeq, - SelectMode.Multi, - evaluator.allowPositionalCommandArgs - ) - } - } yield { - resolved.map( - _.ctx.segments.render - ).toSet.intersect(resolvedDiffed.map(_.render).toSet).toArray.sorted - } - - result match { + SelectiveExecution.resolve0(evaluator, tasks) match { case Left(err) => Result.Failure(err) case Right(success) => success.foreach(println) @@ -70,12 +51,10 @@ trait SelectiveExecutionModule extends mill.define.Module { if (!os.exists(evaluator.outPath / OutFiles.millSelectiveExecution)) { Result.Failure("`selective.run` can only be run after `selective.prepare`") } else { - RunScript.evaluateTasksNamed( - evaluator, - tasks, - Separated, - selectiveExecution = true - ) match { + SelectiveExecution.resolve0(evaluator, tasks).flatMap { resolved => + if (resolved.isEmpty) Right((Nil, Right(Nil))) + else RunScript.evaluateTasksNamed(evaluator, resolved, SelectMode.Multi) + } match { case Left(err) => Result.Failure(err) case Right((watched, Left(err))) => Result.Failure(err) case Right((watched, Right(res))) => Result.Success(res) diff --git a/runner/src/mill/runner/MillBuildBootstrap.scala b/runner/src/mill/runner/MillBuildBootstrap.scala index 8ffa43bdf49..79735a66b74 100644 --- a/runner/src/mill/runner/MillBuildBootstrap.scala +++ b/runner/src/mill/runner/MillBuildBootstrap.scala @@ -353,7 +353,8 @@ class MillBuildBootstrap( disableCallgraph = disableCallgraph, allowPositionalCommandArgs = allowPositionalCommandArgs, systemExit = systemExit, - exclusiveSystemStreams = streams0 + exclusiveSystemStreams = streams0, + selectiveExecution = selectiveExecution ) } @@ -406,8 +407,7 @@ object MillBuildBootstrap { evaluator, targetsAndParams, SelectMode.Separated, - selectiveExecution = selectiveExecution, - selectiveExecutionSave = selectiveExecution + selectiveExecution = selectiveExecution ) } finally Thread.currentThread().setContextClassLoader(previousClassloader) diff --git a/testkit/src/mill/testkit/UnitTester.scala b/testkit/src/mill/testkit/UnitTester.scala index 5303b9c8804..d8e9cf791bc 100644 --- a/testkit/src/mill/testkit/UnitTester.scala +++ b/testkit/src/mill/testkit/UnitTester.scala @@ -102,7 +102,8 @@ class UnitTester( disableCallgraph = false, allowPositionalCommandArgs = false, systemExit = i => ???, - exclusiveSystemStreams = new SystemStreams(outStream, errStream, inStream) + exclusiveSystemStreams = new SystemStreams(outStream, errStream, inStream), + selectiveExecution = false ) def apply(args: String*): Either[Result.Failing[_], UnitTester.Result[Seq[_]]] = {