diff --git a/build.mill b/build.mill index 1b753bdd3f9..307653d4782 100644 --- a/build.mill +++ b/build.mill @@ -488,6 +488,7 @@ trait MillStableScalaModule extends MillPublishScalaModule with Mima { // (5x) MIMA doesn't properly ignore things which are nested inside other private things // so we have to put explicit ignores here (https://github.com/lightbend/mima/issues/771) ProblemFilter.exclude[Problem]("mill.eval.ProfileLogger*"), + ProblemFilter.exclude[Problem]("mill.eval.ChromeProfileLogger*"), ProblemFilter.exclude[Problem]("mill.eval.GroupEvaluator*"), ProblemFilter.exclude[Problem]("mill.eval.EvaluatorCore*"), ProblemFilter.exclude[Problem]("mill.eval.Tarjans*"), diff --git a/main/define/src/mill/define/Task.scala b/main/define/src/mill/define/Task.scala index 21f3faa7412..63bc772111d 100644 --- a/main/define/src/mill/define/Task.scala +++ b/main/define/src/mill/define/Task.scala @@ -41,6 +41,10 @@ abstract class Task[+T] extends Task.Ops[T] with Applyable[Task, T] { def asCommand: Option[Command[T]] = None def asWorker: Option[Worker[T]] = None def self: Task[T] = this + def isExclusiveCommand: Boolean = this match { + case c: Command[_] if c.exclusive => true + case _ => false + } } object Task extends TaskBase { diff --git a/main/eval/src/mill/eval/EvaluatorCore.scala b/main/eval/src/mill/eval/EvaluatorCore.scala index 95b7c47826a..1d220fbbe03 100644 --- a/main/eval/src/mill/eval/EvaluatorCore.scala +++ b/main/eval/src/mill/eval/EvaluatorCore.scala @@ -5,13 +5,12 @@ import mill.api.Strict.Agg import mill.api._ import mill.define._ import mill.eval.Evaluator.TaskResult -import mill.main.client.OutFiles import mill.util._ +import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger} import scala.collection.mutable import scala.concurrent._ -import scala.jdk.CollectionConverters.EnumerationHasAsScala /** * Core logic of evaluating tasks, without any user-facing helper methods @@ -79,17 +78,8 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { val count = new AtomicInteger(1) val indexToTerminal = sortedGroups.keys().toArray val terminalToIndex = indexToTerminal.zipWithIndex.toMap - val upstreamIndexEdges = - indexToTerminal.map(t => interGroupDeps.getOrElse(t, Nil).map(terminalToIndex).toArray) - os.write.over( - outPath / OutFiles.millDependencyTree, - SpanningForest.spanningTreeToJsonTree( - SpanningForest(upstreamIndexEdges, indexToTerminal.indices.toSet, true), - i => indexToTerminal(i).render - ).render(indent = 2) - ) - val futures = mutable.Map.empty[Terminal, Future[Option[GroupEvaluator.Results]]] + EvaluatorLogs.logDependencyTree(interGroupDeps, indexToTerminal, terminalToIndex, outPath) // Prepare a lookup tables up front of all the method names that each class owns, // and the class hierarchy, so during evaluation it is cheap to look up what class @@ -97,8 +87,10 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { val (classToTransitiveClasses, allTransitiveClassMethods) = CodeSigUtils.precomputeMethodNamesPerClass(sortedGroups) - val uncached = new java.util.concurrent.ConcurrentHashMap[Terminal, Unit]() - val changedValueHash = new java.util.concurrent.ConcurrentHashMap[Terminal, Unit]() + val uncached = new ConcurrentHashMap[Terminal, Unit]() + val changedValueHash = new ConcurrentHashMap[Terminal, Unit]() + + val futures = mutable.Map.empty[Terminal, Future[Option[GroupEvaluator.Results]]] def evaluateTerminals( terminals: Seq[Terminal], @@ -114,21 +106,19 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { // due to the topological order of traversal. for (terminal <- terminals) { val deps = interGroupDeps(terminal) - def isExclusiveCommand(t: Task[_]) = t match { - case c: Command[_] if c.exclusive => true - case _ => false - } val group = sortedGroups.lookupKey(terminal) - val exclusiveDeps = deps.filter(d => isExclusiveCommand(d.task)) + val exclusiveDeps = deps.filter(d => d.task.isExclusiveCommand) - if (!isExclusiveCommand(terminal.task) && exclusiveDeps.nonEmpty) { + if (!terminal.task.isExclusiveCommand && exclusiveDeps.nonEmpty) { val failure = Result.Failure( s"Non-exclusive task ${terminal.render} cannot depend on exclusive task " + exclusiveDeps.map(_.render).mkString(", ") ) - val taskResults = - group.map(t => (t, TaskResult[(Val, Int)](failure, () => failure))).toMap + val taskResults = group + .map(t => (t, TaskResult[(Val, Int)](failure, () => failure))) + .toMap + futures(terminal) = Future.successful( Some(GroupEvaluator.Results(taskResults, group.toSeq, false, -1, -1, false)) ) @@ -150,19 +140,13 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { .toMap val startTime = System.nanoTime() / 1000 - val targetLabel = terminal match { - case Terminal.Task(task) => None - case t: Terminal.Labelled[_] => Some(Terminal.printTerm(t)) - } // should we log progress? - val logRun = targetLabel.isDefined && { - val inputResults = for { - target <- group.indexed.filterNot(upstreamResults.contains) - item <- target.inputs.filterNot(group.contains) - } yield upstreamResults(item).map(_._1) - inputResults.forall(_.result.isInstanceOf[Result.Success[_]]) - } + val inputResults = for { + target <- group.indexed.filterNot(upstreamResults.contains) + item <- target.inputs.filterNot(group.contains) + } yield upstreamResults(item).map(_._1) + val logRun = inputResults.forall(_.result.isInstanceOf[Result.Success[_]]) val tickerPrefix = terminal.render.collect { case targetLabel if logRun && logger.enableTicker => targetLabel @@ -195,31 +179,15 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { failed.set(true) val endTime = System.nanoTime() / 1000 - val duration = endTime - startTime - chromeProfileLogger.log( - task = Terminal.printTerm(terminal), - cat = "job", - startTime = startTime, - duration = duration, - threadId = threadNumberer.getThreadId(Thread.currentThread()), - cached = res.cached - ) + val threadId = threadNumberer.getThreadId(Thread.currentThread()) + chromeProfileLogger.log(terminal, "job", startTime, duration, threadId, res.cached) + if (!res.cached) uncached.put(terminal, ()) if (res.valueHashChanged) changedValueHash.put(terminal, ()) - profileLogger.log( - ProfileLogger.Timing( - terminal.render, - (duration / 1000).toInt, - res.cached, - res.valueHashChanged, - deps.map(_.render), - res.inputsHash, - res.previousInputsHash - ) - ) + profileLogger.log(terminal, duration, res, deps) Some(res) } @@ -241,12 +209,7 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { val tasksTransitive = tasksTransitive0.toSet val (tasks, leafExclusiveCommands) = terminals0.partition { - case Terminal.Labelled(t, _) => - if (tasksTransitive.contains(t)) true - else t match { - case t: Command[_] => !t.exclusive - case _ => false - } + case Terminal.Labelled(t, _) => tasksTransitive.contains(t) || !t.isExclusiveCommand case _ => !serialCommandExec } @@ -260,6 +223,15 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { val finishedOptsMap = (nonExclusiveResults ++ exclusiveResults).toMap + EvaluatorLogs.logInvalidationTree( + interGroupDeps, + indexToTerminal, + terminalToIndex, + outPath, + uncached, + changedValueHash + ) + val results0: Vector[(Task[_], TaskResult[(Val, Int)])] = terminals0 .flatMap { t => sortedGroups.lookupKey(t).flatMap { t0 => @@ -272,50 +244,6 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { val results: Map[Task[_], TaskResult[(Val, Int)]] = results0.toMap - val reverseInterGroupDeps = interGroupDeps - .iterator - .flatMap { case (k, vs) => vs.map(_ -> k) } - .toSeq - .groupMap(_._1)(_._2) - - val changedTerminalIndices = changedValueHash.keys().asScala.toSet - val downstreamIndexEdges = indexToTerminal - .map(t => - if (changedTerminalIndices(t)) - reverseInterGroupDeps.getOrElse(t, Nil).map(terminalToIndex).toArray - else Array.empty[Int] - ) - - val edgeSourceIndices = downstreamIndexEdges - .zipWithIndex - .collect { case (es, i) if es.nonEmpty => i } - .toSet - - os.write.over( - outPath / OutFiles.millInvalidationTree, - SpanningForest.spanningTreeToJsonTree( - SpanningForest( - downstreamIndexEdges, - uncached.keys().asScala - .flatMap { uncachedTask => - val uncachedIndex = terminalToIndex(uncachedTask) - Option.when( - // Filter out input and source tasks which do not cause downstream invalidations - // from the invalidation tree, because most of them are un-interesting and the - // user really only cares about (a) inputs that cause downstream tasks to invalidate - // or (b) non-input tasks that were invalidated alone (e.g. due to a codesig change) - !uncachedTask.task.isInstanceOf[InputImpl[_]] || edgeSourceIndices(uncachedIndex) - ) { - uncachedIndex - } - } - .toSet, - true - ), - i => indexToTerminal(i).render - ).render(indent = 2) - ) - EvaluatorCore.Results( goals.indexed.map(results(_).map(_._1).result), // result of flatMap may contain non-distinct entries, diff --git a/main/eval/src/mill/eval/EvaluatorLogs.scala b/main/eval/src/mill/eval/EvaluatorLogs.scala new file mode 100644 index 00000000000..830bae9f37a --- /dev/null +++ b/main/eval/src/mill/eval/EvaluatorLogs.scala @@ -0,0 +1,71 @@ +package mill.eval + +import mill.define.InputImpl +import mill.main.client.OutFiles +import mill.util.SpanningForest +import java.util.concurrent.ConcurrentHashMap +import scala.jdk.CollectionConverters.EnumerationHasAsScala + +private[mill] object EvaluatorLogs { + def logDependencyTree( + interGroupDeps: Map[Terminal, Seq[Terminal]], + indexToTerminal: Array[Terminal], + terminalToIndex: Map[Terminal, Int], + outPath: os.Path + ): Unit = { + SpanningForest.writeJsonFile( + outPath / OutFiles.millDependencyTree, + indexToTerminal.map(t => interGroupDeps.getOrElse(t, Nil).map(terminalToIndex).toArray), + indexToTerminal.indices.toSet, + indexToTerminal(_).render + ) + } + def logInvalidationTree( + interGroupDeps: Map[Terminal, Seq[Terminal]], + indexToTerminal: Array[Terminal], + terminalToIndex: Map[Terminal, Int], + outPath: os.Path, + uncached: ConcurrentHashMap[Terminal, Unit], + changedValueHash: ConcurrentHashMap[Terminal, Unit] + ): Unit = { + + val reverseInterGroupDeps = interGroupDeps + .iterator + .flatMap { case (k, vs) => vs.map(_ -> k) } + .toSeq + .groupMap(_._1)(_._2) + + val changedTerminalIndices = changedValueHash.keys().asScala.toSet + val downstreamIndexEdges = indexToTerminal + .map(t => + if (changedTerminalIndices(t)) + reverseInterGroupDeps.getOrElse(t, Nil).map(terminalToIndex).toArray + else Array.empty[Int] + ) + + val edgeSourceIndices = downstreamIndexEdges + .zipWithIndex + .collect { case (es, i) if es.nonEmpty => i } + .toSet + + SpanningForest.writeJsonFile( + outPath / OutFiles.millInvalidationTree, + downstreamIndexEdges, + uncached.keys().asScala + .flatMap { uncachedTask => + val uncachedIndex = terminalToIndex(uncachedTask) + Option.when( + // Filter out input and source tasks which do not cause downstream invalidations + // from the invalidation tree, because most of them are un-interesting and the + // user really only cares about (a) inputs that cause downstream tasks to invalidate + // or (b) non-input tasks that were invalidated alone (e.g. due to a codesig change) + !uncachedTask.task.isInstanceOf[InputImpl[_]] || edgeSourceIndices(uncachedIndex) + ) { + uncachedIndex + } + } + .toSet, + indexToTerminal(_).render + ) + } +} diff --git a/main/eval/src/mill/eval/GroupEvaluator.scala b/main/eval/src/mill/eval/GroupEvaluator.scala index ec8a064324a..99671fdd9ae 100644 --- a/main/eval/src/mill/eval/GroupEvaluator.scala +++ b/main/eval/src/mill/eval/GroupEvaluator.scala @@ -55,12 +55,6 @@ private[mill] trait GroupEvaluator { executionContext: mill.api.Ctx.Fork.Api, exclusive: Boolean ): GroupEvaluator.Results = { - - val targetLabel = terminal match { - case Terminal.Task(task) => None - case t: Terminal.Labelled[_] => Some(Terminal.printTerm(t)) - } - logger.withPrompt { val externalInputsHash = MurmurHash3.orderedHash( group.items.flatMap(_.inputs).filter(!group.contains(_)) @@ -71,19 +65,20 @@ private[mill] trait GroupEvaluator { val scriptsHash = if (disableCallgraph) 0 - else group - .iterator - .collect { case namedTask: NamedTask[_] => - CodeSigUtils.codeSigForTask( - namedTask, - classToTransitiveClasses, - allTransitiveClassMethods, - methodCodeHashSignatures, - constructorHashSignatures - ) - } - .flatten - .sum + else MurmurHash3.orderedHash( + group + .iterator + .collect { case namedTask: NamedTask[_] => + CodeSigUtils.codeSigForTask( + namedTask, + classToTransitiveClasses, + allTransitiveClassMethods, + methodCodeHashSignatures, + constructorHashSignatures + ) + } + .flatten + ) val inputsHash = externalInputsHash + sideHashes + classLoaderSigHash + scriptsHash @@ -113,25 +108,20 @@ private[mill] trait GroupEvaluator { ) case labelled: Terminal.Labelled[_] => - val out = - if (!labelled.task.ctx.external) outPath - else externalOutPath - + val out = if (!labelled.task.ctx.external) outPath else externalOutPath val paths = EvaluatorPaths.resolveDestPaths(out, Terminal.destSegments(labelled)) - val cached = loadCachedJson(logger, inputsHash, labelled, paths) - val upToDateWorker = loadUpToDateWorker( - logger, - inputsHash, - labelled, - // worker metadata file removed by user, let's recompute the worker - forceDiscard = cached.isEmpty - ) + // `cached.isEmpty` means worker metadata file removed by user so recompute the worker + val upToDateWorker = loadUpToDateWorker(logger, inputsHash, labelled, cached.isEmpty) + + val cachedValueAndHash = + upToDateWorker.map((_, inputsHash)) + .orElse(cached.flatMap { case (inputHash, valOpt, valueHash) => + valOpt.map((_, valueHash)) + }) - upToDateWorker.map((_, inputsHash)) orElse cached.flatMap { - case (inputHash, valOpt, valueHash) => valOpt.map((_, valueHash)) - } match { + cachedValueAndHash match { case Some((v, hashCode)) => val res = Result.Success((v, hashCode)) val newResults: Map[Task[_], TaskResult[(Val, Int)]] = @@ -156,7 +146,7 @@ private[mill] trait GroupEvaluator { results, inputsHash, paths = Some(paths), - maybeTargetLabel = targetLabel, + maybeTargetLabel = Some(terminal.render), counterMsg = countMsg, verboseKeySuffix = verboseKeySuffix, zincProblemReporter, @@ -168,11 +158,12 @@ private[mill] trait GroupEvaluator { val valueHash = newResults(labelled.task) match { case TaskResult(Result.Failure(_, Some((v, _))), _) => - val valueHash = if (terminal.task.asWorker.isEmpty) v.## else inputsHash + val valueHash = getValueHash(v, terminal.task, inputsHash) handleTaskResult(v, valueHash, paths.meta, inputsHash, labelled) + valueHash case TaskResult(Result.Success((v, _)), _) => - val valueHash = if (terminal.task.asWorker.isEmpty) v.## else inputsHash + val valueHash = getValueHash(v, terminal.task, inputsHash) handleTaskResult(v, valueHash, paths.meta, inputsHash, labelled) valueHash @@ -218,7 +209,6 @@ private[mill] trait GroupEvaluator { val newResults = mutable.Map.empty[Task[_], Result[(Val, Int)]] val nonEvaluatedTargets = group.indexed.filterNot(results.contains) - val multiLogger = resolveLogger(paths.map(_.log), logger) var usedDest = Option.empty[os.Path] @@ -258,17 +248,17 @@ private[mill] trait GroupEvaluator { } def wrap[T](t: => T): T = { - val (streams, destFunc) = if (exclusive) (exclusiveSystemStreams, () => workspace) else (multiLogger.systemStreams, () => makeDest()) os.dynamicPwdFunction.withValue(destFunc) { SystemStreams.withStreams(streams) { - if (exclusive) { + if (!exclusive) t + else { logger.reportKey(Seq(counterMsg)) logger.withPromptPaused { t } - } else t + } } } } @@ -287,13 +277,7 @@ private[mill] trait GroupEvaluator { } } - newResults(task) = for (v <- res) yield { - ( - v, - if (task.isInstanceOf[Worker[_]]) inputsHash - else v.## - ) - } + newResults(task) = for (v <- res) yield (v, getValueHash(v, task, inputsHash)) } multiLogger.close() @@ -421,6 +405,9 @@ private[mill] trait GroupEvaluator { ) } + def getValueHash(v: Val, task: Task[_], inputsHash: Int): Int = { + if (task.isInstanceOf[Worker[_]]) inputsHash else v.## + } private def loadUpToDateWorker( logger: ColorLogger, inputsHash: Int, diff --git a/main/eval/src/mill/eval/JsonArrayLogger.scala b/main/eval/src/mill/eval/JsonArrayLogger.scala index 209c82e4e9d..deee2399018 100644 --- a/main/eval/src/mill/eval/JsonArrayLogger.scala +++ b/main/eval/src/mill/eval/JsonArrayLogger.scala @@ -36,7 +36,26 @@ private class JsonArrayLogger[T: upickle.default.Writer](outPath: os.Path, inden } private[eval] class ProfileLogger(outPath: os.Path) - extends JsonArrayLogger[ProfileLogger.Timing](outPath, indent = 2) + extends JsonArrayLogger[ProfileLogger.Timing](outPath, indent = 2) { + def log( + terminal: Terminal, + duration: Long, + res: GroupEvaluator.Results, + deps: Seq[Terminal] + ): Unit = { + log( + ProfileLogger.Timing( + terminal.render, + (duration / 1000).toInt, + res.cached, + res.valueHashChanged, + deps.map(_.render), + res.inputsHash, + res.previousInputsHash + ) + ) + } +} private object ProfileLogger { case class Timing( @@ -58,7 +77,7 @@ private[eval] class ChromeProfileLogger(outPath: os.Path) extends JsonArrayLogger[ChromeProfileLogger.TraceEvent](outPath, indent = -1) { def log( - task: String, + terminal: Terminal, cat: String, startTime: Long, duration: Long, @@ -67,7 +86,7 @@ private[eval] class ChromeProfileLogger(outPath: os.Path) ): Unit = { val event = ChromeProfileLogger.TraceEvent( - name = task, + name = Terminal.printTerm(terminal), cat = cat, ph = "X", ts = startTime, diff --git a/main/eval/src/mill/eval/Terminal.scala b/main/eval/src/mill/eval/Terminal.scala index 23955c82e05..3618a2ad712 100644 --- a/main/eval/src/mill/eval/Terminal.scala +++ b/main/eval/src/mill/eval/Terminal.scala @@ -1,6 +1,6 @@ package mill.eval -import mill.define.{NamedTask, Segment, Segments} +import mill.define.{NamedTask, Segments} /** * A terminal or terminal target is some important work unit, that in most cases has a name (Right[Labelled]) @@ -28,14 +28,6 @@ object Terminal { } } - def printTerm(term: Terminal): String = term match { - case Terminal.Task(task) => task.toString() - case labelled: Terminal.Labelled[_] => - val Seq(first, rest @ _*) = destSegments(labelled).value - val msgParts = Seq(first.asInstanceOf[Segment.Label].value) ++ rest.map { - case Segment.Label(s) => "." + s - case Segment.Cross(s) => "[" + s.mkString(",") + "]" - } - msgParts.mkString - } + @deprecated("User Terminal#render instead") + def printTerm(term: Terminal): String = term.render } diff --git a/main/src/mill/main/RunScript.scala b/main/src/mill/main/RunScript.scala index 67b2048281f..8fecc11ccb8 100644 --- a/main/src/mill/main/RunScript.scala +++ b/main/src/mill/main/RunScript.scala @@ -66,9 +66,7 @@ object RunScript { val (sortedGroups, transitive) = Plan.plan(targets) val terminals = sortedGroups.keys.map(t => (t.task, t)).toMap - val selectiveExecutionEnabled = selectiveExecution && targets - .collectFirst { case c: Command[_] if c.exclusive => true } - .isEmpty + val selectiveExecutionEnabled = selectiveExecution && !targets.exists(_.isExclusiveCommand) val selectedTargetsOrErr = if ( @@ -80,10 +78,7 @@ object RunScript { val (selected, results) = x val selectedSet = selected.toSet ( - targets.filter { - case c: Command[_] if c.exclusive => true - case t => selectedSet(terminals(t).render) - }, + targets.filter(t => t.isExclusiveCommand || selectedSet(terminals(t).render)), results ) } diff --git a/main/util/src/mill/util/SpanningForest.scala b/main/util/src/mill/util/SpanningForest.scala index 2627d12a0c9..4b062cafa79 100644 --- a/main/util/src/mill/util/SpanningForest.scala +++ b/main/util/src/mill/util/SpanningForest.scala @@ -13,6 +13,20 @@ import scala.collection.mutable * the roots of the forest */ private[mill] object SpanningForest { + def writeJsonFile( + path: os.Path, + indexEdges: Array[Array[Int]], + interestingIndices: Set[Int], + render: Int => String + ): Unit = { + os.write.over( + path, + SpanningForest.spanningTreeToJsonTree( + SpanningForest(indexEdges, interestingIndices, true), + render + ).render(indent = 2) + ) + } def spanningTreeToJsonTree(node: SpanningForest.Node, stringify: Int => String): ujson.Obj = { ujson.Obj.from( node.values.map { case (k, v) => stringify(k) -> spanningTreeToJsonTree(v, stringify) }