From 5b75db32fa20fd993aa11b1dc8de083c80e4ca30 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sat, 14 Dec 2024 20:04:28 +0800 Subject: [PATCH 01/13] . --- build.mill | 2 +- docs/modules/ROOT/nav.adoc | 5 ++- .../ROOT/pages/comparisons/unique.adoc | 4 +- docs/modules/ROOT/pages/index.adoc | 4 +- .../large-builds.adoc => large/large.adoc} | 27 +++--------- .../ROOT/pages/large/multi-file-builds.adoc | 16 +++++++ .../ROOT/pages/large/selective-execution.adoc | 44 +++++++++++++++++++ .../ROOT/pages/migrating/migrating.adoc | 2 +- .../10-multi-file-builds/bar/package.mill | 0 .../bar/qux/mymodule/src/BarQux.scala | 0 .../10-multi-file-builds/bar/qux/package.mill | 0 .../multi}/10-multi-file-builds/build.mill | 0 .../10-multi-file-builds/foo/package.mill | 0 .../10-multi-file-builds/foo/src/Foo.scala | 0 .../multi}/11-helper-files/build.mill | 0 .../multi}/11-helper-files/foo/package.mill | 0 .../multi}/11-helper-files/foo/src/Foo.scala | 0 .../multi}/11-helper-files/foo/versions.mill | 0 .../multi}/11-helper-files/src/Main.scala | 0 .../multi}/11-helper-files/util.mill | 0 .../multi}/12-helper-files-sc/build.sc | 0 .../multi}/12-helper-files-sc/foo/package.sc | 0 .../12-helper-files-sc/foo/src/Foo.scala | 0 .../multi}/12-helper-files-sc/foo/versions.sc | 0 .../multi}/12-helper-files-sc/src/Main.scala | 0 .../multi}/12-helper-files-sc/util.sc | 0 .../build.mill.scala | 0 .../foo/package.mill.scala | 0 .../foo/src/Foo.scala | 0 .../foo/versions.mill.scala | 0 .../13-helper-files-mill-scala/src/Main.scala | 0 .../util.mill.scala | 0 .../bar/src/bar/Bar.java | 0 .../bar/test/src/bar/BarTests.java | 0 .../9-selective-execution/build.mill | 3 +- .../foo/src/foo/Foo.java | 0 .../foo/test/src/bar/FooTests.java | 0 .../qux/src/qux/Qux.java | 0 .../qux/test/src/qux/QuxTests.java | 0 example/package.mill | 7 ++- 40 files changed, 83 insertions(+), 31 deletions(-) rename docs/modules/ROOT/pages/{depth/large-builds.adoc => large/large.adoc} (59%) create mode 100644 docs/modules/ROOT/pages/large/multi-file-builds.adoc create mode 100644 docs/modules/ROOT/pages/large/selective-execution.adoc rename example/{depth/large => large/multi}/10-multi-file-builds/bar/package.mill (100%) rename example/{depth/large => large/multi}/10-multi-file-builds/bar/qux/mymodule/src/BarQux.scala (100%) rename example/{depth/large => large/multi}/10-multi-file-builds/bar/qux/package.mill (100%) rename example/{depth/large => large/multi}/10-multi-file-builds/build.mill (100%) rename example/{depth/large => large/multi}/10-multi-file-builds/foo/package.mill (100%) rename example/{depth/large => large/multi}/10-multi-file-builds/foo/src/Foo.scala (100%) rename example/{depth/large => large/multi}/11-helper-files/build.mill (100%) rename example/{depth/large => large/multi}/11-helper-files/foo/package.mill (100%) rename example/{depth/large => large/multi}/11-helper-files/foo/src/Foo.scala (100%) rename example/{depth/large => large/multi}/11-helper-files/foo/versions.mill (100%) rename example/{depth/large => large/multi}/11-helper-files/src/Main.scala (100%) rename example/{depth/large => large/multi}/11-helper-files/util.mill (100%) rename example/{depth/large => large/multi}/12-helper-files-sc/build.sc (100%) rename example/{depth/large => large/multi}/12-helper-files-sc/foo/package.sc (100%) rename example/{depth/large => large/multi}/12-helper-files-sc/foo/src/Foo.scala (100%) rename example/{depth/large => large/multi}/12-helper-files-sc/foo/versions.sc (100%) rename example/{depth/large => large/multi}/12-helper-files-sc/src/Main.scala (100%) rename example/{depth/large => large/multi}/12-helper-files-sc/util.sc (100%) rename example/{depth/large => large/multi}/13-helper-files-mill-scala/build.mill.scala (100%) rename example/{depth/large => large/multi}/13-helper-files-mill-scala/foo/package.mill.scala (100%) rename example/{depth/large => large/multi}/13-helper-files-mill-scala/foo/src/Foo.scala (100%) rename example/{depth/large => large/multi}/13-helper-files-mill-scala/foo/versions.mill.scala (100%) rename example/{depth/large => large/multi}/13-helper-files-mill-scala/src/Main.scala (100%) rename example/{depth/large => large/multi}/13-helper-files-mill-scala/util.mill.scala (100%) rename example/{depth/large => large/selective}/9-selective-execution/bar/src/bar/Bar.java (100%) rename example/{depth/large => large/selective}/9-selective-execution/bar/test/src/bar/BarTests.java (100%) rename example/{depth/large => large/selective}/9-selective-execution/build.mill (99%) rename example/{depth/large => large/selective}/9-selective-execution/foo/src/foo/Foo.java (100%) rename example/{depth/large => large/selective}/9-selective-execution/foo/test/src/bar/FooTests.java (100%) rename example/{depth/large => large/selective}/9-selective-execution/qux/src/qux/Qux.java (100%) rename example/{depth/large => large/selective}/9-selective-execution/qux/test/src/qux/QuxTests.java (100%) diff --git a/build.mill b/build.mill index f114a1fe068..060db96f4fa 100644 --- a/build.mill +++ b/build.mill @@ -256,7 +256,7 @@ object Deps { } } -def millVersion: T[String] = Task { +def millVersion: T[String] = Task.Input { if (Task.env.contains("MILL_STABLE_VERSION")) VcsVersion.calcVcsState(Task.log).format() else "SNAPSHOT" } diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index 53bd6fa510c..8d848c59e04 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -95,8 +95,11 @@ // These are things that most Mill developers would not encounter day to day, // but people developing Mill plugins or working on particularly large or // sophisticated Mill builds will need to understand. +* xref:large/large.adoc[] +** xref:large/selective-execution.adoc[] +** xref:large/multi-file-builds.adoc[] + * Mill In Depth -** xref:depth/large-builds.adoc[] ** xref:depth/sandboxing.adoc[] ** xref:depth/evaluation-model.adoc[] ** xref:depth/design-principles.adoc[] diff --git a/docs/modules/ROOT/pages/comparisons/unique.adoc b/docs/modules/ROOT/pages/comparisons/unique.adoc index be471c12b8e..b174510ba67 100644 --- a/docs/modules/ROOT/pages/comparisons/unique.adoc +++ b/docs/modules/ROOT/pages/comparisons/unique.adoc @@ -85,8 +85,8 @@ xref:android/java.adoc[Android], and has demonstrated the ability to branch out more distant toolchains like xref:extending/example-typescript-support.adoc[Typescript] and xref:extending/example-python-support.adoc[Python]. -Mill also works well with xref:depth/large-builds.adoc[large builds]: its build logic can be -split into multiple folders, is incrementally compiled, +Mill also works well with xref:large/large.adoc[large builds]: its build logic can be +xref:large/multi-file-builds.adoc[split into multiple folders], is incrementally compiled, lazily initialized, and automatically cached and parallelized. That means that even large codebases can remain fast and responsive: Mill's own build easily manages over 400 modules, and the tool can likely handle thousands of modules without issue. diff --git a/docs/modules/ROOT/pages/index.adoc b/docs/modules/ROOT/pages/index.adoc index 2bec2abd99e..c34ad0f7da1 100644 --- a/docs/modules/ROOT/pages/index.adoc +++ b/docs/modules/ROOT/pages/index.adoc @@ -13,7 +13,7 @@ or xref:comparisons/gradle.adoc[2-4x faster than Gradle] helps keep builds clean and understandable * Mill is an easier alternative to https://bazel.build/[Bazel] -for xref:depth/large-builds.adoc[large multi-language monorepos] with hundreds of modules +for xref:large/large.adoc[large multi-language monorepos] with hundreds of modules To get started using Mill, see the language-specific introductory documentation linked below: @@ -37,7 +37,7 @@ Java platform's performance and usability: xref:depth/evaluation-model.adoc#_caching_at_each_layer_of_the_evaluation_model[caches] and xref:cli/flags.adoc#_jobs_j[parallelizes] build tasks to keep local development fast, and avoids the long configuration times seen in other tools like Gradle or SBT. -xref:depth/large-builds.adoc#_selective_execution[Selective execution] keeps +xref:large/selective-execution.adoc[Selective execution] keeps CI validation times short by only running the tests necessary to validate a code change. * *Maintainability*: Mill's config and xref:javalib/intro.adoc#_custom_build_logic[custom logic] diff --git a/docs/modules/ROOT/pages/depth/large-builds.adoc b/docs/modules/ROOT/pages/large/large.adoc similarity index 59% rename from docs/modules/ROOT/pages/depth/large-builds.adoc rename to docs/modules/ROOT/pages/large/large.adoc index bdfa62d5334..9c0c428adab 100644 --- a/docs/modules/ROOT/pages/depth/large-builds.adoc +++ b/docs/modules/ROOT/pages/large/large.adoc @@ -1,7 +1,4 @@ = Large Builds and Monorepos -:page-aliases: Structuring_Large_Builds.adoc - -include::partial$gtag-config.adoc[] This section walks through Mill features and techniques used for managing large builds. While Mill works great for small single-module projects, it is also able to work @@ -10,25 +7,13 @@ https://github.com/com-lihaoyi/mill[com-lihaoyi/mill] project has ~400 modules, other proprietary projects may have many more. Mill modules are cheap. Having more modules does not significantly impact performance -or resource usage, build files are incrementally re-compiled when modified, and modules are -lazily loaded and initialized only when needed. So you are encouraged to break up your project +or resource usage, build files are incrementally re-compiled when modified, and modules are +lazily loaded and initialized only when needed. So you are encouraged to break up your project into modules to manage the layering of your codebase or benefit from parallelism. -== Selective Execution - - -include::partial$example/depth/large/9-selective-execution.adoc[] - -== Multi-file Builds - -include::partial$example/depth/large/10-multi-file-builds.adoc[] - -== Helper Files - -include::partial$example/depth/large/11-helper-files.adoc[] - -== Legacy `.sc` extension - -include::partial$example/depth/large/12-helper-files-sc.adoc[] +Apart from Mill's basic scalability and performance, Mill also comes with many features +that can be utilized to help you manage the build system of a large project or codebase: +* xref:large/selective-execution.adoc[] +* xref:large/multi-file-builds.adoc[] \ No newline at end of file diff --git a/docs/modules/ROOT/pages/large/multi-file-builds.adoc b/docs/modules/ROOT/pages/large/multi-file-builds.adoc new file mode 100644 index 00000000000..b2f363c51d3 --- /dev/null +++ b/docs/modules/ROOT/pages/large/multi-file-builds.adoc @@ -0,0 +1,16 @@ += Multi-File Builds +:page-aliases: Structuring_Large_Builds.adoc + +include::partial$gtag-config.adoc[] + +include::partial$example/large/multi/10-multi-file-builds.adoc[] + +== Helper Files + +include::partial$example/large/multi/11-helper-files.adoc[] + +== Legacy `.sc` extension + +include::partial$example/large/multi/12-helper-files-sc.adoc[] + + diff --git a/docs/modules/ROOT/pages/large/selective-execution.adoc b/docs/modules/ROOT/pages/large/selective-execution.adoc new file mode 100644 index 00000000000..75d7c41965b --- /dev/null +++ b/docs/modules/ROOT/pages/large/selective-execution.adoc @@ -0,0 +1,44 @@ += Selective Execution + +include::partial$gtag-config.adoc[] + + +include::partial$example/large/selective/9-selective-execution.adoc[] + + +== Reproducibility and Determinism + +Selective execution relies on the inputs to your project being deterministic +and reproducible, except for the code changes between the two versions, so that +Mill can compare the state of the build inputs before and after and only run +tasks downstream of those that changed. This is usually the case, but there are +some subtleties to be aware of: + +- *Dynamic `Task.Input` to capture Git metadata must be disabled*, e.g. using + https://github.com/lefou/mill-vcs-version[mill-vcs-version]. The easiest way to do + this is to guard such dynamic inputs on an environment variable, such that + in most scenarios it returns a constant `"SNAPSHOT"` string, and only when + necessary do you pass in the environment variable to compute a real version (e.g. + during publishing) + +```scala +def myProjectVersion: T[String] = Task.Input { + if (Task.env.contains("MY_PROJECT_STABLE_VERSION")) VcsVersion.calcVcsState(Task.log).format() + else "SNAPSHOT" +} +``` + +- *The filesystem layout and position of the before/after codebases must be exactly + the same*. This is not an issue when running `selective.prepare`/`selective.run` on + the same folder on one machine, but if the two calls are run on separate machines + you need to make sure the directory path is the same. + +- *You must use the same Operating System amd Filesystem*, as differences there will + cause the filesystem signatures to change and thus spuriously trigger downstream tasks. + e.g. you cannot run `selective.prepare` on a Windows machine and `selective.run` on Linux + +- *Filesystem permissions must be preserved before/after*. e.g. running `selective,run}` + on different Github Actions machines sharing artifacts can cause issues as + `upload-artifact`/`download-artifact` https://github.com/actions/download-artifact#permission-loss[does not preserve filesystem permissions]. + If this is an issue, you can run `chmod -R . 777` before each of `selective.{prepare,run}` + to ensure they have the exact same filesystem permissions. diff --git a/docs/modules/ROOT/pages/migrating/migrating.adoc b/docs/modules/ROOT/pages/migrating/migrating.adoc index 639ce84a3aa..aca914c3ee3 100644 --- a/docs/modules/ROOT/pages/migrating/migrating.adoc +++ b/docs/modules/ROOT/pages/migrating/migrating.adoc @@ -283,7 +283,7 @@ to see which ones may help: * xref:fundamentals/modules.adoc#_trait_modules[Trait Modules] to centralize common config -* xref:depth/large-builds.adoc#_multi_file_builds[Multi-File Builds] to let you co-locate +* xref:large/multi-file-builds.adoc[Multi-File Builds] to let you co-locate build logic and the code being built * xref:extending/writing-plugins.adoc[Writing and Publishing your own Mill Plugins] diff --git a/example/depth/large/10-multi-file-builds/bar/package.mill b/example/large/multi/10-multi-file-builds/bar/package.mill similarity index 100% rename from example/depth/large/10-multi-file-builds/bar/package.mill rename to example/large/multi/10-multi-file-builds/bar/package.mill diff --git a/example/depth/large/10-multi-file-builds/bar/qux/mymodule/src/BarQux.scala b/example/large/multi/10-multi-file-builds/bar/qux/mymodule/src/BarQux.scala similarity index 100% rename from example/depth/large/10-multi-file-builds/bar/qux/mymodule/src/BarQux.scala rename to example/large/multi/10-multi-file-builds/bar/qux/mymodule/src/BarQux.scala diff --git a/example/depth/large/10-multi-file-builds/bar/qux/package.mill b/example/large/multi/10-multi-file-builds/bar/qux/package.mill similarity index 100% rename from example/depth/large/10-multi-file-builds/bar/qux/package.mill rename to example/large/multi/10-multi-file-builds/bar/qux/package.mill diff --git a/example/depth/large/10-multi-file-builds/build.mill b/example/large/multi/10-multi-file-builds/build.mill similarity index 100% rename from example/depth/large/10-multi-file-builds/build.mill rename to example/large/multi/10-multi-file-builds/build.mill diff --git a/example/depth/large/10-multi-file-builds/foo/package.mill b/example/large/multi/10-multi-file-builds/foo/package.mill similarity index 100% rename from example/depth/large/10-multi-file-builds/foo/package.mill rename to example/large/multi/10-multi-file-builds/foo/package.mill diff --git a/example/depth/large/10-multi-file-builds/foo/src/Foo.scala b/example/large/multi/10-multi-file-builds/foo/src/Foo.scala similarity index 100% rename from example/depth/large/10-multi-file-builds/foo/src/Foo.scala rename to example/large/multi/10-multi-file-builds/foo/src/Foo.scala diff --git a/example/depth/large/11-helper-files/build.mill b/example/large/multi/11-helper-files/build.mill similarity index 100% rename from example/depth/large/11-helper-files/build.mill rename to example/large/multi/11-helper-files/build.mill diff --git a/example/depth/large/11-helper-files/foo/package.mill b/example/large/multi/11-helper-files/foo/package.mill similarity index 100% rename from example/depth/large/11-helper-files/foo/package.mill rename to example/large/multi/11-helper-files/foo/package.mill diff --git a/example/depth/large/11-helper-files/foo/src/Foo.scala b/example/large/multi/11-helper-files/foo/src/Foo.scala similarity index 100% rename from example/depth/large/11-helper-files/foo/src/Foo.scala rename to example/large/multi/11-helper-files/foo/src/Foo.scala diff --git a/example/depth/large/11-helper-files/foo/versions.mill b/example/large/multi/11-helper-files/foo/versions.mill similarity index 100% rename from example/depth/large/11-helper-files/foo/versions.mill rename to example/large/multi/11-helper-files/foo/versions.mill diff --git a/example/depth/large/11-helper-files/src/Main.scala b/example/large/multi/11-helper-files/src/Main.scala similarity index 100% rename from example/depth/large/11-helper-files/src/Main.scala rename to example/large/multi/11-helper-files/src/Main.scala diff --git a/example/depth/large/11-helper-files/util.mill b/example/large/multi/11-helper-files/util.mill similarity index 100% rename from example/depth/large/11-helper-files/util.mill rename to example/large/multi/11-helper-files/util.mill diff --git a/example/depth/large/12-helper-files-sc/build.sc b/example/large/multi/12-helper-files-sc/build.sc similarity index 100% rename from example/depth/large/12-helper-files-sc/build.sc rename to example/large/multi/12-helper-files-sc/build.sc diff --git a/example/depth/large/12-helper-files-sc/foo/package.sc b/example/large/multi/12-helper-files-sc/foo/package.sc similarity index 100% rename from example/depth/large/12-helper-files-sc/foo/package.sc rename to example/large/multi/12-helper-files-sc/foo/package.sc diff --git a/example/depth/large/12-helper-files-sc/foo/src/Foo.scala b/example/large/multi/12-helper-files-sc/foo/src/Foo.scala similarity index 100% rename from example/depth/large/12-helper-files-sc/foo/src/Foo.scala rename to example/large/multi/12-helper-files-sc/foo/src/Foo.scala diff --git a/example/depth/large/12-helper-files-sc/foo/versions.sc b/example/large/multi/12-helper-files-sc/foo/versions.sc similarity index 100% rename from example/depth/large/12-helper-files-sc/foo/versions.sc rename to example/large/multi/12-helper-files-sc/foo/versions.sc diff --git a/example/depth/large/12-helper-files-sc/src/Main.scala b/example/large/multi/12-helper-files-sc/src/Main.scala similarity index 100% rename from example/depth/large/12-helper-files-sc/src/Main.scala rename to example/large/multi/12-helper-files-sc/src/Main.scala diff --git a/example/depth/large/12-helper-files-sc/util.sc b/example/large/multi/12-helper-files-sc/util.sc similarity index 100% rename from example/depth/large/12-helper-files-sc/util.sc rename to example/large/multi/12-helper-files-sc/util.sc diff --git a/example/depth/large/13-helper-files-mill-scala/build.mill.scala b/example/large/multi/13-helper-files-mill-scala/build.mill.scala similarity index 100% rename from example/depth/large/13-helper-files-mill-scala/build.mill.scala rename to example/large/multi/13-helper-files-mill-scala/build.mill.scala diff --git a/example/depth/large/13-helper-files-mill-scala/foo/package.mill.scala b/example/large/multi/13-helper-files-mill-scala/foo/package.mill.scala similarity index 100% rename from example/depth/large/13-helper-files-mill-scala/foo/package.mill.scala rename to example/large/multi/13-helper-files-mill-scala/foo/package.mill.scala diff --git a/example/depth/large/13-helper-files-mill-scala/foo/src/Foo.scala b/example/large/multi/13-helper-files-mill-scala/foo/src/Foo.scala similarity index 100% rename from example/depth/large/13-helper-files-mill-scala/foo/src/Foo.scala rename to example/large/multi/13-helper-files-mill-scala/foo/src/Foo.scala diff --git a/example/depth/large/13-helper-files-mill-scala/foo/versions.mill.scala b/example/large/multi/13-helper-files-mill-scala/foo/versions.mill.scala similarity index 100% rename from example/depth/large/13-helper-files-mill-scala/foo/versions.mill.scala rename to example/large/multi/13-helper-files-mill-scala/foo/versions.mill.scala diff --git a/example/depth/large/13-helper-files-mill-scala/src/Main.scala b/example/large/multi/13-helper-files-mill-scala/src/Main.scala similarity index 100% rename from example/depth/large/13-helper-files-mill-scala/src/Main.scala rename to example/large/multi/13-helper-files-mill-scala/src/Main.scala diff --git a/example/depth/large/13-helper-files-mill-scala/util.mill.scala b/example/large/multi/13-helper-files-mill-scala/util.mill.scala similarity index 100% rename from example/depth/large/13-helper-files-mill-scala/util.mill.scala rename to example/large/multi/13-helper-files-mill-scala/util.mill.scala diff --git a/example/depth/large/9-selective-execution/bar/src/bar/Bar.java b/example/large/selective/9-selective-execution/bar/src/bar/Bar.java similarity index 100% rename from example/depth/large/9-selective-execution/bar/src/bar/Bar.java rename to example/large/selective/9-selective-execution/bar/src/bar/Bar.java diff --git a/example/depth/large/9-selective-execution/bar/test/src/bar/BarTests.java b/example/large/selective/9-selective-execution/bar/test/src/bar/BarTests.java similarity index 100% rename from example/depth/large/9-selective-execution/bar/test/src/bar/BarTests.java rename to example/large/selective/9-selective-execution/bar/test/src/bar/BarTests.java diff --git a/example/depth/large/9-selective-execution/build.mill b/example/large/selective/9-selective-execution/build.mill similarity index 99% rename from example/depth/large/9-selective-execution/build.mill rename to example/large/selective/9-selective-execution/build.mill index ea5f136b14d..095ee7675a3 100644 --- a/example/depth/large/9-selective-execution/build.mill +++ b/example/large/selective/9-selective-execution/build.mill @@ -22,7 +22,7 @@ // ```bash // > git checkout main # start from the target branch of the PR // -// > ./mill selective.prepare __.test +// > ./mill selective.prepare // // > git checkout pull-request-branch # go to the pull request branch // @@ -105,3 +105,4 @@ Test run bar.BarTests finished: 0 failed, 0 ignored, 1 total, ... // tasks non-selectively, which is convenient if you want to conditionally disable selective // execution (e.g. perhaps you want to perform selective execution on pre-merge on pull // requests but not post-merge on the main branch) +// diff --git a/example/depth/large/9-selective-execution/foo/src/foo/Foo.java b/example/large/selective/9-selective-execution/foo/src/foo/Foo.java similarity index 100% rename from example/depth/large/9-selective-execution/foo/src/foo/Foo.java rename to example/large/selective/9-selective-execution/foo/src/foo/Foo.java diff --git a/example/depth/large/9-selective-execution/foo/test/src/bar/FooTests.java b/example/large/selective/9-selective-execution/foo/test/src/bar/FooTests.java similarity index 100% rename from example/depth/large/9-selective-execution/foo/test/src/bar/FooTests.java rename to example/large/selective/9-selective-execution/foo/test/src/bar/FooTests.java diff --git a/example/depth/large/9-selective-execution/qux/src/qux/Qux.java b/example/large/selective/9-selective-execution/qux/src/qux/Qux.java similarity index 100% rename from example/depth/large/9-selective-execution/qux/src/qux/Qux.java rename to example/large/selective/9-selective-execution/qux/src/qux/Qux.java diff --git a/example/depth/large/9-selective-execution/qux/test/src/qux/QuxTests.java b/example/large/selective/9-selective-execution/qux/test/src/qux/QuxTests.java similarity index 100% rename from example/depth/large/9-selective-execution/qux/test/src/qux/QuxTests.java rename to example/large/selective/9-selective-execution/qux/test/src/qux/QuxTests.java diff --git a/example/package.mill b/example/package.mill index bbbd900a963..79d9e84f908 100644 --- a/example/package.mill +++ b/example/package.mill @@ -89,11 +89,14 @@ object `package` extends RootModule with Module { object depth extends Module { - object large extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "large")) - object sandbox extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "sandbox")) object javahome extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "javahome")) } + object large extends Module { + + object selective extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "selective")) + object multi extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "multi")) + } object extending extends Module { object imports extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "imports")) From 3b29531208e7b56ae2fef74c1f41594e93520df5 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sat, 14 Dec 2024 23:50:02 +0800 Subject: [PATCH 02/13] basic invalidation and dependency forests --- .../client/src/mill/main/client/OutFiles.java | 3 ++ main/codesig/src/ReachabilityAnalysis.scala | 17 ++++----- main/codesig/src/ResolvedCalls.scala | 17 +-------- main/eval/src/mill/eval/EvaluatorCore.scala | 30 +++++++++++++++- main/src/mill/main/SelectiveExecution.scala | 18 +--------- .../src/mill/util}/SpanningForest.scala | 35 ++++++++++++++++--- 6 files changed, 70 insertions(+), 50 deletions(-) rename main/{codesig/src => util/src/mill/util}/SpanningForest.scala (72%) diff --git a/main/client/src/mill/main/client/OutFiles.java b/main/client/src/mill/main/client/OutFiles.java index 51be8714b36..ddc226c12e4 100644 --- a/main/client/src/mill/main/client/OutFiles.java +++ b/main/client/src/mill/main/client/OutFiles.java @@ -73,4 +73,7 @@ public class OutFiles { * root tasks changed so Mill can decide which tasks to execute. */ public static final String millSelectiveExecution = "mill-selective-execution.json"; + + public static final String millDependencyForest = "mill-dependency-forest.json"; + public static final String millInvalidationForest = "mill-invalidation-forest.json"; } diff --git a/main/codesig/src/ReachabilityAnalysis.scala b/main/codesig/src/ReachabilityAnalysis.scala index 116848bab9d..1a1dc22b354 100644 --- a/main/codesig/src/ReachabilityAnalysis.scala +++ b/main/codesig/src/ReachabilityAnalysis.scala @@ -1,7 +1,7 @@ package mill.codesig -import mill.util.Tarjans +import mill.util.{SpanningForest, Tarjans} import upickle.default.{Writer, writer} -import JvmModel._ +import JvmModel.* import scala.collection.immutable.SortedMap import ujson.Obj @@ -126,15 +126,10 @@ object CallGraphAnalysis { } .toSet - def spanningTreeToJsonTree(node: SpanningForest.Node): ujson.Obj = { - ujson.Obj.from( - node.values.map { case (k, v) => - indexToNodes(k).toString -> spanningTreeToJsonTree(v) - } - ) - } - - spanningTreeToJsonTree(SpanningForest.apply(indexGraphEdges, nodesWithChangedHashes)) + SpanningForest.spanningTreeToJsonTree( + SpanningForest.apply(indexGraphEdges, nodesWithChangedHashes), + k => indexToNodes(k).toString + ) } def indexGraphEdges( diff --git a/main/codesig/src/ResolvedCalls.scala b/main/codesig/src/ResolvedCalls.scala index 6bb5b5698f7..5a32697a17b 100644 --- a/main/codesig/src/ResolvedCalls.scala +++ b/main/codesig/src/ResolvedCalls.scala @@ -1,6 +1,7 @@ package mill.codesig import JvmModel._ import JType.{Cls => JCls} +import mill.util.SpanningForest.breadthFirst import upickle.default.{ReadWriter, macroRW} case class ResolvedCalls( @@ -188,20 +189,4 @@ object ResolvedCalls { ) } - def breadthFirst[T](start: IterableOnce[T])(edges: T => IterableOnce[T]): Seq[T] = { - val seen = collection.mutable.Set.empty[T] - val seenList = collection.mutable.Buffer.empty[T] - val queued = collection.mutable.Queue.from(start) - - while (queued.nonEmpty) { - val current = queued.dequeue() - seen.add(current) - seenList.append(current) - - for (next <- edges(current).iterator) { - if (!seen.contains(next)) queued.enqueue(next) - } - } - seenList.toSeq - } } diff --git a/main/eval/src/mill/eval/EvaluatorCore.scala b/main/eval/src/mill/eval/EvaluatorCore.scala index 4bdc2f48fbd..b4ad304618f 100644 --- a/main/eval/src/mill/eval/EvaluatorCore.scala +++ b/main/eval/src/mill/eval/EvaluatorCore.scala @@ -5,12 +5,13 @@ import mill.api.Strict.Agg import mill.api._ import mill.define._ import mill.eval.Evaluator.TaskResult - +import mill.main.client.OutFiles import mill.util._ import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger} import scala.collection.mutable import scala.concurrent._ +import scala.jdk.CollectionConverters.EnumerationHasAsScala /** * Core logic of evaluating tasks, without any user-facing helper methods @@ -85,6 +86,7 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { val (classToTransitiveClasses, allTransitiveClassMethods) = CodeSigUtils.precomputeMethodNamesPerClass(sortedGroups) + val uncached = new java.util.concurrent.ConcurrentHashMap[Terminal, Unit]() def evaluateTerminals( terminals: Seq[Terminal], forkExecutionContext: mill.api.Ctx.Fork.Impl, @@ -191,6 +193,7 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { threadId = threadNumberer.getThreadId(Thread.currentThread()), cached = res.cached ) + if (!res.cached) uncached.put(terminal, ()) profileLogger.log( ProfileLogger.Timing( @@ -254,6 +257,31 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { val results: Map[Task[_], TaskResult[(Val, Int)]] = results0.toMap + val reverseInterGroupDeps = interGroupDeps + .iterator + .flatMap{case (k, vs) => vs.map(_ -> k)} + .toSeq + .groupMap(_._1)(_._2) + + val indexToTerminal = sortedGroups.keys().toArray + val terminalToIndex = indexToTerminal.zipWithIndex.toMap + val downstreamIndexEdges = indexToTerminal.map(t => reverseInterGroupDeps.getOrElse(t, Nil).map(terminalToIndex).toArray) + val upstreamIndexEdges = indexToTerminal.map(t => interGroupDeps.getOrElse(t, Nil).map(terminalToIndex).toArray) + os.write.over( + outPath / OutFiles.millInvalidationForest, + SpanningForest.spanningTreeToJsonTree( + SpanningForest(upstreamIndexEdges, uncached.keys().asScala.map(terminalToIndex).toSet), + i => indexToTerminal(i).render + ).render(indent = 2) + ) + os.write.over( + outPath / OutFiles.millDependencyForest, + SpanningForest.spanningTreeToJsonTree( + SpanningForest.apply(downstreamIndexEdges, indexToTerminal.indices.toSet), + i => indexToTerminal(i).render + ).render(indent = 2) + ) + EvaluatorCore.Results( goals.indexed.map(results(_).map(_._1).result), // result of flatMap may contain non-distinct entries, diff --git a/main/src/mill/main/SelectiveExecution.scala b/main/src/mill/main/SelectiveExecution.scala index 5b0f1a71c27..f310fa15785 100644 --- a/main/src/mill/main/SelectiveExecution.scala +++ b/main/src/mill/main/SelectiveExecution.scala @@ -4,6 +4,7 @@ import mill.api.Strict import mill.define.{InputImpl, NamedTask, Task} import mill.eval.{CodeSigUtils, Evaluator, Plan, Terminal} import mill.main.client.OutFiles +import mill.util.SpanningForest.breadthFirst import mill.resolve.{Resolve, SelectMode} private[mill] object SelectiveExecution { @@ -114,23 +115,6 @@ private[mill] object SelectiveExecution { breadthFirst(changedRootTasks)(downstreamEdgeMap.getOrElse(_, Nil)) } - def breadthFirst[T](start: IterableOnce[T])(edges: T => IterableOnce[T]): Seq[T] = { - val seen = collection.mutable.Set.empty[T] - val seenList = collection.mutable.Buffer.empty[T] - val queued = collection.mutable.Queue.from(start) - - while (queued.nonEmpty) { - val current = queued.dequeue() - seen.add(current) - seenList.append(current) - - for (next <- edges(current).iterator) { - if (!seen.contains(next)) queued.enqueue(next) - } - } - seenList.toSeq - } - def saveMetadata(evaluator: Evaluator, metadata: SelectiveExecution.Metadata): Unit = { os.write.over( evaluator.outPath / OutFiles.millSelectiveExecution, diff --git a/main/codesig/src/SpanningForest.scala b/main/util/src/mill/util/SpanningForest.scala similarity index 72% rename from main/codesig/src/SpanningForest.scala rename to main/util/src/mill/util/SpanningForest.scala index a642cae82db..c2e3aeffb4a 100644 --- a/main/codesig/src/SpanningForest.scala +++ b/main/util/src/mill/util/SpanningForest.scala @@ -1,5 +1,6 @@ -package mill.codesig -import collection.mutable +package mill.util + +import scala.collection.mutable /** * Algorithm to compute the minimal spanning forest of a directed acyclic graph @@ -11,8 +12,14 @@ import collection.mutable * Returns the forest as a [[Node]] structure with the top-level node containing * the roots of the forest */ -object SpanningForest { - +private[mill] object SpanningForest { + def spanningTreeToJsonTree(node: SpanningForest.Node, stringify: Int => String): ujson.Obj = { + ujson.Obj.from( + node.values.map { case (k, v) => + stringify(k) -> spanningTreeToJsonTree(v, stringify) + } + ) + } case class Node(values: mutable.Map[Int, Node] = mutable.Map()) def apply(indexGraphEdges: Array[Array[Int]], importantVertices: Set[Int]): Node = { // Find all importantVertices which are "roots" with no incoming edges @@ -35,7 +42,7 @@ object SpanningForest { .flatMap { case (vs, k) => vs.map((_, k)) } .groupMap(_._1)(_._2) - ResolvedCalls.breadthFirst(rootChangedNodeIndices) { index => + breadthFirst(rootChangedNodeIndices) { index => val nextIndices = downstreamGraphEdges.getOrElse( index, @@ -54,4 +61,22 @@ object SpanningForest { } spanningForest } + + def breadthFirst[T](start: IterableOnce[T])(edges: T => IterableOnce[T]): Seq[T] = { + val seen = collection.mutable.Set.empty[T] + val seenList = collection.mutable.Buffer.empty[T] + val queued = collection.mutable.Queue.from(start) + + while (queued.nonEmpty) { + val current = queued.dequeue() + seen.add(current) + seenList.append(current) + + for (next <- edges(current).iterator) { + if (!seen.contains(next)) queued.enqueue(next) + } + } + seenList.toSeq + } + } From 453754c5e7c68b3a100dec7f1cec66d127c5fd09 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 15 Dec 2024 00:29:40 +0800 Subject: [PATCH 03/13] spanning forest fixes --- main/eval/src/mill/eval/EvaluatorCore.scala | 4 +- main/util/src/mill/util/SpanningForest.scala | 14 +++--- .../src/mill/util/SpanningForestTests.scala | 46 +++++++++++++++++++ 3 files changed, 55 insertions(+), 9 deletions(-) create mode 100644 main/util/test/src/mill/util/SpanningForestTests.scala diff --git a/main/eval/src/mill/eval/EvaluatorCore.scala b/main/eval/src/mill/eval/EvaluatorCore.scala index b4ad304618f..b09261ea50f 100644 --- a/main/eval/src/mill/eval/EvaluatorCore.scala +++ b/main/eval/src/mill/eval/EvaluatorCore.scala @@ -87,6 +87,7 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { CodeSigUtils.precomputeMethodNamesPerClass(sortedGroups) val uncached = new java.util.concurrent.ConcurrentHashMap[Terminal, Unit]() + def evaluateTerminals( terminals: Seq[Terminal], forkExecutionContext: mill.api.Ctx.Fork.Impl, @@ -267,6 +268,7 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { val terminalToIndex = indexToTerminal.zipWithIndex.toMap val downstreamIndexEdges = indexToTerminal.map(t => reverseInterGroupDeps.getOrElse(t, Nil).map(terminalToIndex).toArray) val upstreamIndexEdges = indexToTerminal.map(t => interGroupDeps.getOrElse(t, Nil).map(terminalToIndex).toArray) + os.write.over( outPath / OutFiles.millInvalidationForest, SpanningForest.spanningTreeToJsonTree( @@ -277,7 +279,7 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { os.write.over( outPath / OutFiles.millDependencyForest, SpanningForest.spanningTreeToJsonTree( - SpanningForest.apply(downstreamIndexEdges, indexToTerminal.indices.toSet), + SpanningForest(downstreamIndexEdges, indexToTerminal.indices.toSet), i => indexToTerminal(i).render ).render(indent = 2) ) diff --git a/main/util/src/mill/util/SpanningForest.scala b/main/util/src/mill/util/SpanningForest.scala index c2e3aeffb4a..b594d2f830e 100644 --- a/main/util/src/mill/util/SpanningForest.scala +++ b/main/util/src/mill/util/SpanningForest.scala @@ -15,9 +15,7 @@ import scala.collection.mutable private[mill] object SpanningForest { def spanningTreeToJsonTree(node: SpanningForest.Node, stringify: Int => String): ujson.Obj = { ujson.Obj.from( - node.values.map { case (k, v) => - stringify(k) -> spanningTreeToJsonTree(v, stringify) - } + node.values.map { case (k, v) => stringify(k) -> spanningTreeToJsonTree(v, stringify) } ) } case class Node(values: mutable.Map[Int, Node] = mutable.Map()) @@ -43,11 +41,11 @@ private[mill] object SpanningForest { .groupMap(_._1)(_._2) breadthFirst(rootChangedNodeIndices) { index => - val nextIndices = - downstreamGraphEdges.getOrElse( - index, - Array[Int]() - ) // needed to add explicit type for Scala 3.5.0-RC6 + // needed to add explicit type for Scala 3.5.0-RC6 + val nextIndices = downstreamGraphEdges + .getOrElse(index, Array[Int]()) + .filter(importantVertices) + // We build up the spanningForest during a normal breadth first search, // using the `nodeMapping` to quickly find a vertice's tree node so we // can add children to it. We need to duplicate the `seen.contains` logic diff --git a/main/util/test/src/mill/util/SpanningForestTests.scala b/main/util/test/src/mill/util/SpanningForestTests.scala new file mode 100644 index 00000000000..a5e1de396c6 --- /dev/null +++ b/main/util/test/src/mill/util/SpanningForestTests.scala @@ -0,0 +1,46 @@ +package mill.util + +import utest.{TestSuite, Tests, test} +import collection.mutable +import SpanningForest.Node +object SpanningForestTests extends TestSuite { + + val tests = Tests { + + test("test") { + val forest = SpanningForest.apply( + Array( + Array(1), + Array(2), + Array(3), + Array(), + Array(), + ), + Set(0) + ) + + val expected = Node( + mutable.Map( + 0 -> Node( + mutable.Map( + 1 -> Node( + mutable.Map( + 2 -> Node( + mutable.Map( + 3 -> Node(mutable.Map()) + ) + ) + ) + ) + ) + ) + ) + ) + + assert(forest == expected + ) + } + + } + +} From 9650199fcfa30f91fcfce1219b3c3d62bfcaef48 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 15 Dec 2024 00:42:03 +0800 Subject: [PATCH 04/13] wip --- main/codesig/src/ReachabilityAnalysis.scala | 10 +++++++++- main/eval/src/mill/eval/EvaluatorCore.scala | 4 ++-- main/util/src/mill/util/SpanningForest.scala | 13 +++---------- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/main/codesig/src/ReachabilityAnalysis.scala b/main/codesig/src/ReachabilityAnalysis.scala index 1a1dc22b354..9cf7d591b37 100644 --- a/main/codesig/src/ReachabilityAnalysis.scala +++ b/main/codesig/src/ReachabilityAnalysis.scala @@ -126,8 +126,16 @@ object CallGraphAnalysis { } .toSet + val reverseGraphMap = indexGraphEdges + .zipWithIndex + .flatMap { case (vs, k) => vs.map((_, k)) } + .groupMap(_._1)(_._2) + SpanningForest.spanningTreeToJsonTree( - SpanningForest.apply(indexGraphEdges, nodesWithChangedHashes), + SpanningForest.apply( + indexGraphEdges.indices.map(reverseGraphMap(_)).toArray, + nodesWithChangedHashes + ), k => indexToNodes(k).toString ) } diff --git a/main/eval/src/mill/eval/EvaluatorCore.scala b/main/eval/src/mill/eval/EvaluatorCore.scala index b09261ea50f..e5c6ca81bfb 100644 --- a/main/eval/src/mill/eval/EvaluatorCore.scala +++ b/main/eval/src/mill/eval/EvaluatorCore.scala @@ -272,14 +272,14 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { os.write.over( outPath / OutFiles.millInvalidationForest, SpanningForest.spanningTreeToJsonTree( - SpanningForest(upstreamIndexEdges, uncached.keys().asScala.map(terminalToIndex).toSet), + SpanningForest(downstreamIndexEdges, uncached.keys().asScala.map(terminalToIndex).toSet), i => indexToTerminal(i).render ).render(indent = 2) ) os.write.over( outPath / OutFiles.millDependencyForest, SpanningForest.spanningTreeToJsonTree( - SpanningForest(downstreamIndexEdges, indexToTerminal.indices.toSet), + SpanningForest(upstreamIndexEdges, indexToTerminal.indices.toSet), i => indexToTerminal(i).render ).render(indent = 2) ) diff --git a/main/util/src/mill/util/SpanningForest.scala b/main/util/src/mill/util/SpanningForest.scala index b594d2f830e..bfd46878313 100644 --- a/main/util/src/mill/util/SpanningForest.scala +++ b/main/util/src/mill/util/SpanningForest.scala @@ -22,9 +22,8 @@ private[mill] object SpanningForest { def apply(indexGraphEdges: Array[Array[Int]], importantVertices: Set[Int]): Node = { // Find all importantVertices which are "roots" with no incoming edges // from other importantVertices - val rootChangedNodeIndices = importantVertices.filter(i => - !indexGraphEdges(i).exists(importantVertices.contains(_)) - ) + val destinations = importantVertices.flatMap(indexGraphEdges(_)) + val rootChangedNodeIndices = importantVertices.filter(!destinations.contains(_)) // Prepare a mutable tree structure that we will return, pre-populated with // just the first level of nodes from the `rootChangedNodeIndices`, as well @@ -35,15 +34,9 @@ private[mill] object SpanningForest { // Do a breadth first search from the `rootChangedNodeIndices` across the // reverse edges of the graph to build up the spanning forest - val downstreamGraphEdges = indexGraphEdges - .zipWithIndex - .flatMap { case (vs, k) => vs.map((_, k)) } - .groupMap(_._1)(_._2) - breadthFirst(rootChangedNodeIndices) { index => // needed to add explicit type for Scala 3.5.0-RC6 - val nextIndices = downstreamGraphEdges - .getOrElse(index, Array[Int]()) + val nextIndices = indexGraphEdges(index) .filter(importantVertices) // We build up the spanningForest during a normal breadth first search, From 770379e49dc163500244a9b8c28265719419cf96 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 15 Dec 2024 01:21:59 +0800 Subject: [PATCH 05/13] wip --- main/eval/src/mill/eval/EvaluatorCore.scala | 11 ++++++-- main/eval/src/mill/eval/GroupEvaluator.scala | 28 ++++++++++++++------ 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/main/eval/src/mill/eval/EvaluatorCore.scala b/main/eval/src/mill/eval/EvaluatorCore.scala index e5c6ca81bfb..7711c318285 100644 --- a/main/eval/src/mill/eval/EvaluatorCore.scala +++ b/main/eval/src/mill/eval/EvaluatorCore.scala @@ -87,6 +87,7 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { CodeSigUtils.precomputeMethodNamesPerClass(sortedGroups) val uncached = new java.util.concurrent.ConcurrentHashMap[Terminal, Unit]() + val changedValueHash = new java.util.concurrent.ConcurrentHashMap[Terminal, Unit]() def evaluateTerminals( terminals: Seq[Terminal], @@ -118,7 +119,7 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { val taskResults = group.map(t => (t, TaskResult[(Val, Int)](failure, () => failure))).toMap futures(terminal) = Future.successful( - Some(GroupEvaluator.Results(taskResults, group.toSeq, false, -1, -1)) + Some(GroupEvaluator.Results(taskResults, group.toSeq, false, -1, -1, false)) ) } else { futures(terminal) = Future.sequence(deps.map(futures)).map { upstreamValues => @@ -195,6 +196,7 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { cached = res.cached ) if (!res.cached) uncached.put(terminal, ()) + if (res.changedValueHash) changedValueHash.put(terminal, ()) profileLogger.log( ProfileLogger.Timing( @@ -266,7 +268,12 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { val indexToTerminal = sortedGroups.keys().toArray val terminalToIndex = indexToTerminal.zipWithIndex.toMap - val downstreamIndexEdges = indexToTerminal.map(t => reverseInterGroupDeps.getOrElse(t, Nil).map(terminalToIndex).toArray) + val changedTerminalIndices = changedValueHash.keys().asScala.toSet + val downstreamIndexEdges = indexToTerminal + .map(t => + if (changedTerminalIndices(t)) reverseInterGroupDeps.getOrElse(t, Nil).map(terminalToIndex).toArray + else Array.empty[Int] + ) val upstreamIndexEdges = indexToTerminal.map(t => interGroupDeps.getOrElse(t, Nil).map(terminalToIndex).toArray) os.write.over( diff --git a/main/eval/src/mill/eval/GroupEvaluator.scala b/main/eval/src/mill/eval/GroupEvaluator.scala index 7ef94d96e7b..2486d90f22b 100644 --- a/main/eval/src/mill/eval/GroupEvaluator.scala +++ b/main/eval/src/mill/eval/GroupEvaluator.scala @@ -103,7 +103,7 @@ private[mill] trait GroupEvaluator { executionContext, exclusive ) - GroupEvaluator.Results(newResults, newEvaluated.toSeq, null, inputsHash, -1) + GroupEvaluator.Results(newResults, newEvaluated.toSeq, null, inputsHash, -1, changedValueHash = false) case labelled: Terminal.Labelled[_] => val out = @@ -126,7 +126,7 @@ private[mill] trait GroupEvaluator { cached.isEmpty ) - upToDateWorker.map((_, inputsHash)) orElse cached.flatMap(_._2) match { + upToDateWorker.map((_, inputsHash)) orElse cached.flatMap { case (inputHash, valOpt, valueHash) => valOpt.map((_, valueHash))} match { case Some((v, hashCode)) => val res = Result.Success((v, hashCode)) val newResults: Map[Task[_], TaskResult[(Val, Int)]] = @@ -137,7 +137,8 @@ private[mill] trait GroupEvaluator { Nil, cached = true, inputsHash, - -1 + -1, + changedValueHash = false ) case _ => @@ -160,12 +161,14 @@ private[mill] trait GroupEvaluator { exclusive ) - newResults(labelled.task) match { + val valueHash = newResults(labelled.task) match { case TaskResult(Result.Failure(_, Some((v, _))), _) => handleTaskResult(v, v.##, paths.meta, inputsHash, labelled) + v.## case TaskResult(Result.Success((v, _)), _) => handleTaskResult(v, v.##, paths.meta, inputsHash, labelled) + v.## case _ => // Wipe out any cached meta.json file that exists, so @@ -173,14 +176,21 @@ private[mill] trait GroupEvaluator { // assume it's associated with the possibly-borked state of the // destPath after an evaluation failure. os.remove.all(paths.meta) + 0 } + mill.main.client.DebugLog.println("") + mill.main.client.DebugLog.println(labelled.render) + mill.main.client.DebugLog.println(cached.map(_._3).toString) + mill.main.client.DebugLog.println(valueHash.toString) + GroupEvaluator.Results( newResults, newEvaluated.toSeq, cached = if (labelled.task.isInstanceOf[InputImpl[_]]) null else false, inputsHash, - cached.map(_._1).getOrElse(-1) + cached.map(_._1).getOrElse(-1), + !cached.map(_._3).contains(valueHash) ) } } @@ -383,7 +393,7 @@ private[mill] trait GroupEvaluator { inputsHash: Int, labelled: Terminal.Labelled[_], paths: EvaluatorPaths - ): Option[(Int, Option[(Val, Int)])] = { + ): Option[(Int, Option[Val], Int)] = { for { cached <- try Some(upickle.default.read[Evaluator.Cached](paths.meta.toIO)) @@ -405,7 +415,8 @@ private[mill] trait GroupEvaluator { None case NonFatal(_) => None } - } yield (Val(parsed), cached.valueHash) + } yield Val(parsed), + cached.valueHash ) } @@ -457,6 +468,7 @@ private[mill] object GroupEvaluator { newEvaluated: Seq[Task[_]], cached: java.lang.Boolean, inputsHash: Int, - previousInputsHash: Int + previousInputsHash: Int, + changedValueHash: Boolean ) } From 2b3c53c0269a594716b74ac2cfb5d2c3f9ab75e2 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 15 Dec 2024 09:29:27 +0800 Subject: [PATCH 06/13] wip --- kotlinlib/package.mill | 1 + main/eval/src/mill/eval/EvaluatorCore.scala | 22 +++++++++++--------- main/eval/src/mill/eval/GroupEvaluator.scala | 5 ----- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/kotlinlib/package.mill b/kotlinlib/package.mill index f0fce59c5e8..bd551f16dca 100644 --- a/kotlinlib/package.mill +++ b/kotlinlib/package.mill @@ -25,6 +25,7 @@ object `package` extends RootModule with build.MillPublishScalaModule with Build BuildInfo.Value("freemarkerDep", Dep.unparse(build.Deps.RuntimeDeps.freemarker).get, "freemarker dependency (used for Dokka)") ) + trait MillKotlinModule extends build.MillPublishScalaModule { override def javacOptions = { val release = diff --git a/main/eval/src/mill/eval/EvaluatorCore.scala b/main/eval/src/mill/eval/EvaluatorCore.scala index 7711c318285..4be359f4e03 100644 --- a/main/eval/src/mill/eval/EvaluatorCore.scala +++ b/main/eval/src/mill/eval/EvaluatorCore.scala @@ -77,6 +77,16 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { val terminals0 = sortedGroups.keys().toVector val failed = new AtomicBoolean(false) val count = new AtomicInteger(1) + val indexToTerminal = sortedGroups.keys().toArray + val terminalToIndex = indexToTerminal.zipWithIndex.toMap + val upstreamIndexEdges = indexToTerminal.map(t => interGroupDeps.getOrElse(t, Nil).map(terminalToIndex).toArray) + os.write.over( + outPath / OutFiles.millDependencyForest, + SpanningForest.spanningTreeToJsonTree( + SpanningForest(upstreamIndexEdges, indexToTerminal.indices.toSet), + i => indexToTerminal(i).render + ).render(indent = 2) + ) val futures = mutable.Map.empty[Terminal, Future[Option[GroupEvaluator.Results]]] @@ -266,15 +276,14 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { .toSeq .groupMap(_._1)(_._2) - val indexToTerminal = sortedGroups.keys().toArray - val terminalToIndex = indexToTerminal.zipWithIndex.toMap + val changedTerminalIndices = changedValueHash.keys().asScala.toSet val downstreamIndexEdges = indexToTerminal .map(t => if (changedTerminalIndices(t)) reverseInterGroupDeps.getOrElse(t, Nil).map(terminalToIndex).toArray else Array.empty[Int] ) - val upstreamIndexEdges = indexToTerminal.map(t => interGroupDeps.getOrElse(t, Nil).map(terminalToIndex).toArray) + os.write.over( outPath / OutFiles.millInvalidationForest, @@ -283,13 +292,6 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { i => indexToTerminal(i).render ).render(indent = 2) ) - os.write.over( - outPath / OutFiles.millDependencyForest, - SpanningForest.spanningTreeToJsonTree( - SpanningForest(upstreamIndexEdges, indexToTerminal.indices.toSet), - i => indexToTerminal(i).render - ).render(indent = 2) - ) EvaluatorCore.Results( goals.indexed.map(results(_).map(_._1).result), diff --git a/main/eval/src/mill/eval/GroupEvaluator.scala b/main/eval/src/mill/eval/GroupEvaluator.scala index 2486d90f22b..11bff00d26e 100644 --- a/main/eval/src/mill/eval/GroupEvaluator.scala +++ b/main/eval/src/mill/eval/GroupEvaluator.scala @@ -179,11 +179,6 @@ private[mill] trait GroupEvaluator { 0 } - mill.main.client.DebugLog.println("") - mill.main.client.DebugLog.println(labelled.render) - mill.main.client.DebugLog.println(cached.map(_._3).toString) - mill.main.client.DebugLog.println(valueHash.toString) - GroupEvaluator.Results( newResults, newEvaluated.toSeq, From 65673826d292d6ac340a1e01f5f91e5ec3f44f5e Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 15 Dec 2024 10:07:14 +0800 Subject: [PATCH 07/13] mandatory methodCodeHashSignatures spanningInvalidationForest --- kotlinlib/package.mill | 10 +++++++- .../kotlinlib/worker/api/KotlinWorker.scala | 2 +- main/codesig/src/Logger.scala | 25 +++++++++++-------- main/codesig/src/ReachabilityAnalysis.scala | 12 ++++----- main/codesig/test/src/Util.scala | 2 +- .../src/mill/runner/MillBuildRootModule.scala | 5 +++- 6 files changed, 36 insertions(+), 20 deletions(-) diff --git a/kotlinlib/package.mill b/kotlinlib/package.mill index bd551f16dca..d714b916ce3 100644 --- a/kotlinlib/package.mill +++ b/kotlinlib/package.mill @@ -12,10 +12,19 @@ object `package` extends RootModule with build.MillPublishScalaModule with Build def moduleDeps = Seq(build.main, build.scalalib, build.testrunner, worker) def testTransitiveDeps = super.testTransitiveDeps() ++ Seq(worker.impl.testDep()) + val x = 1 + val y = 1 + val z = 1 + val z2 = 1 + val z3 = 1 + val z4 = 1 def buildInfoPackageName = "mill.kotlinlib" def buildInfoObjectName = "Versions" def buildInfoMembers = Seq( BuildInfo.Value("kotlinVersion", build.Deps.kotlinVersion, "Version of Kotlin"), + BuildInfo.Value("kotlinVersion2", build.Deps.kotlinVersion, "Version of Kotlin"), + BuildInfo.Value("kotlinVersion3", build.Deps.kotlinVersion, "Version of Kotlin"), + BuildInfo.Value("kotlinVersion4", build.Deps.kotlinVersion, "Version of Kotlin"), BuildInfo.Value("koverVersion", build.Deps.RuntimeDeps.koverVersion, "Version of Kover."), BuildInfo.Value("ktfmtVersion", build.Deps.RuntimeDeps.ktfmt.version, "Version of Ktfmt."), BuildInfo.Value("ktlintVersion", build.Deps.RuntimeDeps.ktlint.version, "Version of ktlint."), @@ -25,7 +34,6 @@ object `package` extends RootModule with build.MillPublishScalaModule with Build BuildInfo.Value("freemarkerDep", Dep.unparse(build.Deps.RuntimeDeps.freemarker).get, "freemarker dependency (used for Dokka)") ) - trait MillKotlinModule extends build.MillPublishScalaModule { override def javacOptions = { val release = diff --git a/kotlinlib/worker/src/mill/kotlinlib/worker/api/KotlinWorker.scala b/kotlinlib/worker/src/mill/kotlinlib/worker/api/KotlinWorker.scala index 112f8b19fc0..97f1f9ad4c3 100644 --- a/kotlinlib/worker/src/mill/kotlinlib/worker/api/KotlinWorker.scala +++ b/kotlinlib/worker/src/mill/kotlinlib/worker/api/KotlinWorker.scala @@ -10,7 +10,7 @@ import mill.api.{Ctx, Result} trait KotlinWorker { def compile(target: KotlinWorkerTarget, args: Seq[String])(implicit ctx: Ctx): Result[Unit] - + val x = 1 } sealed class KotlinWorkerTarget diff --git a/main/codesig/src/Logger.scala b/main/codesig/src/Logger.scala index c0a8e2a1038..d54c30edc9d 100644 --- a/main/codesig/src/Logger.scala +++ b/main/codesig/src/Logger.scala @@ -1,18 +1,23 @@ package mill.codesig -class Logger(logFolder: Option[os.Path]) { +class Logger(mandatoryLogFolder: os.Path, logFolder: Option[os.Path]) { logFolder.foreach(os.remove.all(_)) + os.remove.all(mandatoryLogFolder) private var count = 1 - def log[T: upickle.default.Writer](t: => sourcecode.Text[T], prefix: String = ""): Unit = { + def log0[T: upickle.default.Writer](p: os.Path, t: => sourcecode.Text[T], prefix: String = "") = { lazy val res = t - logFolder.foreach { p => - os.write( - p / s"$count-$prefix${res.source}.json", - upickle.default.stream(res.value, indent = 4), - createFolders = true - ) - count += 1 - } + os.write( + p / s"$count-$prefix${res.source}.json", + upickle.default.stream(res.value, indent = 4), + createFolders = true + ) + count += 1 + } + def log[T: upickle.default.Writer](t: => sourcecode.Text[T], prefix: String = ""): Unit = { + logFolder.foreach(log0(_, t, prefix)) + } + def mandatoryLog[T: upickle.default.Writer](t: => sourcecode.Text[T], prefix: String = ""): Unit = { + log0(mandatoryLogFolder, t, prefix) } } diff --git a/main/codesig/src/ReachabilityAnalysis.scala b/main/codesig/src/ReachabilityAnalysis.scala index 9cf7d591b37..aa587b80e61 100644 --- a/main/codesig/src/ReachabilityAnalysis.scala +++ b/main/codesig/src/ReachabilityAnalysis.scala @@ -77,7 +77,7 @@ class CallGraphAnalysis( .collect { case (CallGraphAnalysis.LocalDef(d), v) => (d.toString, v) } .to(SortedMap) - logger.log(transitiveCallGraphHashes) + logger.mandatoryLog(transitiveCallGraphHashes) lazy val spanningInvalidationForest: Obj = prevTransitiveCallGraphHashesOpt() match { case Some(prevTransitiveCallGraphHashes) => @@ -90,7 +90,7 @@ class CallGraphAnalysis( case None => ujson.Obj() } - logger.log(spanningInvalidationForest) + logger.mandatoryLog(spanningInvalidationForest) } object CallGraphAnalysis { @@ -131,11 +131,11 @@ object CallGraphAnalysis { .flatMap { case (vs, k) => vs.map((_, k)) } .groupMap(_._1)(_._2) + val reverseGraphEdges = indexGraphEdges.indices.map(reverseGraphMap.getOrElse(_, Array())).toArray + mill.main.client.DebugLog.println(pprint.apply(nodesWithChangedHashes).toString()) + mill.main.client.DebugLog.println(pprint.apply(reverseGraphEdges).toString()) SpanningForest.spanningTreeToJsonTree( - SpanningForest.apply( - indexGraphEdges.indices.map(reverseGraphMap(_)).toArray, - nodesWithChangedHashes - ), + SpanningForest.apply(reverseGraphEdges, nodesWithChangedHashes), k => indexToNodes(k).toString ) } diff --git a/main/codesig/test/src/Util.scala b/main/codesig/test/src/Util.scala index 23b19eafbd5..503f1dfb3b6 100644 --- a/main/codesig/test/src/Util.scala +++ b/main/codesig/test/src/Util.scala @@ -19,7 +19,7 @@ object TestUtil { .map(os.Path(_)) ), (_, _) => false, - new Logger(Some(testLogFolder)), + new Logger(testLogFolder, Some(testLogFolder)), () => None ) } diff --git a/runner/src/mill/runner/MillBuildRootModule.scala b/runner/src/mill/runner/MillBuildRootModule.scala index 814228f4790..d9d49cfb4fd 100644 --- a/runner/src/mill/runner/MillBuildRootModule.scala +++ b/runner/src/mill/runner/MillBuildRootModule.scala @@ -178,7 +178,10 @@ abstract class MillBuildRootModule()(implicit (isSimpleTarget && !isForwarderCallsite) || isCommand || isMillDiscover }, - logger = new mill.codesig.Logger(Option.when(debugEnabled)(T.dest / "current")), + logger = new mill.codesig.Logger( + T.dest / "current", + Option.when(debugEnabled)(T.dest / "current") + ), prevTransitiveCallGraphHashesOpt = () => Option.when(os.exists(T.dest / "previous/result.json"))( upickle.default.read[Map[String, Int]]( From 2b49d34f53c91e8860a48b3baf4854fe06a4d44f Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 15 Dec 2024 11:15:10 +0800 Subject: [PATCH 08/13] . --- kotlinlib/package.mill | 3 +++ main/codesig/src/Logger.scala | 7 +++---- main/codesig/src/ReachabilityAnalysis.scala | 11 ++++++----- .../test/src/{Util.scala => TestUtil.scala} | 0 main/eval/src/mill/eval/EvaluatorCore.scala | 4 ++-- main/util/src/mill/util/SpanningForest.scala | 6 ++++-- .../test/src/mill/util/SpanningForestTests.scala | 3 ++- runner/src/mill/runner/MillBuildRootModule.scala | 16 +++------------- 8 files changed, 23 insertions(+), 27 deletions(-) rename main/codesig/test/src/{Util.scala => TestUtil.scala} (100%) diff --git a/kotlinlib/package.mill b/kotlinlib/package.mill index d714b916ce3..b3b148b29f5 100644 --- a/kotlinlib/package.mill +++ b/kotlinlib/package.mill @@ -18,6 +18,9 @@ object `package` extends RootModule with build.MillPublishScalaModule with Build val z2 = 1 val z3 = 1 val z4 = 1 + val z5 = 1 + val z6 = 1 + val z7 = 7 def buildInfoPackageName = "mill.kotlinlib" def buildInfoObjectName = "Versions" def buildInfoMembers = Seq( diff --git a/main/codesig/src/Logger.scala b/main/codesig/src/Logger.scala index d54c30edc9d..2c722a6c4eb 100644 --- a/main/codesig/src/Logger.scala +++ b/main/codesig/src/Logger.scala @@ -5,17 +5,16 @@ class Logger(mandatoryLogFolder: os.Path, logFolder: Option[os.Path]) { os.remove.all(mandatoryLogFolder) private var count = 1 - def log0[T: upickle.default.Writer](p: os.Path, t: => sourcecode.Text[T], prefix: String = "") = { - lazy val res = t + def log0[T: upickle.default.Writer](p: os.Path, res: sourcecode.Text[T], prefix: String = "") = { os.write( - p / s"$count-$prefix${res.source}.json", + p / s"$prefix${res.source}.json", upickle.default.stream(res.value, indent = 4), createFolders = true ) count += 1 } def log[T: upickle.default.Writer](t: => sourcecode.Text[T], prefix: String = ""): Unit = { - logFolder.foreach(log0(_, t, prefix)) + logFolder.foreach(log0(_, t, s"$count-$prefix")) } def mandatoryLog[T: upickle.default.Writer](t: => sourcecode.Text[T], prefix: String = ""): Unit = { log0(mandatoryLogFolder, t, prefix) diff --git a/main/codesig/src/ReachabilityAnalysis.scala b/main/codesig/src/ReachabilityAnalysis.scala index aa587b80e61..74a5e41a8df 100644 --- a/main/codesig/src/ReachabilityAnalysis.scala +++ b/main/codesig/src/ReachabilityAnalysis.scala @@ -39,7 +39,7 @@ class CallGraphAnalysis( lazy val methodCodeHashes: SortedMap[String, Int] = methods.map { case (k, vs) => (k.toString, vs.codeHash) }.to(SortedMap) - logger.log(methodCodeHashes) + logger.mandatoryLog(methodCodeHashes) lazy val prettyCallGraph: SortedMap[String, Array[CallGraphAnalysis.Node]] = { indexGraphEdges.zip(indexToNodes).map { case (vs, k) => @@ -48,7 +48,7 @@ class CallGraphAnalysis( .to(SortedMap) } - logger.log(prettyCallGraph) + logger.mandatoryLog(prettyCallGraph) def transitiveCallGraphValues[V: scala.reflect.ClassTag]( nodeValues: Array[V], @@ -132,10 +132,11 @@ object CallGraphAnalysis { .groupMap(_._1)(_._2) val reverseGraphEdges = indexGraphEdges.indices.map(reverseGraphMap.getOrElse(_, Array())).toArray - mill.main.client.DebugLog.println(pprint.apply(nodesWithChangedHashes).toString()) - mill.main.client.DebugLog.println(pprint.apply(reverseGraphEdges).toString()) +// pprint.log(nodesWithChangedHashes) +// pprint.log(reverseGraphEdges) + SpanningForest.spanningTreeToJsonTree( - SpanningForest.apply(reverseGraphEdges, nodesWithChangedHashes), + SpanningForest.apply(reverseGraphEdges, nodesWithChangedHashes, false), k => indexToNodes(k).toString ) } diff --git a/main/codesig/test/src/Util.scala b/main/codesig/test/src/TestUtil.scala similarity index 100% rename from main/codesig/test/src/Util.scala rename to main/codesig/test/src/TestUtil.scala diff --git a/main/eval/src/mill/eval/EvaluatorCore.scala b/main/eval/src/mill/eval/EvaluatorCore.scala index 4be359f4e03..eeccaf7cd02 100644 --- a/main/eval/src/mill/eval/EvaluatorCore.scala +++ b/main/eval/src/mill/eval/EvaluatorCore.scala @@ -83,7 +83,7 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { os.write.over( outPath / OutFiles.millDependencyForest, SpanningForest.spanningTreeToJsonTree( - SpanningForest(upstreamIndexEdges, indexToTerminal.indices.toSet), + SpanningForest(upstreamIndexEdges, indexToTerminal.indices.toSet, true), i => indexToTerminal(i).render ).render(indent = 2) ) @@ -288,7 +288,7 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { os.write.over( outPath / OutFiles.millInvalidationForest, SpanningForest.spanningTreeToJsonTree( - SpanningForest(downstreamIndexEdges, uncached.keys().asScala.map(terminalToIndex).toSet), + SpanningForest(downstreamIndexEdges, uncached.keys().asScala.map(terminalToIndex).toSet, true), i => indexToTerminal(i).render ).render(indent = 2) ) diff --git a/main/util/src/mill/util/SpanningForest.scala b/main/util/src/mill/util/SpanningForest.scala index bfd46878313..442fe0e36d9 100644 --- a/main/util/src/mill/util/SpanningForest.scala +++ b/main/util/src/mill/util/SpanningForest.scala @@ -19,7 +19,9 @@ private[mill] object SpanningForest { ) } case class Node(values: mutable.Map[Int, Node] = mutable.Map()) - def apply(indexGraphEdges: Array[Array[Int]], importantVertices: Set[Int]): Node = { + def apply(indexGraphEdges: Array[Array[Int]], + importantVertices: Set[Int], + limitToImportantVertices: Boolean): Node = { // Find all importantVertices which are "roots" with no incoming edges // from other importantVertices val destinations = importantVertices.flatMap(indexGraphEdges(_)) @@ -37,7 +39,7 @@ private[mill] object SpanningForest { breadthFirst(rootChangedNodeIndices) { index => // needed to add explicit type for Scala 3.5.0-RC6 val nextIndices = indexGraphEdges(index) - .filter(importantVertices) + .filter(e => !limitToImportantVertices || importantVertices(e)) // We build up the spanningForest during a normal breadth first search, // using the `nodeMapping` to quickly find a vertice's tree node so we diff --git a/main/util/test/src/mill/util/SpanningForestTests.scala b/main/util/test/src/mill/util/SpanningForestTests.scala index a5e1de396c6..832620275f4 100644 --- a/main/util/test/src/mill/util/SpanningForestTests.scala +++ b/main/util/test/src/mill/util/SpanningForestTests.scala @@ -16,7 +16,8 @@ object SpanningForestTests extends TestSuite { Array(), Array(), ), - Set(0) + Set(0), + limitToImportantVertices = false ) val expected = Node( diff --git a/runner/src/mill/runner/MillBuildRootModule.scala b/runner/src/mill/runner/MillBuildRootModule.scala index d9d49cfb4fd..bea05d234f0 100644 --- a/runner/src/mill/runner/MillBuildRootModule.scala +++ b/runner/src/mill/runner/MillBuildRootModule.scala @@ -183,24 +183,14 @@ abstract class MillBuildRootModule()(implicit Option.when(debugEnabled)(T.dest / "current") ), prevTransitiveCallGraphHashesOpt = () => - Option.when(os.exists(T.dest / "previous/result.json"))( + Option.when(os.exists(T.dest / "previous/transitiveCallGraphHashes.json"))( upickle.default.read[Map[String, Int]]( - os.read.stream(T.dest / "previous/result.json") + os.read.stream(T.dest / "previous/transitiveCallGraphHashes.json") ) ) ) - val result = codesig.transitiveCallGraphHashes - if (debugEnabled) { - os.write( - T.dest / "current/result.json", - upickle.default.stream( - SortedMap.from(codesig.transitiveCallGraphHashes0.map { case (k, v) => (k.toString, v) }), - indent = 4 - ) - ) - } - result + codesig.transitiveCallGraphHashes } override def sources: T[Seq[PathRef]] = Task { From 74820450eb5da4c66d788770b1d8e9d0ceb44aa1 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 15 Dec 2024 18:30:35 +0800 Subject: [PATCH 09/13] fix spanningInvalidationForest.json --- example/javalib/basic/1-simple/build.mill | 1 + main/codesig/src/ReachabilityAnalysis.scala | 10 ++++------ runner/src/mill/runner/MillBuildRootModule.scala | 4 ++-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/example/javalib/basic/1-simple/build.mill b/example/javalib/basic/1-simple/build.mill index 67abcd1e759..2bd2abad280 100644 --- a/example/javalib/basic/1-simple/build.mill +++ b/example/javalib/basic/1-simple/build.mill @@ -9,6 +9,7 @@ object foo extends JavaModule { ) object test extends JavaTests with TestModule.Junit4 { + val x = 8 def ivyDeps = super.ivyDeps() ++ Agg( ivy"com.google.guava:guava:33.3.0-jre" ) diff --git a/main/codesig/src/ReachabilityAnalysis.scala b/main/codesig/src/ReachabilityAnalysis.scala index 74a5e41a8df..b5ecaa30bc9 100644 --- a/main/codesig/src/ReachabilityAnalysis.scala +++ b/main/codesig/src/ReachabilityAnalysis.scala @@ -39,7 +39,7 @@ class CallGraphAnalysis( lazy val methodCodeHashes: SortedMap[String, Int] = methods.map { case (k, vs) => (k.toString, vs.codeHash) }.to(SortedMap) - logger.mandatoryLog(methodCodeHashes) + logger.log(methodCodeHashes) lazy val prettyCallGraph: SortedMap[String, Array[CallGraphAnalysis.Node]] = { indexGraphEdges.zip(indexToNodes).map { case (vs, k) => @@ -48,7 +48,7 @@ class CallGraphAnalysis( .to(SortedMap) } - logger.mandatoryLog(prettyCallGraph) + logger.log(prettyCallGraph) def transitiveCallGraphValues[V: scala.reflect.ClassTag]( nodeValues: Array[V], @@ -77,7 +77,8 @@ class CallGraphAnalysis( .collect { case (CallGraphAnalysis.LocalDef(d), v) => (d.toString, v) } .to(SortedMap) - logger.mandatoryLog(transitiveCallGraphHashes) + logger.mandatoryLog(transitiveCallGraphHashes0) + logger.log(transitiveCallGraphHashes) lazy val spanningInvalidationForest: Obj = prevTransitiveCallGraphHashesOpt() match { case Some(prevTransitiveCallGraphHashes) => @@ -121,7 +122,6 @@ object CallGraphAnalysis { .filter { nodeIndex => val currentValue = transitiveCallGraphHashes0Map(indexToNodes(nodeIndex)) val prevValue = prevTransitiveCallGraphHashes.get(indexToNodes(nodeIndex).toString) - !prevValue.contains(currentValue) } .toSet @@ -132,8 +132,6 @@ object CallGraphAnalysis { .groupMap(_._1)(_._2) val reverseGraphEdges = indexGraphEdges.indices.map(reverseGraphMap.getOrElse(_, Array())).toArray -// pprint.log(nodesWithChangedHashes) -// pprint.log(reverseGraphEdges) SpanningForest.spanningTreeToJsonTree( SpanningForest.apply(reverseGraphEdges, nodesWithChangedHashes, false), diff --git a/runner/src/mill/runner/MillBuildRootModule.scala b/runner/src/mill/runner/MillBuildRootModule.scala index bea05d234f0..3a2b793fe29 100644 --- a/runner/src/mill/runner/MillBuildRootModule.scala +++ b/runner/src/mill/runner/MillBuildRootModule.scala @@ -183,9 +183,9 @@ abstract class MillBuildRootModule()(implicit Option.when(debugEnabled)(T.dest / "current") ), prevTransitiveCallGraphHashesOpt = () => - Option.when(os.exists(T.dest / "previous/transitiveCallGraphHashes.json"))( + Option.when(os.exists(T.dest / "previous/transitiveCallGraphHashes0.json"))( upickle.default.read[Map[String, Int]]( - os.read.stream(T.dest / "previous/transitiveCallGraphHashes.json") + os.read.stream(T.dest / "previous/transitiveCallGraphHashes0.json") ) ) ) From 841a95b907ccb1534af3f91c979653cabf497b19 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 15 Dec 2024 18:43:09 +0800 Subject: [PATCH 10/13] more consistently special case worker valueHash as inputHash --- example/javalib/basic/1-simple/build.mill | 2 +- main/eval/src/mill/eval/EvaluatorCore.scala | 3 +- main/eval/src/mill/eval/GroupEvaluator.scala | 37 +++++++++---------- main/eval/src/mill/eval/JsonArrayLogger.scala | 1 + 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/example/javalib/basic/1-simple/build.mill b/example/javalib/basic/1-simple/build.mill index 2bd2abad280..2fedbe96320 100644 --- a/example/javalib/basic/1-simple/build.mill +++ b/example/javalib/basic/1-simple/build.mill @@ -9,7 +9,7 @@ object foo extends JavaModule { ) object test extends JavaTests with TestModule.Junit4 { - val x = 8 + val x = 81 def ivyDeps = super.ivyDeps() ++ Agg( ivy"com.google.guava:guava:33.3.0-jre" ) diff --git a/main/eval/src/mill/eval/EvaluatorCore.scala b/main/eval/src/mill/eval/EvaluatorCore.scala index eeccaf7cd02..6054402e74c 100644 --- a/main/eval/src/mill/eval/EvaluatorCore.scala +++ b/main/eval/src/mill/eval/EvaluatorCore.scala @@ -206,13 +206,14 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { cached = res.cached ) if (!res.cached) uncached.put(terminal, ()) - if (res.changedValueHash) changedValueHash.put(terminal, ()) + if (res.valueHashChanged) changedValueHash.put(terminal, ()) profileLogger.log( ProfileLogger.Timing( terminal.render, (duration / 1000).toInt, res.cached, + res.valueHashChanged, deps.map(_.render), res.inputsHash, res.previousInputsHash diff --git a/main/eval/src/mill/eval/GroupEvaluator.scala b/main/eval/src/mill/eval/GroupEvaluator.scala index 11bff00d26e..476224d4ee9 100644 --- a/main/eval/src/mill/eval/GroupEvaluator.scala +++ b/main/eval/src/mill/eval/GroupEvaluator.scala @@ -103,17 +103,14 @@ private[mill] trait GroupEvaluator { executionContext, exclusive ) - GroupEvaluator.Results(newResults, newEvaluated.toSeq, null, inputsHash, -1, changedValueHash = false) + GroupEvaluator.Results(newResults, newEvaluated.toSeq, null, inputsHash, -1, valueHashChanged = false) case labelled: Terminal.Labelled[_] => val out = if (!labelled.task.ctx.external) outPath else externalOutPath - val paths = EvaluatorPaths.resolveDestPaths( - out, - Terminal.destSegments(labelled) - ) + val paths = EvaluatorPaths.resolveDestPaths(out, Terminal.destSegments(labelled)) val cached = loadCachedJson(logger, inputsHash, labelled, paths) @@ -121,9 +118,8 @@ private[mill] trait GroupEvaluator { logger, inputsHash, labelled, - forceDiscard = - // worker metadata file removed by user, let's recompute the worker - cached.isEmpty + // worker metadata file removed by user, let's recompute the worker + forceDiscard = cached.isEmpty ) upToDateWorker.map((_, inputsHash)) orElse cached.flatMap { case (inputHash, valOpt, valueHash) => valOpt.map((_, valueHash))} match { @@ -138,7 +134,7 @@ private[mill] trait GroupEvaluator { cached = true, inputsHash, -1, - changedValueHash = false + valueHashChanged = false ) case _ => @@ -161,14 +157,17 @@ private[mill] trait GroupEvaluator { exclusive ) + val valueHash = newResults(labelled.task) match { case TaskResult(Result.Failure(_, Some((v, _))), _) => - handleTaskResult(v, v.##, paths.meta, inputsHash, labelled) - v.## + val valueHash = if (terminal.task.asWorker.isEmpty) v.## else inputsHash + handleTaskResult(v, valueHash, paths.meta, inputsHash, labelled) + case TaskResult(Result.Success((v, _)), _) => - handleTaskResult(v, v.##, paths.meta, inputsHash, labelled) - v.## + val valueHash = if (terminal.task.asWorker.isEmpty) v.## else inputsHash + handleTaskResult(v, valueHash, paths.meta, inputsHash, labelled) + valueHash case _ => // Wipe out any cached meta.json file that exists, so @@ -459,11 +458,11 @@ private[mill] trait GroupEvaluator { private[mill] object GroupEvaluator { case class Results( - newResults: Map[Task[_], TaskResult[(Val, Int)]], - newEvaluated: Seq[Task[_]], - cached: java.lang.Boolean, - inputsHash: Int, - previousInputsHash: Int, - changedValueHash: Boolean + newResults: Map[Task[_], TaskResult[(Val, Int)]], + newEvaluated: Seq[Task[_]], + cached: java.lang.Boolean, + inputsHash: Int, + previousInputsHash: Int, + valueHashChanged: Boolean ) } diff --git a/main/eval/src/mill/eval/JsonArrayLogger.scala b/main/eval/src/mill/eval/JsonArrayLogger.scala index 87292aef7bd..209c82e4e9d 100644 --- a/main/eval/src/mill/eval/JsonArrayLogger.scala +++ b/main/eval/src/mill/eval/JsonArrayLogger.scala @@ -43,6 +43,7 @@ private object ProfileLogger { label: String, millis: Int, cached: java.lang.Boolean = null, + valueHashChanged: java.lang.Boolean = null, dependencies: Seq[String] = Nil, inputsHash: Int, previousInputsHash: Int = -1 From d90facc14b7bdb120f2b92c62e5f843ba000ced5 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 15 Dec 2024 19:02:20 +0800 Subject: [PATCH 11/13] . --- example/javalib/basic/1-simple/build.mill | 1 - kotlinlib/package.mill | 9 --------- 2 files changed, 10 deletions(-) diff --git a/example/javalib/basic/1-simple/build.mill b/example/javalib/basic/1-simple/build.mill index 2fedbe96320..67abcd1e759 100644 --- a/example/javalib/basic/1-simple/build.mill +++ b/example/javalib/basic/1-simple/build.mill @@ -9,7 +9,6 @@ object foo extends JavaModule { ) object test extends JavaTests with TestModule.Junit4 { - val x = 81 def ivyDeps = super.ivyDeps() ++ Agg( ivy"com.google.guava:guava:33.3.0-jre" ) diff --git a/kotlinlib/package.mill b/kotlinlib/package.mill index b3b148b29f5..2913cb65eb2 100644 --- a/kotlinlib/package.mill +++ b/kotlinlib/package.mill @@ -12,15 +12,6 @@ object `package` extends RootModule with build.MillPublishScalaModule with Build def moduleDeps = Seq(build.main, build.scalalib, build.testrunner, worker) def testTransitiveDeps = super.testTransitiveDeps() ++ Seq(worker.impl.testDep()) - val x = 1 - val y = 1 - val z = 1 - val z2 = 1 - val z3 = 1 - val z4 = 1 - val z5 = 1 - val z6 = 1 - val z7 = 7 def buildInfoPackageName = "mill.kotlinlib" def buildInfoObjectName = "Versions" def buildInfoMembers = Seq( From 1273d3ec23857f4b15a391ac75c61a07c1bd5128 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 15 Dec 2024 19:05:27 +0800 Subject: [PATCH 12/13] . --- main/codesig/src/Logger.scala | 5 +++- main/codesig/src/ReachabilityAnalysis.scala | 3 ++- main/eval/src/mill/eval/EvaluatorCore.scala | 16 ++++++----- main/eval/src/mill/eval/GroupEvaluator.scala | 27 ++++++++++++------- main/util/src/mill/util/SpanningForest.scala | 8 +++--- .../src/mill/util/SpanningForestTests.scala | 7 +++-- 6 files changed, 41 insertions(+), 25 deletions(-) diff --git a/main/codesig/src/Logger.scala b/main/codesig/src/Logger.scala index 2c722a6c4eb..315f2475124 100644 --- a/main/codesig/src/Logger.scala +++ b/main/codesig/src/Logger.scala @@ -16,7 +16,10 @@ class Logger(mandatoryLogFolder: os.Path, logFolder: Option[os.Path]) { def log[T: upickle.default.Writer](t: => sourcecode.Text[T], prefix: String = ""): Unit = { logFolder.foreach(log0(_, t, s"$count-$prefix")) } - def mandatoryLog[T: upickle.default.Writer](t: => sourcecode.Text[T], prefix: String = ""): Unit = { + def mandatoryLog[T: upickle.default.Writer]( + t: => sourcecode.Text[T], + prefix: String = "" + ): Unit = { log0(mandatoryLogFolder, t, prefix) } } diff --git a/main/codesig/src/ReachabilityAnalysis.scala b/main/codesig/src/ReachabilityAnalysis.scala index b5ecaa30bc9..f58c2fcf6ea 100644 --- a/main/codesig/src/ReachabilityAnalysis.scala +++ b/main/codesig/src/ReachabilityAnalysis.scala @@ -131,7 +131,8 @@ object CallGraphAnalysis { .flatMap { case (vs, k) => vs.map((_, k)) } .groupMap(_._1)(_._2) - val reverseGraphEdges = indexGraphEdges.indices.map(reverseGraphMap.getOrElse(_, Array())).toArray + val reverseGraphEdges = + indexGraphEdges.indices.map(reverseGraphMap.getOrElse(_, Array())).toArray SpanningForest.spanningTreeToJsonTree( SpanningForest.apply(reverseGraphEdges, nodesWithChangedHashes, false), diff --git a/main/eval/src/mill/eval/EvaluatorCore.scala b/main/eval/src/mill/eval/EvaluatorCore.scala index 6054402e74c..62215659950 100644 --- a/main/eval/src/mill/eval/EvaluatorCore.scala +++ b/main/eval/src/mill/eval/EvaluatorCore.scala @@ -79,7 +79,8 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { val count = new AtomicInteger(1) val indexToTerminal = sortedGroups.keys().toArray val terminalToIndex = indexToTerminal.zipWithIndex.toMap - val upstreamIndexEdges = indexToTerminal.map(t => interGroupDeps.getOrElse(t, Nil).map(terminalToIndex).toArray) + val upstreamIndexEdges = + indexToTerminal.map(t => interGroupDeps.getOrElse(t, Nil).map(terminalToIndex).toArray) os.write.over( outPath / OutFiles.millDependencyForest, SpanningForest.spanningTreeToJsonTree( @@ -273,23 +274,26 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { val reverseInterGroupDeps = interGroupDeps .iterator - .flatMap{case (k, vs) => vs.map(_ -> k)} + .flatMap { case (k, vs) => vs.map(_ -> k) } .toSeq .groupMap(_._1)(_._2) - val changedTerminalIndices = changedValueHash.keys().asScala.toSet val downstreamIndexEdges = indexToTerminal .map(t => - if (changedTerminalIndices(t)) reverseInterGroupDeps.getOrElse(t, Nil).map(terminalToIndex).toArray + if (changedTerminalIndices(t)) + reverseInterGroupDeps.getOrElse(t, Nil).map(terminalToIndex).toArray else Array.empty[Int] ) - os.write.over( outPath / OutFiles.millInvalidationForest, SpanningForest.spanningTreeToJsonTree( - SpanningForest(downstreamIndexEdges, uncached.keys().asScala.map(terminalToIndex).toSet, true), + SpanningForest( + downstreamIndexEdges, + uncached.keys().asScala.map(terminalToIndex).toSet, + true + ), i => indexToTerminal(i).render ).render(indent = 2) ) diff --git a/main/eval/src/mill/eval/GroupEvaluator.scala b/main/eval/src/mill/eval/GroupEvaluator.scala index 476224d4ee9..ec8a064324a 100644 --- a/main/eval/src/mill/eval/GroupEvaluator.scala +++ b/main/eval/src/mill/eval/GroupEvaluator.scala @@ -103,7 +103,14 @@ private[mill] trait GroupEvaluator { executionContext, exclusive ) - GroupEvaluator.Results(newResults, newEvaluated.toSeq, null, inputsHash, -1, valueHashChanged = false) + GroupEvaluator.Results( + newResults, + newEvaluated.toSeq, + null, + inputsHash, + -1, + valueHashChanged = false + ) case labelled: Terminal.Labelled[_] => val out = @@ -122,7 +129,9 @@ private[mill] trait GroupEvaluator { forceDiscard = cached.isEmpty ) - upToDateWorker.map((_, inputsHash)) orElse cached.flatMap { case (inputHash, valOpt, valueHash) => valOpt.map((_, valueHash))} match { + upToDateWorker.map((_, inputsHash)) orElse cached.flatMap { + case (inputHash, valOpt, valueHash) => valOpt.map((_, valueHash)) + } match { case Some((v, hashCode)) => val res = Result.Success((v, hashCode)) val newResults: Map[Task[_], TaskResult[(Val, Int)]] = @@ -157,13 +166,11 @@ private[mill] trait GroupEvaluator { exclusive ) - val valueHash = newResults(labelled.task) match { case TaskResult(Result.Failure(_, Some((v, _))), _) => val valueHash = if (terminal.task.asWorker.isEmpty) v.## else inputsHash handleTaskResult(v, valueHash, paths.meta, inputsHash, labelled) - case TaskResult(Result.Success((v, _)), _) => val valueHash = if (terminal.task.asWorker.isEmpty) v.## else inputsHash handleTaskResult(v, valueHash, paths.meta, inputsHash, labelled) @@ -458,11 +465,11 @@ private[mill] trait GroupEvaluator { private[mill] object GroupEvaluator { case class Results( - newResults: Map[Task[_], TaskResult[(Val, Int)]], - newEvaluated: Seq[Task[_]], - cached: java.lang.Boolean, - inputsHash: Int, - previousInputsHash: Int, - valueHashChanged: Boolean + newResults: Map[Task[_], TaskResult[(Val, Int)]], + newEvaluated: Seq[Task[_]], + cached: java.lang.Boolean, + inputsHash: Int, + previousInputsHash: Int, + valueHashChanged: Boolean ) } diff --git a/main/util/src/mill/util/SpanningForest.scala b/main/util/src/mill/util/SpanningForest.scala index 442fe0e36d9..2627d12a0c9 100644 --- a/main/util/src/mill/util/SpanningForest.scala +++ b/main/util/src/mill/util/SpanningForest.scala @@ -19,9 +19,11 @@ private[mill] object SpanningForest { ) } case class Node(values: mutable.Map[Int, Node] = mutable.Map()) - def apply(indexGraphEdges: Array[Array[Int]], - importantVertices: Set[Int], - limitToImportantVertices: Boolean): Node = { + def apply( + indexGraphEdges: Array[Array[Int]], + importantVertices: Set[Int], + limitToImportantVertices: Boolean + ): Node = { // Find all importantVertices which are "roots" with no incoming edges // from other importantVertices val destinations = importantVertices.flatMap(indexGraphEdges(_)) diff --git a/main/util/test/src/mill/util/SpanningForestTests.scala b/main/util/test/src/mill/util/SpanningForestTests.scala index 832620275f4..afab8147a49 100644 --- a/main/util/test/src/mill/util/SpanningForestTests.scala +++ b/main/util/test/src/mill/util/SpanningForestTests.scala @@ -14,13 +14,13 @@ object SpanningForestTests extends TestSuite { Array(2), Array(3), Array(), - Array(), + Array() ), Set(0), limitToImportantVertices = false ) - val expected = Node( + val expected = Node( mutable.Map( 0 -> Node( mutable.Map( @@ -38,8 +38,7 @@ object SpanningForestTests extends TestSuite { ) ) - assert(forest == expected - ) + assert(forest == expected) } } From 281cabdaa75c80f4f468539ca18867bbcde2b202 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 15 Dec 2024 19:20:28 +0800 Subject: [PATCH 13/13] . --- main/codesig/src/Logger.scala | 6 +++++- runner/src/mill/runner/MillBuildRootModule.scala | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/main/codesig/src/Logger.scala b/main/codesig/src/Logger.scala index 315f2475124..b30de439433 100644 --- a/main/codesig/src/Logger.scala +++ b/main/codesig/src/Logger.scala @@ -5,7 +5,11 @@ class Logger(mandatoryLogFolder: os.Path, logFolder: Option[os.Path]) { os.remove.all(mandatoryLogFolder) private var count = 1 - def log0[T: upickle.default.Writer](p: os.Path, res: sourcecode.Text[T], prefix: String = "") = { + def log0[T: upickle.default.Writer]( + p: os.Path, + res: sourcecode.Text[T], + prefix: String = "" + ): Unit = { os.write( p / s"$prefix${res.source}.json", upickle.default.stream(res.value, indent = 4), diff --git a/runner/src/mill/runner/MillBuildRootModule.scala b/runner/src/mill/runner/MillBuildRootModule.scala index 3a2b793fe29..e36732bca6c 100644 --- a/runner/src/mill/runner/MillBuildRootModule.scala +++ b/runner/src/mill/runner/MillBuildRootModule.scala @@ -12,7 +12,6 @@ import mill.main.client.OutFiles._ import mill.main.client.CodeGenConstants.buildFileExtensions import mill.main.{BuildInfo, RootModule} -import scala.collection.immutable.SortedMap import scala.util.Try import mill.define.Target