Improve Android SDK handling (#3913)

This PR seeks to improve Android SDK handling approach.

The current issues are:

* Android SDK setup is [limited only to
Linux](https://github.com/com-lihaoyi/mill/blob/9d11962eaf529474dd66e095620a97cff66ebcd3/scalalib/src/mill/javalib/android/AndroidSdkModule.scala#L24)
and is also using [tooling which may be not available on other
platforms](https://github.com/com-lihaoyi/mill/blob/9d11962eaf529474dd66e095620a97cff66ebcd3/scalalib/src/mill/javalib/android/AndroidSdkModule.scala#L102-L109).
* Each module [will have its own copy of
SDK](https://github.com/com-lihaoyi/mill/blob/9d11962eaf529474dd66e095620a97cff66ebcd3/scalalib/src/mill/javalib/android/AndroidSdkModule.scala#L114).
Imagine a project with 20 Android modules (19 lib modules + 1 app
module), it will be 20x space usage than it should be. This is quite
significant, considering that the basic setup (`platforms:android-35`,
`platform-tools` and `build-tools;35.0.0`) is taking ~300 MB.
* Licenses are [accepted on the end-user
behalf](https://github.com/com-lihaoyi/mill/blob/9d11962eaf529474dd66e095620a97cff66ebcd3/scalalib/src/mill/javalib/android/AndroidSdkModule.scala#L108),
without the consent.

This PR:

* Aligns the SDK setup experience with AGP (Android Gradle Plugin):
* Expects Android SDK (at least command line tools) to be present on the
end-user machine and registered with the env variable (`ANDROID_HOME`).
* Expects command line tools to be present there to utilize
[sdkmanager](https://developer.android.com/tools/sdkmanager) to setup
the necessary SDK components.
* Expects user to accept the necessary licenses by themselves. If they
are not accepted, `sdkmanager` will block `stdin` waiting for the user
input before proceeding with installation. **Note**: when downloading
[Android Studio](https://developer.android.com/studio), it will come
with the necessary basic license already accepted + some basic SDK
components. If downloading command line tools, it is necessary to accept
the main license (`android-sdk-license`) manually.
* Installs all the necessary components to the location pointed by the
`ANDROID_HOME` env variable, so that they are shared by all modules.
* Adds the necessary step on the CI to install Android SDK for the jobs
where it is needed. **Note**: I was also thinking about extracting
Android-related examples into a dedicated job, but then for the existing
jobs doing something like `example.kotlinlib[__].local.testCached`
->`example.kotlinlib[__:^AndroidAppKotlinModule].local.testCached`
doesn't work (at least `mill resolve` still shows Android module), all
cross-segments should be listed manually.

Speaking of `sdkmanager`: it is also possible to utilize directly the
[following
artifact](https://mvnrepository.com/artifact/com.android.tools/sdklib)
and put it in a worker, but for this one it is impossible to find a
version on the Android Developers website, so end-user will have
troubles updating it and anyway it is one-shot action normally, so
keeping it even in the separate classpath looks like an overkill.
[Command line tools](https://developer.android.com/tools) will be needed
anyway further for the other tools it provides.

Co-authored-by: 0xnm <0xnm@users.noreply.github.com>

.github/workflows/run-mill-action.yml

@@ -27,6 +27,9 @@ on:
       env-bridge-versions:
         default: 'none'
         type: string
+      install-android-sdk:
+        default: false
+        type: boolean
 
 jobs:
   run:
@@ -62,6 +65,10 @@ jobs:
         with:
           node-version: '22'
 
+      - uses: android-actions/setup-android@v3
+        if: ${{ inputs.install-android-sdk }}
+        with:
+          log-accepted-android-sdk-licenses: false
 
       - name: Prepare git config
         run: |

.github/workflows/run-tests.yml

@@ -65,10 +65,12 @@ jobs:
             millargs: "contrib.__.testCached"
 
           - java-version: 17
+            install-android-sdk: true
             millargs: "'example.javalib.__.local.testCached'"
           - java-version: 17
             millargs: "'example.scalalib.__.local.testCached'"
           - java-version: 17
+            install-android-sdk: true
             millargs: "'example.kotlinlib.__.local.testCached'"
           - java-version: '11'
             millargs: "'example.thirdparty[{mockito,acyclic,commons-io}].local.testCached'"
@@ -93,6 +95,7 @@ jobs:
 
     uses: ./.github/workflows/run-mill-action.yml
     with:
+      install-android-sdk: ${{ matrix.install-android-sdk }}
       java-version: ${{ matrix.java-version }}
       millargs: ${{ matrix.millargs }}
 

scalalib/src/mill/javalib/android/AndroidSdkModule.scala

@@ -2,6 +2,11 @@ package mill.javalib.android
 
 import mill._
 
+import java.math.BigInteger
+import java.nio.charset.StandardCharsets
+import java.security.MessageDigest
+import scala.xml.XML
+
 /**
  * Trait for managing the Android SDK in a Mill build system.
  *
@@ -17,12 +22,9 @@ import mill._
 @mill.api.experimental
 trait AndroidSdkModule extends Module {
 
-  /**
-   * Provides the URL to download the Android SDK command-line tools.
-   */
-  def sdkUrl: T[String] = Task {
-    "https://dl.google.com/android/repository/commandlinetools-linux-11076708_latest.zip"
-  }
+  // this has a format `repository2-%d`, where the last number is schema version. For the needs of this module it
+  // is okay to stick with the particular version.
+  private val remotePackagesUrl = "https://dl.google.com/android/repository/repository2-3.xml"
 
   /**
    * Specifies the version of the Android build tools to be used.
@@ -38,14 +40,17 @@ trait AndroidSdkModule extends Module {
    * Provides the path to the `android.jar` file, necessary for compiling Android apps.
    */
   def androidJarPath: T[PathRef] = Task {
-    PathRef(installAndroidSdk().path / "platforms" / platformsVersion().toString / "android.jar")
+    installAndroidSdkComponents()
+    PathRef(sdkPath().path / "platforms" / platformsVersion() / "android.jar")
   }
 
   /**
    * Provides path to the Android build tools for the selected version.
    */
-  def buildToolsPath: T[PathRef] =
-    Task { PathRef(installAndroidSdk().path / "build-tools" / buildToolsVersion().toString) }
+  def buildToolsPath: T[PathRef] = Task {
+    installAndroidSdkComponents()
+    PathRef(sdkPath().path / "build-tools" / buildToolsVersion())
+  }
 
   /**
    * Provides path to D8 Dex compiler, used for converting Java bytecode into Dalvik bytecode.
@@ -76,46 +81,135 @@ trait AndroidSdkModule extends Module {
   }
 
   /**
-   * Installs the Android SDK by performing the following actions:
+   * Installs the necessary Android SDK components such as platform-tools, build-tools, and Android platforms.
    *
-   * - Downloads the SDK command-line tools from the specified URL.
-   *
-   * - Extracts the downloaded zip file into the SDK directory.
-   *
-   * - Accepts the required SDK licenses.
-   *
-   * - Installs essential SDK components such as platform-tools, build-tools, and Android platforms.
-   *
-   * For more details on the sdkmanager tool, refer to:
+   * For more details on the `sdkmanager` tool, refer to:
    * [[https://developer.android.com/tools/sdkmanager sdkmanager Documentation]]
    *
-   * @return A task containing a `PathRef` pointing to the installed SDK directory.
+   * @return A task containing a [[PathRef]] pointing to the SDK directory.
    */
-  def installAndroidSdk: T[PathRef] = Task {
-    val zipFilePath: os.Path = T.dest / "commandlinetools.zip"
-    val sdkManagerPath: os.Path = T.dest / "cmdline-tools/bin/sdkmanager"
-
-    // Download SDK command-line tools
-    os.write(zipFilePath, requests.get(sdkUrl().toString).bytes)
-
-    // Extract the downloaded SDK tools into the destination directory
-    os.call(Seq("unzip", zipFilePath.toString, "-d", T.dest.toString))
-
-    // Automatically accept the SDK licenses
-    os.call(Seq(
-      "bash",
-      "-c",
-      s"yes | $sdkManagerPath --licenses --sdk_root=${T.dest}"
-    ))
-
-    // Install platform-tools, build-tools, and the Android platform
-    os.call(Seq(
-      sdkManagerPath.toString,
-      s"--sdk_root=${T.dest}",
+  def installAndroidSdkComponents: T[Unit] = Task {
+    val sdkPath0 = sdkPath()
+    val sdkManagerPath = findLatestSdkManager(sdkPath0.path) match {
+      case Some(x) => x
+      case _ => throw new IllegalStateException(
+          s"Cannot locate cmdline-tools in Android SDK $sdkPath0. Download" +
+            " it at https://developer.android.com/studio#command-tools. See https://developer.android.com/tools" +
+            " for more details."
+        )
+    }
+
+    val packages = Seq(
       "platform-tools",
-      s"build-tools;${buildToolsVersion().toString}",
-      s"platforms;${platformsVersion().toString}"
-    ))
-    PathRef(T.dest)
+      s"build-tools;${buildToolsVersion()}",
+      s"platforms;${platformsVersion()}"
+    )
+    // sdkmanager executable and state of the installed package is a shared resource, which can be accessed
+    // from the different Android SDK modules.
+    AndroidSdkLock.synchronized {
+      val missingPackages = packages.filter(p => !isPackageInstalled(sdkPath0.path, p))
+      val packagesWithoutLicense = missingPackages
+        .map(p => (p, isLicenseAccepted(sdkPath0.path, remoteReposInfo().path, p)))
+        .filter(!_._2)
+      if (packagesWithoutLicense.nonEmpty) {
+        throw new IllegalStateException(
+          "Failed to install the following SDK packages, because their respective" +
+            s" licenses are not accepted:\n\n${packagesWithoutLicense.map(_._1).mkString("\n")}"
+        )
+      }
+
+      if (missingPackages.nonEmpty) {
+        val callResult = os.call(
+          // Install platform-tools, build-tools, and the Android platform
+          Seq(sdkManagerPath.toString) ++ missingPackages,
+          stdout = os.Inherit
+        )
+        if (callResult.exitCode != 0) {
+          throw new IllegalStateException(
+            "Failed to install Android SDK components. Check logs for more details."
+          )
+        }
+      }
+    }
+  }
+
+  private def sdkPath: T[PathRef] = Task {
+    T.env.get("ANDROID_HOME")
+      .orElse(T.env.get("ANDROID_SDK_ROOT")) match {
+      case Some(x) => PathRef(os.Path(x))
+      case _ => throw new IllegalStateException("Android SDK location not found. Define a valid" +
+          " SDK location with an ANDROID_HOME environment variable.")
+    }
+  }
+
+  private def isPackageInstalled(sdkPath: os.Path, packageName: String): Boolean =
+    os.exists(sdkPath / os.SubPath(packageName.replaceAll(";", "/")))
+
+  private def isLicenseAccepted(
+      sdkPath: os.Path,
+      remoteReposInfo: os.Path,
+      packageName: String
+  ): Boolean = {
+    val (licenseName, licenseHash) = licenseForPackage(remoteReposInfo, packageName)
+    val licenseFile = sdkPath / "licenses" / licenseName
+    os.exists(licenseFile) && os.isFile(licenseFile) && os.read(licenseFile).contains(licenseHash)
+  }
+
+  private def licenseForPackage(remoteReposInfo: os.Path, packageName: String): (String, String) = {
+    val repositoryInfo = XML.loadFile(remoteReposInfo.toIO)
+    val remotePackage = (repositoryInfo \ "remotePackage")
+      .filter(_ \@ "path" == packageName)
+      .head
+    val licenseName = (remotePackage \ "uses-license").head \@ "ref"
+    val licenseText = (repositoryInfo \ "license")
+      .filter(_ \@ "id" == licenseName)
+      .text
+      .replaceAll(
+        "(?<=\\s)[ \t]*",
+        ""
+      ) // remove spaces and tabs preceded by space, tab, or newline.
+      .replaceAll("(?<!\n)\n(?!\n)", " ") // replace lone newlines with space
+      .replaceAll(" +", " ")
+      .trim
+    val licenseHash = hexArray(sha1.digest(licenseText.getBytes(StandardCharsets.UTF_8)))
+    (licenseName, licenseHash)
+  }
+
+  private def remoteReposInfo: Command[PathRef] = Task.Command {
+    // shouldn't be persistent, allow it to be re-downloaded again.
+    // it will be called only if some packages are not installed.
+    val path = T.dest / "repository.xml"
+    os.write(
+      T.dest / "repository.xml",
+      requests.get(remotePackagesUrl).bytes
+    )
+    PathRef(path)
+  }
+
+  private def sha1 = MessageDigest.getInstance("sha1")
+
+  private def hexArray(arr: Array[Byte]) =
+    String.format("%0" + (arr.length << 1) + "x", new BigInteger(1, arr))
+
+  private def findLatestSdkManager(sdkPath: os.Path): Option[os.Path] = {
+    var sdkManagerPath = sdkPath / "cmdline-tools/latest/bin/sdkmanager"
+    if (!os.exists(sdkManagerPath)) {
+      // overall it can be cmdline-tools/<version>
+      val candidates = os.list(sdkPath / "cmdline-tools")
+        .filter(os.isDir)
+      if (candidates.nonEmpty) {
+        val latestCmdlineToolsPath = candidates
+          .map(p => (p, p.baseName.split('.')))
+          .filter(_._2 match {
+            case Array(_, _) => true
+            case _ => false
+          })
+          .maxBy(_._2.head.toInt)._1
+        sdkManagerPath = latestCmdlineToolsPath / "bin" / "sdkmanager"
+      }
+    }
+    Some(sdkManagerPath).filter(os.exists)
   }
 }
+
+private object AndroidSdkLock