Merge pull request #4 from elvispy/multiple_workflows

Add support for caching multiple workflows

.github/workflows/CI.yml

@@ -1,5 +1,7 @@
 name: CI
-on: push
+on:
+ push:
+ pull_request:
 concurrency:
  # Skip intermediate builds: always.
  # Cancel intermediate builds: only if it is a pull request build.

.gitignore

@@ -1 +1,4 @@
 /Manifest.toml
+
+test/MultipleWorkflows/**/Manifest.toml
+.vscode/*

README.md

@@ -10,9 +10,12 @@
 
 - Bundles all necessary Julia code and artifacts needed to run without internet access.
 - Build for platforms other than the host platform.
+- Can build multiple packages/projects into a single path.
+- Can precompile all dependencies to built path.
 
 ## Usage
 
+### Example 1
 ```julia
 using DepotDelivery: build
 
@@ -25,6 +28,20 @@ path = build(path_to_project; platform = Base.BinaryPlatforms.HostPlatform())
 - The build settings live in `$path/config/depot_build.toml`
 - Run this in the production environment to get started: `include("$path/config/depot_startup.jl")`.
 
+### Example 2
+This example shows how to build a depo path from different Project.toml files, enabling precompilation as needed.
+```julia
+using DepotDelivery: build
+
+# We can provide a depot_path to share DEPOT_PATH 
+depot_path = "path/to/depot/"
+
+# Assumes `path/Project.toml` exists (or `path/JuliaProject.toml`) in each entry of first argument, and force precompilation.
+path = build(["path/project-1", "path-2/project-2"]; depot=depot_path, precompiled=true)
+```
+
+Be aware that `build` will copy everything inside those directories to `depot_path/dev/`. Avoid populating those directories with unnecessary files. For example, when starting a new project, it's better to run `julia --project=./isolated_folder/` rather than `julia --project=.`, as in the latter case the `Project.toml` file will coexist with other stuff.
+
 ## Building for Non-Host Platforms
 
 - Use any `Base.BinaryPlatforms.AbstractPlatform` as the `platform` argument.

src/DepotDelivery.jl

@@ -27,29 +27,50 @@ function sandbox(f::Function)
  end
 end
 
-#-----------------------------------------------------------------------------# build
-function build(path::String; platform = Base.BinaryPlatforms.HostPlatform(), verbose=true)
+#-----------------------------------------------------------------------------#
+"""
+Builds the depot for a specified project.
+
+Arguments:
+- `path::String`: The path to the project directory containing `Project.toml` or `JuliaProject.toml`.
+- `platform::AbstractPlatform`: The target platform for building (default is the host platform).
+- `verbose::Bool`: Whether to display verbose output during the build process (default is `true`).
+- `depot::String`: The path to the depot directory (default is a temporary directory).
+- `precompiled::Bool`: Whether to enable precompilation of packages (default is `false`).
+
+Returns:
+- The path to the built depot.
+
+Example:
+```julia
+depot_path = build("/path/to/your/project")
+"""
+function build(path::String; platform = Base.BinaryPlatforms.HostPlatform(), verbose=true, depot = mktempdir(), precompiled=false)
  path = abspath(path)
- depot = mktempdir()
+ mkpath(depot)
  sandbox() do
  proj_file = joinpath(path, "Project.toml")
  proj_file = isfile(proj_file) ? proj_file : joinpath(path, "JuliaProject.toml")
  isfile(proj_file) || error("No Project.toml or JuliaProject.toml found in `$path`.")
  proj = TOML.parsefile(proj_file)
- name = proj["name"]
+ # Defines a project name for Project.toml that don't come from a package
+ name = haskey(proj, "name") ? proj["name"] : Base.basename(Base.dirname(proj_file))
  build_spec = Dict(
  :datetime => Dates.now(),
  :versioninfo => sprint(InteractiveUtils.versioninfo),
  :project_file => proj_file,
  :project => proj,
  :platform => string(platform)
  )
- mkdir(joinpath(depot, "config"))
- mkdir(joinpath(depot, "dev"))
+ mkpath(joinpath(depot, "config"))
+ mkpath(joinpath(depot, "dev", name))
  push!(empty!(DEPOT_PATH), depot)
- ENV["JULIA_PKG_PRECOMPILE_AUTO"] = "0" # Needed when building for non-host platforms
+
+ # Disabling precompile for non-host platforms
+ precompiled ? delete!(ENV, "JULIA_PKG_PRECOMPILE_AUTO") : ENV["JULIA_PKG_PRECOMPILE_AUTO"] = "0" 
 
- cp(path, joinpath(depot, "dev", name)) # Copy project into dev/
+ cp(path, joinpath(depot, "dev", name), force=true) # Copy project into dev/
+
  Pkg.activate(joinpath(depot, "dev", name))
  Pkg.instantiate(; platform, verbose)
 
@@ -64,10 +85,33 @@ function build(path::String; platform = Base.BinaryPlatforms.HostPlatform(), ver
  open(io -> TOML.print(io, build_spec), joinpath(depot, "config", "depot_build.toml"), "w")
  open(io -> print(io, startup_script(name)), joinpath(depot, "config", "depot_startup.jl"), "w")
  end
-
+ 
  return depot
 end
 
+"""
+Builds depots for multiple projects specified by their paths.
+
+Arguments:
+- `paths::Vector{String}`: An array of directories of project paths.
+- `platform::AbstractPlatform`: The target platform for building (default is the host platform).
+- `verbose::Bool`: Whether to display verbose output during the build process (default is `true`).
+- `depot::String`: The path to the depot directory (default is a temporary directory).
+- `precompiled::Bool`: Whether to enable precompilation (default is `false`).
+
+Example:
+```julia
+project_paths = ["/path/to/project1", "/path/to/project2"]
+build(project_paths)
+"""
+function build(paths::Vector{String}; platform = Base.BinaryPlatforms.HostPlatform(), verbose=true, depot=mktempdir(), precompiled=false)
+ for path in paths
+ build(path; platform=platform, verbose=verbose, depot=depot, precompiled=precompiled)
+ end
+ return depot
+ end
+
+
 #-----------------------------------------------------------------------------# startup_script
 startup_script(name) = """
  import Pkg

test/MultipleWorkflows/BitFlags/Project.toml

@@ -0,0 +1,2 @@
+[deps]
+BitFlags = "d1d4a3ce-64b1-5f1a-9ba4-7e7e69966f35"

test/MultipleWorkflows/URIs/Project.toml

@@ -0,0 +1,2 @@
+[deps]
+URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"

test/MultipleWorkflows/Unzip/Project.toml

@@ -0,0 +1,2 @@
+[deps]
+Unzip = "41fe7b60-77ed-43a1-b4f0-825fd5a5650d"

test/runtests.jl

@@ -2,6 +2,7 @@ using DepotDelivery
 using Pkg
 using Test
 
+
 depot = DepotDelivery.build(joinpath(@__DIR__, "TestProject"))
 
 @test DepotDelivery.test(depot)
@@ -18,3 +19,29 @@ end
 depot2 = DepotDelivery.build(joinpath(@__DIR__, "TestProject"), platform = Pkg.BinaryPlatforms.Windows(:x86_64))
 
 path = joinpath(depot2, "packages", "HDF5_jll")
+
+
+#-----------------------------------------------------------------------------# Testing multiple workflows
+packages_list = readdir(joinpath(@__DIR__, "MultipleWorkflows/"));
+proj_paths = joinpath.(@__DIR__, "MultipleWorkflows/", packages_list);
+depot = DepotDelivery.build(proj_paths, precompiled=true)
+
+DepotDelivery.sandbox() do 
+ push!(empty!(DEPOT_PATH), depot)
+
+ # Test that for every project instantiated, their dependencies exist 
+ # and the depot path does not point to the default value
+ @testset for (proj, package) in zip(proj_paths, packages_list)
+ Pkg.activate(proj)
+ Pkg.instantiate()
+ package_symbol = Symbol(package)
+ @eval using $package_symbol
+ package_value = eval(package_symbol)
+ @test !occursin(".julia", pathof(package_value))
+ end
+
+ # Ensure compiled folders are populated
+ @testset for package in packages_list
+ @test length(readdir(joinpath(depot, "compiled", "v$(VERSION.major).$(VERSION.minor)", package))) > 0
+ end
+end