From 8a9f5345cf56bf5b2afebff3e712c8dc56af5c50 Mon Sep 17 00:00:00 2001 From: Tomas Fiers Date: Wed, 11 Jan 2023 23:32:11 +0100 Subject: [PATCH] =?UTF-8?q?Crawl=20registry=20(=E2=86=92=20to=20rm=20restr?= =?UTF-8?q?iction=20"Only=20works=20for=20pkgs=20in=20active=20project")?= =?UTF-8?q?=20Close=20#5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit finally! 🍾πŸ₯‚ missing: more proper testing, code cleanup (there's some non dry in proj toml parsing and keys "deps") --- Changelog.md | 4 ++ Project.toml | 2 + ReadMe.md | 14 +------ docs/Manifest.toml | 2 +- docs/make.jl | 2 +- docs/src/ref/internals.md | 1 - src/enduser.jl | 4 -- src/internals/Internals.jl | 9 ++++- src/internals/depgraph.jl | 80 ++++++-------------------------------- src/internals/project.jl | 78 +++++++++++++++++++++++++++++++++++++ src/internals/registry.jl | 50 ++++++++++++++++++++++++ src/internals/stdlib.jl | 26 +++++++++++++ test/integration.jl | 1 + test/unit.jl | 2 +- 14 files changed, 185 insertions(+), 90 deletions(-) create mode 100644 src/internals/project.jl create mode 100644 src/internals/registry.jl create mode 100644 src/internals/stdlib.jl diff --git a/Changelog.md b/Changelog.md index 0af8a1a..7e6a6dd 100644 --- a/Changelog.md +++ b/Changelog.md @@ -38,6 +38,10 @@ The version numbers roughly follow SemVer Possible categories: [Added, Changed, Fixed, Removed, Security, Deprecated (for soon-to-be removed features)] --> +- Remove limitation "package must be installed in active project". + Any package in the General registry (and standard library) can now be + queried from anywhere. + - This (re)introduced dependencies: Pkg (a big one; but stdlib), and UUIDS. - Dark-mode option for _all_ generated images\ (not just local SVGs; also PNGs and webapp URLs) - Pass the `mode=:dark` keyword argument to `open` and `create` for this. diff --git a/Project.toml b/Project.toml index 3fa375c..c903344 100644 --- a/Project.toml +++ b/Project.toml @@ -6,8 +6,10 @@ version = "0.4.0-dev" [deps] DefaultApplication = "3f0dd361-4fe0-5fc6-8523-80b14ec94d85" EzXML = "8f5d6c58-4d21-5cfd-889c-e3ad7ee6a615" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" +UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] DefaultApplication = "1" diff --git a/ReadMe.md b/ReadMe.md index d3ebbcd..643f5a7 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -30,19 +30,7 @@ This will open the browser to [this url][dotlink], which renders something like width=680 alt="Dependency graph of Unitful, rendered with Graphviz dot"> -
-
- - The given package (here: [Unitful][unitful]) must be installed in the currently active project for this to work. - - Note that `PkgGraph` does not have to be installed in the same project however:\ - you can switch projects _after_ `PkgGraph` has been imported (using `pkg> activate …`). - - Even easier is to install `PkgGraph` in your base environment (see [Global Install](#global-install)), - so you don't have to switch projects at all. - -
To filter out binary dependencies ([JLL packages]) or packages from the Julia standard library, you can set the keyword arguments `jll = false` and `stdlib = false`. @@ -85,7 +73,7 @@ pkg> add PkgGraph You might want to install `PkgGraph` in your base environment (e.g. `v1.8`).\ You can then use it in any project, without having to install it in that project -or having to always switch projects. +or having to switch projects.
diff --git a/docs/Manifest.toml b/docs/Manifest.toml index 4f7c516..0cddc25 100644 --- a/docs/Manifest.toml +++ b/docs/Manifest.toml @@ -164,7 +164,7 @@ uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" version = "1.8.0" [[deps.PkgGraph]] -deps = ["DefaultApplication", "EzXML", "TOML", "URIs"] +deps = ["DefaultApplication", "EzXML", "Pkg", "TOML", "URIs", "UUIDs"] path = ".." uuid = "f9c1b9e4-72e8-4a14-ade5-14f45fc35f11" version = "0.4.0-dev" diff --git a/docs/make.jl b/docs/make.jl index 618f270..6144212 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -22,7 +22,7 @@ DocMeta.setdocmeta!(PkgGraph, :DocTestSetup, :(using PkgGraph); recursive=true, println("") # ..including Documenter.HTML(…) construction call makedocs( source = srcmod, - modules = [PkgGraph], + # modules = [PkgGraph], # β†ͺ To get a warning if there are any docstrings not mentioned in the markdown. sitename = "PkgGraph.jl", # β†ͺ Displayed in page title and navbar. diff --git a/docs/src/ref/internals.md b/docs/src/ref/internals.md index ee07a84..fb48d1e 100644 --- a/docs/src/ref/internals.md +++ b/docs/src/ref/internals.md @@ -13,7 +13,6 @@ Internals ```@docs depgraph -packages_in_active_manifest is_jll is_in_stdlib ``` diff --git a/src/enduser.jl b/src/enduser.jl index 1aa99b2..49d2ed8 100644 --- a/src/enduser.jl +++ b/src/enduser.jl @@ -4,8 +4,6 @@ Open the browser to an image of `pkgname`'s dependency graph. -The given package must be installed in the currently active project. - See [Settings](@ref) for possible keyword arguments. """ function open(pkgname; dryrun = false, kw...) @@ -27,8 +25,6 @@ and open it with your default image viewer. Uses the external program '`dot`' (see [graphviz.org](https://graphviz.org)), which must be available on `PATH`. -The given package must be installed in the currently active project. - `fmt` is an output file format supported by dot, such as `:svg` or `:png`.\\ If `fmt` is `:svg`, the generated SVG file is post-processed, to add light and dark-mode CSS. diff --git a/src/internals/Internals.jl b/src/internals/Internals.jl index b48b7e3..ccef58e 100644 --- a/src/internals/Internals.jl +++ b/src/internals/Internals.jl @@ -11,10 +11,17 @@ can be accessed as `PkgGraph.depgraph`, e.g. module Internals using TOML +using UUIDs +include("stdlib.jl") + using Base: active_project +include("project.jl") + +using Pkg +include("registry.jl") + include("depgraph.jl") export depgraph, - packages_in_active_manifest, should_be_included, is_jll, is_in_stdlib, diff --git a/src/internals/depgraph.jl b/src/internals/depgraph.jl index 0c6da3c..bf096e7 100644 --- a/src/internals/depgraph.jl +++ b/src/internals/depgraph.jl @@ -28,23 +28,20 @@ julia> depgraph(:Test) "Test" => "Serialization" ``` """ -depgraph(pkgname; jll = true, stdlib = true) = begin +depgraph(pkgname; jll = true, stdlib = true, verbose = false) = begin rootpkg = string(pkgname) - packages = packages_in_active_manifest() - include_jll = jll - include_stdlib = stdlib - if rootpkg βˆ‰ keys(packages) - error(""" - The given package ($pkgname) must be installed in the active project - (which is currently `$(active_project())`)""") + if is_in_project(rootpkg) + verbose && @info "Package `$rootpkg` found in active project. Using Manifest.toml" + direct_deps = direct_deps_from_project() + else + verbose && @info "Package `$rootpkg` not found in active project. Using General registry" + direct_deps = direct_deps_from_registry end deps = Vector{Pair{String, String}}() - add_deps_of(name) = begin - pkg_info = only(packages[name]) # Two packages with same name not supported. - direct_deps = get(pkg_info, "deps", []) - for dep in direct_deps - if should_be_included(dep; include_jll, include_stdlib) - push!(deps, name => dep) + add_deps_of(pkg) = begin + for dep in direct_deps(pkg) + if should_be_included(dep, include_jll=jll, include_stdlib=stdlib) + push!(deps, pkg => dep) add_deps_of(dep) end end @@ -53,40 +50,6 @@ depgraph(pkgname; jll = true, stdlib = true) = begin return unique!(deps) # Could use a SortedSet instead; but this spares a pkg load. end -manifest(proj_path) = replace(proj_path, "Project.toml" => "Manifest.toml") - -if VERSION β‰₯ v"1.7" - packages_in(manifest) = TOML.parsefile(manifest)["deps"] -else - packages_in(manifest) = TOML.parsefile(manifest) -end - -""" - packages_in_active_manifest() - -Read and parse the `Manifest.toml` of the active project, and return its -'deps' table (as a dictionary indexed by package names). - -Every entry in this dictionary is a list. This is for when multiple -packages would share the same name. - -## Example: - -```jldoctest; filter = r" => .*\$"m -julia> using PkgGraph.Internals - -julia> packages = packages_in_active_manifest(); - -julia> only(packages["PkgGraph"]) -Dict{String, Any} with 4 entries: - "deps" => ["DefaultApplication", "TOML", "URIs"] - "uuid" => "f9c1b9e4-72e8-4a14-ade5-14f45fc35f11" - "version" => "0.1.0" - "path" => "C:\\Users\\tfiers\\.julia\\dev\\PkgGraph" -``` -""" -packages_in_active_manifest() = packages_in(manifest(active_project())) - should_be_included(pkg; include_jll = true, include_stdlib = true) = if !include_jll && is_jll(pkg) @@ -102,26 +65,7 @@ should_be_included(pkg; include_jll = true, include_stdlib = true) = """ is_jll(pkg) = endswith(pkg, "_jll") - """ is_in_stdlib(pkg)::Bool """ -is_in_stdlib(pkg) = pkg in STDLIB - -stdlib_packages() = begin - packages = Set{String}() - for path in readdir(Sys.STDLIB; join = true) - # β†ͺ `join` gets us complete paths - if isdir(path) - push!(packages, pkgname(path)) - end - end - packages -end -pkgname(pkgdir) = begin - proj_file = joinpath(pkgdir, "Project.toml") - toml_dict = TOML.parsefile(proj_file) - pkgname = toml_dict["name"] -end - -const STDLIB = stdlib_packages() +is_in_stdlib(pkg) = pkg in STDLIB_NAMES diff --git a/src/internals/project.jl b/src/internals/project.jl new file mode 100644 index 0000000..eec92f5 --- /dev/null +++ b/src/internals/project.jl @@ -0,0 +1,78 @@ + +using TOML +using Base: active_project + +is_in_project(pkg, proj = active_project()) = + isfile(proj) && (name(proj) == pkg || pkg in keys(all_deps(proj))) + +name(proj_path) = name(dict(proj_path)) +dict(proj_path) = TOML.parsefile(proj_path) +name(toml::Dict) = get(toml, "name", nothing) + +all_deps(project) = begin + mani = manifest(project) + if !isfile(mani) + return Dict() + end + @static if VERSION β‰₯ v"1.7" + deps = TOML.parsefile(mani)["deps"] + else + deps = TOML.parsefile(mani) + end +end +manifest(project) = replace(project, "Project.toml" => "Manifest.toml") + +direct_deps_from_project(proj = active_project()) = begin + proj_dict = dict(proj) + proj_name = name(proj_dict) + all_deps_ = all_deps(proj) + direct_deps(pkgname) = + if pkgname == proj_name + get(proj_dict, "deps", []) |> keys + else + deps_with_name = all_deps_[pkgname] + check_only(deps_with_name) + dep_dict = only(deps_with_name) + get(dep_dict, "deps", []) + end + direct_deps +end +check_only(packages_with_same_name) = @assert( + length(packages_with_same_name) == 1, + """ + Different packages with same name not supported + (The offending packages:) + $packages_with_same_name + """ +) + +# The above is poop: we want dep tree of entire thing, also if no top specified +# no name. then there's multiple roots, sure (or take as name, the directory) +# + + +""" + packages_in_active_manifest() + +Read and parse the `Manifest.toml` of the given project, and return its +'deps' table (as a dictionary indexed by package names). + +Every entry in this dictionary is a list. This is for when multiple +packages would share the same name. + +## Example: + +```jldoctest; filter = r" => .*\$"m +julia> using PkgGraph.Internals + +julia> packages = packages_in_active_manifest(); + +julia> only(packages["PkgGraph"]) +Dict{String, Any} with 4 entries: + "deps" => ["DefaultApplication", "TOML", "URIs"] + "uuid" => "f9c1b9e4-72e8-4a14-ade5-14f45fc35f11" + "version" => "0.1.0" + "path" => "C:\\Users\\tfiers\\.julia\\dev\\PkgGraph" +``` +""" +packages_in_active_manifest() = all_deps(active_project()) diff --git a/src/internals/registry.jl b/src/internals/registry.jl new file mode 100644 index 0000000..d1d3f56 --- /dev/null +++ b/src/internals/registry.jl @@ -0,0 +1,50 @@ +using Pkg.Registry: reachable_registries, + uuids_from_name, + init_package_info!, + initialize_uncompressed!, + JULIA_UUID + + +const reg = first(reachable_registries()) +@assert reg.name == "General" + +name(uuid::UUID) = + if uuid in STDLIB_UUIDS + STDLIB[uuid] + elseif uuid in keys(reg.pkgs) + reg.pkgs[uuid].name + else + error() + end + +uuid(name::AbstractString) = + if name in STDLIB_NAMES + findfirst(==(name), STDLIB) + else + uuids = uuids_from_name(reg, name) + if isempty(uuids) + error("Package `$name` not found") + elseif length(uuids) > 1 + error("Multiple packages with the same name (`$name`) not supported") + else + return only(uuids) + end + end + +direct_deps_from_registry(pkg) = begin + if pkg in STDLIB_NAMES + return direct_deps_of_stdlib_pkg(pkg) + end + pkgentry = reg.pkgs[uuid(pkg)] + p = init_package_info!(pkgentry) + versions = keys(p.version_info) + v = maximum(versions) + initialize_uncompressed!(p, [v]) + vinfo = p.version_info[v] + compat_info = vinfo.uncompressed_compat + # β†ͺ All direct deps will be here, even if author didn't them + # [compat] (their versionspec will just be "*"). + direct_dep_uuids = collect(keys(compat_info)) + filter!(!=(JULIA_UUID), direct_dep_uuids) + return name.(direct_dep_uuids) +end diff --git a/src/internals/stdlib.jl b/src/internals/stdlib.jl new file mode 100644 index 0000000..ceeecf9 --- /dev/null +++ b/src/internals/stdlib.jl @@ -0,0 +1,26 @@ +using UUIDs +using TOML + +stdlib() = begin + packages = Dict{UUID,String}() + for path in readdir(Sys.STDLIB; join = true) + # β†ͺ `join` gets us complete paths + if isdir(path) + toml = proj_dict(path) + push!(packages, UUID(toml["uuid"]) => toml["name"]) + end + end + packages +end +proj_dict(pkgdir) = TOML.parsefile(proj_file(pkgdir)) +proj_file(pkgdir) = joinpath(pkgdir, "Project.toml") + +const STDLIB = stdlib() +const STDLIB_UUIDS = keys(STDLIB) +const STDLIB_NAMES = values(STDLIB) + +direct_deps_of_stdlib_pkg(name) = begin + pkgdir = joinpath(Sys.STDLIB, name) + d = proj_dict(pkgdir) + keys(get(d, "deps", [])) +end diff --git a/test/integration.jl b/test/integration.jl index 278703b..bd60773 100644 --- a/test/integration.jl +++ b/test/integration.jl @@ -7,6 +7,7 @@ using Test @test isnothing(PkgGraph.open("Test", dryrun = true)) @test isnothing(PkgGraph.create("Test", dryrun = true)) + @test isnothing(PkgGraph.create("PyPlot", dryrun = true)) end diff --git a/test/unit.jl b/test/unit.jl index 0ac0fc7..9a85c74 100644 --- a/test/unit.jl +++ b/test/unit.jl @@ -13,7 +13,7 @@ using Test if VERSION β‰₯ v"1.8" # Julia v1.7 does not support error string matching @test_throws( - "The given package (DinnaeExist) must be installed in the active project", + "Package `DinnaeExist` not found", depgraph("DinnaeExist") ) end