Skip to content

Commit

Permalink
feat(pypi): improve resolving suitable python version (#1725)
Browse files Browse the repository at this point in the history
  • Loading branch information
williamboman authored Jun 1, 2024
1 parent 0fb4e56 commit 0950b15
Show file tree
Hide file tree
Showing 10 changed files with 300 additions and 19 deletions.
86 changes: 70 additions & 16 deletions lua/mason-core/installer/managers/pypi.lua
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,20 @@ local a = require "mason-core.async"
local installer = require "mason-core.installer"
local log = require "mason-core.log"
local path = require "mason-core.path"
local pep440 = require "mason-core.pep440"
local platform = require "mason-core.platform"
local providers = require "mason-core.providers"
local semver = require "mason-core.semver"
local spawn = require "mason-core.spawn"

local M = {}

local VENV_DIR = "venv"

local is_executable = _.compose(_.equals(1), vim.fn.executable)

---@async
---@param candidates string[]
local function resolve_python3(candidates)
local is_executable = _.compose(_.equals(1), vim.fn.executable)
a.scheduler()
local available_candidates = _.filter(is_executable, candidates)
for __, candidate in ipairs(available_candidates) do
Expand All @@ -31,16 +32,33 @@ local function resolve_python3(candidates)
return nil
end

---@param min_version? Semver
local function get_versioned_candidates(min_version)
---@param version string
---@param specifiers string
local function pep440_check_version(version, specifiers)
-- The version check only implements a subset of the PEP440 specification and may error with certain inputs.
local ok, result = pcall(pep440.check_version, version, specifiers)
if not ok then
log.fmt_warn(
"Failed to check PEP440 version compatibility for version %s with specifiers %s: %s",
version,
specifiers,
result
)
return false
end
return result
end

---@param supported_python_versions string
local function get_versioned_candidates(supported_python_versions)
return _.filter_map(function(pair)
local version, executable = unpack(pair)
if not min_version or version > min_version then
return Optional.of(executable)
else
if not pep440_check_version(tostring(version), supported_python_versions) then
return Optional.empty()
end
return Optional.of(executable)
end, {
{ semver.new "3.12.0", "python3.12" },
{ semver.new "3.11.0", "python3.11" },
{ semver.new "3.10.0", "python3.10" },
{ semver.new "3.9.0", "python3.9" },
Expand All @@ -51,24 +69,60 @@ local function get_versioned_candidates(min_version)
end

---@async
local function create_venv()
---@param pkg { name: string, version: string }
local function create_venv(pkg)
local ctx = installer.context()
---@type string?
local supported_python_versions = providers.pypi.get_supported_python_versions(pkg.name, pkg.version):get_or_nil()

-- 1. Resolve stock python3 installation.
local stock_candidates = platform.is.win and { "python", "python3" } or { "python3", "python" }
local stock_target = resolve_python3(stock_candidates)
if stock_target then
log.fmt_debug("Resolved stock python3 installation version %s", stock_target.version)
end
local versioned_candidates = get_versioned_candidates(stock_target and stock_target.version)
log.debug("Resolving versioned python3 candidates", versioned_candidates)

-- 2. Resolve suitable versioned python3 installation (python3.12, python3.11, etc.).
local versioned_candidates = {}
if supported_python_versions ~= nil then
log.fmt_debug("Finding versioned candidates for %s", supported_python_versions)
versioned_candidates = get_versioned_candidates(supported_python_versions)
end
local target = resolve_python3(versioned_candidates) or stock_target
local ctx = installer.context()

if not target then
ctx.stdio_sink.stderr(
("Unable to find python3 installation. Tried the following candidates: %s.\n"):format(
return Result.failure(
("Unable to find python3 installation in PATH. Tried the following candidates: %s."):format(
_.join(", ", _.concat(stock_candidates, versioned_candidates))
)
)
return Result.failure "Failed to find python3 installation."
end

-- 3. If a versioned python3 installation was not found, warn the user if the stock python3 installation is outside
-- the supported version range.
if
target == stock_target
and supported_python_versions ~= nil
and not pep440_check_version(tostring(target.version), supported_python_versions)
then
if ctx.opts.force then
ctx.stdio_sink.stderr(
("Warning: The resolved python3 version %s is not compatible with the required Python versions: %s.\n"):format(
target.version,
supported_python_versions
)
)
else
ctx.stdio_sink.stderr "Run with :MasonInstall --force to bypass this version validation.\n"
return Result.failure(
("Failed to find a python3 installation in PATH that meets the required versions (%s). Found version: %s."):format(
supported_python_versions,
target.version
)
)
end
end

log.fmt_debug("Found python3 installation version=%s, executable=%s", target.version, target.executable)
ctx.stdio_sink.stdout "Creating virtual environment…\n"
return ctx.spawn[target.executable] { "-m", "venv", VENV_DIR }
Expand Down Expand Up @@ -118,15 +172,15 @@ local function pip_install(pkgs, extra_args)
end

---@async
---@param opts { upgrade_pip: boolean, install_extra_args?: string[] }
---@param opts { package: { name: string, version: string }, upgrade_pip: boolean, install_extra_args?: string[] }
function M.init(opts)
return Result.try(function(try)
log.fmt_debug("pypi: init", opts)
local ctx = installer.context()

-- pip3 will hardcode the full path to venv executables, so we need to promote cwd to make sure pip uses the final destination path.
ctx:promote_cwd()
try(create_venv())
try(create_venv(opts.package))

if opts.upgrade_pip then
ctx.stdio_sink.stdout "Upgrading pip inside the virtual environment…\n"
Expand Down
6 changes: 5 additions & 1 deletion lua/mason-core/installer/registry/providers/pypi.lua
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ function M.parse(source, purl)
---@class ParsedPypiSource : ParsedPackageSource
local parsed_source = {
package = purl.name,
version = purl.version,
version = purl.version --[[ @as string ]],
extra = _.path({ "qualifiers", "extra" }, purl),
extra_packages = source.extra_packages,
pip = {
Expand All @@ -42,6 +42,10 @@ function M.install(ctx, source)

return Result.try(function(try)
try(pypi.init {
package = {
name = source.package,
version = source.version,
},
upgrade_pip = source.pip.upgrade,
install_extra_args = source.pip.extra_args,
})
Expand Down
64 changes: 64 additions & 0 deletions lua/mason-core/pep440/init.lua
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
-- Function to split a version string into its components
local function split_version(version)
local parts = {}
for part in version:gmatch "[^.]+" do
table.insert(parts, tonumber(part) or part)
end
return parts
end

-- Function to compare two versions
local function compare_versions(version1, version2)
local v1_parts = split_version(version1)
local v2_parts = split_version(version2)
local len = math.max(#v1_parts, #v2_parts)

for i = 1, len do
local v1_part = v1_parts[i] or 0
local v2_part = v2_parts[i] or 0

if v1_part < v2_part then
return -1
elseif v1_part > v2_part then
return 1
end
end

return 0
end

-- Function to check a version against a single specifier
local function check_single_specifier(version, specifier)
local operator, spec_version = specifier:match "^([<>=!]+)%s*(.+)$"
local comp_result = compare_versions(version, spec_version)

if operator == "==" then
return comp_result == 0
elseif operator == "!=" then
return comp_result ~= 0
elseif operator == "<=" then
return comp_result <= 0
elseif operator == "<" then
return comp_result < 0
elseif operator == ">=" then
return comp_result >= 0
elseif operator == ">" then
return comp_result > 0
else
error("Invalid operator in version specifier: " .. operator)
end
end

-- Function to check a version against multiple specifiers
local function check_version(version, specifiers)
for specifier in specifiers:gmatch "[^,]+" do
if not check_single_specifier(version, specifier:match "^%s*(.-)%s*$") then
return false
end
end
return true
end

return {
check_version = check_version,
}
1 change: 1 addition & 0 deletions lua/mason-core/providers/init.lua
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ local settings = require "mason.settings"
---@class PyPiProvider
---@field get_latest_version? async fun(pkg: string): Result # Result<PyPiPackage>
---@field get_all_versions? async fun(pkg: string): Result # Result<string[]> # Sorting should not be relied upon due to "proprietary" sorting algo in pip that is difficult to replicate in mason-registry-api.
---@field get_supported_python_versions? async fun(pkg: string, version: string): Result # Result<string> # Returns a version specifier as provided by the PyPI API (see PEP440).

---@alias RubyGem { name: string, version: string }

Expand Down
2 changes: 2 additions & 0 deletions lua/mason-registry/api.lua
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,8 @@ api.pypi = {
latest = get "/api/pypi/{package}/versions/latest",
---@type ApiSignature<{ package: string }>
all = get "/api/pypi/{package}/versions/all",
---@type ApiSignature<{ package: string, version: string }>
get = get "/api/pypi/{package}/versions/{version}",
},
}

Expand Down
14 changes: 14 additions & 0 deletions lua/mason/providers/client/pypi.lua
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
local Optional = require "mason-core.optional"
local Result = require "mason-core.result"
local _ = require "mason-core.functional"
local a = require "mason-core.async"
local fetch = require "mason-core.fetch"
local fs = require "mason-core.fs"
local platform = require "mason-core.platform"
local spawn = require "mason-core.spawn"
Expand Down Expand Up @@ -50,4 +52,16 @@ return {
return get_all_versions(pkg):map(_.compose(Optional.of_nilable, _.last)):and_then(synthesize_pkg(pkg))
end,
get_all_versions = get_all_versions,
get_supported_python_versions = function(pkg, version)
return fetch(("https://pypi.org/pypi/%s/%s/json"):format(pkg, version))
:map_catching(vim.json.decode)
:map(_.path { "info", "requires_python" })
:and_then(function(requires_python)
if type(requires_python) ~= "string" or requires_python == "" then
return Result.failure "Package does not specify supported Python versions."
else
return Result.success(requires_python)
end
end)
end,
}
14 changes: 14 additions & 0 deletions lua/mason/providers/registry-api/init.lua
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
local Result = require "mason-core.result"
local _ = require "mason-core.functional"
local api = require "mason-registry.api"

---@type Provider
Expand Down Expand Up @@ -31,6 +33,18 @@ return {
get_all_versions = function(pkg)
return api.pypi.versions.all { package = pkg }
end,
get_supported_python_versions = function(pkg, version)
return api.pypi.versions
.get({ package = pkg, version = version })
:map(_.prop "requires_python")
:and_then(function(requires_python)
if type(requires_python) ~= "string" or requires_python == "" then
return Result.failure "Package does not specify supported Python versions."
else
return Result.success(requires_python)
end
end)
end,
},
rubygems = {
get_latest_version = function(gem)
Expand Down
Loading

0 comments on commit 0950b15

Please sign in to comment.