diff --git a/.gitignore b/.gitignore index 32a7e01..a05adb0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .vscode/ /*.bat deno.lock +plug/ diff --git a/README.md b/README.md index dc669b6..3b63ab8 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,15 @@ [![checks](https://github.com/denosaurs/deno_python/actions/workflows/checks.yml/badge.svg)](https://github.com/denosaurs/deno_python/actions/workflows/checks.yml) [![License](https://img.shields.io/github/license/denosaurs/deno_python)](https://github.com/denosaurs/deno_python/blob/master/LICENSE) -Python interpreter bindings for Deno. +This module provides a seamless integration between deno and python by +integrating with the [Python/C API](https://docs.python.org/3/c-api/index.html). +It acts as a bridge between the two languages, enabling you to pass data and +execute python code from within your deno applications. This enables access to +the large and wonderful [python ecosystem](https://pypi.org/) while remaining +native (unlike a runtime like the wondeful +[pyodide](https://github.com/pyodide/pyodide) which is compiled to wasm, +sandboxed and may not work with all python packages) and simply using the +existing python installation. ## Example @@ -32,6 +40,43 @@ permissions since enabling FFI effectively escapes the permissions sandbox. deno run -A --unstable ``` +### Dependencies + +Normally deno_python follows the default python way of resolving imports, going +through `sys.path` resolving them globally, locally or scoped to a virtual +environment. This is ~~great~~ and allows you to manage your python dependencies +for `deno_python` projects in the same way you would any other python project +using your favorite package manager, be it +[`pip`](https://pip.pypa.io/en/stable/), +[`conda`](https://docs.conda.io/en/latest/) or +[`poetry`](https://python-poetry.org/). + +This may not be a good thing though, especially for something like a deno module +which may depend on a python package. That is why the [`ext/pip`](./ext/pip.ts) +utility exists for this project. It allows you to install python dependencies +using pip, scoped to either the global deno installation or if defined the +`--location` passed to deno without leaking to the global python scope. It uses +the same caching location and algorithm as +[plug](https://github.com/denosaurs/deno) and +[deno cache](https://github.com/denoland/deno_cache). + +To use [`ext/pip`](./ext/pip.ts) for python package management you simply use +the provided `import` or `install` methods. The rest is handled automatically +for you! Just take a look! + +```ts +import { pip } from "https://deno.land/x/python/ext/pip.ts"; + +const np = await pip.import("numpy"); +const plt = await pip.import("matplotlib", "matplotlib.pyplot"); + +const xpoints = np.array([1, 8]); +const ypoints = np.array([3, 10]); + +plt.plot(xpoints, ypoints); +plt.show(); +``` + ## Documentation Check out the docs diff --git a/examples/pip_import.ts b/examples/pip_import.ts new file mode 100644 index 0000000..027b4e4 --- /dev/null +++ b/examples/pip_import.ts @@ -0,0 +1,10 @@ +import { pip } from "../ext/pip.ts"; + +const np = await pip.import("numpy"); +const plt = await pip.import("matplotlib", "matplotlib.pyplot"); + +const xpoints = np.array([1, 8]); +const ypoints = np.array([3, 10]); + +plt.plot(xpoints, ypoints); +plt.show(); diff --git a/ext/pip.ts b/ext/pip.ts new file mode 100644 index 0000000..2077edf --- /dev/null +++ b/ext/pip.ts @@ -0,0 +1,170 @@ +import { kw, python, PythonError } from "../mod.ts"; + +import { join } from "https://deno.land/std@0.198.0/path/mod.ts"; +import { ensureDir } from "https://deno.land/std@0.198.0/fs/mod.ts"; +import { green, yellow } from "https://deno.land/std@0.198.0/fmt/colors.ts"; + +import type { CacheLocation } from "https://deno.land/x/plug@1.0.2/types.ts"; +import { ensureCacheLocation } from "https://deno.land/x/plug@1.0.2/download.ts"; +import { hash } from "https://deno.land/x/plug@1.0.2/util.ts"; + +const sys = python.import("sys"); +const runpy = python.import("runpy"); +const importlib = python.import("importlib"); + +// https://packaging.python.org/en/latest/specifications/name-normalization/ +const MODULE_REGEX = + /^([a-z0-9]|[a-z0-9][a-z0-9._-]*[a-z0-9])([^a-z0-9._-].*)?$/i; + +function normalizeModuleName(name: string) { + return name.replaceAll(/[-_.]+/g, "-").toLowerCase(); +} + +function getModuleNameAndVersion(module: string): { + name: string; + version?: string; +} { + const match = module.match(MODULE_REGEX); + const name = match?.[1]; + const version = match?.[2]; + + if (name == null) { + throw new TypeError("Could not match any valid pip module name"); + } + + return { + name: normalizeModuleName(name), + version, + }; +} + +export class Pip { + #cacheLocation: Promise; + + constructor(location: CacheLocation) { + this.#cacheLocation = Promise.all([ + ensureCacheLocation(location), + globalThis.location !== undefined + ? hash(globalThis.location.href) + : Promise.resolve("pip"), + ]).then(async (parts) => { + const cacheLocation = join(...parts); + await ensureDir(cacheLocation); + + if (!(cacheLocation in sys.path)) { + sys.path.insert(0, cacheLocation); + } + + return cacheLocation; + }); + } + + /** + * Install a Python module using the `pip` package manager. + * + * @param module The Python module which you wish to install + * + * @example + * ```ts + * import { python } from "https://deno.land/x/python/mod.ts"; + * import { install } from "https://deno.land/x/python/ext/pip.ts"; + * + * await install("numpy"); + * const numpy = python.import("numpy"); + * + * ``` + */ + async install(module: string) { + const argv = sys.argv; + sys.argv = [ + "pip", + "install", + "-q", + "-t", + await this.#cacheLocation, + module, + ]; + + console.log(`${green("Installing")} ${module}`); + + try { + runpy.run_module("pip", kw`run_name=${"__main__"}`); + } catch (error) { + if ( + !( + error instanceof PythonError && + error.type.isInstance(python.builtins.SystemExit()) && + error.value.asLong() === 0 + ) + ) { + throw error; + } + } finally { + sys.argv = argv; + } + } + + /** + * Install and import a Python module using the `pip` package manager. + * + * @param module The Python module which you wish to install + * + * @example + * ```ts + * import { python } from "https://deno.land/x/python/mod.ts"; + * import { pip } from "https://deno.land/x/python/ext/pip.ts"; + * + * const numpy = await pip.import("numpy==1.25.2"); + * + * ``` + */ + async import(module: string, entrypoint?: string) { + const { name } = getModuleNameAndVersion(module); + + await this.install(module); + + if (entrypoint) { + return python.import(entrypoint); + } + + const packages = importlib.metadata.packages_distributions(); + const entrypoints = []; + + for (const entry of packages) { + if (packages[entry].valueOf().includes(name)) { + entrypoints.push(entry.valueOf()); + } + } + + if (entrypoints.length === 0) { + throw new TypeError( + `Failed to import module ${module}, could not find import name ${name}`, + ); + } + + entrypoint = entrypoints[0]; + + if (entrypoints.length > 1) { + if (entrypoints.includes(name)) { + entrypoint = entrypoints[entrypoints.indexOf(name)]; + } else { + console.warn( + `${ + yellow( + "Warning", + ) + } could not determine a single entrypoint for module ${module}, please specify one of: ${ + entrypoints.join( + ", ", + ) + }. Importing ${entrypoint}`, + ); + } + } + + return python.import(entrypoint!); + } +} + +export const pip = new Pip(); +export default pip; diff --git a/test/deps.ts b/test/deps.ts index 8afbb88..5d1ae69 100644 --- a/test/deps.ts +++ b/test/deps.ts @@ -1 +1 @@ -export * from "https://deno.land/std@0.178.0/testing/asserts.ts"; +export * from "https://deno.land/std@0.198.0/testing/asserts.ts";