feat(ext/pip): pip install and import (#43)

.gitignore

@@ -1,3 +1,4 @@
 .vscode/
 /*.bat
 deno.lock
+plug/

README.md

@@ -5,7 +5,15 @@
 [![checks](https://github.com/denosaurs/deno_python/actions/workflows/checks.yml/badge.svg)](https://github.com/denosaurs/deno_python/actions/workflows/checks.yml)
 [![License](https://img.shields.io/github/license/denosaurs/deno_python)](https://github.com/denosaurs/deno_python/blob/master/LICENSE)
 
-Python interpreter bindings for Deno.
+This module provides a seamless integration between deno and python by
+integrating with the [Python/C API](https://docs.python.org/3/c-api/index.html).
+It acts as a bridge between the two languages, enabling you to pass data and
+execute python code from within your deno applications. This enables access to
+the large and wonderful [python ecosystem](https://pypi.org/) while remaining
+native (unlike a runtime like the wondeful
+[pyodide](https://github.com/pyodide/pyodide) which is compiled to wasm,
+sandboxed and may not work with all python packages) and simply using the
+existing python installation.
 
 ## Example
 
@@ -32,6 +40,43 @@ permissions since enabling FFI effectively escapes the permissions sandbox.
 deno run -A --unstable <file>
 ```
 
+### Dependencies
+
+Normally deno_python follows the default python way of resolving imports, going
+through `sys.path` resolving them globally, locally or scoped to a virtual
+environment. This is ~~great~~ and allows you to manage your python dependencies
+for `deno_python` projects in the same way you would any other python project
+using your favorite package manager, be it
+[`pip`](https://pip.pypa.io/en/stable/),
+[`conda`](https://docs.conda.io/en/latest/) or
+[`poetry`](https://python-poetry.org/).
+
+This may not be a good thing though, especially for something like a deno module
+which may depend on a python package. That is why the [`ext/pip`](./ext/pip.ts)
+utility exists for this project. It allows you to install python dependencies
+using pip, scoped to either the global deno installation or if defined the
+`--location` passed to deno without leaking to the global python scope. It uses
+the same caching location and algorithm as
+[plug](https://github.com/denosaurs/deno) and
+[deno cache](https://github.com/denoland/deno_cache).
+
+To use [`ext/pip`](./ext/pip.ts) for python package management you simply use
+the provided `import` or `install` methods. The rest is handled automatically
+for you! Just take a look!
+
+```ts
+import { pip } from "https://deno.land/x/python/ext/pip.ts";
+
+const np = await pip.import("numpy");
+const plt = await pip.import("matplotlib", "matplotlib.pyplot");
+
+const xpoints = np.array([1, 8]);
+const ypoints = np.array([3, 10]);
+
+plt.plot(xpoints, ypoints);
+plt.show();
+```
+
 ## Documentation
 
 Check out the docs

examples/pip_import.ts

@@ -0,0 +1,10 @@
+import { pip } from "../ext/pip.ts";
+
+const np = await pip.import("numpy");
+const plt = await pip.import("matplotlib", "matplotlib.pyplot");
+
+const xpoints = np.array([1, 8]);
+const ypoints = np.array([3, 10]);
+
+plt.plot(xpoints, ypoints);
+plt.show();

ext/pip.ts

@@ -0,0 +1,170 @@
+import { kw, python, PythonError } from "../mod.ts";
+
+import { join } from "https://deno.land/std@0.198.0/path/mod.ts";
+import { ensureDir } from "https://deno.land/std@0.198.0/fs/mod.ts";
+import { green, yellow } from "https://deno.land/std@0.198.0/fmt/colors.ts";
+
+import type { CacheLocation } from "https://deno.land/x/plug@1.0.2/types.ts";
+import { ensureCacheLocation } from "https://deno.land/x/plug@1.0.2/download.ts";
+import { hash } from "https://deno.land/x/plug@1.0.2/util.ts";
+
+const sys = python.import("sys");
+const runpy = python.import("runpy");
+const importlib = python.import("importlib");
+
+// https://packaging.python.org/en/latest/specifications/name-normalization/
+const MODULE_REGEX =
+ /^([a-z0-9]|[a-z0-9][a-z0-9._-]*[a-z0-9])([^a-z0-9._-].*)?$/i;
+
+function normalizeModuleName(name: string) {
+ return name.replaceAll(/[-_.]+/g, "-").toLowerCase();
+}
+
+function getModuleNameAndVersion(module: string): {
+ name: string;
+ version?: string;
+} {
+ const match = module.match(MODULE_REGEX);
+ const name = match?.[1];
+ const version = match?.[2];
+
+ if (name == null) {
+ throw new TypeError("Could not match any valid pip module name");
+ }
+
+ return {
+ name: normalizeModuleName(name),
+ version,
+ };
+}
+
+export class Pip {
+ #cacheLocation: Promise<string>;
+
+ constructor(location: CacheLocation) {
+ this.#cacheLocation = Promise.all([
+ ensureCacheLocation(location),
+ globalThis.location !== undefined
+ ? hash(globalThis.location.href)
+ : Promise.resolve("pip"),
+ ]).then(async (parts) => {
+ const cacheLocation = join(...parts);
+ await ensureDir(cacheLocation);
+
+ if (!(cacheLocation in sys.path)) {
+ sys.path.insert(0, cacheLocation);
+ }
+
+ return cacheLocation;
+ });
+ }
+
+ /**
+ * Install a Python module using the `pip` package manager.
+ *
+ * @param module The Python module which you wish to install
+ *
+ * @example
+ * ```ts
+ * import { python } from "https://deno.land/x/python/mod.ts";
+ * import { install } from "https://deno.land/x/python/ext/pip.ts";
+ *
+ * await install("numpy");
+ * const numpy = python.import("numpy");
+ *
+ * ```
+ */
+ async install(module: string) {
+ const argv = sys.argv;
+ sys.argv = [
+ "pip",
+ "install",
+ "-q",
+ "-t",
+ await this.#cacheLocation,
+ module,
+ ];
+
+ console.log(`${green("Installing")} ${module}`);
+
+ try {
+ runpy.run_module("pip", kw`run_name=${"__main__"}`);
+ } catch (error) {
+ if (
+ !(
+ error instanceof PythonError &&
+ error.type.isInstance(python.builtins.SystemExit()) &&
+ error.value.asLong() === 0
+ )
+ ) {
+ throw error;
+ }
+ } finally {
+ sys.argv = argv;
+ }
+ }
+
+ /**
+ * Install and import a Python module using the `pip` package manager.
+ *
+ * @param module The Python module which you wish to install
+ *
+ * @example
+ * ```ts
+ * import { python } from "https://deno.land/x/python/mod.ts";
+ * import { pip } from "https://deno.land/x/python/ext/pip.ts";
+ *
+ * const numpy = await pip.import("numpy==1.25.2");
+ *
+ * ```
+ */
+ async import(module: string, entrypoint?: string) {
+ const { name } = getModuleNameAndVersion(module);
+
+ await this.install(module);
+
+ if (entrypoint) {
+ return python.import(entrypoint);
+ }
+
+ const packages = importlib.metadata.packages_distributions();
+ const entrypoints = [];
+
+ for (const entry of packages) {
+ if (packages[entry].valueOf().includes(name)) {
+ entrypoints.push(entry.valueOf());
+ }
+ }
+
+ if (entrypoints.length === 0) {
+ throw new TypeError(
+ `Failed to import module ${module}, could not find import name ${name}`,
+ );
+ }
+
+ entrypoint = entrypoints[0];
+
+ if (entrypoints.length > 1) {
+ if (entrypoints.includes(name)) {
+ entrypoint = entrypoints[entrypoints.indexOf(name)];
+ } else {
+ console.warn(
+ `${
+ yellow(
+ "Warning",
+ )
+ } could not determine a single entrypoint for module ${module}, please specify one of: ${
+ entrypoints.join(
+ ", ",
+ )
+ }. Importing ${entrypoint}`,
+ );
+ }
+ }
+
+ return python.import(entrypoint!);
+ }
+}
+
+export const pip = new Pip();
+export default pip;

test/deps.ts

@@ -1 +1 @@
-export * from "https://deno.land/std@0.178.0/testing/asserts.ts";
+export * from "https://deno.land/std@0.198.0/testing/asserts.ts";