extension metadata

.github/workflows/build-so.yaml

@@ -42,7 +42,7 @@ jobs:
       with:
         name: extensions-linux-x64
         if-no-files-found: error
-        path: ~/.jyafn/extensions/*.so
+        path: ~/.jyafn/extensions/*-latest.so
         retention-days: 1
 
   build-dylib:
@@ -79,7 +79,7 @@ jobs:
       with:
         name: extensions-macos-arm64
         if-no-files-found: error
-        path: ~/.jyafn/extensions/*.dylib
+        path: ~/.jyafn/extensions/*-latest.dylib
         retention-days: 1
 
 

jyafn-ext/extensions/dummy/Makefile

@@ -24,4 +24,5 @@ build:
 
 install: build
 	mkdir -p ~/.jyafn/extensions/
-	mv $(ROOT)/target/release/lib$(LIBNAME).$(SO_EXT) ~/.jyafn/extensions/$(NAME)-$(VERSION).$(SO_EXT)
+	cp $(ROOT)/target/release/lib$(LIBNAME).$(SO_EXT) ~/.jyafn/extensions/$(NAME)-$(VERSION).$(SO_EXT)
+	cp $(ROOT)/target/release/lib$(LIBNAME).$(SO_EXT) ~/.jyafn/extensions/$(NAME)-latest.$(SO_EXT)

jyafn-ext/extensions/lightgbm/Makefile

@@ -24,4 +24,5 @@ build:
 
 install: build
 	mkdir -p ~/.jyafn/extensions/
-	mv $(ROOT)/target/release/lib$(LIBNAME).$(SO_EXT) ~/.jyafn/extensions/$(NAME)-$(VERSION).$(SO_EXT)
+	cp $(ROOT)/target/release/lib$(LIBNAME).$(SO_EXT) ~/.jyafn/extensions/$(NAME)-$(VERSION).$(SO_EXT)
+	cp $(ROOT)/target/release/lib$(LIBNAME).$(SO_EXT) ~/.jyafn/extensions/$(NAME)-latest.$(SO_EXT)

jyafn-ext/src/lib.rs

@@ -98,6 +98,10 @@ macro_rules! extension {
         pub extern "C" fn extension_init() -> *const c_char {
             fn safe_extension_init() -> String {
                 let manifest = $crate::serde_json::json!({
+                    "metatada": {
+                        "name": env!("CARGO_PKG_NAME"),
+                        "version": env!("CARGO_PKG_VERSION"),
+                    },
                     "outcome": {
                         "fn_get_err": "outcome_get_err",
                         "fn_get_ok": "outcome_get_ok",

jyafn-python/pyproject.toml

@@ -18,6 +18,7 @@ dependencies = [
     "numpy>=1.13",
     "click>=8.1",
     "click_default_group>=1.2",
+    "semver>=3",
 ]
 
 [project.scripts]

jyafn-python/python/jyafn/cli.py

@@ -1,9 +1,19 @@
-from http.server import BaseHTTPRequestHandler, HTTPServer
-
 import click
-from click_default_group import DefaultGroup  # type:ignore
 import timeit as pytimeit
 import jyafn as fn
+import platform
+import re
+import os
+import semver
+import tempfile
+import shutil
+import ctypes
+import json
+
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from urllib.parse import urlparse
+from urllib.request import urlretrieve
+from click_default_group import DefaultGroup  # type:ignore
 
 from .describe import describe, describe_graph  # type:ignore
 
@@ -106,3 +116,75 @@ def do_POST(self):
     finally:
         httpd.server_close()
         click.echo("Stopping httpd...")
+
+
+@main.command(help="Dowloads an extension from an URL")
+@click.option(
+    "--force",
+    "-f",
+    is_flag=True,
+    help="forces the substition of the extension if it exists",
+)
+@click.argument(
+    "origin",
+    help="the place to fetch the extension from. Can be a URL or the name of an extension "
+    "in the registry",
+)
+def get(force, origin):
+    match platform.system():
+        case "Linux":
+            extension = "so"
+        case "Darwin":
+            extension = "dylib"
+        case "Windows":
+            extension = "dll"
+        case p:
+            click.echo(f"platform {p!r} not supported")
+            exit(1)
+
+    install_to = os.getenv(
+        "JYAFN_PATH", os.path.expanduser("~/.jyafn/extensions")
+    ).split(",")[0]
+
+    click.echo(f"Downloading {origin} to {install_to}...")
+    with tempfile.NamedTemporaryFile() as temp:
+        urlretrieve(origin, temp.name)
+
+        # This runs arbitrary code downloaded from the internet. I hope you know what you
+        # are doing.
+        so = ctypes.cdll.LoadLibrary(temp.name)
+
+        if not hasattr(so, "extension_init"):
+            click.echo("extension is missing symbol `extension_init`")
+            exit(1)
+        so.extension_init.restype = ctypes.c_char_p
+
+        manifest = json.loads(so.extension_init())
+        name = manifest["metadata"]["name"]
+        version = manifest["metadata"]["version"]
+
+        if not re.match(r"[a-z][a-z0-9_]*", name):
+            click.echo(f"invalid extension name {name!r}.")
+            click.echo(
+                "hint: extension names should contain only lowercase letters and digits and "
+                "should start with a letter"
+            )
+            exit(1)
+
+        try:
+            semver.parse_version_info(version)
+        except ValueError:
+            click.echo(f"version {version!r} not a valid semantic version")
+            exit(1)
+
+        click.echo(f"Collected {name}-{version}.{extension}")
+
+        target_path = f"{install_to}/{name}-{version}.{extension}"
+        if os.path.exists(target_path) and not force:
+            click.echo(f"extension {name}-{version} is already installed")
+            click.echo("hint: pass -f to force reinstall")
+            exit(1)
+
+        shutil.copy(temp.name, target_path)
+
+    click.echo(f"You now have {name}-{version} installed")

jyafn/src/extension.rs

@@ -3,9 +3,10 @@
 use lazy_static::lazy_static;
 use libloading::{Library, Symbol};
 use serde_derive::{Deserialize, Serialize};
+use serde_with::{serde_as, DisplayFromStr};
 use std::collections::HashMap;
 use std::ffi::{c_char, CStr, CString};
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::sync::{Arc, RwLock};
 
 use crate::layout::{Layout, Struct};
@@ -38,6 +39,8 @@ pub struct Dumped(pub(crate) *mut ());
 /// this extension.
 #[derive(Debug, Serialize, Deserialize)]
 pub struct ExtensionManifest {
+    /// Tells us what this extension is.
+    metadata: ExtensionMetadata,
     /// Describes the symbols to be used when accessing outcomes of fallible operations.
     outcome: OutcomeManifest,
     /// Describes the symbols to be used when accessing buffers of binary memory.
@@ -47,6 +50,17 @@ pub struct ExtensionManifest {
     resources: HashMap<String, ResourceManifest>,
 }
 
+#[serde_as]
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ExtensionMetadata {
+    /// This is the name of the extension and must match the file name in the filesystem.
+    name: String,
+    /// This is the version of the extension and must be a valid semantic version and
+    /// must match the file name in the filesystem.
+    #[serde_as(as = "DisplayFromStr")]
+    version: semver::Version,
+}
+
 /// Lists the names of the symbols needed to create the interface between an outcome and
 /// jyafn. See [`OutcomeSymbols`] for detailed information on the contract for each
 /// symbol.
@@ -230,6 +244,8 @@ lazy_static! {
 pub struct Extension {
     /// The shared object handle.
     _library: Library,
+    /// The metedata of this extension.
+    metadata: ExtensionMetadata,
     /// Describes the symbols to be used when accessing outcomes of fallible operations.
     outcome: OutcomeSymbols,
     /// Describes the symbols to be used when accessing buffers of binary memory.
@@ -242,7 +258,7 @@ pub struct Extension {
 impl Extension {
     /// Loads an extension, given a path. This path is OS-specific and will be resolved
     /// by the OS acording to its own quirky rules.
-    pub(crate) fn load(path: PathBuf) -> Result<Extension, Error> {
+    pub(crate) fn load(path: &Path) -> Result<Extension, Error> {
         unsafe {
             // Safety: we can only pray nobody loads anything funny here. However, it's
             // not my responsibilty what kind of crap you install in your computer.
@@ -276,6 +292,7 @@ impl Extension {
 
             Ok(Extension {
                 _library: library,
+                metadata: manifest.metadata,
                 outcome,
                 dumped,
                 resources,
@@ -426,7 +443,24 @@ pub fn try_get(name: &str, version_req: &semver::VersionReq) -> Result<Arc<Exten
     }
 
     let extension =
-        Arc::new(Extension::load(path).with_context(|| format!("loading extension {name:?}"))?);
+        Arc::new(Extension::load(&path).with_context(|| format!("loading extension {name:?}"))?);
+
+    // Check if what you got is what was actually advertised:
+    if extension.metadata.name != name {
+        return Err(format!(
+            "file {path:?} should provide {name:?} but provides {:?}",
+            extension.metadata.name
+        )
+        .into());
+    }
+    if extension.metadata.version != version {
+        return Err(format!(
+            "file {path:?} should provide version {version} but provides {}",
+            extension.metadata.version
+        )
+        .into());
+    }
+
     loaded_extensions.insert(version, extension.clone());
 
     Ok(extension)