diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 030dfcc..1653aae 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -14,6 +14,12 @@ jobs: authToken: "${{ secrets.CACHIX_AUTH_TOKEN }}" - name: Build Nix run: nix build -j8 .#devShells.x86_64-linux.default + - name: Zig Cache + uses: actions/cache@v4 + with: + path: rose-zig/zig-cache + key: ${{ runner.os }}-rose-zig + save-always: true - name: Typecheck if: success() || failure() # Means that we run all steps even if one fails. run: nix develop --command make typecheck diff --git a/Makefile b/Makefile index b09574a..9fbe266 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,22 @@ check: typecheck test lintcheck -typecheck: +# Build the Zig library for development. +build-zig: + cd rose-zig && zig build -Doptimize=Debug + +typecheck: build-zig mypy . -test: +test-py: build-zig pytest -n logical . coverage html -snapshot: +test-zig: + cd rose-zig && zig build test --summary all + +test: test-zig test-py + +snapshot: build-zig pytest --snapshot-update . lintcheck: @@ -23,4 +32,7 @@ lint: clean: git clean -xdf -.PHONY: check test typecheck lintcheck lint clean +nixify-zig-deps: + cd rose-zig && zon2nix > deps.nix + +.PHONY: check build-zig test-py test-zig test typecheck lintcheck lint clean nixify-zig-deps diff --git a/flake.nix b/flake.nix index 8f669e5..0c2a7a7 100644 --- a/flake.nix +++ b/flake.nix @@ -31,6 +31,7 @@ inherit # Runtime deps. appdirs + cffi click jinja2 llfuse @@ -63,6 +64,7 @@ echo "$path" } export ROSE_ROOT="$(find-up flake.nix)" + export ROSE_SO_PATH="$ROSE_ROOT/rose-zig/zig-out/lib/librose.so" export PYTHONPATH="$ROSE_ROOT/rose-py:''${PYTHONPATH:-}" export PYTHONPATH="$ROSE_ROOT/rose-watch:$PYTHONPATH" export PYTHONPATH="$ROSE_ROOT/rose-vfs:$PYTHONPATH" @@ -75,19 +77,22 @@ pkgs.ruff pkgs.nodePackages.pyright pkgs.nodePackages.prettier + pkgs.zig + pkgs.zls python-with-deps ]; }) ]; }; packages = rec { - rose-py = pkgs.callPackage ./rose-py { inherit version python-pin py-deps; }; + rose-zig = pkgs.callPackage ./rose-zig { inherit version; }; + rose-py = pkgs.callPackage ./rose-py { inherit version python-pin py-deps rose-zig; }; rose-watch = pkgs.callPackage ./rose-watch { inherit version python-pin py-deps rose-py; }; rose-vfs = pkgs.callPackage ./rose-vfs { inherit version python-pin py-deps rose-py; }; rose-cli = pkgs.callPackage ./rose-cli { inherit version python-pin py-deps rose-py rose-vfs rose-watch; }; all = pkgs.buildEnv { name = "rose-all"; - paths = [ rose-py rose-watch rose-vfs rose-cli ]; + paths = [ rose-zig rose-py rose-watch rose-vfs rose-cli ]; }; }; }); diff --git a/pyproject.toml b/pyproject.toml index 19a1c65..12698bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,6 +89,9 @@ ignore_missing_imports = true [[tool.mypy.overrides]] module = "snapshottest" ignore_missing_imports = true +[[tool.mypy.overrides]] +module = "cffi" +ignore_missing_imports = true [tool.pytest.ini_options] addopts = [ diff --git a/rose-cli/rose_cli/cli.py b/rose-cli/rose_cli/cli.py index 6d8b56b..8696ee8 100644 --- a/rose-cli/rose_cli/cli.py +++ b/rose-cli/rose_cli/cli.py @@ -119,6 +119,14 @@ def version() -> None: click.echo(VERSION) +@cli.command() +def tmp() -> None: + """Temporary development command for FFI testing. If you see this, it's a bug.""" + from rose.ffi import get_release + + print(get_release()) + + @cli.group() def config() -> None: """Utilites for configuring Rosé.""" diff --git a/rose-cli/rose_cli/cli_test.py b/rose-cli/rose_cli/cli_test.py index e36367d..fc50e36 100644 --- a/rose-cli/rose_cli/cli_test.py +++ b/rose-cli/rose_cli/cli_test.py @@ -6,6 +6,7 @@ import pytest from click.testing import CliRunner from rose import AudioTags, Config +from rose.ffi import get_release from rose_vfs.virtualfs_test import start_virtual_fs from rose_cli.cli import ( @@ -125,3 +126,7 @@ def mock_exit(x: int) -> None: # Assert that we can't kill a non-existent watchdog. res = runner.invoke(unwatch, obj=ctx) assert res.exit_code == 1 + + +def test_ffi() -> None: + assert get_release() == "hello" diff --git a/rose-py/default.nix b/rose-py/default.nix index 09c1082..9a36267 100644 --- a/rose-py/default.nix +++ b/rose-py/default.nix @@ -1,6 +1,7 @@ { python-pin , version , py-deps +, rose-zig }: python-pin.pkgs.buildPythonPackage { @@ -8,7 +9,9 @@ python-pin.pkgs.buildPythonPackage { version = version; src = ./.; propagatedBuildInputs = [ + rose-zig py-deps.appdirs + py-deps.cffi py-deps.click py-deps.jinja2 py-deps.mutagen @@ -17,4 +20,5 @@ python-pin.pkgs.buildPythonPackage { py-deps.uuid6 ]; doCheck = false; + ROSE_SO_PATH = "${rose-zig}/lib/librose.so"; } diff --git a/rose-py/rose/ffi.py b/rose-py/rose/ffi.py new file mode 100644 index 0000000..25dc15c --- /dev/null +++ b/rose-py/rose/ffi.py @@ -0,0 +1,22 @@ +import os + +from cffi import FFI + +from rose.common import RoseError + +try: + so_path = os.environ["ROSE_SO_PATH"] +except KeyError as e: + raise RoseError("ROSE_SO_PATH unset: cannot load underlying Zig library") from e + +ffi = FFI() +lib = ffi.dlopen(so_path) +ffi.cdef(""" + void free_str(void *str); + + char *getRelease(); +""") + + +def get_release() -> str: + return ffi.string(ffi.gc(lib.getRelease(), lib.free_str)).decode("utf-8") # type: ignore diff --git a/rose-zig/build.zig b/rose-zig/build.zig new file mode 100644 index 0000000..d8f487a --- /dev/null +++ b/rose-zig/build.zig @@ -0,0 +1,47 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + // Dependencies. + const sqlite = b.dependency("sqlite", .{ + .target = target, + .optimize = optimize, + }); + // TODO: This is really expensive, so uncomment it only when we start using it. + // const ffmpeg = b.dependency("ffmpeg", .{ + // .target = target, + // .optimize = optimize, + // }); + + // Specify the core library module. + const rose = b.addModule("rose", .{ + .root_source_file = b.path("rose/root.zig"), + .target = target, + .optimize = optimize, + .imports = &[_]std.Build.Module.Import{ + // .{ .name = "av", .module = ffmpeg.module("av") }, + .{ .name = "sqlite", .module = sqlite.module("sqlite") }, + }, + }); + + // Tests for the core library module. + const test_step = b.step("test", "Run unit tests"); + const unit_tests = b.addTest(.{ + .root_source_file = .{ .path = "rose/root.zig" }, + .target = target, + }); + const run_unit_tests = b.addRunArtifact(unit_tests); + test_step.dependOn(&run_unit_tests.step); + + // Shared library for compatibility with other languages. + const librose = b.addSharedLibrary(.{ + .name = "rose", + .root_source_file = .{ .path = "ffi/root.zig" }, + .target = target, + .optimize = optimize, + }); + librose.root_module.addImport("rose", rose); + b.installArtifact(librose); +} diff --git a/rose-zig/build.zig.zon b/rose-zig/build.zig.zon new file mode 100644 index 0000000..862a1af --- /dev/null +++ b/rose-zig/build.zig.zon @@ -0,0 +1,15 @@ +.{ + .name = "rose", + .version = "0.5.0", + .dependencies = .{ + .sqlite = .{ + .url = "https://github.com/vrischmann/zig-sqlite/archive/dc339b7cf3bca82a12c2169231dd247587766781.tar.gz", + .hash = "1220e0961c135c5aa3af77a043dbc5890a18235a157238df0e2882fe84a8c8439c7a", + }, + .ffmpeg = .{ + .url = "https://github.com/andrewrk/ffmpeg/archive/1704e8898ea6217df91e6afc2a2de3f2b82a98d9.tar.gz", + .hash = "122032707cdf94da394e309978146ee33c61a285300eeb916928af376ec1638a95f1", + }, + }, + .paths = .{""}, +} diff --git a/rose-zig/default.nix b/rose-zig/default.nix new file mode 100644 index 0000000..235af1d --- /dev/null +++ b/rose-zig/default.nix @@ -0,0 +1,15 @@ +{ callPackage +, stdenv +, zig +, version +}: + +stdenv.mkDerivation { + pname = "rose-zig"; + version = version; + src = ./.; + nativeBuildInputs = [ zig.hook ]; + postPatch = '' + ln -s ${callPackage ./deps.nix { }} $ZIG_GLOBAL_CACHE_DIR/p + ''; +} diff --git a/rose-zig/deps.nix b/rose-zig/deps.nix new file mode 100644 index 0000000..4d915b7 --- /dev/null +++ b/rose-zig/deps.nix @@ -0,0 +1,62 @@ +# generated by zon2nix (https://github.com/nix-community/zon2nix) + +{ linkFarm, fetchzip }: + +linkFarm "zig-packages" [ + { + name = "122004fa7e2ff0b3d472049743358f8fdf065cdf63bc0e5e3d54c6bb8d81d93e40da"; + path = fetchzip { + url = "https://github.com/andrewrk/nasm/archive/b5f62392d56baf6aa02567f28e0da70664609262.tar.gz"; + hash = "sha256-tPBQixxG+phvEfRHeOLHMjY1Ynp7r9zNSgRy4R/ILQM="; + }; + } + { + name = "1220138f4aba0c01e66b68ed9e1e1e74614c06e4743d88bc58af4f1c3dd0aae5fea7"; + path = fetchzip { + url = "https://github.com/allyourcodebase/zlib/archive/refs/tags/1.3.1-3.tar.gz"; + hash = "sha256-R1tB+ORO3qeV/cNxsp5GqsiOyKUXjaj4Pd1v5AfWYz4="; + }; + } + { + name = "122032707cdf94da394e309978146ee33c61a285300eeb916928af376ec1638a95f1"; + path = fetchzip { + url = "https://github.com/andrewrk/ffmpeg/archive/1704e8898ea6217df91e6afc2a2de3f2b82a98d9.tar.gz"; + hash = "sha256-6EYu1QT76cJQMW9F41DvXsVLulH1YwGrkhwSV6IsBes="; + }; + } + { + name = "122074e0bf09c3622780e697c11c6744e763dd63777e480baf2b583ee3ab6a02ff14"; + path = fetchzip { + url = "https://github.com/andrewrk/libvorbis/archive/refs/tags/1.3.8-3.tar.gz"; + hash = "sha256-KHKYT3tmab9qYu8N2iJwm1rS+mU7Cwnn8Jp0cfOdnIg="; + }; + } + { + name = "12207d353609d95cee9da7891919e6d9582e97b7aa2831bd50f33bf523a582a08547"; + path = fetchzip { + url = "https://github.com/madler/zlib/archive/refs/tags/v1.3.tar.gz"; + hash = "sha256-eUuXV5zfy+fmiMNdWw5QCqDloBkaxy1tgi7by9nYHNA="; + }; + } + { + name = "1220b3e1fb33317c92f9ead09630f6b4be59e80d0a8780754f8aa4ee7da61cb7b47a"; + path = fetchzip { + url = "https://github.com/andrewrk/libogg/archive/refs/tags/1.3.6-2.tar.gz"; + hash = "sha256-3dFDBo4Af58bW8Gf+sHLigwo8CO2siwzWWtAoYe5opI="; + }; + } + { + name = "1220bee0fcf98bf6ad75b7bb09ff1f873ca38547a15b1e7a4532d20d94107d8d330a"; + path = fetchzip { + url = "https://github.com/andrewrk/libmp3lame/archive/refs/tags/3.100.1-3.tar.gz"; + hash = "sha256-kMI7JACnIVAdUHp5DUKx2XfKgIb1ftr6x/oYJdsTKyI="; + }; + } + { + name = "1220e0961c135c5aa3af77a043dbc5890a18235a157238df0e2882fe84a8c8439c7a"; + path = fetchzip { + url = "https://github.com/vrischmann/zig-sqlite/archive/dc339b7cf3bca82a12c2169231dd247587766781.tar.gz"; + hash = "sha256-YouCVidJqhI5+joTSe1aSbnjVC+qVG2aIz6MyibRmQk="; + }; + } +] diff --git a/rose-zig/ffi/root.zig b/rose-zig/ffi/root.zig new file mode 100644 index 0000000..999221b --- /dev/null +++ b/rose-zig/ffi/root.zig @@ -0,0 +1,17 @@ +const std = @import("std"); +const rose = @import("rose"); + +var gpa = std.heap.GeneralPurposeAllocator(.{}){}; +const allocator = gpa.allocator(); + +export fn getRelease() [*:0]const u8 { + const message = rose.getRelease(allocator) catch |err| switch (err) { + error.OutOfMemory => @panic("Out of memory"), + }; + return message.ptr; +} + +export fn free_str(str: [*:0]const u8) void { + const len = std.mem.len(str); + allocator.free(str[0 .. len + 1]); +} diff --git a/rose-zig/rose/root.zig b/rose-zig/rose/root.zig new file mode 100644 index 0000000..46ed947 --- /dev/null +++ b/rose-zig/rose/root.zig @@ -0,0 +1,26 @@ +const std = @import("std"); +const sqlite = @import("sqlite"); +const testing = std.testing; + +pub fn getRelease(allocator: std.mem.Allocator) ![:0]const u8 { + return try allocator.dupeZ(u8, "hello"); +} + +pub fn getTrack(allocator: std.mem.Allocator) void { + const db = try sqlite.Db.init(.{ + .mode = sqlite.Db.Mode{ .File = "/home/blissful/.cache/rose/cache.sqlite3" }, + .open_flags = .{ + .write = true, + .create = true, + }, + .threading_mode = .MultiThread, + }); + _ = db; + _ = allocator; +} + +test "basic add functionality" { + const message = try getRelease(testing.allocator); + try testing.expect(std.mem.eql(u8, message, "hello")); + testing.allocator.free(message); +}