From e319892a6a5d8d5726101cc60ed9ed23a973d540 Mon Sep 17 00:00:00 2001 From: Liam Date: Tue, 24 Dec 2024 19:16:59 -0500 Subject: [PATCH] Pull out media server and handling into separate container --- config/runtime.exs | 1 + docker-compose.yml | 14 + docker/app/Dockerfile | 21 +- docker/broker/Dockerfile | 27 + docker/{app => broker}/safe-rsvg-convert | 0 lib/philomena_media/analyzers/gif.ex | 3 +- lib/philomena_media/analyzers/jpeg.ex | 3 +- lib/philomena_media/analyzers/png.ex | 3 +- lib/philomena_media/analyzers/svg.ex | 3 +- lib/philomena_media/analyzers/webm.ex | 3 +- lib/philomena_media/broker.ex | 13 + lib/philomena_media/gif_preview.ex | 4 +- lib/philomena_media/intensities.ex | 4 +- lib/philomena_media/mime.ex | 2 +- lib/philomena_media/processors/gif.ex | 13 +- lib/philomena_media/processors/jpeg.ex | 13 +- lib/philomena_media/processors/png.ex | 9 +- lib/philomena_media/processors/svg.ex | 7 +- lib/philomena_media/processors/webm.ex | 9 +- native/broker/.dockerignore | 1 + native/broker/Cargo.lock | 1217 ++++++++++++++++++++++ native/broker/Cargo.toml | 23 + native/broker/src/broker_client.rs | 79 ++ native/broker/src/broker_server.rs | 65 ++ native/broker/src/command_client.rs | 119 +++ native/broker/src/command_server.rs | 63 ++ native/broker/src/lib.rs | 56 + native/broker/src/signal.rs | 15 + 28 files changed, 1743 insertions(+), 47 deletions(-) create mode 100644 docker/broker/Dockerfile rename docker/{app => broker}/safe-rsvg-convert (100%) create mode 100644 lib/philomena_media/broker.ex create mode 100644 native/broker/.dockerignore create mode 100644 native/broker/Cargo.lock create mode 100644 native/broker/Cargo.toml create mode 100644 native/broker/src/broker_client.rs create mode 100644 native/broker/src/broker_server.rs create mode 100644 native/broker/src/command_client.rs create mode 100644 native/broker/src/command_server.rs create mode 100644 native/broker/src/lib.rs create mode 100644 native/broker/src/signal.rs diff --git a/config/runtime.exs b/config/runtime.exs index e5c35a870..118f83a95 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -30,6 +30,7 @@ config :philomena, tag_file_root: System.fetch_env!("TAG_FILE_ROOT"), site_domains: System.fetch_env!("SITE_DOMAINS"), tag_url_root: System.fetch_env!("TAG_URL_ROOT"), + broker_addr: System.fetch_env!("BROKER_ADDR"), redis_host: System.get_env("REDIS_HOST", "localhost"), proxy_host: System.get_env("PROXY_HOST"), camo_host: System.get_env("CAMO_HOST"), diff --git a/docker-compose.yml b/docker-compose.yml index a12334188..ac1a82d40 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,6 +30,7 @@ services: - IMAGE_URL_ROOT=/img - BADGE_URL_ROOT=/badge-img - TAG_URL_ROOT=/tag-img + - BROKER_ADDR=broker:1500 - OPENSEARCH_URL=http://opensearch:9200 - REDIS_HOST=valkey - DATABASE_URL=ecto://postgres:postgres@postgres/philomena_dev @@ -52,6 +53,7 @@ services: - app_deps_data:/srv/philomena/deps - app_native_data:/srv/philomena/priv/native depends_on: + - broker - postgres - opensearch - valkey @@ -89,6 +91,18 @@ services: - .:/srv/philomena attach: false + broker: + build: + context: . + dockerfile: ./docker/broker/Dockerfile + attach: false + deploy: + resources: + limits: + cpus: 8 + memory: 4gb + pids: 8192 + web: build: context: . diff --git a/docker/app/Dockerfile b/docker/app/Dockerfile index 1b9a8d69e..3048ced98 100644 --- a/docker/app/Dockerfile +++ b/docker/app/Dockerfile @@ -1,27 +1,16 @@ FROM elixir:1.17.2-alpine -ADD https://api.github.com/repos/philomena-dev/FFmpeg/git/refs/heads/release/6.1 /tmp/ffmpeg_version.json -RUN (echo "https://github.com/philomena-dev/prebuilt-ffmpeg/raw/master"; cat /etc/apk/repositories) > /tmp/repositories \ - && cp /tmp/repositories /etc/apk/repositories \ - && apk update --allow-untrusted \ - && apk add inotify-tools build-base git ffmpeg ffmpeg-dev npm nodejs file-dev libjpeg-turbo-dev libpng-dev gifsicle optipng libjpeg-turbo-utils librsvg rsvg-convert imagemagick postgresql16-client wget rust cargo --allow-untrusted \ +RUN apk add inotify-tools build-base git npm nodejs postgresql16-client wget rust cargo \ && mix local.hex --force \ && mix local.rebar --force -ADD https://api.github.com/repos/philomena-dev/cli_intensities/git/refs/heads/master /tmp/cli_intensities_version.json -RUN git clone --depth 1 https://github.com/philomena-dev/cli_intensities /tmp/cli_intensities \ - && cd /tmp/cli_intensities \ - && make -j$(nproc) install - -ADD https://api.github.com/repos/philomena-dev/mediatools/git/refs/heads/master /tmp/mediatools_version.json -RUN git clone --depth 1 https://github.com/philomena-dev/mediatools /tmp/mediatools \ - && ln -s /usr/lib/librsvg-2.so.2 /usr/lib/librsvg-2.so \ - && cd /tmp/mediatools \ - && make -j$(nproc) install +COPY native/broker /tmp/broker +RUN cd /tmp/broker \ + && cargo build --release \ + && cp target/release/broker /usr/local/bin/broker COPY docker/app/run-development /usr/local/bin/run-development COPY docker/app/run-test /usr/local/bin/run-test -COPY docker/app/safe-rsvg-convert /usr/local/bin/safe-rsvg-convert COPY docker/app/purge-cache /usr/local/bin/purge-cache ENV PATH=$PATH:/root/.cargo/bin EXPOSE 5173 diff --git a/docker/broker/Dockerfile b/docker/broker/Dockerfile new file mode 100644 index 000000000..2d17e6d41 --- /dev/null +++ b/docker/broker/Dockerfile @@ -0,0 +1,27 @@ +FROM rust:1.83-alpine + +ADD https://api.github.com/repos/philomena-dev/FFmpeg/git/refs/heads/release/6.1 /tmp/ffmpeg_version.json +ADD https://api.github.com/repos/philomena-dev/cli_intensities/git/refs/heads/master /tmp/cli_intensities_version.json +ADD https://api.github.com/repos/philomena-dev/mediatools/git/refs/heads/master /tmp/mediatools_version.json + +RUN (echo "https://github.com/philomena-dev/prebuilt-ffmpeg/raw/master"; cat /etc/apk/repositories) > /tmp/repositories \ + && cp /tmp/repositories /etc/apk/repositories \ + && apk update --allow-untrusted \ + && apk add build-base git ffmpeg ffmpeg-dev file-dev libjpeg-turbo-dev libpng-dev \ + gifsicle optipng libjpeg-turbo-utils librsvg rsvg-convert imagemagick --allow-untrusted \ + && git clone --depth 1 https://github.com/philomena-dev/cli_intensities /tmp/cli_intensities \ + && cd /tmp/cli_intensities \ + && make -j$(nproc) install \ + && git clone --depth 1 https://github.com/philomena-dev/mediatools /tmp/mediatools \ + && ln -s /usr/lib/librsvg-2.so.2 /usr/lib/librsvg-2.so \ + && cd /tmp/mediatools \ + && make -j$(nproc) install + +COPY native/broker /tmp/broker +RUN cd /tmp/broker \ + && cargo build --release \ + && cp target/release/broker_server /usr/local/bin/broker_server + +COPY docker/broker/safe-rsvg-convert /usr/local/bin/safe-rsvg-convert +ENV RUST_LOG=trace +CMD ["/usr/local/bin/broker_server", "0.0.0.0:1500"] diff --git a/docker/app/safe-rsvg-convert b/docker/broker/safe-rsvg-convert similarity index 100% rename from docker/app/safe-rsvg-convert rename to docker/broker/safe-rsvg-convert diff --git a/lib/philomena_media/analyzers/gif.ex b/lib/philomena_media/analyzers/gif.ex index 982d7a319..edb88fb1c 100644 --- a/lib/philomena_media/analyzers/gif.ex +++ b/lib/philomena_media/analyzers/gif.ex @@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Analyzers.Gif do alias PhilomenaMedia.Analyzers.Analyzer alias PhilomenaMedia.Analyzers.Result + alias PhilomenaMedia.Broker @behaviour Analyzer @@ -20,7 +21,7 @@ defmodule PhilomenaMedia.Analyzers.Gif do end defp stats(file) do - case System.cmd("mediastat", [file]) do + case Broker.cmd("mediastat", [file]) do {output, 0} -> [_size, frames, width, height, num, den] = output diff --git a/lib/philomena_media/analyzers/jpeg.ex b/lib/philomena_media/analyzers/jpeg.ex index 60b29e04f..5df34f60d 100644 --- a/lib/philomena_media/analyzers/jpeg.ex +++ b/lib/philomena_media/analyzers/jpeg.ex @@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Analyzers.Jpeg do alias PhilomenaMedia.Analyzers.Analyzer alias PhilomenaMedia.Analyzers.Result + alias PhilomenaMedia.Broker @behaviour Analyzer @@ -20,7 +21,7 @@ defmodule PhilomenaMedia.Analyzers.Jpeg do end defp stats(file) do - case System.cmd("mediastat", [file]) do + case Broker.cmd("mediastat", [file]) do {output, 0} -> [_size, _frames, width, height, num, den] = output diff --git a/lib/philomena_media/analyzers/png.ex b/lib/philomena_media/analyzers/png.ex index 83cb506f4..bfa79f5a7 100644 --- a/lib/philomena_media/analyzers/png.ex +++ b/lib/philomena_media/analyzers/png.ex @@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Analyzers.Png do alias PhilomenaMedia.Analyzers.Analyzer alias PhilomenaMedia.Analyzers.Result + alias PhilomenaMedia.Broker @behaviour Analyzer @@ -20,7 +21,7 @@ defmodule PhilomenaMedia.Analyzers.Png do end defp stats(file) do - case System.cmd("mediastat", [file]) do + case Broker.cmd("mediastat", [file]) do {output, 0} -> [_size, frames, width, height, num, den] = output diff --git a/lib/philomena_media/analyzers/svg.ex b/lib/philomena_media/analyzers/svg.ex index f83a55f00..2c5c2a04a 100644 --- a/lib/philomena_media/analyzers/svg.ex +++ b/lib/philomena_media/analyzers/svg.ex @@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Analyzers.Svg do alias PhilomenaMedia.Analyzers.Analyzer alias PhilomenaMedia.Analyzers.Result + alias PhilomenaMedia.Broker @behaviour Analyzer @@ -20,7 +21,7 @@ defmodule PhilomenaMedia.Analyzers.Svg do end defp stats(file) do - case System.cmd("svgstat", [file]) do + case Broker.cmd("svgstat", [file]) do {output, 0} -> [_size, _frames, width, height, _num, _den] = output diff --git a/lib/philomena_media/analyzers/webm.ex b/lib/philomena_media/analyzers/webm.ex index b215e01e1..d0ece54f9 100644 --- a/lib/philomena_media/analyzers/webm.ex +++ b/lib/philomena_media/analyzers/webm.ex @@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Analyzers.Webm do alias PhilomenaMedia.Analyzers.Analyzer alias PhilomenaMedia.Analyzers.Result + alias PhilomenaMedia.Broker @behaviour Analyzer @@ -20,7 +21,7 @@ defmodule PhilomenaMedia.Analyzers.Webm do end defp stats(file) do - case System.cmd("mediastat", [file]) do + case Broker.cmd("mediastat", [file]) do {output, 0} -> [_size, frames, width, height, num, den] = output diff --git a/lib/philomena_media/broker.ex b/lib/philomena_media/broker.ex new file mode 100644 index 000000000..000be6fd8 --- /dev/null +++ b/lib/philomena_media/broker.ex @@ -0,0 +1,13 @@ +defmodule PhilomenaMedia.Broker do + @doc """ + Out-of-process replacement for `System.cmd/2` that calls the requested + command elsewhere, translating file accesses, and returns the result. + """ + def cmd(command, args) do + System.cmd("broker", [broker_addr(), "execute-command", "--", command] ++ args) + end + + defp broker_addr do + Application.get_env(:philomena, :broker_addr) + end +end diff --git a/lib/philomena_media/gif_preview.ex b/lib/philomena_media/gif_preview.ex index fe3f79140..d9e0ac63e 100644 --- a/lib/philomena_media/gif_preview.ex +++ b/lib/philomena_media/gif_preview.ex @@ -3,6 +3,8 @@ defmodule PhilomenaMedia.GifPreview do GIF preview generation for video files. """ + alias PhilomenaMedia.Broker + @type duration :: float() @type dimensions :: {pos_integer(), pos_integer()} @@ -46,7 +48,7 @@ defmodule PhilomenaMedia.GifPreview do end) {_output, 0} = - System.cmd( + Broker.cmd( "ffmpeg", commands(video, gif, clamp(duration), dimensions, num_images, target_framerate) ) diff --git a/lib/philomena_media/intensities.ex b/lib/philomena_media/intensities.ex index ea0952952..19e792913 100644 --- a/lib/philomena_media/intensities.ex +++ b/lib/philomena_media/intensities.ex @@ -17,6 +17,8 @@ defmodule PhilomenaMedia.Intensities do of image dimensions, with poor precision and a poor-to-fair accuracy. """ + alias PhilomenaMedia.Broker + @type t :: %__MODULE__{ nw: float(), ne: float(), @@ -50,7 +52,7 @@ defmodule PhilomenaMedia.Intensities do """ @spec file(Path.t()) :: {:ok, t()} | :error def file(input) do - System.cmd("image-intensities", [input]) + Broker.cmd("image-intensities", [input]) |> case do {output, 0} -> [nw, ne, sw, se] = diff --git a/lib/philomena_media/mime.ex b/lib/philomena_media/mime.ex index 1b29aa759..6305f599b 100644 --- a/lib/philomena_media/mime.ex +++ b/lib/philomena_media/mime.ex @@ -27,7 +27,7 @@ defmodule PhilomenaMedia.Mime do """ @spec file(Path.t()) :: {:ok, t()} | {:unsupported_mime, t()} | :error def file(path) do - System.cmd("file", ["-b", "--mime-type", path]) + PhilomenaMedia.Broker.cmd("file", ["-b", "--mime-type", path]) |> case do {output, 0} -> true_mime(String.trim(output)) diff --git a/lib/philomena_media/processors/gif.ex b/lib/philomena_media/processors/gif.ex index 6e185f9fb..6e45b8ccc 100644 --- a/lib/philomena_media/processors/gif.ex +++ b/lib/philomena_media/processors/gif.ex @@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Processors.Gif do alias PhilomenaMedia.Intensities alias PhilomenaMedia.Analyzers.Result + alias PhilomenaMedia.Broker alias PhilomenaMedia.Processors.Processor alias PhilomenaMedia.Processors @@ -46,7 +47,7 @@ defmodule PhilomenaMedia.Processors.Gif do defp optimize(file) do optimized = Briefly.create!(extname: ".gif") - {_output, 0} = System.cmd("gifsicle", ["--careful", "-O2", file, "-o", optimized]) + {_output, 0} = Broker.cmd("gifsicle", ["--careful", "-O2", file, "-o", optimized]) optimized end @@ -54,7 +55,7 @@ defmodule PhilomenaMedia.Processors.Gif do defp preview(duration, file) do preview = Briefly.create!(extname: ".png") - {_output, 0} = System.cmd("mediathumb", [file, to_string(duration / 2), preview]) + {_output, 0} = Broker.cmd("mediathumb", [file, to_string(duration / 2), preview]) preview end @@ -63,7 +64,7 @@ defmodule PhilomenaMedia.Processors.Gif do palette = Briefly.create!(extname: ".png") {_output, 0} = - System.cmd("ffmpeg", [ + Broker.cmd("ffmpeg", [ "-loglevel", "0", "-y", @@ -88,7 +89,7 @@ defmodule PhilomenaMedia.Processors.Gif do filter_graph = "[0:v]#{scale_filter}[x];[x][1:v]#{palette_filter}" {_output, 0} = - System.cmd("ffmpeg", [ + Broker.cmd("ffmpeg", [ "-loglevel", "0", "-y", @@ -109,7 +110,7 @@ defmodule PhilomenaMedia.Processors.Gif do mp4 = Briefly.create!(extname: ".mp4") {_output, 0} = - System.cmd("ffmpeg", [ + Broker.cmd("ffmpeg", [ "-loglevel", "0", "-y", @@ -127,7 +128,7 @@ defmodule PhilomenaMedia.Processors.Gif do ]) {_output, 0} = - System.cmd("ffmpeg", [ + Broker.cmd("ffmpeg", [ "-loglevel", "0", "-y", diff --git a/lib/philomena_media/processors/jpeg.ex b/lib/philomena_media/processors/jpeg.ex index 604442571..948bb68ab 100644 --- a/lib/philomena_media/processors/jpeg.ex +++ b/lib/philomena_media/processors/jpeg.ex @@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do alias PhilomenaMedia.Intensities alias PhilomenaMedia.Analyzers.Result + alias PhilomenaMedia.Broker alias PhilomenaMedia.Processors.Processor alias PhilomenaMedia.Processors @@ -42,7 +43,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do defp requires_lossy_transformation?(file) do with {output, 0} <- - System.cmd("identify", ["-format", "%[orientation]\t%[profile:icc]", file]), + Broker.cmd("identify", ["-format", "%[orientation]\t%[profile:icc]", file]), [orientation, profile] <- String.split(output, "\t") do orientation not in ["Undefined", "TopLeft"] or profile != "" else @@ -60,7 +61,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do true -> # Transcode: strip EXIF, embedded profile and reorient image {_output, 0} = - System.cmd("convert", [ + Broker.cmd("convert", [ file, "-profile", srgb_profile(), @@ -71,7 +72,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do _ -> # Transmux only: Strip EXIF without touching orientation - validate_return(System.cmd("jpegtran", ["-copy", "none", "-outfile", stripped, file])) + validate_return(Broker.cmd("jpegtran", ["-copy", "none", "-outfile", stripped, file])) end stripped @@ -80,7 +81,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do defp optimize(file) do optimized = Briefly.create!(extname: ".jpg") - validate_return(System.cmd("jpegtran", ["-optimize", "-outfile", optimized, file])) + validate_return(Broker.cmd("jpegtran", ["-optimize", "-outfile", optimized, file])) optimized end @@ -90,7 +91,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do scale_filter = "scale=w=#{width}:h=#{height}:force_original_aspect_ratio=decrease" {_output, 0} = - System.cmd("ffmpeg", [ + Broker.cmd("ffmpeg", [ "-loglevel", "0", "-y", @@ -103,7 +104,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do scaled ]) - {_output, 0} = System.cmd("jpegtran", ["-optimize", "-outfile", scaled, scaled]) + {_output, 0} = Broker.cmd("jpegtran", ["-optimize", "-outfile", scaled, scaled]) [{:copy, scaled, "#{thumb_name}.jpg"}] end diff --git a/lib/philomena_media/processors/png.ex b/lib/philomena_media/processors/png.ex index 0fc4c50d3..344a6538a 100644 --- a/lib/philomena_media/processors/png.ex +++ b/lib/philomena_media/processors/png.ex @@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Processors.Png do alias PhilomenaMedia.Intensities alias PhilomenaMedia.Analyzers.Result + alias PhilomenaMedia.Broker alias PhilomenaMedia.Processors.Processor alias PhilomenaMedia.Processors @@ -49,7 +50,7 @@ defmodule PhilomenaMedia.Processors.Png do optimized = Briefly.create!(extname: ".png") {_output, 0} = - System.cmd("optipng", ["-fix", "-i0", "-o2", "-quiet", "-clobber", file, "-out", optimized]) + Broker.cmd("optipng", ["-fix", "-i0", "-o2", "-quiet", "-clobber", file, "-out", optimized]) # Remove useless .bak file File.rm(optimized <> ".bak") @@ -66,7 +67,7 @@ defmodule PhilomenaMedia.Processors.Png do {_output, 0} = cond do animated? -> - System.cmd("ffmpeg", [ + Broker.cmd("ffmpeg", [ "-loglevel", "0", "-y", @@ -82,10 +83,10 @@ defmodule PhilomenaMedia.Processors.Png do ]) true -> - System.cmd("ffmpeg", ["-loglevel", "0", "-y", "-i", file, "-vf", scale_filter, scaled]) + Broker.cmd("ffmpeg", ["-loglevel", "0", "-y", "-i", file, "-vf", scale_filter, scaled]) end - System.cmd("optipng", ["-i0", "-o1", "-quiet", "-clobber", scaled]) + Broker.cmd("optipng", ["-i0", "-o1", "-quiet", "-clobber", scaled]) [{:copy, scaled, "#{thumb_name}.png"}] end diff --git a/lib/philomena_media/processors/svg.ex b/lib/philomena_media/processors/svg.ex index aaa3dd5ca..b43dc6ef9 100644 --- a/lib/philomena_media/processors/svg.ex +++ b/lib/philomena_media/processors/svg.ex @@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Processors.Svg do alias PhilomenaMedia.Intensities alias PhilomenaMedia.Analyzers.Result + alias PhilomenaMedia.Broker alias PhilomenaMedia.Processors.Processor alias PhilomenaMedia.Processors @@ -42,7 +43,7 @@ defmodule PhilomenaMedia.Processors.Svg do defp preview(file) do preview = Briefly.create!(extname: ".png") - {_output, 0} = System.cmd("safe-rsvg-convert", [file, preview]) + {_output, 0} = Broker.cmd("safe-rsvg-convert", [file, preview]) preview end @@ -52,9 +53,9 @@ defmodule PhilomenaMedia.Processors.Svg do scale_filter = "scale=w=#{width}:h=#{height}:force_original_aspect_ratio=decrease" {_output, 0} = - System.cmd("ffmpeg", ["-loglevel", "0", "-y", "-i", preview, "-vf", scale_filter, scaled]) + Broker.cmd("ffmpeg", ["-loglevel", "0", "-y", "-i", preview, "-vf", scale_filter, scaled]) - {_output, 0} = System.cmd("optipng", ["-i0", "-o1", "-quiet", "-clobber", scaled]) + {_output, 0} = Broker.cmd("optipng", ["-i0", "-o1", "-quiet", "-clobber", scaled]) [{:copy, scaled, "#{thumb_name}.png"}] end diff --git a/lib/philomena_media/processors/webm.ex b/lib/philomena_media/processors/webm.ex index 22c54a6d8..6f825e30d 100644 --- a/lib/philomena_media/processors/webm.ex +++ b/lib/philomena_media/processors/webm.ex @@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Processors.Webm do alias PhilomenaMedia.Intensities alias PhilomenaMedia.Analyzers.Result + alias PhilomenaMedia.Broker alias PhilomenaMedia.GifPreview alias PhilomenaMedia.Processors.Processor alias PhilomenaMedia.Processors @@ -55,7 +56,7 @@ defmodule PhilomenaMedia.Processors.Webm do defp preview(duration, file) do preview = Briefly.create!(extname: ".png") - {_output, 0} = System.cmd("mediathumb", [file, to_string(duration / 2), preview]) + {_output, 0} = Broker.cmd("mediathumb", [file, to_string(duration / 2), preview]) preview end @@ -64,7 +65,7 @@ defmodule PhilomenaMedia.Processors.Webm do stripped = Briefly.create!(extname: ".webm") {_output, 0} = - System.cmd("ffmpeg", [ + Broker.cmd("ffmpeg", [ "-loglevel", "0", "-y", @@ -109,7 +110,7 @@ defmodule PhilomenaMedia.Processors.Webm do mp4 = Briefly.create!(extname: ".mp4") {_output, 0} = - System.cmd("ffmpeg", [ + Broker.cmd("ffmpeg", [ "-loglevel", "0", "-y", @@ -167,7 +168,7 @@ defmodule PhilomenaMedia.Processors.Webm do mp4 = Briefly.create!(extname: ".mp4") {_output, 0} = - System.cmd("ffmpeg", [ + Broker.cmd("ffmpeg", [ "-loglevel", "0", "-y", diff --git a/native/broker/.dockerignore b/native/broker/.dockerignore new file mode 100644 index 000000000..eb5a316cb --- /dev/null +++ b/native/broker/.dockerignore @@ -0,0 +1 @@ +target diff --git a/native/broker/Cargo.lock b/native/broker/Cargo.lock new file mode 100644 index 000000000..79f4c74d1 --- /dev/null +++ b/native/broker/Cargo.lock @@ -0,0 +1,1217 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +dependencies = [ + "anstyle", + "windows-sys 0.59.0", +] + +[[package]] +name = "anyhow" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" + +[[package]] +name = "async-trait" +version = "0.1.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "backtrace" +version = "0.3.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets", +] + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bitflags" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" + +[[package]] +name = "broker" +version = "0.1.0" +dependencies = [ + "clap", + "env_logger", + "futures", + "once_cell", + "serde", + "tarpc", + "tempfile", + "tokio", + "tracing", +] + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clap" +version = "4.5.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" + +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + +[[package]] +name = "educe" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4bd92664bf78c4d3dba9b7cdafce6fa15b13ed3ed16175218196942e99168a8" +dependencies = [ + "enum-ordinalize", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "enum-ordinalize" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea0dcfa4e54eeb516fe454635a95753ddd39acda650ce703031c6973e315dd5" +dependencies = [ + "enum-ordinalize-derive", +] + +[[package]] +name = "enum-ordinalize-derive" +version = "4.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d28318a75d4aead5c4db25382e8ef717932d0346600cacae6357eb5941bc5ff" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "env_filter" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcaee3d8e3cfc3fd92428d477bc97fc29ec8716d180c0d74c643bb26166660e0" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "humantime", + "log", +] + +[[package]] +name = "errno" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "itoa" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" + +[[package]] +name = "js-sys" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.169" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" + +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "miniz_oxide" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ffbe83022cedc1d264172192511ae958937694cd57ce297164951b8b3568394" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.52.0", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + +[[package]] +name = "opentelemetry" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "570074cc999d1a58184080966e5bd3bf3a9a4af650c3b05047c2621e7405cd17" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cefe0543875379e47eb5f1e68ff83f45cc41366a92dfd0d073d513bf68e9a05" + +[[package]] +name = "opentelemetry_sdk" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2c627d9f4c9cdc1f21a29ee4bfbd6028fcb8bcf2a857b43f3abdf72c9c862f3" +dependencies = [ + "async-trait", + "futures-channel", + "futures-executor", + "futures-util", + "once_cell", + "opentelemetry", + "percent-encoding", + "rand", + "thiserror", +] + +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + +[[package]] +name = "rustix" +version = "0.38.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.216" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.216" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.134" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "socket2" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tarpc" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14d1be17be018ebeec4c489449adb5ef227746775974c311ce79e09886ef83c7" +dependencies = [ + "anyhow", + "fnv", + "futures", + "humantime", + "opentelemetry", + "opentelemetry-semantic-conventions", + "pin-project", + "rand", + "serde", + "static_assertions", + "tarpc-plugins", + "thiserror", + "tokio", + "tokio-serde", + "tokio-util", + "tracing", + "tracing-opentelemetry", +] + +[[package]] +name = "tarpc-plugins" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0e3d9676af494694e11a3e367a4bfa7f6d1d5566bd0fe9aceb4aa9281122ab8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tempfile" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" +dependencies = [ + "cfg-if", + "fastrand", + "once_cell", + "rustix", + "windows-sys 0.59.0", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "tokio" +version = "1.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-macros" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-serde" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf600e7036b17782571dd44fa0a5cea3c82f60db5137f774a325a76a0d6852b" +dependencies = [ + "bincode", + "bytes", + "educe", + "futures-core", + "futures-sink", + "pin-project", + "serde", + "serde_json", +] + +[[package]] +name = "tokio-util" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "slab", + "tokio", +] + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-opentelemetry" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc58af5d3f6c5811462cabb3289aec0093f7338e367e5a33d28c0433b3c7360b" +dependencies = [ + "js-sys", + "once_cell", + "opentelemetry", + "opentelemetry_sdk", + "tracing", + "tracing-core", + "tracing-subscriber", + "web-time", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +dependencies = [ + "sharded-slab", + "thread_local", + "tracing-core", +] + +[[package]] +name = "unicode-ident" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" +dependencies = [ + "cfg-if", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/native/broker/Cargo.toml b/native/broker/Cargo.toml new file mode 100644 index 000000000..5be375efa --- /dev/null +++ b/native/broker/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "broker" +version = "0.1.0" +edition = "2021" + +[[bin]] +name = "broker" +path = "src/broker_client.rs" + +[[bin]] +name = "broker_server" +path = "src/broker_server.rs" + +[dependencies] +env_logger = "0.11" +clap = { version = "4.5", features = ["derive"] } +futures = "0.3" +once_cell = "1.20" +serde = { version = "1.0", features = ["derive"] } +tarpc = { version = "0.35", features = ["full"] } +tempfile = "3" +tokio = { version = "1.0", features = ["full"] } +tracing = "0.1" diff --git a/native/broker/src/broker_client.rs b/native/broker/src/broker_client.rs new file mode 100644 index 000000000..47015367a --- /dev/null +++ b/native/broker/src/broker_client.rs @@ -0,0 +1,79 @@ +use std::io::Write; +use std::process::ExitCode; + +use broker::BrokerClient; +use clap::{Parser, Subcommand}; + +mod command_client; + +#[derive(Parser, Debug)] +#[command(version, about = "RPC Broker", long_about = None)] +struct Arguments { + /// Server address to connect to, like localhost:1500 + server_addr: String, + + /// Subcommand to execute. + #[command(subcommand)] + invocation_type: InvocationType, +} + +#[derive(Subcommand, Debug)] +enum InvocationType { + /// Execute a command with the given arguments on the remote server. + ExecuteCommand { + /// Program name to execute. + /// + /// One of convert, ffprobe, ffmpeg, file, gifsicle, identify, + /// image-intensities, jpegtran, mediastat, optipng, safe-rsvg-convert. + program: String, + /// Arguments to pass to program. + args: Vec, + }, +} + +#[tokio::main(flavor = "current_thread")] +async fn main() -> ExitCode { + let args = Arguments::parse(); + let client = connect_to_socket_server(&args) + .await + .expect("failed to connect to server"); + + match args.invocation_type { + InvocationType::ExecuteCommand { program, args } => { + run_command_client(&client, program, args).await + } + } +} + +async fn connect_to_socket_server(args: &Arguments) -> Option { + let codec = tarpc::tokio_serde::formats::Json::default; + + for addr in tokio::net::lookup_host(&args.server_addr).await.ok()? { + let mut transport = tarpc::serde_transport::tcp::connect(addr, codec); + transport.config_mut().max_frame_length(usize::MAX); + + let transport = match transport.await { + Ok(transport) => transport, + _ => continue, + }; + + return Some(BrokerClient::new(tarpc::client::Config::default(), transport).spawn()); + } + + None +} + +async fn run_command_client(client: &BrokerClient, program: String, args: Vec) -> ExitCode { + let reply = command_client::execute_command(client, program, args) + .await + .unwrap(); + + write_then_drop(std::io::stderr(), reply.stderr); + write_then_drop(std::io::stdout(), reply.stdout); + + reply.status.into() +} + +fn write_then_drop(mut stream: impl Write, data: Vec) { + stream.write_all(&data).unwrap() +} diff --git a/native/broker/src/broker_server.rs b/native/broker/src/broker_server.rs new file mode 100644 index 000000000..4ad7b8de3 --- /dev/null +++ b/native/broker/src/broker_server.rs @@ -0,0 +1,65 @@ +use std::net::SocketAddr; + +use broker::{Broker, CommandReply, ExecuteCommandError, FileMap}; +use clap::Parser; +use futures::{future, Future, StreamExt}; +use tarpc::context; +use tarpc::server::Channel; + +mod command_server; +mod signal; + +#[derive(Parser, Debug)] +#[command(version, about = "RPC Broker Server", long_about = None)] +struct Arguments { + /// Socket address to bind to, like 127.0.0.1:1500 + server_addr: SocketAddr, +} + +#[derive(Clone)] +struct BrokerServer; + +impl Broker for BrokerServer { + async fn execute_command( + self, + _: context::Context, + program: String, + arguments: Vec, + file_map: FileMap, + ) -> Result<(CommandReply, FileMap), ExecuteCommandError> { + command_server::execute_command(program, arguments, file_map).await + } +} + +fn main() { + env_logger::init(); + + let args = Arguments::parse(); + + serve(&args); +} + +async fn spawn(fut: impl Future + Send + 'static) { + tokio::spawn(fut); +} + +#[tokio::main] +async fn serve(args: &Arguments) { + signal::install_handlers(); + + let codec = tarpc::tokio_serde::formats::Json::default; + let mut listener = tarpc::serde_transport::tcp::listen(args.server_addr, codec) + .await + .unwrap(); + + listener.config_mut().max_frame_length(usize::MAX); + listener + // Ignore accept errors. + .filter_map(|r| future::ready(r.ok())) + .map(tarpc::server::BaseChannel::with_defaults) + .map(|channel| { + tokio::spawn(channel.execute(BrokerServer.serve()).for_each(spawn)); + }) + .collect() + .await +} diff --git a/native/broker/src/command_client.rs b/native/broker/src/command_client.rs new file mode 100644 index 000000000..9096ea6f3 --- /dev/null +++ b/native/broker/src/command_client.rs @@ -0,0 +1,119 @@ +use std::collections::{HashMap, HashSet}; +use std::ffi::OsString; +use std::path::Path; +use std::time::{Duration, Instant}; + +use broker::{BrokerClient, CommandReply, ExecuteCommandError, FileMap}; +use once_cell::sync::Lazy; +use tarpc::context::Context; + +#[derive(Default)] +struct CallParameters { + /// Mapping from replaced name to original name. + replacements: HashMap, + /// List of post-processed arguments. + arguments: Vec, + /// Mapping of replaced name to file contents. + file_map: FileMap, +} + +/// List of file extensions which can be forwarded. +static FORWARDED_EXTS: Lazy> = Lazy::new(|| { + vec![ + "gif", "jpg", "jpeg", "png", "svg", "webm", "webp", "mp4", "icc", + ] + .into_iter() + .map(Into::into) + .collect() +}); + +fn forwarded_ext(path: &Path) -> Option<&str> { + match path.extension() { + Some(ext) if FORWARDED_EXTS.contains(ext) => ext.to_str(), + _ => None, + } +} + +fn create_replacements(arguments: impl Iterator) -> CallParameters { + use std::fs::read; + + // Maps original name to replaced name. + let mut processed = HashMap::::new(); + let mut counter: usize = 0; + + let mut output = CallParameters::default(); + + output.arguments = arguments + .map(|arg| { + let path = Path::new(&arg); + + // Avoid adding additional replacements if the same file is passed multiple times. + if let Some(replaced_name) = processed.get(&arg) { + return replaced_name.clone(); + } + + // Only try things that look like paths. + if !path.is_absolute() { + return arg; + } + + // Only forward if file is in extension allow list. + let Some(ext) = forwarded_ext(path) else { + return arg; + }; + + // Don't forward paths that don't exist or can't be read. + let Ok(contents) = read(path) else { + return arg; + }; + + let replaced_name = format!("{}.{}", counter, ext); + counter = counter.checked_add(1).unwrap(); + + processed.insert(arg.clone(), replaced_name.clone()); // original -> replaced + output.replacements.insert(replaced_name.clone(), arg); // replaced -> original + output.file_map.insert(replaced_name.clone(), contents); // replaced -> [contents] + + replaced_name + }) + .collect(); + + output +} + +fn update_replacements(replacements: HashMap, file_map: FileMap) { + use std::fs::write; + + for (replaced_name, contents) in file_map { + // Intentionally panic if the original name didn't exist or writing fails. + let original_name = replacements.get(&replaced_name).unwrap(); + write(original_name, contents).unwrap(); + } +} + +fn context_with_1_hour_deadline() -> Context { + let mut context = Context::current(); + context.deadline = Instant::now() + Duration::from_secs(60 * 60); + context +} + +pub async fn execute_command( + client: &BrokerClient, + program: String, + arguments: Vec, +) -> Result { + let call_params = create_replacements(arguments.into_iter()); + let (reply, file_map) = client + .execute_command( + context_with_1_hour_deadline(), + program, + call_params.arguments, + call_params.file_map, + ) + .await + .unwrap()?; + + update_replacements(call_params.replacements, file_map); + + Ok(reply) +} diff --git a/native/broker/src/command_server.rs b/native/broker/src/command_server.rs new file mode 100644 index 000000000..9400ae400 --- /dev/null +++ b/native/broker/src/command_server.rs @@ -0,0 +1,63 @@ +use std::collections::HashSet; +use std::os::unix::process::ExitStatusExt; + +use broker::{CommandReply, ExecuteCommandError, FileMap, PERMITTED_PROGRAMS}; +use tokio::process::Command; + +fn validate_name(name: &str) -> Result<(), ExecuteCommandError> { + if name == "." || name.contains("..") || name.contains('/') || name.contains('\\') { + return Err(ExecuteCommandError::InvalidFileMapName); + } + + Ok(()) +} + +pub async fn execute_command( + program: String, + arguments: Vec, + file_map: FileMap, +) -> Result<(CommandReply, FileMap), ExecuteCommandError> { + use std::fs::{read, write}; + + // Check program name. + if !PERMITTED_PROGRAMS.contains(&program.as_ref()) { + return Err(ExecuteCommandError::UnpermittedProgram(program)); + } + + // Create a new temporary directory which we will work in. + let dir = tempfile::tempdir().map_err(|_| ExecuteCommandError::RemoteFilesystemError)?; + + // Verify and write out all files. + let mut files = HashSet::::new(); + for (name, contents) in file_map { + validate_name(&name)?; + files.insert(name.clone()); + + let path = dir.path().join(name); + write(path, contents).map_err(|_| ExecuteCommandError::RemoteFilesystemError)?; + } + + // Run the command. + let output = Command::new(program) + .args(arguments) + .current_dir(dir.path()) + .output() + .await + .map_err(|_| ExecuteCommandError::ExecutionError)?; + + // Read back all files. + let mut file_map = FileMap::new(); + for name in files { + let path = dir.path().join(name.clone()); + let contents = read(path).map_err(|_| ExecuteCommandError::RemoteFilesystemError)?; + file_map.insert(name, contents); + } + + let reply = CommandReply { + status: output.status.into_raw() as u8, + stdout: output.stdout, + stderr: output.stderr, + }; + + Ok((reply, file_map)) +} diff --git a/native/broker/src/lib.rs b/native/broker/src/lib.rs new file mode 100644 index 000000000..5163f49b3 --- /dev/null +++ b/native/broker/src/lib.rs @@ -0,0 +1,56 @@ +use once_cell::sync::Lazy; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; + +#[tarpc::service] +pub trait Broker { + /// Executes a command on the broker server. + async fn execute_command( + program: String, + arguments: Vec, + file_map: FileMap, + ) -> Result<(CommandReply, FileMap), ExecuteCommandError>; +} + +/// Errors which can occur during command execution. +#[derive(Debug, Deserialize, Serialize)] +pub enum ExecuteCommandError { + /// Requested program was not allowed to be executed. + UnpermittedProgram(String), + /// Failed to launch program. + ExecutionError, + /// File map name character was not allowed ('..', '/', '\\'). + InvalidFileMapName, + /// Generic filesystem error. + RemoteFilesystemError, +} + +/// Enumeration of permitted program names. +pub static PERMITTED_PROGRAMS: Lazy> = Lazy::new(|| { + vec![ + "convert", + "ffprobe", + "ffmpeg", + "file", + "gifsicle", + "identify", + "image-intensities", + "jpegtran", + "mediastat", + "optipng", + "safe-rsvg-convert", + ] + .into_iter() + .collect() +}); + +/// Mapping between file name and file contents. +pub type FileMap = HashMap>; + +/// Output reply from broker after command execution has finished. +#[derive(Debug, Deserialize, Serialize)] +pub struct CommandReply { + pub status: u8, + pub stdout: Vec, + pub stderr: Vec, +} diff --git a/native/broker/src/signal.rs b/native/broker/src/signal.rs new file mode 100644 index 000000000..68135d362 --- /dev/null +++ b/native/broker/src/signal.rs @@ -0,0 +1,15 @@ +use tokio::signal::unix::{signal, SignalKind}; + +pub fn install_handlers() { + let mut sigterm = signal(SignalKind::terminate()).unwrap(); + let mut sigint = signal(SignalKind::interrupt()).unwrap(); + + tokio::spawn(async move { + tokio::select! { + _ = sigterm.recv() => tracing::debug!("Received SIGTERM"), + _ = sigint.recv() => tracing::debug!("Received SIGINT"), + }; + + std::process::exit(1); + }); +}