Multi GPU and encoders improvements

.devcontainer/dev-setup.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -e
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get update
+
+# Install debugger
+apt-get install -y gdb
+
+# Install wayland-protocols
+apt-get install -y wayland-protocols
+
+# Build and install nvtop
+apt-get install -y libdrm-dev libsystemd-dev libncurses5-dev libncursesw5-dev
+cd /tmp/
+git clone https://github.com/Syllo/nvtop.git
+mkdir -p nvtop/build && cd nvtop/build
+CXX=/usr/bin/clang++ cmake .. -DNVIDIA_SUPPORT=ON -DAMDGPU_SUPPORT=ON -DINTEL_SUPPORT=ON
+cmake --build . --target install --config Release
+
+# Setup nvidia
+bash /etc/cont-init.d/30-nvidia.sh

.devcontainer/devcontainer.json

@@ -0,0 +1,69 @@
+{
+  "name": "wolf-dev",
+  "build": {
+    "context": "..",
+    "dockerfile": "../docker/wolf.Dockerfile",
+    "target": "wolf-builder"
+  },
+  "postStartCommand": "bash /workspaces/wolf/.devcontainer/dev-setup.sh",
+  "features": {
+    "ghcr.io/nils-geistmann/devcontainers-features/zsh:0": {
+      "plugins": [
+        "git",
+        "zsh-autosuggestions",
+        "history",
+        "docker"
+      ]
+    },
+    "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {
+      "moby": true,
+      "installDockerBuildx": false
+    }
+  },
+  "runArgs": [
+    "--network=host",
+    "-v",
+    "/tmp/sockets:/tmp/sockets:rw",
+    "-v",
+    "/etc/wolf:/etc/wolf:rw",
+    "-v",
+    "/run/udev:/run/udev:ro",
+    "--rm"
+  ],
+  "hostRequirements": {
+    "gpu": true
+  },
+  "privileged": true,
+  "containerEnv": {
+    "NVIDIA_DRIVER_CAPABILITIES": "all",
+    "NVIDIA_VISIBLE_DEVICES": "all"
+  },
+  "remoteEnv": {
+    "XDG_RUNTIME_DIR": "/tmp/sockets",
+    "WOLF_LOG_LEVEL": "DEBUG",
+    "HOST_APPS_STATE_FOLDER": "/etc/wolf",
+    "WOLF_CFG_FILE": "/etc/wolf/cfg/config.toml",
+    "WOLF_PRIVATE_KEY_FILE": "/etc/wolf/cfg/key.pem",
+    "WOLF_PRIVATE_CERT_FILE": "/etc/wolf/cfg/cert.pem",
+    "WOLF_PULSE_IMAGE": "ghcr.io/games-on-whales/pulseaudio:master",
+    "WOLF_RENDER_NODE": "/dev/dri/renderD128",
+    "WOLF_STOP_CONTAINER_ON_EXIT": "TRUE",
+    "WOLF_DOCKER_SOCKET": "/var/run/docker.sock",
+    "WOLF_DOCKER_FAKE_UDEV_PATH": "/wolf/fake-udev",
+    "RUST_BACKTRACE": "full",
+    "RUST_LOG": "INFO",
+    "GST_DEBUG": "2",
+    "CMAKE_GENERATOR": "Ninja"
+  },
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "ms-vscode.cpptools-extension-pack",
+        "ms-azuretools.vscode-docker",
+        "github.vscode-github-actions",
+        "jeff-hykin.better-cpp-syntax",
+        "tamasfe.even-better-toml"
+      ]
+    }
+  }
+}

.github/workflows/docker-build.yml

@@ -109,9 +109,9 @@ jobs:
           file: docker/gstreamer.Dockerfile
           push: true
           build-args: |
-            GSTREAMER_VERSION=1.22.7
+            GSTREAMER_VERSION=1.24.6
             BASE_IMAGE=ghcr.io/${{ github.repository_owner }}/gpu-drivers:2023.11
-          tags: ghcr.io/${{ github.repository_owner }}/gstreamer:1.22.7,gameonwhales/gstreamer:1.22.7 # TODO: set gstreamer version as param
+          tags: ghcr.io/${{ github.repository_owner }}/gstreamer:1.24.6,gameonwhales/gstreamer:1.24.6 # TODO: set gstreamer version as param
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/gstreamer:buildcache
           cache-to: type=registry,ref=ghcr.io/${{ github.repository_owner }}/gstreamer:buildcache,mode=max
@@ -126,7 +126,7 @@ jobs:
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           build-args: |
-            BASE_IMAGE=ghcr.io/${{ github.repository_owner }}/gstreamer:1.22.7
+            BASE_IMAGE=ghcr.io/${{ github.repository_owner }}/gstreamer:1.24.6
             IMAGE_SOURCE=${{ steps.prep.outputs.github_server_url }}/${{ github.repository }}
           cache-from: ${{ steps.prep.outputs.cache_from }}
-          cache-to: ${{ steps.prep.outputs.cache_to }}
+          cache-to: ${{ steps.prep.outputs.cache_to }}

.gitignore

@@ -18,6 +18,6 @@ DartConfiguration.tcl
 .run/
 target
 Cargo.lock
-
+build/
 .clj-kondo
 .lsp

docker/gstreamer.Dockerfile

@@ -4,8 +4,11 @@ ENV DEBIAN_FRONTEND=noninteractive
 ENV BUILD_ARCHITECTURE=amd64
 ENV DEB_BUILD_OPTIONS=noddebs
 
-ARG GSTREAMER_VERSION=1.22.7
+ARG GSTREAMER_VERSION=1.25.1
 ENV GSTREAMER_VERSION=$GSTREAMER_VERSION
+# Change this to 1.25.1 once released
+ARG GSTREAMER_SHA_COMMIT=671281d860899e9a236f604076831a9ce72186b8
+ENV GSTREAMER_SHA_COMMIT=$GSTREAMER_SHA_COMMIT
 
 ENV SOURCE_PATH=/sources/
 WORKDIR $SOURCE_PATH
@@ -42,8 +45,10 @@ RUN <<_GSTREAMER_INSTALL
     apt-get install -y --no-install-recommends $DEV_PACKAGES
 
     # Build gstreamer
-    git clone -b $GSTREAMER_VERSION --depth=1 https://gitlab.freedesktop.org/gstreamer/gstreamer.git $SOURCE_PATH/gstreamer
+    git clone https://gitlab.freedesktop.org/gstreamer/gstreamer.git $SOURCE_PATH/gstreamer
     cd ${SOURCE_PATH}/gstreamer
+    git checkout $GSTREAMER_SHA_COMMIT
+    git submodule update --recursive --remote
     # see the list of possible options here: https://gitlab.freedesktop.org/gstreamer/gstreamer/-/blob/main/meson_options.txt \
     meson setup \
         --buildtype=release \

docker/wolf.Dockerfile

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=ghcr.io/games-on-whales/gstreamer:1.22.7
+ARG BASE_IMAGE=ghcr.io/games-on-whales/gstreamer:1.24.6
 ########################################################
 FROM $BASE_IMAGE AS wolf-builder
 

docs/modules/dev/pages/manual_build.adoc

@@ -32,7 +32,7 @@ apt install -y build-essential ninja-build gcc meson cmake ccache bison equivs \
 .Build gstreamer
 [source,bash]
 ....
-git clone -b 1.22.7 --depth=1 https://gitlab.freedesktop.org/gstreamer/gstreamer.git
+git clone -b 1.24.6 --depth=1 https://gitlab.freedesktop.org/gstreamer/gstreamer.git
 cd gstreamer
 # Setup a place where we'll put the libraries
 mkdir -p $HOME/gstreamer/include -p $HOME/gstreamer/usr/local/include

docs/modules/user/pages/configuration.adoc

@@ -51,6 +51,10 @@ Wolf is configured via a TOML config file and some additional optional ENV varia
 |/dev/dri/renderD128
 |The default render node used for virtual desktops; see: <<_multiple_gpu>>
 
+|WOLF_ENCODER_NODE
+|$WOLF_RENDER_NODE
+|The default render node used for the Gstreamer pipelines; see: <<_multiple_gpu>>
+
 |WOLF_DOCKER_FAKE_UDEV_PATH
 |$HOST_APPS_STATE_FOLDER/fake-udev
 |The path on the host for the fake-udev CLI tool
@@ -92,6 +96,12 @@ Can be changed in the `config.toml` file
 
 === Share home folder with multiple clients
 
+[WARNING]
+====
+This will break isolation, if you want to connect with multiple clients at the same time you should not share the home folder.
+You can follow development of that feature https://github.com/games-on-whales/wolf/issues/83[here]
+====
+
 By default, Wolf will create a new home folder for each client, but if you want to share the same home folder with multiple clients, you can set the `app_state_folder` to the same value for each paired client; example: +
 
 [source,toml]
@@ -259,43 +269,19 @@ You can read more about gstreamer and custom pipelines in the xref:gstreamer.ado
 When you have multiple GPUs installed in your host, you might want to have better control over which one is used by Wolf and how. +
 There are two main separated parts that make use of HW acceleration in Wolf:
 
-* Gstreamer video encoding: this will use HW acceleration in order to efficiently encode the video stream with H.264 or HEVC.
-* App render node: this will use HW acceleration in order to create virtual Wayland desktops and run the chosen app (ex: Firefox, Steam, ...)
+* *App render node*: this will use HW acceleration in order to create virtual Wayland desktops and run the chosen app (ex: Firefox, Steam, ...).
+Use the `WOLF_RENDER_NODE` (defaults to `/dev/dri/renderD128`) env variable to control this.
+* *Gstreamer video encoding*: this will use HW acceleration in order to efficiently encode the video stream with H.264 or HEVC.
+Use the `WOLF_ENCODER_NODE` (defaults to `WOLF_RENDER_NODE`) env variable to control this.
 
-They can be configured separately, and ideally you could even *use two GPUs at the same time* for different jobs; a common setup would be to use the integrated GPU just for the streaming part and use a powerful GPU to play apps/games.
-
-=== Gstreamer video encoding
-
-The streaming video encoding pipeline is fully controlled by the `config.toml` file; here the order in which entries are listed is important because Wolf will just try each listed plugin; the first one that works is the one that will be used.
+They can be configured separately, and in theory you could even *use two GPUs at the same time* for different jobs; ex: use the integrated GPU just for the streaming part and use a powerful GPU to play apps/games.
 
-[NOTE,caption=EXAMPLE]
+[WARNING]
 ====
-If you have an Intel iGPU and a Nvidia card in the same host, and you would like to use QuickSync in order to do the encoding, you can either:
-
-* Delete the `nvcodec` entries under `gstreamer.video.hevc_encoders`
-* Cut the `qsv` entry and paste it above the `nvcodec` entry
+This isn't recommended, it might introduce additional latency and it's not optimal.
+HW encoding on modern GPUs is very lightweight and it's better to use the same GPU for both jobs.
 ====
 
-On top of that, each single `apps` entry support overriding the default streaming pipeline; for example:
-
-[source,toml]
-....
-[[apps]]
-title = "Test ball"
-
-# More options here, removed for brevity...
-
-[apps.video]
-source = """
-videotestsrc pattern=ball flip=true is-live=true !
-video/x-raw, framerate={fps}/1
-\
-"""
-....
-
-In case you have two GPUs that will use the same encoder pipeline (example: an AMD iGPU and an AMD GPU card) you can override the `encoder_pipeline` with the corresponding encoder plugin; see:
-https://gitlab.freedesktop.org/gstreamer/gstreamer/-/issues/1167[gstreamer/issues/1167].
-
 === App render node
 
 Each application that Wolf will start will have access only to a specific render node even if the host has multiple GPUs connected. +
@@ -331,6 +317,40 @@ image = "ghcr.io/games-on-whales/steam:edge"
 render_node = "/dev/dri/renderD129"
 ....
 
+=== Gstreamer video encoding
+
+The easy way to control this is to just edit the env variable `WOLF_ENCODER_NODE` (defaults to match `WOLF_RENDER_NODE` in order to use the same GPU for both), this will set the default render node used for the Gstreamer pipelines.
+
+The streaming video encoding pipeline is fully controlled by the `config.toml` file; here the order in which entries are listed is important because Wolf will just try each listed plugin; the first one that works is the one that will be used.
+
+[NOTE,caption=EXAMPLE]
+====
+If you have an Intel iGPU and a Nvidia card in the same host, and you would like to use QuickSync in order to do the encoding, you can either:
+
+* Delete the `nvcodec` entries under `gstreamer.video.hevc_encoders`
+* Cut the `qsv` entry and paste it above the `nvcodec` entry
+====
+
+On top of that, each single `apps` entry support overriding the default streaming pipeline; for example:
+
+[source,toml]
+....
+[[apps]]
+title = "Test ball"
+
+# More options here, removed for brevity...
+
+[apps.video]
+source = """
+videotestsrc pattern=ball flip=true is-live=true !
+video/x-raw, framerate={fps}/1
+\
+"""
+....
+
+In case you have two GPUs that will use the same encoder pipeline (example: an AMD iGPU and an AMD GPU card) you can override the `video_params` with the corresponding encoder plugin; see:
+https://gitlab.freedesktop.org/gstreamer/gstreamer/-/issues/1167[gstreamer/issues/1167].
+
 == Directly launch a Steam game
 
 In order to directly launch a Steam game from Moonlight you can just copy the existing `[[apps]]` entry for Steam, change the name and just add the Steam app ID as env variable; example:

src/core/CMakeLists.txt

@@ -56,7 +56,7 @@ if (UNIX AND NOT APPLE)
         FetchContent_Declare(
                 inputtino
                 GIT_REPOSITORY https://github.com/games-on-whales/inputtino.git
-                GIT_TAG 56bc064)
+                GIT_TAG 5d4b8b2)
         FetchContent_MakeAvailable(inputtino)
     endif ()
 

src/core/src/core/audio.hpp

@@ -8,17 +8,41 @@
 #include <functional>
 #include <memory>
 #include <string>
+#include <vector>
 
 namespace wolf::core::audio {
 
 typedef struct Server Server;
 
 std::shared_ptr<Server> connect(std::string_view server = {});
 
+constexpr auto SAMPLE_RATE = 48000;
+
+struct AudioMode {
+
+  enum Speakers {
+    FRONT_LEFT,
+    FRONT_RIGHT,
+    FRONT_CENTER,
+    LOW_FREQUENCY,
+    BACK_LEFT,
+    BACK_RIGHT,
+    SIDE_LEFT,
+    SIDE_RIGHT,
+    MAX_SPEAKERS,
+  };
+
+  int channels{};
+  int streams{};
+  int coupled_streams{};
+  std::vector<Speakers> speakers;
+  int bitrate{};
+  int sample_rate = SAMPLE_RATE;
+};
+
 struct AudioDevice {
   std::string_view sink_name;
-  int n_channels;
-  int bitrate = 48000;
+  AudioMode mode;
 };
 
 struct VSink {