Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement zero copy pipeline #156

Draft
wants to merge 2 commits into
base: stable
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/linux-build-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,7 @@ jobs:
-DCATCH_DEVELOPMENT_BUILD=ON \
-DTEST_VIRTUAL_INPUT=OFF \
-DTEST_DOCKER=OFF \
-DLINK_RUST_WAYLAND=ON \
-DLINK_RUST_WAYLAND=OFF \
-DTEST_RUST_WAYLAND=OFF \
-DTEST_NVIDIA=OFF \
-DTEST_SDL=OFF \
Expand Down
4 changes: 2 additions & 2 deletions docker/gstreamer.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ Depends: libc6, libcap2, libcap2-bin, libdw1, libglib2.0-0, libunwind8,
zlib1g, libdrm2, libva2, libmfx1, libpulse0, libxdamage1, libx265-199, libopus0,
libegl1, libgl1, libgles2, libudev1, libva-drm2, libva-wayland2, libva-x11-2, libva2,
libwayland-client0, libx11-6, libxrandr2, libvpl2, libzxing3, libopenexr-3-1-30, librsvg2-2, libwebp7,
libcairo2, libcairo-gobject2, libjpeg8, libopenjp2-7, liblcms2-2, libzbar0, libaom3
libcairo2, libcairo-gobject2, libjpeg8, libopenjp2-7, liblcms2-2, libzbar0, libaom3, libsoup-2.4-1
Provides: gstreamer, libgstreamer1.0-0
Description: Manually built from git
EOT
Expand All @@ -39,7 +39,7 @@ RUN <<_GSTREAMER_INSTALL
libmfx-dev libvpl-dev libmfx-tools libunwind8 libcap2-bin \
libx11-dev libxfixes-dev libxdamage-dev libwayland-dev libpulse-dev libglib2.0-dev \
libopenjp2-7-dev liblcms2-dev libcairo2-dev libcairo-gobject2 libwebp7 librsvg2-dev libaom-dev \
libharfbuzz-dev libpango1.0-dev
libharfbuzz-dev libpango1.0-dev libsoup-2.4-1
"
apt-get update -y
apt-get install -y --no-install-recommends $DEV_PACKAGES
Expand Down
4 changes: 2 additions & 2 deletions docker/wolf.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,9 @@ RUN <<_GST_WAYLAND_DISPLAY

git clone https://github.com/games-on-whales/gst-wayland-display
cd gst-wayland-display
git checkout a31f5a0
git checkout ae5cf24
cargo install cargo-c
cargo cinstall -p c-bindings --prefix=/usr/local --libdir=/usr/local/lib/
cargo cinstall -p gst-plugin-wayland-display --prefix=/usr/local/lib/x86_64-linux-gnu/ --libdir=/usr/local/lib/x86_64-linux-gnu/gstreamer-1.0
_GST_WAYLAND_DISPLAY

COPY . /wolf/
Expand Down
11 changes: 6 additions & 5 deletions src/core/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ file(GLOB CORE_HEADERS CONFIGURE_DEPENDS src/core/*.hpp)
set(CORE_SRC "")

find_package(PkgConfig)
option(LINK_RUST_WAYLAND "Link to our custom Rust wayland compositor" ON)
option(LINK_RUST_WAYLAND "Link to our custom Rust wayland compositor" OFF)
if (LINK_RUST_WAYLAND AND UNIX AND NOT APPLE)
pkg_check_modules(libgstwaylanddisplay REQUIRED IMPORTED_TARGET libgstwaylanddisplay>=0.3.0)
target_link_libraries(wolf_core PUBLIC PkgConfig::libgstwaylanddisplay)
Expand Down Expand Up @@ -50,9 +50,9 @@ if (UNIX AND NOT APPLE)
target_link_libraries(wolf_core PUBLIC wolf::audio)

option(WOLF_CUSTOM_INPUTTINO_SRC "Use custom inputtino source" OFF)
if(WOLF_CUSTOM_INPUTTINO_SRC)
if (WOLF_CUSTOM_INPUTTINO_SRC)
add_subdirectory(${WOLF_CUSTOM_INPUTTINO_SRC} ${CMAKE_CURRENT_BINARY_DIR}/inputtino EXCLUDE_FROM_ALL)
else()
else ()
FetchContent_Declare(
inputtino
GIT_REPOSITORY https://github.com/games-on-whales/inputtino.git
Expand All @@ -64,10 +64,11 @@ if (UNIX AND NOT APPLE)
target_link_libraries(wolf_core PUBLIC wolf::uinput)

if (LINK_RUST_WAYLAND)
message(STATUS "Using Rust C bindings for Wayland")
list(APPEND CORE_SRC src/platforms/linux/virtual-display/wayland-display.cpp)
else ()
message(WARNING "Missing virtual display implementation for this platform")
list(APPEND CORE_SRC platforms/unknown/no-virtual-display.cpp)
message(STATUS "Using Gstreamer bindings for Wayland")
list(APPEND CORE_SRC src/platforms/linux/virtual-display/gst-wayland-display.cpp)
endif ()
else ()
file(GLOB CORE_SRC SRCS platforms/unknown/*.cpp)
Expand Down
57 changes: 0 additions & 57 deletions src/core/src/core/gstreamer.hpp
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
#pragma once

#include <eventbus/event_bus.hpp>
#include <gst/gst.h>
#include <helpers/logger.hpp>
#include <immer/array.hpp>
#include <immer/box.hpp>

namespace wolf::core::gstreamer {

Expand All @@ -31,60 +28,6 @@ static void pipeline_eos_handler(GstBus *bus, GstMessage *message, gpointer data
g_main_loop_quit(loop);
}

static bool run_pipeline(const std::string &pipeline_desc,
const std::function<immer::array<immer::box<events::EventBusHandlers>>(
gst_element_ptr /* pipeline */, gst_main_loop_ptr /* main_loop */)> &on_pipeline_ready) {
GError *error = nullptr;
gst_element_ptr pipeline(gst_parse_launch(pipeline_desc.c_str(), &error), [](const auto &pipeline) {
logs::log(logs::trace, "~pipeline");
gst_object_unref(pipeline);
});

if (!pipeline) {
logs::log(logs::error, "[GSTREAMER] Pipeline parse error: {}", error->message);
g_error_free(error);
return false;
} else if (error) { // Please note that you might get a return value that is not NULL even though the error is set. In
// this case there was a recoverable parsing error and you can try to play the pipeline.
logs::log(logs::warning, "[GSTREAMER] Pipeline parse error (recovered): {}", error->message);
g_error_free(error);
}

gst_main_context_ptr context = {g_main_context_new(), ::g_main_context_unref};
g_main_context_push_thread_default(context.get());
gst_main_loop_ptr loop(g_main_loop_new(context.get(), FALSE), ::g_main_loop_unref);

/*
* adds a watch for new message on our pipeline's message bus to
* the default GLib main context, which is the main context that our
* GLib main loop is attached to below
*/
auto bus = gst_pipeline_get_bus(GST_PIPELINE(pipeline.get()));
gst_bus_add_signal_watch(bus);
g_signal_connect(bus, "message::error", G_CALLBACK(pipeline_error_handler), loop.get());
g_signal_connect(bus, "message::eos", G_CALLBACK(pipeline_eos_handler), loop.get());
gst_object_unref(bus);

/* Set the pipeline to "playing" state*/
gst_element_set_state(pipeline.get(), GST_STATE_PLAYING);
GST_DEBUG_BIN_TO_DOT_FILE_WITH_TS(reinterpret_cast<GstBin *>(pipeline.get()),
GST_DEBUG_GRAPH_SHOW_ALL,
"pipeline-start");

/* Let the calling thread set extra things */
auto handlers = on_pipeline_ready(pipeline, loop);

/* The main loop will be run until someone calls g_main_loop_quit() */
g_main_loop_run(loop.get());

/* Out of the main loop, clean up nicely */
gst_element_set_state(pipeline.get(), GST_STATE_PAUSED);
gst_element_set_state(pipeline.get(), GST_STATE_READY);
gst_element_set_state(pipeline.get(), GST_STATE_NULL);

return true;
}

/**
* Sends a custom message in the pipeline
*/
Expand Down
14 changes: 3 additions & 11 deletions src/core/src/core/virtual-display.hpp
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#pragma once
#include <core/gstreamer.hpp>
#include <gst/gst.h>
#include <immer/array.hpp>
#include <immer/vector.hpp>
Expand All @@ -16,20 +17,11 @@ struct DisplayMode {
typedef struct WaylandState WaylandState;

using wl_state_ptr = std::shared_ptr<WaylandState>;
using gst_element_ptr = std::shared_ptr<GstElement>;

wl_state_ptr create_wayland_display(const immer::array<std::string> &input_devices,
const std::string &render_node = "/dev/dri/renderD128");
wl_state_ptr create_wayland_display(gstreamer::gst_element_ptr wayland_plugin, const std::string &wayland_socket_name);

std::unique_ptr<GstCaps, decltype(&gst_caps_unref)> set_resolution(WaylandState &w_state,
const DisplayMode &display_mode,
const std::optional<gst_element_ptr> &app_src = {});
std::string get_wayland_socket_name(WaylandState &w_state);

immer::vector<std::string> get_devices(const WaylandState &w_state);
immer::vector<std::string> get_env(const WaylandState &w_state);

static void destroy(WaylandState *w_state);
GstBuffer *get_frame(WaylandState &w_state);
bool add_input_device(WaylandState &w_state, const std::string &device_path);

class WaylandMouse {
Expand Down
196 changes: 196 additions & 0 deletions src/core/src/platforms/linux/virtual-display/gst-wayland-display.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,196 @@
/**
* Instead of directly linking to the Rust C bindings, we can interact with the Gstreamer plugin
*/

#include <core/virtual-display.hpp>
#include <linux/input-event-codes.h>

namespace wolf::core::virtual_display {
struct WaylandState {
/**
* The wayland plugin element,
* we need a reference so that we can send events directly to it (mouse, keyboard, ...)
*/
gstreamer::gst_element_ptr wayland_plugin;

std::string wayland_socket_name;
};

wl_state_ptr create_wayland_display(gstreamer::gst_element_ptr wayland_plugin, const std::string &wayland_socket_name) {
return std::make_shared<WaylandState>(
WaylandState{.wayland_plugin = wayland_plugin, .wayland_socket_name = wayland_socket_name});
}

std::string get_wayland_socket_name(WaylandState &w_state) {
return w_state.wayland_socket_name;
}

bool add_input_device(WaylandState &w_state, const std::string &device_path) {
auto g_devices = g_array_new(FALSE, FALSE, sizeof(GValue));
auto g_device = g_strdup(device_path.c_str());
g_array_insert_val(g_devices, 0, g_device);
gstreamer::send_message(w_state.wayland_plugin.get(),
gst_structure_new("VirtualDevicesReady", "paths", G_TYPE_ARRAY, g_devices, NULL));
g_array_unref(g_devices);
return true;
}

void WaylandMouse::move(int delta_x, int delta_y) {
auto msg = /* clang-format off */
gst_structure_new("MouseMoveRelative",
"pointer_x", G_TYPE_DOUBLE, static_cast<double>(delta_x),
"pointer_y", G_TYPE_DOUBLE, static_cast<double>(delta_y),
NULL);
/* clang-format on */
gstreamer::send_message(w_state->wayland_plugin.get(), msg);
}

void WaylandMouse::move_abs(int x, int y, int screen_width, int screen_height) {
auto msg = /* clang-format off */
gst_structure_new("MouseMoveAbsolute",
"pointer_x", G_TYPE_DOUBLE, static_cast<double>(x),
"pointer_y", G_TYPE_DOUBLE, static_cast<double>(y),
NULL);
/* clang-format on */
gstreamer::send_message(w_state->wayland_plugin.get(), msg);
}

unsigned int moonlight_button_to_linux(unsigned int button) {
switch (button) {
case 1:
return BTN_LEFT;
case 2:
return BTN_MIDDLE;
case 3:
return BTN_RIGHT;
case 4:
return BTN_SIDE;
default:
return BTN_EXTRA;
}
}

void WaylandMouse::press(unsigned int button) {
auto msg = /* clang-format off */
gst_structure_new("MouseButton",
"button", G_TYPE_UINT, moonlight_button_to_linux(button),
"pressed", G_TYPE_BOOLEAN, true,
NULL);
/* clang-format on */
gstreamer::send_message(w_state->wayland_plugin.get(), msg);
}

void WaylandMouse::release(unsigned int button) {
auto msg = /* clang-format off */
gst_structure_new("MouseButton",
"button", G_TYPE_UINT, moonlight_button_to_linux(button),
"pressed", G_TYPE_BOOLEAN, false,
NULL);
/* clang-format on */
gstreamer::send_message(w_state->wayland_plugin.get(), msg);
}
void WaylandMouse::vertical_scroll(int high_res_distance) {
auto msg = /* clang-format off */
gst_structure_new("MouseAxis",
"x", G_TYPE_DOUBLE, static_cast<double>(0),
"y", G_TYPE_DOUBLE, static_cast<double>(-high_res_distance),
NULL);
/* clang-format on */
gstreamer::send_message(w_state->wayland_plugin.get(), msg);
}

void WaylandMouse::horizontal_scroll(int high_res_distance) {
auto msg = /* clang-format off */
gst_structure_new("MouseAxis",
"x", G_TYPE_DOUBLE, static_cast<double>(high_res_distance),
"y", G_TYPE_DOUBLE, static_cast<double>(0),
NULL);
/* clang-format on */
gstreamer::send_message(w_state->wayland_plugin.get(), msg);
}

/**
* A map of Moonlight key codes to Linux key codes
*/
static const std::map<unsigned int, unsigned int> key_mappings = {
{0x08, KEY_BACKSPACE}, {0x09, KEY_TAB},
{0x0C, KEY_CLEAR}, {0x0D, KEY_ENTER},
{0x10, KEY_LEFTSHIFT}, {0x11, KEY_LEFTCTRL},
{0x12, KEY_LEFTALT}, {0x13, KEY_PAUSE},
{0x14, KEY_CAPSLOCK}, {0x15, KEY_KATAKANAHIRAGANA},
{0x16, KEY_HANGEUL}, {0x17, KEY_HANJA},
{0x19, KEY_KATAKANA}, {0x1B, KEY_ESC},
{0x20, KEY_SPACE}, {0x21, KEY_PAGEUP},
{0x22, KEY_PAGEDOWN}, {0x23, KEY_END},
{0x24, KEY_HOME}, {0x25, KEY_LEFT},
{0x26, KEY_UP}, {0x27, KEY_RIGHT},
{0x28, KEY_DOWN}, {0x29, KEY_SELECT},
{0x2A, KEY_PRINT}, {0x2C, KEY_SYSRQ},
{0x2D, KEY_INSERT}, {0x2E, KEY_DELETE},
{0x2F, KEY_HELP}, {0x30, KEY_0},
{0x31, KEY_1}, {0x32, KEY_2},
{0x33, KEY_3}, {0x34, KEY_4},
{0x35, KEY_5}, {0x36, KEY_6},
{0x37, KEY_7}, {0x38, KEY_8},
{0x39, KEY_9}, {0x41, KEY_A},
{0x42, KEY_B}, {0x43, KEY_C},
{0x44, KEY_D}, {0x45, KEY_E},
{0x46, KEY_F}, {0x47, KEY_G},
{0x48, KEY_H}, {0x49, KEY_I},
{0x4A, KEY_J}, {0x4B, KEY_K},
{0x4C, KEY_L}, {0x4D, KEY_M},
{0x4E, KEY_N}, {0x4F, KEY_O},
{0x50, KEY_P}, {0x51, KEY_Q},
{0x52, KEY_R}, {0x53, KEY_S},
{0x54, KEY_T}, {0x55, KEY_U},
{0x56, KEY_V}, {0x57, KEY_W},
{0x58, KEY_X}, {0x59, KEY_Y},
{0x5A, KEY_Z}, {0x5B, KEY_LEFTMETA},
{0x5C, KEY_RIGHTMETA}, {0x5F, KEY_SLEEP},
{0x60, KEY_KP0}, {0x61, KEY_KP1},
{0x62, KEY_KP2}, {0x63, KEY_KP3},
{0x64, KEY_KP4}, {0x65, KEY_KP5},
{0x66, KEY_KP6}, {0x67, KEY_KP7},
{0x68, KEY_KP8}, {0x69, KEY_KP9},
{0x6A, KEY_KPASTERISK}, {0x6B, KEY_KPPLUS},
{0x6C, KEY_KPCOMMA}, {0x6D, KEY_KPMINUS},
{0x6E, KEY_KPDOT}, {0x6F, KEY_KPSLASH},
{0x70, KEY_F1}, {0x71, KEY_F2},
{0x72, KEY_F3}, {0x73, KEY_F4},
{0x74, KEY_F5}, {0x75, KEY_F6},
{0x76, KEY_F7}, {0x77, KEY_F8},
{0x78, KEY_F9}, {0x79, KEY_F10},
{0x7A, KEY_F11}, {0x7B, KEY_F12},
{0x90, KEY_NUMLOCK}, {0x91, KEY_SCROLLLOCK},
{0xA0, KEY_LEFTSHIFT}, {0xA1, KEY_RIGHTSHIFT},
{0xA2, KEY_LEFTCTRL}, {0xA3, KEY_RIGHTCTRL},
{0xA4, KEY_LEFTALT}, {0xA5, KEY_RIGHTALT},
{0xBA, KEY_SEMICOLON}, {0xBB, KEY_EQUAL},
{0xBC, KEY_COMMA}, {0xBD, KEY_MINUS},
{0xBE, KEY_DOT}, {0xBF, KEY_SLASH},
{0xC0, KEY_GRAVE}, {0xDB, KEY_LEFTBRACE},
{0xDC, KEY_BACKSLASH}, {0xDD, KEY_RIGHTBRACE},
{0xDE, KEY_APOSTROPHE}, {0xE2, KEY_102ND},
};

void WaylandKeyboard::press(unsigned int key_code) {
auto msg = /* clang-format off */
gst_structure_new("KeyboardKey",
"key", G_TYPE_UINT, key_mappings.at(key_code),
"pressed", G_TYPE_BOOLEAN, true,
NULL);
/* clang-format on */
gstreamer::send_message(w_state->wayland_plugin.get(), msg);
}

void WaylandKeyboard::release(unsigned int key_code) {
auto msg = /* clang-format off */
gst_structure_new("KeyboardKey",
"key", G_TYPE_UINT, key_mappings.at(key_code),
"pressed", G_TYPE_BOOLEAN, false,
NULL);
/* clang-format on */
gstreamer::send_message(w_state->wayland_plugin.get(), msg);
}

} // namespace wolf::core::virtual_display
17 changes: 0 additions & 17 deletions src/moonlight-server/runners/child_session.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -27,23 +27,6 @@ void RunChildSession::run(std::size_t session_id,
/* inherit the wayland connection, needed in order to advertise some devices (ex: PS5 trackpad) */
if (auto wl = *parent_session->wayland_display->load()) {
child_session->wayland_display = parent_session->wayland_display;

/* Add mouse and keyboards to our wayland display */
if (child_session->mouse->has_value()) {
if (auto mouse = std::get_if<input::Mouse>(&child_session->mouse->value())) {
for (auto path : mouse->get_nodes()) {
add_input_device(*wl, path);
}
}
}

if (child_session->keyboard->has_value()) {
if (auto kb = std::get_if<input::Keyboard>(&child_session->keyboard->value())) {
for (auto path : kb->get_nodes()) {
add_input_device(*wl, path);
}
}
}
}

/* Keep a history of plugged devices so that we can clean up when over */
Expand Down
Loading
Loading