diff --git a/.cargo/config.toml b/.cargo/config.toml index a657ae61b9cf8..043adf6b30e2c 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -13,6 +13,12 @@ rustflags = ["-C", "link-arg=-fuse-ld=mold"] linker = "clang" rustflags = ["-C", "link-arg=-fuse-ld=mold"] +[target.aarch64-apple-darwin] +rustflags = ["-C", "link-args=-Objc -all_load"] + +[target.x86_64-apple-darwin] +rustflags = ["-C", "link-args=-Objc -all_load"] + # This cfg will reduce the size of `windows::core::Error` from 16 bytes to 4 bytes [target.'cfg(target_os = "windows")'] rustflags = ["--cfg", "windows_slim_errors"] diff --git a/Cargo.lock b/Cargo.lock index d21006ee55f59..83231ea906b32 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -954,6 +954,22 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "async-tungstenite" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cca750b12e02c389c1694d35c16539f88b8bbaa5945934fdc1b41a776688589" +dependencies = [ + "async-native-tls", + "async-std", + "async-tls", + "futures-io", + "futures-util", + "log", + "pin-project-lite", + "tungstenite 0.21.0", +] + [[package]] name = "async-tungstenite" version = "0.28.0" @@ -1822,7 +1838,7 @@ dependencies = [ "arrayvec", "cc", "cfg-if", - "constant_time_eq", + "constant_time_eq 0.3.1", ] [[package]] @@ -2007,6 +2023,27 @@ dependencies = [ "either", ] +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "call" version = "0.1.0" @@ -2015,12 +2052,13 @@ dependencies = [ "audio", "client", "collections", + "feature_flags", "fs", "futures 0.3.31", "gpui", "http_client", "language", - "live_kit_client", + "livekit_client", "log", "postage", "project", @@ -2478,7 +2516,7 @@ dependencies = [ "anyhow", "async-native-tls", "async-recursion 0.3.2", - "async-tungstenite", + "async-tungstenite 0.28.0", "chrono", "clock", "cocoa 0.26.0", @@ -2610,7 +2648,7 @@ dependencies = [ "assistant_tool", "async-stripe", "async-trait", - "async-tungstenite", + "async-tungstenite 0.28.0", "audio", "aws-config", "aws-sdk-kinesis", @@ -2648,8 +2686,8 @@ dependencies = [ "jsonwebtoken", "language", "language_model", - "live_kit_client", - "live_kit_server", + "livekit_client", + "livekit_server", "log", "lsp", "menu", @@ -2662,7 +2700,7 @@ dependencies = [ "pretty_assertions", "project", "prometheus", - "prost", + "prost 0.9.0", "rand 0.8.5", "recent_projects", "release_channel", @@ -2862,6 +2900,12 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + [[package]] name = "constant_time_eq" version = "0.3.1" @@ -3069,6 +3113,17 @@ dependencies = [ "coreaudio-sys", ] +[[package]] +name = "coreaudio-rs" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34ca07354f6d0640333ef95f48d460a4bcf34812a7e7967f9b44c728a8f37c28" +dependencies = [ + "bitflags 1.3.2", + "core-foundation-sys", + "coreaudio-sys", +] + [[package]] name = "coreaudio-sys" version = "0.2.16" @@ -3103,12 +3158,11 @@ dependencies = [ [[package]] name = "cpal" version = "0.15.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "873dab07c8f743075e57f524c583985fbaf745602acbe916a01539364369a779" +source = "git+https://github.com/zed-industries/cpal?rev=fd8bc2fd39f1f5fdee5a0690656caff9a26d9d50#fd8bc2fd39f1f5fdee5a0690656caff9a26d9d50" dependencies = [ "alsa", "core-foundation-sys", - "coreaudio-rs", + "coreaudio-rs 0.11.3", "dasp_sample", "jni", "js-sys", @@ -3440,6 +3494,65 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96a6ac251f4a2aca6b3f91340350eab87ae57c3f127ffeb585e92bd336717991" +[[package]] +name = "cxx" +version = "1.0.133" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05e1ec88093d2abd9cf1b09ffd979136b8e922bf31cad966a8fe0d73233112ef" +dependencies = [ + "cc", + "cxxbridge-cmd", + "cxxbridge-flags", + "cxxbridge-macro", + "foldhash", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.133" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afa390d956ee7ccb41aeed7ed7856ab3ffb4fc587e7216be7e0f83e949b4e6c" +dependencies = [ + "cc", + "codespan-reporting", + "proc-macro2", + "quote", + "scratch", + "syn 2.0.87", +] + +[[package]] +name = "cxxbridge-cmd" +version = "1.0.133" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c23bfff654d6227cbc83de8e059d2f8678ede5fc3a6c5a35d5c379983cc61e6" +dependencies = [ + "clap", + "codespan-reporting", + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.133" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c01b36e22051bc6928a78583f1621abaaf7621561c2ada1b00f7878fbe2caa" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.133" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e14013136fac689345d17b9a6df55977251f11d333c0a571e8d963b55e1f95" +dependencies = [ + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.87", +] + [[package]] name = "dashmap" version = "5.5.3" @@ -4696,6 +4809,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "fsevent" version = "0.1.0" @@ -6329,6 +6452,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.12.1" @@ -6454,7 +6586,7 @@ checksum = "58d9afa5bc6eeafb78f710a2efc585f69099f8b6a99dc7eb826581e3773a6e31" dependencies = [ "anyhow", "async-trait", - "async-tungstenite", + "async-tungstenite 0.28.0", "futures 0.3.31", "jupyter-protocol", "serde", @@ -6873,6 +7005,29 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libwebrtc" +version = "0.3.7" +source = "git+https://github.com/zed-industries/rust-sdks?rev=799f10133d93ba2a88642cd480d01ec4da53408c#799f10133d93ba2a88642cd480d01ec4da53408c" +dependencies = [ + "cxx", + "jni", + "js-sys", + "lazy_static", + "livekit-protocol", + "livekit-runtime", + "log", + "parking_lot", + "serde", + "serde_json", + "thiserror 1.0.69", + "tokio", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webrtc-sys", +] + [[package]] name = "libz-sys" version = "1.1.20" @@ -6885,6 +7040,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "link-cplusplus" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d240c6f7e1ba3a28b0249f774e6a9dd0175054b52dfbb61b16eb8505c3785c9" +dependencies = [ + "cc", +] + [[package]] name = "linkify" version = "0.10.0" @@ -6933,17 +7097,97 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" [[package]] -name = "live_kit_client" +name = "livekit" +version = "0.7.0" +source = "git+https://github.com/zed-industries/rust-sdks?rev=799f10133d93ba2a88642cd480d01ec4da53408c#799f10133d93ba2a88642cd480d01ec4da53408c" +dependencies = [ + "chrono", + "futures-util", + "lazy_static", + "libwebrtc", + "livekit-api", + "livekit-protocol", + "livekit-runtime", + "log", + "parking_lot", + "prost 0.12.6", + "semver", + "serde", + "serde_json", + "thiserror 1.0.69", + "tokio", +] + +[[package]] +name = "livekit-api" +version = "0.4.1" +source = "git+https://github.com/zed-industries/rust-sdks?rev=799f10133d93ba2a88642cd480d01ec4da53408c#799f10133d93ba2a88642cd480d01ec4da53408c" +dependencies = [ + "async-tungstenite 0.25.1", + "futures-util", + "http 0.2.12", + "jsonwebtoken", + "livekit-protocol", + "livekit-runtime", + "log", + "parking_lot", + "prost 0.12.6", + "reqwest 0.11.27", + "scopeguard", + "serde", + "serde_json", + "sha2", + "thiserror 1.0.69", + "tokio", + "tokio-tungstenite 0.20.1", + "url", +] + +[[package]] +name = "livekit-protocol" +version = "0.3.6" +source = "git+https://github.com/zed-industries/rust-sdks?rev=799f10133d93ba2a88642cd480d01ec4da53408c#799f10133d93ba2a88642cd480d01ec4da53408c" +dependencies = [ + "futures-util", + "livekit-runtime", + "parking_lot", + "pbjson", + "pbjson-types", + "prost 0.12.6", + "prost-types 0.12.6", + "serde", + "thiserror 1.0.69", + "tokio", +] + +[[package]] +name = "livekit-runtime" +version = "0.3.1" +source = "git+https://github.com/zed-industries/rust-sdks?rev=799f10133d93ba2a88642cd480d01ec4da53408c#799f10133d93ba2a88642cd480d01ec4da53408c" +dependencies = [ + "async-io 2.4.0", + "async-std", + "async-task", + "futures 0.3.31", +] + +[[package]] +name = "livekit_client" version = "0.1.0" dependencies = [ "anyhow", - "async-broadcast", "async-trait", "collections", "core-foundation 0.9.4", + "coreaudio-rs 0.12.1", + "cpal", "futures 0.3.31", "gpui", - "live_kit_server", + "http 0.2.12", + "http_client", + "image", + "livekit", + "livekit_server", "log", "media", "nanoid", @@ -6953,19 +7197,21 @@ dependencies = [ "serde_json", "sha2", "simplelog", + "smallvec", + "util", ] [[package]] -name = "live_kit_server" +name = "livekit_server" version = "0.1.0" dependencies = [ "anyhow", "async-trait", "jsonwebtoken", "log", - "prost", - "prost-build", - "prost-types", + "prost 0.9.0", + "prost-build 0.9.0", + "prost-types 0.9.0", "reqwest 0.12.8", "serde", ] @@ -7254,6 +7500,7 @@ dependencies = [ "anyhow", "bindgen", "core-foundation 0.9.4", + "ctor", "foreign-types 0.5.0", "metal", "objc", @@ -7946,7 +8193,7 @@ dependencies = [ "md-5", "num", "num-bigint-dig", - "pbkdf2", + "pbkdf2 0.12.2", "rand 0.8.5", "serde", "sha2", @@ -8266,6 +8513,17 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "password-hash" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" +dependencies = [ + "base64ct", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "password-hash" version = "0.5.0" @@ -8316,6 +8574,55 @@ dependencies = [ "util", ] +[[package]] +name = "pbjson" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1030c719b0ec2a2d25a5df729d6cff1acf3cc230bf766f4f97833591f7577b90" +dependencies = [ + "base64 0.21.7", + "serde", +] + +[[package]] +name = "pbjson-build" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2580e33f2292d34be285c5bc3dba5259542b083cfad6037b6d70345f24dcb735" +dependencies = [ + "heck 0.4.1", + "itertools 0.11.0", + "prost 0.12.6", + "prost-types 0.12.6", +] + +[[package]] +name = "pbjson-types" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18f596653ba4ac51bdecbb4ef6773bc7f56042dc13927910de1684ad3d32aa12" +dependencies = [ + "bytes 1.8.0", + "chrono", + "pbjson", + "pbjson-build", + "prost 0.12.6", + "prost-build 0.12.6", + "serde", +] + +[[package]] +name = "pbkdf2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +dependencies = [ + "digest", + "hmac", + "password-hash 0.4.2", + "sha2", +] + [[package]] name = "pbkdf2" version = "0.12.2" @@ -9345,7 +9652,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001" dependencies = [ "bytes 1.8.0", - "prost-derive", + "prost-derive 0.9.0", +] + +[[package]] +name = "prost" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" +dependencies = [ + "bytes 1.8.0", + "prost-derive 0.12.6", ] [[package]] @@ -9361,13 +9678,34 @@ dependencies = [ "log", "multimap", "petgraph", - "prost", - "prost-types", + "prost 0.9.0", + "prost-types 0.9.0", "regex", "tempfile", "which 4.4.2", ] +[[package]] +name = "prost-build" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" +dependencies = [ + "bytes 1.8.0", + "heck 0.5.0", + "itertools 0.12.1", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost 0.12.6", + "prost-types 0.12.6", + "regex", + "syn 2.0.87", + "tempfile", +] + [[package]] name = "prost-derive" version = "0.9.0" @@ -9381,6 +9719,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "prost-derive" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" +dependencies = [ + "anyhow", + "itertools 0.12.1", + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "prost-types" version = "0.9.0" @@ -9388,7 +9739,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a" dependencies = [ "bytes 1.8.0", - "prost", + "prost 0.9.0", +] + +[[package]] +name = "prost-types" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" +dependencies = [ + "prost 0.12.6", ] [[package]] @@ -9397,8 +9757,8 @@ version = "0.1.0" dependencies = [ "anyhow", "collections", - "prost", - "prost-build", + "prost 0.9.0", + "prost-build 0.9.0", "serde", ] @@ -9898,7 +10258,7 @@ dependencies = [ "log", "parking_lot", "paths", - "prost", + "prost 0.9.0", "release_channel", "rpc", "serde", @@ -10033,6 +10393,7 @@ dependencies = [ "http 0.2.12", "http-body 0.4.6", "hyper 0.14.31", + "hyper-rustls 0.24.2", "hyper-tls", "ipnet", "js-sys", @@ -10042,6 +10403,8 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", + "rustls 0.21.12", + "rustls-native-certs 0.6.3", "rustls-pemfile 1.0.4", "serde", "serde_json", @@ -10050,6 +10413,7 @@ dependencies = [ "system-configuration 0.5.1", "tokio", "tokio-native-tls", + "tokio-rustls 0.24.1", "tower-service", "url", "wasm-bindgen", @@ -10273,7 +10637,7 @@ name = "rpc" version = "0.1.0" dependencies = [ "anyhow", - "async-tungstenite", + "async-tungstenite 0.28.0", "base64 0.22.1", "chrono", "collections", @@ -10649,14 +11013,20 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scratch" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3cf7c11c38cb994f3d40e8a8cde3bbd1f72a435e4c49e85d6553d8312306152" + [[package]] name = "scrypt" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" dependencies = [ - "password-hash", - "pbkdf2", + "password-hash 0.5.0", + "pbkdf2 0.12.2", "salsa20", "sha2", ] @@ -12815,7 +13185,10 @@ checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" dependencies = [ "futures-util", "log", + "rustls 0.21.12", + "rustls-native-certs 0.6.3", "tokio", + "tokio-rustls 0.24.1", "tungstenite 0.20.1", ] @@ -13323,6 +13696,7 @@ dependencies = [ "httparse", "log", "rand 0.8.5", + "rustls 0.21.12", "sha1", "thiserror 1.0.69", "url", @@ -13341,6 +13715,7 @@ dependencies = [ "http 1.1.0", "httparse", "log", + "native-tls", "rand 0.8.5", "sha1", "thiserror 1.0.69", @@ -14450,6 +14825,32 @@ version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" +[[package]] +name = "webrtc-sys" +version = "0.3.5" +source = "git+https://github.com/zed-industries/rust-sdks?rev=799f10133d93ba2a88642cd480d01ec4da53408c#799f10133d93ba2a88642cd480d01ec4da53408c" +dependencies = [ + "cc", + "cxx", + "cxx-build", + "glob", + "log", + "webrtc-sys-build", +] + +[[package]] +name = "webrtc-sys-build" +version = "0.3.5" +source = "git+https://github.com/zed-industries/rust-sdks?rev=799f10133d93ba2a88642cd480d01ec4da53408c#799f10133d93ba2a88642cd480d01ec4da53408c" +dependencies = [ + "fs2", + "regex", + "reqwest 0.11.27", + "scratch", + "semver", + "zip", +] + [[package]] name = "weezl" version = "0.1.8" @@ -16016,6 +16417,26 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "zip" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +dependencies = [ + "aes", + "byteorder", + "bzip2", + "constant_time_eq 0.1.5", + "crc32fast", + "crossbeam-utils", + "flate2", + "hmac", + "pbkdf2 0.11.0", + "sha1", + "time", + "zstd", +] + [[package]] name = "zstd" version = "0.11.2+zstd.1.5.2" diff --git a/Cargo.toml b/Cargo.toml index 0465545990616..178187deea9e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,8 +65,8 @@ members = [ "crates/language_selector", "crates/language_tools", "crates/languages", - "crates/live_kit_client", - "crates/live_kit_server", + "crates/livekit_client", + "crates/livekit_server", "crates/lsp", "crates/markdown", "crates/markdown_preview", @@ -248,8 +248,8 @@ language_models = { path = "crates/language_models" } language_selector = { path = "crates/language_selector" } language_tools = { path = "crates/language_tools" } languages = { path = "crates/languages" } -live_kit_client = { path = "crates/live_kit_client" } -live_kit_server = { path = "crates/live_kit_server" } +livekit_client = { path = "crates/livekit_client" } +livekit_server = { path = "crates/livekit_server" } lsp = { path = "crates/lsp" } markdown = { path = "crates/markdown" } markdown_preview = { path = "crates/markdown_preview" } @@ -382,6 +382,7 @@ heed = { version = "0.20.1", features = ["read-txn-no-tls"] } hex = "0.4.3" html5ever = "0.27.0" hyper = "0.14" +http = "1.1" ignore = "0.4.22" image = "0.25.1" indexmap = { version = "1.6.2", features = ["serde"] } @@ -392,6 +393,7 @@ jupyter-protocol = { version = "0.5.0" } jupyter-websocket-client = { version = "0.8.0" } libc = "0.2" linkify = "0.10.0" +livekit = { git = "https://github.com/zed-industries/rust-sdks", rev="799f10133d93ba2a88642cd480d01ec4da53408c", features = ["dispatcher", "services-dispatcher", "rustls-tls-native-roots"], default-features = false } log = { version = "0.4.16", features = ["kv_unstable_serde", "serde"] } markup5ever_rcdom = "0.3.0" nanoid = "0.4" @@ -570,6 +572,10 @@ features = [ "Win32_UI_WindowsAndMessaging", ] +# TODO livekit https://github.com/RustAudio/cpal/pull/891 +[patch.crates-io] +cpal = { git = "https://github.com/zed-industries/cpal", rev = "fd8bc2fd39f1f5fdee5a0690656caff9a26d9d50" } + [profile.dev] split-debuginfo = "unpacked" debug = "limited" diff --git a/crates/call/Cargo.toml b/crates/call/Cargo.toml index 974c860c08bdb..f93f0819c679c 100644 --- a/crates/call/Cargo.toml +++ b/crates/call/Cargo.toml @@ -17,7 +17,7 @@ test-support = [ "client/test-support", "collections/test-support", "gpui/test-support", - "live_kit_client/test-support", + "livekit_client/test-support", "project/test-support", "util/test-support" ] @@ -27,11 +27,12 @@ anyhow.workspace = true audio.workspace = true client.workspace = true collections.workspace = true +feature_flags.workspace = true fs.workspace = true futures.workspace = true gpui.workspace = true language.workspace = true -live_kit_client.workspace = true +livekit_client.workspace = true log.workspace = true postage.workspace = true project.workspace = true @@ -47,7 +48,7 @@ collections = { workspace = true, features = ["test-support"] } fs = { workspace = true, features = ["test-support"] } gpui = { workspace = true, features = ["test-support"] } language = { workspace = true, features = ["test-support"] } -live_kit_client = { workspace = true, features = ["test-support"] } +livekit_client = { workspace = true, features = ["test-support"] } project = { workspace = true, features = ["test-support"] } util = { workspace = true, features = ["test-support"] } http_client = { workspace = true, features = ["test-support"] } diff --git a/crates/call/src/call.rs b/crates/call/src/call.rs index c7993f365880d..608fbae4e9031 100644 --- a/crates/call/src/call.rs +++ b/crates/call/src/call.rs @@ -18,6 +18,11 @@ use room::Event; use settings::Settings; use std::sync::Arc; +#[cfg(not(target_os = "windows"))] +pub use livekit_client::play_remote_video_track; +pub use livekit_client::{ + track::RemoteVideoTrack, RemoteVideoTrackView, RemoteVideoTrackViewEvent, +}; pub use participant::ParticipantLocation; pub use room::Room; @@ -26,6 +31,10 @@ struct GlobalActiveCall(Model); impl Global for GlobalActiveCall {} pub fn init(client: Arc, user_store: Model, cx: &mut AppContext) { + livekit_client::init( + cx.background_executor().dispatcher.clone(), + cx.http_client(), + ); CallSettings::register(cx); let active_call = cx.new_model(|cx| ActiveCall::new(client, user_store, cx)); diff --git a/crates/call/src/participant.rs b/crates/call/src/participant.rs index 9faefc63c3697..2ca33be728e5d 100644 --- a/crates/call/src/participant.rs +++ b/crates/call/src/participant.rs @@ -1,13 +1,17 @@ +#![cfg_attr(target_os = "windows", allow(unused))] + use anyhow::{anyhow, Result}; -use client::ParticipantIndex; -use client::{proto, User}; +use client::{proto, ParticipantIndex, User}; use collections::HashMap; use gpui::WeakModel; -pub use live_kit_client::Frame; -pub use live_kit_client::{RemoteAudioTrack, RemoteVideoTrack}; +use livekit_client::AudioStream; use project::Project; use std::sync::Arc; +#[cfg(not(target_os = "windows"))] +pub use livekit_client::id::TrackSid; +pub use livekit_client::track::{RemoteAudioTrack, RemoteVideoTrack}; + #[derive(Copy, Clone, Debug, Eq, PartialEq)] pub enum ParticipantLocation { SharedProject { project_id: u64 }, @@ -39,7 +43,6 @@ pub struct LocalParticipant { pub role: proto::ChannelRole, } -#[derive(Clone, Debug)] pub struct RemoteParticipant { pub user: Arc, pub peer_id: proto::PeerId, @@ -49,6 +52,17 @@ pub struct RemoteParticipant { pub participant_index: ParticipantIndex, pub muted: bool, pub speaking: bool, - pub video_tracks: HashMap>, - pub audio_tracks: HashMap>, + #[cfg(not(target_os = "windows"))] + pub video_tracks: HashMap, + #[cfg(not(target_os = "windows"))] + pub audio_tracks: HashMap, +} + +impl RemoteParticipant { + pub fn has_video_tracks(&self) -> bool { + #[cfg(not(target_os = "windows"))] + return !self.video_tracks.is_empty(); + #[cfg(target_os = "windows")] + return false; + } } diff --git a/crates/call/src/room.rs b/crates/call/src/room.rs index 3eb98f3109ff4..11033098f7a7e 100644 --- a/crates/call/src/room.rs +++ b/crates/call/src/room.rs @@ -1,3 +1,5 @@ +#![cfg_attr(target_os = "windows", allow(unused))] + use crate::{ call_settings::CallSettings, participant::{LocalParticipant, ParticipantLocation, RemoteParticipant}, @@ -15,11 +17,23 @@ use gpui::{ AppContext, AsyncAppContext, Context, EventEmitter, Model, ModelContext, Task, WeakModel, }; use language::LanguageRegistry; -use live_kit_client::{LocalAudioTrack, LocalTrackPublication, LocalVideoTrack, RoomUpdate}; +#[cfg(not(target_os = "windows"))] +use livekit::{ + capture_local_audio_track, capture_local_video_track, + id::ParticipantIdentity, + options::{TrackPublishOptions, VideoCodec}, + play_remote_audio_track, + publication::LocalTrackPublication, + track::{TrackKind, TrackSource}, + RoomEvent, RoomOptions, +}; +#[cfg(target_os = "windows")] +use livekit::{publication::LocalTrackPublication, RoomEvent}; +use livekit_client as livekit; use postage::{sink::Sink, stream::Stream, watch}; use project::Project; use settings::Settings as _; -use std::{future::Future, mem, sync::Arc, time::Duration}; +use std::{any::Any, future::Future, mem, sync::Arc, time::Duration}; use util::{post_inc, ResultExt, TryFutureExt}; pub const RECONNECT_TIMEOUT: Duration = Duration::from_secs(30); @@ -92,13 +106,10 @@ impl Room { !self.shared_projects.is_empty() } - #[cfg(any(test, feature = "test-support"))] + #[cfg(all(any(test, feature = "test-support"), not(target_os = "windows")))] pub fn is_connected(&self) -> bool { if let Some(live_kit) = self.live_kit.as_ref() { - matches!( - *live_kit.room.status().borrow(), - live_kit_client::ConnectionState::Connected { .. } - ) + live_kit.room.connection_state() == livekit::ConnectionState::Connected } else { false } @@ -107,82 +118,12 @@ impl Room { fn new( id: u64, channel_id: Option, - live_kit_connection_info: Option, + livekit_connection_info: Option, client: Arc, user_store: Model, cx: &mut ModelContext, ) -> Self { - let live_kit_room = if let Some(connection_info) = live_kit_connection_info { - let room = live_kit_client::Room::new(); - let mut status = room.status(); - // Consume the initial status of the room. - let _ = status.try_recv(); - let _maintain_room = cx.spawn(|this, mut cx| async move { - while let Some(status) = status.next().await { - let this = if let Some(this) = this.upgrade() { - this - } else { - break; - }; - - if status == live_kit_client::ConnectionState::Disconnected { - this.update(&mut cx, |this, cx| this.leave(cx).log_err()) - .ok(); - break; - } - } - }); - - let _handle_updates = cx.spawn({ - let room = room.clone(); - move |this, mut cx| async move { - let mut updates = room.updates(); - while let Some(update) = updates.next().await { - let this = if let Some(this) = this.upgrade() { - this - } else { - break; - }; - - this.update(&mut cx, |this, cx| { - this.live_kit_room_updated(update, cx).log_err() - }) - .ok(); - } - } - }); - - let connect = room.connect(&connection_info.server_url, &connection_info.token); - cx.spawn(|this, mut cx| async move { - connect.await?; - this.update(&mut cx, |this, cx| { - if this.can_use_microphone() { - if let Some(live_kit) = &this.live_kit { - if !live_kit.muted_by_user && !live_kit.deafened { - return this.share_microphone(cx); - } - } - } - Task::ready(Ok(())) - })? - .await - }) - .detach_and_log_err(cx); - - Some(LiveKitRoom { - room, - screen_track: LocalTrack::None, - microphone_track: LocalTrack::None, - next_publish_id: 0, - muted_by_user: Self::mute_on_join(cx), - deafened: false, - speaking: false, - _maintain_room, - _handle_updates, - }) - } else { - None - }; + spawn_room_connection(livekit_connection_info, cx); let maintain_connection = cx.spawn({ let client = client.clone(); @@ -196,7 +137,7 @@ impl Room { Self { id, channel_id, - live_kit: live_kit_room, + live_kit: None, status: RoomStatus::Online, shared_projects: Default::default(), joined_projects: Default::default(), @@ -706,11 +647,45 @@ impl Room { this.update(&mut cx, |this, cx| this.apply_room_update(room, cx))? } - fn apply_room_update( - &mut self, + fn apply_room_update(&mut self, room: proto::Room, cx: &mut ModelContext) -> Result<()> { + log::trace!( + "client {:?}. room update: {:?}", + self.client.user_id(), + &room + ); + + self.pending_room_update = Some(self.start_room_connection(room, cx)); + + cx.notify(); + Ok(()) + } + + pub fn room_update_completed(&mut self) -> impl Future { + let mut done_rx = self.room_update_completed_rx.clone(); + async move { + while let Some(result) = done_rx.next().await { + if result.is_some() { + break; + } + } + } + } + + #[cfg(target_os = "windows")] + fn start_room_connection( + &self, mut room: proto::Room, cx: &mut ModelContext, - ) -> Result<()> { + ) -> Task<()> { + Task::ready(()) + } + + #[cfg(not(target_os = "windows"))] + fn start_room_connection( + &self, + mut room: proto::Room, + cx: &mut ModelContext, + ) -> Task<()> { // Filter ourselves out from the room's participants. let local_participant_ix = room .participants @@ -737,8 +712,7 @@ impl Room { user_store.get_users(pending_participant_user_ids, cx), ) }); - - self.pending_room_update = Some(cx.spawn(|this, mut cx| async move { + cx.spawn(|this, mut cx| async move { let (remote_participants, pending_participants) = futures::join!(remote_participants, pending_participants); @@ -758,8 +732,8 @@ impl Room { } } this.local_participant.projects.clear(); - if let Some(live_kit_room) = &mut this.live_kit { - live_kit_room.stop_publishing(cx); + if let Some(livekit_room) = &mut this.live_kit { + livekit_room.stop_publishing(cx); } } @@ -776,6 +750,11 @@ impl Room { this.local_participant.projects.clear(); } + let livekit_participants = this + .live_kit + .as_ref() + .map(|live_kit| live_kit.room.remote_participants()); + if let Some(participants) = remote_participants.log_err() { for (participant, user) in room.participants.into_iter().zip(participants) { let Some(peer_id) = participant.peer_id else { @@ -858,40 +837,31 @@ impl Room { muted: true, speaking: false, video_tracks: Default::default(), + #[cfg(not(target_os = "windows"))] audio_tracks: Default::default(), }, ); Audio::play_sound(Sound::Joined, cx); - - if let Some(live_kit) = this.live_kit.as_ref() { - let video_tracks = - live_kit.room.remote_video_tracks(&user.id.to_string()); - let audio_tracks = - live_kit.room.remote_audio_tracks(&user.id.to_string()); - let publications = live_kit - .room - .remote_audio_track_publications(&user.id.to_string()); - - for track in video_tracks { - this.live_kit_room_updated( - RoomUpdate::SubscribedToRemoteVideoTrack(track), - cx, - ) - .log_err(); - } - - for (track, publication) in - audio_tracks.iter().zip(publications.iter()) + if let Some(livekit_participants) = &livekit_participants { + if let Some(livekit_participant) = livekit_participants + .get(&ParticipantIdentity(user.id.to_string())) { - this.live_kit_room_updated( - RoomUpdate::SubscribedToRemoteAudioTrack( - track.clone(), - publication.clone(), - ), - cx, - ) - .log_err(); + for publication in + livekit_participant.track_publications().into_values() + { + if let Some(track) = publication.track() { + this.livekit_room_updated( + RoomEvent::TrackSubscribed { + track, + publication, + participant: livekit_participant.clone(), + }, + cx, + ) + .warn_on_err(); + } + } } } } @@ -959,61 +929,89 @@ impl Room { cx.notify(); }) .ok(); - })); - - cx.notify(); - Ok(()) - } - - pub fn room_update_completed(&mut self) -> impl Future { - let mut done_rx = self.room_update_completed_rx.clone(); - async move { - while let Some(result) = done_rx.next().await { - if result.is_some() { - break; - } - } - } + }) } - fn live_kit_room_updated( + fn livekit_room_updated( &mut self, - update: RoomUpdate, + event: RoomEvent, cx: &mut ModelContext, ) -> Result<()> { - match update { - RoomUpdate::SubscribedToRemoteVideoTrack(track) => { - let user_id = track.publisher_id().parse()?; - let track_id = track.sid().to_string(); - let participant = self - .remote_participants - .get_mut(&user_id) - .ok_or_else(|| anyhow!("subscribed to track by unknown participant"))?; - participant.video_tracks.insert(track_id.clone(), track); - cx.emit(Event::RemoteVideoTracksChanged { - participant_id: participant.peer_id, - }); + log::trace!( + "client {:?}. livekit event: {:?}", + self.client.user_id(), + &event + ); + + match event { + #[cfg(not(target_os = "windows"))] + RoomEvent::TrackSubscribed { + track, + participant, + publication, + } => { + let user_id = participant.identity().0.parse()?; + let track_id = track.sid(); + let participant = self.remote_participants.get_mut(&user_id).ok_or_else(|| { + anyhow!( + "{:?} subscribed to track by unknown participant {user_id}", + self.client.user_id() + ) + })?; + if self.live_kit.as_ref().map_or(true, |kit| kit.deafened) { + track.rtc_track().set_enabled(false); + } + match track { + livekit::track::RemoteTrack::Audio(track) => { + cx.emit(Event::RemoteAudioTracksChanged { + participant_id: participant.peer_id, + }); + let stream = play_remote_audio_track(&track, cx.background_executor())?; + participant.audio_tracks.insert(track_id, (track, stream)); + participant.muted = publication.is_muted(); + } + livekit::track::RemoteTrack::Video(track) => { + cx.emit(Event::RemoteVideoTracksChanged { + participant_id: participant.peer_id, + }); + participant.video_tracks.insert(track_id, track); + } + } } - RoomUpdate::UnsubscribedFromRemoteVideoTrack { - publisher_id, - track_id, + #[cfg(not(target_os = "windows"))] + RoomEvent::TrackUnsubscribed { + track, participant, .. } => { - let user_id = publisher_id.parse()?; - let participant = self - .remote_participants - .get_mut(&user_id) - .ok_or_else(|| anyhow!("unsubscribed from track by unknown participant"))?; - participant.video_tracks.remove(&track_id); - cx.emit(Event::RemoteVideoTracksChanged { - participant_id: participant.peer_id, - }); + let user_id = participant.identity().0.parse()?; + let participant = self.remote_participants.get_mut(&user_id).ok_or_else(|| { + anyhow!( + "{:?}, unsubscribed from track by unknown participant {user_id}", + self.client.user_id() + ) + })?; + match track { + livekit::track::RemoteTrack::Audio(track) => { + participant.audio_tracks.remove(&track.sid()); + participant.muted = true; + cx.emit(Event::RemoteAudioTracksChanged { + participant_id: participant.peer_id, + }); + } + livekit::track::RemoteTrack::Video(track) => { + participant.video_tracks.remove(&track.sid()); + cx.emit(Event::RemoteVideoTracksChanged { + participant_id: participant.peer_id, + }); + } + } } - RoomUpdate::ActiveSpeakersChanged { speakers } => { + #[cfg(not(target_os = "windows"))] + RoomEvent::ActiveSpeakersChanged { speakers } => { let mut speaker_ids = speakers .into_iter() - .filter_map(|speaker_sid| speaker_sid.parse().ok()) + .filter_map(|speaker| speaker.identity().0.parse().ok()) .collect::>(); speaker_ids.sort_unstable(); for (sid, participant) in &mut self.remote_participants { @@ -1026,82 +1024,65 @@ impl Room { } } - RoomUpdate::RemoteAudioTrackMuteChanged { track_id, muted } => { + #[cfg(not(target_os = "windows"))] + RoomEvent::TrackMuted { + participant, + publication, + } + | RoomEvent::TrackUnmuted { + participant, + publication, + } => { let mut found = false; - for participant in &mut self.remote_participants.values_mut() { - for track in participant.audio_tracks.values() { + let user_id = participant.identity().0.parse()?; + let track_id = publication.sid(); + if let Some(participant) = self.remote_participants.get_mut(&user_id) { + for (track, _) in participant.audio_tracks.values() { if track.sid() == track_id { found = true; break; } } if found { - participant.muted = muted; - break; - } - } - } - - RoomUpdate::SubscribedToRemoteAudioTrack(track, publication) => { - if let Some(live_kit) = &self.live_kit { - if live_kit.deafened { - track.stop(); - cx.foreground_executor() - .spawn(publication.set_enabled(false)) - .detach(); + participant.muted = publication.is_muted(); } } - - let user_id = track.publisher_id().parse()?; - let track_id = track.sid().to_string(); - let participant = self - .remote_participants - .get_mut(&user_id) - .ok_or_else(|| anyhow!("subscribed to track by unknown participant"))?; - participant.audio_tracks.insert(track_id.clone(), track); - participant.muted = publication.is_muted(); - - cx.emit(Event::RemoteAudioTracksChanged { - participant_id: participant.peer_id, - }); - } - - RoomUpdate::UnsubscribedFromRemoteAudioTrack { - publisher_id, - track_id, - } => { - let user_id = publisher_id.parse()?; - let participant = self - .remote_participants - .get_mut(&user_id) - .ok_or_else(|| anyhow!("unsubscribed from track by unknown participant"))?; - participant.audio_tracks.remove(&track_id); - cx.emit(Event::RemoteAudioTracksChanged { - participant_id: participant.peer_id, - }); - } - - RoomUpdate::LocalAudioTrackUnpublished { publication } => { - log::info!("unpublished audio track {}", publication.sid()); - if let Some(room) = &mut self.live_kit { - room.microphone_track = LocalTrack::None; - } } - RoomUpdate::LocalVideoTrackUnpublished { publication } => { - log::info!("unpublished video track {}", publication.sid()); + #[cfg(not(target_os = "windows"))] + RoomEvent::LocalTrackUnpublished { publication, .. } => { + log::info!("unpublished track {}", publication.sid()); if let Some(room) = &mut self.live_kit { - room.screen_track = LocalTrack::None; + if let LocalTrack::Published { + track_publication, .. + } = &room.microphone_track + { + if track_publication.sid() == publication.sid() { + room.microphone_track = LocalTrack::None; + } + } + if let LocalTrack::Published { + track_publication, .. + } = &room.screen_track + { + if track_publication.sid() == publication.sid() { + room.screen_track = LocalTrack::None; + } + } } } - RoomUpdate::LocalAudioTrackPublished { publication } => { - log::info!("published audio track {}", publication.sid()); + #[cfg(not(target_os = "windows"))] + RoomEvent::LocalTrackPublished { publication, .. } => { + log::info!("published track {:?}", publication.sid()); } - RoomUpdate::LocalVideoTrackPublished { publication } => { - log::info!("published video track {}", publication.sid()); + #[cfg(not(target_os = "windows"))] + RoomEvent::Disconnected { reason } => { + log::info!("disconnected from room: {reason:?}"); + self.leave(cx).detach_and_log_err(cx); } + _ => {} } cx.notify(); @@ -1317,8 +1298,17 @@ impl Room { self.live_kit.as_ref().map(|live_kit| live_kit.deafened) } - pub fn can_use_microphone(&self) -> bool { + pub fn can_use_microphone(&self, _cx: &AppContext) -> bool { use proto::ChannelRole::*; + + #[cfg(not(any(test, feature = "test-support")))] + { + use feature_flags::FeatureFlagAppExt as _; + if cfg!(target_os = "windows") || (cfg!(target_os = "linux") && !_cx.is_staff()) { + return false; + } + } + match self.local_participant.role { Admin | Member | Talker => true, Guest | Banned => false, @@ -1333,161 +1323,177 @@ impl Room { } } + #[cfg(target_os = "windows")] + pub fn share_microphone(&mut self, cx: &mut ModelContext) -> Task> { + Task::ready(Err(anyhow!("Windows is not supported yet"))) + } + + #[cfg(not(target_os = "windows"))] #[track_caller] pub fn share_microphone(&mut self, cx: &mut ModelContext) -> Task> { if self.status.is_offline() { return Task::ready(Err(anyhow!("room is offline"))); } - let publish_id = if let Some(live_kit) = self.live_kit.as_mut() { + let (participant, publish_id) = if let Some(live_kit) = self.live_kit.as_mut() { let publish_id = post_inc(&mut live_kit.next_publish_id); live_kit.microphone_track = LocalTrack::Pending { publish_id }; cx.notify(); - publish_id + (live_kit.room.local_participant(), publish_id) } else { return Task::ready(Err(anyhow!("live-kit was not initialized"))); }; cx.spawn(move |this, mut cx| async move { - let publish_track = async { - let track = LocalAudioTrack::create(); - this.upgrade() - .ok_or_else(|| anyhow!("room was dropped"))? - .update(&mut cx, |this, _| { - this.live_kit - .as_ref() - .map(|live_kit| live_kit.room.publish_audio_track(track)) - })? - .ok_or_else(|| anyhow!("live-kit was not initialized"))? - .await - }; - let publication = publish_track.await; - this.upgrade() - .ok_or_else(|| anyhow!("room was dropped"))? - .update(&mut cx, |this, cx| { - let live_kit = this - .live_kit - .as_mut() - .ok_or_else(|| anyhow!("live-kit was not initialized"))?; - - let canceled = if let LocalTrack::Pending { - publish_id: cur_publish_id, - } = &live_kit.microphone_track - { - *cur_publish_id != publish_id - } else { - true - }; - - match publication { - Ok(publication) => { - if canceled { - live_kit.room.unpublish_track(publication); - } else { - if live_kit.muted_by_user || live_kit.deafened { - cx.background_executor() - .spawn(publication.set_mute(true)) - .detach(); - } - live_kit.microphone_track = LocalTrack::Published { - track_publication: publication, - }; - cx.notify(); + let (track, stream) = capture_local_audio_track(cx.background_executor())?.await; + + let publication = participant + .publish_track( + livekit::track::LocalTrack::Audio(track), + TrackPublishOptions { + source: TrackSource::Microphone, + ..Default::default() + }, + ) + .await + .map_err(|error| anyhow!("failed to publish track: {error}")); + this.update(&mut cx, |this, cx| { + let live_kit = this + .live_kit + .as_mut() + .ok_or_else(|| anyhow!("live-kit was not initialized"))?; + + let canceled = if let LocalTrack::Pending { + publish_id: cur_publish_id, + } = &live_kit.microphone_track + { + *cur_publish_id != publish_id + } else { + true + }; + + match publication { + Ok(publication) => { + if canceled { + cx.background_executor() + .spawn(async move { + participant.unpublish_track(&publication.sid()).await + }) + .detach_and_log_err(cx) + } else { + if live_kit.muted_by_user || live_kit.deafened { + publication.mute(); } - Ok(()) + live_kit.microphone_track = LocalTrack::Published { + track_publication: publication, + _stream: Box::new(stream), + }; + cx.notify(); } - Err(error) => { - if canceled { - Ok(()) - } else { - live_kit.microphone_track = LocalTrack::None; - cx.notify(); - Err(error) - } + Ok(()) + } + Err(error) => { + if canceled { + Ok(()) + } else { + live_kit.microphone_track = LocalTrack::None; + cx.notify(); + Err(error) } } - })? + } + })? }) } + #[cfg(target_os = "windows")] + pub fn share_screen(&mut self, cx: &mut ModelContext) -> Task> { + Task::ready(Err(anyhow!("Windows is not supported yet"))) + } + + #[cfg(not(target_os = "windows"))] pub fn share_screen(&mut self, cx: &mut ModelContext) -> Task> { if self.status.is_offline() { return Task::ready(Err(anyhow!("room is offline"))); - } else if self.is_screen_sharing() { + } + if self.is_screen_sharing() { return Task::ready(Err(anyhow!("screen was already shared"))); } - let (displays, publish_id) = if let Some(live_kit) = self.live_kit.as_mut() { + let (participant, publish_id) = if let Some(live_kit) = self.live_kit.as_mut() { let publish_id = post_inc(&mut live_kit.next_publish_id); live_kit.screen_track = LocalTrack::Pending { publish_id }; cx.notify(); - (live_kit.room.display_sources(), publish_id) + (live_kit.room.local_participant(), publish_id) } else { return Task::ready(Err(anyhow!("live-kit was not initialized"))); }; - cx.spawn(move |this, mut cx| async move { - let publish_track = async { - let displays = displays.await?; - let display = displays - .first() - .ok_or_else(|| anyhow!("no display found"))?; - let track = LocalVideoTrack::screen_share_for_display(display); - this.upgrade() - .ok_or_else(|| anyhow!("room was dropped"))? - .update(&mut cx, |this, _| { - this.live_kit - .as_ref() - .map(|live_kit| live_kit.room.publish_video_track(track)) - })? - .ok_or_else(|| anyhow!("live-kit was not initialized"))? - .await - }; - - let publication = publish_track.await; - this.upgrade() - .ok_or_else(|| anyhow!("room was dropped"))? - .update(&mut cx, |this, cx| { - let live_kit = this - .live_kit - .as_mut() - .ok_or_else(|| anyhow!("live-kit was not initialized"))?; - - let canceled = if let LocalTrack::Pending { - publish_id: cur_publish_id, - } = &live_kit.screen_track - { - *cur_publish_id != publish_id - } else { - true - }; + let sources = cx.screen_capture_sources(); - match publication { - Ok(publication) => { - if canceled { - live_kit.room.unpublish_track(publication); - } else { - live_kit.screen_track = LocalTrack::Published { - track_publication: publication, - }; - cx.notify(); - } + cx.spawn(move |this, mut cx| async move { + let sources = sources.await??; + let source = sources.first().ok_or_else(|| anyhow!("no display found"))?; + + let (track, stream) = capture_local_video_track(&**source).await?; + + let publication = participant + .publish_track( + livekit::track::LocalTrack::Video(track), + TrackPublishOptions { + source: TrackSource::Screenshare, + video_codec: VideoCodec::H264, + ..Default::default() + }, + ) + .await + .map_err(|error| anyhow!("error publishing screen track {error:?}")); - Audio::play_sound(Sound::StartScreenshare, cx); + this.update(&mut cx, |this, cx| { + let live_kit = this + .live_kit + .as_mut() + .ok_or_else(|| anyhow!("live-kit was not initialized"))?; + + let canceled = if let LocalTrack::Pending { + publish_id: cur_publish_id, + } = &live_kit.screen_track + { + *cur_publish_id != publish_id + } else { + true + }; + + match publication { + Ok(publication) => { + if canceled { + cx.background_executor() + .spawn(async move { + participant.unpublish_track(&publication.sid()).await + }) + .detach() + } else { + live_kit.screen_track = LocalTrack::Published { + track_publication: publication, + _stream: Box::new(stream), + }; + cx.notify(); + } + Audio::play_sound(Sound::StartScreenshare, cx); + Ok(()) + } + Err(error) => { + if canceled { Ok(()) - } - Err(error) => { - if canceled { - Ok(()) - } else { - live_kit.screen_track = LocalTrack::None; - cx.notify(); - Err(error) - } + } else { + live_kit.screen_track = LocalTrack::None; + cx.notify(); + Err(error) } } - })? + } + })? }) } @@ -1512,9 +1518,7 @@ impl Room { } if should_undeafen { - if let Some(task) = self.set_deafened(false, cx) { - task.detach_and_log_err(cx); - } + self.set_deafened(false, cx); } } } @@ -1527,9 +1531,7 @@ impl Room { live_kit.deafened = deafened; let should_change_mute = !live_kit.muted_by_user; - if let Some(task) = self.set_deafened(deafened, cx) { - task.detach_and_log_err(cx); - } + self.set_deafened(deafened, cx); if should_change_mute { if let Some(task) = self.set_mute(deafened, cx) { @@ -1557,47 +1559,36 @@ impl Room { LocalTrack::Published { track_publication, .. } => { - live_kit.room.unpublish_track(track_publication); - cx.notify(); - + #[cfg(not(target_os = "windows"))] + { + let local_participant = live_kit.room.local_participant(); + let sid = track_publication.sid(); + cx.background_executor() + .spawn(async move { local_participant.unpublish_track(&sid).await }) + .detach_and_log_err(cx); + cx.notify(); + } Audio::play_sound(Sound::StopScreenshare, cx); Ok(()) } } } - fn set_deafened( - &mut self, - deafened: bool, - cx: &mut ModelContext, - ) -> Option>> { - let live_kit = self.live_kit.as_mut()?; - cx.notify(); - - let mut track_updates = Vec::new(); - for participant in self.remote_participants.values() { - for publication in live_kit - .room - .remote_audio_track_publications(&participant.user.id.to_string()) - { - track_updates.push(publication.set_enabled(!deafened)); - } - - for track in participant.audio_tracks.values() { - if deafened { - track.stop(); - } else { - track.start(); + fn set_deafened(&mut self, deafened: bool, cx: &mut ModelContext) -> Option<()> { + #[cfg(not(target_os = "windows"))] + { + let live_kit = self.live_kit.as_mut()?; + cx.notify(); + for (_, participant) in live_kit.room.remote_participants() { + for (_, publication) in participant.track_publications() { + if publication.kind() == TrackKind::Audio { + publication.set_enabled(!deafened); + } } } } - Some(cx.foreground_executor().spawn(async move { - for result in futures::future::join_all(track_updates).await { - result?; - } - Ok(()) - })) + None } fn set_mute( @@ -1623,25 +1614,84 @@ impl Room { } } LocalTrack::Pending { .. } => None, - LocalTrack::Published { track_publication } => Some( - cx.foreground_executor() - .spawn(track_publication.set_mute(should_mute)), - ), + LocalTrack::Published { + track_publication, .. + } => { + #[cfg(not(target_os = "windows"))] + { + if should_mute { + track_publication.mute() + } else { + track_publication.unmute() + } + } + None + } } } +} - #[cfg(any(test, feature = "test-support"))] - pub fn set_display_sources(&self, sources: Vec) { - self.live_kit - .as_ref() - .unwrap() - .room - .set_display_sources(sources); +#[cfg(target_os = "windows")] +fn spawn_room_connection( + livekit_connection_info: Option, + cx: &mut ModelContext<'_, Room>, +) { +} + +#[cfg(not(target_os = "windows"))] +fn spawn_room_connection( + livekit_connection_info: Option, + cx: &mut ModelContext<'_, Room>, +) { + if let Some(connection_info) = livekit_connection_info { + cx.spawn(|this, mut cx| async move { + let (room, mut events) = livekit::Room::connect( + &connection_info.server_url, + &connection_info.token, + RoomOptions::default(), + ) + .await?; + + this.update(&mut cx, |this, cx| { + let _handle_updates = cx.spawn(|this, mut cx| async move { + while let Some(event) = events.recv().await { + if this + .update(&mut cx, |this, cx| { + this.livekit_room_updated(event, cx).warn_on_err(); + }) + .is_err() + { + break; + } + } + }); + + let muted_by_user = Room::mute_on_join(cx); + this.live_kit = Some(LiveKitRoom { + room: Arc::new(room), + screen_track: LocalTrack::None, + microphone_track: LocalTrack::None, + next_publish_id: 0, + muted_by_user, + deafened: false, + speaking: false, + _handle_updates, + }); + + if !muted_by_user && this.can_use_microphone(cx) { + this.share_microphone(cx) + } else { + Task::ready(Ok(())) + } + })? + .await + }) + .detach_and_log_err(cx); } } struct LiveKitRoom { - room: Arc, + room: Arc, screen_track: LocalTrack, microphone_track: LocalTrack, /// Tracks whether we're currently in a muted state due to auto-mute from deafening or manual mute performed by user. @@ -1649,17 +1699,21 @@ struct LiveKitRoom { deafened: bool, speaking: bool, next_publish_id: usize, - _maintain_room: Task<()>, _handle_updates: Task<()>, } impl LiveKitRoom { + #[cfg(target_os = "windows")] + fn stop_publishing(&mut self, _cx: &mut ModelContext) {} + + #[cfg(not(target_os = "windows"))] fn stop_publishing(&mut self, cx: &mut ModelContext) { + let mut tracks_to_unpublish = Vec::new(); if let LocalTrack::Published { track_publication, .. } = mem::replace(&mut self.microphone_track, LocalTrack::None) { - self.room.unpublish_track(track_publication); + tracks_to_unpublish.push(track_publication.sid()); cx.notify(); } @@ -1667,9 +1721,18 @@ impl LiveKitRoom { track_publication, .. } = mem::replace(&mut self.screen_track, LocalTrack::None) { - self.room.unpublish_track(track_publication); + tracks_to_unpublish.push(track_publication.sid()); cx.notify(); } + + let participant = self.room.local_participant(); + cx.background_executor() + .spawn(async move { + for sid in tracks_to_unpublish { + participant.unpublish_track(&sid).await.log_err(); + } + }) + .detach(); } } @@ -1680,6 +1743,7 @@ enum LocalTrack { }, Published { track_publication: LocalTrackPublication, + _stream: Box, }, } diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index c8e1c8d3ed30a..a01711f0e5edc 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -548,7 +548,7 @@ mod mac_os { }, LocalPath { executable: PathBuf, - plist: InfoPlist, + short_version_string: String, }, } @@ -587,17 +587,9 @@ mod mac_os { } _ => { println!("Bundle path {bundle_path:?} has no *.app extension, attempting to locate a dev build"); - let plist_path = bundle_path - .parent() - .with_context(|| format!("Bundle path {bundle_path:?} has no parent"))? - .join("WebRTC.framework/Resources/Info.plist"); - let plist = - plist::from_file::<_, InfoPlist>(&plist_path).with_context(|| { - format!("Reading dev bundle plist file at {plist_path:?}") - })?; Ok(Bundle::LocalPath { executable: bundle_path, - plist, + short_version_string: "test-dev-version".to_string(), }) } } @@ -607,9 +599,16 @@ mod mac_os { impl InstalledApp for Bundle { fn zed_version_string(&self) -> String { let is_dev = matches!(self, Self::LocalPath { .. }); + let version = match self { + Bundle::App { plist, .. } => &plist.bundle_short_version_string, + Bundle::LocalPath { + short_version_string, + .. + } => &short_version_string, + }; format!( "Zed {}{} – {}", - self.plist().bundle_short_version_string, + version, if is_dev { " (dev)" } else { "" }, self.path().display(), ) @@ -691,13 +690,6 @@ mod mac_os { } impl Bundle { - fn plist(&self) -> &InfoPlist { - match self { - Self::App { plist, .. } => plist, - Self::LocalPath { plist, .. } => plist, - } - } - fn path(&self) -> &Path { match self { Self::App { app_bundle, .. } => app_bundle, diff --git a/crates/collab/.env.toml b/crates/collab/.env.toml index f542422e950a3..5d292387cbda0 100644 --- a/crates/collab/.env.toml +++ b/crates/collab/.env.toml @@ -5,9 +5,9 @@ HTTP_PORT = 8080 API_TOKEN = "secret" INVITE_LINK_PREFIX = "http://localhost:3000/invites/" ZED_ENVIRONMENT = "development" -LIVE_KIT_SERVER = "http://localhost:7880" -LIVE_KIT_KEY = "devkey" -LIVE_KIT_SECRET = "secret" +LIVEKIT_SERVER = "http://localhost:7880" +LIVEKIT_KEY = "devkey" +LIVEKIT_SECRET = "secret" BLOB_STORE_ACCESS_KEY = "the-blob-store-access-key" BLOB_STORE_SECRET_KEY = "the-blob-store-secret-key" BLOB_STORE_BUCKET = "the-extensions-bucket" diff --git a/crates/collab/Cargo.toml b/crates/collab/Cargo.toml index e56507c007289..ea6020fa68975 100644 --- a/crates/collab/Cargo.toml +++ b/crates/collab/Cargo.toml @@ -40,7 +40,7 @@ google_ai.workspace = true hex.workspace = true http_client.workspace = true jsonwebtoken.workspace = true -live_kit_server.workspace = true +livekit_server.workspace = true log.workspace = true nanoid.workspace = true open_ai.workspace = true @@ -101,7 +101,7 @@ hyper.workspace = true indoc.workspace = true language = { workspace = true, features = ["test-support"] } language_model = { workspace = true, features = ["test-support"] } -live_kit_client = { workspace = true, features = ["test-support"] } +livekit_client = { workspace = true, features = ["test-support"] } lsp = { workspace = true, features = ["test-support"] } menu.workspace = true multi_buffer = { workspace = true, features = ["test-support"] } diff --git a/crates/collab/k8s/collab.template.yml b/crates/collab/k8s/collab.template.yml index fb5d4ed6eccf6..2bd250771dfb3 100644 --- a/crates/collab/k8s/collab.template.yml +++ b/crates/collab/k8s/collab.template.yml @@ -109,17 +109,17 @@ spec: secretKeyRef: name: zed-client key: checksum-seed - - name: LIVE_KIT_SERVER + - name: LIVEKIT_SERVER valueFrom: secretKeyRef: name: livekit key: server - - name: LIVE_KIT_KEY + - name: livekit_KEY valueFrom: secretKeyRef: name: livekit key: key - - name: LIVE_KIT_SECRET + - name: livekit_SECRET valueFrom: secretKeyRef: name: livekit diff --git a/crates/collab/src/db/queries/channels.rs b/crates/collab/src/db/queries/channels.rs index 10120ea814301..b107358eff6fe 100644 --- a/crates/collab/src/db/queries/channels.rs +++ b/crates/collab/src/db/queries/channels.rs @@ -154,9 +154,9 @@ impl Database { } let role = role.unwrap(); - let live_kit_room = format!("channel-{}", nanoid::nanoid!(30)); + let livekit_room = format!("channel-{}", nanoid::nanoid!(30)); let room_id = self - .get_or_create_channel_room(channel_id, &live_kit_room, &tx) + .get_or_create_channel_room(channel_id, &livekit_room, &tx) .await?; self.join_channel_room_internal(room_id, user_id, connection, role, &tx) @@ -896,7 +896,7 @@ impl Database { pub(crate) async fn get_or_create_channel_room( &self, channel_id: ChannelId, - live_kit_room: &str, + livekit_room: &str, tx: &DatabaseTransaction, ) -> Result { let room = room::Entity::find() @@ -909,7 +909,7 @@ impl Database { } else { let result = room::Entity::insert(room::ActiveModel { channel_id: ActiveValue::Set(Some(channel_id)), - live_kit_room: ActiveValue::Set(live_kit_room.to_string()), + live_kit_room: ActiveValue::Set(livekit_room.to_string()), ..Default::default() }) .exec(tx) diff --git a/crates/collab/src/db/queries/rooms.rs b/crates/collab/src/db/queries/rooms.rs index 682c4ed38949e..a3a99bee71a44 100644 --- a/crates/collab/src/db/queries/rooms.rs +++ b/crates/collab/src/db/queries/rooms.rs @@ -103,11 +103,11 @@ impl Database { &self, user_id: UserId, connection: ConnectionId, - live_kit_room: &str, + livekit_room: &str, ) -> Result { self.transaction(|tx| async move { let room = room::ActiveModel { - live_kit_room: ActiveValue::set(live_kit_room.into()), + live_kit_room: ActiveValue::set(livekit_room.into()), ..Default::default() } .insert(&*tx) @@ -1316,7 +1316,7 @@ impl Database { channel, proto::Room { id: db_room.id.to_proto(), - live_kit_room: db_room.live_kit_room, + livekit_room: db_room.live_kit_room, participants: participants.into_values().collect(), pending_participants, followers, diff --git a/crates/collab/src/lib.rs b/crates/collab/src/lib.rs index f595cff89011d..cfa0e1631ebca 100644 --- a/crates/collab/src/lib.rs +++ b/crates/collab/src/lib.rs @@ -156,9 +156,9 @@ pub struct Config { pub clickhouse_password: Option, pub clickhouse_database: Option, pub invite_link_prefix: String, - pub live_kit_server: Option, - pub live_kit_key: Option, - pub live_kit_secret: Option, + pub livekit_server: Option, + pub livekit_key: Option, + pub livekit_secret: Option, pub llm_database_url: Option, pub llm_database_max_connections: Option, pub llm_database_migrations_path: Option, @@ -210,9 +210,9 @@ impl Config { database_max_connections: 0, api_token: "".into(), invite_link_prefix: "".into(), - live_kit_server: None, - live_kit_key: None, - live_kit_secret: None, + livekit_server: None, + livekit_key: None, + livekit_secret: None, llm_database_url: None, llm_database_max_connections: None, llm_database_migrations_path: None, @@ -277,7 +277,7 @@ impl ServiceMode { pub struct AppState { pub db: Arc, pub llm_db: Option>, - pub live_kit_client: Option>, + pub livekit_client: Option>, pub blob_store_client: Option, pub stripe_client: Option>, pub stripe_billing: Option>, @@ -309,17 +309,17 @@ impl AppState { None }; - let live_kit_client = if let Some(((server, key), secret)) = config - .live_kit_server + let livekit_client = if let Some(((server, key), secret)) = config + .livekit_server .as_ref() - .zip(config.live_kit_key.as_ref()) - .zip(config.live_kit_secret.as_ref()) + .zip(config.livekit_key.as_ref()) + .zip(config.livekit_secret.as_ref()) { - Some(Arc::new(live_kit_server::api::LiveKitClient::new( + Some(Arc::new(livekit_server::api::LiveKitClient::new( server.clone(), key.clone(), secret.clone(), - )) as Arc) + )) as Arc) } else { None }; @@ -329,7 +329,7 @@ impl AppState { let this = Self { db: db.clone(), llm_db, - live_kit_client, + livekit_client, blob_store_client: build_blob_store_client(&config).await.log_err(), stripe_billing: stripe_client .clone() diff --git a/crates/collab/src/rpc.rs b/crates/collab/src/rpc.rs index a17d4924b72d5..5aca7ec52ba91 100644 --- a/crates/collab/src/rpc.rs +++ b/crates/collab/src/rpc.rs @@ -418,7 +418,7 @@ impl Server { let peer = self.peer.clone(); let timeout = self.app_state.executor.sleep(CLEANUP_TIMEOUT); let pool = self.connection_pool.clone(); - let live_kit_client = self.app_state.live_kit_client.clone(); + let livekit_client = self.app_state.livekit_client.clone(); let span = info_span!("start server"); self.app_state.executor.spawn_detached( @@ -463,8 +463,8 @@ impl Server { for room_id in room_ids { let mut contacts_to_update = HashSet::default(); let mut canceled_calls_to_user_ids = Vec::new(); - let mut live_kit_room = String::new(); - let mut delete_live_kit_room = false; + let mut livekit_room = String::new(); + let mut delete_livekit_room = false; if let Some(mut refreshed_room) = app_state .db @@ -487,8 +487,8 @@ impl Server { .extend(refreshed_room.canceled_calls_to_user_ids.iter().copied()); canceled_calls_to_user_ids = mem::take(&mut refreshed_room.canceled_calls_to_user_ids); - live_kit_room = mem::take(&mut refreshed_room.room.live_kit_room); - delete_live_kit_room = refreshed_room.room.participants.is_empty(); + livekit_room = mem::take(&mut refreshed_room.room.livekit_room); + delete_livekit_room = refreshed_room.room.participants.is_empty(); } { @@ -539,9 +539,9 @@ impl Server { } } - if let Some(live_kit) = live_kit_client.as_ref() { - if delete_live_kit_room { - live_kit.delete_room(live_kit_room).await.trace_err(); + if let Some(live_kit) = livekit_client.as_ref() { + if delete_livekit_room { + live_kit.delete_room(livekit_room).await.trace_err(); } } } @@ -1210,15 +1210,15 @@ async fn create_room( response: Response, session: Session, ) -> Result<()> { - let live_kit_room = nanoid::nanoid!(30); + let livekit_room = nanoid::nanoid!(30); let live_kit_connection_info = util::maybe!(async { - let live_kit = session.app_state.live_kit_client.as_ref(); + let live_kit = session.app_state.livekit_client.as_ref(); let live_kit = live_kit?; let user_id = session.user_id().to_string(); let token = live_kit - .room_token(&live_kit_room, &user_id.to_string()) + .room_token(&livekit_room, &user_id.to_string()) .trace_err()?; Some(proto::LiveKitConnectionInfo { @@ -1232,7 +1232,7 @@ async fn create_room( let room = session .db() .await - .create_room(session.user_id(), session.connection_id, &live_kit_room) + .create_room(session.user_id(), session.connection_id, &livekit_room) .await?; response.send(proto::CreateRoomResponse { @@ -1284,22 +1284,22 @@ async fn join_room( .trace_err(); } - let live_kit_connection_info = - if let Some(live_kit) = session.app_state.live_kit_client.as_ref() { - live_kit - .room_token( - &joined_room.room.live_kit_room, - &session.user_id().to_string(), - ) - .trace_err() - .map(|token| proto::LiveKitConnectionInfo { - server_url: live_kit.url().into(), - token, - can_publish: true, - }) - } else { - None - }; + let live_kit_connection_info = if let Some(live_kit) = session.app_state.livekit_client.as_ref() + { + live_kit + .room_token( + &joined_room.room.livekit_room, + &session.user_id().to_string(), + ) + .trace_err() + .map(|token| proto::LiveKitConnectionInfo { + server_url: live_kit.url().into(), + token, + can_publish: true, + }) + } else { + None + }; response.send(proto::JoinRoomResponse { room: Some(joined_room.room), @@ -1506,7 +1506,7 @@ async fn set_room_participant_role( let user_id = UserId::from_proto(request.user_id); let role = ChannelRole::from(request.role()); - let (live_kit_room, can_publish) = { + let (livekit_room, can_publish) = { let room = session .db() .await @@ -1518,18 +1518,18 @@ async fn set_room_participant_role( ) .await?; - let live_kit_room = room.live_kit_room.clone(); + let livekit_room = room.livekit_room.clone(); let can_publish = ChannelRole::from(request.role()).can_use_microphone(); room_updated(&room, &session.peer); - (live_kit_room, can_publish) + (livekit_room, can_publish) }; - if let Some(live_kit) = session.app_state.live_kit_client.as_ref() { + if let Some(live_kit) = session.app_state.livekit_client.as_ref() { live_kit .update_participant( - live_kit_room.clone(), + livekit_room.clone(), request.user_id.to_string(), - live_kit_server::proto::ParticipantPermission { + livekit_server::proto::ParticipantPermission { can_subscribe: true, can_publish, can_publish_data: can_publish, @@ -3091,7 +3091,7 @@ async fn join_channel_internal( let live_kit_connection_info = session .app_state - .live_kit_client + .livekit_client .as_ref() .and_then(|live_kit| { let (can_publish, token) = if role == ChannelRole::Guest { @@ -3099,7 +3099,7 @@ async fn join_channel_internal( false, live_kit .guest_token( - &joined_room.room.live_kit_room, + &joined_room.room.livekit_room, &session.user_id().to_string(), ) .trace_err()?, @@ -3109,7 +3109,7 @@ async fn join_channel_internal( true, live_kit .room_token( - &joined_room.room.live_kit_room, + &joined_room.room.livekit_room, &session.user_id().to_string(), ) .trace_err()?, @@ -4313,8 +4313,8 @@ async fn leave_room_for_session(session: &Session, connection_id: ConnectionId) let room_id; let canceled_calls_to_user_ids; - let live_kit_room; - let delete_live_kit_room; + let livekit_room; + let delete_livekit_room; let room; let channel; @@ -4327,8 +4327,8 @@ async fn leave_room_for_session(session: &Session, connection_id: ConnectionId) room_id = RoomId::from_proto(left_room.room.id); canceled_calls_to_user_ids = mem::take(&mut left_room.canceled_calls_to_user_ids); - live_kit_room = mem::take(&mut left_room.room.live_kit_room); - delete_live_kit_room = left_room.deleted; + livekit_room = mem::take(&mut left_room.room.livekit_room); + delete_livekit_room = left_room.deleted; room = mem::take(&mut left_room.room); channel = mem::take(&mut left_room.channel); @@ -4368,14 +4368,14 @@ async fn leave_room_for_session(session: &Session, connection_id: ConnectionId) update_user_contacts(contact_user_id, session).await?; } - if let Some(live_kit) = session.app_state.live_kit_client.as_ref() { + if let Some(live_kit) = session.app_state.livekit_client.as_ref() { live_kit - .remove_participant(live_kit_room.clone(), session.user_id().to_string()) + .remove_participant(livekit_room.clone(), session.user_id().to_string()) .await .trace_err(); - if delete_live_kit_room { - live_kit.delete_room(live_kit_room).await.trace_err(); + if delete_livekit_room { + live_kit.delete_room(livekit_room).await.trace_err(); } } diff --git a/crates/collab/src/tests.rs b/crates/collab/src/tests.rs index 29373bc6ea170..2ce69efc9b406 100644 --- a/crates/collab/src/tests.rs +++ b/crates/collab/src/tests.rs @@ -1,3 +1,6 @@ +// todo(windows): Actually run the tests +#![cfg(not(target_os = "windows"))] + use std::sync::Arc; use call::Room; diff --git a/crates/collab/src/tests/channel_guest_tests.rs b/crates/collab/src/tests/channel_guest_tests.rs index 5a091fe3083b1..006a3e5d1cf22 100644 --- a/crates/collab/src/tests/channel_guest_tests.rs +++ b/crates/collab/src/tests/channel_guest_tests.rs @@ -107,7 +107,9 @@ async fn test_channel_guest_promotion(cx_a: &mut TestAppContext, cx_b: &mut Test }); assert!(project_b.read_with(cx_b, |project, cx| project.is_read_only(cx))); assert!(editor_b.update(cx_b, |e, cx| e.read_only(cx))); - assert!(room_b.read_with(cx_b, |room, _| !room.can_use_microphone())); + cx_b.update(|cx_b| { + assert!(room_b.read_with(cx_b, |room, cx| !room.can_use_microphone(cx))); + }); assert!(room_b .update(cx_b, |room, cx| room.share_microphone(cx)) .await @@ -133,7 +135,9 @@ async fn test_channel_guest_promotion(cx_a: &mut TestAppContext, cx_b: &mut Test assert!(editor_b.update(cx_b, |editor, cx| !editor.read_only(cx))); // B sees themselves as muted, and can unmute. - assert!(room_b.read_with(cx_b, |room, _| room.can_use_microphone())); + cx_b.update(|cx_b| { + assert!(room_b.read_with(cx_b, |room, cx| room.can_use_microphone(cx))); + }); room_b.read_with(cx_b, |room, _| assert!(room.is_muted())); room_b.update(cx_b, |room, cx| room.toggle_mute(cx)); cx_a.run_until_parked(); @@ -226,7 +230,9 @@ async fn test_channel_requires_zed_cla(cx_a: &mut TestAppContext, cx_b: &mut Tes let room_b = cx_b .read(ActiveCall::global) .update(cx_b, |call, _| call.room().unwrap().clone()); - assert!(room_b.read_with(cx_b, |room, _| !room.can_use_microphone())); + cx_b.update(|cx_b| { + assert!(room_b.read_with(cx_b, |room, cx| !room.can_use_microphone(cx))); + }); // A tries to grant write access to B, but cannot because B has not // yet signed the zed CLA. @@ -244,7 +250,9 @@ async fn test_channel_requires_zed_cla(cx_a: &mut TestAppContext, cx_b: &mut Tes .unwrap_err(); cx_a.run_until_parked(); assert!(room_b.read_with(cx_b, |room, _| !room.can_share_projects())); - assert!(room_b.read_with(cx_b, |room, _| !room.can_use_microphone())); + cx_b.update(|cx_b| { + assert!(room_b.read_with(cx_b, |room, cx| !room.can_use_microphone(cx))); + }); // A tries to grant write access to B, but cannot because B has not // yet signed the zed CLA. @@ -262,7 +270,9 @@ async fn test_channel_requires_zed_cla(cx_a: &mut TestAppContext, cx_b: &mut Tes .unwrap(); cx_a.run_until_parked(); assert!(room_b.read_with(cx_b, |room, _| !room.can_share_projects())); - assert!(room_b.read_with(cx_b, |room, _| room.can_use_microphone())); + cx_b.update(|cx_b| { + assert!(room_b.read_with(cx_b, |room, cx| room.can_use_microphone(cx))); + }); // User B signs the zed CLA. server @@ -287,5 +297,7 @@ async fn test_channel_requires_zed_cla(cx_a: &mut TestAppContext, cx_b: &mut Tes .unwrap(); cx_a.run_until_parked(); assert!(room_b.read_with(cx_b, |room, _| room.can_share_projects())); - assert!(room_b.read_with(cx_b, |room, _| room.can_use_microphone())); + cx_b.update(|cx_b| { + assert!(room_b.read_with(cx_b, |room, cx| room.can_use_microphone(cx))); + }); } diff --git a/crates/collab/src/tests/following_tests.rs b/crates/collab/src/tests/following_tests.rs index d708194f58396..778d67b81dcad 100644 --- a/crates/collab/src/tests/following_tests.rs +++ b/crates/collab/src/tests/following_tests.rs @@ -9,10 +9,9 @@ use collab_ui::{ use editor::{Editor, ExcerptRange, MultiBuffer}; use gpui::{ point, BackgroundExecutor, BorrowAppContext, Context, Entity, SharedString, TestAppContext, - View, VisualContext, VisualTestContext, + TestScreenCaptureSource, View, VisualContext, VisualTestContext, }; use language::Capability; -use live_kit_client::MacOSDisplay; use project::WorktreeSettings; use rpc::proto::PeerId; use serde_json::json; @@ -429,17 +428,17 @@ async fn test_basic_following( ); // Client B activates an external window, which causes a new screen-sharing item to be added to the pane. - let display = MacOSDisplay::new(); + let display = TestScreenCaptureSource::new(); active_call_b .update(cx_b, |call, cx| call.set_location(None, cx)) .await .unwrap(); + cx_b.set_screen_capture_sources(vec![display]); active_call_b .update(cx_b, |call, cx| { - call.room().unwrap().update(cx, |room, cx| { - room.set_display_sources(vec![display.clone()]); - room.share_screen(cx) - }) + call.room() + .unwrap() + .update(cx, |room, cx| room.share_screen(cx)) }) .await .unwrap(); diff --git a/crates/collab/src/tests/integration_tests.rs b/crates/collab/src/tests/integration_tests.rs index b6a02474245e9..4b0e44d0ed148 100644 --- a/crates/collab/src/tests/integration_tests.rs +++ b/crates/collab/src/tests/integration_tests.rs @@ -16,7 +16,7 @@ use futures::{channel::mpsc, StreamExt as _}; use git::repository::GitFileStatus; use gpui::{ px, size, AppContext, BackgroundExecutor, Model, Modifiers, MouseButton, MouseDownEvent, - TestAppContext, UpdateGlobal, + TestAppContext, TestScreenCaptureSource, UpdateGlobal, }; use language::{ language_settings::{ @@ -25,7 +25,6 @@ use language::{ tree_sitter_rust, tree_sitter_typescript, Diagnostic, DiagnosticEntry, FakeLspAdapter, Language, LanguageConfig, LanguageMatcher, LineEnding, OffsetRangeExt, Point, Rope, }; -use live_kit_client::MacOSDisplay; use lsp::LanguageServerId; use parking_lot::Mutex; use project::lsp_store::FormatTarget; @@ -242,15 +241,15 @@ async fn test_basic_calls( ); // User A shares their screen - let display = MacOSDisplay::new(); + let display = TestScreenCaptureSource::new(); let events_b = active_call_events(cx_b); let events_c = active_call_events(cx_c); + cx_a.set_screen_capture_sources(vec![display]); active_call_a .update(cx_a, |call, cx| { - call.room().unwrap().update(cx, |room, cx| { - room.set_display_sources(vec![display.clone()]); - room.share_screen(cx) - }) + call.room() + .unwrap() + .update(cx, |room, cx| room.share_screen(cx)) }) .await .unwrap(); @@ -329,7 +328,7 @@ async fn test_basic_calls( // to automatically leave the room. User C leaves the room as well because // nobody else is in there. server - .test_live_kit_server + .test_livekit_server .disconnect_client(client_b.user_id().unwrap().to_string()) .await; executor.run_until_parked(); @@ -844,7 +843,7 @@ async fn test_client_disconnecting_from_room( // User B gets disconnected from the LiveKit server, which causes it // to automatically leave the room. server - .test_live_kit_server + .test_livekit_server .disconnect_client(client_b.user_id().unwrap().to_string()) .await; executor.run_until_parked(); @@ -1943,7 +1942,7 @@ async fn test_mute_deafen( room_a.read_with(cx_a, |room, _| assert!(!room.is_muted())); room_b.read_with(cx_b, |room, _| assert!(!room.is_muted())); - // Users A and B are both muted. + // Users A and B are both unmuted. assert_eq!( participant_audio_state(&room_a, cx_a), &[ParticipantAudioState { @@ -2075,7 +2074,7 @@ async fn test_mute_deafen( audio_tracks_playing: participant .audio_tracks .values() - .map(|track| track.is_playing()) + .map(|(track, _)| track.rtc_track().enabled()) .collect(), }) .collect::>() @@ -6058,13 +6057,13 @@ async fn test_join_call_after_screen_was_shared( assert_eq!(call_b.calling_user.github_login, "user_a"); // User A shares their screen - let display = MacOSDisplay::new(); + let display = TestScreenCaptureSource::new(); + cx_a.set_screen_capture_sources(vec![display]); active_call_a .update(cx_a, |call, cx| { - call.room().unwrap().update(cx, |room, cx| { - room.set_display_sources(vec![display.clone()]); - room.share_screen(cx) - }) + call.room() + .unwrap() + .update(cx, |room, cx| room.share_screen(cx)) }) .await .unwrap(); diff --git a/crates/collab/src/tests/test_server.rs b/crates/collab/src/tests/test_server.rs index c93cce9770e58..d897ef30ff347 100644 --- a/crates/collab/src/tests/test_server.rs +++ b/crates/collab/src/tests/test_server.rs @@ -47,7 +47,7 @@ use workspace::{Workspace, WorkspaceStore}; pub struct TestServer { pub app_state: Arc, - pub test_live_kit_server: Arc, + pub test_livekit_server: Arc, server: Arc, next_github_user_id: i32, connection_killers: Arc>>>, @@ -79,7 +79,7 @@ pub struct ContactsSummary { impl TestServer { pub async fn start(deterministic: BackgroundExecutor) -> Self { - static NEXT_LIVE_KIT_SERVER_ID: AtomicUsize = AtomicUsize::new(0); + static NEXT_LIVEKIT_SERVER_ID: AtomicUsize = AtomicUsize::new(0); let use_postgres = env::var("USE_POSTGRES").ok(); let use_postgres = use_postgres.as_deref(); @@ -88,16 +88,16 @@ impl TestServer { } else { TestDb::sqlite(deterministic.clone()) }; - let live_kit_server_id = NEXT_LIVE_KIT_SERVER_ID.fetch_add(1, SeqCst); - let live_kit_server = live_kit_client::TestServer::create( - format!("http://livekit.{}.test", live_kit_server_id), - format!("devkey-{}", live_kit_server_id), - format!("secret-{}", live_kit_server_id), + let livekit_server_id = NEXT_LIVEKIT_SERVER_ID.fetch_add(1, SeqCst); + let livekit_server = livekit_client::test::TestServer::create( + format!("http://livekit.{}.test", livekit_server_id), + format!("devkey-{}", livekit_server_id), + format!("secret-{}", livekit_server_id), deterministic.clone(), ) .unwrap(); let executor = Executor::Deterministic(deterministic.clone()); - let app_state = Self::build_app_state(&test_db, &live_kit_server, executor.clone()).await; + let app_state = Self::build_app_state(&test_db, &livekit_server, executor.clone()).await; let epoch = app_state .db .create_server(&app_state.config.zed_environment) @@ -114,7 +114,7 @@ impl TestServer { forbid_connections: Default::default(), next_github_user_id: 0, _test_db: test_db, - test_live_kit_server: live_kit_server, + test_livekit_server: livekit_server, } } @@ -500,13 +500,13 @@ impl TestServer { pub async fn build_app_state( test_db: &TestDb, - live_kit_test_server: &live_kit_client::TestServer, + livekit_test_server: &livekit_client::test::TestServer, executor: Executor, ) -> Arc { Arc::new(AppState { db: test_db.db().clone(), llm_db: None, - live_kit_client: Some(Arc::new(live_kit_test_server.create_api_client())), + livekit_client: Some(Arc::new(livekit_test_server.create_api_client())), blob_store_client: None, stripe_client: None, stripe_billing: None, @@ -520,9 +520,9 @@ impl TestServer { database_max_connections: 0, api_token: "".into(), invite_link_prefix: "".into(), - live_kit_server: None, - live_kit_key: None, - live_kit_secret: None, + livekit_server: None, + livekit_key: None, + livekit_secret: None, llm_database_url: None, llm_database_max_connections: None, llm_database_migrations_path: None, @@ -572,7 +572,7 @@ impl Deref for TestServer { impl Drop for TestServer { fn drop(&mut self) { self.server.teardown(); - self.test_live_kit_server.teardown().unwrap(); + self.test_livekit_server.teardown().unwrap(); } } diff --git a/crates/collab_ui/src/collab_panel.rs b/crates/collab_ui/src/collab_panel.rs index c93a48096a4b4..fa3ab0219b77d 100644 --- a/crates/collab_ui/src/collab_panel.rs +++ b/crates/collab_ui/src/collab_panel.rs @@ -474,11 +474,10 @@ impl CollabPanel { project_id: project.id, worktree_root_names: project.worktree_root_names.clone(), host_user_id: participant.user.id, - is_last: projects.peek().is_none() - && participant.video_tracks.is_empty(), + is_last: projects.peek().is_none() && !participant.has_video_tracks(), }); } - if !participant.video_tracks.is_empty() { + if participant.has_video_tracks() { self.entries.push(ListEntry::ParticipantScreen { peer_id: Some(participant.peer_id), is_last: true, diff --git a/crates/evals/build.rs b/crates/evals/build.rs index 8175d90494c84..4b2cc0d76a260 100644 --- a/crates/evals/build.rs +++ b/crates/evals/build.rs @@ -3,12 +3,5 @@ fn main() { println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=10.15.7"); println!("cargo:rerun-if-env-changed=ZED_BUNDLE"); - if std::env::var("ZED_BUNDLE").ok().as_deref() == Some("true") { - // Find WebRTC.framework in the Frameworks folder when running as part of an application bundle. - println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path/../Frameworks"); - } else { - // Find WebRTC.framework as a sibling of the executable when running outside of an application bundle. - println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path"); - } } } diff --git a/crates/gpui/build.rs b/crates/gpui/build.rs index ef29d7cc8222a..045372b73ca67 100644 --- a/crates/gpui/build.rs +++ b/crates/gpui/build.rs @@ -50,6 +50,7 @@ mod macos { fn generate_dispatch_bindings() { println!("cargo:rustc-link-lib=framework=System"); + println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); println!("cargo:rerun-if-changed=src/platform/mac/dispatch.h"); let bindings = bindgen::Builder::default() diff --git a/crates/gpui/src/app.rs b/crates/gpui/src/app.rs index 87ee3942dd44a..ca787587b917d 100644 --- a/crates/gpui/src/app.rs +++ b/crates/gpui/src/app.rs @@ -33,8 +33,8 @@ use crate::{ Entity, EventEmitter, ForegroundExecutor, Global, KeyBinding, Keymap, Keystroke, LayoutId, Menu, MenuItem, OwnedMenu, PathPromptOptions, Pixels, Platform, PlatformDisplay, Point, PromptBuilder, PromptHandle, PromptLevel, Render, RenderablePromptHandle, Reservation, - SharedString, SubscriberSet, Subscription, SvgRenderer, Task, TextSystem, View, ViewContext, - Window, WindowAppearance, WindowContext, WindowHandle, WindowId, + ScreenCaptureSource, SharedString, SubscriberSet, Subscription, SvgRenderer, Task, TextSystem, + View, ViewContext, Window, WindowAppearance, WindowContext, WindowHandle, WindowId, }; mod async_context; @@ -599,6 +599,13 @@ impl AppContext { self.platform.primary_display() } + /// Returns a list of available screen capture sources. + pub fn screen_capture_sources( + &self, + ) -> oneshot::Receiver>>> { + self.platform.screen_capture_sources() + } + /// Returns the display with the given ID, if one exists. pub fn find_display(&self, id: DisplayId) -> Option> { self.displays() diff --git a/crates/gpui/src/app/test_context.rs b/crates/gpui/src/app/test_context.rs index 2fea804301792..04ca7764c5bf2 100644 --- a/crates/gpui/src/app/test_context.rs +++ b/crates/gpui/src/app/test_context.rs @@ -4,8 +4,8 @@ use crate::{ Element, Empty, Entity, EventEmitter, ForegroundExecutor, Global, InputEvent, Keystroke, Model, ModelContext, Modifiers, ModifiersChangedEvent, MouseButton, MouseDownEvent, MouseMoveEvent, MouseUpEvent, Pixels, Platform, Point, Render, Result, Size, Task, TestDispatcher, - TestPlatform, TestWindow, TextSystem, View, ViewContext, VisualContext, WindowBounds, - WindowContext, WindowHandle, WindowOptions, + TestPlatform, TestScreenCaptureSource, TestWindow, TextSystem, View, ViewContext, + VisualContext, WindowBounds, WindowContext, WindowHandle, WindowOptions, }; use anyhow::{anyhow, bail}; use futures::{channel::oneshot, Stream, StreamExt}; @@ -287,6 +287,12 @@ impl TestAppContext { self.test_window(window_handle).simulate_resize(size); } + /// Causes the given sources to be returned if the application queries for screen + /// capture sources. + pub fn set_screen_capture_sources(&self, sources: Vec) { + self.test_platform.set_screen_capture_sources(sources); + } + /// Returns all windows open in the test. pub fn windows(&self) -> Vec { self.app.borrow().windows().clone() diff --git a/crates/gpui/src/geometry.rs b/crates/gpui/src/geometry.rs index 9e0b9b9014039..b636c95a614e6 100644 --- a/crates/gpui/src/geometry.rs +++ b/crates/gpui/src/geometry.rs @@ -704,6 +704,11 @@ pub struct Bounds { pub size: Size, } +/// Create a bounds with the given origin and size +pub fn bounds(origin: Point, size: Size) -> Bounds { + Bounds { origin, size } +} + impl Bounds { /// Generate a centered bounds for the given display or primary display if none is provided pub fn centered(display_id: Option, size: Size, cx: &AppContext) -> Self { diff --git a/crates/gpui/src/platform.rs b/crates/gpui/src/platform.rs index 8228d44bb4f49..f3ffa323d8981 100644 --- a/crates/gpui/src/platform.rs +++ b/crates/gpui/src/platform.rs @@ -71,6 +71,9 @@ pub(crate) use test::*; #[cfg(target_os = "windows")] pub(crate) use windows::*; +#[cfg(any(test, feature = "test-support"))] +pub use test::TestScreenCaptureSource; + #[cfg(target_os = "macos")] pub(crate) fn current_platform(headless: bool) -> Rc { Rc::new(MacPlatform::new(headless)) @@ -150,6 +153,10 @@ pub(crate) trait Platform: 'static { None } + fn screen_capture_sources( + &self, + ) -> oneshot::Receiver>>>; + fn open_window( &self, handle: AnyWindowHandle, @@ -229,6 +236,25 @@ pub trait PlatformDisplay: Send + Sync + Debug { } } +/// A source of on-screen video content that can be captured. +pub trait ScreenCaptureSource { + /// Returns the video resolution of this source. + fn resolution(&self) -> Result>; + + /// Start capture video from this source, invoking the given callback + /// with each frame. + fn stream( + &self, + frame_callback: Box, + ) -> oneshot::Receiver>>; +} + +/// A video stream captured from a screen. +pub trait ScreenCaptureStream {} + +/// A frame of video captured from a screen. +pub struct ScreenCaptureFrame(pub PlatformScreenCaptureFrame); + /// An opaque identifier for a hardware display #[derive(PartialEq, Eq, Hash, Copy, Clone)] pub struct DisplayId(pub(crate) u32); diff --git a/crates/gpui/src/platform/linux.rs b/crates/gpui/src/platform/linux.rs index 04998693616fd..089b52cf1e730 100644 --- a/crates/gpui/src/platform/linux.rs +++ b/crates/gpui/src/platform/linux.rs @@ -20,3 +20,5 @@ pub(crate) use text_system::*; pub(crate) use wayland::*; #[cfg(feature = "x11")] pub(crate) use x11::*; + +pub(crate) type PlatformScreenCaptureFrame = (); diff --git a/crates/gpui/src/platform/linux/platform.rs b/crates/gpui/src/platform/linux/platform.rs index 650ed70af8c6d..a85052a4f0bcd 100644 --- a/crates/gpui/src/platform/linux/platform.rs +++ b/crates/gpui/src/platform/linux/platform.rs @@ -35,8 +35,8 @@ use crate::{ px, Action, AnyWindowHandle, BackgroundExecutor, ClipboardItem, CursorStyle, DisplayId, ForegroundExecutor, Keymap, Keystroke, LinuxDispatcher, Menu, MenuItem, Modifiers, OwnedMenu, PathPromptOptions, Pixels, Platform, PlatformDisplay, PlatformInputHandler, PlatformTextSystem, - PlatformWindow, Point, PromptLevel, Result, SemanticVersion, SharedString, Size, Task, - WindowAppearance, WindowOptions, WindowParams, + PlatformWindow, Point, PromptLevel, Result, ScreenCaptureSource, SemanticVersion, SharedString, + Size, Task, WindowAppearance, WindowOptions, WindowParams, }; pub(crate) const SCROLL_LINES: f32 = 3.0; @@ -242,6 +242,14 @@ impl Platform for P { self.displays() } + fn screen_capture_sources( + &self, + ) -> oneshot::Receiver>>> { + let (mut tx, rx) = oneshot::channel(); + tx.send(Err(anyhow!("screen capture not implemented"))).ok(); + rx + } + fn active_window(&self) -> Option { self.active_window() } diff --git a/crates/gpui/src/platform/mac.rs b/crates/gpui/src/platform/mac.rs index 396fd49d04896..bd3d8f35ac9d1 100644 --- a/crates/gpui/src/platform/mac.rs +++ b/crates/gpui/src/platform/mac.rs @@ -4,12 +4,14 @@ mod dispatcher; mod display; mod display_link; mod events; +mod screen_capture; #[cfg(not(feature = "macos-blade"))] mod metal_atlas; #[cfg(not(feature = "macos-blade"))] pub mod metal_renderer; +use media::core_video::CVImageBuffer; #[cfg(not(feature = "macos-blade"))] use metal_renderer as renderer; @@ -49,6 +51,9 @@ pub(crate) use window::*; #[cfg(feature = "font-kit")] pub(crate) use text_system::*; +/// A frame of video captured from a screen. +pub(crate) type PlatformScreenCaptureFrame = CVImageBuffer; + trait BoolExt { fn to_objc(self) -> BOOL; } diff --git a/crates/gpui/src/platform/mac/platform.rs b/crates/gpui/src/platform/mac/platform.rs index 28f427af1b8a8..f0fe560ca479e 100644 --- a/crates/gpui/src/platform/mac/platform.rs +++ b/crates/gpui/src/platform/mac/platform.rs @@ -1,14 +1,14 @@ use super::{ attributed_string::{NSAttributedString, NSMutableAttributedString}, events::key_to_native, - BoolExt, + renderer, screen_capture, BoolExt, }; use crate::{ hash, Action, AnyWindowHandle, BackgroundExecutor, ClipboardEntry, ClipboardItem, ClipboardString, CursorStyle, ForegroundExecutor, Image, ImageFormat, Keymap, MacDispatcher, MacDisplay, MacWindow, Menu, MenuItem, PathPromptOptions, Platform, PlatformDisplay, - PlatformTextSystem, PlatformWindow, Result, SemanticVersion, Task, WindowAppearance, - WindowParams, + PlatformTextSystem, PlatformWindow, Result, ScreenCaptureSource, SemanticVersion, Task, + WindowAppearance, WindowParams, }; use anyhow::anyhow; use block::ConcreteBlock; @@ -58,8 +58,6 @@ use std::{ }; use strum::IntoEnumIterator; -use super::renderer; - #[allow(non_upper_case_globals)] const NSUTF8StringEncoding: NSUInteger = 4; @@ -552,6 +550,12 @@ impl Platform for MacPlatform { .collect() } + fn screen_capture_sources( + &self, + ) -> oneshot::Receiver>>> { + screen_capture::get_sources() + } + fn active_window(&self) -> Option { MacWindow::active_window() } diff --git a/crates/gpui/src/platform/mac/screen_capture.rs b/crates/gpui/src/platform/mac/screen_capture.rs new file mode 100644 index 0000000000000..a2b535996fa1a --- /dev/null +++ b/crates/gpui/src/platform/mac/screen_capture.rs @@ -0,0 +1,239 @@ +use crate::{ + platform::{ScreenCaptureFrame, ScreenCaptureSource, ScreenCaptureStream}, + px, size, Pixels, Size, +}; +use anyhow::{anyhow, Result}; +use block::ConcreteBlock; +use cocoa::{ + base::{id, nil, YES}, + foundation::NSArray, +}; +use core_foundation::base::TCFType; +use ctor::ctor; +use futures::channel::oneshot; +use media::core_media::{CMSampleBuffer, CMSampleBufferRef}; +use metal::NSInteger; +use objc::{ + class, + declare::ClassDecl, + msg_send, + runtime::{Class, Object, Sel}, + sel, sel_impl, +}; +use std::{cell::RefCell, ffi::c_void, mem, ptr, rc::Rc}; + +#[derive(Clone)] +pub struct MacScreenCaptureSource { + sc_display: id, +} + +pub struct MacScreenCaptureStream { + sc_stream: id, + sc_stream_output: id, +} + +#[link(name = "ScreenCaptureKit", kind = "framework")] +extern "C" {} + +static mut DELEGATE_CLASS: *const Class = ptr::null(); +static mut OUTPUT_CLASS: *const Class = ptr::null(); +const FRAME_CALLBACK_IVAR: &str = "frame_callback"; + +#[allow(non_upper_case_globals)] +const SCStreamOutputTypeScreen: NSInteger = 0; + +impl ScreenCaptureSource for MacScreenCaptureSource { + fn resolution(&self) -> Result> { + unsafe { + let width: i64 = msg_send![self.sc_display, width]; + let height: i64 = msg_send![self.sc_display, height]; + Ok(size(px(width as f32), px(height as f32))) + } + } + + fn stream( + &self, + frame_callback: Box, + ) -> oneshot::Receiver>> { + unsafe { + let stream: id = msg_send![class!(SCStream), alloc]; + let filter: id = msg_send![class!(SCContentFilter), alloc]; + let configuration: id = msg_send![class!(SCStreamConfiguration), alloc]; + let delegate: id = msg_send![DELEGATE_CLASS, alloc]; + let output: id = msg_send![OUTPUT_CLASS, alloc]; + + let excluded_windows = NSArray::array(nil); + let filter: id = msg_send![filter, initWithDisplay:self.sc_display excludingWindows:excluded_windows]; + let configuration: id = msg_send![configuration, init]; + let delegate: id = msg_send![delegate, init]; + let output: id = msg_send![output, init]; + + output.as_mut().unwrap().set_ivar( + FRAME_CALLBACK_IVAR, + Box::into_raw(Box::new(frame_callback)) as *mut c_void, + ); + + let stream: id = msg_send![stream, initWithFilter:filter configuration:configuration delegate:delegate]; + + let (mut tx, rx) = oneshot::channel(); + + let mut error: id = nil; + let _: () = msg_send![stream, addStreamOutput:output type:SCStreamOutputTypeScreen sampleHandlerQueue:0 error:&mut error as *mut id]; + if error != nil { + let message: id = msg_send![error, localizedDescription]; + tx.send(Err(anyhow!("failed to add stream output {message:?}"))) + .ok(); + return rx; + } + + let tx = Rc::new(RefCell::new(Some(tx))); + let handler = ConcreteBlock::new({ + move |error: id| { + let result = if error == nil { + let stream = MacScreenCaptureStream { + sc_stream: stream, + sc_stream_output: output, + }; + Ok(Box::new(stream) as Box) + } else { + let message: id = msg_send![error, localizedDescription]; + Err(anyhow!("failed to stop screen capture stream {message:?}")) + }; + if let Some(tx) = tx.borrow_mut().take() { + tx.send(result).ok(); + } + } + }); + let handler = handler.copy(); + let _: () = msg_send![stream, startCaptureWithCompletionHandler:handler]; + rx + } + } +} + +impl Drop for MacScreenCaptureSource { + fn drop(&mut self) { + unsafe { + let _: () = msg_send![self.sc_display, release]; + } + } +} + +impl ScreenCaptureStream for MacScreenCaptureStream {} + +impl Drop for MacScreenCaptureStream { + fn drop(&mut self) { + unsafe { + let mut error: id = nil; + let _: () = msg_send![self.sc_stream, removeStreamOutput:self.sc_stream_output type:SCStreamOutputTypeScreen error:&mut error as *mut _]; + if error != nil { + let message: id = msg_send![error, localizedDescription]; + log::error!("failed to add stream output {message:?}"); + } + + let handler = ConcreteBlock::new(move |error: id| { + if error != nil { + let message: id = msg_send![error, localizedDescription]; + log::error!("failed to stop screen capture stream {message:?}"); + } + }); + let block = handler.copy(); + let _: () = msg_send![self.sc_stream, stopCaptureWithCompletionHandler:block]; + let _: () = msg_send![self.sc_stream, release]; + let _: () = msg_send![self.sc_stream_output, release]; + } + } +} + +pub(crate) fn get_sources() -> oneshot::Receiver>>> { + unsafe { + let (mut tx, rx) = oneshot::channel(); + let tx = Rc::new(RefCell::new(Some(tx))); + + let block = ConcreteBlock::new(move |shareable_content: id, error: id| { + let Some(mut tx) = tx.borrow_mut().take() else { + return; + }; + let result = if error == nil { + let displays: id = msg_send![shareable_content, displays]; + let mut result = Vec::new(); + for i in 0..displays.count() { + let display = displays.objectAtIndex(i); + let source = MacScreenCaptureSource { + sc_display: msg_send![display, retain], + }; + result.push(Box::new(source) as Box); + } + Ok(result) + } else { + let msg: id = msg_send![error, localizedDescription]; + Err(anyhow!("Failed to register: {:?}", msg)) + }; + tx.send(result).ok(); + }); + let block = block.copy(); + + let _: () = msg_send![ + class!(SCShareableContent), + getShareableContentExcludingDesktopWindows:YES + onScreenWindowsOnly:YES + completionHandler:block]; + rx + } +} + +#[ctor] +unsafe fn build_classes() { + let mut decl = ClassDecl::new("GPUIStreamDelegate", class!(NSObject)).unwrap(); + decl.add_method( + sel!(outputVideoEffectDidStartForStream:), + output_video_effect_did_start_for_stream as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(outputVideoEffectDidStopForStream:), + output_video_effect_did_stop_for_stream as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(stream:didStopWithError:), + stream_did_stop_with_error as extern "C" fn(&Object, Sel, id, id), + ); + DELEGATE_CLASS = decl.register(); + + let mut decl = ClassDecl::new("GPUIStreamOutput", class!(NSObject)).unwrap(); + decl.add_method( + sel!(stream:didOutputSampleBuffer:ofType:), + stream_did_output_sample_buffer_of_type as extern "C" fn(&Object, Sel, id, id, NSInteger), + ); + decl.add_ivar::<*mut c_void>(FRAME_CALLBACK_IVAR); + + OUTPUT_CLASS = decl.register(); +} + +extern "C" fn output_video_effect_did_start_for_stream(_this: &Object, _: Sel, _stream: id) {} + +extern "C" fn output_video_effect_did_stop_for_stream(_this: &Object, _: Sel, _stream: id) {} + +extern "C" fn stream_did_stop_with_error(_this: &Object, _: Sel, _stream: id, _error: id) {} + +extern "C" fn stream_did_output_sample_buffer_of_type( + this: &Object, + _: Sel, + _stream: id, + sample_buffer: id, + buffer_type: NSInteger, +) { + if buffer_type != SCStreamOutputTypeScreen { + return; + } + + unsafe { + let sample_buffer = sample_buffer as CMSampleBufferRef; + let sample_buffer = CMSampleBuffer::wrap_under_get_rule(sample_buffer); + if let Some(buffer) = sample_buffer.image_buffer() { + let callback: Box> = + Box::from_raw(*this.get_ivar::<*mut c_void>(FRAME_CALLBACK_IVAR) as *mut _); + callback(ScreenCaptureFrame(buffer)); + mem::forget(callback); + } + } +} diff --git a/crates/gpui/src/platform/test.rs b/crates/gpui/src/platform/test.rs index d17739239eede..70462cb5e2cae 100644 --- a/crates/gpui/src/platform/test.rs +++ b/crates/gpui/src/platform/test.rs @@ -7,3 +7,5 @@ pub(crate) use dispatcher::*; pub(crate) use display::*; pub(crate) use platform::*; pub(crate) use window::*; + +pub use platform::TestScreenCaptureSource; diff --git a/crates/gpui/src/platform/test/platform.rs b/crates/gpui/src/platform/test/platform.rs index aadbe9b5953d8..67227b60fec04 100644 --- a/crates/gpui/src/platform/test/platform.rs +++ b/crates/gpui/src/platform/test/platform.rs @@ -1,7 +1,7 @@ use crate::{ - AnyWindowHandle, BackgroundExecutor, ClipboardItem, CursorStyle, ForegroundExecutor, Keymap, - Platform, PlatformDisplay, PlatformTextSystem, Task, TestDisplay, TestWindow, WindowAppearance, - WindowParams, + px, size, AnyWindowHandle, BackgroundExecutor, ClipboardItem, CursorStyle, ForegroundExecutor, + Keymap, Platform, PlatformDisplay, PlatformTextSystem, ScreenCaptureFrame, ScreenCaptureSource, + ScreenCaptureStream, Task, TestDisplay, TestWindow, WindowAppearance, WindowParams, }; use anyhow::Result; use collections::VecDeque; @@ -31,6 +31,7 @@ pub(crate) struct TestPlatform { #[cfg(any(target_os = "linux", target_os = "freebsd"))] current_primary_item: Mutex>, pub(crate) prompts: RefCell, + screen_capture_sources: RefCell>, pub opened_url: RefCell>, pub text_system: Arc, #[cfg(target_os = "windows")] @@ -38,6 +39,31 @@ pub(crate) struct TestPlatform { weak: Weak, } +#[derive(Clone)] +/// A fake screen capture source, used for testing. +pub struct TestScreenCaptureSource {} + +pub struct TestScreenCaptureStream {} + +impl ScreenCaptureSource for TestScreenCaptureSource { + fn resolution(&self) -> Result> { + Ok(size(px(1.), px(1.))) + } + + fn stream( + &self, + _frame_callback: Box, + ) -> oneshot::Receiver>> { + let (mut tx, rx) = oneshot::channel(); + let stream = TestScreenCaptureStream {}; + tx.send(Ok(Box::new(stream) as Box)) + .ok(); + rx + } +} + +impl ScreenCaptureStream for TestScreenCaptureStream {} + #[derive(Default)] pub(crate) struct TestPrompts { multiple_choice: VecDeque>, @@ -72,6 +98,7 @@ impl TestPlatform { background_executor: executor, foreground_executor, prompts: Default::default(), + screen_capture_sources: Default::default(), active_cursor: Default::default(), active_display: Rc::new(TestDisplay::new()), active_window: Default::default(), @@ -114,6 +141,10 @@ impl TestPlatform { !self.prompts.borrow().multiple_choice.is_empty() } + pub(crate) fn set_screen_capture_sources(&self, sources: Vec) { + *self.screen_capture_sources.borrow_mut() = sources; + } + pub(crate) fn prompt(&self, msg: &str, detail: Option<&str>) -> oneshot::Receiver { let (tx, rx) = oneshot::channel(); self.background_executor() @@ -202,6 +233,20 @@ impl Platform for TestPlatform { Some(self.active_display.clone()) } + fn screen_capture_sources( + &self, + ) -> oneshot::Receiver>>> { + let (mut tx, rx) = oneshot::channel(); + tx.send(Ok(self + .screen_capture_sources + .borrow() + .iter() + .map(|source| Box::new(source.clone()) as Box) + .collect())) + .ok(); + rx + } + fn active_window(&self) -> Option { self.active_window .borrow() @@ -330,6 +375,13 @@ impl Platform for TestPlatform { } } +impl TestScreenCaptureSource { + /// Create a fake screen capture source, for testing. + pub fn new() -> Self { + Self {} + } +} + #[cfg(target_os = "windows")] impl Drop for TestPlatform { fn drop(&mut self) { diff --git a/crates/gpui/src/platform/windows.rs b/crates/gpui/src/platform/windows.rs index 84cf107c70516..51d09f0013f96 100644 --- a/crates/gpui/src/platform/windows.rs +++ b/crates/gpui/src/platform/windows.rs @@ -21,3 +21,5 @@ pub(crate) use window::*; pub(crate) use wrapper::*; pub(crate) use windows::Win32::Foundation::HWND; + +pub(crate) type PlatformScreenCaptureFrame = (); diff --git a/crates/gpui/src/platform/windows/platform.rs b/crates/gpui/src/platform/windows/platform.rs index 389b90765df26..0c23a4ef7ad14 100644 --- a/crates/gpui/src/platform/windows/platform.rs +++ b/crates/gpui/src/platform/windows/platform.rs @@ -325,6 +325,14 @@ impl Platform for WindowsPlatform { WindowsDisplay::primary_monitor().map(|display| Rc::new(display) as Rc) } + fn screen_capture_sources( + &self, + ) -> oneshot::Receiver>>> { + let (mut tx, rx) = oneshot::channel(); + tx.send(Err(anyhow!("screen capture not implemented"))).ok(); + rx + } + fn active_window(&self) -> Option { let active_window_hwnd = unsafe { GetActiveWindow() }; self.try_get_windows_inner_from_hwnd(active_window_hwnd) diff --git a/crates/http_client/Cargo.toml b/crates/http_client/Cargo.toml index ac8e254b84f60..a4f10cff18082 100644 --- a/crates/http_client/Cargo.toml +++ b/crates/http_client/Cargo.toml @@ -20,7 +20,7 @@ bytes.workspace = true anyhow.workspace = true derive_more.workspace = true futures.workspace = true -http = "1.1" +http.workspace = true log.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/crates/live_kit_client/LiveKitBridge/Package.resolved b/crates/live_kit_client/LiveKitBridge/Package.resolved deleted file mode 100644 index b925bc8f0d5ef..0000000000000 --- a/crates/live_kit_client/LiveKitBridge/Package.resolved +++ /dev/null @@ -1,52 +0,0 @@ -{ - "object": { - "pins": [ - { - "package": "LiveKit", - "repositoryURL": "https://github.com/livekit/client-sdk-swift.git", - "state": { - "branch": null, - "revision": "7331b813a5ab8a95cfb81fb2b4ed10519428b9ff", - "version": "1.0.12" - } - }, - { - "package": "Promises", - "repositoryURL": "https://github.com/google/promises.git", - "state": { - "branch": null, - "revision": "ec957ccddbcc710ccc64c9dcbd4c7006fcf8b73a", - "version": "2.2.0" - } - }, - { - "package": "WebRTC", - "repositoryURL": "https://github.com/webrtc-sdk/Specs.git", - "state": { - "branch": null, - "revision": "2f6bab30c8df0fe59ab3e58bc99097f757f85f65", - "version": "104.5112.17" - } - }, - { - "package": "swift-log", - "repositoryURL": "https://github.com/apple/swift-log.git", - "state": { - "branch": null, - "revision": "32e8d724467f8fe623624570367e3d50c5638e46", - "version": "1.5.2" - } - }, - { - "package": "SwiftProtobuf", - "repositoryURL": "https://github.com/apple/swift-protobuf.git", - "state": { - "branch": null, - "revision": "ce20dc083ee485524b802669890291c0d8090170", - "version": "1.22.1" - } - } - ] - }, - "version": 1 -} diff --git a/crates/live_kit_client/LiveKitBridge/Package.swift b/crates/live_kit_client/LiveKitBridge/Package.swift deleted file mode 100644 index d7b5c271b9549..0000000000000 --- a/crates/live_kit_client/LiveKitBridge/Package.swift +++ /dev/null @@ -1,27 +0,0 @@ -// swift-tools-version: 5.5 - -import PackageDescription - -let package = Package( - name: "LiveKitBridge", - platforms: [ - .macOS(.v10_15) - ], - products: [ - // Products define the executables and libraries a package produces, and make them visible to other packages. - .library( - name: "LiveKitBridge", - type: .static, - targets: ["LiveKitBridge"]), - ], - dependencies: [ - .package(url: "https://github.com/livekit/client-sdk-swift.git", .exact("1.0.12")), - ], - targets: [ - // Targets are the basic building blocks of a package. A target can define a module or a test suite. - // Targets can depend on other targets in this package, and on products in packages this package depends on. - .target( - name: "LiveKitBridge", - dependencies: [.product(name: "LiveKit", package: "client-sdk-swift")]), - ] -) diff --git a/crates/live_kit_client/LiveKitBridge/README.md b/crates/live_kit_client/LiveKitBridge/README.md deleted file mode 100644 index b982c672866a3..0000000000000 --- a/crates/live_kit_client/LiveKitBridge/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# LiveKitBridge - -A description of this package. diff --git a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift deleted file mode 100644 index 7468c08791f8b..0000000000000 --- a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ /dev/null @@ -1,383 +0,0 @@ -import Foundation -import LiveKit -import WebRTC -import ScreenCaptureKit - -class LKRoomDelegate: RoomDelegate { - var data: UnsafeRawPointer - var onDidDisconnect: @convention(c) (UnsafeRawPointer) -> Void - var onDidSubscribeToRemoteAudioTrack: @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer, UnsafeRawPointer) -> Void - var onDidUnsubscribeFromRemoteAudioTrack: @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void - var onMuteChangedFromRemoteAudioTrack: @convention(c) (UnsafeRawPointer, CFString, Bool) -> Void - var onActiveSpeakersChanged: @convention(c) (UnsafeRawPointer, CFArray) -> Void - var onDidSubscribeToRemoteVideoTrack: @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer) -> Void - var onDidUnsubscribeFromRemoteVideoTrack: @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void - var onDidPublishOrUnpublishLocalAudioTrack: @convention(c) (UnsafeRawPointer, UnsafeRawPointer, Bool) -> Void - var onDidPublishOrUnpublishLocalVideoTrack: @convention(c) (UnsafeRawPointer, UnsafeRawPointer, Bool) -> Void - - init( - data: UnsafeRawPointer, - onDidDisconnect: @escaping @convention(c) (UnsafeRawPointer) -> Void, - onDidSubscribeToRemoteAudioTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer, UnsafeRawPointer) -> Void, - onDidUnsubscribeFromRemoteAudioTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void, - onMuteChangedFromRemoteAudioTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, Bool) -> Void, - onActiveSpeakersChanged: @convention(c) (UnsafeRawPointer, CFArray) -> Void, - onDidSubscribeToRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer) -> Void, - onDidUnsubscribeFromRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void, - onDidPublishOrUnpublishLocalAudioTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer, Bool) -> Void, - onDidPublishOrUnpublishLocalVideoTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer, Bool) -> Void - ) - { - self.data = data - self.onDidDisconnect = onDidDisconnect - self.onDidSubscribeToRemoteAudioTrack = onDidSubscribeToRemoteAudioTrack - self.onDidUnsubscribeFromRemoteAudioTrack = onDidUnsubscribeFromRemoteAudioTrack - self.onDidSubscribeToRemoteVideoTrack = onDidSubscribeToRemoteVideoTrack - self.onDidUnsubscribeFromRemoteVideoTrack = onDidUnsubscribeFromRemoteVideoTrack - self.onMuteChangedFromRemoteAudioTrack = onMuteChangedFromRemoteAudioTrack - self.onActiveSpeakersChanged = onActiveSpeakersChanged - self.onDidPublishOrUnpublishLocalAudioTrack = onDidPublishOrUnpublishLocalAudioTrack - self.onDidPublishOrUnpublishLocalVideoTrack = onDidPublishOrUnpublishLocalVideoTrack - } - - func room(_ room: Room, didUpdate connectionState: ConnectionState, oldValue: ConnectionState) { - if connectionState.isDisconnected { - self.onDidDisconnect(self.data) - } - } - - func room(_ room: Room, participant: RemoteParticipant, didSubscribe publication: RemoteTrackPublication, track: Track) { - if track.kind == .video { - self.onDidSubscribeToRemoteVideoTrack(self.data, participant.identity as CFString, track.sid! as CFString, Unmanaged.passUnretained(track).toOpaque()) - } else if track.kind == .audio { - self.onDidSubscribeToRemoteAudioTrack(self.data, participant.identity as CFString, track.sid! as CFString, Unmanaged.passUnretained(track).toOpaque(), Unmanaged.passUnretained(publication).toOpaque()) - } - } - - func room(_ room: Room, participant: Participant, didUpdate publication: TrackPublication, muted: Bool) { - if publication.kind == .audio { - self.onMuteChangedFromRemoteAudioTrack(self.data, publication.sid as CFString, muted) - } - } - - func room(_ room: Room, didUpdate speakers: [Participant]) { - guard let speaker_ids = speakers.compactMap({ $0.identity as CFString }) as CFArray? else { return } - self.onActiveSpeakersChanged(self.data, speaker_ids) - } - - func room(_ room: Room, participant: RemoteParticipant, didUnsubscribe publication: RemoteTrackPublication, track: Track) { - if track.kind == .video { - self.onDidUnsubscribeFromRemoteVideoTrack(self.data, participant.identity as CFString, track.sid! as CFString) - } else if track.kind == .audio { - self.onDidUnsubscribeFromRemoteAudioTrack(self.data, participant.identity as CFString, track.sid! as CFString) - } - } - - func room(_ room: Room, localParticipant: LocalParticipant, didPublish publication: LocalTrackPublication) { - if publication.kind == .video { - self.onDidPublishOrUnpublishLocalVideoTrack(self.data, Unmanaged.passUnretained(publication).toOpaque(), true) - } else if publication.kind == .audio { - self.onDidPublishOrUnpublishLocalAudioTrack(self.data, Unmanaged.passUnretained(publication).toOpaque(), true) - } - } - - func room(_ room: Room, localParticipant: LocalParticipant, didUnpublish publication: LocalTrackPublication) { - if publication.kind == .video { - self.onDidPublishOrUnpublishLocalVideoTrack(self.data, Unmanaged.passUnretained(publication).toOpaque(), false) - } else if publication.kind == .audio { - self.onDidPublishOrUnpublishLocalAudioTrack(self.data, Unmanaged.passUnretained(publication).toOpaque(), false) - } - } -} - -class LKVideoRenderer: NSObject, VideoRenderer { - var data: UnsafeRawPointer - var onFrame: @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Bool - var onDrop: @convention(c) (UnsafeRawPointer) -> Void - var adaptiveStreamIsEnabled: Bool = false - var adaptiveStreamSize: CGSize = .zero - weak var track: VideoTrack? - - init(data: UnsafeRawPointer, onFrame: @escaping @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Bool, onDrop: @escaping @convention(c) (UnsafeRawPointer) -> Void) { - self.data = data - self.onFrame = onFrame - self.onDrop = onDrop - } - - deinit { - self.onDrop(self.data) - } - - func setSize(_ size: CGSize) { - } - - func renderFrame(_ frame: RTCVideoFrame?) { - let buffer = frame?.buffer as? RTCCVPixelBuffer - if let pixelBuffer = buffer?.pixelBuffer { - if !self.onFrame(self.data, pixelBuffer) { - DispatchQueue.main.async { - self.track?.remove(videoRenderer: self) - } - } - } - } -} - -@_cdecl("LKRoomDelegateCreate") -public func LKRoomDelegateCreate( - data: UnsafeRawPointer, - onDidDisconnect: @escaping @convention(c) (UnsafeRawPointer) -> Void, - onDidSubscribeToRemoteAudioTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer, UnsafeRawPointer) -> Void, - onDidUnsubscribeFromRemoteAudioTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void, - onMuteChangedFromRemoteAudioTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, Bool) -> Void, - onActiveSpeakerChanged: @escaping @convention(c) (UnsafeRawPointer, CFArray) -> Void, - onDidSubscribeToRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer) -> Void, - onDidUnsubscribeFromRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void, - onDidPublishOrUnpublishLocalAudioTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer, Bool) -> Void, - onDidPublishOrUnpublishLocalVideoTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer, Bool) -> Void -) -> UnsafeMutableRawPointer { - let delegate = LKRoomDelegate( - data: data, - onDidDisconnect: onDidDisconnect, - onDidSubscribeToRemoteAudioTrack: onDidSubscribeToRemoteAudioTrack, - onDidUnsubscribeFromRemoteAudioTrack: onDidUnsubscribeFromRemoteAudioTrack, - onMuteChangedFromRemoteAudioTrack: onMuteChangedFromRemoteAudioTrack, - onActiveSpeakersChanged: onActiveSpeakerChanged, - onDidSubscribeToRemoteVideoTrack: onDidSubscribeToRemoteVideoTrack, - onDidUnsubscribeFromRemoteVideoTrack: onDidUnsubscribeFromRemoteVideoTrack, - onDidPublishOrUnpublishLocalAudioTrack: onDidPublishOrUnpublishLocalAudioTrack, - onDidPublishOrUnpublishLocalVideoTrack: onDidPublishOrUnpublishLocalVideoTrack - ) - return Unmanaged.passRetained(delegate).toOpaque() -} - -@_cdecl("LKRoomCreate") -public func LKRoomCreate(delegate: UnsafeRawPointer) -> UnsafeMutableRawPointer { - let delegate = Unmanaged.fromOpaque(delegate).takeUnretainedValue() - return Unmanaged.passRetained(Room(delegate: delegate)).toOpaque() -} - -@_cdecl("LKRoomConnect") -public func LKRoomConnect(room: UnsafeRawPointer, url: CFString, token: CFString, callback: @escaping @convention(c) (UnsafeRawPointer, CFString?) -> Void, callback_data: UnsafeRawPointer) { - let room = Unmanaged.fromOpaque(room).takeUnretainedValue() - - room.connect(url as String, token as String).then { _ in - callback(callback_data, UnsafeRawPointer(nil) as! CFString?) - }.catch { error in - callback(callback_data, error.localizedDescription as CFString) - } -} - -@_cdecl("LKRoomDisconnect") -public func LKRoomDisconnect(room: UnsafeRawPointer) { - let room = Unmanaged.fromOpaque(room).takeUnretainedValue() - room.disconnect() -} - -@_cdecl("LKRoomPublishVideoTrack") -public func LKRoomPublishVideoTrack(room: UnsafeRawPointer, track: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer, UnsafeMutableRawPointer?, CFString?) -> Void, callback_data: UnsafeRawPointer) { - let room = Unmanaged.fromOpaque(room).takeUnretainedValue() - let track = Unmanaged.fromOpaque(track).takeUnretainedValue() - room.localParticipant?.publishVideoTrack(track: track).then { publication in - callback(callback_data, Unmanaged.passRetained(publication).toOpaque(), nil) - }.catch { error in - callback(callback_data, nil, error.localizedDescription as CFString) - } -} - -@_cdecl("LKRoomPublishAudioTrack") -public func LKRoomPublishAudioTrack(room: UnsafeRawPointer, track: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer, UnsafeMutableRawPointer?, CFString?) -> Void, callback_data: UnsafeRawPointer) { - let room = Unmanaged.fromOpaque(room).takeUnretainedValue() - let track = Unmanaged.fromOpaque(track).takeUnretainedValue() - room.localParticipant?.publishAudioTrack(track: track).then { publication in - callback(callback_data, Unmanaged.passRetained(publication).toOpaque(), nil) - }.catch { error in - callback(callback_data, nil, error.localizedDescription as CFString) - } -} - - -@_cdecl("LKRoomUnpublishTrack") -public func LKRoomUnpublishTrack(room: UnsafeRawPointer, publication: UnsafeRawPointer) { - let room = Unmanaged.fromOpaque(room).takeUnretainedValue() - let publication = Unmanaged.fromOpaque(publication).takeUnretainedValue() - let _ = room.localParticipant?.unpublish(publication: publication) -} - -@_cdecl("LKRoomAudioTracksForRemoteParticipant") -public func LKRoomAudioTracksForRemoteParticipant(room: UnsafeRawPointer, participantId: CFString) -> CFArray? { - let room = Unmanaged.fromOpaque(room).takeUnretainedValue() - - for (_, participant) in room.remoteParticipants { - if participant.identity == participantId as String { - return participant.audioTracks.compactMap { $0.track as? RemoteAudioTrack } as CFArray? - } - } - - return nil; -} - -@_cdecl("LKRoomAudioTrackPublicationsForRemoteParticipant") -public func LKRoomAudioTrackPublicationsForRemoteParticipant(room: UnsafeRawPointer, participantId: CFString) -> CFArray? { - let room = Unmanaged.fromOpaque(room).takeUnretainedValue() - - for (_, participant) in room.remoteParticipants { - if participant.identity == participantId as String { - return participant.audioTracks.compactMap { $0 as? RemoteTrackPublication } as CFArray? - } - } - - return nil; -} - -@_cdecl("LKRoomVideoTracksForRemoteParticipant") -public func LKRoomVideoTracksForRemoteParticipant(room: UnsafeRawPointer, participantId: CFString) -> CFArray? { - let room = Unmanaged.fromOpaque(room).takeUnretainedValue() - - for (_, participant) in room.remoteParticipants { - if participant.identity == participantId as String { - return participant.videoTracks.compactMap { $0.track as? RemoteVideoTrack } as CFArray? - } - } - - return nil; -} - -@_cdecl("LKLocalAudioTrackCreateTrack") -public func LKLocalAudioTrackCreateTrack() -> UnsafeMutableRawPointer { - let track = LocalAudioTrack.createTrack(options: AudioCaptureOptions( - echoCancellation: true, - noiseSuppression: true - )) - - return Unmanaged.passRetained(track).toOpaque() -} - - -@_cdecl("LKCreateScreenShareTrackForDisplay") -public func LKCreateScreenShareTrackForDisplay(display: UnsafeMutableRawPointer) -> UnsafeMutableRawPointer { - let display = Unmanaged.fromOpaque(display).takeUnretainedValue() - let track = LocalVideoTrack.createMacOSScreenShareTrack(source: display, preferredMethod: .legacy) - return Unmanaged.passRetained(track).toOpaque() -} - -@_cdecl("LKVideoRendererCreate") -public func LKVideoRendererCreate(data: UnsafeRawPointer, onFrame: @escaping @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Bool, onDrop: @escaping @convention(c) (UnsafeRawPointer) -> Void) -> UnsafeMutableRawPointer { - Unmanaged.passRetained(LKVideoRenderer(data: data, onFrame: onFrame, onDrop: onDrop)).toOpaque() -} - -@_cdecl("LKVideoTrackAddRenderer") -public func LKVideoTrackAddRenderer(track: UnsafeRawPointer, renderer: UnsafeRawPointer) { - let track = Unmanaged.fromOpaque(track).takeUnretainedValue() as! VideoTrack - let renderer = Unmanaged.fromOpaque(renderer).takeRetainedValue() - renderer.track = track - track.add(videoRenderer: renderer) -} - -@_cdecl("LKRemoteVideoTrackGetSid") -public func LKRemoteVideoTrackGetSid(track: UnsafeRawPointer) -> CFString { - let track = Unmanaged.fromOpaque(track).takeUnretainedValue() - return track.sid! as CFString -} - -@_cdecl("LKRemoteAudioTrackGetSid") -public func LKRemoteAudioTrackGetSid(track: UnsafeRawPointer) -> CFString { - let track = Unmanaged.fromOpaque(track).takeUnretainedValue() - return track.sid! as CFString -} - -@_cdecl("LKRemoteAudioTrackStart") -public func LKRemoteAudioTrackStart(track: UnsafeRawPointer) { - let track = Unmanaged.fromOpaque(track).takeUnretainedValue() - track.start() -} - -@_cdecl("LKRemoteAudioTrackStop") -public func LKRemoteAudioTrackStop(track: UnsafeRawPointer) { - let track = Unmanaged.fromOpaque(track).takeUnretainedValue() - track.stop() -} - -@_cdecl("LKDisplaySources") -public func LKDisplaySources(data: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer, CFArray?, CFString?) -> Void) { - MacOSScreenCapturer.sources(for: .display, includeCurrentApplication: false, preferredMethod: .legacy).then { displaySources in - callback(data, displaySources as CFArray, nil) - }.catch { error in - callback(data, nil, error.localizedDescription as CFString) - } -} - -@_cdecl("LKLocalTrackPublicationSetMute") -public func LKLocalTrackPublicationSetMute( - publication: UnsafeRawPointer, - muted: Bool, - on_complete: @escaping @convention(c) (UnsafeRawPointer, CFString?) -> Void, - callback_data: UnsafeRawPointer -) { - let publication = Unmanaged.fromOpaque(publication).takeUnretainedValue() - - if muted { - publication.mute().then { - on_complete(callback_data, nil) - }.catch { error in - on_complete(callback_data, error.localizedDescription as CFString) - } - } else { - publication.unmute().then { - on_complete(callback_data, nil) - }.catch { error in - on_complete(callback_data, error.localizedDescription as CFString) - } - } -} - -@_cdecl("LKLocalTrackPublicationIsMuted") -public func LKLocalTrackPublicationIsMuted( - publication: UnsafeRawPointer -) -> Bool { - let publication = Unmanaged.fromOpaque(publication).takeUnretainedValue() - return publication.muted -} - -@_cdecl("LKRemoteTrackPublicationSetEnabled") -public func LKRemoteTrackPublicationSetEnabled( - publication: UnsafeRawPointer, - enabled: Bool, - on_complete: @escaping @convention(c) (UnsafeRawPointer, CFString?) -> Void, - callback_data: UnsafeRawPointer -) { - let publication = Unmanaged.fromOpaque(publication).takeUnretainedValue() - - publication.set(enabled: enabled).then { - on_complete(callback_data, nil) - }.catch { error in - on_complete(callback_data, error.localizedDescription as CFString) - } -} - -@_cdecl("LKRemoteTrackPublicationIsMuted") -public func LKRemoteTrackPublicationIsMuted( - publication: UnsafeRawPointer -) -> Bool { - let publication = Unmanaged.fromOpaque(publication).takeUnretainedValue() - - return publication.muted -} - -@_cdecl("LKRemoteTrackPublicationGetSid") -public func LKRemoteTrackPublicationGetSid( - publication: UnsafeRawPointer -) -> CFString { - let publication = Unmanaged.fromOpaque(publication).takeUnretainedValue() - - return publication.sid as CFString -} - -@_cdecl("LKLocalTrackPublicationGetSid") -public func LKLocalTrackPublicationGetSid( - publication: UnsafeRawPointer -) -> CFString { - let publication = Unmanaged.fromOpaque(publication).takeUnretainedValue() - - return publication.sid as CFString -} diff --git a/crates/live_kit_client/build.rs b/crates/live_kit_client/build.rs deleted file mode 100644 index 2fdfd982bf9c1..0000000000000 --- a/crates/live_kit_client/build.rs +++ /dev/null @@ -1,185 +0,0 @@ -use serde::Deserialize; -use std::{ - env, - path::{Path, PathBuf}, - process::Command, -}; - -const SWIFT_PACKAGE_NAME: &str = "LiveKitBridge"; - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SwiftTargetInfo { - pub triple: String, - pub unversioned_triple: String, - pub module_triple: String, - pub swift_runtime_compatibility_version: String, - #[serde(rename = "librariesRequireRPath")] - pub libraries_require_rpath: bool, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SwiftPaths { - pub runtime_library_paths: Vec, - pub runtime_library_import_paths: Vec, - pub runtime_resource_path: String, -} - -#[derive(Debug, Deserialize)] -pub struct SwiftTarget { - pub target: SwiftTargetInfo, - pub paths: SwiftPaths, -} - -const MACOS_TARGET_VERSION: &str = "10.15.7"; - -fn main() { - if cfg!(all( - target_os = "macos", - not(any(test, feature = "test-support", feature = "no-webrtc")), - )) { - let swift_target = get_swift_target(); - - build_bridge(&swift_target); - link_swift_stdlib(&swift_target); - link_webrtc_framework(&swift_target); - - // Register exported Objective-C selectors, protocols, etc when building example binaries. - println!("cargo:rustc-link-arg=-Wl,-ObjC"); - } -} - -fn build_bridge(swift_target: &SwiftTarget) { - println!("cargo:rerun-if-env-changed=MACOSX_DEPLOYMENT_TARGET"); - println!("cargo:rerun-if-changed={}/Sources", SWIFT_PACKAGE_NAME); - println!( - "cargo:rerun-if-changed={}/Package.swift", - SWIFT_PACKAGE_NAME - ); - println!( - "cargo:rerun-if-changed={}/Package.resolved", - SWIFT_PACKAGE_NAME - ); - - let swift_package_root = swift_package_root(); - let swift_target_folder = swift_target_folder(); - let swift_cache_folder = swift_cache_folder(); - if !Command::new("swift") - .arg("build") - .arg("--disable-automatic-resolution") - .args(["--configuration", &env::var("PROFILE").unwrap()]) - .args(["--triple", &swift_target.target.triple]) - .args(["--build-path".into(), swift_target_folder]) - .args(["--cache-path".into(), swift_cache_folder]) - .current_dir(&swift_package_root) - .status() - .unwrap() - .success() - { - panic!( - "Failed to compile swift package in {}", - swift_package_root.display() - ); - } - - println!( - "cargo:rustc-link-search=native={}", - swift_target.out_dir_path().display() - ); - println!("cargo:rustc-link-lib=static={}", SWIFT_PACKAGE_NAME); -} - -fn link_swift_stdlib(swift_target: &SwiftTarget) { - for path in &swift_target.paths.runtime_library_paths { - println!("cargo:rustc-link-search=native={}", path); - } -} - -fn link_webrtc_framework(swift_target: &SwiftTarget) { - let swift_out_dir_path = swift_target.out_dir_path(); - println!("cargo:rustc-link-lib=framework=WebRTC"); - println!( - "cargo:rustc-link-search=framework={}", - swift_out_dir_path.display() - ); - // Find WebRTC.framework as a sibling of the executable when running tests. - println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path"); - // Find WebRTC.framework in parent directory of the executable when running examples. - println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path/.."); - - let source_path = swift_out_dir_path.join("WebRTC.framework"); - let deps_dir_path = - PathBuf::from(env::var("OUT_DIR").unwrap()).join("../../../deps/WebRTC.framework"); - let target_dir_path = - PathBuf::from(env::var("OUT_DIR").unwrap()).join("../../../WebRTC.framework"); - copy_dir(&source_path, &deps_dir_path); - copy_dir(&source_path, &target_dir_path); -} - -fn get_swift_target() -> SwiftTarget { - let mut arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap(); - if arch == "aarch64" { - arch = "arm64".into(); - } - let target = format!("{}-apple-macosx{}", arch, MACOS_TARGET_VERSION); - - let swift_target_info_str = Command::new("swift") - .args(["-target", &target, "-print-target-info"]) - .output() - .unwrap() - .stdout; - - serde_json::from_slice(&swift_target_info_str).unwrap() -} - -fn swift_package_root() -> PathBuf { - env::current_dir().unwrap().join(SWIFT_PACKAGE_NAME) -} - -fn swift_target_folder() -> PathBuf { - let target = env::var("TARGET").unwrap(); - env::current_dir() - .unwrap() - .join(format!("../../target/{target}/{SWIFT_PACKAGE_NAME}_target")) -} - -fn swift_cache_folder() -> PathBuf { - let target = env::var("TARGET").unwrap(); - env::current_dir() - .unwrap() - .join(format!("../../target/{target}/{SWIFT_PACKAGE_NAME}_cache")) -} - -fn copy_dir(source: &Path, destination: &Path) { - assert!( - Command::new("rm") - .arg("-rf") - .arg(destination) - .status() - .unwrap() - .success(), - "could not remove {:?} before copying", - destination - ); - - assert!( - Command::new("cp") - .arg("-R") - .args([source, destination]) - .status() - .unwrap() - .success(), - "could not copy {:?} to {:?}", - source, - destination - ); -} - -impl SwiftTarget { - fn out_dir_path(&self) -> PathBuf { - swift_target_folder() - .join(&self.target.unversioned_triple) - .join(env::var("PROFILE").unwrap()) - } -} diff --git a/crates/live_kit_client/examples/test_app.rs b/crates/live_kit_client/examples/test_app.rs deleted file mode 100644 index de8be97e86c59..0000000000000 --- a/crates/live_kit_client/examples/test_app.rs +++ /dev/null @@ -1,172 +0,0 @@ -use std::time::Duration; - -use futures::StreamExt; -use gpui::{actions, KeyBinding, Menu, MenuItem}; -use live_kit_client::{LocalAudioTrack, LocalVideoTrack, Room, RoomUpdate}; -use live_kit_server::token::{self, VideoGrant}; -use log::LevelFilter; -use simplelog::SimpleLogger; - -actions!(live_kit_client, [Quit]); - -fn main() { - SimpleLogger::init(LevelFilter::Info, Default::default()).expect("could not initialize logger"); - - gpui::App::new().run(|cx| { - #[cfg(any(test, feature = "test-support"))] - println!("USING TEST LIVEKIT"); - - #[cfg(not(any(test, feature = "test-support")))] - println!("USING REAL LIVEKIT"); - - cx.activate(true); - - cx.on_action(quit); - cx.bind_keys([KeyBinding::new("cmd-q", Quit, None)]); - - cx.set_menus(vec![Menu { - name: "Zed".into(), - items: vec![MenuItem::Action { - name: "Quit".into(), - action: Box::new(Quit), - os_action: None, - }], - }]); - - let live_kit_url = std::env::var("LIVE_KIT_URL").unwrap_or("http://localhost:7880".into()); - let live_kit_key = std::env::var("LIVE_KIT_KEY").unwrap_or("devkey".into()); - let live_kit_secret = std::env::var("LIVE_KIT_SECRET").unwrap_or("secret".into()); - - cx.spawn(|cx| async move { - let user_a_token = token::create( - &live_kit_key, - &live_kit_secret, - Some("test-participant-1"), - VideoGrant::to_join("test-room"), - ) - .unwrap(); - let room_a = Room::new(); - room_a.connect(&live_kit_url, &user_a_token).await.unwrap(); - - let user2_token = token::create( - &live_kit_key, - &live_kit_secret, - Some("test-participant-2"), - VideoGrant::to_join("test-room"), - ) - .unwrap(); - let room_b = Room::new(); - room_b.connect(&live_kit_url, &user2_token).await.unwrap(); - - let mut room_updates = room_b.updates(); - let audio_track = LocalAudioTrack::create(); - let audio_track_publication = room_a.publish_audio_track(audio_track).await.unwrap(); - - if let RoomUpdate::SubscribedToRemoteAudioTrack(track, _) = - room_updates.next().await.unwrap() - { - let remote_tracks = room_b.remote_audio_tracks("test-participant-1"); - assert_eq!(remote_tracks.len(), 1); - assert_eq!(remote_tracks[0].publisher_id(), "test-participant-1"); - assert_eq!(track.publisher_id(), "test-participant-1"); - } else { - panic!("unexpected message"); - } - - audio_track_publication.set_mute(true).await.unwrap(); - - println!("waiting for mute changed!"); - if let RoomUpdate::RemoteAudioTrackMuteChanged { track_id, muted } = - room_updates.next().await.unwrap() - { - let remote_tracks = room_b.remote_audio_tracks("test-participant-1"); - assert_eq!(remote_tracks[0].sid(), track_id); - assert!(muted); - } else { - panic!("unexpected message"); - } - - audio_track_publication.set_mute(false).await.unwrap(); - - if let RoomUpdate::RemoteAudioTrackMuteChanged { track_id, muted } = - room_updates.next().await.unwrap() - { - let remote_tracks = room_b.remote_audio_tracks("test-participant-1"); - assert_eq!(remote_tracks[0].sid(), track_id); - assert!(!muted); - } else { - panic!("unexpected message"); - } - - println!("Pausing for 5 seconds to test audio, make some noise!"); - let timer = cx.background_executor().timer(Duration::from_secs(5)); - timer.await; - let remote_audio_track = room_b - .remote_audio_tracks("test-participant-1") - .pop() - .unwrap(); - room_a.unpublish_track(audio_track_publication); - - // Clear out any active speakers changed messages - let mut next = room_updates.next().await.unwrap(); - while let RoomUpdate::ActiveSpeakersChanged { speakers } = next { - println!("Speakers changed: {:?}", speakers); - next = room_updates.next().await.unwrap(); - } - - if let RoomUpdate::UnsubscribedFromRemoteAudioTrack { - publisher_id, - track_id, - } = next - { - assert_eq!(publisher_id, "test-participant-1"); - assert_eq!(remote_audio_track.sid(), track_id); - assert_eq!(room_b.remote_audio_tracks("test-participant-1").len(), 0); - } else { - panic!("unexpected message"); - } - - let displays = room_a.display_sources().await.unwrap(); - let display = displays.into_iter().next().unwrap(); - - let local_video_track = LocalVideoTrack::screen_share_for_display(&display); - let local_video_track_publication = - room_a.publish_video_track(local_video_track).await.unwrap(); - - if let RoomUpdate::SubscribedToRemoteVideoTrack(track) = - room_updates.next().await.unwrap() - { - let remote_video_tracks = room_b.remote_video_tracks("test-participant-1"); - assert_eq!(remote_video_tracks.len(), 1); - assert_eq!(remote_video_tracks[0].publisher_id(), "test-participant-1"); - assert_eq!(track.publisher_id(), "test-participant-1"); - } else { - panic!("unexpected message"); - } - - let remote_video_track = room_b - .remote_video_tracks("test-participant-1") - .pop() - .unwrap(); - room_a.unpublish_track(local_video_track_publication); - if let RoomUpdate::UnsubscribedFromRemoteVideoTrack { - publisher_id, - track_id, - } = room_updates.next().await.unwrap() - { - assert_eq!(publisher_id, "test-participant-1"); - assert_eq!(remote_video_track.sid(), track_id); - assert_eq!(room_b.remote_video_tracks("test-participant-1").len(), 0); - } else { - panic!("unexpected message"); - } - - cx.update(|cx| cx.shutdown()).ok(); - }) - .detach(); - }); -} - -fn quit(_: &Quit, cx: &mut gpui::AppContext) { - cx.quit(); -} diff --git a/crates/live_kit_client/src/live_kit_client.rs b/crates/live_kit_client/src/live_kit_client.rs deleted file mode 100644 index 4820a4eedb854..0000000000000 --- a/crates/live_kit_client/src/live_kit_client.rs +++ /dev/null @@ -1,37 +0,0 @@ -#![allow(clippy::arc_with_non_send_sync)] - -use std::sync::Arc; - -#[cfg(all(target_os = "macos", not(any(test, feature = "test-support"))))] -pub mod prod; - -#[cfg(all(target_os = "macos", not(any(test, feature = "test-support"))))] -pub use prod::*; - -#[cfg(any(test, feature = "test-support", not(target_os = "macos")))] -pub mod test; - -#[cfg(any(test, feature = "test-support", not(target_os = "macos")))] -pub use test::*; - -pub type Sid = String; - -#[derive(Clone, Eq, PartialEq)] -pub enum ConnectionState { - Disconnected, - Connected { url: String, token: String }, -} - -#[derive(Clone)] -pub enum RoomUpdate { - ActiveSpeakersChanged { speakers: Vec }, - RemoteAudioTrackMuteChanged { track_id: Sid, muted: bool }, - SubscribedToRemoteVideoTrack(Arc), - SubscribedToRemoteAudioTrack(Arc, Arc), - UnsubscribedFromRemoteVideoTrack { publisher_id: Sid, track_id: Sid }, - UnsubscribedFromRemoteAudioTrack { publisher_id: Sid, track_id: Sid }, - LocalAudioTrackPublished { publication: LocalTrackPublication }, - LocalAudioTrackUnpublished { publication: LocalTrackPublication }, - LocalVideoTrackPublished { publication: LocalTrackPublication }, - LocalVideoTrackUnpublished { publication: LocalTrackPublication }, -} diff --git a/crates/live_kit_client/src/prod.rs b/crates/live_kit_client/src/prod.rs deleted file mode 100644 index f7452da71036d..0000000000000 --- a/crates/live_kit_client/src/prod.rs +++ /dev/null @@ -1,981 +0,0 @@ -use crate::{ConnectionState, RoomUpdate, Sid}; -use anyhow::{anyhow, Context, Result}; -use core_foundation::{ - array::{CFArray, CFArrayRef}, - base::{CFRelease, CFRetain, TCFType}, - string::{CFString, CFStringRef}, -}; -use futures::{ - channel::{mpsc, oneshot}, - Future, -}; -pub use media::core_video::CVImageBuffer; -use media::core_video::CVImageBufferRef; -use parking_lot::Mutex; -use postage::watch; -use std::{ - ffi::c_void, - sync::{Arc, Weak}, -}; - -macro_rules! pointer_type { - ($pointer_name:ident) => { - #[repr(transparent)] - #[derive(Copy, Clone, Debug)] - pub struct $pointer_name(pub *const std::ffi::c_void); - unsafe impl Send for $pointer_name {} - }; -} - -mod swift { - pointer_type!(Room); - pointer_type!(LocalAudioTrack); - pointer_type!(RemoteAudioTrack); - pointer_type!(LocalVideoTrack); - pointer_type!(RemoteVideoTrack); - pointer_type!(LocalTrackPublication); - pointer_type!(RemoteTrackPublication); - pointer_type!(MacOSDisplay); - pointer_type!(RoomDelegate); -} - -extern "C" { - fn LKRoomDelegateCreate( - callback_data: *mut c_void, - on_did_disconnect: extern "C" fn(callback_data: *mut c_void), - on_did_subscribe_to_remote_audio_track: extern "C" fn( - callback_data: *mut c_void, - publisher_id: CFStringRef, - track_id: CFStringRef, - remote_track: swift::RemoteAudioTrack, - remote_publication: swift::RemoteTrackPublication, - ), - on_did_unsubscribe_from_remote_audio_track: extern "C" fn( - callback_data: *mut c_void, - publisher_id: CFStringRef, - track_id: CFStringRef, - ), - on_mute_changed_from_remote_audio_track: extern "C" fn( - callback_data: *mut c_void, - track_id: CFStringRef, - muted: bool, - ), - on_active_speakers_changed: extern "C" fn( - callback_data: *mut c_void, - participants: CFArrayRef, - ), - on_did_subscribe_to_remote_video_track: extern "C" fn( - callback_data: *mut c_void, - publisher_id: CFStringRef, - track_id: CFStringRef, - remote_track: swift::RemoteVideoTrack, - ), - on_did_unsubscribe_from_remote_video_track: extern "C" fn( - callback_data: *mut c_void, - publisher_id: CFStringRef, - track_id: CFStringRef, - ), - on_did_publish_or_unpublish_local_audio_track: extern "C" fn( - callback_data: *mut c_void, - publication: swift::LocalTrackPublication, - is_published: bool, - ), - on_did_publish_or_unpublish_local_video_track: extern "C" fn( - callback_data: *mut c_void, - publication: swift::LocalTrackPublication, - is_published: bool, - ), - ) -> swift::RoomDelegate; - - fn LKRoomCreate(delegate: swift::RoomDelegate) -> swift::Room; - fn LKRoomConnect( - room: swift::Room, - url: CFStringRef, - token: CFStringRef, - callback: extern "C" fn(*mut c_void, CFStringRef), - callback_data: *mut c_void, - ); - fn LKRoomDisconnect(room: swift::Room); - fn LKRoomPublishVideoTrack( - room: swift::Room, - track: swift::LocalVideoTrack, - callback: extern "C" fn(*mut c_void, swift::LocalTrackPublication, CFStringRef), - callback_data: *mut c_void, - ); - fn LKRoomPublishAudioTrack( - room: swift::Room, - track: swift::LocalAudioTrack, - callback: extern "C" fn(*mut c_void, swift::LocalTrackPublication, CFStringRef), - callback_data: *mut c_void, - ); - fn LKRoomUnpublishTrack(room: swift::Room, publication: swift::LocalTrackPublication); - - fn LKRoomAudioTracksForRemoteParticipant( - room: swift::Room, - participant_id: CFStringRef, - ) -> CFArrayRef; - - fn LKRoomAudioTrackPublicationsForRemoteParticipant( - room: swift::Room, - participant_id: CFStringRef, - ) -> CFArrayRef; - - fn LKRoomVideoTracksForRemoteParticipant( - room: swift::Room, - participant_id: CFStringRef, - ) -> CFArrayRef; - - fn LKVideoRendererCreate( - callback_data: *mut c_void, - on_frame: extern "C" fn(callback_data: *mut c_void, frame: CVImageBufferRef) -> bool, - on_drop: extern "C" fn(callback_data: *mut c_void), - ) -> *const c_void; - - fn LKRemoteAudioTrackGetSid(track: swift::RemoteAudioTrack) -> CFStringRef; - fn LKRemoteVideoTrackGetSid(track: swift::RemoteVideoTrack) -> CFStringRef; - fn LKRemoteAudioTrackStart(track: swift::RemoteAudioTrack); - fn LKRemoteAudioTrackStop(track: swift::RemoteAudioTrack); - fn LKVideoTrackAddRenderer(track: swift::RemoteVideoTrack, renderer: *const c_void); - - fn LKDisplaySources( - callback_data: *mut c_void, - callback: extern "C" fn( - callback_data: *mut c_void, - sources: CFArrayRef, - error: CFStringRef, - ), - ); - fn LKCreateScreenShareTrackForDisplay(display: swift::MacOSDisplay) -> swift::LocalVideoTrack; - fn LKLocalAudioTrackCreateTrack() -> swift::LocalAudioTrack; - - fn LKLocalTrackPublicationSetMute( - publication: swift::LocalTrackPublication, - muted: bool, - on_complete: extern "C" fn(callback_data: *mut c_void, error: CFStringRef), - callback_data: *mut c_void, - ); - - fn LKRemoteTrackPublicationSetEnabled( - publication: swift::RemoteTrackPublication, - enabled: bool, - on_complete: extern "C" fn(callback_data: *mut c_void, error: CFStringRef), - callback_data: *mut c_void, - ); - - fn LKLocalTrackPublicationIsMuted(publication: swift::LocalTrackPublication) -> bool; - fn LKRemoteTrackPublicationIsMuted(publication: swift::RemoteTrackPublication) -> bool; - fn LKLocalTrackPublicationGetSid(publication: swift::LocalTrackPublication) -> CFStringRef; - fn LKRemoteTrackPublicationGetSid(publication: swift::RemoteTrackPublication) -> CFStringRef; -} - -pub struct Room { - native_room: swift::Room, - connection: Mutex<( - watch::Sender, - watch::Receiver, - )>, - update_subscribers: Mutex>>, - _delegate: RoomDelegate, -} - -impl Room { - pub fn new() -> Arc { - Arc::new_cyclic(|weak_room| { - let delegate = RoomDelegate::new(weak_room.clone()); - Self { - native_room: unsafe { LKRoomCreate(delegate.native_delegate) }, - connection: Mutex::new(watch::channel_with(ConnectionState::Disconnected)), - update_subscribers: Default::default(), - _delegate: delegate, - } - }) - } - - pub fn status(&self) -> watch::Receiver { - self.connection.lock().1.clone() - } - - pub fn connect(self: &Arc, url: &str, token: &str) -> impl Future> { - let url = CFString::new(url); - let token = CFString::new(token); - let (did_connect, tx, rx) = Self::build_done_callback(); - unsafe { - LKRoomConnect( - self.native_room, - url.as_concrete_TypeRef(), - token.as_concrete_TypeRef(), - did_connect, - tx, - ) - } - - let this = self.clone(); - let url = url.to_string(); - let token = token.to_string(); - async move { - rx.await.unwrap().context("error connecting to room")?; - *this.connection.lock().0.borrow_mut() = ConnectionState::Connected { url, token }; - Ok(()) - } - } - - fn did_disconnect(&self) { - *self.connection.lock().0.borrow_mut() = ConnectionState::Disconnected; - } - - pub fn display_sources(self: &Arc) -> impl Future>> { - extern "C" fn callback(tx: *mut c_void, sources: CFArrayRef, error: CFStringRef) { - unsafe { - let tx = Box::from_raw(tx as *mut oneshot::Sender>>); - - if sources.is_null() { - let _ = tx.send(Err(anyhow!("{}", CFString::wrap_under_get_rule(error)))); - } else { - let sources = CFArray::wrap_under_get_rule(sources) - .into_iter() - .map(|source| MacOSDisplay::new(swift::MacOSDisplay(*source))) - .collect(); - - let _ = tx.send(Ok(sources)); - } - } - } - - let (tx, rx) = oneshot::channel(); - - unsafe { - LKDisplaySources(Box::into_raw(Box::new(tx)) as *mut _, callback); - } - - async move { rx.await.unwrap() } - } - - pub fn publish_video_track( - self: &Arc, - track: LocalVideoTrack, - ) -> impl Future> { - let (tx, rx) = oneshot::channel::>(); - extern "C" fn callback( - tx: *mut c_void, - publication: swift::LocalTrackPublication, - error: CFStringRef, - ) { - let tx = - unsafe { Box::from_raw(tx as *mut oneshot::Sender>) }; - if error.is_null() { - let _ = tx.send(Ok(LocalTrackPublication::new(publication))); - } else { - let error = unsafe { CFString::wrap_under_get_rule(error).to_string() }; - let _ = tx.send(Err(anyhow!(error))); - } - } - unsafe { - LKRoomPublishVideoTrack( - self.native_room, - track.0, - callback, - Box::into_raw(Box::new(tx)) as *mut c_void, - ); - } - async { rx.await.unwrap().context("error publishing video track") } - } - - pub fn publish_audio_track( - self: &Arc, - track: LocalAudioTrack, - ) -> impl Future> { - let (tx, rx) = oneshot::channel::>(); - extern "C" fn callback( - tx: *mut c_void, - publication: swift::LocalTrackPublication, - error: CFStringRef, - ) { - let tx = - unsafe { Box::from_raw(tx as *mut oneshot::Sender>) }; - if error.is_null() { - let _ = tx.send(Ok(LocalTrackPublication::new(publication))); - } else { - let error = unsafe { CFString::wrap_under_get_rule(error).to_string() }; - let _ = tx.send(Err(anyhow!(error))); - } - } - unsafe { - LKRoomPublishAudioTrack( - self.native_room, - track.0, - callback, - Box::into_raw(Box::new(tx)) as *mut c_void, - ); - } - async { rx.await.unwrap().context("error publishing audio track") } - } - - pub fn unpublish_track(&self, publication: LocalTrackPublication) { - unsafe { - LKRoomUnpublishTrack(self.native_room, publication.0); - } - } - - pub fn remote_video_tracks(&self, participant_id: &str) -> Vec> { - unsafe { - let tracks = LKRoomVideoTracksForRemoteParticipant( - self.native_room, - CFString::new(participant_id).as_concrete_TypeRef(), - ); - - if tracks.is_null() { - Vec::new() - } else { - let tracks = CFArray::wrap_under_get_rule(tracks); - tracks - .into_iter() - .map(|native_track| { - let native_track = swift::RemoteVideoTrack(*native_track); - let id = - CFString::wrap_under_get_rule(LKRemoteVideoTrackGetSid(native_track)) - .to_string(); - Arc::new(RemoteVideoTrack::new( - native_track, - id, - participant_id.into(), - )) - }) - .collect() - } - } - } - - pub fn remote_audio_tracks(&self, participant_id: &str) -> Vec> { - unsafe { - let tracks = LKRoomAudioTracksForRemoteParticipant( - self.native_room, - CFString::new(participant_id).as_concrete_TypeRef(), - ); - - if tracks.is_null() { - Vec::new() - } else { - let tracks = CFArray::wrap_under_get_rule(tracks); - tracks - .into_iter() - .map(|native_track| { - let native_track = swift::RemoteAudioTrack(*native_track); - let id = - CFString::wrap_under_get_rule(LKRemoteAudioTrackGetSid(native_track)) - .to_string(); - Arc::new(RemoteAudioTrack::new( - native_track, - id, - participant_id.into(), - )) - }) - .collect() - } - } - } - - pub fn remote_audio_track_publications( - &self, - participant_id: &str, - ) -> Vec> { - unsafe { - let tracks = LKRoomAudioTrackPublicationsForRemoteParticipant( - self.native_room, - CFString::new(participant_id).as_concrete_TypeRef(), - ); - - if tracks.is_null() { - Vec::new() - } else { - let tracks = CFArray::wrap_under_get_rule(tracks); - tracks - .into_iter() - .map(|native_track_publication| { - let native_track_publication = - swift::RemoteTrackPublication(*native_track_publication); - Arc::new(RemoteTrackPublication::new(native_track_publication)) - }) - .collect() - } - } - } - - pub fn updates(&self) -> mpsc::UnboundedReceiver { - let (tx, rx) = mpsc::unbounded(); - self.update_subscribers.lock().push(tx); - rx - } - - fn did_subscribe_to_remote_audio_track( - &self, - track: RemoteAudioTrack, - publication: RemoteTrackPublication, - ) { - let track = Arc::new(track); - let publication = Arc::new(publication); - self.update_subscribers.lock().retain(|tx| { - tx.unbounded_send(RoomUpdate::SubscribedToRemoteAudioTrack( - track.clone(), - publication.clone(), - )) - .is_ok() - }); - } - - fn did_unsubscribe_from_remote_audio_track(&self, publisher_id: String, track_id: String) { - self.update_subscribers.lock().retain(|tx| { - tx.unbounded_send(RoomUpdate::UnsubscribedFromRemoteAudioTrack { - publisher_id: publisher_id.clone(), - track_id: track_id.clone(), - }) - .is_ok() - }); - } - - fn mute_changed_from_remote_audio_track(&self, track_id: String, muted: bool) { - self.update_subscribers.lock().retain(|tx| { - tx.unbounded_send(RoomUpdate::RemoteAudioTrackMuteChanged { - track_id: track_id.clone(), - muted, - }) - .is_ok() - }); - } - - fn active_speakers_changed(&self, speakers: Vec) { - self.update_subscribers.lock().retain(move |tx| { - tx.unbounded_send(RoomUpdate::ActiveSpeakersChanged { - speakers: speakers.clone(), - }) - .is_ok() - }); - } - - fn did_subscribe_to_remote_video_track(&self, track: RemoteVideoTrack) { - let track = Arc::new(track); - self.update_subscribers.lock().retain(|tx| { - tx.unbounded_send(RoomUpdate::SubscribedToRemoteVideoTrack(track.clone())) - .is_ok() - }); - } - - fn did_unsubscribe_from_remote_video_track(&self, publisher_id: String, track_id: String) { - self.update_subscribers.lock().retain(|tx| { - tx.unbounded_send(RoomUpdate::UnsubscribedFromRemoteVideoTrack { - publisher_id: publisher_id.clone(), - track_id: track_id.clone(), - }) - .is_ok() - }); - } - - fn build_done_callback() -> ( - extern "C" fn(*mut c_void, CFStringRef), - *mut c_void, - oneshot::Receiver>, - ) { - let (tx, rx) = oneshot::channel(); - extern "C" fn done_callback(tx: *mut c_void, error: CFStringRef) { - let tx = unsafe { Box::from_raw(tx as *mut oneshot::Sender>) }; - if error.is_null() { - let _ = tx.send(Ok(())); - } else { - let error = unsafe { CFString::wrap_under_get_rule(error).to_string() }; - let _ = tx.send(Err(anyhow!(error))); - } - } - ( - done_callback, - Box::into_raw(Box::new(tx)) as *mut c_void, - rx, - ) - } - - pub fn set_display_sources(&self, _: Vec) { - unreachable!("This is a test-only function") - } -} - -impl Drop for Room { - fn drop(&mut self) { - unsafe { - LKRoomDisconnect(self.native_room); - CFRelease(self.native_room.0); - } - } -} - -struct RoomDelegate { - native_delegate: swift::RoomDelegate, - weak_room: *mut c_void, -} - -impl RoomDelegate { - fn new(weak_room: Weak) -> Self { - let weak_room = weak_room.into_raw() as *mut c_void; - let native_delegate = unsafe { - LKRoomDelegateCreate( - weak_room, - Self::on_did_disconnect, - Self::on_did_subscribe_to_remote_audio_track, - Self::on_did_unsubscribe_from_remote_audio_track, - Self::on_mute_change_from_remote_audio_track, - Self::on_active_speakers_changed, - Self::on_did_subscribe_to_remote_video_track, - Self::on_did_unsubscribe_from_remote_video_track, - Self::on_did_publish_or_unpublish_local_audio_track, - Self::on_did_publish_or_unpublish_local_video_track, - ) - }; - Self { - native_delegate, - weak_room, - } - } - - extern "C" fn on_did_disconnect(room: *mut c_void) { - let room = unsafe { Weak::from_raw(room as *mut Room) }; - if let Some(room) = room.upgrade() { - room.did_disconnect(); - } - let _ = Weak::into_raw(room); - } - - extern "C" fn on_did_subscribe_to_remote_audio_track( - room: *mut c_void, - publisher_id: CFStringRef, - track_id: CFStringRef, - track: swift::RemoteAudioTrack, - publication: swift::RemoteTrackPublication, - ) { - let room = unsafe { Weak::from_raw(room as *mut Room) }; - let publisher_id = unsafe { CFString::wrap_under_get_rule(publisher_id).to_string() }; - let track_id = unsafe { CFString::wrap_under_get_rule(track_id).to_string() }; - let track = RemoteAudioTrack::new(track, track_id, publisher_id); - let publication = RemoteTrackPublication::new(publication); - if let Some(room) = room.upgrade() { - room.did_subscribe_to_remote_audio_track(track, publication); - } - let _ = Weak::into_raw(room); - } - - extern "C" fn on_did_unsubscribe_from_remote_audio_track( - room: *mut c_void, - publisher_id: CFStringRef, - track_id: CFStringRef, - ) { - let room = unsafe { Weak::from_raw(room as *mut Room) }; - let publisher_id = unsafe { CFString::wrap_under_get_rule(publisher_id).to_string() }; - let track_id = unsafe { CFString::wrap_under_get_rule(track_id).to_string() }; - if let Some(room) = room.upgrade() { - room.did_unsubscribe_from_remote_audio_track(publisher_id, track_id); - } - let _ = Weak::into_raw(room); - } - - extern "C" fn on_mute_change_from_remote_audio_track( - room: *mut c_void, - track_id: CFStringRef, - muted: bool, - ) { - let room = unsafe { Weak::from_raw(room as *mut Room) }; - let track_id = unsafe { CFString::wrap_under_get_rule(track_id).to_string() }; - if let Some(room) = room.upgrade() { - room.mute_changed_from_remote_audio_track(track_id, muted); - } - let _ = Weak::into_raw(room); - } - - extern "C" fn on_active_speakers_changed(room: *mut c_void, participants: CFArrayRef) { - if participants.is_null() { - return; - } - - let room = unsafe { Weak::from_raw(room as *mut Room) }; - let speakers = unsafe { - CFArray::wrap_under_get_rule(participants) - .into_iter() - .map( - |speaker: core_foundation::base::ItemRef<'_, *const c_void>| { - CFString::wrap_under_get_rule(*speaker as CFStringRef).to_string() - }, - ) - .collect() - }; - - if let Some(room) = room.upgrade() { - room.active_speakers_changed(speakers); - } - let _ = Weak::into_raw(room); - } - - extern "C" fn on_did_subscribe_to_remote_video_track( - room: *mut c_void, - publisher_id: CFStringRef, - track_id: CFStringRef, - track: swift::RemoteVideoTrack, - ) { - let room = unsafe { Weak::from_raw(room as *mut Room) }; - let publisher_id = unsafe { CFString::wrap_under_get_rule(publisher_id).to_string() }; - let track_id = unsafe { CFString::wrap_under_get_rule(track_id).to_string() }; - let track = RemoteVideoTrack::new(track, track_id, publisher_id); - if let Some(room) = room.upgrade() { - room.did_subscribe_to_remote_video_track(track); - } - let _ = Weak::into_raw(room); - } - - extern "C" fn on_did_unsubscribe_from_remote_video_track( - room: *mut c_void, - publisher_id: CFStringRef, - track_id: CFStringRef, - ) { - let room = unsafe { Weak::from_raw(room as *mut Room) }; - let publisher_id = unsafe { CFString::wrap_under_get_rule(publisher_id).to_string() }; - let track_id = unsafe { CFString::wrap_under_get_rule(track_id).to_string() }; - if let Some(room) = room.upgrade() { - room.did_unsubscribe_from_remote_video_track(publisher_id, track_id); - } - let _ = Weak::into_raw(room); - } - - extern "C" fn on_did_publish_or_unpublish_local_audio_track( - room: *mut c_void, - publication: swift::LocalTrackPublication, - is_published: bool, - ) { - let room = unsafe { Weak::from_raw(room as *mut Room) }; - if let Some(room) = room.upgrade() { - let publication = LocalTrackPublication::new(publication); - let update = if is_published { - RoomUpdate::LocalAudioTrackPublished { publication } - } else { - RoomUpdate::LocalAudioTrackUnpublished { publication } - }; - room.update_subscribers - .lock() - .retain(|tx| tx.unbounded_send(update.clone()).is_ok()); - } - let _ = Weak::into_raw(room); - } - - extern "C" fn on_did_publish_or_unpublish_local_video_track( - room: *mut c_void, - publication: swift::LocalTrackPublication, - is_published: bool, - ) { - let room = unsafe { Weak::from_raw(room as *mut Room) }; - if let Some(room) = room.upgrade() { - let publication = LocalTrackPublication::new(publication); - let update = if is_published { - RoomUpdate::LocalVideoTrackPublished { publication } - } else { - RoomUpdate::LocalVideoTrackUnpublished { publication } - }; - room.update_subscribers - .lock() - .retain(|tx| tx.unbounded_send(update.clone()).is_ok()); - } - let _ = Weak::into_raw(room); - } -} - -impl Drop for RoomDelegate { - fn drop(&mut self) { - unsafe { - CFRelease(self.native_delegate.0); - let _ = Weak::from_raw(self.weak_room as *mut Room); - } - } -} - -pub struct LocalAudioTrack(swift::LocalAudioTrack); - -impl LocalAudioTrack { - pub fn create() -> Self { - Self(unsafe { LKLocalAudioTrackCreateTrack() }) - } -} - -impl Drop for LocalAudioTrack { - fn drop(&mut self) { - unsafe { CFRelease(self.0 .0) } - } -} - -pub struct LocalVideoTrack(swift::LocalVideoTrack); - -impl LocalVideoTrack { - pub fn screen_share_for_display(display: &MacOSDisplay) -> Self { - Self(unsafe { LKCreateScreenShareTrackForDisplay(display.0) }) - } -} - -impl Drop for LocalVideoTrack { - fn drop(&mut self) { - unsafe { CFRelease(self.0 .0) } - } -} - -pub struct LocalTrackPublication(swift::LocalTrackPublication); - -impl LocalTrackPublication { - pub fn new(native_track_publication: swift::LocalTrackPublication) -> Self { - unsafe { - CFRetain(native_track_publication.0); - } - Self(native_track_publication) - } - - pub fn sid(&self) -> String { - unsafe { CFString::wrap_under_get_rule(LKLocalTrackPublicationGetSid(self.0)).to_string() } - } - - pub fn set_mute(&self, muted: bool) -> impl Future> { - let (tx, rx) = futures::channel::oneshot::channel(); - - extern "C" fn complete_callback(callback_data: *mut c_void, error: CFStringRef) { - let tx = unsafe { Box::from_raw(callback_data as *mut oneshot::Sender>) }; - if error.is_null() { - tx.send(Ok(())).ok(); - } else { - let error = unsafe { CFString::wrap_under_get_rule(error).to_string() }; - tx.send(Err(anyhow!(error))).ok(); - } - } - - unsafe { - LKLocalTrackPublicationSetMute( - self.0, - muted, - complete_callback, - Box::into_raw(Box::new(tx)) as *mut c_void, - ) - } - - async move { rx.await.unwrap() } - } - - pub fn is_muted(&self) -> bool { - unsafe { LKLocalTrackPublicationIsMuted(self.0) } - } -} - -impl Clone for LocalTrackPublication { - fn clone(&self) -> Self { - unsafe { - CFRetain(self.0 .0); - } - Self(self.0) - } -} - -impl Drop for LocalTrackPublication { - fn drop(&mut self) { - unsafe { CFRelease(self.0 .0) } - } -} - -pub struct RemoteTrackPublication(swift::RemoteTrackPublication); - -impl RemoteTrackPublication { - pub fn new(native_track_publication: swift::RemoteTrackPublication) -> Self { - unsafe { - CFRetain(native_track_publication.0); - } - Self(native_track_publication) - } - - pub fn sid(&self) -> String { - unsafe { CFString::wrap_under_get_rule(LKRemoteTrackPublicationGetSid(self.0)).to_string() } - } - - pub fn is_muted(&self) -> bool { - unsafe { LKRemoteTrackPublicationIsMuted(self.0) } - } - - pub fn set_enabled(&self, enabled: bool) -> impl Future> { - let (tx, rx) = futures::channel::oneshot::channel(); - - extern "C" fn complete_callback(callback_data: *mut c_void, error: CFStringRef) { - let tx = unsafe { Box::from_raw(callback_data as *mut oneshot::Sender>) }; - if error.is_null() { - tx.send(Ok(())).ok(); - } else { - let error = unsafe { CFString::wrap_under_get_rule(error).to_string() }; - tx.send(Err(anyhow!(error))).ok(); - } - } - - unsafe { - LKRemoteTrackPublicationSetEnabled( - self.0, - enabled, - complete_callback, - Box::into_raw(Box::new(tx)) as *mut c_void, - ) - } - - async move { rx.await.unwrap() } - } -} - -impl Drop for RemoteTrackPublication { - fn drop(&mut self) { - unsafe { CFRelease(self.0 .0) } - } -} - -#[derive(Debug)] -pub struct RemoteAudioTrack { - native_track: swift::RemoteAudioTrack, - sid: Sid, - publisher_id: String, -} - -impl RemoteAudioTrack { - fn new(native_track: swift::RemoteAudioTrack, sid: Sid, publisher_id: String) -> Self { - unsafe { - CFRetain(native_track.0); - } - Self { - native_track, - sid, - publisher_id, - } - } - - pub fn sid(&self) -> &str { - &self.sid - } - - pub fn publisher_id(&self) -> &str { - &self.publisher_id - } - - pub fn start(&self) { - unsafe { LKRemoteAudioTrackStart(self.native_track) } - } - - pub fn stop(&self) { - unsafe { LKRemoteAudioTrackStop(self.native_track) } - } -} - -impl Drop for RemoteAudioTrack { - fn drop(&mut self) { - // todo: uncomment this `CFRelease`, unless we find that it was causing - // the crash in the `livekit.multicast` thread. - // - // unsafe { CFRelease(self.native_track.0) } - let _ = self.native_track; - } -} - -#[derive(Debug)] -pub struct RemoteVideoTrack { - native_track: swift::RemoteVideoTrack, - sid: Sid, - publisher_id: String, -} - -impl RemoteVideoTrack { - fn new(native_track: swift::RemoteVideoTrack, sid: Sid, publisher_id: String) -> Self { - unsafe { - CFRetain(native_track.0); - } - Self { - native_track, - sid, - publisher_id, - } - } - - pub fn sid(&self) -> &str { - &self.sid - } - - pub fn publisher_id(&self) -> &str { - &self.publisher_id - } - - pub fn frames(&self) -> async_broadcast::Receiver { - extern "C" fn on_frame(callback_data: *mut c_void, frame: CVImageBufferRef) -> bool { - unsafe { - let tx = Box::from_raw(callback_data as *mut async_broadcast::Sender); - let buffer = CVImageBuffer::wrap_under_get_rule(frame); - let result = tx.try_broadcast(Frame(buffer)); - let _ = Box::into_raw(tx); - match result { - Ok(_) => true, - Err(async_broadcast::TrySendError::Closed(_)) - | Err(async_broadcast::TrySendError::Inactive(_)) => { - log::warn!("no active receiver for frame"); - false - } - Err(async_broadcast::TrySendError::Full(_)) => { - log::warn!("skipping frame as receiver is not keeping up"); - true - } - } - } - } - - extern "C" fn on_drop(callback_data: *mut c_void) { - unsafe { - let _ = Box::from_raw(callback_data as *mut async_broadcast::Sender); - } - } - - let (tx, rx) = async_broadcast::broadcast(64); - unsafe { - let renderer = LKVideoRendererCreate( - Box::into_raw(Box::new(tx)) as *mut c_void, - on_frame, - on_drop, - ); - LKVideoTrackAddRenderer(self.native_track, renderer); - rx - } - } -} - -impl Drop for RemoteVideoTrack { - fn drop(&mut self) { - unsafe { CFRelease(self.native_track.0) } - } -} - -pub struct MacOSDisplay(swift::MacOSDisplay); - -impl MacOSDisplay { - fn new(ptr: swift::MacOSDisplay) -> Self { - unsafe { - CFRetain(ptr.0); - } - Self(ptr) - } -} - -impl Drop for MacOSDisplay { - fn drop(&mut self) { - unsafe { CFRelease(self.0 .0) } - } -} - -#[derive(Clone)] -pub struct Frame(CVImageBuffer); - -impl Frame { - pub fn width(&self) -> usize { - self.0.width() - } - - pub fn height(&self) -> usize { - self.0.height() - } - - pub fn image(&self) -> CVImageBuffer { - self.0.clone() - } -} diff --git a/crates/live_kit_client/src/test.rs b/crates/live_kit_client/src/test.rs deleted file mode 100644 index 2c26c88f72346..0000000000000 --- a/crates/live_kit_client/src/test.rs +++ /dev/null @@ -1,882 +0,0 @@ -use crate::{ConnectionState, RoomUpdate, Sid}; -use anyhow::{anyhow, Context, Result}; -use async_trait::async_trait; -use collections::{btree_map::Entry as BTreeEntry, hash_map::Entry, BTreeMap, HashMap, HashSet}; -use futures::Stream; -use gpui::{BackgroundExecutor, SurfaceSource}; -use live_kit_server::{proto, token}; - -use parking_lot::Mutex; -use postage::watch; -use std::{ - future::Future, - mem, - sync::{ - atomic::{AtomicBool, Ordering::SeqCst}, - Arc, Weak, - }, -}; - -static SERVERS: Mutex>> = Mutex::new(BTreeMap::new()); - -pub struct TestServer { - pub url: String, - pub api_key: String, - pub secret_key: String, - rooms: Mutex>, - executor: BackgroundExecutor, -} - -impl TestServer { - pub fn create( - url: String, - api_key: String, - secret_key: String, - executor: BackgroundExecutor, - ) -> Result> { - let mut servers = SERVERS.lock(); - if let BTreeEntry::Vacant(e) = servers.entry(url.clone()) { - let server = Arc::new(TestServer { - url, - api_key, - secret_key, - rooms: Default::default(), - executor, - }); - e.insert(server.clone()); - Ok(server) - } else { - Err(anyhow!("a server with url {:?} already exists", url)) - } - } - - fn get(url: &str) -> Result> { - Ok(SERVERS - .lock() - .get(url) - .ok_or_else(|| anyhow!("no server found for url"))? - .clone()) - } - - pub fn teardown(&self) -> Result<()> { - SERVERS - .lock() - .remove(&self.url) - .ok_or_else(|| anyhow!("server with url {:?} does not exist", self.url))?; - Ok(()) - } - - pub fn create_api_client(&self) -> TestApiClient { - TestApiClient { - url: self.url.clone(), - } - } - - pub async fn create_room(&self, room: String) -> Result<()> { - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] - self.executor.simulate_random_delay().await; - let mut server_rooms = self.rooms.lock(); - if let Entry::Vacant(e) = server_rooms.entry(room.clone()) { - e.insert(Default::default()); - Ok(()) - } else { - Err(anyhow!("room {:?} already exists", room)) - } - } - - async fn delete_room(&self, room: String) -> Result<()> { - // TODO: clear state associated with all `Room`s. - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] - self.executor.simulate_random_delay().await; - let mut server_rooms = self.rooms.lock(); - server_rooms - .remove(&room) - .ok_or_else(|| anyhow!("room {:?} does not exist", room))?; - Ok(()) - } - - async fn join_room(&self, token: String, client_room: Arc) -> Result<()> { - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] - self.executor.simulate_random_delay().await; - - let claims = live_kit_server::token::validate(&token, &self.secret_key)?; - let identity = claims.sub.unwrap().to_string(); - let room_name = claims.video.room.unwrap(); - let mut server_rooms = self.rooms.lock(); - let room = (*server_rooms).entry(room_name.to_string()).or_default(); - - if let Entry::Vacant(e) = room.client_rooms.entry(identity.clone()) { - for track in &room.video_tracks { - client_room - .0 - .lock() - .updates_tx - .try_broadcast(RoomUpdate::SubscribedToRemoteVideoTrack(Arc::new( - RemoteVideoTrack { - server_track: track.clone(), - }, - ))) - .unwrap(); - } - for track in &room.audio_tracks { - client_room - .0 - .lock() - .updates_tx - .try_broadcast(RoomUpdate::SubscribedToRemoteAudioTrack( - Arc::new(RemoteAudioTrack { - server_track: track.clone(), - room: Arc::downgrade(&client_room), - }), - Arc::new(RemoteTrackPublication), - )) - .unwrap(); - } - e.insert(client_room); - Ok(()) - } else { - Err(anyhow!( - "{:?} attempted to join room {:?} twice", - identity, - room_name - )) - } - } - - async fn leave_room(&self, token: String) -> Result<()> { - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] - self.executor.simulate_random_delay().await; - let claims = live_kit_server::token::validate(&token, &self.secret_key)?; - let identity = claims.sub.unwrap().to_string(); - let room_name = claims.video.room.unwrap(); - let mut server_rooms = self.rooms.lock(); - let room = server_rooms - .get_mut(&*room_name) - .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; - room.client_rooms.remove(&identity).ok_or_else(|| { - anyhow!( - "{:?} attempted to leave room {:?} before joining it", - identity, - room_name - ) - })?; - Ok(()) - } - - async fn remove_participant(&self, room_name: String, identity: String) -> Result<()> { - // TODO: clear state associated with the `Room`. - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] - self.executor.simulate_random_delay().await; - - let mut server_rooms = self.rooms.lock(); - let room = server_rooms - .get_mut(&room_name) - .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; - room.client_rooms.remove(&identity).ok_or_else(|| { - anyhow!( - "participant {:?} did not join room {:?}", - identity, - room_name - ) - })?; - Ok(()) - } - - async fn update_participant( - &self, - room_name: String, - identity: String, - permission: proto::ParticipantPermission, - ) -> Result<()> { - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] - self.executor.simulate_random_delay().await; - let mut server_rooms = self.rooms.lock(); - let room = server_rooms - .get_mut(&room_name) - .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; - room.participant_permissions.insert(identity, permission); - Ok(()) - } - - pub async fn disconnect_client(&self, client_identity: String) { - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] - self.executor.simulate_random_delay().await; - let mut server_rooms = self.rooms.lock(); - for room in server_rooms.values_mut() { - if let Some(room) = room.client_rooms.remove(&client_identity) { - *room.0.lock().connection.0.borrow_mut() = ConnectionState::Disconnected; - } - } - } - - async fn publish_video_track( - &self, - token: String, - local_track: LocalVideoTrack, - ) -> Result { - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] - self.executor.simulate_random_delay().await; - let claims = live_kit_server::token::validate(&token, &self.secret_key)?; - let identity = claims.sub.unwrap().to_string(); - let room_name = claims.video.room.unwrap(); - - let mut server_rooms = self.rooms.lock(); - let room = server_rooms - .get_mut(&*room_name) - .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; - - let can_publish = room - .participant_permissions - .get(&identity) - .map(|permission| permission.can_publish) - .or(claims.video.can_publish) - .unwrap_or(true); - - if !can_publish { - return Err(anyhow!("user is not allowed to publish")); - } - - let sid = nanoid::nanoid!(17); - let track = Arc::new(TestServerVideoTrack { - sid: sid.clone(), - publisher_id: identity.clone(), - frames_rx: local_track.frames_rx.clone(), - }); - - room.video_tracks.push(track.clone()); - - for (id, client_room) in &room.client_rooms { - if *id != identity { - let _ = client_room - .0 - .lock() - .updates_tx - .try_broadcast(RoomUpdate::SubscribedToRemoteVideoTrack(Arc::new( - RemoteVideoTrack { - server_track: track.clone(), - }, - ))) - .unwrap(); - } - } - - Ok(sid) - } - - async fn publish_audio_track( - &self, - token: String, - _local_track: &LocalAudioTrack, - ) -> Result { - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] - self.executor.simulate_random_delay().await; - - let claims = live_kit_server::token::validate(&token, &self.secret_key)?; - let identity = claims.sub.unwrap().to_string(); - let room_name = claims.video.room.unwrap(); - - let mut server_rooms = self.rooms.lock(); - let room = server_rooms - .get_mut(&*room_name) - .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; - - let can_publish = room - .participant_permissions - .get(&identity) - .map(|permission| permission.can_publish) - .or(claims.video.can_publish) - .unwrap_or(true); - - if !can_publish { - return Err(anyhow!("user is not allowed to publish")); - } - - let sid = nanoid::nanoid!(17); - let track = Arc::new(TestServerAudioTrack { - sid: sid.clone(), - publisher_id: identity.clone(), - muted: AtomicBool::new(false), - }); - - let publication = Arc::new(RemoteTrackPublication); - - room.audio_tracks.push(track.clone()); - - for (id, client_room) in &room.client_rooms { - if *id != identity { - let _ = client_room - .0 - .lock() - .updates_tx - .try_broadcast(RoomUpdate::SubscribedToRemoteAudioTrack( - Arc::new(RemoteAudioTrack { - server_track: track.clone(), - room: Arc::downgrade(client_room), - }), - publication.clone(), - )) - .unwrap(); - } - } - - Ok(sid) - } - - fn set_track_muted(&self, token: &str, track_sid: &str, muted: bool) -> Result<()> { - let claims = live_kit_server::token::validate(token, &self.secret_key)?; - let room_name = claims.video.room.unwrap(); - let identity = claims.sub.unwrap(); - let mut server_rooms = self.rooms.lock(); - let room = server_rooms - .get_mut(&*room_name) - .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; - if let Some(track) = room - .audio_tracks - .iter_mut() - .find(|track| track.sid == track_sid) - { - track.muted.store(muted, SeqCst); - for (id, client_room) in room.client_rooms.iter() { - if *id != identity { - client_room - .0 - .lock() - .updates_tx - .try_broadcast(RoomUpdate::RemoteAudioTrackMuteChanged { - track_id: track_sid.to_string(), - muted, - }) - .unwrap(); - } - } - } - Ok(()) - } - - fn is_track_muted(&self, token: &str, track_sid: &str) -> Option { - let claims = live_kit_server::token::validate(token, &self.secret_key).ok()?; - let room_name = claims.video.room.unwrap(); - - let mut server_rooms = self.rooms.lock(); - let room = server_rooms.get_mut(&*room_name)?; - room.audio_tracks.iter().find_map(|track| { - if track.sid == track_sid { - Some(track.muted.load(SeqCst)) - } else { - None - } - }) - } - - fn video_tracks(&self, token: String) -> Result>> { - let claims = live_kit_server::token::validate(&token, &self.secret_key)?; - let room_name = claims.video.room.unwrap(); - let identity = claims.sub.unwrap(); - - let mut server_rooms = self.rooms.lock(); - let room = server_rooms - .get_mut(&*room_name) - .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; - room.client_rooms - .get(identity.as_ref()) - .ok_or_else(|| anyhow!("not a participant in room"))?; - Ok(room - .video_tracks - .iter() - .map(|track| { - Arc::new(RemoteVideoTrack { - server_track: track.clone(), - }) - }) - .collect()) - } - - fn audio_tracks(&self, token: String) -> Result>> { - let claims = live_kit_server::token::validate(&token, &self.secret_key)?; - let room_name = claims.video.room.unwrap(); - let identity = claims.sub.unwrap(); - - let mut server_rooms = self.rooms.lock(); - let room = server_rooms - .get_mut(&*room_name) - .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; - let client_room = room - .client_rooms - .get(identity.as_ref()) - .ok_or_else(|| anyhow!("not a participant in room"))?; - Ok(room - .audio_tracks - .iter() - .map(|track| { - Arc::new(RemoteAudioTrack { - server_track: track.clone(), - room: Arc::downgrade(client_room), - }) - }) - .collect()) - } -} - -#[derive(Default)] -struct TestServerRoom { - client_rooms: HashMap>, - video_tracks: Vec>, - audio_tracks: Vec>, - participant_permissions: HashMap, -} - -#[derive(Debug)] -struct TestServerVideoTrack { - sid: Sid, - publisher_id: Sid, - frames_rx: async_broadcast::Receiver, -} - -#[derive(Debug)] -struct TestServerAudioTrack { - sid: Sid, - publisher_id: Sid, - muted: AtomicBool, -} - -impl TestServerRoom {} - -pub struct TestApiClient { - url: String, -} - -#[async_trait] -impl live_kit_server::api::Client for TestApiClient { - fn url(&self) -> &str { - &self.url - } - - async fn create_room(&self, name: String) -> Result<()> { - let server = TestServer::get(&self.url)?; - server.create_room(name).await?; - Ok(()) - } - - async fn delete_room(&self, name: String) -> Result<()> { - let server = TestServer::get(&self.url)?; - server.delete_room(name).await?; - Ok(()) - } - - async fn remove_participant(&self, room: String, identity: String) -> Result<()> { - let server = TestServer::get(&self.url)?; - server.remove_participant(room, identity).await?; - Ok(()) - } - - async fn update_participant( - &self, - room: String, - identity: String, - permission: live_kit_server::proto::ParticipantPermission, - ) -> Result<()> { - let server = TestServer::get(&self.url)?; - server - .update_participant(room, identity, permission) - .await?; - Ok(()) - } - - fn room_token(&self, room: &str, identity: &str) -> Result { - let server = TestServer::get(&self.url)?; - token::create( - &server.api_key, - &server.secret_key, - Some(identity), - token::VideoGrant::to_join(room), - ) - } - - fn guest_token(&self, room: &str, identity: &str) -> Result { - let server = TestServer::get(&self.url)?; - token::create( - &server.api_key, - &server.secret_key, - Some(identity), - token::VideoGrant::for_guest(room), - ) - } -} - -struct RoomState { - connection: ( - watch::Sender, - watch::Receiver, - ), - display_sources: Vec, - paused_audio_tracks: HashSet, - updates_tx: async_broadcast::Sender, - updates_rx: async_broadcast::Receiver, -} - -pub struct Room(Mutex); - -impl Room { - pub fn new() -> Arc { - let (updates_tx, updates_rx) = async_broadcast::broadcast(128); - Arc::new(Self(Mutex::new(RoomState { - connection: watch::channel_with(ConnectionState::Disconnected), - display_sources: Default::default(), - paused_audio_tracks: Default::default(), - updates_tx, - updates_rx, - }))) - } - - pub fn status(&self) -> watch::Receiver { - self.0.lock().connection.1.clone() - } - - pub fn connect(self: &Arc, url: &str, token: &str) -> impl Future> { - let this = self.clone(); - let url = url.to_string(); - let token = token.to_string(); - async move { - let server = TestServer::get(&url)?; - server - .join_room(token.clone(), this.clone()) - .await - .context("room join")?; - *this.0.lock().connection.0.borrow_mut() = ConnectionState::Connected { url, token }; - Ok(()) - } - } - - pub fn display_sources(self: &Arc) -> impl Future>> { - let this = self.clone(); - async move { - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] - { - let server = this.test_server(); - server.executor.simulate_random_delay().await; - } - - Ok(this.0.lock().display_sources.clone()) - } - } - - pub fn publish_video_track( - self: &Arc, - track: LocalVideoTrack, - ) -> impl Future> { - let this = self.clone(); - let track = track.clone(); - async move { - let sid = this - .test_server() - .publish_video_track(this.token(), track) - .await?; - Ok(LocalTrackPublication { - room: Arc::downgrade(&this), - sid, - }) - } - } - - pub fn publish_audio_track( - self: &Arc, - track: LocalAudioTrack, - ) -> impl Future> { - let this = self.clone(); - let track = track.clone(); - async move { - let sid = this - .test_server() - .publish_audio_track(this.token(), &track) - .await?; - Ok(LocalTrackPublication { - room: Arc::downgrade(&this), - sid, - }) - } - } - - pub fn unpublish_track(&self, _publication: LocalTrackPublication) {} - - pub fn remote_audio_tracks(&self, publisher_id: &str) -> Vec> { - if !self.is_connected() { - return Vec::new(); - } - - self.test_server() - .audio_tracks(self.token()) - .unwrap() - .into_iter() - .filter(|track| track.publisher_id() == publisher_id) - .collect() - } - - pub fn remote_audio_track_publications( - &self, - publisher_id: &str, - ) -> Vec> { - if !self.is_connected() { - return Vec::new(); - } - - self.test_server() - .audio_tracks(self.token()) - .unwrap() - .into_iter() - .filter(|track| track.publisher_id() == publisher_id) - .map(|_track| Arc::new(RemoteTrackPublication {})) - .collect() - } - - pub fn remote_video_tracks(&self, publisher_id: &str) -> Vec> { - if !self.is_connected() { - return Vec::new(); - } - - self.test_server() - .video_tracks(self.token()) - .unwrap() - .into_iter() - .filter(|track| track.publisher_id() == publisher_id) - .collect() - } - - pub fn updates(&self) -> impl Stream { - self.0.lock().updates_rx.clone() - } - - pub fn set_display_sources(&self, sources: Vec) { - self.0.lock().display_sources = sources; - } - - fn test_server(&self) -> Arc { - match self.0.lock().connection.1.borrow().clone() { - ConnectionState::Disconnected => panic!("must be connected to call this method"), - ConnectionState::Connected { url, .. } => TestServer::get(&url).unwrap(), - } - } - - fn token(&self) -> String { - match self.0.lock().connection.1.borrow().clone() { - ConnectionState::Disconnected => panic!("must be connected to call this method"), - ConnectionState::Connected { token, .. } => token, - } - } - - fn is_connected(&self) -> bool { - match *self.0.lock().connection.1.borrow() { - ConnectionState::Disconnected => false, - ConnectionState::Connected { .. } => true, - } - } -} - -impl Drop for Room { - fn drop(&mut self) { - if let ConnectionState::Connected { token, .. } = mem::replace( - &mut *self.0.lock().connection.0.borrow_mut(), - ConnectionState::Disconnected, - ) { - if let Ok(server) = TestServer::get(&token) { - let executor = server.executor.clone(); - executor - .spawn(async move { server.leave_room(token).await.unwrap() }) - .detach(); - } - } - } -} - -#[derive(Clone)] -pub struct LocalTrackPublication { - sid: String, - room: Weak, -} - -impl LocalTrackPublication { - pub fn set_mute(&self, mute: bool) -> impl Future> { - let sid = self.sid.clone(); - let room = self.room.clone(); - async move { - if let Some(room) = room.upgrade() { - room.test_server() - .set_track_muted(&room.token(), &sid, mute) - } else { - Err(anyhow!("no such room")) - } - } - } - - pub fn is_muted(&self) -> bool { - if let Some(room) = self.room.upgrade() { - room.test_server() - .is_track_muted(&room.token(), &self.sid) - .unwrap_or(false) - } else { - false - } - } - - pub fn sid(&self) -> String { - self.sid.clone() - } -} - -pub struct RemoteTrackPublication; - -impl RemoteTrackPublication { - pub fn set_enabled(&self, _enabled: bool) -> impl Future> { - async { Ok(()) } - } - - pub fn is_muted(&self) -> bool { - false - } - - pub fn sid(&self) -> String { - "".to_string() - } -} - -#[derive(Clone)] -pub struct LocalVideoTrack { - frames_rx: async_broadcast::Receiver, -} - -impl LocalVideoTrack { - pub fn screen_share_for_display(display: &MacOSDisplay) -> Self { - Self { - frames_rx: display.frames.1.clone(), - } - } -} - -#[derive(Clone)] -pub struct LocalAudioTrack; - -impl LocalAudioTrack { - pub fn create() -> Self { - Self - } -} - -#[derive(Debug)] -pub struct RemoteVideoTrack { - server_track: Arc, -} - -impl RemoteVideoTrack { - pub fn sid(&self) -> &str { - &self.server_track.sid - } - - pub fn publisher_id(&self) -> &str { - &self.server_track.publisher_id - } - - pub fn frames(&self) -> async_broadcast::Receiver { - self.server_track.frames_rx.clone() - } -} - -#[derive(Debug)] -pub struct RemoteAudioTrack { - server_track: Arc, - room: Weak, -} - -impl RemoteAudioTrack { - pub fn sid(&self) -> &str { - &self.server_track.sid - } - - pub fn publisher_id(&self) -> &str { - &self.server_track.publisher_id - } - - pub fn start(&self) { - if let Some(room) = self.room.upgrade() { - room.0 - .lock() - .paused_audio_tracks - .remove(&self.server_track.sid); - } - } - - pub fn stop(&self) { - if let Some(room) = self.room.upgrade() { - room.0 - .lock() - .paused_audio_tracks - .insert(self.server_track.sid.clone()); - } - } - - pub fn is_playing(&self) -> bool { - !self - .room - .upgrade() - .unwrap() - .0 - .lock() - .paused_audio_tracks - .contains(&self.server_track.sid) - } -} - -#[derive(Clone)] -pub struct MacOSDisplay { - frames: ( - async_broadcast::Sender, - async_broadcast::Receiver, - ), -} - -impl Default for MacOSDisplay { - fn default() -> Self { - Self::new() - } -} - -impl MacOSDisplay { - pub fn new() -> Self { - Self { - frames: async_broadcast::broadcast(128), - } - } - - pub fn send_frame(&self, frame: Frame) { - self.frames.0.try_broadcast(frame).unwrap(); - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Frame { - pub label: String, - pub width: usize, - pub height: usize, -} - -impl Frame { - pub fn width(&self) -> usize { - self.width - } - - pub fn height(&self) -> usize { - self.height - } - - pub fn image(&self) -> SurfaceSource { - unimplemented!("you can't call this in test mode") - } -} diff --git a/crates/live_kit_client/.cargo/config.toml b/crates/livekit_client/.cargo/config.toml similarity index 62% rename from crates/live_kit_client/.cargo/config.toml rename to crates/livekit_client/.cargo/config.toml index b33fe211bd576..77f7c9dd6c48b 100644 --- a/crates/live_kit_client/.cargo/config.toml +++ b/crates/livekit_client/.cargo/config.toml @@ -1,2 +1,2 @@ -[live_kit_client_test] +[livekit_client_test] rustflags = ["-C", "link-args=-ObjC"] diff --git a/crates/live_kit_client/Cargo.toml b/crates/livekit_client/Cargo.toml similarity index 60% rename from crates/live_kit_client/Cargo.toml rename to crates/livekit_client/Cargo.toml index e23c63453e176..ac0c3b5740418 100644 --- a/crates/live_kit_client/Cargo.toml +++ b/crates/livekit_client/Cargo.toml @@ -1,8 +1,8 @@ [package] -name = "live_kit_client" +name = "livekit_client" version = "0.1.0" edition = "2021" -description = "Bindings to LiveKit Swift client SDK" +description = "Logic for using LiveKit with GPUI" publish = false license = "GPL-3.0-or-later" @@ -10,7 +10,7 @@ license = "GPL-3.0-or-later" workspace = true [lib] -path = "src/live_kit_client.rs" +path = "src/livekit_client.rs" doctest = false [[example]] @@ -19,42 +19,40 @@ name = "test_app" [features] no-webrtc = [] test-support = [ - "async-trait", "collections/test-support", "gpui/test-support", - "live_kit_server", "nanoid", ] [dependencies] anyhow.workspace = true -async-broadcast = "0.7" -async-trait = { workspace = true, optional = true } -collections = { workspace = true, optional = true } +async-trait.workspace = true +collections.workspace = true +cpal = "0.15" futures.workspace = true -gpui = { workspace = true, optional = true } -live_kit_server = { workspace = true, optional = true } +gpui.workspace = true +http_2 = { package = "http", version = "0.2.1" } +livekit_server.workspace = true log.workspace = true media.workspace = true nanoid = { workspace = true, optional = true} parking_lot.workspace = true postage.workspace = true +util.workspace = true +http_client.workspace = true +smallvec.workspace = true +image.workspace = true + +[target.'cfg(not(target_os = "windows"))'.dependencies] +livekit.workspace = true [target.'cfg(target_os = "macos")'.dependencies] core-foundation.workspace = true - -[target.'cfg(all(not(target_os = "macos")))'.dependencies] -async-trait = { workspace = true } -collections = { workspace = true } -gpui = { workspace = true } -live_kit_server.workspace = true -nanoid.workspace = true +coreaudio-rs = "0.12.1" [dev-dependencies] -async-trait.workspace = true collections = { workspace = true, features = ["test-support"] } gpui = { workspace = true, features = ["test-support"] } -live_kit_server.workspace = true nanoid.workspace = true sha2.workspace = true simplelog.workspace = true diff --git a/crates/live_kit_client/LICENSE-GPL b/crates/livekit_client/LICENSE-GPL similarity index 100% rename from crates/live_kit_client/LICENSE-GPL rename to crates/livekit_client/LICENSE-GPL diff --git a/crates/livekit_client/examples/test_app.rs b/crates/livekit_client/examples/test_app.rs new file mode 100644 index 0000000000000..54384ac5ee2f3 --- /dev/null +++ b/crates/livekit_client/examples/test_app.rs @@ -0,0 +1,438 @@ +#![cfg_attr(windows, allow(unused))] + +use gpui::{ + actions, bounds, div, point, + prelude::{FluentBuilder as _, IntoElement}, + px, rgb, size, AsyncAppContext, Bounds, InteractiveElement, KeyBinding, Menu, MenuItem, + ParentElement, Pixels, Render, ScreenCaptureStream, SharedString, + StatefulInteractiveElement as _, Styled, Task, View, ViewContext, VisualContext, WindowBounds, + WindowHandle, WindowOptions, +}; +#[cfg(not(target_os = "windows"))] +use livekit_client::{ + capture_local_audio_track, capture_local_video_track, + id::ParticipantIdentity, + options::{TrackPublishOptions, VideoCodec}, + participant::{Participant, RemoteParticipant}, + play_remote_audio_track, + publication::{LocalTrackPublication, RemoteTrackPublication}, + track::{LocalTrack, RemoteTrack, RemoteVideoTrack, TrackSource}, + AudioStream, RemoteVideoTrackView, Room, RoomEvent, RoomOptions, +}; + +#[cfg(target_os = "windows")] +use livekit_client::{ + participant::{Participant, RemoteParticipant}, + publication::{LocalTrackPublication, RemoteTrackPublication}, + track::{LocalTrack, RemoteTrack, RemoteVideoTrack}, + AudioStream, RemoteVideoTrackView, Room, RoomEvent, +}; + +use livekit_server::token::{self, VideoGrant}; +use log::LevelFilter; +use postage::stream::Stream as _; +use simplelog::SimpleLogger; + +actions!(livekit_client, [Quit]); + +#[cfg(windows)] +fn main() {} + +#[cfg(not(windows))] +fn main() { + SimpleLogger::init(LevelFilter::Info, Default::default()).expect("could not initialize logger"); + + gpui::App::new().run(|cx| { + livekit_client::init( + cx.background_executor().dispatcher.clone(), + cx.http_client(), + ); + + #[cfg(any(test, feature = "test-support"))] + println!("USING TEST LIVEKIT"); + + #[cfg(not(any(test, feature = "test-support")))] + println!("USING REAL LIVEKIT"); + + cx.activate(true); + cx.on_action(quit); + cx.bind_keys([KeyBinding::new("cmd-q", Quit, None)]); + cx.set_menus(vec![Menu { + name: "Zed".into(), + items: vec![MenuItem::Action { + name: "Quit".into(), + action: Box::new(Quit), + os_action: None, + }], + }]); + + let livekit_url = std::env::var("LIVEKIT_URL").unwrap_or("http://localhost:7880".into()); + let livekit_key = std::env::var("LIVEKIT_KEY").unwrap_or("devkey".into()); + let livekit_secret = std::env::var("LIVEKIT_SECRET").unwrap_or("secret".into()); + let height = px(800.); + let width = px(800.); + + cx.spawn(|cx| async move { + let mut windows = Vec::new(); + for i in 0..2 { + let token = token::create( + &livekit_key, + &livekit_secret, + Some(&format!("test-participant-{i}")), + VideoGrant::to_join("test-room"), + ) + .unwrap(); + + let bounds = bounds(point(width * i, px(0.0)), size(width, height)); + let window = + LivekitWindow::new(livekit_url.as_str(), token.as_str(), bounds, cx.clone()) + .await; + windows.push(window); + } + }) + .detach(); + }); +} + +fn quit(_: &Quit, cx: &mut gpui::AppContext) { + cx.quit(); +} + +struct LivekitWindow { + room: Room, + microphone_track: Option, + screen_share_track: Option, + microphone_stream: Option, + screen_share_stream: Option>, + #[cfg(not(target_os = "windows"))] + remote_participants: Vec<(ParticipantIdentity, ParticipantState)>, + _events_task: Task<()>, +} + +#[derive(Default)] +struct ParticipantState { + audio_output_stream: Option<(RemoteTrackPublication, AudioStream)>, + muted: bool, + screen_share_output_view: Option<(RemoteVideoTrack, View)>, + speaking: bool, +} + +#[cfg(not(windows))] +impl LivekitWindow { + async fn new( + url: &str, + token: &str, + bounds: Bounds, + cx: AsyncAppContext, + ) -> WindowHandle { + let (room, mut events) = Room::connect(url, token, RoomOptions::default()) + .await + .unwrap(); + + cx.update(|cx| { + cx.open_window( + WindowOptions { + window_bounds: Some(WindowBounds::Windowed(bounds)), + ..Default::default() + }, + |cx| { + cx.new_view(|cx| { + let _events_task = cx.spawn(|this, mut cx| async move { + while let Some(event) = events.recv().await { + this.update(&mut cx, |this: &mut LivekitWindow, cx| { + this.handle_room_event(event, cx) + }) + .ok(); + } + }); + + Self { + room, + microphone_track: None, + microphone_stream: None, + screen_share_track: None, + screen_share_stream: None, + remote_participants: Vec::new(), + _events_task, + } + }) + }, + ) + .unwrap() + }) + .unwrap() + } + + fn handle_room_event(&mut self, event: RoomEvent, cx: &mut ViewContext) { + eprintln!("event: {event:?}"); + + match event { + RoomEvent::TrackUnpublished { + publication, + participant, + } => { + let output = self.remote_participant(participant); + let unpublish_sid = publication.sid(); + if output + .audio_output_stream + .as_ref() + .map_or(false, |(track, _)| track.sid() == unpublish_sid) + { + output.audio_output_stream.take(); + } + if output + .screen_share_output_view + .as_ref() + .map_or(false, |(track, _)| track.sid() == unpublish_sid) + { + output.screen_share_output_view.take(); + } + cx.notify(); + } + + RoomEvent::TrackSubscribed { + publication, + participant, + track, + } => { + let output = self.remote_participant(participant); + match track { + RemoteTrack::Audio(track) => { + output.audio_output_stream = Some(( + publication.clone(), + play_remote_audio_track(&track, cx.background_executor()).unwrap(), + )); + } + RemoteTrack::Video(track) => { + output.screen_share_output_view = Some(( + track.clone(), + cx.new_view(|cx| RemoteVideoTrackView::new(track, cx)), + )); + } + } + cx.notify(); + } + + RoomEvent::TrackMuted { participant, .. } => { + if let Participant::Remote(participant) = participant { + self.remote_participant(participant).muted = true; + cx.notify(); + } + } + + RoomEvent::TrackUnmuted { participant, .. } => { + if let Participant::Remote(participant) = participant { + self.remote_participant(participant).muted = false; + cx.notify(); + } + } + + RoomEvent::ActiveSpeakersChanged { speakers } => { + for (identity, output) in &mut self.remote_participants { + output.speaking = speakers.iter().any(|speaker| { + if let Participant::Remote(speaker) = speaker { + speaker.identity() == *identity + } else { + false + } + }); + } + cx.notify(); + } + + _ => {} + } + + cx.notify(); + } + + fn remote_participant(&mut self, participant: RemoteParticipant) -> &mut ParticipantState { + match self + .remote_participants + .binary_search_by_key(&&participant.identity(), |row| &row.0) + { + Ok(ix) => &mut self.remote_participants[ix].1, + Err(ix) => { + self.remote_participants + .insert(ix, (participant.identity(), ParticipantState::default())); + &mut self.remote_participants[ix].1 + } + } + } + + fn toggle_mute(&mut self, cx: &mut ViewContext) { + if let Some(track) = &self.microphone_track { + if track.is_muted() { + track.unmute(); + } else { + track.mute(); + } + cx.notify(); + } else { + let participant = self.room.local_participant(); + cx.spawn(|this, mut cx| async move { + let (track, stream) = capture_local_audio_track(cx.background_executor())?.await; + let publication = participant + .publish_track( + LocalTrack::Audio(track), + TrackPublishOptions { + source: TrackSource::Microphone, + ..Default::default() + }, + ) + .await + .unwrap(); + this.update(&mut cx, |this, cx| { + this.microphone_track = Some(publication); + this.microphone_stream = Some(stream); + cx.notify(); + }) + }) + .detach(); + } + } + + fn toggle_screen_share(&mut self, cx: &mut ViewContext) { + if let Some(track) = self.screen_share_track.take() { + self.screen_share_stream.take(); + let participant = self.room.local_participant(); + cx.background_executor() + .spawn(async move { + participant.unpublish_track(&track.sid()).await.unwrap(); + }) + .detach(); + cx.notify(); + } else { + let participant = self.room.local_participant(); + let sources = cx.screen_capture_sources(); + cx.spawn(|this, mut cx| async move { + let sources = sources.await.unwrap()?; + let source = sources.into_iter().next().unwrap(); + let (track, stream) = capture_local_video_track(&*source).await?; + let publication = participant + .publish_track( + LocalTrack::Video(track), + TrackPublishOptions { + source: TrackSource::Screenshare, + video_codec: VideoCodec::H264, + ..Default::default() + }, + ) + .await + .unwrap(); + this.update(&mut cx, |this, cx| { + this.screen_share_track = Some(publication); + this.screen_share_stream = Some(stream); + cx.notify(); + }) + }) + .detach(); + } + } + + fn toggle_remote_audio_for_participant( + &mut self, + identity: &ParticipantIdentity, + cx: &mut ViewContext, + ) -> Option<()> { + let participant = self.remote_participants.iter().find_map(|(id, state)| { + if id == identity { + Some(state) + } else { + None + } + })?; + let publication = &participant.audio_output_stream.as_ref()?.0; + publication.set_enabled(!publication.is_enabled()); + cx.notify(); + Some(()) + } +} + +#[cfg(not(windows))] +impl Render for LivekitWindow { + fn render(&mut self, cx: &mut ViewContext) -> impl IntoElement { + fn button() -> gpui::Div { + div() + .w(px(180.0)) + .h(px(30.0)) + .px_2() + .m_2() + .bg(rgb(0x8888ff)) + } + + div() + .bg(rgb(0xffffff)) + .size_full() + .flex() + .flex_col() + .child( + div().bg(rgb(0xffd4a8)).flex().flex_row().children([ + button() + .id("toggle-mute") + .child(if let Some(track) = &self.microphone_track { + if track.is_muted() { + "Unmute" + } else { + "Mute" + } + } else { + "Publish mic" + }) + .on_click(cx.listener(|this, _, cx| this.toggle_mute(cx))), + button() + .id("toggle-screen-share") + .child(if self.screen_share_track.is_none() { + "Share screen" + } else { + "Unshare screen" + }) + .on_click(cx.listener(|this, _, cx| this.toggle_screen_share(cx))), + ]), + ) + .child( + div() + .id("remote-participants") + .overflow_y_scroll() + .flex() + .flex_col() + .flex_grow() + .children(self.remote_participants.iter().map(|(identity, state)| { + div() + .h(px(300.0)) + .flex() + .flex_col() + .m_2() + .px_2() + .bg(rgb(0x8888ff)) + .child(SharedString::from(if state.speaking { + format!("{} (speaking)", &identity.0) + } else if state.muted { + format!("{} (muted)", &identity.0) + } else { + identity.0.clone() + })) + .when_some(state.audio_output_stream.as_ref(), |el, state| { + el.child( + button() + .id(SharedString::from(identity.0.clone())) + .child(if state.0.is_enabled() { + "Deafen" + } else { + "Undeafen" + }) + .on_click(cx.listener({ + let identity = identity.clone(); + move |this, _, cx| { + this.toggle_remote_audio_for_participant( + &identity, cx, + ); + } + })), + ) + }) + .children(state.screen_share_output_view.as_ref().map(|e| e.1.clone())) + })), + ) + } +} diff --git a/crates/livekit_client/src/livekit_client.rs b/crates/livekit_client/src/livekit_client.rs new file mode 100644 index 0000000000000..836618e3ac62d --- /dev/null +++ b/crates/livekit_client/src/livekit_client.rs @@ -0,0 +1,657 @@ +#![cfg_attr(target_os = "windows", allow(unused))] + +mod remote_video_track_view; +#[cfg(any(test, feature = "test-support", target_os = "windows"))] +pub mod test; + +use anyhow::{anyhow, Context as _, Result}; +use cpal::traits::{DeviceTrait, HostTrait, StreamTrait as _}; +use futures::{io, Stream, StreamExt as _}; +use gpui::{ + BackgroundExecutor, ScreenCaptureFrame, ScreenCaptureSource, ScreenCaptureStream, Task, +}; +use parking_lot::Mutex; +use std::{borrow::Cow, collections::VecDeque, future::Future, pin::Pin, sync::Arc, thread}; +use util::{debug_panic, ResultExt as _}; +#[cfg(not(target_os = "windows"))] +use webrtc::{ + audio_frame::AudioFrame, + audio_source::{native::NativeAudioSource, AudioSourceOptions, RtcAudioSource}, + audio_stream::native::NativeAudioStream, + video_frame::{VideoBuffer, VideoFrame, VideoRotation}, + video_source::{native::NativeVideoSource, RtcVideoSource, VideoResolution}, + video_stream::native::NativeVideoStream, +}; + +#[cfg(all(not(any(test, feature = "test-support")), not(target_os = "windows")))] +use livekit::track::RemoteAudioTrack; +#[cfg(all(not(any(test, feature = "test-support")), not(target_os = "windows")))] +pub use livekit::*; +#[cfg(any(test, feature = "test-support", target_os = "windows"))] +use test::track::RemoteAudioTrack; +#[cfg(any(test, feature = "test-support", target_os = "windows"))] +pub use test::*; + +pub use remote_video_track_view::{RemoteVideoTrackView, RemoteVideoTrackViewEvent}; + +pub enum AudioStream { + Input { + _thread_handle: std::sync::mpsc::Sender<()>, + _transmit_task: Task<()>, + }, + Output { + _task: Task<()>, + }, +} + +struct Dispatcher(Arc); + +#[cfg(not(target_os = "windows"))] +impl livekit::dispatcher::Dispatcher for Dispatcher { + fn dispatch(&self, runnable: livekit::dispatcher::Runnable) { + self.0.dispatch(runnable, None); + } + + fn dispatch_after( + &self, + duration: std::time::Duration, + runnable: livekit::dispatcher::Runnable, + ) { + self.0.dispatch_after(duration, runnable); + } +} + +struct HttpClientAdapter(Arc); + +fn http_2_status(status: http_client::http::StatusCode) -> http_2::StatusCode { + http_2::StatusCode::from_u16(status.as_u16()) + .expect("valid status code to status code conversion") +} + +#[cfg(not(target_os = "windows"))] +impl livekit::dispatcher::HttpClient for HttpClientAdapter { + fn get( + &self, + url: &str, + ) -> Pin> + Send>> { + let http_client = self.0.clone(); + let url = url.to_string(); + Box::pin(async move { + let response = http_client + .get(&url, http_client::AsyncBody::empty(), false) + .await + .map_err(io::Error::other)?; + Ok(livekit::dispatcher::Response { + status: http_2_status(response.status()), + body: Box::pin(response.into_body()), + }) + }) + } + + fn send_async( + &self, + request: http_2::Request>, + ) -> Pin> + Send>> { + let http_client = self.0.clone(); + let mut builder = http_client::http::Request::builder() + .method(request.method().as_str()) + .uri(request.uri().to_string()); + + for (key, value) in request.headers().iter() { + builder = builder.header(key.as_str(), value.as_bytes()); + } + + if !request.extensions().is_empty() { + debug_panic!( + "Livekit sent an HTTP request with a protocol extension that Zed doesn't support!" + ); + } + + let request = builder + .body(http_client::AsyncBody::from_bytes( + request.into_body().into(), + )) + .unwrap(); + + Box::pin(async move { + let response = http_client.send(request).await.map_err(io::Error::other)?; + Ok(livekit::dispatcher::Response { + status: http_2_status(response.status()), + body: Box::pin(response.into_body()), + }) + }) + } +} + +#[cfg(target_os = "windows")] +pub fn init( + dispatcher: Arc, + http_client: Arc, +) { +} + +#[cfg(not(target_os = "windows"))] +pub fn init( + dispatcher: Arc, + http_client: Arc, +) { + livekit::dispatcher::set_dispatcher(Dispatcher(dispatcher)); + livekit::dispatcher::set_http_client(HttpClientAdapter(http_client)); +} + +#[cfg(not(target_os = "windows"))] +pub async fn capture_local_video_track( + capture_source: &dyn ScreenCaptureSource, +) -> Result<(track::LocalVideoTrack, Box)> { + let resolution = capture_source.resolution()?; + let track_source = NativeVideoSource::new(VideoResolution { + width: resolution.width.0 as u32, + height: resolution.height.0 as u32, + }); + + let capture_stream = capture_source + .stream({ + let track_source = track_source.clone(); + Box::new(move |frame| { + if let Some(buffer) = video_frame_buffer_to_webrtc(frame) { + track_source.capture_frame(&VideoFrame { + rotation: VideoRotation::VideoRotation0, + timestamp_us: 0, + buffer, + }); + } + }) + }) + .await??; + + Ok(( + track::LocalVideoTrack::create_video_track( + "screen share", + RtcVideoSource::Native(track_source), + ), + capture_stream, + )) +} + +#[cfg(not(target_os = "windows"))] +pub fn capture_local_audio_track( + background_executor: &BackgroundExecutor, +) -> Result> { + use util::maybe; + + let (frame_tx, mut frame_rx) = futures::channel::mpsc::unbounded(); + let (thread_handle, thread_kill_rx) = std::sync::mpsc::channel::<()>(); + let sample_rate; + let channels; + + if cfg!(any(test, feature = "test-support")) { + sample_rate = 2; + channels = 1; + } else { + let (device, config) = default_device(true)?; + sample_rate = config.sample_rate().0; + channels = config.channels() as u32; + thread::spawn(move || { + maybe!({ + if let Some(name) = device.name().ok() { + log::info!("Using microphone: {}", name) + } else { + log::info!("Using microphone: "); + } + + let stream = device + .build_input_stream_raw( + &config.config(), + cpal::SampleFormat::I16, + move |data, _: &_| { + frame_tx + .unbounded_send(AudioFrame { + data: Cow::Owned(data.as_slice::().unwrap().to_vec()), + sample_rate, + num_channels: channels, + samples_per_channel: data.len() as u32 / channels, + }) + .ok(); + }, + |err| log::error!("error capturing audio track: {:?}", err), + None, + ) + .context("failed to build input stream")?; + + stream.play()?; + // Keep the thread alive and holding onto the `stream` + thread_kill_rx.recv().ok(); + anyhow::Ok(Some(())) + }) + .log_err(); + }); + } + + Ok(background_executor.spawn({ + let background_executor = background_executor.clone(); + async move { + let source = NativeAudioSource::new( + AudioSourceOptions { + echo_cancellation: true, + noise_suppression: true, + auto_gain_control: true, + }, + sample_rate, + channels, + 100, + ); + let transmit_task = background_executor.spawn({ + let source = source.clone(); + async move { + while let Some(frame) = frame_rx.next().await { + source.capture_frame(&frame).await.log_err(); + } + } + }); + + let track = track::LocalAudioTrack::create_audio_track( + "microphone", + RtcAudioSource::Native(source), + ); + + ( + track, + AudioStream::Input { + _thread_handle: thread_handle, + _transmit_task: transmit_task, + }, + ) + } + })) +} + +#[cfg(not(target_os = "windows"))] +pub fn play_remote_audio_track( + track: &RemoteAudioTrack, + background_executor: &BackgroundExecutor, +) -> Result { + let track = track.clone(); + // We track device changes in our output because Livekit has a resampler built in, + // and it's easy to create a new native audio stream when the device changes. + if cfg!(any(test, feature = "test-support")) { + Ok(AudioStream::Output { + _task: background_executor.spawn(async {}), + }) + } else { + let mut default_change_listener = DeviceChangeListener::new(false)?; + let (output_device, output_config) = default_device(false)?; + + let _task = background_executor.spawn({ + let background_executor = background_executor.clone(); + async move { + let (mut _receive_task, mut _thread) = + start_output_stream(output_config, output_device, &track, &background_executor); + + while let Some(_) = default_change_listener.next().await { + let Some((output_device, output_config)) = get_default_output().log_err() + else { + continue; + }; + + if let Ok(name) = output_device.name() { + log::info!("Using speaker: {}", name) + } else { + log::info!("Using speaker: ") + } + + (_receive_task, _thread) = start_output_stream( + output_config, + output_device, + &track, + &background_executor, + ); + } + + futures::future::pending::<()>().await; + } + }); + + Ok(AudioStream::Output { _task }) + } +} + +fn default_device(input: bool) -> anyhow::Result<(cpal::Device, cpal::SupportedStreamConfig)> { + let device; + let config; + if input { + device = cpal::default_host() + .default_input_device() + .ok_or_else(|| anyhow!("no audio input device available"))?; + config = device + .default_input_config() + .context("failed to get default input config")?; + } else { + device = cpal::default_host() + .default_output_device() + .ok_or_else(|| anyhow!("no audio output device available"))?; + config = device + .default_output_config() + .context("failed to get default output config")?; + } + Ok((device, config)) +} + +#[cfg(not(target_os = "windows"))] +fn get_default_output() -> anyhow::Result<(cpal::Device, cpal::SupportedStreamConfig)> { + let host = cpal::default_host(); + let output_device = host + .default_output_device() + .context("failed to read default output device")?; + let output_config = output_device.default_output_config()?; + Ok((output_device, output_config)) +} + +#[cfg(not(target_os = "windows"))] +fn start_output_stream( + output_config: cpal::SupportedStreamConfig, + output_device: cpal::Device, + track: &track::RemoteAudioTrack, + background_executor: &BackgroundExecutor, +) -> (Task<()>, std::sync::mpsc::Sender<()>) { + let buffer = Arc::new(Mutex::new(VecDeque::::new())); + let sample_rate = output_config.sample_rate(); + + let mut stream = NativeAudioStream::new( + track.rtc_track(), + sample_rate.0 as i32, + output_config.channels() as i32, + ); + + let receive_task = background_executor.spawn({ + let buffer = buffer.clone(); + async move { + const MS_OF_BUFFER: u32 = 100; + const MS_IN_SEC: u32 = 1000; + while let Some(frame) = stream.next().await { + let frame_size = frame.samples_per_channel * frame.num_channels; + debug_assert!(frame.data.len() == frame_size as usize); + + let buffer_size = + ((frame.sample_rate * frame.num_channels) / MS_IN_SEC * MS_OF_BUFFER) as usize; + + let mut buffer = buffer.lock(); + let new_size = buffer.len() + frame.data.len(); + if new_size > buffer_size { + let overflow = new_size - buffer_size; + buffer.drain(0..overflow); + } + + buffer.extend(frame.data.iter()); + } + } + }); + + // The _output_stream needs to be on it's own thread because it's !Send + // and we experienced a deadlock when it's created on the main thread. + let (thread, end_on_drop_rx) = std::sync::mpsc::channel::<()>(); + thread::spawn(move || { + if cfg!(any(test, feature = "test-support")) { + // Can't play audio in tests + return; + } + + let output_stream = output_device.build_output_stream( + &output_config.config(), + { + let buffer = buffer.clone(); + move |data, _info| { + let mut buffer = buffer.lock(); + if buffer.len() < data.len() { + data.fill(0); + } else { + // SAFETY: We know that buffer has at least data.len() values in it. + // because we just checked + let mut drain = buffer.drain(..data.len()); + data.fill_with(|| unsafe { drain.next().unwrap_unchecked() }); + } + } + }, + |error| log::error!("error playing audio track: {:?}", error), + None, + ); + + let Some(output_stream) = output_stream.log_err() else { + return; + }; + + output_stream.play().log_err(); + // Block forever to keep the output stream alive + end_on_drop_rx.recv().ok(); + }); + + (receive_task, thread) +} + +#[cfg(target_os = "windows")] +pub fn play_remote_video_track( + track: &track::RemoteVideoTrack, +) -> impl Stream { + futures::stream::empty() +} + +#[cfg(not(target_os = "windows"))] +pub fn play_remote_video_track( + track: &track::RemoteVideoTrack, +) -> impl Stream { + NativeVideoStream::new(track.rtc_track()) + .filter_map(|frame| async move { video_frame_buffer_from_webrtc(frame.buffer) }) +} + +#[cfg(target_os = "macos")] +pub type RemoteVideoFrame = media::core_video::CVImageBuffer; + +#[cfg(target_os = "macos")] +fn video_frame_buffer_from_webrtc(buffer: Box) -> Option { + use core_foundation::base::TCFType as _; + use media::core_video::CVImageBuffer; + + let buffer = buffer.as_native()?; + let pixel_buffer = buffer.get_cv_pixel_buffer(); + if pixel_buffer.is_null() { + return None; + } + + unsafe { Some(CVImageBuffer::wrap_under_get_rule(pixel_buffer as _)) } +} + +#[cfg(not(target_os = "macos"))] +pub type RemoteVideoFrame = Arc; + +#[cfg(not(any(target_os = "macos", target_os = "windows")))] +fn video_frame_buffer_from_webrtc(buffer: Box) -> Option { + use gpui::RenderImage; + use image::{Frame, RgbaImage}; + use livekit::webrtc::prelude::VideoFormatType; + use smallvec::SmallVec; + use std::alloc::{alloc, Layout}; + + let width = buffer.width(); + let height = buffer.height(); + let stride = width * 4; + let byte_len = (stride * height) as usize; + let argb_image = unsafe { + // Motivation for this unsafe code is to avoid initializing the frame data, since to_argb + // will write all bytes anyway. + let start_ptr = alloc(Layout::array::(byte_len).log_err()?); + if start_ptr.is_null() { + return None; + } + let bgra_frame_slice = std::slice::from_raw_parts_mut(start_ptr, byte_len); + buffer.to_argb( + VideoFormatType::ARGB, // For some reason, this displays correctly while RGBA (the correct format) does not + bgra_frame_slice, + stride, + width as i32, + height as i32, + ); + Vec::from_raw_parts(start_ptr, byte_len, byte_len) + }; + + Some(Arc::new(RenderImage::new(SmallVec::from_elem( + Frame::new( + RgbaImage::from_raw(width, height, argb_image) + .with_context(|| "Bug: not enough bytes allocated for image.") + .log_err()?, + ), + 1, + )))) +} + +#[cfg(target_os = "macos")] +fn video_frame_buffer_to_webrtc(frame: ScreenCaptureFrame) -> Option> { + use core_foundation::base::TCFType as _; + + let pixel_buffer = frame.0.as_concrete_TypeRef(); + std::mem::forget(frame.0); + unsafe { + Some(webrtc::video_frame::native::NativeBuffer::from_cv_pixel_buffer(pixel_buffer as _)) + } +} + +#[cfg(not(any(target_os = "macos", target_os = "windows")))] +fn video_frame_buffer_to_webrtc(_frame: ScreenCaptureFrame) -> Option> { + None as Option> +} + +trait DeviceChangeListenerApi: Stream + Sized { + fn new(input: bool) -> Result; +} + +#[cfg(target_os = "macos")] +mod macos { + + use coreaudio::sys::{ + kAudioHardwarePropertyDefaultInputDevice, kAudioHardwarePropertyDefaultOutputDevice, + kAudioObjectPropertyElementMaster, kAudioObjectPropertyScopeGlobal, + kAudioObjectSystemObject, AudioObjectAddPropertyListener, AudioObjectID, + AudioObjectPropertyAddress, AudioObjectRemovePropertyListener, OSStatus, + }; + use futures::{channel::mpsc::UnboundedReceiver, StreamExt}; + + use crate::DeviceChangeListenerApi; + + /// Implementation from: https://github.com/zed-industries/cpal/blob/fd8bc2fd39f1f5fdee5a0690656caff9a26d9d50/src/host/coreaudio/macos/property_listener.rs#L15 + pub struct CoreAudioDefaultDeviceChangeListener { + rx: UnboundedReceiver<()>, + callback: Box, + input: bool, + } + + trait _AssertSend: Send {} + impl _AssertSend for CoreAudioDefaultDeviceChangeListener {} + + struct PropertyListenerCallbackWrapper(Box); + + unsafe extern "C" fn property_listener_handler_shim( + _: AudioObjectID, + _: u32, + _: *const AudioObjectPropertyAddress, + callback: *mut ::std::os::raw::c_void, + ) -> OSStatus { + let wrapper = callback as *mut PropertyListenerCallbackWrapper; + (*wrapper).0(); + 0 + } + + impl DeviceChangeListenerApi for CoreAudioDefaultDeviceChangeListener { + fn new(input: bool) -> gpui::Result { + let (tx, rx) = futures::channel::mpsc::unbounded(); + + let callback = Box::new(PropertyListenerCallbackWrapper(Box::new(move || { + tx.unbounded_send(()).ok(); + }))); + + unsafe { + coreaudio::Error::from_os_status(AudioObjectAddPropertyListener( + kAudioObjectSystemObject, + &AudioObjectPropertyAddress { + mSelector: if input { + kAudioHardwarePropertyDefaultInputDevice + } else { + kAudioHardwarePropertyDefaultOutputDevice + }, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMaster, + }, + Some(property_listener_handler_shim), + &*callback as *const _ as *mut _, + ))?; + } + + Ok(Self { + rx, + callback, + input, + }) + } + } + + impl Drop for CoreAudioDefaultDeviceChangeListener { + fn drop(&mut self) { + unsafe { + AudioObjectRemovePropertyListener( + kAudioObjectSystemObject, + &AudioObjectPropertyAddress { + mSelector: if self.input { + kAudioHardwarePropertyDefaultInputDevice + } else { + kAudioHardwarePropertyDefaultOutputDevice + }, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMaster, + }, + Some(property_listener_handler_shim), + &*self.callback as *const _ as *mut _, + ); + } + } + } + + impl futures::Stream for CoreAudioDefaultDeviceChangeListener { + type Item = (); + + fn poll_next( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + self.rx.poll_next_unpin(cx) + } + } +} + +#[cfg(target_os = "macos")] +type DeviceChangeListener = macos::CoreAudioDefaultDeviceChangeListener; + +#[cfg(not(target_os = "macos"))] +mod noop_change_listener { + use std::task::Poll; + + use crate::DeviceChangeListenerApi; + + pub struct NoopOutputDeviceChangelistener {} + + impl DeviceChangeListenerApi for NoopOutputDeviceChangelistener { + fn new(_input: bool) -> anyhow::Result { + Ok(NoopOutputDeviceChangelistener {}) + } + } + + impl futures::Stream for NoopOutputDeviceChangelistener { + type Item = (); + + fn poll_next( + self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + ) -> Poll> { + Poll::Pending + } + } +} + +#[cfg(not(target_os = "macos"))] +type DeviceChangeListener = noop_change_listener::NoopOutputDeviceChangelistener; diff --git a/crates/livekit_client/src/remote_video_track_view.rs b/crates/livekit_client/src/remote_video_track_view.rs new file mode 100644 index 0000000000000..d7618391d68a5 --- /dev/null +++ b/crates/livekit_client/src/remote_video_track_view.rs @@ -0,0 +1,99 @@ +use crate::track::RemoteVideoTrack; +use anyhow::Result; +use futures::StreamExt as _; +use gpui::{Empty, EventEmitter, IntoElement, Render, Task, View, ViewContext, VisualContext as _}; + +pub struct RemoteVideoTrackView { + track: RemoteVideoTrack, + latest_frame: Option, + #[cfg(not(target_os = "macos"))] + current_rendered_frame: Option, + #[cfg(not(target_os = "macos"))] + previous_rendered_frame: Option, + _maintain_frame: Task>, +} + +#[derive(Debug)] +pub enum RemoteVideoTrackViewEvent { + Close, +} + +impl RemoteVideoTrackView { + pub fn new(track: RemoteVideoTrack, cx: &mut ViewContext) -> Self { + cx.focus_handle(); + let frames = super::play_remote_video_track(&track); + + Self { + track, + latest_frame: None, + _maintain_frame: cx.spawn(|this, mut cx| async move { + futures::pin_mut!(frames); + while let Some(frame) = frames.next().await { + this.update(&mut cx, |this, cx| { + this.latest_frame = Some(frame); + cx.notify(); + })?; + } + this.update(&mut cx, |_this, cx| { + #[cfg(not(target_os = "macos"))] + { + use util::ResultExt as _; + if let Some(frame) = _this.previous_rendered_frame.take() { + cx.window_context().drop_image(frame).log_err(); + } + // TODO(mgsloan): This might leak the last image of the screenshare if + // render is called after the screenshare ends. + if let Some(frame) = _this.current_rendered_frame.take() { + cx.window_context().drop_image(frame).log_err(); + } + } + cx.emit(RemoteVideoTrackViewEvent::Close) + })?; + Ok(()) + }), + #[cfg(not(target_os = "macos"))] + current_rendered_frame: None, + #[cfg(not(target_os = "macos"))] + previous_rendered_frame: None, + } + } + + pub fn clone(&self, cx: &mut ViewContext) -> View { + cx.new_view(|cx| Self::new(self.track.clone(), cx)) + } +} + +impl EventEmitter for RemoteVideoTrackView {} + +impl Render for RemoteVideoTrackView { + fn render(&mut self, _cx: &mut ViewContext) -> impl IntoElement { + #[cfg(target_os = "macos")] + if let Some(latest_frame) = &self.latest_frame { + use gpui::Styled as _; + return gpui::surface(latest_frame.clone()) + .size_full() + .into_any_element(); + } + + #[cfg(not(target_os = "macos"))] + if let Some(latest_frame) = &self.latest_frame { + use gpui::Styled as _; + if let Some(current_rendered_frame) = self.current_rendered_frame.take() { + if let Some(frame) = self.previous_rendered_frame.take() { + // Only drop the frame if it's not also the current frame. + if frame.id != current_rendered_frame.id { + use util::ResultExt as _; + _cx.window_context().drop_image(frame).log_err(); + } + } + self.previous_rendered_frame = Some(current_rendered_frame) + } + self.current_rendered_frame = Some(latest_frame.clone()); + return gpui::img(latest_frame.clone()) + .size_full() + .into_any_element(); + } + + Empty.into_any_element() + } +} diff --git a/crates/livekit_client/src/test.rs b/crates/livekit_client/src/test.rs new file mode 100644 index 0000000000000..e67189c09cfbb --- /dev/null +++ b/crates/livekit_client/src/test.rs @@ -0,0 +1,825 @@ +pub mod participant; +pub mod publication; +pub mod track; + +#[cfg(not(windows))] +pub mod webrtc; + +#[cfg(not(windows))] +use self::id::*; +use self::{participant::*, publication::*, track::*}; +use anyhow::{anyhow, Context, Result}; +use async_trait::async_trait; +use collections::{btree_map::Entry as BTreeEntry, hash_map::Entry, BTreeMap, HashMap, HashSet}; +use gpui::BackgroundExecutor; +#[cfg(not(windows))] +use livekit::options::TrackPublishOptions; +use livekit_server::{proto, token}; +use parking_lot::Mutex; +use postage::{mpsc, sink::Sink}; +use std::sync::{ + atomic::{AtomicBool, Ordering::SeqCst}, + Arc, Weak, +}; + +#[cfg(not(windows))] +pub use livekit::{id, options, ConnectionState, DisconnectReason, RoomOptions}; + +static SERVERS: Mutex>> = Mutex::new(BTreeMap::new()); + +pub struct TestServer { + pub url: String, + pub api_key: String, + pub secret_key: String, + #[cfg(not(target_os = "windows"))] + rooms: Mutex>, + executor: BackgroundExecutor, +} + +#[cfg(not(target_os = "windows"))] +impl TestServer { + pub fn create( + url: String, + api_key: String, + secret_key: String, + executor: BackgroundExecutor, + ) -> Result> { + let mut servers = SERVERS.lock(); + if let BTreeEntry::Vacant(e) = servers.entry(url.clone()) { + let server = Arc::new(TestServer { + url, + api_key, + secret_key, + rooms: Default::default(), + executor, + }); + e.insert(server.clone()); + Ok(server) + } else { + Err(anyhow!("a server with url {:?} already exists", url)) + } + } + + fn get(url: &str) -> Result> { + Ok(SERVERS + .lock() + .get(url) + .ok_or_else(|| anyhow!("no server found for url"))? + .clone()) + } + + pub fn teardown(&self) -> Result<()> { + SERVERS + .lock() + .remove(&self.url) + .ok_or_else(|| anyhow!("server with url {:?} does not exist", self.url))?; + Ok(()) + } + + pub fn create_api_client(&self) -> TestApiClient { + TestApiClient { + url: self.url.clone(), + } + } + + pub async fn create_room(&self, room: String) -> Result<()> { + self.executor.simulate_random_delay().await; + + let mut server_rooms = self.rooms.lock(); + if let Entry::Vacant(e) = server_rooms.entry(room.clone()) { + e.insert(Default::default()); + Ok(()) + } else { + Err(anyhow!("room {:?} already exists", room)) + } + } + + async fn delete_room(&self, room: String) -> Result<()> { + self.executor.simulate_random_delay().await; + + let mut server_rooms = self.rooms.lock(); + server_rooms + .remove(&room) + .ok_or_else(|| anyhow!("room {:?} does not exist", room))?; + Ok(()) + } + + async fn join_room(&self, token: String, client_room: Room) -> Result { + self.executor.simulate_random_delay().await; + + let claims = livekit_server::token::validate(&token, &self.secret_key)?; + let identity = ParticipantIdentity(claims.sub.unwrap().to_string()); + let room_name = claims.video.room.unwrap(); + let mut server_rooms = self.rooms.lock(); + let room = (*server_rooms).entry(room_name.to_string()).or_default(); + + if let Entry::Vacant(e) = room.client_rooms.entry(identity.clone()) { + for server_track in &room.video_tracks { + let track = RemoteTrack::Video(RemoteVideoTrack { + server_track: server_track.clone(), + _room: client_room.downgrade(), + }); + client_room + .0 + .lock() + .updates_tx + .blocking_send(RoomEvent::TrackSubscribed { + track: track.clone(), + publication: RemoteTrackPublication { + sid: server_track.sid.clone(), + room: client_room.downgrade(), + track, + }, + participant: RemoteParticipant { + room: client_room.downgrade(), + identity: server_track.publisher_id.clone(), + }, + }) + .unwrap(); + } + for server_track in &room.audio_tracks { + let track = RemoteTrack::Audio(RemoteAudioTrack { + server_track: server_track.clone(), + room: client_room.downgrade(), + }); + client_room + .0 + .lock() + .updates_tx + .blocking_send(RoomEvent::TrackSubscribed { + track: track.clone(), + publication: RemoteTrackPublication { + sid: server_track.sid.clone(), + room: client_room.downgrade(), + track, + }, + participant: RemoteParticipant { + room: client_room.downgrade(), + identity: server_track.publisher_id.clone(), + }, + }) + .unwrap(); + } + e.insert(client_room); + Ok(identity) + } else { + Err(anyhow!( + "{:?} attempted to join room {:?} twice", + identity, + room_name + )) + } + } + + async fn leave_room(&self, token: String) -> Result<()> { + self.executor.simulate_random_delay().await; + + let claims = livekit_server::token::validate(&token, &self.secret_key)?; + let identity = ParticipantIdentity(claims.sub.unwrap().to_string()); + let room_name = claims.video.room.unwrap(); + let mut server_rooms = self.rooms.lock(); + let room = server_rooms + .get_mut(&*room_name) + .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; + room.client_rooms.remove(&identity).ok_or_else(|| { + anyhow!( + "{:?} attempted to leave room {:?} before joining it", + identity, + room_name + ) + })?; + Ok(()) + } + + fn remote_participants( + &self, + token: String, + ) -> Result> { + let claims = livekit_server::token::validate(&token, &self.secret_key)?; + let local_identity = ParticipantIdentity(claims.sub.unwrap().to_string()); + let room_name = claims.video.room.unwrap().to_string(); + + if let Some(server_room) = self.rooms.lock().get(&room_name) { + let room = server_room + .client_rooms + .get(&local_identity) + .unwrap() + .downgrade(); + Ok(server_room + .client_rooms + .iter() + .filter(|(identity, _)| *identity != &local_identity) + .map(|(identity, _)| { + ( + identity.clone(), + RemoteParticipant { + room: room.clone(), + identity: identity.clone(), + }, + ) + }) + .collect()) + } else { + Ok(Default::default()) + } + } + + async fn remove_participant( + &self, + room_name: String, + identity: ParticipantIdentity, + ) -> Result<()> { + self.executor.simulate_random_delay().await; + + let mut server_rooms = self.rooms.lock(); + let room = server_rooms + .get_mut(&room_name) + .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; + room.client_rooms.remove(&identity).ok_or_else(|| { + anyhow!( + "participant {:?} did not join room {:?}", + identity, + room_name + ) + })?; + Ok(()) + } + + async fn update_participant( + &self, + room_name: String, + identity: String, + permission: proto::ParticipantPermission, + ) -> Result<()> { + self.executor.simulate_random_delay().await; + + let mut server_rooms = self.rooms.lock(); + let room = server_rooms + .get_mut(&room_name) + .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; + room.participant_permissions + .insert(ParticipantIdentity(identity), permission); + Ok(()) + } + + pub async fn disconnect_client(&self, client_identity: String) { + let client_identity = ParticipantIdentity(client_identity); + + self.executor.simulate_random_delay().await; + + let mut server_rooms = self.rooms.lock(); + for room in server_rooms.values_mut() { + if let Some(room) = room.client_rooms.remove(&client_identity) { + let mut room = room.0.lock(); + room.connection_state = ConnectionState::Disconnected; + room.updates_tx + .blocking_send(RoomEvent::Disconnected { + reason: DisconnectReason::SignalClose, + }) + .ok(); + } + } + } + + async fn publish_video_track( + &self, + token: String, + _local_track: LocalVideoTrack, + ) -> Result { + self.executor.simulate_random_delay().await; + + let claims = livekit_server::token::validate(&token, &self.secret_key)?; + let identity = ParticipantIdentity(claims.sub.unwrap().to_string()); + let room_name = claims.video.room.unwrap(); + + let mut server_rooms = self.rooms.lock(); + let room = server_rooms + .get_mut(&*room_name) + .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; + + let can_publish = room + .participant_permissions + .get(&identity) + .map(|permission| permission.can_publish) + .or(claims.video.can_publish) + .unwrap_or(true); + + if !can_publish { + return Err(anyhow!("user is not allowed to publish")); + } + + let sid: TrackSid = format!("TR_{}", nanoid::nanoid!(17)).try_into().unwrap(); + let server_track = Arc::new(TestServerVideoTrack { + sid: sid.clone(), + publisher_id: identity.clone(), + }); + + room.video_tracks.push(server_track.clone()); + + for (room_identity, client_room) in &room.client_rooms { + if *room_identity != identity { + let track = RemoteTrack::Video(RemoteVideoTrack { + server_track: server_track.clone(), + _room: client_room.downgrade(), + }); + let publication = RemoteTrackPublication { + sid: sid.clone(), + room: client_room.downgrade(), + track: track.clone(), + }; + let participant = RemoteParticipant { + identity: identity.clone(), + room: client_room.downgrade(), + }; + client_room + .0 + .lock() + .updates_tx + .blocking_send(RoomEvent::TrackSubscribed { + track, + publication, + participant, + }) + .unwrap(); + } + } + + Ok(sid) + } + + async fn publish_audio_track( + &self, + token: String, + _local_track: &LocalAudioTrack, + ) -> Result { + self.executor.simulate_random_delay().await; + + let claims = livekit_server::token::validate(&token, &self.secret_key)?; + let identity = ParticipantIdentity(claims.sub.unwrap().to_string()); + let room_name = claims.video.room.unwrap(); + + let mut server_rooms = self.rooms.lock(); + let room = server_rooms + .get_mut(&*room_name) + .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; + + let can_publish = room + .participant_permissions + .get(&identity) + .map(|permission| permission.can_publish) + .or(claims.video.can_publish) + .unwrap_or(true); + + if !can_publish { + return Err(anyhow!("user is not allowed to publish")); + } + + let sid: TrackSid = format!("TR_{}", nanoid::nanoid!(17)).try_into().unwrap(); + let server_track = Arc::new(TestServerAudioTrack { + sid: sid.clone(), + publisher_id: identity.clone(), + muted: AtomicBool::new(false), + }); + + room.audio_tracks.push(server_track.clone()); + + for (room_identity, client_room) in &room.client_rooms { + if *room_identity != identity { + let track = RemoteTrack::Audio(RemoteAudioTrack { + server_track: server_track.clone(), + room: client_room.downgrade(), + }); + let publication = RemoteTrackPublication { + sid: sid.clone(), + room: client_room.downgrade(), + track: track.clone(), + }; + let participant = RemoteParticipant { + identity: identity.clone(), + room: client_room.downgrade(), + }; + client_room + .0 + .lock() + .updates_tx + .blocking_send(RoomEvent::TrackSubscribed { + track, + publication, + participant, + }) + .ok(); + } + } + + Ok(sid) + } + + async fn unpublish_track(&self, _token: String, _track: &TrackSid) -> Result<()> { + Ok(()) + } + + fn set_track_muted(&self, token: &str, track_sid: &TrackSid, muted: bool) -> Result<()> { + let claims = livekit_server::token::validate(&token, &self.secret_key)?; + let room_name = claims.video.room.unwrap(); + let identity = ParticipantIdentity(claims.sub.unwrap().to_string()); + let mut server_rooms = self.rooms.lock(); + let room = server_rooms + .get_mut(&*room_name) + .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; + if let Some(track) = room + .audio_tracks + .iter_mut() + .find(|track| track.sid == *track_sid) + { + track.muted.store(muted, SeqCst); + for (id, client_room) in room.client_rooms.iter() { + if *id != identity { + let participant = Participant::Remote(RemoteParticipant { + identity: identity.clone(), + room: client_room.downgrade(), + }); + let track = RemoteTrack::Audio(RemoteAudioTrack { + server_track: track.clone(), + room: client_room.downgrade(), + }); + let publication = TrackPublication::Remote(RemoteTrackPublication { + sid: track_sid.clone(), + room: client_room.downgrade(), + track, + }); + + let event = if muted { + RoomEvent::TrackMuted { + participant, + publication, + } + } else { + RoomEvent::TrackUnmuted { + participant, + publication, + } + }; + + client_room + .0 + .lock() + .updates_tx + .blocking_send(event) + .unwrap(); + } + } + } + Ok(()) + } + + fn is_track_muted(&self, token: &str, track_sid: &TrackSid) -> Option { + let claims = livekit_server::token::validate(&token, &self.secret_key).ok()?; + let room_name = claims.video.room.unwrap(); + + let mut server_rooms = self.rooms.lock(); + let room = server_rooms.get_mut(&*room_name)?; + room.audio_tracks.iter().find_map(|track| { + if track.sid == *track_sid { + Some(track.muted.load(SeqCst)) + } else { + None + } + }) + } + + fn video_tracks(&self, token: String) -> Result> { + let claims = livekit_server::token::validate(&token, &self.secret_key)?; + let room_name = claims.video.room.unwrap(); + let identity = ParticipantIdentity(claims.sub.unwrap().to_string()); + + let mut server_rooms = self.rooms.lock(); + let room = server_rooms + .get_mut(&*room_name) + .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; + let client_room = room + .client_rooms + .get(&identity) + .ok_or_else(|| anyhow!("not a participant in room"))?; + Ok(room + .video_tracks + .iter() + .map(|track| RemoteVideoTrack { + server_track: track.clone(), + _room: client_room.downgrade(), + }) + .collect()) + } + + fn audio_tracks(&self, token: String) -> Result> { + let claims = livekit_server::token::validate(&token, &self.secret_key)?; + let room_name = claims.video.room.unwrap(); + let identity = ParticipantIdentity(claims.sub.unwrap().to_string()); + + let mut server_rooms = self.rooms.lock(); + let room = server_rooms + .get_mut(&*room_name) + .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; + let client_room = room + .client_rooms + .get(&identity) + .ok_or_else(|| anyhow!("not a participant in room"))?; + Ok(room + .audio_tracks + .iter() + .map(|track| RemoteAudioTrack { + server_track: track.clone(), + room: client_room.downgrade(), + }) + .collect()) + } +} + +#[cfg(not(target_os = "windows"))] +#[derive(Default, Debug)] +struct TestServerRoom { + client_rooms: HashMap, + video_tracks: Vec>, + audio_tracks: Vec>, + participant_permissions: HashMap, +} + +#[cfg(not(target_os = "windows"))] +#[derive(Debug)] +struct TestServerVideoTrack { + sid: TrackSid, + publisher_id: ParticipantIdentity, + // frames_rx: async_broadcast::Receiver, +} + +#[cfg(not(target_os = "windows"))] +#[derive(Debug)] +struct TestServerAudioTrack { + sid: TrackSid, + publisher_id: ParticipantIdentity, + muted: AtomicBool, +} + +pub struct TestApiClient { + url: String, +} + +#[derive(Clone, Debug)] +#[non_exhaustive] +pub enum RoomEvent { + ParticipantConnected(RemoteParticipant), + ParticipantDisconnected(RemoteParticipant), + LocalTrackPublished { + publication: LocalTrackPublication, + track: LocalTrack, + participant: LocalParticipant, + }, + LocalTrackUnpublished { + publication: LocalTrackPublication, + participant: LocalParticipant, + }, + TrackSubscribed { + track: RemoteTrack, + publication: RemoteTrackPublication, + participant: RemoteParticipant, + }, + TrackUnsubscribed { + track: RemoteTrack, + publication: RemoteTrackPublication, + participant: RemoteParticipant, + }, + TrackSubscriptionFailed { + participant: RemoteParticipant, + error: String, + #[cfg(not(target_os = "windows"))] + track_sid: TrackSid, + }, + TrackPublished { + publication: RemoteTrackPublication, + participant: RemoteParticipant, + }, + TrackUnpublished { + publication: RemoteTrackPublication, + participant: RemoteParticipant, + }, + TrackMuted { + participant: Participant, + publication: TrackPublication, + }, + TrackUnmuted { + participant: Participant, + publication: TrackPublication, + }, + RoomMetadataChanged { + old_metadata: String, + metadata: String, + }, + ParticipantMetadataChanged { + participant: Participant, + old_metadata: String, + metadata: String, + }, + ParticipantNameChanged { + participant: Participant, + old_name: String, + name: String, + }, + ActiveSpeakersChanged { + speakers: Vec, + }, + #[cfg(not(target_os = "windows"))] + ConnectionStateChanged(ConnectionState), + Connected { + participants_with_tracks: Vec<(RemoteParticipant, Vec)>, + }, + #[cfg(not(target_os = "windows"))] + Disconnected { + reason: DisconnectReason, + }, + Reconnecting, + Reconnected, +} + +#[cfg(not(target_os = "windows"))] +#[async_trait] +impl livekit_server::api::Client for TestApiClient { + fn url(&self) -> &str { + &self.url + } + + async fn create_room(&self, name: String) -> Result<()> { + let server = TestServer::get(&self.url)?; + server.create_room(name).await?; + Ok(()) + } + + async fn delete_room(&self, name: String) -> Result<()> { + let server = TestServer::get(&self.url)?; + server.delete_room(name).await?; + Ok(()) + } + + async fn remove_participant(&self, room: String, identity: String) -> Result<()> { + let server = TestServer::get(&self.url)?; + server + .remove_participant(room, ParticipantIdentity(identity)) + .await?; + Ok(()) + } + + async fn update_participant( + &self, + room: String, + identity: String, + permission: livekit_server::proto::ParticipantPermission, + ) -> Result<()> { + let server = TestServer::get(&self.url)?; + server + .update_participant(room, identity, permission) + .await?; + Ok(()) + } + + fn room_token(&self, room: &str, identity: &str) -> Result { + let server = TestServer::get(&self.url)?; + token::create( + &server.api_key, + &server.secret_key, + Some(identity), + token::VideoGrant::to_join(room), + ) + } + + fn guest_token(&self, room: &str, identity: &str) -> Result { + let server = TestServer::get(&self.url)?; + token::create( + &server.api_key, + &server.secret_key, + Some(identity), + token::VideoGrant::for_guest(room), + ) + } +} + +struct RoomState { + url: String, + token: String, + #[cfg(not(target_os = "windows"))] + local_identity: ParticipantIdentity, + #[cfg(not(target_os = "windows"))] + connection_state: ConnectionState, + #[cfg(not(target_os = "windows"))] + paused_audio_tracks: HashSet, + updates_tx: mpsc::Sender, +} + +#[derive(Clone, Debug)] +pub struct Room(Arc>); + +#[derive(Clone, Debug)] +pub(crate) struct WeakRoom(Weak>); + +#[cfg(not(target_os = "windows"))] +impl std::fmt::Debug for RoomState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Room") + .field("url", &self.url) + .field("token", &self.token) + .field("local_identity", &self.local_identity) + .field("connection_state", &self.connection_state) + .field("paused_audio_tracks", &self.paused_audio_tracks) + .finish() + } +} + +#[cfg(target_os = "windows")] +impl std::fmt::Debug for RoomState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Room") + .field("url", &self.url) + .field("token", &self.token) + .finish() + } +} + +#[cfg(not(target_os = "windows"))] +impl Room { + fn downgrade(&self) -> WeakRoom { + WeakRoom(Arc::downgrade(&self.0)) + } + + pub fn connection_state(&self) -> ConnectionState { + self.0.lock().connection_state + } + + pub fn local_participant(&self) -> LocalParticipant { + let identity = self.0.lock().local_identity.clone(); + LocalParticipant { + identity, + room: self.clone(), + } + } + + pub async fn connect( + url: &str, + token: &str, + _options: RoomOptions, + ) -> Result<(Self, mpsc::Receiver)> { + let server = TestServer::get(&url)?; + let (updates_tx, updates_rx) = mpsc::channel(1024); + let this = Self(Arc::new(Mutex::new(RoomState { + local_identity: ParticipantIdentity(String::new()), + url: url.to_string(), + token: token.to_string(), + connection_state: ConnectionState::Disconnected, + paused_audio_tracks: Default::default(), + updates_tx, + }))); + + let identity = server + .join_room(token.to_string(), this.clone()) + .await + .context("room join")?; + { + let mut state = this.0.lock(); + state.local_identity = identity; + state.connection_state = ConnectionState::Connected; + } + + Ok((this, updates_rx)) + } + + pub fn remote_participants(&self) -> HashMap { + self.test_server() + .remote_participants(self.0.lock().token.clone()) + .unwrap() + } + + fn test_server(&self) -> Arc { + TestServer::get(&self.0.lock().url).unwrap() + } + + fn token(&self) -> String { + self.0.lock().token.clone() + } +} + +#[cfg(not(target_os = "windows"))] +impl Drop for RoomState { + fn drop(&mut self) { + if self.connection_state == ConnectionState::Connected { + if let Ok(server) = TestServer::get(&self.url) { + let executor = server.executor.clone(); + let token = self.token.clone(); + executor + .spawn(async move { server.leave_room(token).await.ok() }) + .detach(); + } + } + } +} + +impl WeakRoom { + fn upgrade(&self) -> Option { + self.0.upgrade().map(Room) + } +} diff --git a/crates/livekit_client/src/test/participant.rs b/crates/livekit_client/src/test/participant.rs new file mode 100644 index 0000000000000..8d476b15379dd --- /dev/null +++ b/crates/livekit_client/src/test/participant.rs @@ -0,0 +1,111 @@ +use super::*; + +#[derive(Clone, Debug)] +pub enum Participant { + Local(LocalParticipant), + Remote(RemoteParticipant), +} + +#[derive(Clone, Debug)] +pub struct LocalParticipant { + #[cfg(not(target_os = "windows"))] + pub(super) identity: ParticipantIdentity, + pub(super) room: Room, +} + +#[derive(Clone, Debug)] +pub struct RemoteParticipant { + #[cfg(not(target_os = "windows"))] + pub(super) identity: ParticipantIdentity, + pub(super) room: WeakRoom, +} + +#[cfg(not(target_os = "windows"))] +impl Participant { + pub fn identity(&self) -> ParticipantIdentity { + match self { + Participant::Local(participant) => participant.identity.clone(), + Participant::Remote(participant) => participant.identity.clone(), + } + } +} + +#[cfg(not(target_os = "windows"))] +impl LocalParticipant { + pub async fn unpublish_track(&self, track: &TrackSid) -> Result<()> { + self.room + .test_server() + .unpublish_track(self.room.token(), track) + .await + } + + pub async fn publish_track( + &self, + track: LocalTrack, + _options: TrackPublishOptions, + ) -> Result { + let this = self.clone(); + let track = track.clone(); + let server = this.room.test_server(); + let sid = match track { + LocalTrack::Video(track) => { + server.publish_video_track(this.room.token(), track).await? + } + LocalTrack::Audio(track) => { + server + .publish_audio_track(this.room.token(), &track) + .await? + } + }; + Ok(LocalTrackPublication { + room: self.room.downgrade(), + sid, + }) + } +} + +#[cfg(not(target_os = "windows"))] +impl RemoteParticipant { + pub fn track_publications(&self) -> HashMap { + if let Some(room) = self.room.upgrade() { + let server = room.test_server(); + let audio = server + .audio_tracks(room.token()) + .unwrap() + .into_iter() + .filter(|track| track.publisher_id() == self.identity) + .map(|track| { + ( + track.sid(), + RemoteTrackPublication { + sid: track.sid(), + room: self.room.clone(), + track: RemoteTrack::Audio(track), + }, + ) + }); + let video = server + .video_tracks(room.token()) + .unwrap() + .into_iter() + .filter(|track| track.publisher_id() == self.identity) + .map(|track| { + ( + track.sid(), + RemoteTrackPublication { + sid: track.sid(), + room: self.room.clone(), + track: RemoteTrack::Video(track), + }, + ) + }); + audio.chain(video).collect() + } else { + HashMap::default() + } + } + + pub fn identity(&self) -> ParticipantIdentity { + self.identity.clone() + } +} diff --git a/crates/livekit_client/src/test/publication.rs b/crates/livekit_client/src/test/publication.rs new file mode 100644 index 0000000000000..6a3dfa0a51ce9 --- /dev/null +++ b/crates/livekit_client/src/test/publication.rs @@ -0,0 +1,116 @@ +use super::*; + +#[derive(Clone, Debug)] +pub enum TrackPublication { + Local(LocalTrackPublication), + Remote(RemoteTrackPublication), +} + +#[derive(Clone, Debug)] +pub struct LocalTrackPublication { + #[cfg(not(target_os = "windows"))] + pub(crate) sid: TrackSid, + pub(crate) room: WeakRoom, +} + +#[derive(Clone, Debug)] +pub struct RemoteTrackPublication { + #[cfg(not(target_os = "windows"))] + pub(crate) sid: TrackSid, + pub(crate) room: WeakRoom, + pub(crate) track: RemoteTrack, +} + +#[cfg(not(target_os = "windows"))] +impl TrackPublication { + pub fn sid(&self) -> TrackSid { + match self { + TrackPublication::Local(track) => track.sid(), + TrackPublication::Remote(track) => track.sid(), + } + } + + pub fn is_muted(&self) -> bool { + match self { + TrackPublication::Local(track) => track.is_muted(), + TrackPublication::Remote(track) => track.is_muted(), + } + } +} + +#[cfg(not(target_os = "windows"))] +impl LocalTrackPublication { + pub fn sid(&self) -> TrackSid { + self.sid.clone() + } + + pub fn mute(&self) { + self.set_mute(true) + } + + pub fn unmute(&self) { + self.set_mute(false) + } + + fn set_mute(&self, mute: bool) { + if let Some(room) = self.room.upgrade() { + room.test_server() + .set_track_muted(&room.token(), &self.sid, mute) + .ok(); + } + } + + pub fn is_muted(&self) -> bool { + if let Some(room) = self.room.upgrade() { + room.test_server() + .is_track_muted(&room.token(), &self.sid) + .unwrap_or(false) + } else { + false + } + } +} + +#[cfg(not(target_os = "windows"))] +impl RemoteTrackPublication { + pub fn sid(&self) -> TrackSid { + self.sid.clone() + } + + pub fn track(&self) -> Option { + Some(self.track.clone()) + } + + pub fn kind(&self) -> TrackKind { + self.track.kind() + } + + pub fn is_muted(&self) -> bool { + if let Some(room) = self.room.upgrade() { + room.test_server() + .is_track_muted(&room.token(), &self.sid) + .unwrap_or(false) + } else { + false + } + } + + pub fn is_enabled(&self) -> bool { + if let Some(room) = self.room.upgrade() { + !room.0.lock().paused_audio_tracks.contains(&self.sid) + } else { + false + } + } + + pub fn set_enabled(&self, enabled: bool) { + if let Some(room) = self.room.upgrade() { + let paused_audio_tracks = &mut room.0.lock().paused_audio_tracks; + if enabled { + paused_audio_tracks.remove(&self.sid); + } else { + paused_audio_tracks.insert(self.sid.clone()); + } + } + } +} diff --git a/crates/livekit_client/src/test/track.rs b/crates/livekit_client/src/test/track.rs new file mode 100644 index 0000000000000..302177a10a6f6 --- /dev/null +++ b/crates/livekit_client/src/test/track.rs @@ -0,0 +1,201 @@ +use super::*; +#[cfg(not(windows))] +use webrtc::{audio_source::RtcAudioSource, video_source::RtcVideoSource}; + +#[cfg(not(windows))] +pub use livekit::track::{TrackKind, TrackSource}; + +#[derive(Clone, Debug)] +pub enum LocalTrack { + Audio(LocalAudioTrack), + Video(LocalVideoTrack), +} + +#[derive(Clone, Debug)] +pub enum RemoteTrack { + Audio(RemoteAudioTrack), + Video(RemoteVideoTrack), +} + +#[derive(Clone, Debug)] +pub struct LocalVideoTrack {} + +#[derive(Clone, Debug)] +pub struct LocalAudioTrack {} + +#[derive(Clone, Debug)] +pub struct RemoteVideoTrack { + #[cfg(not(target_os = "windows"))] + pub(super) server_track: Arc, + pub(super) _room: WeakRoom, +} + +#[derive(Clone, Debug)] +pub struct RemoteAudioTrack { + #[cfg(not(target_os = "windows"))] + pub(super) server_track: Arc, + pub(super) room: WeakRoom, +} + +pub enum RtcTrack { + Audio(RtcAudioTrack), + Video(RtcVideoTrack), +} + +pub struct RtcAudioTrack { + #[cfg(not(target_os = "windows"))] + pub(super) server_track: Arc, + pub(super) room: WeakRoom, +} + +pub struct RtcVideoTrack { + #[cfg(not(target_os = "windows"))] + pub(super) _server_track: Arc, +} + +#[cfg(not(target_os = "windows"))] +impl RemoteTrack { + pub fn sid(&self) -> TrackSid { + match self { + RemoteTrack::Audio(track) => track.sid(), + RemoteTrack::Video(track) => track.sid(), + } + } + + pub fn kind(&self) -> TrackKind { + match self { + RemoteTrack::Audio(_) => TrackKind::Audio, + RemoteTrack::Video(_) => TrackKind::Video, + } + } + + pub fn publisher_id(&self) -> ParticipantIdentity { + match self { + RemoteTrack::Audio(track) => track.publisher_id(), + RemoteTrack::Video(track) => track.publisher_id(), + } + } + + pub fn rtc_track(&self) -> RtcTrack { + match self { + RemoteTrack::Audio(track) => RtcTrack::Audio(track.rtc_track()), + RemoteTrack::Video(track) => RtcTrack::Video(track.rtc_track()), + } + } +} + +#[cfg(not(windows))] +impl LocalVideoTrack { + pub fn create_video_track(_name: &str, _source: RtcVideoSource) -> Self { + Self {} + } +} + +#[cfg(not(windows))] +impl LocalAudioTrack { + pub fn create_audio_track(_name: &str, _source: RtcAudioSource) -> Self { + Self {} + } +} + +#[cfg(not(target_os = "windows"))] +impl RemoteAudioTrack { + pub fn sid(&self) -> TrackSid { + self.server_track.sid.clone() + } + + pub fn publisher_id(&self) -> ParticipantIdentity { + self.server_track.publisher_id.clone() + } + + pub fn start(&self) { + if let Some(room) = self.room.upgrade() { + room.0 + .lock() + .paused_audio_tracks + .remove(&self.server_track.sid); + } + } + + pub fn stop(&self) { + if let Some(room) = self.room.upgrade() { + room.0 + .lock() + .paused_audio_tracks + .insert(self.server_track.sid.clone()); + } + } + + pub fn rtc_track(&self) -> RtcAudioTrack { + RtcAudioTrack { + server_track: self.server_track.clone(), + room: self.room.clone(), + } + } +} + +#[cfg(not(target_os = "windows"))] +impl RemoteVideoTrack { + pub fn sid(&self) -> TrackSid { + self.server_track.sid.clone() + } + + pub fn publisher_id(&self) -> ParticipantIdentity { + self.server_track.publisher_id.clone() + } + + pub fn rtc_track(&self) -> RtcVideoTrack { + RtcVideoTrack { + _server_track: self.server_track.clone(), + } + } +} + +#[cfg(not(target_os = "windows"))] +impl RtcTrack { + pub fn enabled(&self) -> bool { + match self { + RtcTrack::Audio(track) => track.enabled(), + RtcTrack::Video(track) => track.enabled(), + } + } + + pub fn set_enabled(&self, enabled: bool) { + match self { + RtcTrack::Audio(track) => track.set_enabled(enabled), + RtcTrack::Video(_) => {} + } + } +} + +#[cfg(not(target_os = "windows"))] +impl RtcAudioTrack { + pub fn set_enabled(&self, enabled: bool) { + if let Some(room) = self.room.upgrade() { + let paused_audio_tracks = &mut room.0.lock().paused_audio_tracks; + if enabled { + paused_audio_tracks.remove(&self.server_track.sid); + } else { + paused_audio_tracks.insert(self.server_track.sid.clone()); + } + } + } + + pub fn enabled(&self) -> bool { + if let Some(room) = self.room.upgrade() { + !room + .0 + .lock() + .paused_audio_tracks + .contains(&self.server_track.sid) + } else { + false + } + } +} + +impl RtcVideoTrack { + pub fn enabled(&self) -> bool { + true + } +} diff --git a/crates/livekit_client/src/test/webrtc.rs b/crates/livekit_client/src/test/webrtc.rs new file mode 100644 index 0000000000000..6ac06e04846d7 --- /dev/null +++ b/crates/livekit_client/src/test/webrtc.rs @@ -0,0 +1,136 @@ +use super::track::{RtcAudioTrack, RtcVideoTrack}; +use futures::Stream; +use livekit::webrtc as real; +use std::{ + pin::Pin, + task::{Context, Poll}, +}; + +pub mod video_stream { + use super::*; + + pub mod native { + use super::*; + use real::video_frame::BoxVideoFrame; + + pub struct NativeVideoStream { + pub track: RtcVideoTrack, + } + + impl NativeVideoStream { + pub fn new(track: RtcVideoTrack) -> Self { + Self { track } + } + } + + impl Stream for NativeVideoStream { + type Item = BoxVideoFrame; + + fn poll_next(self: Pin<&mut Self>, _cx: &mut Context) -> Poll> { + Poll::Pending + } + } + } +} + +pub mod audio_stream { + use super::*; + + pub mod native { + use super::*; + use real::audio_frame::AudioFrame; + + pub struct NativeAudioStream { + pub track: RtcAudioTrack, + } + + impl NativeAudioStream { + pub fn new(track: RtcAudioTrack, _sample_rate: i32, _num_channels: i32) -> Self { + Self { track } + } + } + + impl Stream for NativeAudioStream { + type Item = AudioFrame<'static>; + + fn poll_next(self: Pin<&mut Self>, _cx: &mut Context) -> Poll> { + Poll::Pending + } + } + } +} + +pub mod audio_source { + use super::*; + + pub use real::audio_source::AudioSourceOptions; + + pub mod native { + use std::sync::Arc; + + use super::*; + use real::{audio_frame::AudioFrame, RtcError}; + + #[derive(Clone)] + pub struct NativeAudioSource { + pub options: Arc, + pub sample_rate: u32, + pub num_channels: u32, + } + + impl NativeAudioSource { + pub fn new( + options: AudioSourceOptions, + sample_rate: u32, + num_channels: u32, + _queue_size_ms: u32, + ) -> Self { + Self { + options: Arc::new(options), + sample_rate, + num_channels, + } + } + + pub async fn capture_frame(&self, _frame: &AudioFrame<'_>) -> Result<(), RtcError> { + Ok(()) + } + } + } + + pub enum RtcAudioSource { + Native(native::NativeAudioSource), + } +} + +pub use livekit::webrtc::audio_frame; +pub use livekit::webrtc::video_frame; + +pub mod video_source { + use super::*; + pub use real::video_source::VideoResolution; + + pub struct RTCVideoSource; + + pub mod native { + use super::*; + use real::video_frame::{VideoBuffer, VideoFrame}; + + #[derive(Clone)] + pub struct NativeVideoSource { + pub resolution: VideoResolution, + } + + impl NativeVideoSource { + pub fn new(resolution: super::VideoResolution) -> Self { + Self { resolution } + } + + pub fn capture_frame>(&self, _frame: &VideoFrame) {} + } + } + + pub enum RtcVideoSource { + Native(native::NativeVideoSource), + } +} diff --git a/crates/live_kit_server/Cargo.toml b/crates/livekit_server/Cargo.toml similarity index 90% rename from crates/live_kit_server/Cargo.toml rename to crates/livekit_server/Cargo.toml index 4b4b5e13dad43..c76cb1580cf4f 100644 --- a/crates/live_kit_server/Cargo.toml +++ b/crates/livekit_server/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "live_kit_server" +name = "livekit_server" version = "0.1.0" edition = "2021" description = "SDK for the LiveKit server API" @@ -10,7 +10,7 @@ license = "AGPL-3.0-or-later" workspace = true [lib] -path = "src/live_kit_server.rs" +path = "src/livekit_server.rs" doctest = false [dependencies] diff --git a/crates/live_kit_server/LICENSE-AGPL b/crates/livekit_server/LICENSE-AGPL similarity index 100% rename from crates/live_kit_server/LICENSE-AGPL rename to crates/livekit_server/LICENSE-AGPL diff --git a/crates/live_kit_server/build.rs b/crates/livekit_server/build.rs similarity index 100% rename from crates/live_kit_server/build.rs rename to crates/livekit_server/build.rs diff --git a/crates/live_kit_server/src/api.rs b/crates/livekit_server/src/api.rs similarity index 100% rename from crates/live_kit_server/src/api.rs rename to crates/livekit_server/src/api.rs diff --git a/crates/live_kit_server/src/live_kit_server.rs b/crates/livekit_server/src/livekit_server.rs similarity index 100% rename from crates/live_kit_server/src/live_kit_server.rs rename to crates/livekit_server/src/livekit_server.rs diff --git a/crates/live_kit_server/src/proto.rs b/crates/livekit_server/src/proto.rs similarity index 100% rename from crates/live_kit_server/src/proto.rs rename to crates/livekit_server/src/proto.rs diff --git a/crates/live_kit_server/src/token.rs b/crates/livekit_server/src/token.rs similarity index 100% rename from crates/live_kit_server/src/token.rs rename to crates/livekit_server/src/token.rs diff --git a/crates/live_kit_server/vendored/protocol/README.md b/crates/livekit_server/vendored/protocol/README.md similarity index 100% rename from crates/live_kit_server/vendored/protocol/README.md rename to crates/livekit_server/vendored/protocol/README.md diff --git a/crates/live_kit_server/vendored/protocol/livekit_analytics.proto b/crates/livekit_server/vendored/protocol/livekit_analytics.proto similarity index 100% rename from crates/live_kit_server/vendored/protocol/livekit_analytics.proto rename to crates/livekit_server/vendored/protocol/livekit_analytics.proto diff --git a/crates/live_kit_server/vendored/protocol/livekit_egress.proto b/crates/livekit_server/vendored/protocol/livekit_egress.proto similarity index 100% rename from crates/live_kit_server/vendored/protocol/livekit_egress.proto rename to crates/livekit_server/vendored/protocol/livekit_egress.proto diff --git a/crates/live_kit_server/vendored/protocol/livekit_ingress.proto b/crates/livekit_server/vendored/protocol/livekit_ingress.proto similarity index 100% rename from crates/live_kit_server/vendored/protocol/livekit_ingress.proto rename to crates/livekit_server/vendored/protocol/livekit_ingress.proto diff --git a/crates/live_kit_server/vendored/protocol/livekit_internal.proto b/crates/livekit_server/vendored/protocol/livekit_internal.proto similarity index 100% rename from crates/live_kit_server/vendored/protocol/livekit_internal.proto rename to crates/livekit_server/vendored/protocol/livekit_internal.proto diff --git a/crates/live_kit_server/vendored/protocol/livekit_models.proto b/crates/livekit_server/vendored/protocol/livekit_models.proto similarity index 100% rename from crates/live_kit_server/vendored/protocol/livekit_models.proto rename to crates/livekit_server/vendored/protocol/livekit_models.proto diff --git a/crates/live_kit_server/vendored/protocol/livekit_room.proto b/crates/livekit_server/vendored/protocol/livekit_room.proto similarity index 100% rename from crates/live_kit_server/vendored/protocol/livekit_room.proto rename to crates/livekit_server/vendored/protocol/livekit_room.proto diff --git a/crates/live_kit_server/vendored/protocol/livekit_rpc_internal.proto b/crates/livekit_server/vendored/protocol/livekit_rpc_internal.proto similarity index 100% rename from crates/live_kit_server/vendored/protocol/livekit_rpc_internal.proto rename to crates/livekit_server/vendored/protocol/livekit_rpc_internal.proto diff --git a/crates/live_kit_server/vendored/protocol/livekit_rtc.proto b/crates/livekit_server/vendored/protocol/livekit_rtc.proto similarity index 100% rename from crates/live_kit_server/vendored/protocol/livekit_rtc.proto rename to crates/livekit_server/vendored/protocol/livekit_rtc.proto diff --git a/crates/live_kit_server/vendored/protocol/livekit_webhook.proto b/crates/livekit_server/vendored/protocol/livekit_webhook.proto similarity index 100% rename from crates/live_kit_server/vendored/protocol/livekit_webhook.proto rename to crates/livekit_server/vendored/protocol/livekit_webhook.proto diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index 92940d1c526e8..70478eeb759f8 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -17,6 +17,7 @@ anyhow.workspace = true [target.'cfg(target_os = "macos")'.dependencies] core-foundation.workspace = true +ctor.workspace = true foreign-types = "0.5" metal = "0.29" objc = "0.2" diff --git a/crates/media/src/media.rs b/crates/media/src/media.rs index 8757249c31940..3f55475589814 100644 --- a/crates/media/src/media.rs +++ b/crates/media/src/media.rs @@ -253,11 +253,14 @@ pub mod core_media { } } - pub fn image_buffer(&self) -> CVImageBuffer { + pub fn image_buffer(&self) -> Option { unsafe { - CVImageBuffer::wrap_under_get_rule(CMSampleBufferGetImageBuffer( - self.as_concrete_TypeRef(), - )) + let ptr = CMSampleBufferGetImageBuffer(self.as_concrete_TypeRef()); + if ptr.is_null() { + None + } else { + Some(CVImageBuffer::wrap_under_get_rule(ptr)) + } } } diff --git a/crates/proto/proto/zed.proto b/crates/proto/proto/zed.proto index 178d88ad26742..d84196a9da271 100644 --- a/crates/proto/proto/zed.proto +++ b/crates/proto/proto/zed.proto @@ -429,7 +429,7 @@ message Room { repeated Participant participants = 2; repeated PendingParticipant pending_participants = 3; repeated Follower followers = 4; - string live_kit_room = 5; + string livekit_room = 5; } message Participant { diff --git a/crates/storybook/build.rs b/crates/storybook/build.rs index ec0e8944f5abf..66791cae4218e 100644 --- a/crates/storybook/build.rs +++ b/crates/storybook/build.rs @@ -1,11 +1,4 @@ fn main() { - // Find WebRTC.framework as a sibling of the executable when running outside of an application bundle. - // TODO: We shouldn't depend on WebRTC in editor - #[cfg(target_os = "macos")] - { - println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path"); - } - #[cfg(target_os = "windows")] { #[cfg(target_env = "msvc")] diff --git a/crates/title_bar/src/collab.rs b/crates/title_bar/src/collab.rs index 649dfb34f70ac..7d977bb458add 100644 --- a/crates/title_bar/src/collab.rs +++ b/crates/title_bar/src/collab.rs @@ -294,9 +294,9 @@ impl TitleBar { let is_muted = room.is_muted(); let is_deafened = room.is_deafened().unwrap_or(false); let is_screen_sharing = room.is_screen_sharing(); - let can_use_microphone = room.can_use_microphone(); + let can_use_microphone = room.can_use_microphone(cx); let can_share_projects = room.can_share_projects(); - let platform_supported = match self.platform_style { + let screen_sharing_supported = match self.platform_style { PlatformStyle::Mac => true, PlatformStyle::Linux | PlatformStyle::Windows => false, }; @@ -363,9 +363,7 @@ impl TitleBar { ) .tooltip(move |cx| { Tooltip::text( - if !platform_supported { - "Cannot share microphone" - } else if is_muted { + if is_muted { "Unmute microphone" } else { "Mute microphone" @@ -375,56 +373,45 @@ impl TitleBar { }) .style(ButtonStyle::Subtle) .icon_size(IconSize::Small) - .selected(platform_supported && is_muted) - .disabled(!platform_supported) + .selected(is_muted) .selected_style(ButtonStyle::Tinted(TintColor::Negative)) .on_click(move |_, cx| { toggle_mute(&Default::default(), cx); }) .into_any_element(), ); - } - children.push( - IconButton::new( - "mute-sound", - if is_deafened { - ui::IconName::AudioOff - } else { - ui::IconName::AudioOn - }, - ) - .style(ButtonStyle::Subtle) - .selected_style(ButtonStyle::Tinted(TintColor::Negative)) - .icon_size(IconSize::Small) - .selected(is_deafened) - .disabled(!platform_supported) - .tooltip(move |cx| { - if !platform_supported { - Tooltip::text("Cannot share microphone", cx) - } else if can_use_microphone { + children.push( + IconButton::new( + "mute-sound", + if is_deafened { + ui::IconName::AudioOff + } else { + ui::IconName::AudioOn + }, + ) + .style(ButtonStyle::Subtle) + .selected_style(ButtonStyle::Tinted(TintColor::Negative)) + .icon_size(IconSize::Small) + .selected(is_deafened) + .tooltip(move |cx| { Tooltip::with_meta("Deafen Audio", None, "Mic will be muted", cx) - } else { - Tooltip::text("Deafen Audio", cx) - } - }) - .on_click(move |_, cx| toggle_deafen(&Default::default(), cx)) - .into_any_element(), - ); + }) + .on_click(move |_, cx| toggle_deafen(&Default::default(), cx)) + .into_any_element(), + ); + } - if can_share_projects { + if screen_sharing_supported { children.push( IconButton::new("screen-share", ui::IconName::Screen) .style(ButtonStyle::Subtle) .icon_size(IconSize::Small) .selected(is_screen_sharing) - .disabled(!platform_supported) .selected_style(ButtonStyle::Tinted(TintColor::Accent)) .tooltip(move |cx| { Tooltip::text( - if !platform_supported { - "Cannot share screen" - } else if is_screen_sharing { + if is_screen_sharing { "Stop Sharing Screen" } else { "Share Screen" diff --git a/crates/workspace/src/shared_screen.rs b/crates/workspace/src/shared_screen.rs index 59df859488093..285946cce0709 100644 --- a/crates/workspace/src/shared_screen.rs +++ b/crates/workspace/src/shared_screen.rs @@ -2,16 +2,13 @@ use crate::{ item::{Item, ItemEvent}, ItemNavHistory, WorkspaceId, }; -use anyhow::Result; -use call::participant::{Frame, RemoteVideoTrack}; +use call::{RemoteVideoTrack, RemoteVideoTrackView}; use client::{proto::PeerId, User}; -use futures::StreamExt; use gpui::{ - div, surface, AppContext, EventEmitter, FocusHandle, FocusableView, InteractiveElement, - ParentElement, Render, SharedString, Styled, Task, View, ViewContext, VisualContext, - WindowContext, + div, AppContext, EventEmitter, FocusHandle, FocusableView, InteractiveElement, ParentElement, + Render, SharedString, Styled, View, ViewContext, VisualContext, WindowContext, }; -use std::sync::{Arc, Weak}; +use std::sync::Arc; use ui::{prelude::*, Icon, IconName}; pub enum Event { @@ -19,40 +16,30 @@ pub enum Event { } pub struct SharedScreen { - track: Weak, - frame: Option, pub peer_id: PeerId, user: Arc, nav_history: Option, - _maintain_frame: Task>, + view: View, focus: FocusHandle, } impl SharedScreen { pub fn new( - track: &Arc, + track: RemoteVideoTrack, peer_id: PeerId, user: Arc, cx: &mut ViewContext, ) -> Self { - cx.focus_handle(); - let mut frames = track.frames(); + let view = cx.new_view(|cx| RemoteVideoTrackView::new(track.clone(), cx)); + cx.subscribe(&view, |_, _, ev, cx| match ev { + call::RemoteVideoTrackViewEvent::Close => cx.emit(Event::Close), + }) + .detach(); Self { - track: Arc::downgrade(track), - frame: None, + view, peer_id, user, nav_history: Default::default(), - _maintain_frame: cx.spawn(|this, mut cx| async move { - while let Some(frame) = frames.next().await { - this.update(&mut cx, |this, cx| { - this.frame = Some(frame); - cx.notify(); - })?; - } - this.update(&mut cx, |_, cx| cx.emit(Event::Close))?; - Ok(()) - }), focus: cx.focus_handle(), } } @@ -72,11 +59,7 @@ impl Render for SharedScreen { .track_focus(&self.focus) .key_context("SharedScreen") .size_full() - .children( - self.frame - .as_ref() - .map(|frame| surface(frame.image()).size_full()), - ) + .child(self.view.clone()) } } @@ -114,8 +97,13 @@ impl Item for SharedScreen { _workspace_id: Option, cx: &mut ViewContext, ) -> Option> { - let track = self.track.upgrade()?; - Some(cx.new_view(|cx| Self::new(&track, self.peer_id, self.user.clone(), cx))) + Some(cx.new_view(|cx| Self { + view: self.view.update(cx, |view, cx| view.clone(cx)), + peer_id: self.peer_id, + user: self.user.clone(), + nav_history: Default::default(), + focus: cx.focus_handle(), + })) } fn to_item_events(event: &Self::Event, mut f: impl FnMut(ItemEvent)) { diff --git a/crates/workspace/src/workspace.rs b/crates/workspace/src/workspace.rs index a8681f22c5b48..b8056a4bf8988 100644 --- a/crates/workspace/src/workspace.rs +++ b/crates/workspace/src/workspace.rs @@ -3944,6 +3944,17 @@ impl Workspace { None } + #[cfg(target_os = "windows")] + fn shared_screen_for_peer( + &self, + _peer_id: PeerId, + _pane: &View, + _cx: &mut WindowContext, + ) -> Option> { + None + } + + #[cfg(not(target_os = "windows"))] fn shared_screen_for_peer( &self, peer_id: PeerId, @@ -3962,7 +3973,7 @@ impl Workspace { } } - Some(cx.new_view(|cx| SharedScreen::new(&track, peer_id, user.clone(), cx))) + Some(cx.new_view(|cx| SharedScreen::new(track, peer_id, user.clone(), cx))) } pub fn on_window_activation_changed(&mut self, cx: &mut ViewContext) { diff --git a/crates/zed/build.rs b/crates/zed/build.rs index 3013773f91834..b59492287fea1 100644 --- a/crates/zed/build.rs +++ b/crates/zed/build.rs @@ -5,13 +5,6 @@ fn main() { println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=10.15.7"); println!("cargo:rerun-if-env-changed=ZED_BUNDLE"); - if std::env::var("ZED_BUNDLE").ok().as_deref() == Some("true") { - // Find WebRTC.framework in the Frameworks folder when running as part of an application bundle. - println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path/../Frameworks"); - } else { - // Find WebRTC.framework as a sibling of the executable when running outside of an application bundle. - println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path"); - } // Weakly link ReplayKit to ensure Zed can be used on macOS 10.15+. println!("cargo:rustc-link-arg=-Wl,-weak_framework,ReplayKit"); diff --git a/script/bundle-linux b/script/bundle-linux index 98b49ae4da945..c05037b6ccabb 100755 --- a/script/bundle-linux +++ b/script/bundle-linux @@ -92,7 +92,7 @@ cp "${target_dir}/${target_triple}/release/cli" "${zed_dir}/bin/zed" find_libs() { ldd ${target_dir}/${target_triple}/release/zed |\ cut -d' ' -f3 |\ - grep -v '\<\(libstdc++.so\|libc.so\|libgcc_s.so\|libm.so\|libpthread.so\|libdl.so\)' + grep -v '\<\(libstdc++.so\|libc.so\|libgcc_s.so\|libm.so\|libpthread.so\|libdl.so\|libasound.so\)' } mkdir -p "${zed_dir}/lib" diff --git a/script/bundle-mac b/script/bundle-mac index 54247645ccfd5..85058b03dce6b 100755 --- a/script/bundle-mac +++ b/script/bundle-mac @@ -219,13 +219,7 @@ function sign_app_binaries() { local app_path=$1 local architecture=$2 local architecture_dir=$3 - echo "Copying WebRTC.framework into the frameworks folder" - rm -rf "${app_path}/Contents/Frameworks" - mkdir -p "${app_path}/Contents/Frameworks" - if [ "$local_arch" = false ]; then - cp -R target/${local_target_triple}/${target_dir}/WebRTC.framework "${app_path}/Contents/Frameworks/" - else - cp -R target/${target_dir}/WebRTC.framework "${app_path}/Contents/Frameworks/" + if [ "$local_arch" = true ]; then cp -R target/${target_dir}/cli "${app_path}/Contents/MacOS/" fi @@ -238,7 +232,6 @@ function sign_app_binaries() { if [[ $can_code_sign = true ]]; then echo "Code signing binaries" # sequence of codesign commands modeled after this example: https://developer.apple.com/forums/thread/701514 - /usr/bin/codesign --deep --force --timestamp --sign "$IDENTITY" "${app_path}/Contents/Frameworks/WebRTC.framework" -v /usr/bin/codesign --deep --force --timestamp --options runtime --sign "$IDENTITY" "${app_path}/Contents/MacOS/cli" -v /usr/bin/codesign --deep --force --timestamp --options runtime --sign "$IDENTITY" "${app_path}/Contents/MacOS/git" -v /usr/bin/codesign --deep --force --timestamp --options runtime --entitlements crates/zed/resources/zed.entitlements --sign "$IDENTITY" "${app_path}/Contents/MacOS/zed" -v diff --git a/typos.toml b/typos.toml index 0682d0a3a9e18..dc724dd50d5b8 100644 --- a/typos.toml +++ b/typos.toml @@ -22,7 +22,7 @@ extend-exclude = [ # Stripe IDs are flagged as typos. "crates/collab/src/db/tests/processed_stripe_event_tests.rs", # Not our typos. - "crates/live_kit_server/", + "crates/livekit_server/", # Vim makes heavy use of partial typing tables. "crates/vim/", # Editor and file finder rely on partial typing and custom in-string syntax.